[feature]: allow to install scoped packages (#2943)

foxriver76 · web-flow · commit 784d891be6c2 · 2024-11-13T14:42:14.000+01:00
* allow to install scoped packages

* optimize code

* added documentation
diff --git a/README.md b/README.md
@@ -102,6 +102,7 @@ The main configuration is stored in `iobroker-data/iobroker.json`. Normally, the
 - [js-controller Host Messages](#js-controller-host-messages)
 - [Adapter Development](#adapter-development)
 - [Environment Variables](#environment-variables)
+- [Vendor Packages Workflow](#vendor-packages-workflow)
 
 ### Admin UI
 **Feature status:** stable
@@ -1318,6 +1319,74 @@ However, on upgrades of Node.js these get lost. If js-controller detects a Node.
 
 In some scenarios, e.g. during development it may be useful to deactivate this feature. You can do so by settings the `IOB_NO_SETCAP` environment variable to `true`.
 
+### Vendor Packages Workflow
+Feature status: New in 7.0.0
+
+This feature is only of interest for vendors which aim to provide a package which is published to a private package registry (e.g. GitHub Packages).
+This may be desirable if the adapter is only relevant for a specific customer and/or contains business logic which needs to be kept secret.
+
+In the following, information is provided how private packages can be installed into the ioBroker ecosystem.
+The information is tested with GitHub packages. However, it should work in a similar fashion with other registries, like GitLab Packages.
+
+#### Package Registry
+You can use e.g. the GitHub package registry. Simply scope your adapter to your organization or personal scope by changing the package name in the `package.json` 
+and configuring the `publishConfig`:
+
+```json
+{
+  "name": "@org/vendorAdapter",
+  "publishConfig": {
+    "registry": "https://npm.pkg.github.com"
+  }
+}
+```
+
+Note, that you need to configure `npm` to authenticate via your registry.
+Find more information in the [documentation](https://docs.npmjs.com/cli/v9/configuring-npm/npmrc#auth-related-configuration).
+
+Example `.npmrc` file (can be in your project or in the users home directory):
+
+```
+//npm.pkg.github.com/:_authToken=<YOUR_TOKEN>
+@org:registry=https://npm.pkg.github.com
+```
+
+Where `YOUR_TOKEN` is an access token which has the permissions to write packages.
+
+If you then execute `npm publish`, the package will be published to your custom registry instead of the `npm` registry.
+
+#### Vendor Repository
+In your vendor-specific repository, each adapter can have a separate field called `packetName`.
+This represents the real name of the npm packet. E.g.
+
+```json
+{
+  "vendorAdapter": {
+    "version": "1.0.0",
+    "name": "vendorAdapter",
+    "packetName": "@org/vendorAdapter"
+  }
+}
+```
+
+The js-controller will alias the package name to the adapter name on installation.
+This has one drawback, which is normally not relevant for vendor setups. You can not install the adapter via the `npm url` command, meaning no installation from GitHub or local tarballs.
+
+#### Token setup
+On the customers ioBroker host, create a `.npmrc` file inside of `/home/iobroker/`.
+It should look like: 
+
+```
+//npm.pkg.github.com/:_authToken=<YOUR_TOKEN>
+@org:registry=https://npm.pkg.github.com
+```
+
+Where `YOUR_TOKEN` is an access token which has the permissions to read packages.
+A best practice working with multiple customers is, to create an organization for each customer instead of using your personal scope. 
+Hence, you can scope them to not have access to packages of other customers or your own.
+
+Find more information in the [documentation](https://docs.npmjs.com/cli/v9/configuring-npm/npmrc#auth-related-configuration).
+
 ## Release cycle and Development process overview
 The goal is to release an update for the js-controller roughly all 6 months (April/September). The main reasons for this are shorter iterations and fewer changes that can be problematic for the users (and getting fast feedback) and also trying to stay up-to-date with the dependencies.
 
diff --git a/packages/cli/src/lib/setup/setupInstall.ts b/packages/cli/src/lib/setup/setupInstall.ts
@@ -148,29 +148,28 @@ export class Install {
     /**
      * Download given packet
      *
-     * @param repoUrl
-     * @param packetName
+     * @param repoUrlOrRepo repository url or already the repository object
+     * @param packetName name of the package to install
      * @param options options.stopDb will stop the db before upgrade ONLY use it for controller upgrade - db is gone afterwards, does not work with stoppedList
-     * @param stoppedList
+     * @param stoppedList list of stopped instances (as instance objects)
      */
     async downloadPacket(
-        repoUrl: string | undefined | Record<string, any>,
+        repoUrlOrRepo: string | undefined | Record<string, any>,
         packetName: string,
         options?: CLIDownloadPacketOptions,
         stoppedList?: ioBroker.InstanceObject[],
     ): Promise<DownloadPacketReturnObject> {
-        let url;
         if (!options || typeof options !== 'object') {
             options = {};
         }
 
         stoppedList = stoppedList || [];
-        let sources: Record<string, any>;
+        let sources: Record<string, ioBroker.RepositoryJsonAdapterContent>;
 
-        if (!repoUrl || !tools.isObject(repoUrl)) {
-            sources = await getRepository({ repoName: repoUrl, objects: this.objects });
+        if (!repoUrlOrRepo || !tools.isObject(repoUrlOrRepo)) {
+            sources = await getRepository({ repoName: repoUrlOrRepo, objects: this.objects });
         } else {
-            sources = repoUrl;
+            sources = repoUrlOrRepo;
         }
 
         if (options.stopDb && stoppedList.length) {
@@ -194,97 +193,54 @@ export class Install {
                 version = '';
             }
         }
-        options.packetName = packetName;
 
-        options.unsafePerm = sources[packetName]?.unsafePerm;
+        const source = sources[packetName];
+
+        if (!source) {
+            const errMessage = `Unknown packet name ${packetName}. Please install packages from outside the repository using "${tools.appNameLowerCase} url <url-or-package>"!`;
+            console.error(`host.${hostname} ${errMessage}`);
+            throw new IoBrokerError({
+                code: EXIT_CODES.UNKNOWN_PACKET_NAME,
+                message: errMessage,
+            });
+        }
+
+        options.packetName = packetName;
+        options.unsafePerm = source.unsafePerm;
 
         // Check if flag stopBeforeUpdate is true or on windows we stop because of issue #1436
-        if ((sources[packetName]?.stopBeforeUpdate || osPlatform === 'win32') && !stoppedList.length) {
+        if ((source.stopBeforeUpdate || osPlatform === 'win32') && !stoppedList.length) {
             stoppedList = await this._getInstancesOfAdapter(packetName);
             await this.enableInstances(stoppedList, false);
         }
 
-        // try to extract the information from local sources-dist.json
-        if (!sources[packetName]) {
-            try {
-                const sourcesDist = fs.readJsonSync(`${tools.getControllerDir()}/conf/sources-dist.json`);
-                sources[packetName] = sourcesDist[packetName];
-            } catch {
-                // OK
+        if (options.stopDb) {
+            if (this.objects.destroy) {
+                await this.objects.destroy();
+                console.log('Stopped Objects DB');
             }
-        }
-
-        if (sources[packetName]) {
-            url = sources[packetName].url;
-
-            if (
-                url &&
-                packetName === 'js-controller' &&
-                fs.pathExistsSync(
-                    `${tools.getControllerDir()}/../../node_modules/${tools.appName.toLowerCase()}.js-controller`,
-                )
-            ) {
-                url = null;
+            if (this.states.destroy) {
+                await this.states.destroy();
+                console.log('Stopped States DB');
             }
+        }
 
-            if (!url && packetName !== 'example') {
-                if (options.stopDb) {
-                    if (this.objects.destroy) {
-                        await this.objects.destroy();
-                        console.log('Stopped Objects DB');
-                    }
-                    if (this.states.destroy) {
-                        await this.states.destroy();
-                        console.log('Stopped States DB');
-                    }
-                }
-
-                // Install node modules
-                await this._npmInstallWithCheck(
-                    `${tools.appName.toLowerCase()}.${packetName}${version ? `@${version}` : ''}`,
-                    options,
-                    debug,
-                );
+        // vendor packages could be scoped and thus differ in the package name
+        const npmPacketName = source.packetName
+            ? `${tools.appName.toLowerCase()}.${packetName}@npm:${source.packetName}`
+            : `${tools.appName.toLowerCase()}.${packetName}`;
 
-                return { packetName, stoppedList };
-            } else if (url && url.match(this.tarballRegex)) {
-                if (options.stopDb) {
-                    if (this.objects.destroy) {
-                        await this.objects.destroy();
-                        console.log('Stopped Objects DB');
-                    }
-                    if (this.states.destroy) {
-                        await this.states.destroy();
-                        console.log('Stopped States DB');
-                    }
-                }
+        // Install node modules
+        await this._npmInstallWithCheck(`${npmPacketName}${version ? `@${version}` : ''}`, options, debug);
 
-                // Install node modules
-                await this._npmInstallWithCheck(url, options, debug);
-                return { packetName, stoppedList };
-            } else if (!url) {
-                // Adapter
-                console.warn(
-                    `host.${hostname} Adapter "${packetName}" can be updated only together with ${tools.appName.toLowerCase()}.js-controller`,
-                );
-                return { packetName, stoppedList };
-            }
-        }
-
-        console.error(
-            `host.${hostname} Unknown packet name ${packetName}. Please install packages from outside the repository using "${tools.appNameLowerCase} url <url-or-package>"!`,
-        );
-        throw new IoBrokerError({
-            code: EXIT_CODES.UNKNOWN_PACKET_NAME,
-            message: `Unknown packetName ${packetName}. Please install packages from outside the repository using npm!`,
-        });
+        return { packetName, stoppedList };
     }
 
     /**
      * Install npm module from url
      *
-     * @param npmUrl
-     * @param options
+     * @param npmUrl parameter passed to `npm install <npmUrl>`
+     * @param options additional packet download options
      * @param debug if debug output should be printed
      */
     private async _npmInstallWithCheck(
@@ -337,8 +293,8 @@ export class Install {
 
         try {
             return await this._npmInstall({ npmUrl, options, debug, isRetry: false });
-        } catch (err) {
-            console.error(`Could not install ${npmUrl}: ${err.message}`);
+        } catch (e) {
+            console.error(`Could not install ${npmUrl}: ${e.message}`);
         }
     }
 
@@ -379,7 +335,7 @@ export class Install {
         const { npmUrl, debug, isRetry } = installOptions;
         let { options } = installOptions;
 
-        if (typeof options !== 'object') {
+        if (!tools.isObject(options)) {
             options = {};
         }
 
diff --git a/packages/cli/src/lib/setup/utils.ts b/packages/cli/src/lib/setup/utils.ts
@@ -16,21 +16,23 @@ interface GetRepositoryOptions {
  *
  * @param options Repository specific options
  */
-export async function getRepository(options: GetRepositoryOptions): Promise<Record<string, any>> {
+export async function getRepository(
+    options: GetRepositoryOptions,
+): Promise<Record<string, ioBroker.RepositoryJsonAdapterContent>> {
     const { objects } = options;
     const { repoName } = options;
 
     let repoNameOrArray: string | string[] | undefined = repoName;
     if (!repoName || repoName === 'auto') {
-        const systemConfig = await objects.getObjectAsync('system.config');
+        const systemConfig = await objects.getObject('system.config');
         repoNameOrArray = systemConfig!.common.activeRepo;
     }
 
     const repoArr = !Array.isArray(repoNameOrArray) ? [repoNameOrArray!] : repoNameOrArray;
 
-    const systemRepos = (await objects.getObjectAsync('system.repositories'))!;
+    const systemRepos = (await objects.getObject('system.repositories'))!;
 
-    const allSources = {};
+    const allSources: Record<string, ioBroker.RepositoryJsonAdapterContent> = {};
     let changed = false;
     let anyFound = false;
     for (const repoUrl of repoArr) {
@@ -62,7 +64,7 @@ export async function getRepository(options: GetRepositoryOptions): Promise<Reco
         }
 
         if (changed) {
-            await objects.setObjectAsync('system.repositories', systemRepos);
+            await objects.setObject('system.repositories', systemRepos);
         }
     }
 
diff --git a/packages/types-dev/objects.d.ts b/packages/types-dev/objects.d.ts
@@ -998,6 +998,10 @@ declare global {
             version: string;
             /** Array of blocked versions, each entry represents a semver range */
             blockedVersions: string[];
+            /** If true the unsafe perm flag is needed on install */
+            unsafePerm?: boolean;
+            /** If given, the packet name differs from the adapter name, e.g. because it is a scoped package */
+            packetName?: string;
 
             /** Other Adapter related properties, not important for this implementation */
             [other: string]: unknown;
