From 303b5eb47624c330ae40ef96e02d7c6e7f4cf8b9 Mon Sep 17 00:00:00 2001 From: Eric Fornaciari Date: Mon, 24 Nov 2025 13:55:33 -0800 Subject: [PATCH] DF-22512: Adds tooling to generate EA fields to a CSV for review --- package.json | 1 + .../generate-adapter-field-csv/generator.ts | 216 ++++++++++++++++++ .../src/generate-adapter-field-csv/index.ts | 160 +++++++++++++ .../readmeBlacklist.json | 59 +++++ packages/scripts/src/shared/docGenTypes.d.ts | 2 + 5 files changed, 438 insertions(+) create mode 100644 packages/scripts/src/generate-adapter-field-csv/generator.ts create mode 100644 packages/scripts/src/generate-adapter-field-csv/index.ts create mode 100644 packages/scripts/src/generate-adapter-field-csv/readmeBlacklist.json diff --git a/package.json b/package.json index 2b4d439b0c..8ecd2a05ae 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "test:ci:integration": "EA_PORT=0 METRICS_ENABLED=false jest --coverage integration", "generate:master-list": "METRICS_ENABLED=false ts-node-transpile-only ./packages/scripts/src/generate-master-list", "generate:readme": "METRICS_ENABLED=false ts-node-transpile-only ./packages/scripts/src/generate-readme", + "generate:adapter-field-csv": "METRICS_ENABLED=false ts-node-transpile-only ./packages/scripts/src/generate-adapter-field-csv", "postinstall": "husky install", "cm": "cz", "ci-changeset": "git reset --hard $CURRENT_SHA && yarn changeset version", diff --git a/packages/scripts/src/generate-adapter-field-csv/generator.ts b/packages/scripts/src/generate-adapter-field-csv/generator.ts new file mode 100644 index 0000000000..7594cc2ce1 --- /dev/null +++ b/packages/scripts/src/generate-adapter-field-csv/generator.ts @@ -0,0 +1,216 @@ +import { Adapter } from '@chainlink/external-adapter-framework/adapter' +import { SettingsDefinitionMap } from '@chainlink/external-adapter-framework/config' +import fs from 'fs' +import path from 'path' +import process from 'process' +import { test } from 'shelljs' +import * as generatorPack from '../../package.json' +import { + EndpointDetails, + EnvVars, + Package, + RateLimits, + RateLimitsSchema, + Schema, +} from '../shared/docGenTypes' +import { getJsonFile, getMdFile } from '../shared/docGenUtils' +import { WorkspaceAdapter } from '../workspace' + +const testEnvOverrides = { + API_VERBOSE: 'true', + EA_PORT: '0', + LOG_LEVEL: 'debug', + METRICS_ENABLED: 'false', + NODE_ENV: undefined, + RECORD: undefined, + WS_ENABLED: undefined, +} + +const TRUNCATE_EXAMPLE_LINES = 500 + +const genSig = + 'This document was generated automatically. Please see [README Generator](../../scripts#readme-generator) for more info.' + +const exampleTextHeader = '### Example\n\n' + +const noExampleText = 'There are no examples for this endpoint.' + +const checkFilePaths = (filePaths: string[]): string => { + for (const filePath of filePaths) { + if (test('-f', filePath)) return filePath + } + throw Error(`No file found in the following paths: ${filePaths.join(',')}`) +} + +// Checks if an object appears to be an instance of the given class while +// allowing for the possibility that the class is from a different version +// of the package. +const objectHasShapeOfClass = (object, Class): boolean => { + if (object instanceof Class) { + return true + } + + const objectProps = Object.getOwnPropertyNames(object.constructor.prototype) + const classProps = Object.getOwnPropertyNames(Class.prototype) + + if (objectProps.length !== classProps.length) { + return false + } + + for (const prop of classProps) { + if (!objectProps.includes(prop)) { + return false + } + } + return true +} + +export class AdapterFieldGenerator { + schemaDescription: string + adapterPath: string + adapterType: string + schemaPath: string + packageJson: Package + defaultEndpoint = '' + defaultBaseUrl = '' + endpointDetails: EndpointDetails = {} + envVars: EnvVars + rateLimitsPath: string + rateLimits: RateLimits + integrationTestPath: string | null + name: string + readmeText = '' + requiredEnvVars: string[] + verbose: boolean + version: string + versionBadgeUrl: string + license: string + frameworkVersion: 'v2' | 'v3' + frameworkVersionBadgeUrl: string + knownIssuesPath: string + knownIssuesSection: string | null + + constructor(adapter: WorkspaceAdapter, verbose = false) { + this.verbose = verbose + this.adapterPath = adapter.location + + if (!this.adapterPath.endsWith('/')) this.adapterPath += '/' + if (!test('-d', this.adapterPath)) throw Error(`${this.adapterPath} is not a directory`) + if (verbose) console.log(`${this.adapterPath}: Checking package.json`) + const packagePath = checkFilePaths([this.adapterPath + 'package.json']) + const packageJson = getJsonFile(packagePath) as Package + + if (packageJson.dependencies) { + const adapterEA = packageJson.dependencies['@chainlink/external-adapter-framework'] + if (adapterEA) { + this.frameworkVersion = 'v3' + const generatorEA = generatorPack.dependencies['@chainlink/external-adapter-framework'] + if (adapterEA != generatorEA) { + console.log(`This generator uses ${generatorEA} but ${adapter.name} uses ${adapterEA}`) + } + } else { + this.frameworkVersion = 'v2' + } + } + + this.version = packageJson.version ?? '' + this.versionBadgeUrl = `https://img.shields.io/github/package-json/v/smartcontractkit/external-adapters-js?filename=${packagePath}` + this.frameworkVersionBadgeUrl = `https://img.shields.io/badge/framework%20version-${this.frameworkVersion}-blueviolet` + this.license = packageJson.license ?? '' + + this.knownIssuesPath = this.adapterPath + 'docs/known-issues.md' + this.schemaPath = this.adapterPath + 'schemas/env.json' + this.rateLimitsPath = this.adapterPath + 'src/config/limits.json' + this.integrationTestPath = this.adapterPath + 'test/integration/*.test.ts' + this.packageJson = packageJson + } + + // We need to require/import adapter contents to generate the README. + // We use this function instead of the constructor because we need to fetch, and constructors can't be async. + async loadAdapterContent(): Promise { + if (fs.existsSync(this.schemaPath)) { + const configPath = checkFilePaths([ + this.adapterPath + 'src/config.ts', + this.adapterPath + 'src/config/index.ts', + ]) + const configFile = await require(path.join(process.cwd(), configPath)) + + //Is V2. Populate self w/ env.json content + if (this.verbose) console.log(`${this.adapterPath}: Checking schema/env.json`) + const schema = getJsonFile(this.schemaPath) as Schema + let rateLimits + try { + rateLimits = getJsonFile(this.rateLimitsPath) + } catch (e) { + rateLimits = { http: {} } + } + this.frameworkVersion = 'v2' + this.schemaDescription = schema.description ?? '' + this.name = schema.title ?? this.packageJson.name ?? '' + this.envVars = schema.properties ?? {} + this.rateLimits = (rateLimits as RateLimitsSchema)?.http || {} + this.requiredEnvVars = schema.required ?? [] + this.defaultEndpoint = configFile.DEFAULT_ENDPOINT + this.defaultBaseUrl = configFile.DEFAULT_BASE_URL || configFile.DEFAULT_WS_API_ENDPOINT + + if (this.verbose) console.log(`${this.adapterPath}: Importing src/endpoint/index.ts`) + const endpointPath = checkFilePaths([this.adapterPath + 'src/endpoint/index.ts']) + this.endpointDetails = await require(path.join(process.cwd(), endpointPath)) + } else { + this.frameworkVersion = 'v3' + if (this.verbose) + console.log(`${this.adapterPath}: Importing framework adapter to read properties`) + + //Framework adapters don't use env.json. Instead, populate "schema" with import + const adapterImport = await import( + path.join(process.cwd(), this.adapterPath, 'dist', 'index.js') + ) + + const adapter = adapterImport.adapter as Adapter + const adapterSettings = ( + adapter.config as unknown as { settingsDefinition: SettingsDefinitionMap } + ).settingsDefinition + this.name = adapter.name + this.envVars = adapterSettings || {} + this.rateLimits = adapter.rateLimiting?.tiers || {} + + this.endpointDetails = adapter.endpoints?.length + ? adapter.endpoints.reduce( + (accumulator, endpoint) => + Object.assign(accumulator, { + [endpoint.name]: { + ...endpoint, + supportedEndpoints: [endpoint.name, ...(endpoint.aliases || [])], + }, + }), + {}, + ) + : {} + + this.requiredEnvVars = adapterSettings + ? Object.keys(adapterSettings).filter((k) => adapterSettings[k].required === true) ?? [] // Keys of required customSettings + : [] + //Note, not populating description, doesn't exist in framework adapters + this.defaultEndpoint = adapter.defaultEndpoint ?? '' + } + + if (fs.existsSync(this.knownIssuesPath)) { + this.knownIssuesSection = getMdFile(this.knownIssuesPath) || null + } + } + + buildCSV(): string[] { + if (this.verbose) console.log(`${this.adapterPath}: Generating csv`) + + const envVars = this.envVars + + if (this.frameworkVersion === 'v2') { + return [] + } + + return Object.entries(envVars).map(([field, v]) => { + const sensitive = v.sensitive ? 'true' : 'false' + return [this.name, field, sensitive].map(String).join(',') + }) + } +} diff --git a/packages/scripts/src/generate-adapter-field-csv/index.ts b/packages/scripts/src/generate-adapter-field-csv/index.ts new file mode 100644 index 0000000000..bf76e8c9b1 --- /dev/null +++ b/packages/scripts/src/generate-adapter-field-csv/index.ts @@ -0,0 +1,160 @@ +import commandLineArgs from 'command-line-args' +import commandLineUsage from 'command-line-usage' +import { writeFileSync } from 'fs' +import { Blacklist, BooleanMap } from '../shared/docGenTypes' +import { getJsonFile } from '../shared/docGenUtils' +import { getWorkspaceAdapters, getWorkspacePackages } from '../workspace' +import { AdapterFieldGenerator } from './generator' + +const pathToBlacklist = 'packages/scripts/src/generate-readme/readmeBlacklist.json' + +const findTypeAndName = new RegExp( + // For example, packages/(sources)/(coinbase) + /packages\/(sources|composites|examples|targets|non-deployable)\/(.*)/, +) +export async function main(): Promise { + try { + // Define CLI options + const commandLineOptions = [ + { + name: 'all', + alias: 'a', + type: Boolean, + description: 'Generate READMEs for all source EAs not in blacklist', + }, + { + name: 'verbose', + alias: 'v', + type: Boolean, + description: 'Include extra logs for each generation process', + }, + { + name: 'testPath', + alias: 't', + type: String, + description: 'Run script as test for EA along given path', + }, + { + name: 'adapters', + multiple: true, + defaultOption: true, + description: 'Generate READMEs for all source EAs given by name', + }, + { name: 'help', alias: 'h', type: Boolean, description: 'Display usage guide' }, + ] + const options = commandLineArgs(commandLineOptions) + + // Generate usage guide + if (options.help) { + const usage = commandLineUsage([ + { + header: 'README Generator Script', + content: + 'This script is run from the root of the external-adapter-js/ repo to generate READMEs for supported external adapters. This functionality is currently limited to a subset of source adapters only.', + }, + { + header: 'Options', + optionList: commandLineOptions, + }, + { + content: + 'Source code: {underline https://github.com/smartcontractkit/external-adapters-\njs/packages/scripts/src/generate-readme/}', + }, + ]) + console.log(usage) + return + } + + const shouldBuildAll = + options.all || + getWorkspacePackages(process.env['UPSTREAM_BRANCH']).find( + (p) => p.type === 'core' || p.type === 'scripts', + ) + + let adapters = shouldBuildAll + ? getWorkspaceAdapters() + : getWorkspaceAdapters([], process.env['UPSTREAM_BRANCH']) + + // Test setting + if (options.testPath) { + const adapter = adapters.find((a) => a.location === options.testPath) + if (!adapter) { + console.error(`Adapter at ${options.testPath} was not found`) + return + } + const generator = new AdapterFieldGenerator(adapter, options.verbose) + await generator.loadAdapterContent() + const csv = generator.buildCSV().join('\n') + writeFileSync(`${adapter.descopedName}.csv`, csv) + return + } + + options.verbose && + console.log( + `Adapters being considered for readme generation: `, + adapters.map((a) => `${a.name}: ${a.location}`), + ) + + const initialAdapterLength = adapters.length + + // If specific adapters are passed to the command line, only select those + if (options.adapters?.length) { + options.verbose && + console.log(`Reducing list of adapters to ones specified on the command line`) + adapters = adapters.filter((p) => { + return ( + (options.adapters as string[]).includes(p.descopedName) || // p.descopedName example: "coinbase-adapter" + (options.adapters as string[]).includes(p.descopedName.replace(/-adapter$/, '')) // "coinbase" (without "-adapter") + ) + }) + } + + const blacklist = (getJsonFile(pathToBlacklist) as Blacklist).blacklist + const adapterInBlacklist = blacklist.reduce((map: BooleanMap, a) => { + const adapterName = `@chainlink/${a}-adapter` + map[adapterName] = true + return map + }, {}) + options.verbose && console.log(`Removing blacklisted and non-source adapters from the list`) + adapters = adapters + .filter((a) => !adapterInBlacklist[a.name]) // Remove blacklisted adapters + .filter((p) => p.type === 'sources' || p.type === 'composites') // Remove non-source and non-composite adapters + + options.verbose && + console.log(`Filtered ${initialAdapterLength - adapters.length} adapters from the list`) // Verbose because this message is confusing if you're not familiar with generate-readme + console.log( + 'Generating README(s) for the following adapters: ', + adapters.map((a) => a.name), + ) + + // Collect CSVs from each generator + const csvQueue = await Promise.all( + adapters + .map(async (adapter) => { + const generator = new AdapterFieldGenerator(adapter, options.verbose) + await generator.loadAdapterContent() + return generator + }) + .filter(async (generator) => { + return (await generator).frameworkVersion === 'v3' + }) + .map(async (generator) => { + return (await generator).buildCSV().join('\n') + }), + ) + + // Combine all CSVs into one + const combinedCSV = csvQueue.join('\n') + + // Save the combined CSV to a file + writeFileSync('combined_adapters.csv', combinedCSV) + console.log(`Combined CSV generated with ${csvQueue.length} adapters.`) + + process.exit(0) + } catch (error) { + console.error({ error: error.message, stack: error.stack }) + process.exit(1) + } +} + +main() diff --git a/packages/scripts/src/generate-adapter-field-csv/readmeBlacklist.json b/packages/scripts/src/generate-adapter-field-csv/readmeBlacklist.json new file mode 100644 index 0000000000..b624df48fa --- /dev/null +++ b/packages/scripts/src/generate-adapter-field-csv/readmeBlacklist.json @@ -0,0 +1,59 @@ +{ + "blacklist": [ + "ada-balance", + "alphachain", + "alpine", + "anchor", + "anyblock", + "apy-finance", + "augur", + "anchorage", + "bitcoin-json-rpc", + "blockchain.com", + "blockchair", + "chain-reserve-wallet", + "circuit-breaker", + "coingecko", + "crypto-volatility-index", + "curve-3pool", + "defi-dozen", + "defi-pulse", + "dlc-btc-por", + "dxdao", + "dydx-stark", + "generic-api", + "graphql", + "harmony", + "historical-average", + "implied-price", + "implied-price-test", + "json-rpc", + "layer2-sequencer-health", + "linear-finance", + "linkpool", + "lition", + "market-closure", + "medianizer", + "messari", + "multi-address-list", + "nav-generic", + "onchain", + "onchain-gas", + "outlier-detection", + "paypal", + "proof-of-reserves", + "reduce", + "savax-price", + "set-token-index", + "sportsdataio", + "synthetix-debt-pool", + "the-graph", + "token-allocation", + "vesper", + "view-function-multi-chain", + "xsushi-price", + "coinbase-prime", + "harris-and-trotter", + "nav-consulting" + ] +} diff --git a/packages/scripts/src/shared/docGenTypes.d.ts b/packages/scripts/src/shared/docGenTypes.d.ts index ec08a745e7..ff4e4c5c70 100644 --- a/packages/scripts/src/shared/docGenTypes.d.ts +++ b/packages/scripts/src/shared/docGenTypes.d.ts @@ -28,6 +28,8 @@ export type EnvVars = { description?: string options?: readonly string[] type?: string + sensitive?: boolean + required?: boolean } }