CDS Extractor Rewrite Phase 2 : Improve Performance and Precision

.gitignore

@@ -71,3 +71,5 @@ tmp/
 **.testproj
 dbs
 *.cds.json
+.cds-extractor-cache
+

extractors/README.md

@@ -30,16 +30,20 @@ pre-finalize.sh`"]
     JSE[[javascript extractor]]
     DTRAC[codeql database<br>trace-command]
     SPF[[pre-finalize.sh]]
-    DIDX[codeql database index-files<br> --language=cds<br>--include-extension=.cds]
-    SIF[[index-files.sh]]
-    SIT[[index-files.ts/js]]
-    NPM[[npm install & build]]
-    DETS[[Determine CDS command]]
-    FIND[[Find package.json dirs]]
-    INST[[Install dependencies]]
-    CC[[cds compiler]]
+    ABCMD[[autobuild.sh/cmd]]
+    ABT[[cds-extractor.ts/js]]
+    ENV[[setup & validate<br>environment]]
+    PDG[[build project<br>dependency graph]]
+    INSTC[[install dependencies<br>with caching]]
+    PROC[[process CDS files<br>to JSON]]
+    PMAP[[project-aware<br>dependency resolution]]
+    FIND[[find project for<br>CDS file]]
+    CDCMD[[determine CDS<br>command for project]]
+    COMP[[compile CDS<br>to JSON]]
     CDJ([.cds.json files])
+    FILT[[configure LGTM<br>index filters]]
     JSA[[javascript extractor<br>autobuild script]]
+    DIAG[[add compilation<br>diagnostics]]
     TF([CodeQL TRAP files])
     DBF[codeql database finalize<br> -- /path/to/database]
 
@@ -54,20 +58,30 @@ pre-finalize.sh`"]
     JSE ==> |run autobuild within<br>the javascript extractor| DTRAC
 
     DTRAC ==> |run the build --command| SPF
-    SPF ==> |run codeql index-files<br>for CDS files| DIDX
-    DIDX ==> |invoke script via<br>--search-path| SIF
-    SIF ==> |runs TypeScript version<br>after npm install| NPM
-    NPM ==> |executes compiled<br>index-files.js| SIT
+    SPF ==> |run autobuilder<br>for CDS files| ABCMD
+    ABCMD ==> |runs TypeScript version<br>of CDS extractor| ABT
 
-    SIT ==> |finds project directories<br>with package.json| FIND
-    FIND ==> |install CDS dependencies<br>in project directories| INST
-    SIT ==> |determines which<br>cds command to use| DETS
-    DETS ==> |processes each CDS file| CC
+    ABT ==> |setup and validate<br>environment first| ENV
+    ABT ==> |build project dependency<br>graph for source root| PDG
+    PDG ==> |analyze CDS projects<br>structure & relationships| PMAP
+
+    ABT ==> |efficiently install<br>required dependencies| INSTC
+    INSTC ==> |use cached approach for<br>dependency installation| PMAP
+
+    ABT ==> |process each CDS file<br>to generate JSON files| PROC
+    PROC ==> |find which project<br>contains this CDS file| FIND
+    FIND ==> |uses project-aware<br>dependency resolution| PMAP
+    FIND ==> |determine appropriate<br>CDS command for project| CDCMD
+
+    CDCMD ==> |compile CDS file to JSON<br>with project context| COMP
+    COMP ==> |generate JSON representation<br>with project awareness| CDJ
+    COMP --x |if compilation fails,<br>report diagnostics| DIAG
+    DIAG -.-> |diagnostics stored<br>in database| DB
 
-    CC ==> |compile .cds files to<br>create .cds.json files| CDJ
     CDJ -.-> |stored in same location<br>as original .cds files| DB
 
-    SIT ==> |configures extraction<br>filters for JSON files| JSA
+    ABT ==> |configure extraction<br>filters for JSON files| FILT
+    ABT ==> |run JavaScript extractor<br>to process JSON files| JSA
     JSA ==> |processes .cds.json files<br>via javascript extractor| CDJ
 
     CDJ ==> |javascript extractor<br>generates TRAP files| TF

extractors/cds/tools/.gitignore

@@ -1,3 +1,6 @@
+# Ignore files create just for debugging the CDS extractor.
+debug/
+
 # Ignore the entire "out" directory as this is for the .js and .js.map files
 # which are generated by the `tsc` build process. In the current project config,
 # we require the platform-specific "index-files" shell/cmd script to run the

extractors/cds/tools/cds-extractor.ts

@@ -0,0 +1,170 @@
+import {
+  buildCdsProjectDependencyGraph,
+  compileCdsToJson,
+  determineCdsCommand,
+  findProjectForCdsFile,
+} from './src/cds';
+import { CdsProjectMapWithDebugSignals } from './src/cds/parser/types';
+import { runJavaScriptExtractor } from './src/codeql';
+import { addCompilationDiagnostic } from './src/diagnostics';
+import { configureLgtmIndexFilters, setupAndValidateEnvironment } from './src/environment';
+import { installDependencies } from './src/packageManager';
+import { RunMode } from './src/runMode';
+import { validateArguments } from './src/utils';
+
+// Validate arguments to this script.
+// The first argument we pass is the expected run mode, which will be extracted from process.argv[2]
+// This will determine the correct minimum argument count for validation
+const validationResult = validateArguments(process.argv, RunMode.AUTOBUILD);
+if (!validationResult.isValid) {
+  console.warn(validationResult.usageMessage);
+  // Exit with an error code on invalid use of this script.
+  process.exit(1);
+}
+
+// Get the validated and sanitized arguments
+const { runMode, sourceRoot } = validationResult.args!;
+
+// Check for autobuild mode
+if (runMode === (RunMode.AUTOBUILD as string)) {
+  console.log('Autobuild mode is not implemented yet.');
+  process.exit(1);
+}
+
+// Setup the environment and validate all requirements first, before changing directory
+// This ensures we can properly locate the CodeQL tools
+const {
+  success: envSetupSuccess,
+  errorMessages,
+  codeqlExePath,
+  autobuildScriptPath,
+  platformInfo,
+} = setupAndValidateEnvironment(sourceRoot);
+
+if (!envSetupSuccess) {
+  const codeqlExe = platformInfo.isWindows ? 'codeql.exe' : 'codeql';
+  console.warn(
+    `'${codeqlExe} database index-files --language cds' terminated early due to: ${errorMessages.join(
+      ', ',
+    )}.`,
+  );
+  // Exit with an error code when environment setup fails.
+  process.exit(1);
+}
+
+// Force this script, and any process it spawns, to use the project (source) root
+// directory as the current working directory.
+process.chdir(sourceRoot);
+
+console.log(
+  `INFO: CodeQL CDS extractor using run mode '${runMode}' for scan of project source root directory '${sourceRoot}'.`,
+);
+
+// Using the new project-aware approach to find CDS projects and their dependencies
+console.log('Detecting CDS projects and analyzing their structure...');
+
+// Build the project dependency graph using the project-aware parser
+// Pass the script directory (__dirname) to support debug-parser mode internally
+const projectMap = buildCdsProjectDependencyGraph(sourceRoot, runMode, __dirname);
+
+// Cast to the interface with debug signals to properly handle debug mode
+const typedProjectMap = projectMap as CdsProjectMapWithDebugSignals;
+
+// Check if we're in debug-parser mode and should exit (based on signals from buildCdsProjectDependencyGraph)
+if (typedProjectMap.__debugParserSuccess) {
+  console.log('Debug parser mode completed successfully.');
+  process.exit(0);
+} else if (typedProjectMap.__debugParserFailure) {
+  console.warn('No CDS projects found. Cannot generate debug information.');
+  process.exit(1);
+}
+
+// Install dependencies of discovered CAP/CDS projects
+console.log('Ensuring depencencies are installed in cache for required CDS compiler versions...');
+const projectCacheDirMap = installDependencies(projectMap, sourceRoot, codeqlExePath);
+
+const cdsFilePathsToProcess: string[] = [];
+
+console.log('Extracting CDS files from discovered projects...');
+
+// Use the project map to collect all `.cds` files from each project.
+// We want to "extract" all `.cds` files from all projects so that we have a copy
+// of each `.cds` source file in the CodeQL database.
+for (const [, project] of projectMap.entries()) {
+  cdsFilePathsToProcess.push(...project.cdsFiles);
+}
+
+console.log('Processing CDS files to JSON ...');
+
+// Collect files that need compilation, handling project-level compilation
+const cdsFilesToCompile: string[] = [];
+const projectsForProjectLevelCompilation = new Set<string>();
+
+for (const [projectDir, project] of projectMap.entries()) {
+  if (project.cdsFilesToCompile.includes('__PROJECT_LEVEL_COMPILATION__')) {
+    // This project needs project-level compilation
+    projectsForProjectLevelCompilation.add(projectDir);
+    // We'll only compile one file per project to trigger project-level compilation
+    // Use the first CDS file as a representative
+    if (project.cdsFiles.length > 0) {
+      cdsFilesToCompile.push(project.cdsFiles[0]);
+    }
+  } else {
+    // Normal individual file compilation
+    cdsFilesToCompile.push(...project.cdsFilesToCompile);
+  }
+}
+
+console.log(
+  `Found ${cdsFilePathsToProcess.length} total CDS files, ${cdsFilesToCompile.length} files to compile (${projectsForProjectLevelCompilation.size} project-level compilations)`,
+);
+
+// Evaluate each `.cds` source file that should be compiled to JSON.
+for (const rawCdsFilePath of cdsFilesToCompile) {
+  try {
+    // Find which project this CDS file belongs to, to use the correct cache directory
+    const projectDir = findProjectForCdsFile(rawCdsFilePath, sourceRoot, projectMap);
+    const cacheDir = projectDir ? projectCacheDirMap.get(projectDir) : undefined;
+
+    // Determine the CDS command to use based on the cache directory for this specific file
+    const cdsCommand = determineCdsCommand(cacheDir);
+
+    // Use resolved path directly instead of passing through getArg
+    // Pass the project dependency information to enable project-aware compilation
+    const compilationResult = compileCdsToJson(
+      rawCdsFilePath,
+      sourceRoot,
+      cdsCommand,
+      cacheDir,
+      projectMap,
+      projectDir,
+    );
+
+    if (!compilationResult.success && compilationResult.message) {
+      console.error(
+        `ERROR: adding diagnostic for source file=${rawCdsFilePath} : ${compilationResult.message} ...`,
+      );
+      addCompilationDiagnostic(rawCdsFilePath, compilationResult.message, codeqlExePath);
+    }
+  } catch (errorMessage) {
+    console.error(
+      `ERROR: adding diagnostic for source file=${rawCdsFilePath} : ${String(errorMessage)} ...`,
+    );
+    addCompilationDiagnostic(rawCdsFilePath, String(errorMessage), codeqlExePath);
+  }
+}
+
+// Configure the "LGTM" index filters for proper extraction.
+configureLgtmIndexFilters();
+
+// Run CodeQL's JavaScript extractor to process the compiled JSON files.
+const extractorResult = runJavaScriptExtractor(sourceRoot, autobuildScriptPath, codeqlExePath);
+if (!extractorResult.success && extractorResult.error) {
+  console.error(`Error running JavaScript extractor: ${extractorResult.error}`);
+}
+
+// Use the `cds-extractor.js` name in the log message as that is the name of the script
+// that is actually run by the `codeql database index-files` command. This TypeScript
+// file is where the code/logic is edited/implemented, but the runnable script is
+// generated by the TypeScript compiler and is named `cds-extractor.js`.
+console.log(`Completed run of cds-extractor.js script for CDS extractor.`);