diff --git a/java/justfile b/java/justfile new file mode 100644 index 000000000000..e67ad738bfb7 --- /dev/null +++ b/java/justfile @@ -0,0 +1,3 @@ +import '../lib.just' + +build: (_build "java") diff --git a/java/ql/justfile b/java/ql/justfile new file mode 100644 index 000000000000..7a46396fb856 --- /dev/null +++ b/java/ql/justfile @@ -0,0 +1,6 @@ +import "../../lib.just" + +[no-cd] +format *ARGS=".": (_ql_format ARGS) + +consistency_queries := source_dir() / "consistency-queries" diff --git a/java/ql/test/justfile b/java/ql/test/justfile new file mode 100644 index 000000000000..3ce20e4a32ee --- /dev/null +++ b/java/ql/test/justfile @@ -0,0 +1,15 @@ +import "../justfile" + +base_flags := """\ + CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT="\\ " \ +""" + +all_checks := default_db_checks + """\ + --check-undefined-labels \ + --check-repeated-labels \ + --check-redefined-labels \ + --check-use-before-definition \ + --consistency-queries=""" + consistency_queries + +[no-cd] +test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS) \ No newline at end of file diff --git a/justfile b/justfile new file mode 100644 index 000000000000..a25d51fd9a5d --- /dev/null +++ b/justfile @@ -0,0 +1,2 @@ +import 'misc/just/lib.just' +import 'misc/just/forward.just' diff --git a/lib.just b/lib.just new file mode 100644 index 000000000000..0ddd926bcda5 --- /dev/null +++ b/lib.just @@ -0,0 +1 @@ +import "misc/just/lib.just" diff --git a/misc/bazel/justfile b/misc/bazel/justfile new file mode 100644 index 000000000000..55b2eb9dbd8e --- /dev/null +++ b/misc/bazel/justfile @@ -0,0 +1,5 @@ +import '../just/lib.just' + +[no-cd, positional-arguments, no-exit-message] +hello +ARGS: + @echo "hello from bzl" "$@" diff --git a/misc/codegen/justfile b/misc/codegen/justfile new file mode 100644 index 000000000000..9dd2019b5fac --- /dev/null +++ b/misc/codegen/justfile @@ -0,0 +1,5 @@ +import "../just/lib.just" + +test *ARGS="": (_bazel "test" "@codeql//misc/codegen/...") + +format *ARGS=".": (_black ARGS) \ No newline at end of file diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts new file mode 100644 index 000000000000..3002e8d394dc --- /dev/null +++ b/misc/just/codeql-test-run.ts @@ -0,0 +1,138 @@ +import * as child_process from "child_process"; +import * as path from "path"; +import * as os from "os"; +import * as fs from "fs"; + +function invoke( + invocation: string[], + options: { cwd?: string; log_prefix?: string } = {}, +): number { + const log_prefix = + options.log_prefix && options.log_prefix !== "" + ? `${options.log_prefix} ` + : ""; + console.log( + `${process.env["CMD_BEGIN"] || ""}${log_prefix}${invocation.join(" ")}${process.env["CMD_END"] || ""}`, + ); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { + stdio: "inherit", + cwd: options.cwd, + }); + } catch (error) { + return 1; + } + return 0; +} + +type Args = { + tests: string[]; + flags: string[]; + env: string[]; + codeql: string; + all: boolean; +}; + +function parseArgs(args: Args, argv: string) { + argv.split(/(? arg.replace("\\ ", " ")) + .forEach((arg) => { + if (arg.startsWith("--codeql=")) { + args.codeql = arg.split("=")[1]; + } else if (arg === "+" || arg === "--all-checks") { + args.all = true; + } else if (arg.startsWith("-")) { + args.flags.push(arg); + } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) { + args.env.push(arg); + } else if (arg !== "") { + args.tests.push(arg); + } + }); +} + +function codeqlTestRun(argv: string[]): number { + const semmle_code = process.env["SEMMLE_CODE"]; + const [language, base_args, all_args, extra_args] = argv; + const ram_per_thread = process.platform === "linux" ? 3000 : 2048; + const cpus = os.cpus().length; + let args: Args = { + tests: [], + flags: [`--ram=${ram_per_thread * cpus}`, `-j${cpus}`], + env: [], + codeql: semmle_code ? "build" : "host", + all: false, + }; + parseArgs(args, base_args); + parseArgs(args, extra_args); + if (args.all) { + parseArgs(args, all_args); + } + if (!semmle_code && (args.codeql === "build" || args.codeql === "built")) { + console.error( + "Using `--codeql=build` or `--codeql=built` requires working with the internal repository", + ); + return 1; + } + if (args.tests.length === 0) { + args.tests.push("."); + } + if (args.codeql === "build") { + if ( + invoke(["python3", "build", `target/intree/codeql-${language}`], { + cwd: semmle_code, + }) !== 0 + ) { + return 1; + } + } + if (args.codeql !== "host") { + // disable the default implicit config file, but keep an explicit one + // this is the same behavior wrt to `--codeql` as the integration test runner + process.env["CODEQL_CONFIG_FILE"] ||= "."; + } + // Set and unset environment variables + args.env.forEach((envVar) => { + const [key, value] = envVar.split("=", 2); + if (key) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } else { + console.error(`Invalid environment variable assignment: ${envVar}`); + process.exit(1); + } + }); + let codeql; + if (args.codeql === "built" || args.codeql === "build") { + codeql = path.join( + semmle_code!, + "target", + "intree", + `codeql-${language}`, + "codeql", + ); + } else if (args.codeql === "host") { + codeql = "codeql"; + } else { + codeql = args.codeql; + if (fs.lstatSync(codeql).isDirectory()) { + codeql = path.join(codeql, "codeql"); + if (process.platform === "win32") { + codeql += ".exe"; + } + } + if (!fs.existsSync(codeql)) { + console.error(`CodeQL executable not found: ${codeql}`); + return 1; + } + } + + return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], { + log_prefix: args.env.join(" "), + }); +} + +process.exit(codeqlTestRun(process.argv.slice(2))); diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts new file mode 100644 index 000000000000..7f8264e6e757 --- /dev/null +++ b/misc/just/forward-command.ts @@ -0,0 +1,92 @@ +import * as child_process from "child_process"; +import * as path from "path"; +import * as fs from "fs"; + +function commonDir(paths: string[]): string { + if (paths.length === 0) return ""; + const splitPaths = paths.map((p) => p.split(path.sep)); + let i; + for (i = 0; i < splitPaths[0].length; i++) { + if (!splitPaths.every((parts) => parts[i] === splitPaths[0][i])) { + break; + } + } + const commonParts = splitPaths[0].slice(0, i); + let ret = commonParts.join(path.sep); + if (!fs.existsSync(ret)) { + throw new Error(`Common directory does not exist: ${ret}`); + } + if (!fs.lstatSync(ret).isDirectory()) { + ret = path.dirname(ret); + } + return ret; +} + +function forwardCommand(args: string[]): number { + // Avoid infinite recursion + if (args.length == 0) { + console.error("No command provided"); + return 1; + } + const cmd = args[0]; + const envVariable = `__JUST_FORWARD_${cmd}`; + if (process.env[envVariable]) { + console.error(`No common ${cmd} handler found`); + return 1; + } + process.env[envVariable] = "true"; + const cmdArgs = args.slice(1); + // non-positional arguments are flags, + (used by language tests) or environment variable settings + const is_non_positional = /^(-.*|\+|[A-Z_][A-Z_0-9]*=.*)$/; + const flags = cmdArgs.filter((arg) => is_non_positional.test(arg)); + const positionalArgs = cmdArgs.filter( + (arg) => !is_non_positional.test(arg), + ); + + if (positionalArgs.length === 0) { + console.error("No positional arguments provided"); + return 1; + } + + const commonPath = commonDir(positionalArgs); + let relativeArgs = positionalArgs.map( + (arg) => path.relative(commonPath, arg) || ".", + ); + if (relativeArgs.length === 1 && relativeArgs[0] === ".") { + relativeArgs = []; + } + let relativeFlags = flags.map((arg) => { + // this might break in specific corner cases, but is good enough for most uses + // workaround if this doesn't work is to not use the forwarder (call just directly in the relevant directory) + if (arg.includes("=") && arg.includes(path.sep)) { + let [flags, flag_arg] = arg.split("=", 2); + flag_arg = flag_arg + .split(path.delimiter) + .map((p) => + path.isAbsolute(p) ? p : path.relative(commonPath, p), + ) + .join(path.delimiter); + return `${flags}=${flag_arg}`; + } + return arg; + }); + + const invocation = [ + process.env["JUST_EXECUTABLE"] || "just", + cmd, + ...relativeFlags, + ...relativeArgs, + ]; + console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { + stdio: "inherit", + cwd: commonPath, + }); + } catch (error) { + return 1; + } + return 0; +} + +process.exit(forwardCommand(process.argv.slice(2))); diff --git a/misc/just/forward.just b/misc/just/forward.just new file mode 100644 index 000000000000..64af38d272fa --- /dev/null +++ b/misc/just/forward.just @@ -0,0 +1,28 @@ +import "lib.just" + +# copy&paste necessary for each command until proper forwarding of multiple args is implemented +# see https://github.com/casey/just/issues/1988 + +_forward := tsx + ' "' + source_dir() + '/forward-command.ts"' + + +[no-cd, positional-arguments, no-exit-message] +@test *ARGS=".": + {{ _forward }} test "$@" + + +[no-cd, positional-arguments, no-exit-message] +@build *ARGS=".": + {{ _forward }} build "$@" + +[no-cd, positional-arguments, no-exit-message] +@generate *ARGS=".": + {{ _forward }} generate "$@" + +[no-cd, positional-arguments, no-exit-message] +@lint *ARGS=".": + {{ _forward }} lint "$@" + +[no-cd, positional-arguments, no-exit-message] +@format *ARGS=".": + {{ _forward }} format "$@" diff --git a/misc/just/justfile b/misc/just/justfile new file mode 100644 index 000000000000..bfa7bed4db2e --- /dev/null +++ b/misc/just/justfile @@ -0,0 +1,2 @@ +format *ARGS=".": + npx prettier --write {{ ARGS }} diff --git a/misc/just/lib.just b/misc/just/lib.just new file mode 100644 index 000000000000..ffb8731a9c09 --- /dev/null +++ b/misc/just/lib.just @@ -0,0 +1,74 @@ +set fallback +set allow-duplicate-recipes +set allow-duplicate-variables +set unstable + +export PATH_SEP := if os() == "windows" { ";" } else { ":" } +export JUST_EXECUTABLE := just_executable() + +error := style("error") + "error" + NORMAL + ": " +cmd_sep := "\n#--------------------------------------------------------\n" +export CMD_BEGIN := style("command") + cmd_sep +export CMD_END := cmd_sep + NORMAL + +tsx := "npx tsx@4.19.0" + +import? '../../../semmle-code.just' # internal repo just file, if present +import 'semmle-code-stub.just' + + +[no-exit-message] +@_require_semmle_code: + {{ if SEMMLE_CODE == "" { ''' + echo "''' + error + ''' running this recipe requires doing so from an internal repository checkout" >&2 + exit 1 + ''' } else { "" } }} + +_build LANGUAGE: _require_semmle_code (_maybe_build LANGUAGE) + +[no-exit-message] +_maybe_build LANGUAGE: + {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd "$SEMMLE_CODE"; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }} + + +default_db_checks := """\ + --check-databases \ + --check-diff-informed \ + --fail-on-trap-errors \ +""" + +[no-cd, positional-arguments, no-exit-message] +@_codeql_test LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS: + {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@" + + +[no-cd, no-exit-message] +_ql_format +ARGS: (_maybe_build "nolang") + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"' } else { 'codeql' } }} query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }} + + +[no-cd, no-exit-message] +_bazel COMMAND *ARGS: + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/tools/bazel"' } else { 'bazel' } }} {{ COMMAND }} {{ ARGS }}{{ cmd_sep }} + +[no-cd, no-exit-message] +_sembuild *ARGS: (_run_in_semmle_code "./build" ARGS) + +[no-cd, no-exit-message] +_integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" ARGS) + +[no-cd] +_run +ARGS: + {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }} + +[no-cd] +_run_in_semmle_code +ARGS: _require_semmle_code + {{ cmd_sep }}cd "$SEMMLE_CODE"; {{ ARGS }}{{ cmd_sep }} + + +[no-cd, positional-arguments, no-exit-message] +_just +ARGS: + "{{ JUST_EXECUTABLE }}" "$@" + +[no-cd] +_black *ARGS=".": (_run "uv" "run" "black" ARGS) diff --git a/misc/just/semmle-code-stub.just b/misc/just/semmle-code-stub.just new file mode 100644 index 000000000000..14733ffb648e --- /dev/null +++ b/misc/just/semmle-code-stub.just @@ -0,0 +1 @@ +export SEMMLE_CODE := "" diff --git a/rust/justfile b/rust/justfile new file mode 100644 index 000000000000..b0f56a7fb00a --- /dev/null +++ b/rust/justfile @@ -0,0 +1,11 @@ +import '../lib.just' + +install: (_bazel "run" "@codeql//rust:install") + +build: generate (_build "rust") + +generate: (_bazel "run" "@codeql//rust/codegen") + +lint: (_run "python3" "lint.py") + +format: (_run "python3" "lint.py" "--format-only") diff --git a/rust/lint.py b/rust/lint.py index 600a888649e9..e3a078635fa5 100755 --- a/rust/lint.py +++ b/rust/lint.py @@ -4,6 +4,14 @@ import pathlib import shutil import sys +import argparse + + +def options(): + parser = argparse.ArgumentParser(description="lint rust language pack code") + parser.add_argument("--format-only", action="store_true", help="Only apply formatting") + return parser.parse_args() + def tool(name): @@ -12,27 +20,35 @@ def tool(name): return ret -this_dir = pathlib.Path(__file__).resolve().parent +def main(): + args = options() + this_dir = pathlib.Path(__file__).resolve().parent + + + cargo = tool("cargo") + bazel = tool("bazel") + + runs = [] -cargo = tool("cargo") -bazel = tool("bazel") -runs = [] + def run(tool, args, *, cwd=this_dir): + print("+", tool, args) + runs.append(subprocess.run([tool] + args.split(), cwd=cwd)) -def run(tool, args, *, cwd=this_dir): - print("+", tool, args) - runs.append(subprocess.run([tool] + args.split(), cwd=cwd)) + # make sure bazel-provided sources are put in tree for `cargo` to work with them + run(bazel, "run ast-generator:inject-sources") + run(cargo, "fmt --all --quiet") + if not args.format_only: + for manifest in this_dir.rglob("Cargo.toml"): + if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"): + run(cargo, + "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings", + cwd=manifest.parent) -# make sure bazel-provided sources are put in tree for `cargo` to work with them -run(bazel, "run ast-generator:inject-sources") -run(cargo, "fmt --all --quiet") + return max(r.returncode for r in runs) -for manifest in this_dir.rglob("Cargo.toml"): - if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"): - run(cargo, - "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings", - cwd=manifest.parent) -sys.exit(max(r.returncode for r in runs)) +if __name__ == "__main__": + sys.exit(main()) diff --git a/rust/ql/integration-tests/justfile b/rust/ql/integration-tests/justfile new file mode 100644 index 000000000000..c8fad074a3f9 --- /dev/null +++ b/rust/ql/integration-tests/justfile @@ -0,0 +1,9 @@ +import "../../../misc/just/lib.just" + + +[no-cd] +test *ARGS=".": (_just "generate") (_integration_test ARGS) + +# TODO in separate PR +# [no-cd] +# format *ARGS=".": (_ql_format ARGS) (_black ARGS) diff --git a/rust/ql/justfile b/rust/ql/justfile new file mode 100644 index 000000000000..3a8b699e2322 --- /dev/null +++ b/rust/ql/justfile @@ -0,0 +1,10 @@ +import "../../lib.just" + +[no-cd] +format *ARGS=".": (_ql_format ARGS) + +all_checks := default_db_checks + """\ + --consistency-queries=""" + source_dir() / "consistency-queries" + +[no-cd] +test *ARGS=".": (_codeql_test "rust" "" all_checks ARGS) diff --git a/swift/justfile b/swift/justfile new file mode 100644 index 000000000000..cd7cf3ce583c --- /dev/null +++ b/swift/justfile @@ -0,0 +1,14 @@ +import '../lib.just' + +install: (_bazel "run" "@codeql//swift:install") + +build: (_build "swift") + +generate: (_bazel "run" "@codeql//swift/codegen") + +@_check_clang_format: + if ! which clang-format > /dev/null; then \ + "{{ JUST_EXECUTABLE }}" _run_in_semmle_code "tools/bazel" "run" "//c/clang-format:install"; \ + fi + +format ARGS=".": _check_clang_format (_run "clang-format" "-i" ("$(find " + ARGS + " -type f -name '*.h' -or -name '*.cpp')")) diff --git a/swift/ql/integration-tests/justfile b/swift/ql/integration-tests/justfile new file mode 100644 index 000000000000..370f7ef87794 --- /dev/null +++ b/swift/ql/integration-tests/justfile @@ -0,0 +1,9 @@ +import "../../../lib.just" + + +[no-cd] +test *ARGS=".": (_just "generate") (_integration_test ARGS) + +# TODO in separate PR +# [no-cd] +# format *ARGS=".": (_ql_format ARGS) (_black ARGS) diff --git a/swift/ql/justfile b/swift/ql/justfile new file mode 100644 index 000000000000..7fe1fd1faa5e --- /dev/null +++ b/swift/ql/justfile @@ -0,0 +1,14 @@ +import "../../lib.just" + +[no-cd] +format *ARGS=".": (_ql_format ARGS) + +all_checks := default_db_checks + """\ + --check-repeated-labels \ + --check-redefined-labels \ + --check-use-before-definition \ + --consistency-queries=""" + source_dir() / "consistency-queries" + + +[no-cd] +test *ARGS=".": (_codeql_test "swift" "" all_checks ARGS)