Just: introduce common "verbs"

java/justfile

@@ -0,0 +1,3 @@
+import '../lib.just'
+
+build: (_build_dist "java")

java/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"

java/ql/test/justfile

@@ -0,0 +1,16 @@
+import "../justfile"
+
+base_flags := """\
+     CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT="\\ " \
+"""
+
+all_checks := default_db_checks + """\
+    --check-undefined-labels \
+    --check-repeated-labels \
+    --check-redefined-labels \
+    --check-use-before-definition \
+    --consistency-queries=""" + consistency_queries
+
+[no-cd]
+test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS)
+

justfile

@@ -0,0 +1,4 @@
+# see misc/just/README.md for an overview
+
+import 'lib.just'
+import 'misc/just/forward.just'

lib.just

@@ -0,0 +1 @@
+import "misc/just/lib.just"

misc/bazel/justfile

@@ -0,0 +1,5 @@
+import '../just/lib.just'
+
+[no-cd, positional-arguments, no-exit-message]
+hello +ARGS:
+    @echo "hello from bzl" "$@"

misc/codegen/justfile

@@ -0,0 +1,5 @@
+import "../just/lib.just"
+
+test *ARGS="": (_bazel "test" "@codeql//misc/codegen/...")
+
+format *ARGS=".": (_format_py ARGS)

misc/just/README.md

@@ -0,0 +1,44 @@
+This directory contains an infrastructure for [`just`](https://github.com/casey/just)
+recipes that can be used throughout this and the internal repository. In particular we
+have common verbs (`build`, `test`, `format`, `lint`, `generate`) that individual parts
+of the project can implement, and some common functionality that can be used to that
+effect.
+
+# Forwarding
+
+The core of the functionality is given by forwarding. The idea is that:
+
+- if you are in the directory where a verb is implemented, you will get that as per
+  standard `just` behaviour (possibly using fallback).
+- if on the other hand you are beneath it, and you run something like
+  `just test ql/rust/ql/test/{a,b}`, then a forwarder script finds a common justfile
+  implementing the verb for all the positional arguments passed there, and then retries
+  calling `just test` from there. So if `test` is implemented beneath that (in that case,
+  it is in `rust/ql/test`), it uses that recipe.
+- even if there isn't a recipe that is common to all the positional arguments, the
+  forwarder will still group the arguments in batches using the same recipe. So
+  `just build ql/rust ql/java`, or
+  `just test ql/rust/ql/test/some/language/test ql/rust/ql/integration-test/some/integration/test`
+  will also work, with corresponding recipes run sequentially.
+
+Another point is how launching QL tests can be tweaked:
+
+- by default, the corresponding CLI is built from the internal repo (nothing is done if
+  working in `codeql` standalone), and no additional database or consistency checks are
+  made
+- `--codeql=built` can be passed to skip the build step (if no changes were made to the
+  CLI/extractors). This is consistent with the same pytest option
+- you can add the additional checks that CI does with `--all-checks` or the `+`
+  abbreviation. These additional checks are configured in justfiles per language, and
+  correspond to all the additional checks that CI adds (but that a dev might not want to
+  run by default).
+
+Some caveats:
+
+- passing arguments with spaces generally doesn't work, although setting arguments with
+  spaces in `justfile`s (for the base arguments) is supported using escaping as in `\\`.
+  This is a known limitation of just (see
+  <https://github.com/casey/just/issues/1988>)
+- when running different recipes for the same verb, non-positional arguments need to be
+  supported by all recipes involved. For example, this will work ok for `--learn` or
+  `--codeql` options in language and integration tests

misc/just/build.just

@@ -0,0 +1,19 @@
+# Helper build recipes
+
+import "defs.just"
+
+# Build the given language-specific CLI distribution
+_build_dist LANGUAGE: _require_semmle_code (_maybe_build_dist LANGUAGE)
+
+# Build the language-specific distribution if we are in an internal repository checkout
+# Otherwise, do nothing
+[no-exit-message]
+_maybe_build_dist LANGUAGE: (_if_in_semmle_code ('cd "$SEMMLE_CODE"; tools/bazel run //language-packs:intree-' + LANGUAGE) '# using codeql from PATH, if any')
+
+# Call bazel. Uses our official bazel wrapper if we are in an internal repository checkout
+[no-cd, no-exit-message]
+_bazel COMMAND *ARGS: (_if_in_semmle_code 'cd "$SEMMLE_CODE"; tools/bazel' 'bazel' COMMAND ARGS)
+
+# Call sembuild (requires an internal repository checkout)
+[no-cd, no-exit-message]
+_sembuild *ARGS: (_run_in_semmle_code "./build" ARGS)

misc/just/codeql-test-run.ts

@@ -0,0 +1,159 @@
+import * as child_process from "child_process";
+import * as path from "path";
+import * as os from "os";
+import * as fs from "fs";
+
+const vars = {
+    just: process.env["JUST_EXECUTABLE"] || "just",
+    error: process.env["JUST_ERROR"] || "error",
+    cmd_begin: process.env["CMD_BEGIN"] || "",
+    cmd_end: process.env["CMD_END"] || "",
+    semmle_code: process.env["SEMMLE_CODE"],
+};
+
+function invoke(
+    invocation: string[],
+    options: { cwd?: string; log_prefix?: string } = {},
+): number {
+    const log_prefix =
+        options.log_prefix && options.log_prefix !== ""
+            ? `${options.log_prefix} `
+            : "";
+    console.log(
+        `${vars.cmd_begin}${log_prefix}${invocation.join(" ")}${vars.cmd_end}`,
+    );
+    try {
+        child_process.execFileSync(invocation[0], invocation.slice(1), {
+            stdio: "inherit",
+            cwd: options.cwd,
+        });
+    } catch (error) {
+        return 1;
+    }
+    return 0;
+}
+
+type Args = {
+    tests: string[];
+    flags: string[];
+    env: string[];
+    codeql: string;
+    all: boolean;
+};
+
+const old_console_error = console.error;
+
+console.error = (message: string) => {
+    old_console_error(vars.error + message);
+};
+
+function parseArgs(args: Args, argv: string) {
+    argv.split(/(?<!\\) /)
+        .map((arg) => arg.replace("\\ ", " "))
+        .forEach((arg) => {
+            if (arg.startsWith("--codeql=")) {
+                args.codeql = arg.split("=")[1];
+            } else if (arg === "+" || arg === "--all-checks") {
+                args.all = true;
+            } else if (arg.startsWith("-")) {
+                args.flags.push(arg);
+            } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) {
+                args.env.push(arg);
+            } else if (arg !== "") {
+                args.tests.push(arg);
+            }
+        });
+}
+
+function codeqlTestRun(argv: string[]): number {
+    const [language, base_args, all_args, extra_args] = argv;
+    const ram_per_thread = process.platform === "linux" ? 3000 : 2048;
+    const cpus = os.cpus().length;
+    let args: Args = {
+        tests: [],
+        flags: [`--ram=${ram_per_thread * cpus}`, `-j${cpus}`],
+        env: [],
+        codeql: vars.semmle_code ? "build" : "host",
+        all: false,
+    };
+    parseArgs(args, base_args);
+    parseArgs(args, extra_args);
+    if (args.all) {
+        parseArgs(args, all_args);
+    }
+    if (
+        !vars.semmle_code &&
+        (args.codeql === "build" || args.codeql === "built")
+    ) {
+        console.error(
+            "Using `--codeql=build` or `--codeql=built` requires working with the internal repository",
+        );
+        return 1;
+    }
+    if (args.tests.length === 0) {
+        args.tests.push(".");
+    }
+    if (args.codeql === "build") {
+        if (
+            invoke([vars.just, language, "build"], {
+                cwd: vars.semmle_code,
+            }) !== 0
+        ) {
+            return 1;
+        }
+    }
+    if (args.codeql !== "host") {
+        // disable the default implicit config file, but keep an explicit one
+        // this is the same behavior wrt to `--codeql` as the integration test runner
+        process.env["CODEQL_CONFIG_FILE"] ||= ".";
+    }
+    // Set and unset environment variables
+    args.env.forEach((envVar) => {
+        const [key, value] = envVar.split("=", 2);
+        if (key) {
+            if (value === undefined) {
+                delete process.env[key];
+            } else {
+                process.env[key] = value;
+            }
+        } else {
+            console.error(`Invalid environment variable assignment: ${envVar}`);
+            process.exit(1);
+        }
+    });
+    let codeql;
+    function check_codeql() {
+        if (!fs.existsSync(codeql)) {
+            console.error(`CodeQL executable not found: ${codeql}`);
+            process.exit(1);
+        }
+    }
+    if (args.codeql === "built" || args.codeql === "build") {
+        codeql = path.join(
+            vars.semmle_code!,
+            "target",
+            "intree",
+            `codeql-${language}`,
+            "codeql",
+        );
+        check_codeql();
+    } else if (args.codeql === "host") {
+        codeql = "codeql";
+    } else {
+        codeql = args.codeql;
+        check_codeql();
+    }
+    if (fs.lstatSync(codeql).isDirectory()) {
+        codeql = path.join(codeql, "codeql");
+        if (process.platform === "win32") {
+            codeql += ".exe";
+        }
+        check_codeql();
+    }
+
+    return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], {
+        log_prefix: args.env.join(" "),
+    });
+}
+
+process.exit(codeqlTestRun(process.argv.slice(2)));

misc/just/defs.just

@@ -0,0 +1,58 @@
+import? '../../../semmle-code.just'  # internal repo just file, if present
+import 'semmle-code-stub.just'
+
+set fallback
+set allow-duplicate-recipes
+set allow-duplicate-variables
+set unstable
+
+export PATH_SEP := if os() == "windows" { ";" } else { ":" }
+export JUST_EXECUTABLE := just_executable()
+
+error := style("error") + "error" + NORMAL + ": "
+cmd_sep := "\n#--------------------------------------------------------\n"
+export CMD_BEGIN := style("command") + cmd_sep
+export CMD_END := cmd_sep + NORMAL
+export JUST_ERROR := error
+
+tsx := "npx [email protected]"
+
+default_db_checks := """\
+    --check-databases \
+    --check-diff-informed \
+    --fail-on-trap-errors \
+"""
+
+[no-exit-message]
+@_require_semmle_code:
+    {{ if SEMMLE_CODE == "" { '''
+        echo "''' + error + ''' running this recipe requires doing so from an internal repository checkout" >&2
+        exit 1
+    ''' } else { "" } }}
+
+[no-cd]
+_run +ARGS:
+    {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }}
+
+[no-cd]
+_run_in DIR +ARGS:
+    {{ cmd_sep }}cd "{{ DIR }}"; {{ ARGS }}{{ cmd_sep }}
+
+[no-cd]
+_run_in_semmle_code +ARGS: _require_semmle_code (_run_in "$SEMMLE_CODE" ARGS)
+
+[no-cd, positional-arguments, no-exit-message]
+@_just +ARGS:
+    echo "-> just $@"
+    "{{ JUST_EXECUTABLE }}" "$@"
+
+[no-cd, positional-arguments]
+@_if_not_on_ci_just +ARGS:
+    if [ "${GITHUB_ACTIONS:-}" != "true" ]; then \
+        echo "-> just $@"; \
+        "$JUST_EXECUTABLE" "$@"; \
+    fi
+
+[no-cd, no-exit-message]
+_if_in_semmle_code THEN ELSE *ARGS:
+    {{ cmd_sep }}{{ if SEMMLE_CODE != "" { THEN } else { ELSE } }} {{ ARGS }}{{ cmd_sep }}

misc/just/format.just

@@ -0,0 +1,20 @@
+import "build.just"
+
+[no-cd, no-exit-message]
+_format_ql +ARGS: (_maybe_build_dist "nolang") (
+    _if_in_semmle_code
+        '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"'
+        'codeql'
+    ("query format --in-place -v $(find " + ARGS + " -type f -name '*.ql' -or -name '*.qll')")
+)
+
+[no-cd, no-exit-message]
+_format_py *ARGS=".": (_if_in_semmle_code "uv run black" "black" ARGS)
+
+[no-cd, no-exit-message]
+_format_cpp *ARGS=".": (
+    _if_in_semmle_code
+        "uv run clang-format"
+        "clang-format"
+    "-i --verbose $(find " + ARGS + " -type f -name '*.h' -or -name '*.cpp')"
+)