Adding the Docker integration changes

Author: deringezgin

.github/workflows/CI_pipeline.yml

@@ -171,13 +171,13 @@ jobs:
       - name: Build project ${{ matrix.project_slug }}
         run: |
           echo "--- Building ${{ matrix.project_slug }} ---"
-          python scripts/fetch_and_build.py --filter ${{ matrix.project_slug }}
+          conda run -n iris python scripts/fetch_and_build.py --filter ${{ matrix.project_slug }}
           echo "--------------------------------"
 
       - name: Generate CodeQL database for ${{ matrix.project_slug }}
         run: |
           echo "--- Generating CodeQL database for ${{ matrix.project_slug }} ---"
-          python scripts/build_codeql_dbs.py --project ${{ matrix.project_slug }}
+          conda run -n iris python scripts/build_codeql_dbs.py --project ${{ matrix.project_slug }}
           echo "----------------------------------"
 
       - name: Run IRIS for ${{ matrix.project_slug }} with CWE ${{ matrix.cwe }}

data/build_cmds.csv

@@ -0,0 +1,6 @@
+project_slug,build_cmd
+spring-projects__spring-security_CVE-2011-2732_2.0.6.RELEASE,mvn -q -B -DskipTests compile
+x-stream__xstream_CVE-2013-7285_1.4.6,mvn -q -B -DskipTests compile
+spring-security_CVE-2025-22223_6.4.3,"./gradlew --no-daemon -S -Dorg.gradle.dependency.verification=off -Dorg.gradle.warning.mode=none -Dorg.gradle.caching=false --rerun-tasks -x compileKotlin -x compileTestKotlin clean classes"
+incubator-seata_CVE-2025-32897_v2.2.0,mvn -q -B -DskipTests compile
+cassandra-lucene-index_CVE-2025-26511_cassandra-4.0.16-1.0.0,mvn clean package -B -V -e -Dfindbugs.skip -Dcheckstyle.skip -Dpmd.skip=true -Dspotbugs.skip -Denforcer.skip -Dmaven.javadoc.skip -DskipTests -Dmaven.test.skip.exec -Dlicense.skip=true -Drat.skip=true -Dspotless.check.skip=true

data/patches/cassandra-lucene-index_CVE-2025-26511_cassandra-4.0.16-1.0.0.patch

@@ -0,0 +1,13 @@
+diff --git a/pom.xml b/pom.xml
+index 9d373d76..654c07a2 100644
+--- a/pom.xml
++++ b/pom.xml
+@@ -25,7 +25,7 @@
+     
+     <groupId>com.instaclustr</groupId>
+     <artifactId>cassandra-lucene-index-parent</artifactId>
+-    <version>4.1.8-1.0.1</version>
++    <version>4.1.8-1.0.0</version>
+     <packaging>pom</packaging>
+     
+     <name>Cassandra Lucene index</name>

environment.yml

@@ -13,3 +13,6 @@ dependencies:
   - transformers
   - pytorch=2.5
   - google-generativeai
+  - pip
+  - pip:
+    - docker

scripts/build_codeql_dbs.py

@@ -7,9 +7,33 @@
 import json
 sys.path.append(str(Path(__file__).parent.parent))
 
-from src.config import CODEQL_DB_PATH, PROJECT_SOURCE_CODE_DIR, IRIS_ROOT_DIR, BUILD_INFO, DEP_CONFIGS, DATA_DIR
+from src.config import CODEQL_DB_PATH, PROJECT_SOURCE_CODE_DIR, IRIS_ROOT_DIR, BUILD_INFO, DEP_CONFIGS, DATA_DIR, CODEQL_DIR, CVES_MAPPED_W_COMMITS_DIR
+from scripts.docker_utils import ensure_image, create_container, exec_in_container, parse_project_image, copy_dir_to_container, copy_from_container
 ALLVERSIONS = json.load(open(DEP_CONFIGS))
 
+# Path to custom build commands CSV
+BUILD_CMDS_CSV = os.path.join(DATA_DIR, "build_cmds.csv")
+
+def load_custom_build_commands(csv_path: str = BUILD_CMDS_CSV) -> dict[str, str]:
+    """
+    Load custom build commands from the build command CSV file
+    Returns a mapping of project_slug -> build_cmd. Missing file or invalid headers yields an empty mapping.
+    """
+    build_cmds: dict[str, str] = {}
+    if not os.path.exists(csv_path):
+        return build_cmds
+    with open(csv_path, "r", newline="") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            slug = (row.get("project_slug") or "").strip()
+            cmd = (row.get("build_cmd") or "").strip()
+            if slug and cmd:
+                build_cmds[slug] = cmd
+    return build_cmds
+
+# Custom build commands for the CodeQL database creation (loaded from CSV)
+CUSTOM_BUILD_COMMANDS: dict[str, str] = load_custom_build_commands()
+
 def setup_environment(row):
     env = os.environ.copy()
 
@@ -48,6 +72,9 @@ def create_codeql_database(project_slug, env, db_base_path, sources_base_path):
     print(f"PATH: {env.get('PATH', 'Not set')}")
     print(f"JAVA_HOME: {env.get('JAVA_HOME', 'Not set')}")
 
+    # Prefer custom build command when available
+    custom_cmd = CUSTOM_BUILD_COMMANDS.get(project_slug)
+    
     try:
         java_version = subprocess.check_output(['java', '-version'], 
                                             stderr=subprocess.STDOUT, 
@@ -67,8 +94,11 @@ def create_codeql_database(project_slug, env, db_base_path, sources_base_path):
         database_path,
         "--source-root", source_path,
         "--language", "java",
-        "--overwrite"
+        "--overwrite",
     ]
+    if custom_cmd:
+        print(f"Using custom build command for {project_slug}: {custom_cmd}")
+        command.extend(["--command", custom_cmd])
 
     try:
         print(f"Creating database at: {database_path}")
@@ -85,11 +115,121 @@ def create_codeql_database(project_slug, env, db_base_path, sources_base_path):
         print(f'Stderr Info:\n{e.stderr.decode()}')
         raise
 
+
+def create_codeql_database_in_container(project_slug: str, row: dict, db_base_path: str, verbose: bool = False) -> None:
+    image = parse_project_image(project_slug)  # Parse the project image from the project slug
+    ensure_image(image)
+
+    # Add -docker suffix to database name when using container
+    db_project_slug = f"{project_slug}-docker"
+
+    # Prepare host and container paths
+    host_db_dir = os.path.abspath(db_base_path)  # Create the host database directory
+    os.makedirs(host_db_dir, exist_ok=True)
+    container_codeql_dir = "/codeql"
+    container_codeql_bin = f"{container_codeql_dir}/codeql"
+    container_out_base = "/out"
+    container_db_dir = f"{container_out_base}/{db_project_slug}"
+    container_source_root = "/workspace/repo"
+
+    # Read repo info from project_info.csv
+    def get_repo_info_from_project_info(slug: str) -> tuple[str, str]:
+        with open(CVES_MAPPED_W_COMMITS_DIR, 'r') as f:
+            reader = csv.reader(f)
+            next(reader, None)  # header
+            for line in reader:
+                if len(line) > 10 and line[1] == slug:
+                    repo_url = line[8]
+                    commit_id = line[10]
+                    return repo_url, commit_id
+        raise RuntimeError(f"Project slug '{slug}' not found in project_info.csv")
+
+    container = create_container(image=image, working_dir=container_source_root)
+    try:
+        container.start()
+        
+        # Copy CodeQL CLI into container and ensure out dir exists
+        copy_dir_to_container(container, CODEQL_DIR, container_codeql_dir)
+        exec_in_container(container, ["bash", "-lc", f"mkdir -p {container_out_base}"])
+
+        # Fresh fetch like fetch_one.py: reclone at desired commit
+        repo_url, commit_id = get_repo_info_from_project_info(project_slug)
+        fetch_cmd = (
+            f"rm -rf repo && mkdir -p repo && cd repo && "
+            f"git init && git remote add origin '{repo_url}' && "
+            f"git fetch --no-tags --depth 1 origin {commit_id} && "
+            f"git reset --hard FETCH_HEAD && "
+            f"git fetch --no-tags --depth 1 origin '+refs/heads/*:refs/remotes/origin/*'"
+        )
+        print(f"Refreshing sources from {repo_url} @ {commit_id}")
+        code, output = exec_in_container(
+            container,
+            ["bash", "-lc", fetch_cmd],
+            workdir="/workspace",
+            stream=verbose,
+        )
+        if code != 0:
+            if output:
+                print(output)
+            raise RuntimeError(f"Failed to fetch sources for {project_slug}")
+
+        # Apply project patch if available (mirror fetch_one.py behavior)
+        patch_dir_host = os.path.join(DATA_DIR, "patches")
+        patch_file_host = os.path.join(patch_dir_host, f"{project_slug}.patch")
+        if os.path.exists(patch_file_host):
+            print(f"Found patch for {project_slug}, applying...")
+            # Copy entire patches dir to container to keep logic simple
+            copy_dir_to_container(container, patch_dir_host, "/patches")
+            code, output = exec_in_container(
+                container,
+                ["bash", "-lc", f"git apply /patches/{project_slug}.patch"],
+                workdir=container_source_root,
+                stream=verbose,
+            )
+            if code != 0:
+                if output:
+                    print(output)
+                raise RuntimeError(f"Failed to apply patch for {project_slug}")
+        else:
+            print("No patch found; skipping patching.")
+
+        # Prefer custom build command when available
+        custom_cmd = CUSTOM_BUILD_COMMANDS.get(project_slug)
+        if custom_cmd:
+            print(f"Using custom build command for {project_slug}: {custom_cmd}")
+            codeql_cmd = (f"{container_codeql_bin} database create {container_db_dir} "
+                          f"--source-root {container_source_root} --language java --overwrite "
+                          f"--command \"{custom_cmd}\"")
+        else:
+            codeql_cmd = (f"{container_codeql_bin} database create {container_db_dir} --source-root {container_source_root} --language java --overwrite")
+
+        print(f"Initializing database at {container_db_dir}.")
+        code, output = exec_in_container(container, ["bash", "-lc", codeql_cmd], workdir=container_source_root, stream=verbose)
+        
+        if code != 0:
+            print(f"CodeQL database creation failed for {project_slug}")
+            if not verbose and output:
+                print("Error output:")
+                print(output)
+            raise RuntimeError(f"CodeQL database creation failed in container for {project_slug}")
+        
+        print(f"Finalizing database at {container_db_dir}.")
+        # Copy database back to host  
+        copy_from_container(container, container_db_dir, host_db_dir)
+        print(f"Successfully created database at {host_db_dir}/{db_project_slug}.")
+    finally:
+        try:
+            container.remove(force=True)
+        except Exception:
+            pass
+
 def main():
     parser = argparse.ArgumentParser(description='Create CodeQL databases for cwe-bench-java projects')
     parser.add_argument('--project', help='Specific project slug', default=None)
     parser.add_argument('--db-path', help='Base path for storing CodeQL databases', default=CODEQL_DB_PATH)
     parser.add_argument('--sources-path', help='Base path for project sources', default=PROJECT_SOURCE_CODE_DIR)
+    parser.add_argument('--use-container', action='store_true', help='Create DB inside the project container using mounted CodeQL')
+    parser.add_argument('--verbose', action='store_true', help='Show verbose output during database creation')
     args = parser.parse_args()
 
     # Load build information
@@ -98,14 +238,20 @@ def main():
     if args.project:
         project = next((p for p in projects if p['project_slug'] == args.project), None)
         if project:
-            env = setup_environment(project)
-            create_codeql_database(project['project_slug'], env, args.db_path, args.sources_path)
+            if args.use_container:
+                create_codeql_database_in_container(project['project_slug'], project, args.db_path, args.verbose)
+            else:
+                env = setup_environment(project)
+                create_codeql_database(project['project_slug'], env, args.db_path, args.sources_path)
         else:
             print(f"Project {args.project} not found in CSV file")
     else:
         for project in projects:
-            env = setup_environment(project)
-            create_codeql_database(project['project_slug'], env, args.db_path, args.sources_path)
+            if args.use_container:
+                create_codeql_database_in_container(project['project_slug'], project, args.db_path, args.verbose)
+            else:
+                env = setup_environment(project)
+                create_codeql_database(project['project_slug'], env, args.db_path, args.sources_path)
 
 # Location of build_info_local.csv file
 LOCAL_BUILD_INFO = os.path.join(DATA_DIR, "build-info", "build_info_local.csv")

scripts/build_one.py

@@ -27,6 +27,7 @@
 sys.path.append(str(ROOT_DIR))
 
 from src.config import DATA_DIR, DEP_CONFIGS
+from scripts.docker_utils import create_container, ensure_image, exec_in_container, parse_project_image
 
 # Load dependency configurations
 ALLVERSIONS = json.load(open(DEP_CONFIGS))
@@ -108,11 +109,11 @@ def get_build_info_from_csv(project_slug, csv_path):
             for row in reader:
                 if row['project_slug'] == project_slug and row['status'] == 'success':
                     specific_attempt = {}
-                    if row['jdk_version'] != 'n/a':
+                    if row['jdk_version'] not in ('n/a', '', None):
                         specific_attempt['jdk'] = row['jdk_version']
-                    if row['mvn_version'] != 'n/a':
+                    if row['mvn_version'] not in ('n/a', '', None):
                         specific_attempt['mvn'] = row['mvn_version']
-                    if row['gradle_version'] != 'n/a':
+                    if row['gradle_version'] not in ('n/a', '', None):
                         specific_attempt['gradle'] = row['gradle_version']
                     if row['use_gradlew'] != 'n/a':
                         if row['use_gradlew'] == "True":
@@ -372,7 +373,32 @@ def validate_and_create_custom_attempt(jdk, mvn, gradle, gradlew):
     return custom_attempt
 
 
-def build_project(project_slug, try_all=False, custom_attempt=None):
+def build_inside_container(project_slug: str, attempt: dict) -> bool:
+    image = parse_project_image(project_slug)
+    ensure_image(image)
+    container = create_container(image=image, working_dir="/workspace/repo")
+    try:
+        container.start()
+        env = {}
+        cmd: list[str]
+        if "mvn" in attempt:
+            cmd = ["bash", "-lc", "mvn -B -e -U -DskipTests clean package"]
+        elif "gradle" in attempt:
+            cmd = ["bash", "-lc", "gradle build --parallel"]
+        elif "gradlew" in attempt:
+            cmd = ["bash", "-lc", "chmod +x ./gradlew && ./gradlew --no-daemon -S -Dorg.gradle.dependency.verification=off clean"]
+        else:
+            return False
+        code, _ = exec_in_container(container, cmd, workdir="/workspace/repo", environment=env, stream=True)
+        return code == 0
+    finally:
+        try:
+            container.remove(force=True)
+        except Exception:
+            pass
+
+
+def build_project(project_slug, try_all=False, custom_attempt=None, use_container: bool = False):
     """Main function to build a project with various strategies."""
     # Handle custom attempt first
     if custom_attempt:
@@ -385,12 +411,12 @@ def build_project(project_slug, try_all=False, custom_attempt=None):
     if not try_all:
         # Try local build info first
         local_build_info = get_build_info_from_csv(project_slug, f"{DATA_DIR}/build-info/build_info_local.csv")
-        if local_build_info and try_build_with_attempt(project_slug, local_build_info, "local"):
+        if local_build_info and (try_build_with_attempt(project_slug, local_build_info, "local") if not use_container else build_inside_container(project_slug, local_build_info)):
             return True
 
         # Try global build info if local failed
         global_build_info = get_build_info_from_csv(project_slug, f"{DATA_DIR}/build_info.csv")
-        if global_build_info and try_build_with_attempt(project_slug, global_build_info, "global"):
+        if global_build_info and (try_build_with_attempt(project_slug, global_build_info, "global") if not use_container else build_inside_container(project_slug, global_build_info)):
             return True
 
     # Try all default attempts
@@ -399,7 +425,7 @@ def build_project(project_slug, try_all=False, custom_attempt=None):
            "No successful build configuration found in CSV files, trying all version combinations..."))
 
     for attempt in ATTEMPTS:
-        if try_build_with_attempt(project_slug, attempt):
+        if (try_build_with_attempt(project_slug, attempt) if not use_container else build_inside_container(project_slug, attempt)):
             return True
 
     print(f"[build_one] All build attempts failed for {project_slug}")
@@ -451,6 +477,11 @@ def main():
         action="store_true", 
         help="Use the project's gradlew script"
     )
+    parser.add_argument(
+        "--use-container",
+        action="store_true",
+        help="Build inside the project's container image",
+    )
 
     args = parser.parse_args()
 
@@ -462,7 +493,7 @@ def main():
             return 1
         custom_attempt = validate_and_create_custom_attempt(args.jdk, args.mvn, args.gradle, args.gradlew)
 
-    success = build_project(args.project_slug, try_all=args.try_all, custom_attempt=custom_attempt)
+    success = build_project(args.project_slug, try_all=args.try_all, custom_attempt=custom_attempt, use_container=args.use_container)
     return 0 if success else 1