Release 0.11.0

bedboss/bedclassifier/bedclassifier.py

@@ -109,6 +109,7 @@ def _read_bed_file(filepath: str, skiprows: int = 0) -> pd.DataFrame | None:
             raise BedTypeException(reason="Input is not a string or dataframe.")
 
     df = df.dropna(axis=1)
+    df.columns = range(len(df.columns))
     num_cols = len(df.columns)
     compliant_columns = 0
     bed_format_named = DATA_FORMAT.UCSC_BED

bedboss/cli.py

@@ -7,6 +7,7 @@
 from pephubclient.helpers import MessageHandler as printm
 
 from bedboss.bbuploader.cli import app_bbuploader
+from bedboss.qdrant_index.qdrant_cli import qdrant_app
 
 # commented and made new const here, because it speeds up help function,
 # from bbconf.const import DEFAULT_LICENSE
@@ -233,6 +234,117 @@ def run_pep(
     pm.stop_pipeline()
 
 
+@app.command(
+    name="run-pep-hpc",
+    help="Split a large PEP into N chunks and submit each as a SLURM job. Idempotent: re-run to resume failed/pending chunks.",
+)
+def run_pep_hpc(
+    pep: str = typer.Option(..., help="PEP file. Local path or PEPhub registry path."),
+    workdir: str = typer.Option(
+        ..., help="Working directory for chunks, sbatch files, manifest, and state."
+    ),
+    n_chunks: int = typer.Option(..., help="Number of chunks to split the PEP into."),
+    # forwarded run-pep options
+    outfolder: str = typer.Option(
+        ..., help="Path to the output folder (shared across chunks)."
+    ),
+    bedbase_config: str = typer.Option(
+        ...,
+        help="Path to the bedbase config file",
+        exists=True,
+        file_okay=True,
+        readable=True,
+    ),
+    create_bedset: bool = typer.Option(True, help="Create a new bedset"),
+    bedset_heavy: bool = typer.Option(False, help="Run heavy bedbuncher"),
+    rfg_config: str = typer.Option(None, help="Path to the rfg config file"),
+    check_qc: bool = typer.Option(True, help="Check the quality of the input file?"),
+    ensdb: str = typer.Option(None, help="Path to the EnsDb database file"),
+    just_db_commit: bool = typer.Option(False, help="Just commit to the database?"),
+    force_overwrite: bool = typer.Option(
+        False, help="Force overwrite the output files"
+    ),
+    update: bool = typer.Option(False, help="Update existing records"),
+    upload_qdrant: bool = typer.Option(True, help="Upload to Qdrant"),
+    upload_s3: bool = typer.Option(True, help="Upload to S3"),
+    upload_pephub: bool = typer.Option(True, help="Upload to PEPHub"),
+    no_fail: bool = typer.Option(False, help="Do not fail on error"),
+    license_id: str = typer.Option(DEFAULT_LICENSE, help="License ID"),
+    standardize_pep: bool = typer.Option(False, help="Standardize the PEP using bedMS"),
+    lite: bool = typer.Option(False, help="Run the pipeline in lite mode."),
+    rerun: bool = typer.Option(False, help="Rerun already processed samples"),
+    multi: bool = typer.Option(False, help="Run multiple samples"),
+    recover: bool = typer.Option(True, help="Recover from previous run"),
+    dirty: bool = typer.Option(False, help="Run without removing existing files"),
+    # SLURM options
+    slurm_template: str = typer.Option(
+        None,
+        help="Path to a custom sbatch template. See bedboss/bedboss_hpc.py for placeholders.",
+    ),
+    slurm_account: str = typer.Option("shefflab", help="SLURM --account"),
+    slurm_partition: str = typer.Option("standard", help="SLURM --partition"),
+    slurm_time: str = typer.Option("72:00:00", help="SLURM --time"),
+    slurm_mem: str = typer.Option("60000", help="SLURM --mem (MB)"),
+    slurm_cpus: int = typer.Option(4, help="SLURM --cpus-per-task"),
+    slurm_ntasks: int = typer.Option(2, help="SLURM --ntasks"),
+    dry_run: bool = typer.Option(
+        False, help="Split and write sbatch files but do not submit."
+    ),
+):
+    from bedboss.bedboss_hpc import RunPepArgs, SlurmConfig
+    from bedboss.bedboss_hpc import run_pep_hpc as _run_pep_hpc
+
+    run_pep_args = RunPepArgs(
+        outfolder=outfolder,
+        bedbase_config=bedbase_config,
+        create_bedset=create_bedset,
+        bedset_heavy=bedset_heavy,
+        rfg_config=rfg_config,
+        check_qc=check_qc,
+        ensdb=ensdb,
+        just_db_commit=just_db_commit,
+        force_overwrite=force_overwrite,
+        update=update,
+        upload_qdrant=upload_qdrant,
+        upload_s3=upload_s3,
+        upload_pephub=upload_pephub,
+        no_fail=no_fail,
+        license_id=license_id,
+        standardize_pep=standardize_pep,
+        lite=lite,
+        rerun=rerun,
+        multi=multi,
+        recover=recover,
+        dirty=dirty,
+    )
+    slurm_cfg = SlurmConfig(
+        account=slurm_account,
+        partition=slurm_partition,
+        time=slurm_time,
+        mem=slurm_mem,
+        cpus_per_task=slurm_cpus,
+        ntasks=slurm_ntasks,
+    )
+    _run_pep_hpc(
+        pep=pep,
+        workdir=workdir,
+        n_chunks=n_chunks,
+        run_pep_args=run_pep_args,
+        slurm_cfg=slurm_cfg,
+        slurm_template=slurm_template,
+        dry_run=dry_run,
+    )
+
+
+@app.command(name="run-pep-hpc-status", help="Show status of a run-pep-hpc workdir.")
+def run_pep_hpc_status(
+    workdir: str = typer.Option(..., help="Working directory created by run-pep-hpc."),
+):
+    from bedboss.bedboss_hpc import run_pep_hpc_status as _status
+
+    _status(workdir)
+
+
 @app.command(
     help="Run unprocessed files or reprocess them. Currently, only hg38, hg19, and mm10 genomes are supported."
 )
@@ -687,9 +799,9 @@ def download_umap(
         "umap",
         help="Dimensionality reduction method to use. Options: 'umap', 'pca', or 'tsne'. To use UMAP, 'umap-learn' package must be installed.",
     ),
-    save_parquet: bool = typer.Option(
-        False,
-        help="Whether to save Parquet tier files alongside JSON",
+    output_format: str = typer.Option(
+        "parquet",
+        help="Output format: 'json', 'parquet', or 'both'",
     ),
 ):
     from bedboss.scripts.make_umap import get_embeddings
@@ -703,11 +815,11 @@ def download_umap(
         top_assays=top_assays,
         top_cell_lines=top_cell_lines,
         method=method,
-        save_parquet=save_parquet,
+        output_format=output_format,
     )
 
 
-@app.command(help="Update UMAP metadata Parquet tiers without regenerating geometry")
+@app.command(help="Update UMAP parquet metadata without regenerating geometry")
 def update_umap_metadata(
     config: str = typer.Option(
         ...,
@@ -716,9 +828,9 @@ def update_umap_metadata(
         file_okay=True,
         readable=True,
     ),
-    output_dir: str = typer.Option(
+    output_path: str = typer.Option(
         ...,
-        help="Directory to write Parquet tier files",
+        help="Path to write parquet file (without extension)",
     ),
     geometry: str = typer.Option(
         None,
@@ -727,7 +839,7 @@ def update_umap_metadata(
 ):
     from bedboss.scripts.make_umap import update_umap_metadata as _update
 
-    _update(bbconf=config, output_dir=output_dir, geometry=geometry)
+    _update(bbconf=config, output_path=output_path, geometry=geometry)
 
 
 @app.command(help="Check installed R packages")
@@ -793,3 +905,4 @@ def common(
 
 
 app.add_typer(app_bbuploader, name="geo")
+app.add_typer(qdrant_app, name="qdrant")

bedboss/const.py

@@ -37,34 +37,18 @@
 
 BED_PEP_REGISTRY: str = "databio/allbeds:bedbase"
 
-# UMAP Parquet tier column definitions
+# UMAP constants
 DB_QUERY_BATCH_SIZE: int = 5000
+UMAP_GENOME: str = "hg38"
 
-TIER1_COLUMNS: list[str] = [
+UMAP_PARQUET_COLUMNS: list[str] = [
     "id",
+    "x",
+    "y",
     "name",
     "description",
     "assay",
-    "target",
     "cell_line",
     "cell_type",
     "tissue",
-    "number_of_regions",
-    "mean_region_width",
-    "gc_content",
-]
-
-TIER2_COLUMNS: list[str] = [
-    "id",
-    "treatment",
-    "antibody",
-    "species_name",
-    "genome_alias",
-    "bed_compliance",
-    "data_format",
-    "median_tss_dist",
-    "library_source",
-    "global_sample_id",
-    "global_experiment_id",
-    "original_file_name",
 ]

bedboss/qdrant_index/__init__.py

@@ -1,3 +1,9 @@
-from bedboss.qdrant_index.qdrant_index import add_to_qdrant
-
 __all__ = ["add_to_qdrant"]
+
+
+def __getattr__(name):
+    if name == "add_to_qdrant":
+        from bedboss.qdrant_index.qdrant_index import add_to_qdrant
+
+        return add_to_qdrant
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")