Skip to content

SLURM Utilities

Functions for interacting with SLURM schedulers.

Overview

This module provides wrapper functions for common SLURM operations:

  • Querying job status via sacct and squeue
  • Submitting jobs via sbatch
  • Finding job output files
  • Parsing SLURM output formats

Functions

Job Status

get_job_status

get_job_status(job_ids: Union[str, List[str]]) -> Dict[str, str]

Get the state of one or more jobs.

Convenience wrapper around get_sacct_info that returns only job states.

Parameters:

Name Type Description Default
job_ids Union[str, List[str]]

Single job ID or list of job IDs.

required

Returns:

Type Description
Dict[str, str]

Dictionary mapping JobID to State string.

Example

get_job_status(["12345", "12346"])

Source code in src/slurmkit/slurm.py
def get_job_status(job_ids: Union[str, List[str]]) -> Dict[str, str]:
    """
    Get the state of one or more jobs.

    Convenience wrapper around get_sacct_info that returns only job states.

    Args:
        job_ids: Single job ID or list of job IDs.

    Returns:
        Dictionary mapping JobID to State string.

    Example:
        >>> get_job_status(["12345", "12346"])
        {'12345': 'COMPLETED', '12346': 'FAILED'}
    """
    info = get_sacct_info(job_ids, fields=["JobID", "State"])
    return {jid: data.get("State", "UNKNOWN") for jid, data in info.items()}

get_sacct_info

get_sacct_info(job_ids: Union[str, List[str]], fields: Optional[List[str]] = None) -> Dict[str, Dict[str, str]]

Query sacct for job metadata.

Queries the SLURM accounting database for information about completed or running jobs. Results are filtered to exclude job steps (e.g., .batch, .extern) and only return the main job entry.

Parameters:

Name Type Description Default
job_ids Union[str, List[str]]

Single job ID or list of job IDs to query.

required
fields Optional[List[str]]

List of sacct fields to retrieve. Defaults to DEFAULT_SACCT_FIELDS. Common fields: State, Elapsed, Start, End, ExitCode, AllocCPUS, ReqMem, NodeList, Partition. Run sacct --helpformat for full list.

None

Returns:

Type Description
Dict[str, Dict[str, str]]

Dictionary mapping JobID to a dict of field values.

Dict[str, Dict[str, str]]

Returns empty dict if sacct is not available or no jobs found.

Example

info = get_sacct_info(["12345", "12346"]) info["12345"]["State"] 'COMPLETED' info["12345"]["Elapsed"] '01:23:45'

Source code in src/slurmkit/slurm.py
def get_sacct_info(
    job_ids: Union[str, List[str]],
    fields: Optional[List[str]] = None,
) -> Dict[str, Dict[str, str]]:
    """
    Query sacct for job metadata.

    Queries the SLURM accounting database for information about completed
    or running jobs. Results are filtered to exclude job steps (e.g., .batch,
    .extern) and only return the main job entry.

    Args:
        job_ids: Single job ID or list of job IDs to query.
        fields: List of sacct fields to retrieve. Defaults to DEFAULT_SACCT_FIELDS.
            Common fields: State, Elapsed, Start, End, ExitCode, AllocCPUS,
            ReqMem, NodeList, Partition. Run `sacct --helpformat` for full list.

    Returns:
        Dictionary mapping JobID to a dict of field values.
        Returns empty dict if sacct is not available or no jobs found.

    Example:
        >>> info = get_sacct_info(["12345", "12346"])
        >>> info["12345"]["State"]
        'COMPLETED'
        >>> info["12345"]["Elapsed"]
        '01:23:45'
    """
    # Normalize to list
    if isinstance(job_ids, str):
        job_ids = [job_ids]

    if not job_ids:
        return {}

    if fields is None:
        fields = DEFAULT_SACCT_FIELDS.copy()

    # Ensure JobID is always first
    if "JobID" not in fields:
        fields = ["JobID"] + list(fields)

    # Build sacct command
    # --parsable2 gives pipe-delimited output without trailing delimiter
    cmd = [
        "sacct",
        "-j", ",".join(job_ids),
        f"--format={','.join(fields)}",
        "--parsable2",
        "--noheader",
    ]

    try:
        result = run_command(cmd)
        info_map = {}

        for line in result.stdout.splitlines():
            if not line.strip():
                continue

            parts = line.strip().split("|")
            if len(parts) < len(fields):
                continue

            job_id_raw = parts[0]

            # Filter out job steps (e.g., 12345.batch, 12345.extern)
            # We only want the main job entry which has no dots
            # (array jobs use underscore: 12345_1, 12345_2, etc.)
            if "." not in job_id_raw:
                info_map[job_id_raw] = {
                    field: parts[i] for i, field in enumerate(fields)
                }

        return info_map

    except FileNotFoundError:
        print("Error: sacct command not found. Is SLURM installed?", file=sys.stderr)
        return {}

get_canonical_sacct_states

get_canonical_sacct_states(job_ids: Union[str, List[str]]) -> Dict[str, Dict[str, Any]]

Query sacct full rows and resolve canonical state per parent SLURM job ID.

Canonical resolution order: 1) Any RUNNING/COMPLETING row => RUNNING 2) Else any PENDING/REQUEUED/SUSPENDED row => PENDING 3) Else terminal mapping using .batch first, then parent fallback

Source code in src/slurmkit/slurm.py
def get_canonical_sacct_states(job_ids: Union[str, List[str]]) -> Dict[str, Dict[str, Any]]:
    """
    Query sacct full rows and resolve canonical state per parent SLURM job ID.

    Canonical resolution order:
    1) Any RUNNING/COMPLETING row => RUNNING
    2) Else any PENDING/REQUEUED/SUSPENDED row => PENDING
    3) Else terminal mapping using `.batch` first, then parent fallback
    """
    # This function is the full-row state resolver used by collections refresh.
    # It intentionally does not replace get_sacct_info() legacy behavior.
    if isinstance(job_ids, str):
        job_ids = [job_ids]

    requested = [str(job_id).strip() for job_id in job_ids if str(job_id).strip()]
    if not requested:
        return {}

    rows = _query_sacct_rows(requested, fields=CANONICAL_SACCT_FIELDS.copy())
    grouped: Dict[str, List[Dict[str, str]]] = {}
    for row in rows:
        row_job_id = str(row.get("JobID") or "").strip()
        if not row_job_id:
            continue
        parent_job_id = row_job_id.split(".", 1)[0]
        grouped.setdefault(parent_job_id, []).append(row)

    resolved: Dict[str, Dict[str, Any]] = {}
    for requested_job_id in requested:
        parent_job_id = requested_job_id.split(".", 1)[0]
        if parent_job_id in resolved:
            continue
        parent_rows = grouped.get(parent_job_id, [])
        if not parent_rows:
            continue
        resolved[parent_job_id] = _resolve_canonical_state_for_rows(parent_job_id, parent_rows)
    return resolved

get_pending_jobs

get_pending_jobs(user: Optional[str] = None) -> List[Dict[str, str]]

Query squeue for currently pending or running jobs.

Parameters:

Name Type Description Default
user Optional[str]

Username to filter jobs. If None, uses current user (--me).

None

Returns:

Type Description
List[Dict[str, str]]

List of dicts with keys: 'job_id', 'job_name', 'state', 'wait_time'.

List[Dict[str, str]]

wait_time is formatted as human-readable string for pending jobs.

Example

pending = get_pending_jobs() pending[0]

Source code in src/slurmkit/slurm.py
def get_pending_jobs(user: Optional[str] = None) -> List[Dict[str, str]]:
    """
    Query squeue for currently pending or running jobs.

    Args:
        user: Username to filter jobs. If None, uses current user (--me).

    Returns:
        List of dicts with keys: 'job_id', 'job_name', 'state', 'wait_time'.
        wait_time is formatted as human-readable string for pending jobs.

    Example:
        >>> pending = get_pending_jobs()
        >>> pending[0]
        {'job_id': '12345', 'job_name': 'train_model', 'state': 'PENDING', 'wait_time': '2h 30m'}
    """
    # Build squeue command
    # --start shows estimated start time for pending jobs
    # Format: job_id|job_name|state|start_time
    cmd = ["squeue", "--start", "--format=%i|%j|%T|%S", "--noheader"]

    if user:
        cmd.extend(["-u", user])
    else:
        cmd.append("--me")

    try:
        result = run_command(cmd)
        pending_jobs = []
        now = datetime.now()

        for line in result.stdout.splitlines():
            if not line.strip():
                continue

            parts = line.strip().split("|")
            if len(parts) < 4:
                continue

            job_id, job_name, state, start_time_str = parts[:4]

            job_info = {
                "job_id": job_id,
                "job_name": job_name,
                "state": state,
            }

            # Calculate wait time for pending jobs
            if state == "PENDING" and start_time_str not in ("N/A", "Unknown"):
                wait_time = _parse_wait_time(start_time_str, now)
                job_info["wait_time"] = wait_time
            elif state == "PENDING":
                job_info["wait_time"] = "N/A"

            pending_jobs.append(job_info)

        return pending_jobs

    except FileNotFoundError:
        print("Error: squeue command not found. Is SLURM installed?", file=sys.stderr)
        return []

get_sacct_info() preserves the legacy parent-row-only view (job_id without step rows). get_canonical_sacct_states() evaluates full sacct rows (job_id, .batch, .extern, other steps), returns canonical state, and includes row-level raw_state diagnostics.

Job Submission

submit_job

submit_job(script_path: Union[str, Path], dry_run: bool = False, extra_args: Optional[List[str]] = None) -> Tuple[bool, Optional[str], str]

Submit a job script using sbatch.

Parameters:

Name Type Description Default
script_path Union[str, Path]

Path to the job script.

required
dry_run bool

If True, print command without executing.

False
extra_args Optional[List[str]]

Additional arguments to pass to sbatch.

None

Returns:

Type Description
bool

Tuple of (success, job_id, message).

Optional[str]
  • success: True if submission succeeded.
str
  • job_id: The submitted job ID (None if failed or dry_run).
Tuple[bool, Optional[str], str]
  • message: Output message from sbatch or error message.
Example

success, job_id, msg = submit_job("jobs/train.job") print(f"Submitted job {job_id}") Submitted job 12345

Source code in src/slurmkit/slurm.py
def submit_job(
    script_path: Union[str, Path],
    dry_run: bool = False,
    extra_args: Optional[List[str]] = None,
) -> Tuple[bool, Optional[str], str]:
    """
    Submit a job script using sbatch.

    Args:
        script_path: Path to the job script.
        dry_run: If True, print command without executing.
        extra_args: Additional arguments to pass to sbatch.

    Returns:
        Tuple of (success, job_id, message).
        - success: True if submission succeeded.
        - job_id: The submitted job ID (None if failed or dry_run).
        - message: Output message from sbatch or error message.

    Example:
        >>> success, job_id, msg = submit_job("jobs/train.job")
        >>> print(f"Submitted job {job_id}")
        Submitted job 12345
    """
    script_path = Path(script_path)

    if not script_path.exists():
        return False, None, f"Script not found: {script_path}"

    cmd = ["sbatch"]
    if extra_args:
        cmd.extend(extra_args)
    cmd.append(str(script_path))

    if dry_run:
        return True, None, f"[dry-run] {' '.join(cmd)}"

    try:
        result = run_command(cmd)

        if result.returncode == 0:
            # Parse job ID from sbatch output
            # Typical output: "Submitted batch job 12345"
            output = result.stdout.strip()
            job_id = None

            match = re.search(r"Submitted batch job (\d+)", output)
            if match:
                job_id = match.group(1)

            return True, job_id, output

        else:
            error_msg = result.stderr.strip() or result.stdout.strip()
            return False, None, f"sbatch failed: {error_msg}"

    except FileNotFoundError:
        return False, None, "sbatch command not found. Is SLURM installed?"

Job Output

find_job_output

find_job_output(job_id: str, jobs_dir: Optional[Path] = None, config: Optional[Config] = None) -> List[Path]

Find job output files matching a job ID.

Searches for .out files containing the job ID in their filename, using configured output patterns to match files.

Parameters:

Name Type Description Default
job_id str

SLURM job ID to search for.

required
jobs_dir Optional[Path]

Directory to search in. If None, uses config's jobs_dir.

None
config Optional[Config]

Configuration object. If None, uses global config.

None

Returns:

Type Description
List[Path]

List of matching file paths, sorted by modification time (newest first).

Example

outputs = find_job_output("12345") outputs[0] PosixPath('jobs/exp1/logs/train_model.12345.out')

Source code in src/slurmkit/slurm.py
def find_job_output(
    job_id: str,
    jobs_dir: Optional[Path] = None,
    config: Optional[Config] = None,
) -> List[Path]:
    """
    Find job output files matching a job ID.

    Searches for .out files containing the job ID in their filename,
    using configured output patterns to match files.

    Args:
        job_id: SLURM job ID to search for.
        jobs_dir: Directory to search in. If None, uses config's jobs_dir.
        config: Configuration object. If None, uses global config.

    Returns:
        List of matching file paths, sorted by modification time (newest first).

    Example:
        >>> outputs = find_job_output("12345")
        >>> outputs[0]
        PosixPath('jobs/exp1/logs/train_model.12345.out')
    """
    if config is None:
        config = get_config()

    if jobs_dir is None:
        jobs_dir = config.get_path("jobs_dir")

    if jobs_dir is None or not jobs_dir.exists():
        return []

    # Search for files containing the job ID
    # Use glob pattern that matches the ID anywhere in the filename
    pattern = f"**/*{job_id}*.out"
    matches = list(jobs_dir.glob(pattern))

    # Sort by modification time (newest first)
    matches.sort(key=lambda p: p.stat().st_mtime, reverse=True)

    return matches

get_job_script_path

get_job_script_path(job_id: str) -> Optional[Path]

Get the script path for a job using scontrol.

This only works for jobs that are still in SLURM's active job table (typically running or recently completed jobs).

Parameters:

Name Type Description Default
job_id str

SLURM job ID.

required

Returns:

Type Description
Optional[Path]

Path to the job script, or None if not found.

Source code in src/slurmkit/slurm.py
def get_job_script_path(job_id: str) -> Optional[Path]:
    """
    Get the script path for a job using scontrol.

    This only works for jobs that are still in SLURM's active job table
    (typically running or recently completed jobs).

    Args:
        job_id: SLURM job ID.

    Returns:
        Path to the job script, or None if not found.
    """
    cmd = ["scontrol", "show", "job", job_id]

    try:
        result = run_command(cmd)
        if result.returncode != 0 or not result.stdout.strip():
            return None

        # Parse key=value output
        for token in result.stdout.split():
            if "=" not in token:
                continue
            key, value = token.split("=", 1)
            if key in ("Command", "BatchScript"):
                if value and value not in ("(null)", "Unknown"):
                    return Path(value)

        return None

    except FileNotFoundError:
        return None

Usage Example

from slurmkit import (
    get_job_status,
    get_sacct_info,
    get_pending_jobs,
    submit_job,
    find_job_output,
)

# Get status of specific jobs
statuses = get_job_status(["12345", "12346", "12347"])
for job_id, status in statuses.items():
    print(f"Job {job_id}: {status}")

# Get detailed accounting info
info = get_sacct_info(["12345"])
print(info)  # DataFrame with job details

# List pending/running jobs
pending = get_pending_jobs()
print(f"You have {len(pending)} jobs in the queue")

# Submit a job script
job_id = submit_job("jobs/exp1/job_scripts/train.sh")
print(f"Submitted job: {job_id}")

# Find output file for a job
output_path = find_job_output(
    job_id="12345",
    job_name="lr0.01_bs32",
    search_dirs=["jobs/exp1/logs"]
)
print(f"Output file: {output_path}")