Skip to content

Collections

Job collection management for tracking related SLURM jobs.

Overview

Collections allow you to group related jobs together, track their states, and perform batch operations. Each collection stores:

  • Job metadata (parameters, script paths)
  • Job IDs and submission history
  • Current job states
  • Timestamps

Collection.refresh_states() derives canonical SLURM state from full sacct rows and stores per-attempt debug details in attempt.raw_state.

Classes

Collection

Collection

Collection(name: str, description: str = '', created_at: Optional[str] = None, updated_at: Optional[str] = None, cluster: Optional[str] = None, parameters: Optional[Dict[str, Any]] = None, jobs: Optional[List[Dict[str, Any]]] = None, generation: Optional[Dict[str, Any]] = None, notifications: Optional[Dict[str, Any]] = None, version: int = COLLECTION_SCHEMA_VERSION)

A collection of related SLURM jobs stored in an attempts-based schema.

Source code in src/slurmkit/collections.py
def __init__(
    self,
    name: str,
    description: str = "",
    created_at: Optional[str] = None,
    updated_at: Optional[str] = None,
    cluster: Optional[str] = None,
    parameters: Optional[Dict[str, Any]] = None,
    jobs: Optional[List[Dict[str, Any]]] = None,
    generation: Optional[Dict[str, Any]] = None,
    notifications: Optional[Dict[str, Any]] = None,
    version: int = COLLECTION_SCHEMA_VERSION,
):
    self.version = version
    self.name = name
    self.description = description
    self.cluster = cluster or socket.gethostname()
    now = _now_iso()
    self.created_at = created_at or now
    self.updated_at = updated_at or now
    self.parameters = dict(parameters or {})
    self.generation = dict(generation or {})
    self.notifications = dict(notifications or {})
    self._jobs: List[Dict[str, Any]] = [
        self._normalize_job(job)
        for job in (jobs or [])
    ]

add_job

add_job(job_name: str, script_path: Optional[Union[str, Path]] = None, output_path: Optional[Union[str, Path]] = None, job_id: Optional[str] = None, state: Optional[str] = None, parameters: Optional[Dict[str, Any]] = None, hostname: Optional[str] = None, submitted_at: Optional[str] = None, started_at: Optional[str] = None, completed_at: Optional[str] = None) -> Dict[str, Any]
Source code in src/slurmkit/collections.py
def add_job(
    self,
    job_name: str,
    script_path: Optional[Union[str, Path]] = None,
    output_path: Optional[Union[str, Path]] = None,
    job_id: Optional[str] = None,
    state: Optional[str] = None,
    parameters: Optional[Dict[str, Any]] = None,
    hostname: Optional[str] = None,
    submitted_at: Optional[str] = None,
    started_at: Optional[str] = None,
    completed_at: Optional[str] = None,
) -> Dict[str, Any]:
    job = {
        "job_name": job_name,
        "parameters": dict(parameters or {}),
        "attempts": [
            self._new_attempt(
                kind="primary",
                job_id=job_id,
                state=state,
                hostname=hostname,
                submitted_at=submitted_at,
                started_at=started_at,
                completed_at=completed_at,
                script_path=script_path,
                output_path=output_path,
                attempt_job_name=job_name,
                parameters=parameters,
                regenerated=False,
            )
        ],
    }
    self._jobs.append(job)
    self._touch()
    return job

get_job

get_job(job_name: str) -> Optional[Dict[str, Any]]
Source code in src/slurmkit/collections.py
def get_job(self, job_name: str) -> Optional[Dict[str, Any]]:
    for job in self._jobs:
        if job["job_name"] == job_name:
            return job
    return None

update_job

update_job(job_name: str, job_id: Optional[str] = None, state: Optional[str] = None, output_path: Optional[Union[str, Path]] = None, submitted_at: Optional[str] = None, started_at: Optional[str] = None, completed_at: Optional[str] = None, **kwargs: Any) -> bool
Source code in src/slurmkit/collections.py
def update_job(
    self,
    job_name: str,
    job_id: Optional[str] = None,
    state: Optional[str] = None,
    output_path: Optional[Union[str, Path]] = None,
    submitted_at: Optional[str] = None,
    started_at: Optional[str] = None,
    completed_at: Optional[str] = None,
    **kwargs: Any,
) -> bool:
    job = self.get_job(job_name)
    if job is None:
        return False
    attempt = self.primary_attempt(job)
    if job_id is not None:
        attempt["job_id"] = _string_or_none(job_id)
    if state is not None:
        attempt["state"] = _string_or_none(state)
    if output_path is not None:
        attempt["output_path"] = str(output_path)
    if submitted_at is not None:
        attempt["submitted_at"] = _string_or_none(submitted_at)
    if started_at is not None:
        attempt["started_at"] = _string_or_none(started_at)
    if completed_at is not None:
        attempt["completed_at"] = _string_or_none(completed_at)
    for key, value in kwargs.items():
        attempt[key] = value
    self._touch()
    return True

add_resubmission

add_resubmission(job_name: str, job_id: str, output_path: Optional[Union[str, Path]] = None, extra_params: Optional[Dict[str, Any]] = None, submission_group: Optional[str] = None, hostname: Optional[str] = None, attempt_job_name: Optional[str] = None, attempt_script_path: Optional[Union[str, Path]] = None, attempt_parameters: Optional[Dict[str, Any]] = None, regenerated: Optional[bool] = None) -> bool
Source code in src/slurmkit/collections.py
def add_resubmission(
    self,
    job_name: str,
    job_id: str,
    output_path: Optional[Union[str, Path]] = None,
    extra_params: Optional[Dict[str, Any]] = None,
    submission_group: Optional[str] = None,
    hostname: Optional[str] = None,
    attempt_job_name: Optional[str] = None,
    attempt_script_path: Optional[Union[str, Path]] = None,
    attempt_parameters: Optional[Dict[str, Any]] = None,
    regenerated: Optional[bool] = None,
) -> bool:
    job = self.get_job(job_name)
    if job is None:
        return False
    attempt = self._new_attempt(
        kind="resubmission",
        job_id=job_id,
        hostname=hostname,
        submitted_at=_now_iso(),
        output_path=output_path,
        submission_group=submission_group,
        attempt_job_name=attempt_job_name or job_name,
        script_path=attempt_script_path,
        parameters=attempt_parameters,
        extra_params=extra_params,
        regenerated=regenerated,
    )
    job["attempts"].append(attempt)
    self._touch()
    return True

remove_job

remove_job(job_name: str) -> bool
Source code in src/slurmkit/collections.py
def remove_job(self, job_name: str) -> bool:
    for index, job in enumerate(self._jobs):
        if job["job_name"] == job_name:
            del self._jobs[index]
            self._touch()
            return True
    return False

filter_jobs

filter_jobs(state: Optional[str] = None, hostname: Optional[str] = None, submitted: Optional[bool] = None) -> List[Dict[str, Any]]
Source code in src/slurmkit/collections.py
def filter_jobs(
    self,
    state: Optional[str] = None,
    hostname: Optional[str] = None,
    submitted: Optional[bool] = None,
) -> List[Dict[str, Any]]:
    result = []
    for job in self._jobs:
        primary = self.primary_attempt(job)
        if submitted is not None:
            is_submitted = primary.get("job_id") is not None
            if submitted != is_submitted:
                continue
        if hostname is not None and primary.get("hostname") != hostname:
            continue
        if state is not None and self._normalize_state(primary.get("state")) != state:
            continue
        result.append(job)
    return result

to_dict

to_dict() -> Dict[str, Any]
Source code in src/slurmkit/collections.py
def to_dict(self) -> Dict[str, Any]:
    return {
        "version": COLLECTION_SCHEMA_VERSION,
        "name": self.name,
        "description": self.description,
        "created_at": self.created_at,
        "updated_at": self.updated_at,
        "cluster": self.cluster,
        "parameters": self.parameters,
        "generation": self.generation,
        "notifications": self.notifications,
        "jobs": self._jobs,
    }

CollectionManager

CollectionManager

CollectionManager(collections_dir: Optional[Union[str, Path]] = None, config: Optional[Config] = None)

Load, save, and organize v2 collections.

Source code in src/slurmkit/collections.py
def __init__(
    self,
    collections_dir: Optional[Union[str, Path]] = None,
    config: Optional[Config] = None,
):
    explicit_config = config is not None
    if config is None:
        config = get_config()
    self.config = config
    if collections_dir is not None:
        self.collections_dir = Path(collections_dir)
    else:
        self.collections_dir = config.collections_dir
    if explicit_config or collections_dir is None:
        self.collection_locks_dir = config.collection_locks_dir
    else:
        self.collection_locks_dir = self.collections_dir.parent / "locks" / "collections"

create

create(name: str, description: str = '', parameters: Optional[Dict[str, Any]] = None, overwrite: bool = False) -> Collection
Source code in src/slurmkit/collections.py
def create(
    self,
    name: str,
    description: str = "",
    parameters: Optional[Dict[str, Any]] = None,
    overwrite: bool = False,
) -> Collection:
    canonical_name = self.normalize_name(name)
    if self.exists(canonical_name) and not overwrite:
        raise FileExistsError(
            f"Collection already exists: {canonical_name}. Use overwrite=True to replace."
        )
    collection = Collection(
        name=canonical_name,
        description=description,
        parameters=parameters,
    )
    self.save(collection)
    return collection

exists

exists(name: str) -> bool
Source code in src/slurmkit/collections.py
def exists(self, name: str) -> bool:
    return self.get_collection_path(name).exists()

delete

delete(name: str) -> bool
Source code in src/slurmkit/collections.py
def delete(self, name: str) -> bool:
    path = self.get_collection_path(name)
    if path.exists():
        path.unlink()
        self._prune_empty_parent_dirs(path.parent)
        return True
    return False

get_collection_path

get_collection_path(name: Any) -> Path
Source code in src/slurmkit/collections.py
def get_collection_path(self, name: Any) -> Path:
    relative_path = collection_id_to_relative_path(name)
    path = self.collections_dir / relative_path
    collections_root = self.collections_dir.resolve(strict=False)
    resolved_path = path.resolve(strict=False)
    try:
        resolved_path.relative_to(collections_root)
    except ValueError as exc:
        raise _invalid_collection_id(name, "resolved path escapes the collections directory") from exc
    return path

Usage Example

from slurmkit import Collection, CollectionManager

# Create a collection manager
manager = CollectionManager()

# Create a new collection
collection = manager.create("my_experiment", description="Training sweep")

# Add a job to the collection
collection.add_job(
    job_name="lr0.01_bs32",
    parameters={"learning_rate": 0.01, "batch_size": 32},
    script_path=".jobs/exp1/job_scripts/lr0.01_bs32.job",
)

# Submit and record job ID
collection.update_job("lr0.01_bs32", job_id="12345", state="PENDING")

# Filter jobs by state
failed_jobs = collection.filter_jobs(state="FAILED")

# Save collection
manager.save(collection)