Collections
Job collection management for tracking related SLURM jobs.
Overview
Collections allow you to group related jobs together, track their states, and perform batch operations. Each collection stores:
- Job metadata (parameters, script paths)
- Job IDs and submission history
- Current job states
- Timestamps
Collection.refresh_states() derives canonical SLURM state from full sacct
rows and stores per-attempt debug details in attempt.raw_state.
Classes
Collection
Collection
Collection(name: str, description: str = '', created_at: Optional[str] = None, updated_at: Optional[str] = None, cluster: Optional[str] = None, parameters: Optional[Dict[str, Any]] = None, jobs: Optional[List[Dict[str, Any]]] = None, generation: Optional[Dict[str, Any]] = None, notifications: Optional[Dict[str, Any]] = None, version: int = COLLECTION_SCHEMA_VERSION)
A collection of related SLURM jobs stored in an attempts-based schema.
Source code in src/slurmkit/collections.py
| def __init__(
self,
name: str,
description: str = "",
created_at: Optional[str] = None,
updated_at: Optional[str] = None,
cluster: Optional[str] = None,
parameters: Optional[Dict[str, Any]] = None,
jobs: Optional[List[Dict[str, Any]]] = None,
generation: Optional[Dict[str, Any]] = None,
notifications: Optional[Dict[str, Any]] = None,
version: int = COLLECTION_SCHEMA_VERSION,
):
self.version = version
self.name = name
self.description = description
self.cluster = cluster or socket.gethostname()
now = _now_iso()
self.created_at = created_at or now
self.updated_at = updated_at or now
self.parameters = dict(parameters or {})
self.generation = dict(generation or {})
self.notifications = dict(notifications or {})
self._jobs: List[Dict[str, Any]] = [
self._normalize_job(job)
for job in (jobs or [])
]
|
add_job
add_job(job_name: str, script_path: Optional[Union[str, Path]] = None, output_path: Optional[Union[str, Path]] = None, job_id: Optional[str] = None, state: Optional[str] = None, parameters: Optional[Dict[str, Any]] = None, hostname: Optional[str] = None, submitted_at: Optional[str] = None, started_at: Optional[str] = None, completed_at: Optional[str] = None) -> Dict[str, Any]
Source code in src/slurmkit/collections.py
| def add_job(
self,
job_name: str,
script_path: Optional[Union[str, Path]] = None,
output_path: Optional[Union[str, Path]] = None,
job_id: Optional[str] = None,
state: Optional[str] = None,
parameters: Optional[Dict[str, Any]] = None,
hostname: Optional[str] = None,
submitted_at: Optional[str] = None,
started_at: Optional[str] = None,
completed_at: Optional[str] = None,
) -> Dict[str, Any]:
job = {
"job_name": job_name,
"parameters": dict(parameters or {}),
"attempts": [
self._new_attempt(
kind="primary",
job_id=job_id,
state=state,
hostname=hostname,
submitted_at=submitted_at,
started_at=started_at,
completed_at=completed_at,
script_path=script_path,
output_path=output_path,
attempt_job_name=job_name,
parameters=parameters,
regenerated=False,
)
],
}
self._jobs.append(job)
self._touch()
return job
|
get_job
get_job(job_name: str) -> Optional[Dict[str, Any]]
Source code in src/slurmkit/collections.py
| def get_job(self, job_name: str) -> Optional[Dict[str, Any]]:
for job in self._jobs:
if job["job_name"] == job_name:
return job
return None
|
update_job
update_job(job_name: str, job_id: Optional[str] = None, state: Optional[str] = None, output_path: Optional[Union[str, Path]] = None, submitted_at: Optional[str] = None, started_at: Optional[str] = None, completed_at: Optional[str] = None, **kwargs: Any) -> bool
Source code in src/slurmkit/collections.py
| def update_job(
self,
job_name: str,
job_id: Optional[str] = None,
state: Optional[str] = None,
output_path: Optional[Union[str, Path]] = None,
submitted_at: Optional[str] = None,
started_at: Optional[str] = None,
completed_at: Optional[str] = None,
**kwargs: Any,
) -> bool:
job = self.get_job(job_name)
if job is None:
return False
attempt = self.primary_attempt(job)
if job_id is not None:
attempt["job_id"] = _string_or_none(job_id)
if state is not None:
attempt["state"] = _string_or_none(state)
if output_path is not None:
attempt["output_path"] = str(output_path)
if submitted_at is not None:
attempt["submitted_at"] = _string_or_none(submitted_at)
if started_at is not None:
attempt["started_at"] = _string_or_none(started_at)
if completed_at is not None:
attempt["completed_at"] = _string_or_none(completed_at)
for key, value in kwargs.items():
attempt[key] = value
self._touch()
return True
|
add_resubmission
add_resubmission(job_name: str, job_id: str, output_path: Optional[Union[str, Path]] = None, extra_params: Optional[Dict[str, Any]] = None, submission_group: Optional[str] = None, hostname: Optional[str] = None, attempt_job_name: Optional[str] = None, attempt_script_path: Optional[Union[str, Path]] = None, attempt_parameters: Optional[Dict[str, Any]] = None, regenerated: Optional[bool] = None) -> bool
Source code in src/slurmkit/collections.py
| def add_resubmission(
self,
job_name: str,
job_id: str,
output_path: Optional[Union[str, Path]] = None,
extra_params: Optional[Dict[str, Any]] = None,
submission_group: Optional[str] = None,
hostname: Optional[str] = None,
attempt_job_name: Optional[str] = None,
attempt_script_path: Optional[Union[str, Path]] = None,
attempt_parameters: Optional[Dict[str, Any]] = None,
regenerated: Optional[bool] = None,
) -> bool:
job = self.get_job(job_name)
if job is None:
return False
attempt = self._new_attempt(
kind="resubmission",
job_id=job_id,
hostname=hostname,
submitted_at=_now_iso(),
output_path=output_path,
submission_group=submission_group,
attempt_job_name=attempt_job_name or job_name,
script_path=attempt_script_path,
parameters=attempt_parameters,
extra_params=extra_params,
regenerated=regenerated,
)
job["attempts"].append(attempt)
self._touch()
return True
|
remove_job
remove_job(job_name: str) -> bool
Source code in src/slurmkit/collections.py
| def remove_job(self, job_name: str) -> bool:
for index, job in enumerate(self._jobs):
if job["job_name"] == job_name:
del self._jobs[index]
self._touch()
return True
return False
|
filter_jobs
filter_jobs(state: Optional[str] = None, hostname: Optional[str] = None, submitted: Optional[bool] = None) -> List[Dict[str, Any]]
Source code in src/slurmkit/collections.py
| def filter_jobs(
self,
state: Optional[str] = None,
hostname: Optional[str] = None,
submitted: Optional[bool] = None,
) -> List[Dict[str, Any]]:
result = []
for job in self._jobs:
primary = self.primary_attempt(job)
if submitted is not None:
is_submitted = primary.get("job_id") is not None
if submitted != is_submitted:
continue
if hostname is not None and primary.get("hostname") != hostname:
continue
if state is not None and self._normalize_state(primary.get("state")) != state:
continue
result.append(job)
return result
|
to_dict
to_dict() -> Dict[str, Any]
Source code in src/slurmkit/collections.py
| def to_dict(self) -> Dict[str, Any]:
return {
"version": COLLECTION_SCHEMA_VERSION,
"name": self.name,
"description": self.description,
"created_at": self.created_at,
"updated_at": self.updated_at,
"cluster": self.cluster,
"parameters": self.parameters,
"generation": self.generation,
"notifications": self.notifications,
"jobs": self._jobs,
}
|
CollectionManager
CollectionManager
CollectionManager(collections_dir: Optional[Union[str, Path]] = None, config: Optional[Config] = None)
Load, save, and organize v2 collections.
Source code in src/slurmkit/collections.py
| def __init__(
self,
collections_dir: Optional[Union[str, Path]] = None,
config: Optional[Config] = None,
):
explicit_config = config is not None
if config is None:
config = get_config()
self.config = config
if collections_dir is not None:
self.collections_dir = Path(collections_dir)
else:
self.collections_dir = config.collections_dir
if explicit_config or collections_dir is None:
self.collection_locks_dir = config.collection_locks_dir
else:
self.collection_locks_dir = self.collections_dir.parent / "locks" / "collections"
|
create
create(name: str, description: str = '', parameters: Optional[Dict[str, Any]] = None, overwrite: bool = False) -> Collection
Source code in src/slurmkit/collections.py
| def create(
self,
name: str,
description: str = "",
parameters: Optional[Dict[str, Any]] = None,
overwrite: bool = False,
) -> Collection:
canonical_name = self.normalize_name(name)
if self.exists(canonical_name) and not overwrite:
raise FileExistsError(
f"Collection already exists: {canonical_name}. Use overwrite=True to replace."
)
collection = Collection(
name=canonical_name,
description=description,
parameters=parameters,
)
self.save(collection)
return collection
|
exists
exists(name: str) -> bool
Source code in src/slurmkit/collections.py
| def exists(self, name: str) -> bool:
return self.get_collection_path(name).exists()
|
delete
delete(name: str) -> bool
Source code in src/slurmkit/collections.py
| def delete(self, name: str) -> bool:
path = self.get_collection_path(name)
if path.exists():
path.unlink()
self._prune_empty_parent_dirs(path.parent)
return True
return False
|
get_collection_path
get_collection_path(name: Any) -> Path
Source code in src/slurmkit/collections.py
| def get_collection_path(self, name: Any) -> Path:
relative_path = collection_id_to_relative_path(name)
path = self.collections_dir / relative_path
collections_root = self.collections_dir.resolve(strict=False)
resolved_path = path.resolve(strict=False)
try:
resolved_path.relative_to(collections_root)
except ValueError as exc:
raise _invalid_collection_id(name, "resolved path escapes the collections directory") from exc
return path
|
Usage Example
from slurmkit import Collection, CollectionManager
# Create a collection manager
manager = CollectionManager()
# Create a new collection
collection = manager.create("my_experiment", description="Training sweep")
# Add a job to the collection
collection.add_job(
job_name="lr0.01_bs32",
parameters={"learning_rate": 0.01, "batch_size": 32},
script_path=".jobs/exp1/job_scripts/lr0.01_bs32.job",
)
# Submit and record job ID
collection.update_job("lr0.01_bs32", job_id="12345", state="PENDING")
# Filter jobs by state
failed_jobs = collection.filter_jobs(state="FAILED")
# Save collection
manager.save(collection)