Skip to content

Sync

Cross-cluster job state synchronization.

Overview

The SyncManager class enables sharing job states across multiple compute clusters via git. This is useful when:

  • Running experiments across multiple clusters
  • Sharing job status with collaborators
  • Maintaining a central record of all job states

Classes

SyncManager

SyncManager

SyncManager(sync_dir: Optional[Union[str, Path]] = None, config: Optional[Config] = None)

Manager for cross-cluster job state synchronization.

Handles writing sync files and optionally committing/pushing them to git for sharing across clusters.

Attributes:

Name Type Description
sync_dir

Directory where sync files are stored.

hostname

Current machine's hostname.

Example

sync_mgr = SyncManager() sync_mgr.sync_all() # Refresh and write sync file sync_mgr.push() # Git commit and push

Initialize the sync manager.

Parameters:

Name Type Description Default
sync_dir Optional[Union[str, Path]]

Directory for sync files. If None, uses config.

None
config Optional[Config]

Configuration object. If None, uses global config.

None
Source code in src/slurmkit/sync.py
def __init__(
    self,
    sync_dir: Optional[Union[str, Path]] = None,
    config: Optional[Config] = None,
):
    """
    Initialize the sync manager.

    Args:
        sync_dir: Directory for sync files. If None, uses config.
        config: Configuration object. If None, uses global config.
    """
    if config is None:
        config = get_config()

    self.config = config
    self.hostname = socket.gethostname()

    if sync_dir is not None:
        self.sync_dir = Path(sync_dir)
    else:
        self.sync_dir = config.sync_dir

    self._collection_manager = CollectionManager(config=config)

sync_collection

sync_collection(name: str) -> Dict[str, Any]

Sync a single collection's job states.

Refreshes job states from SLURM and returns the updated data.

Parameters:

Name Type Description Default
name str

Collection name.

required

Returns:

Type Description
Dict[str, Any]

Collection sync data dictionary.

Source code in src/slurmkit/sync.py
def sync_collection(self, name: str) -> Dict[str, Any]:
    """
    Sync a single collection's job states.

    Refreshes job states from SLURM and returns the updated data.

    Args:
        name: Collection name.

    Returns:
        Collection sync data dictionary.
    """
    collection = self._collection_manager.load(name)

    # Refresh states from SLURM
    updated_count = collection.refresh_states()

    # Save updated collection
    self._collection_manager.save(collection)

    # Build sync data
    summary = collection.get_summary()

    jobs_data = []
    for job in collection.jobs:
        primary = collection.primary_attempt(job)
        job_data = {
            "job_name": job["job_name"],
            "job_id": primary.get("job_id"),
            "state": primary.get("state"),
            "hostname": primary.get("hostname"),
        }

        resubmissions = job.get("attempts", [])[1:]
        if resubmissions:
            job_data["resubmissions"] = [
                {
                    "job_id": r.get("job_id"),
                    "state": r.get("state"),
                    "hostname": r.get("hostname"),
                }
                for r in resubmissions
            ]

        jobs_data.append(job_data)

    return {
        "name": name,
        "description": collection.description,
        "updated_jobs": updated_count,
        **summary,
        "jobs": jobs_data,
    }

sync_all

sync_all(collection_names: Optional[List[str]] = None) -> Dict[str, Any]

Sync all collections and write sync file.

Parameters:

Name Type Description Default
collection_names Optional[List[str]]

List of collection names to sync. If None, syncs all collections.

None

Returns:

Type Description
Dict[str, Any]

Complete sync data dictionary.

Source code in src/slurmkit/sync.py
def sync_all(
    self,
    collection_names: Optional[List[str]] = None,
) -> Dict[str, Any]:
    """
    Sync all collections and write sync file.

    Args:
        collection_names: List of collection names to sync.
            If None, syncs all collections.

    Returns:
        Complete sync data dictionary.
    """
    if collection_names is None:
        collection_names = self._collection_manager.list_collections()

    collections_data = {}
    total_updated = 0

    for name in collection_names:
        try:
            data = self.sync_collection(name)
            collections_data[name] = data
            total_updated += data.get("updated_jobs", 0)
        except FileNotFoundError:
            # Skip collections that don't exist
            continue
        except Exception as e:
            # Log error but continue with other collections
            print(f"Warning: Error syncing collection '{name}': {e}")
            continue

    # Build sync file data
    sync_data = {
        "cluster": self.hostname,
        "synced_at": datetime.now().isoformat(timespec="seconds"),
        "total_collections": len(collections_data),
        "total_jobs_updated": total_updated,
        "collections": collections_data,
    }

    # Write sync file
    self._ensure_dir()
    sync_path = self.get_sync_file_path()

    with open(sync_path, "w") as f:
        yaml.dump(
            sync_data,
            f,
            default_flow_style=False,
            sort_keys=False,
            allow_unicode=True,
        )

    return sync_data

push

push(message: Optional[str] = None) -> bool

Git commit and push sync files.

Parameters:

Name Type Description Default
message Optional[str]

Commit message. If None, uses a default message.

None

Returns:

Type Description
bool

True if successful, False otherwise.

Source code in src/slurmkit/sync.py
def push(self, message: Optional[str] = None) -> bool:
    """
    Git commit and push sync files.

    Args:
        message: Commit message. If None, uses a default message.

    Returns:
        True if successful, False otherwise.
    """
    sync_path = self.get_sync_file_path()

    if not sync_path.exists():
        print("No sync file to push. Run sync first.")
        return False

    if message is None:
        message = f"slurmkit sync: {self.hostname} at {datetime.now().isoformat(timespec='seconds')}"

    try:
        # Git add
        result = subprocess.run(
            ["git", "add", str(sync_path)],
            capture_output=True,
            text=True,
            cwd=self.sync_dir.parent,
        )
        if result.returncode != 0:
            print(f"git add failed: {result.stderr}")
            return False

        # Git commit
        result = subprocess.run(
            ["git", "commit", "-m", message],
            capture_output=True,
            text=True,
            cwd=self.sync_dir.parent,
        )
        if result.returncode != 0:
            if "nothing to commit" in result.stdout:
                print("No changes to commit.")
                return True
            print(f"git commit failed: {result.stderr}")
            return False

        # Git push
        result = subprocess.run(
            ["git", "push"],
            capture_output=True,
            text=True,
            cwd=self.sync_dir.parent,
        )
        if result.returncode != 0:
            print(f"git push failed: {result.stderr}")
            return False

        print(f"Successfully pushed sync file: {sync_path.name}")
        return True

    except FileNotFoundError:
        print("git command not found.")
        return False

get_sync_file_path

get_sync_file_path(hostname: Optional[str] = None) -> Path

Get the path to a sync file for a hostname.

Parameters:

Name Type Description Default
hostname Optional[str]

Hostname for the sync file. Defaults to current host.

None

Returns:

Type Description
Path

Path to the sync file.

Source code in src/slurmkit/sync.py
def get_sync_file_path(self, hostname: Optional[str] = None) -> Path:
    """
    Get the path to a sync file for a hostname.

    Args:
        hostname: Hostname for the sync file. Defaults to current host.

    Returns:
        Path to the sync file.
    """
    hostname = hostname or self.hostname
    return self.sync_dir / f"{hostname}.yaml"

Usage Example

from slurmkit.sync import SyncManager
from slurmkit import CollectionManager

# Initialize sync manager
sync = SyncManager()

# Get a collection
manager = CollectionManager()
collection = manager.get("my_experiment")

# Sync collection state to git-tracked file
sync.sync_collection(collection)

# Push changes to remote
sync.push(message="Update job states from cluster-a")

CLI Usage

The sync functionality is typically used via the CLI:

# Sync all collections and push
slurmkit sync --push

# Sync specific collection
slurmkit sync --collection my_experiment

# On another cluster, pull and view
git pull
slurmkit collections show my_experiment