Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: MambaInstall plugin #8515

Closed
wants to merge 1 commit into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions distributed/diagnostics/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,78 @@
raise RuntimeError(msg)


class MambaInstall(InstallPlugin):
"""A plugin to install a set of packages with Mamba or Micromamba.

This accepts a set of packages to install on the scheduler and all workers,
as well as options to use when installing. You can specify the use of Micromamba
and whether to restart the workers after installation.

Parameters
----------
packages : list[str]
A list of packages (with optional versions) to install using Mamba or Micromamba.
channels : list[str], optional
A list of channels to include for package resolution.
mamba_options : list[str], optional
Additional command-line options to pass to Mamba.
use_micromamba : bool, optional
Whether to use Micromamba instead of Mamba for installation. Defaults to False.
restart_workers : bool, optional
Whether to restart the worker after installing the packages. Defaults to False.

Examples:
--------
>>> from dask.distributed import MambaInstall
>>> plugin = MambaInstall(packages=["numpy"], channels=["conda-forge"], mamba_options=["--strict-channel-priority"])

>>> client.register_plugin(plugin)
"""

def __init__(
self,
packages: list[str],
channels: list[str] | None = None,
mamba_options: list[str] | None = None,
use_micromamba: bool = False,
restart_workers: bool = False,
):
installer = _MambaInstaller(packages, channels, mamba_options, use_micromamba)
super().__init__(installer, restart_workers=restart_workers)

Check warning on line 674 in distributed/diagnostics/plugin.py

View check run for this annotation

Codecov / codecov/patch

distributed/diagnostics/plugin.py#L673-L674

Added lines #L673 - L674 were not covered by tests


class _MambaInstaller:
def __init__(
self,
packages: list[str],
channels: list[str] | None,
mamba_options: list[str] | None,
use_micromamba: bool,
):
self.packages = packages
self.channels = channels or ["conda-forge"]
self.mamba_options = mamba_options or []
self.use_micromamba = use_micromamba

Check warning on line 688 in distributed/diagnostics/plugin.py

View check run for this annotation

Codecov / codecov/patch

distributed/diagnostics/plugin.py#L685-L688

Added lines #L685 - L688 were not covered by tests

def __call__(self) -> None:
installer = "micromamba" if self.use_micromamba else "mamba"
logger.info(

Check warning on line 692 in distributed/diagnostics/plugin.py

View check run for this annotation

Codecov / codecov/patch

distributed/diagnostics/plugin.py#L691-L692

Added lines #L691 - L692 were not covered by tests
"%s installing the following packages: %s",
installer,
", ".join(self.packages),
)
channels_str = " ".join([f"-c {channel}" for channel in self.channels])
options_str = " ".join(self.mamba_options)
packages_str = " ".join(self.packages)
command = f"{installer} install -y {channels_str} {options_str} {packages_str}"
try:
subprocess.run(command, shell=True, check=True, capture_output=True)
except subprocess.CalledProcessError as e:
msg = f"{installer} install failed with '{e.stderr.decode().strip()}'"
logger.error(msg)
raise RuntimeError(msg)

Check warning on line 706 in distributed/diagnostics/plugin.py

View check run for this annotation

Codecov / codecov/patch

distributed/diagnostics/plugin.py#L697-L706

Added lines #L697 - L706 were not covered by tests


class PipInstall(InstallPlugin):
"""A plugin to pip install a set of packages

Expand Down
Loading