diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5eb932ad..f998ab01 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-yaml - id: end-of-file-fixer @@ -8,12 +8,12 @@ repos: - id: check-toml - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.9.6 + rev: v0.13.3 hooks: - id: ruff args: [--fix] - id: ruff-format - repo: https://github.com/numpy/numpydoc - rev: "v1.8.0" + rev: "v1.9.0" hooks: - id: numpydoc-validation diff --git a/pyproject.toml b/pyproject.toml index 75bc4a31..017fd3f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,6 @@ known_first_party = ["lsst"] write_to = "python/lsst/ctrl/mpexec/version.py" [tool.pytest.ini_options] -open_files_ignore = ["*.ttf", "gen3.sqlite3"] [tool.pydocstyle] convention = "numpy" diff --git a/python/lsst/ctrl/mpexec/cli/butler_factory.py b/python/lsst/ctrl/mpexec/cli/butler_factory.py index b9b0362b..593d3a62 100644 --- a/python/lsst/ctrl/mpexec/cli/butler_factory.py +++ b/python/lsst/ctrl/mpexec/cli/butler_factory.py @@ -583,7 +583,7 @@ def make_write_butler( # collection from its chain collection first. with butler.transaction(): butler.collections.redefine_chain(self.output.name, chain_definition) - butler.removeRuns([replaced], unstore=True) + butler.removeRuns([replaced]) elif prune_replaced is not None: raise NotImplementedError(f"Unsupported --prune-replaced option '{prune_replaced}'.") if not self.output.exists: diff --git a/python/lsst/ctrl/mpexec/cli/cmd/commands.py b/python/lsst/ctrl/mpexec/cli/cmd/commands.py index 7116d351..c8ef8283 100644 --- a/python/lsst/ctrl/mpexec/cli/cmd/commands.py +++ b/python/lsst/ctrl/mpexec/cli/cmd/commands.py @@ -141,7 +141,7 @@ def coverage_context(kwargs: dict[str, Any]) -> Iterator[None]: @click.command(cls=PipetaskCommand, epilog=epilog) @click.pass_context @ctrlMpExecOpts.show_option() -@ctrlMpExecOpts.pipeline_build_options() +@ctrlMpExecOpts.pipeline_build_options(skip_butler_config=True) @ctrlMpExecOpts.qgraph_options() @ctrlMpExecOpts.butler_options() @option_section(sectionText="") diff --git a/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py b/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py index f6c82ee9..d2adef28 100644 --- a/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py +++ b/python/lsst/ctrl/mpexec/cli/opt/optionGroups.py @@ -54,9 +54,15 @@ class pipeline_build_options(OptionGroup): # noqa: N801 """Decorator to add options to the command function for building a pipeline. + + Parameters + ---------- + skip_butler_config : `bool`, optional + If `True` the butler configuration option will not be included and will + be assumed to be added explicitly elsewhere. """ - def __init__(self) -> None: + def __init__(self, skip_butler_config: bool = False) -> None: self.decorators = [ option_section(sectionText="Pipeline build options:"), ctrlMpExecOpts.pipeline_option(), @@ -76,8 +82,9 @@ def __init__(self) -> None: ctrlMpExecOpts.pipeline_dot_option(), ctrlMpExecOpts.pipeline_mermaid_option(), pipeBaseOpts.instrument_option(help=instrumentOptionHelp, metavar="instrument", multiple=True), - ctrlMpExecOpts.butler_config_option(required=False), ] + if not skip_butler_config: + self.decorators.append(ctrlMpExecOpts.butler_config_option(required=False)) class coverage_options(OptionGroup): # noqa: N801 @@ -95,9 +102,23 @@ def __init__(self) -> None: class qgraph_options(OptionGroup): # noqa: N801 """Decorator to add options to a command function for creating a quantum graph. + + Parameters + ---------- + skip_coverage : `bool`, optional + If `True` the coverage configuration options will not be included and + will be assumed to be added explicitly elsewhere. + skip_clobber : `bool`, optional + If `True` the clobber configuration option will not be included and + will be assumed to be added explicitly elsewhere. + skip_summary : `bool`, optional + If `True` the summary configuration option will not be included and + will be assumed to be added explicitly elsewhere. """ - def __init__(self) -> None: + def __init__( + self, skip_coverage: bool = False, skip_clobber: bool = False, skip_summary: bool = False + ) -> None: self.decorators = [ option_section(sectionText="Quantum graph building options:"), ctrlMpExecOpts.qgraph_option(), @@ -106,18 +127,21 @@ def __init__(self) -> None: ctrlMpExecOpts.qgraph_datastore_records_option(), ctrlMpExecOpts.skip_existing_in_option(), ctrlMpExecOpts.skip_existing_option(), - ctrlMpExecOpts.clobber_outputs_option(), ctrlMpExecOpts.save_qgraph_option(), ctrlMpExecOpts.qgraph_dot_option(), ctrlMpExecOpts.qgraph_mermaid_option(), - ctrlMpExecOpts.summary_option(), ctrlMpExecOpts.dataset_query_constraint(), ctrlMpExecOpts.data_id_table_option(), ctrlMpExecOpts.mock_option(), ctrlMpExecOpts.mock_failure_option(), ctrlMpExecOpts.unmocked_dataset_types_option(), - coverage_options(), ] + if not skip_clobber: + self.decorators.append(ctrlMpExecOpts.clobber_outputs_option()) + if not skip_summary: + self.decorators.append(ctrlMpExecOpts.summary_option()) + if not skip_coverage: + self.decorators.append(coverage_options()) class butler_options(OptionGroup): # noqa: N801 @@ -187,8 +211,8 @@ def __init__(self) -> None: click.pass_context, ctrlMpExecOpts.debug_option(), ctrlMpExecOpts.show_option(), - pipeline_build_options(), - qgraph_options(), + pipeline_build_options(skip_butler_config=True), + qgraph_options(skip_coverage=True, skip_clobber=True, skip_summary=True), butler_options(), execution_options(), meta_info_options(), diff --git a/python/lsst/ctrl/mpexec/cli/utils.py b/python/lsst/ctrl/mpexec/cli/utils.py index db7cc3bf..510aba78 100644 --- a/python/lsst/ctrl/mpexec/cli/utils.py +++ b/python/lsst/ctrl/mpexec/cli/utils.py @@ -227,7 +227,7 @@ def summarize_quantum_graph(qg: BaseQuantumGraph) -> int: Parameters ---------- qg : `lsst.pipe.base.quantum_graph.BaseQuantumGraph` - Quantum graph object + Quantum graph object. Returns -------