From a6a9cde62cb9396766ec064ebdc23b0c8bc656cc Mon Sep 17 00:00:00 2001 From: Fede Raimondo Date: Mon, 25 Nov 2024 16:10:51 +0100 Subject: [PATCH 01/10] Add config manager capacitt --- junifer/utils/__init__.pyi | 2 ++ junifer/utils/config.py | 71 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 junifer/utils/config.py diff --git a/junifer/utils/__init__.pyi b/junifer/utils/__init__.pyi index b9db68f18..b01470876 100644 --- a/junifer/utils/__init__.pyi +++ b/junifer/utils/__init__.pyi @@ -1,6 +1,7 @@ __all__ = [ "make_executable", "configure_logging", + "config", "logger", "raise_error", "warn_with_log", @@ -11,5 +12,6 @@ __all__ = [ from .fs import make_executable from .logging import configure_logging, logger, raise_error, warn_with_log +from .config import config from .helpers import run_ext_cmd, deep_update from ._yaml import yaml diff --git a/junifer/utils/config.py b/junifer/utils/config.py new file mode 100644 index 000000000..515980448 --- /dev/null +++ b/junifer/utils/config.py @@ -0,0 +1,71 @@ +"""Provide junifer global configuration.""" + +# Authors: Federico Raimondo +# Synchon Mandal +# License: AGPL +import os +from typing import Any + +from .logging import logger +from .singleton import Singleton + + +class ConfigManager(metaclass=Singleton): + """Manage configuration parameters. + + Attributes + ---------- + _config : dict + Configuration parameters. + + """ + + def __init__(self) -> None: + """Initialize the class.""" + self._config = {} + for t_var in os.environ: + if t_var.startswith("JUNIFER_"): + varname = t_var.replace("JUNIFER_", "") + varname = varname.lower() + varname.replace("_", ".") + logger.debug( + f"Setting {varname} from environment to " + f"{os.environ[t_var]}" + ) + self._config[varname] = os.environ[t_var] + + def get(self, key: str, default: Any = None) -> Any: + """Get configuration parameter. + + Parameters + ---------- + key : str + The configuration key. + + default : Any, optional + The default value to return if the key is not found (default None). + + Returns + ------- + Any + The configuration value. + + """ + return self._config.get(key, default) + + def set(self, key: str, value: Any) -> None: + """Set configuration parameter. + + Parameters + ---------- + key : str + The configuration key. + value : Any + The configuration value. + + """ + logger.debug(f"Setting {key} to {value}") + self._config[key] = value + + +config = ConfigManager() -- 2.52.0 From 75334add76c709ad4935fe37620a19a2e4c50969 Mon Sep 17 00:00:00 2001 From: Fede Raimondo Date: Mon, 25 Nov 2024 21:34:32 +0100 Subject: [PATCH 02/10] fix replace + cast to bool/int/float --- junifer/utils/config.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/junifer/utils/config.py b/junifer/utils/config.py index 515980448..637ff40f2 100644 --- a/junifer/utils/config.py +++ b/junifer/utils/config.py @@ -27,12 +27,25 @@ class ConfigManager(metaclass=Singleton): if t_var.startswith("JUNIFER_"): varname = t_var.replace("JUNIFER_", "") varname = varname.lower() - varname.replace("_", ".") + varname = varname.replace("_", ".") + var_value = os.environ[t_var] + if var_value.lower() == "true": + var_value = True + elif var_value.lower() == "false": + var_value = False + else: + try: + var_value = int(var_value) + except ValueError: + try: + var_value = float(var_value) + except ValueError: + pass logger.debug( f"Setting {varname} from environment to " - f"{os.environ[t_var]}" + f"{var_value} (type: {type(var_value)})" ) - self._config[varname] = os.environ[t_var] + self._config[varname] = var_value def get(self, key: str, default: Any = None) -> Any: """Get configuration parameter. -- 2.52.0 From 0a7f12257fd238354ca892c8a0176cdc482efea6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 10:37:57 +0100 Subject: [PATCH 03/10] chore: reorder __all__ using ruff --- junifer/api/decorators.py | 2 +- junifer/api/functions.py | 2 +- junifer/cli/cli.py | 20 ++++++++++---------- junifer/cli/parser.py | 2 +- junifer/data/_dispatch.py | 2 +- junifer/data/masks/_masks.py | 2 +- junifer/data/template_spaces.py | 2 +- junifer/stats.py | 2 +- junifer/storage/utils.py | 6 +++--- junifer/testing/datagrabbers.py | 2 +- junifer/typing/_typing.py | 14 +++++++------- junifer/utils/helpers.py | 2 +- junifer/utils/logging.py | 2 +- junifer/utils/singleton.py | 2 +- 14 files changed, 31 insertions(+), 31 deletions(-) diff --git a/junifer/api/decorators.py b/junifer/api/decorators.py index b063dc24a..bf39c6c5c 100644 --- a/junifer/api/decorators.py +++ b/junifer/api/decorators.py @@ -13,8 +13,8 @@ from ..typing import DataGrabberLike, MarkerLike, PreprocessorLike, StorageLike __all__ = [ "register_datagrabber", "register_datareader", - "register_preprocessor", "register_marker", + "register_preprocessor", "register_storage", ] diff --git a/junifer/api/functions.py b/junifer/api/functions.py index a67a22b62..d721c9dff 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -24,7 +24,7 @@ from ..typing import DataGrabberLike, MarkerLike, PreprocessorLike, StorageLike from ..utils import logger, raise_error, yaml -__all__ = ["run", "collect", "queue", "reset", "list_elements"] +__all__ = ["collect", "list_elements", "queue", "reset", "run"] def _get_datagrabber(datagrabber_config: dict) -> DataGrabberLike: diff --git a/junifer/cli/cli.py b/junifer/cli/cli.py index da0448c9e..2ac97b3e1 100644 --- a/junifer/cli/cli.py +++ b/junifer/cli/cli.py @@ -29,19 +29,19 @@ from .utils import ( __all__ = [ - "cli", - "run", - "collect", - "queue", - "wtf", - "selftest", - "reset", - "list_elements", - "setup", "afni_docker", - "fsl_docker", "ants_docker", + "cli", + "collect", "freesurfer_docker", + "fsl_docker", + "list_elements", + "queue", + "reset", + "run", + "selftest", + "setup", + "wtf", ] diff --git a/junifer/cli/parser.py b/junifer/cli/parser.py index 7d17c9246..99577c102 100644 --- a/junifer/cli/parser.py +++ b/junifer/cli/parser.py @@ -15,7 +15,7 @@ import pandas as pd from ..utils import logger, raise_error, warn_with_log, yaml -__all__ = ["parse_yaml", "parse_elements"] +__all__ = ["parse_elements", "parse_yaml"] def parse_yaml(filepath: Union[str, Path]) -> dict: # noqa: C901 diff --git a/junifer/data/_dispatch.py b/junifer/data/_dispatch.py index b504fbe6a..c8be8cdb2 100644 --- a/junifer/data/_dispatch.py +++ b/junifer/data/_dispatch.py @@ -25,11 +25,11 @@ if TYPE_CHECKING: __all__ = [ + "deregister_data", "get_data", "list_data", "load_data", "register_data", - "deregister_data", ] diff --git a/junifer/data/masks/_masks.py b/junifer/data/masks/_masks.py index e7cd2bda0..779aa3eab 100644 --- a/junifer/data/masks/_masks.py +++ b/junifer/data/masks/_masks.py @@ -35,7 +35,7 @@ if TYPE_CHECKING: from nibabel.nifti1 import Nifti1Image -__all__ = ["compute_brain_mask", "MaskRegistry"] +__all__ = ["MaskRegistry", "compute_brain_mask"] # Path to the masks diff --git a/junifer/data/template_spaces.py b/junifer/data/template_spaces.py index 76475b49a..1ec253580 100644 --- a/junifer/data/template_spaces.py +++ b/junifer/data/template_spaces.py @@ -16,7 +16,7 @@ from ..utils import logger, raise_error from .utils import closest_resolution -__all__ = ["get_xfm", "get_template"] +__all__ = ["get_template", "get_xfm"] def get_xfm( diff --git a/junifer/stats.py b/junifer/stats.py index 63678efef..61a35bdca 100644 --- a/junifer/stats.py +++ b/junifer/stats.py @@ -13,7 +13,7 @@ from scipy.stats.mstats import winsorize from .utils import logger, raise_error -__all__ = ["get_aggfunc_by_name", "count", "winsorized_mean", "select"] +__all__ = ["count", "get_aggfunc_by_name", "select", "winsorized_mean"] def get_aggfunc_by_name( diff --git a/junifer/storage/utils.py b/junifer/storage/utils.py index 21615e740..1d4bb7fe4 100644 --- a/junifer/storage/utils.py +++ b/junifer/storage/utils.py @@ -15,11 +15,11 @@ from ..utils.logging import logger, raise_error __all__ = [ - "get_dependency_version", - "process_meta", "element_to_prefix", - "store_matrix_checks", + "get_dependency_version", "matrix_to_vector", + "process_meta", + "store_matrix_checks", ] diff --git a/junifer/testing/datagrabbers.py b/junifer/testing/datagrabbers.py index e305937a1..63703a9ab 100644 --- a/junifer/testing/datagrabbers.py +++ b/junifer/testing/datagrabbers.py @@ -15,8 +15,8 @@ from ..datagrabber.base import BaseDataGrabber __all__ = [ "OasisVBMTestingDataGrabber", - "SPMAuditoryTestingDataGrabber", "PartlyCloudyTestingDataGrabber", + "SPMAuditoryTestingDataGrabber", ] diff --git a/junifer/typing/_typing.py b/junifer/typing/_typing.py index b7be28415..9e2cd47ca 100644 --- a/junifer/typing/_typing.py +++ b/junifer/typing/_typing.py @@ -19,16 +19,16 @@ if TYPE_CHECKING: __all__ = [ - "DataGrabberLike", - "PreprocessorLike", - "MarkerLike", - "StorageLike", - "PipelineComponent", - "Dependencies", "ConditionalDependencies", + "DataGrabberLike", + "DataGrabberPatterns", + "Dependencies", "ExternalDependencies", "MarkerInOutMappings", - "DataGrabberPatterns", + "MarkerLike", + "PipelineComponent", + "PreprocessorLike", + "StorageLike", ] diff --git a/junifer/utils/helpers.py b/junifer/utils/helpers.py index 229b75042..f4f67dbe0 100644 --- a/junifer/utils/helpers.py +++ b/junifer/utils/helpers.py @@ -10,7 +10,7 @@ import sys from .logging import logger, raise_error -__all__ = ["run_ext_cmd", "deep_update"] +__all__ = ["deep_update", "run_ext_cmd"] def run_ext_cmd(name: str, cmd: list[str]) -> None: diff --git a/junifer/utils/logging.py b/junifer/utils/logging.py index 0fb26bdbe..93ff8618f 100644 --- a/junifer/utils/logging.py +++ b/junifer/utils/logging.py @@ -24,9 +24,9 @@ import datalad __all__ = [ "WrapStdOut", + "configure_logging", "get_versions", "log_versions", - "configure_logging", "raise_error", "warn_with_log", ] diff --git a/junifer/utils/singleton.py b/junifer/utils/singleton.py index 1503fc9c7..e6e31baa2 100644 --- a/junifer/utils/singleton.py +++ b/junifer/utils/singleton.py @@ -8,7 +8,7 @@ from abc import ABCMeta from typing import Any, ClassVar -__all__ = ["Singleton", "ABCSingleton"] +__all__ = ["ABCSingleton", "Singleton"] class Singleton(type): -- 2.52.0 From 04b3b154a2b56677f5d71e101f9f589cfbd13482 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 10:52:44 +0100 Subject: [PATCH 04/10] style: add commentary, __all__; chore: clean up whitespace --- junifer/utils/config.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/junifer/utils/config.py b/junifer/utils/config.py index 637ff40f2..b515fa917 100644 --- a/junifer/utils/config.py +++ b/junifer/utils/config.py @@ -3,6 +3,7 @@ # Authors: Federico Raimondo # Synchon Mandal # License: AGPL + import os from typing import Any @@ -10,6 +11,9 @@ from .logging import logger from .singleton import Singleton +__all__ = ["ConfigManager", "config"] + + class ConfigManager(metaclass=Singleton): """Manage configuration parameters. @@ -54,13 +58,12 @@ class ConfigManager(metaclass=Singleton): ---------- key : str The configuration key. - - default : Any, optional + default : any, optional The default value to return if the key is not found (default None). Returns ------- - Any + any The configuration value. """ @@ -73,7 +76,7 @@ class ConfigManager(metaclass=Singleton): ---------- key : str The configuration key. - value : Any + value : any The configuration value. """ @@ -81,4 +84,5 @@ class ConfigManager(metaclass=Singleton): self._config[key] = value +# Initialize here to access from anywhere config = ConfigManager() -- 2.52.0 From 380eb567b7a2cfbca54eb31e763c1aa17c222017 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 10:53:34 +0100 Subject: [PATCH 05/10] update: add tests for ConfigManager --- junifer/utils/tests/test_config.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 junifer/utils/tests/test_config.py diff --git a/junifer/utils/tests/test_config.py b/junifer/utils/tests/test_config.py new file mode 100644 index 000000000..43c3bdf77 --- /dev/null +++ b/junifer/utils/tests/test_config.py @@ -0,0 +1,27 @@ +"""Provide tests for ConfigManager.""" + +# Authors: Federico Raimondo +# Synchon Mandal +# License: AGPL + +from junifer.utils import config +from junifer.utils.config import ConfigManager + + +def test_config_manager_singleton() -> None: + """Test that ConfigManager is a singleton.""" + config_mgr_1 = ConfigManager() + config_mgr_2 = ConfigManager() + assert id(config_mgr_1) == id(config_mgr_2) + + +def test_config_manager_get_set() -> None: + """Test config getting and setting for ConfigManager.""" + # Get non-existing with default + assert "mystery_machine" == config.get( + key="scooby", default="mystery_machine" + ) + # Set + config.set(key="scooby", value="doo") + # Get existing + assert "doo" == config.get("scooby") -- 2.52.0 From 6396868803797a1eb44621be940a6f992cb5037d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 11:06:44 +0100 Subject: [PATCH 06/10] chore: import ConfigManager to docs ref --- junifer/utils/__init__.pyi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/junifer/utils/__init__.pyi b/junifer/utils/__init__.pyi index b01470876..afcca40b3 100644 --- a/junifer/utils/__init__.pyi +++ b/junifer/utils/__init__.pyi @@ -8,10 +8,11 @@ __all__ = [ "run_ext_cmd", "deep_update", "yaml", + "ConfigManager", ] from .fs import make_executable from .logging import configure_logging, logger, raise_error, warn_with_log -from .config import config +from .config import config, ConfigManager from .helpers import run_ext_cmd, deep_update from ._yaml import yaml -- 2.52.0 From 797b427c6e428e78ec20f174c2d100e6933cb4e6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 14:41:55 +0100 Subject: [PATCH 07/10] chore: rename config.py to _config.py to avoid possible name mangle conflicts; update: add delete method to ConfigManager --- junifer/utils/__init__.pyi | 2 +- junifer/utils/{config.py => _config.py} | 58 +++++++++++++++++-------- junifer/utils/tests/test_config.py | 2 +- 3 files changed, 42 insertions(+), 20 deletions(-) rename junifer/utils/{config.py => _config.py} (55%) diff --git a/junifer/utils/__init__.pyi b/junifer/utils/__init__.pyi index afcca40b3..c840252d3 100644 --- a/junifer/utils/__init__.pyi +++ b/junifer/utils/__init__.pyi @@ -13,6 +13,6 @@ __all__ = [ from .fs import make_executable from .logging import configure_logging, logger, raise_error, warn_with_log -from .config import config, ConfigManager +from ._config import config, ConfigManager from .helpers import run_ext_cmd, deep_update from ._yaml import yaml diff --git a/junifer/utils/config.py b/junifer/utils/_config.py similarity index 55% rename from junifer/utils/config.py rename to junifer/utils/_config.py index b515fa917..b2b8e4643 100644 --- a/junifer/utils/config.py +++ b/junifer/utils/_config.py @@ -5,8 +5,9 @@ # License: AGPL import os -from typing import Any +from typing import Optional +from ..typing import ConfigVal from .logging import logger from .singleton import Singleton @@ -27,16 +28,21 @@ class ConfigManager(metaclass=Singleton): def __init__(self) -> None: """Initialize the class.""" self._config = {} + # Initial setup from process env + self._reload() + + def _reload(self) -> None: + """Reload env vars.""" for t_var in os.environ: if t_var.startswith("JUNIFER_"): - varname = t_var.replace("JUNIFER_", "") - varname = varname.lower() - varname = varname.replace("_", ".") + # Set correct type var_value = os.environ[t_var] + # bool if var_value.lower() == "true": var_value = True elif var_value.lower() == "false": var_value = False + # numeric else: try: var_value = int(var_value) @@ -45,43 +51,59 @@ class ConfigManager(metaclass=Singleton): var_value = float(var_value) except ValueError: pass - logger.debug( - f"Setting {varname} from environment to " - f"{var_value} (type: {type(var_value)})" + # Set value + var_name = ( + t_var.replace("JUNIFER_", "").lower().replace("_", ".") ) - self._config[varname] = var_value + logger.debug( + f"Setting `{var_name}` from environment to " + f"`{var_value}` (type: {type(var_value)})" + ) + self._config[var_name] = var_value - def get(self, key: str, default: Any = None) -> Any: + def get(self, key: str, default: Optional[ConfigVal] = None) -> ConfigVal: """Get configuration parameter. Parameters ---------- key : str - The configuration key. - default : any, optional + The configuration key to get. + default : bool or int or float or None, optional The default value to return if the key is not found (default None). Returns ------- - any + bool or int or float The configuration value. """ return self._config.get(key, default) - def set(self, key: str, value: Any) -> None: + def set(self, key: str, val: ConfigVal) -> None: """Set configuration parameter. Parameters ---------- key : str - The configuration key. - value : any - The configuration value. + The configuration key to set. + val : bool or int or float + The value to set ``key`` to. """ - logger.debug(f"Setting {key} to {value}") - self._config[key] = value + logger.debug(f"Setting `{key}` to `{val}` (type: {type(val)})") + self._config[key] = val + + def delete(self, key: str) -> None: + """Delete configuration parameter. + + Parameters + ---------- + key : str + The configuration key to delete. + + """ + logger.debug(f"Deleting `{key}` from config") + _ = self._config.pop(key) # Initialize here to access from anywhere diff --git a/junifer/utils/tests/test_config.py b/junifer/utils/tests/test_config.py index 43c3bdf77..6ab3b26be 100644 --- a/junifer/utils/tests/test_config.py +++ b/junifer/utils/tests/test_config.py @@ -5,7 +5,7 @@ # License: AGPL from junifer.utils import config -from junifer.utils.config import ConfigManager +from junifer.utils._config import ConfigManager def test_config_manager_singleton() -> None: -- 2.52.0 From 56b36095db4696445cdf0c06c48fbc519e0a0e45 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 14:43:02 +0100 Subject: [PATCH 08/10] chore: add ConfigVal type hint --- junifer/typing/__init__.pyi | 2 ++ junifer/typing/_typing.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/junifer/typing/__init__.pyi b/junifer/typing/__init__.pyi index 43ab84c02..bfe026fda 100644 --- a/junifer/typing/__init__.pyi +++ b/junifer/typing/__init__.pyi @@ -9,6 +9,7 @@ __all__ = [ "ExternalDependencies", "MarkerInOutMappings", "DataGrabberPatterns", + "ConfigVal", ] from ._typing import ( @@ -22,4 +23,5 @@ from ._typing import ( ExternalDependencies, MarkerInOutMappings, DataGrabberPatterns, + ConfigVal, ) diff --git a/junifer/typing/_typing.py b/junifer/typing/_typing.py index 9e2cd47ca..d7275b597 100644 --- a/junifer/typing/_typing.py +++ b/junifer/typing/_typing.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: __all__ = [ "ConditionalDependencies", + "ConfigVal", "DataGrabberLike", "DataGrabberPatterns", "Dependencies", @@ -60,3 +61,4 @@ MarkerInOutMappings = MutableMapping[str, MutableMapping[str, str]] DataGrabberPatterns = dict[ str, Union[dict[str, str], Sequence[dict[str, str]]] ] +ConfigVal = Union[bool, int, float] -- 2.52.0 From 4388cb0071d8a6c7bab5f6547aaf25f995acbda6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 14:43:20 +0100 Subject: [PATCH 09/10] update: improve tests for ConfigManager --- junifer/utils/tests/test_config.py | 46 +++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/junifer/utils/tests/test_config.py b/junifer/utils/tests/test_config.py index 6ab3b26be..209acfa64 100644 --- a/junifer/utils/tests/test_config.py +++ b/junifer/utils/tests/test_config.py @@ -4,6 +4,11 @@ # Synchon Mandal # License: AGPL +import os + +import pytest + +from junifer.typing import ConfigVal from junifer.utils import config from junifer.utils._config import ConfigManager @@ -15,13 +20,40 @@ def test_config_manager_singleton() -> None: assert id(config_mgr_1) == id(config_mgr_2) -def test_config_manager_get_set() -> None: - """Test config getting and setting for ConfigManager.""" +def test_config_manager() -> None: + """Test config operations for ConfigManager.""" # Get non-existing with default - assert "mystery_machine" == config.get( - key="scooby", default="mystery_machine" - ) + assert config.get(key="scooby") is None # Set - config.set(key="scooby", value="doo") + config.set(key="scooby", val=True) # Get existing - assert "doo" == config.get("scooby") + assert config.get("scooby") + # Delete + config.delete("scooby") + # Get non-existing with default + assert config.get(key="scooby") is None + + +@pytest.mark.parametrize( + "val, expected_val", + [("TRUE", True), ("FALSE", False), ("1", 1), ("0.0", 0.0)], +) +def test_config_manager_env_reload(val: str, expected_val: ConfigVal) -> None: + """Test config parsing from env reload. + + Parameters + ---------- + val : str + The parametrized values. + expected_val : bool or int or float + The parametrized expected value. + + """ + # Set env var + os.environ["JUNIFER_TESTME"] = val + # Check + config._reload() + assert config.get("testme") == expected_val + # Cleanup + del os.environ["JUNIFER_TESTME"] + config.delete("testme") -- 2.52.0 From 8b7b59978da001db953781791b10588d058a4df5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 15:49:07 +0100 Subject: [PATCH 10/10] chore: add changelog 401.feature --- docs/changes/newsfragments/401.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/newsfragments/401.feature diff --git a/docs/changes/newsfragments/401.feature b/docs/changes/newsfragments/401.feature new file mode 100644 index 000000000..24e89567e --- /dev/null +++ b/docs/changes/newsfragments/401.feature @@ -0,0 +1 @@ +Introduce :class:`junifer.utils.ConfigManager` singleton class to manage global configuration by `Fede Raimondo`_ -- 2.52.0