From c55b071912734d9b1a0a692e3962d0ab5a89fd0e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 12:19:21 +0200 Subject: [PATCH 01/17] update: replace pyyaml with ruamel.yaml in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index db1ff6a46..f05b5b7c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [ "nibabel>=3.2.0,<4.1", "nilearn>=0.9.0,<=0.10.0", "sqlalchemy>=1.4.27,<= 1.5.0", - "pyyaml>=5.1.2,<7.0", + "ruamel.yaml>=0.17,<0.18", "importlib_metadata; python_version < '3.10'", "h5py>=3.8.0,<3.9", ] -- 2.52.0 From eb37f8ed143a06502af2f34d6f3739e5a6f9f8e8 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 12:27:49 +0200 Subject: [PATCH 02/17] update: replace pyyaml with ruamel.yaml in conda-env.yml --- conda-env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-env.yml b/conda-env.yml index ee5fc5fac..854baa68f 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -10,7 +10,7 @@ dependencies: - nibabel>=3.2.0,<4.1 - nilearn>=0.9.0,<=0.10.0 - sqlalchemy>=1.4.27,<= 1.5.0 - - pyyaml>=5.1.2,<7.0 + - ruamel.yaml=0.17.* - h5py=3.8.* - seaborn=0.11.* - Sphinx=5.3.* -- 2.52.0 From 51ca5a013bfc0263c5cc3f40c6bb5b5152f268d7 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 12:28:23 +0200 Subject: [PATCH 03/17] docs: replace pyyaml with ruamel.yaml in docs/installation.rst --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 645d52220..5a588f827 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -16,7 +16,7 @@ junifer is compatible with `Python`_ >= 3.8 and requires the following packages: * ``nibabel>=3.2.0,<4.1`` * ``nilearn>=0.9.0,<=0.10.0`` * ``sqlalchemy>=1.4.27,<= 1.5.0`` -* ``pyyaml>=5.1.2,<7.0`` +* ``ruamel.yaml>=0.17,<0.18`` * ``h5py>=3.8.0,<3.9`` Depending on the installation method, these packages might be installed -- 2.52.0 From beb20ba2788e926d4be37d54df32f4dc3c1cca92 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 13:51:28 +0200 Subject: [PATCH 04/17] update: use ruamel.yaml in api/utils.py --- junifer/api/utils.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/junifer/api/utils.py b/junifer/api/utils.py index 0e2ea1001..b4d8ba13b 100644 --- a/junifer/api/utils.py +++ b/junifer/api/utils.py @@ -9,10 +9,19 @@ import re from importlib.metadata import distribution from typing import Dict +from ruamel.yaml import YAML + from .._version import __version__ from ..utils.logging import get_versions +# Configure YAML class once for further use +yaml = YAML() +yaml.default_flow_style = False +yaml.allow_unicode = True +yaml.indent(mapping=2, sequence=4, offset=2) + + def _get_junifer_version() -> Dict[str, str]: """Get junifer version information. @@ -71,16 +80,13 @@ def _get_dependency_information(long_: bool) -> Dict[str, str]: # Get dependencies for junifer dist = distribution("junifer") # Compile regex pattern - re_pattern = re.compile("[a-z-]+") + re_pattern = re.compile("[a-z-_.]+") for pkg_with_version in dist.requires: # type: ignore # Perform regex search matches = re.findall(pattern=re_pattern, string=pkg_with_version) - # Fix issue with PyYAML name registration - if matches[0] == "pyyaml": - key = "yaml" - else: - key = matches[0] + # Extract package name + key = matches[0] if key in dependency_versions.keys(): # Check if pkg part of optional dependencies -- 2.52.0 From 40a8e7541c1973e75ab0cf207cc0815578fd816c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 13:52:13 +0200 Subject: [PATCH 05/17] update: adapt utils/logging.py for ruamel.yaml --- junifer/utils/logging.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/junifer/utils/logging.py b/junifer/utils/logging.py index 5e06dcd48..922b248a8 100644 --- a/junifer/utils/logging.py +++ b/junifer/utils/logging.py @@ -90,7 +90,9 @@ def get_versions() -> Dict: """ module_versions = {} for name, module in sys.modules.items(): - if "." in name: + # Bypassing sub-modules of packages and + # allowing ruamel.yaml + if "." in name and name != "ruamel.yaml": continue if name in ["_curses"]: continue -- 2.52.0 From dd3cea051b544a29c43fef3bc085a282a59dc58e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 13:53:11 +0200 Subject: [PATCH 06/17] update: use ruamel.yaml in api/cli.py --- junifer/api/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/api/cli.py b/junifer/api/cli.py index c24482b8c..d40cc9c68 100644 --- a/junifer/api/cli.py +++ b/junifer/api/cli.py @@ -11,7 +11,6 @@ from pathlib import Path from typing import Dict, List, Union import click -import yaml from ..utils.logging import ( configure_logging, @@ -29,6 +28,7 @@ from .utils import ( _get_junifer_version, _get_python_information, _get_system_information, + yaml, ) @@ -275,7 +275,7 @@ def wtf(long_: bool) -> None: "system": _get_system_information(), "environment": _get_environment_information(long_=long_), } - click.echo(yaml.dump(report, sort_keys=False)) + click.echo(yaml.dump(report, stream=sys.stdout)) @cli.command() -- 2.52.0 From 65fbbe9381cdee9c28f7d5a925ee555a11d11c87 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 13:56:58 +0200 Subject: [PATCH 07/17] update: use ruamel.yaml in api/functions.py --- junifer/api/functions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 6c1a2a7cc..32cc2f893 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -12,8 +12,6 @@ import typing from pathlib import Path from typing import Dict, List, Optional, Tuple, Union -import yaml - from ..datagrabber.base import BaseDataGrabber from ..markers.base import BaseMarker from ..markers.collection import MarkerCollection @@ -22,6 +20,7 @@ from ..preprocess.base import BasePreprocessor from ..storage.base import BaseFeatureStorage from ..utils import logger, raise_error from ..utils.fs import make_executable +from .utils import yaml def _get_datagrabber(datagrabber_config: Dict) -> BaseDataGrabber: @@ -270,8 +269,7 @@ def queue( yaml_config = jobdir / "config.yaml" logger.info(f"Writing YAML config to {str(yaml_config.absolute())}") - with open(yaml_config, "w") as f: - f.write(yaml.dump(config)) + yaml.dump(config, stream=yaml_config) # Get list of elements if elements is None: -- 2.52.0 From c45d3a9300798fb32ed8ae47c929d0671d4ea0c9 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 13:57:20 +0200 Subject: [PATCH 08/17] update: use ruamel.yaml in api/parser.py --- junifer/api/parser.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/junifer/api/parser.py b/junifer/api/parser.py index 8e96b365e..fe5301439 100644 --- a/junifer/api/parser.py +++ b/junifer/api/parser.py @@ -10,9 +10,8 @@ import sys from pathlib import Path from typing import Dict, Union -import yaml - from ..utils.logging import logger, raise_error +from .utils import yaml def parse_yaml(filepath: Union[str, Path]) -> Dict: @@ -38,8 +37,7 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict: if not filepath.exists(): raise_error(f"File does not exist: {str(filepath.absolute())}") # Filepath reading - with open(filepath, "r") as f: - contents = yaml.safe_load(f) + contents = yaml.load(filepath) if "elements" in contents: if contents["elements"] is None: raise_error( -- 2.52.0 From 1ec7715341f44a04c1f42f7d1c3462e8aa748680 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 13:58:15 +0200 Subject: [PATCH 09/17] update: fix tests for api/utils.py --- junifer/api/tests/test_api_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/api/tests/test_api_utils.py b/junifer/api/tests/test_api_utils.py index 33dd4db42..57b67624a 100644 --- a/junifer/api/tests/test_api_utils.py +++ b/junifer/api/tests/test_api_utils.py @@ -40,7 +40,7 @@ def test_get_dependency_information_short() -> None: "nibabel", "nilearn", "sqlalchemy", - "yaml", + "ruamel.yaml", ] @@ -58,7 +58,7 @@ def test_get_dependency_information_long() -> None: "nibabel", "nilearn", "sqlalchemy", - "yaml", + "ruamel.yaml", ]: assert key in dependency_information_keys -- 2.52.0 From 6b148c2d39008188bc0fe0424e3c770b2de5e383 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 13:58:49 +0200 Subject: [PATCH 10/17] update: fix tests for api/cli.py --- junifer/api/tests/test_cli.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/junifer/api/tests/test_cli.py b/junifer/api/tests/test_cli.py index 8e90dc765..aa01285e4 100644 --- a/junifer/api/tests/test_cli.py +++ b/junifer/api/tests/test_cli.py @@ -8,12 +8,18 @@ from pathlib import Path from typing import Tuple import pytest -import yaml from click.testing import CliRunner +from ruamel.yaml import YAML from junifer.api.cli import collect, run, selftest, wtf +# Configure YAML class +yaml = YAML() +yaml.default_flow_style = False +yaml.allow_unicode = True +yaml.indent(mapping=2, sequence=4, offset=2) + # Create click test runner runner = CliRunner() @@ -33,19 +39,17 @@ def test_run_and_collect_commands( # Get test config infile = Path(__file__).parent / "data" / "gmd_mean.yaml" # Read test config - with open(infile, mode="r") as f: - contents = yaml.safe_load(f) + contents = yaml.load(infile) # Working directory workdir = tmp_path / "workdir" - contents["workdir"] = str(workdir.absolute()) + contents["workdir"] = str(workdir.resolve()) # Output directory outdir = tmp_path / "outdir" # Storage - contents["storage"]["uri"] = str(outdir.absolute()) + contents["storage"]["uri"] = str(outdir.resolve()) # Write new test config outfile = tmp_path / "in.yaml" - with open(outfile, mode="w") as f: - yaml.dump(contents, f) + yaml.dump(contents, stream=outfile) # Run command arguments run_args = [ str(outfile.absolute()), -- 2.52.0 From 77a29422c04e40b24ace65ee0c4f6b0db4aadc9d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 13:59:07 +0200 Subject: [PATCH 11/17] update: fix tests for api/functions.py --- junifer/api/tests/test_functions.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 8beed2243..02705f1fd 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import List, Tuple, Union import pytest -import yaml +from ruamel.yaml import YAML import junifer.testing.registry # noqa: F401 from junifer.api.functions import collect, queue, run @@ -19,6 +19,12 @@ from junifer.datagrabber.base import BaseDataGrabber from junifer.pipeline.registry import build +# Configure YAML class +yaml = YAML() +yaml.default_flow_style = False +yaml.allow_unicode = True +yaml.indent(mapping=2, sequence=4, offset=2) + # Define datagrabber datagrabber = { "kind": "OasisVBMTestingDatagrabber", @@ -264,8 +270,7 @@ def test_queue_correct_yaml_config( generated_config_yaml_path = Path( tmp_path / "junifer_jobs" / "yaml_config_gen_check" / "config.yaml" ) - with open(generated_config_yaml_path, "r") as f: - yaml_config = yaml.unsafe_load(f) + yaml_config = yaml.load(generated_config_yaml_path) # Check for correct YAML config generation assert all( key in yaml_config.keys() -- 2.52.0 From 72227bcbcea28cbed45043d7cb79d3ec25844beb Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 13:59:20 +0200 Subject: [PATCH 12/17] chore: cosmetic update in api/decorators.py --- junifer/api/decorators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/junifer/api/decorators.py b/junifer/api/decorators.py index eb2f0bda1..87ea51f89 100644 --- a/junifer/api/decorators.py +++ b/junifer/api/decorators.py @@ -4,6 +4,7 @@ # Leonard Sasse # Synchon Mandal # License: AGPL + from typing import Type from ..pipeline.registry import register -- 2.52.0 From dc044ce398fc8effedd362dff4ab076db45e6c73 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 13:59:37 +0200 Subject: [PATCH 13/17] docs: fix conda env instruction in docs/faq.rst --- docs/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index d933e84a7..0a7a7a5dc 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -23,7 +23,7 @@ The following steps are specific to VSCode and you can choose to go with it: .. code-block:: bash - conda env create -n -f conda-env.yml python=3.10 + conda env create -n -f conda-env.yml conda activate The ``conda-env.yml`` can be found at the root of the repository. -- 2.52.0 From 77bc71cbee49da4fc67206b29f4f213e55d1b35c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 24 Apr 2023 15:21:05 +0200 Subject: [PATCH 14/17] update: fix tests for storage/utils.py --- junifer/storage/tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/tests/test_utils.py b/junifer/storage/tests/test_utils.py index dd7bab3bd..75e632533 100644 --- a/junifer/storage/tests/test_utils.py +++ b/junifer/storage/tests/test_utils.py @@ -29,7 +29,7 @@ from junifer.storage.utils import ( ("nibabel", "4.1"), ("nilearn", "0.10.0"), ("sqlalchemy", "1.5.0"), - ("pyyaml", "7.0"), + ("ruamel.yaml", "0.18.0"), ], ) def test_get_dependency_version(dependency: str, max_version: str) -> None: -- 2.52.0 From 4d20e20c3ce6e60af8a466dfac037b61375be9ca Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 2 May 2023 17:35:32 +0200 Subject: [PATCH 15/17] update: fix and improve tests for api/functions.py --- junifer/api/tests/test_functions.py | 37 +++++++++++++++++------------ 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 02705f1fd..d514fbc82 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -237,6 +237,7 @@ def test_run_and_collect(tmp_path: Path) -> None: def test_queue_correct_yaml_config( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: """Test proper YAML config generation for queueing. @@ -246,26 +247,32 @@ def test_queue_correct_yaml_config( The path to the test directory. monkeypatch : pytest.MonkeyPatch The monkeypatch object. + caplog : pytest.LogCaptureFixture + The logcapturefixture object. """ with monkeypatch.context() as m: m.chdir(tmp_path) - queue( - config={ - "with": "junifer.testing.registry", - "workdir": str(Path(tmp_path).resolve()), - "datagrabber": datagrabber, - "markers": markers, - "storage": storage, - "env": { - "kind": "conda", - "name": "junifer", + with caplog.at_level(logging.INFO): + queue( + config={ + "with": "junifer.testing.registry", + "workdir": str(tmp_path.resolve()), + "datagrabber": datagrabber, + "markers": markers, + "storage": {"kind": "SQLiteFeatureStorage"}, + "env": { + "kind": "conda", + "name": "junifer", + }, + "mem": "8G", }, - "mem": "8G", - }, - kind="HTCondor", - jobname="yaml_config_gen_check", - ) + kind="HTCondor", + jobname="yaml_config_gen_check", + ) + assert "Creating job in" in caplog.text + assert "Writing YAML config to" in caplog.text + assert "Queue done" in caplog.text generated_config_yaml_path = Path( tmp_path / "junifer_jobs" / "yaml_config_gen_check" / "config.yaml" -- 2.52.0 From 2e943eba119a1fe773cde03e71490577e376152f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 2 May 2023 17:38:30 +0200 Subject: [PATCH 16/17] docs: Pypi to PyPI in maintaining.rst --- docs/maintaining.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/maintaining.rst b/docs/maintaining.rst index d9a29e68a..2c883005c 100644 --- a/docs/maintaining.rst +++ b/docs/maintaining.rst @@ -16,7 +16,7 @@ This plugin reads the latest tagged version from git and automatically increments the *MICRO* segment and appends *devN*. This is considered a pre-release. -The CI scripts will publish every tag with the format *v.X.Y.Z* to Pypi as +The CI scripts will publish every tag with the format *v.X.Y.Z* to PyPI as version "X.Y.Z". Additionally, for every push to main, it will be published as pre-release to PyPI. -- 2.52.0 From 6d5eca162c8139d65434c53e72a028e4daf645a5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 2 May 2023 17:41:48 +0200 Subject: [PATCH 17/17] chore: add changelogs 223.bugfix and 223.enh --- docs/changes/newsfragments/223.bugfix | 1 + docs/changes/newsfragments/223.enh | 1 + 2 files changed, 2 insertions(+) create mode 100644 docs/changes/newsfragments/223.bugfix create mode 100644 docs/changes/newsfragments/223.enh diff --git a/docs/changes/newsfragments/223.bugfix b/docs/changes/newsfragments/223.bugfix new file mode 100644 index 000000000..a709cb2e0 --- /dev/null +++ b/docs/changes/newsfragments/223.bugfix @@ -0,0 +1 @@ +Enable YAML 1.2 support and allow multiline strings in YAML which would not work earlier by `Synchon Mandal`_ \ No newline at end of file diff --git a/docs/changes/newsfragments/223.enh b/docs/changes/newsfragments/223.enh new file mode 100644 index 000000000..54e3befe5 --- /dev/null +++ b/docs/changes/newsfragments/223.enh @@ -0,0 +1 @@ +Use ``ruamel.yaml`` instead of ``pyyaml`` as YAML I/O library by `Synchon Mandal`_ \ No newline at end of file -- 2.52.0