From 14e4a9d55e1c85ee89075ad8d8a19f8c8b804909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Fri, 21 Feb 2025 14:40:21 +0100 Subject: [PATCH 01/25] Add an about section, linked in shortcuts menu This moves the contributing page and adds an empty copyright page under the "about" page bundle. These pages are listed in the shortcuts menu (theme-specific), and the menu heading (default: "More") is disabled. Because these pages are kind of "meta" information about the website, it felt appropriate to visually distinguish them from the "regular" content. Technically, this is accomplished according to (1): - a headless branch bundle "about" is created - pages inside that bundle are added to the (theme-default) "shortcuts" menu This seems to be the easiest way to do this without declaring decdicated menus in the config - but this could still be done if we need to use shortcuts for actually having shortcuts. Or we could turn the current "about" pages into an actual chapter. (1) https://mcshelby.github.io/hugo-theme-relearn/configuration/sidebar/menus/index.html#displaying-pages-exclusively-in-a-hugo-menu --- content/about/_index.md | 8 ++++++++ content/{contributing/_index.md => about/contributing.md} | 3 +-- content/about/copyright.md | 4 ++++ hugo.toml | 3 ++- 4 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 content/about/_index.md rename content/{contributing/_index.md => about/contributing.md} (96%) create mode 100644 content/about/copyright.md diff --git a/content/about/_index.md b/content/about/_index.md new file mode 100644 index 0000000..50cdedb --- /dev/null +++ b/content/about/_index.md @@ -0,0 +1,8 @@ ++++ +title = 'Extras' + +[_build] + list = 'never' + publishResources = true + render = 'always' ++++ diff --git a/content/contributing/_index.md b/content/about/contributing.md similarity index 96% rename from content/contributing/_index.md rename to content/about/contributing.md index a77da80..d738514 100644 --- a/content/contributing/_index.md +++ b/content/about/contributing.md @@ -1,7 +1,6 @@ +++ title = "Contributing" -type = "chapter" -weight = 10 +menu = "shortcuts" +++ This documentation website is built with the [hugo](https://gohugo.io/) static website generator. diff --git a/content/about/copyright.md b/content/about/copyright.md new file mode 100644 index 0000000..abab14c --- /dev/null +++ b/content/about/copyright.md @@ -0,0 +1,4 @@ ++++ +title = "Copyright" +menu = "shortcuts" ++++ diff --git a/hugo.toml b/hugo.toml index cf4def8..2fe0b4e 100644 --- a/hugo.toml +++ b/hugo.toml @@ -4,4 +4,5 @@ title = 'SFB1451 RDM docs' theme= 'hugo-theme-relearn' [params] - themeVariant = 'zen-light' \ No newline at end of file + disableShortcutsTitle = true + themeVariant = 'zen-light' -- 2.52.0 From 2627f3a911f37ddf181847412598c68bab6c1cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Mon, 3 Mar 2025 14:30:14 +0100 Subject: [PATCH 02/25] Update copyright / license statement This keeps the CC-BY-SA license used by the previous iteration of the docs, as well as the 2021 copyright notice for Alex and Laura. We discussed this issue in today's lab meeting, and the following is my interpretation of the consensus. The 2022-2025 copyright gets assigned to myself as the main contributor in that time period. We see this mainly as the contact person for (potential) relicensing questions and opt for simplicity. We care mostly about the license, less so about the individual contributors (see git shortlog -sne master). --- content/about/copyright.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/content/about/copyright.md b/content/about/copyright.md index abab14c..41ab874 100644 --- a/content/about/copyright.md +++ b/content/about/copyright.md @@ -2,3 +2,11 @@ title = "Copyright" menu = "shortcuts" +++ + +© 2021 by Alex Waite, Laura Waite +© 2022-2025 by Michał Szczepanik + +The content of this SFB1451 RDM documentation website +is licensed under +[Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/). + -- 2.52.0 From 7af5ec632ab0555d1b8971cc6eb45231903a0b3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Mon, 28 Apr 2025 21:44:11 +0200 Subject: [PATCH 03/25] Add funding acknowledgement --- content/about/acknowledgements.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 content/about/acknowledgements.md diff --git a/content/about/acknowledgements.md b/content/about/acknowledgements.md new file mode 100644 index 0000000..dda7ce2 --- /dev/null +++ b/content/about/acknowledgements.md @@ -0,0 +1,11 @@ ++++ +date = '2025-04-28T21:34:35+02:00' +draft = true +title = 'Acknowledgements' +menu = 'shortcuts' ++++ + +This documentation was developed with funding from the Deutsche +Forschungsgemeinschaft (DFG, German Research Foundation) under grant +SFB 1451 ([431549029](https://gepris.dfg.de/gepris/projekt/431549029), +INF project). -- 2.52.0 From a8c9429b2fd633c18b2cd0964c447997672c297a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Mon, 28 Apr 2025 22:47:43 +0200 Subject: [PATCH 04/25] Pin Hugo version in GitLab CI It looks like the build broke on the latest hugo version (0.147.0), due to what might be theme/hugo mismatch. Instead of figuring it out now, we go to the latest version known to be working (0.145.0). --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b25a25c..80895ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ default: image: - name: hugomods/hugo:git + name: hugomods/hugo:git-0.145.0 entrypoint: [""] variables: -- 2.52.0 From 77f915b7ef3eef3737bf4d83b6b0c4d7b448401f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Wed, 18 Jun 2025 13:02:36 +0200 Subject: [PATCH 05/25] Add an example of ls-file-collection In the page about datalad-based deposition to the catalog, the solution based on datalad status is replaced by a more "modern" one which uses DataLad-next's ls-file collection. A script is included directly, as a Python code block. The catppuccin-latte color scheme is used because it fits pretty nicely with the website's light theme -- much more so than Hugo's default monokai (dark). It would appear that the code block style only kicks in when the language is specified. See: https://gohugo.io/quick-reference/syntax-highlighting-styles/ Closes #4 --- content/catalog/datalad-based-deposition.md | 97 ++++++++++++++++----- 1 file changed, 77 insertions(+), 20 deletions(-) diff --git a/content/catalog/datalad-based-deposition.md b/content/catalog/datalad-based-deposition.md index 6126477..545b09b 100644 --- a/content/catalog/datalad-based-deposition.md +++ b/content/catalog/datalad-based-deposition.md @@ -113,31 +113,88 @@ steps 1 through 3. ### 2. \[Optional\] Create the `files` table -The [files -table]({{% relref "register-data#table-files" %}}) is -an optional addition to the tabby record and lists one or more files -that form part of the DataLad dataset. Such a file listing can be easily -generated based on the output of `datalad status` - see the -[status2tabby.py](https://github.com/sfb1451/tabby-utils/blob/main/status2tabby.py) -script, which uses the `datalad-next` extension, for an example -implementation that you can also use directly: +The [files table]({{% relref "register-data#table-files" %}}) is an +optional addition to the tabby record and lists one or more files that +form part of the DataLad dataset. Such a file listing can be easily +generated based on the output of +[ls_file_collection](https://docs.datalad.org/projects/next/en/latest/generated/datalad.api.ls_file_collection.html) +provided by Datalad-next. Below, we present an example implementation, +which you can also use directly. -``` None -usage: status2tabby.py [-h] dataset outfile +Note that files which are present will be checksummed; for annexed +files which are not present locally the checksum will be taken from +the annex key. This assumes that DataLad's default backend (MD5E) is +used; adjust the code for different checksums if necessary. -positional arguments: - dataset Dataset for which files will be listed - outfile Name of tsv file to write -options: - -h, --help show this help message and exit +```python {lineNos=true, style=catppuccin-latte} +"""Generate a TSV file listing the files in a dataset + +Usage: + + python this_file.py dataset outfile + +""" + +from argparse import ArgumentParser +from csv import DictWriter +from pathlib import Path +import re +from datalad.api import ls_file_collection + + +def match_md5(key): + """Report MD5 checksum contained in annex key + + Assumes MD5E backend (DataLad's default). + https://git-annex.branchable.com/internals/key_format/ + + """ + m = re.match(r"MD5E-s\d+--([a-f0-9]{32})", key) + return m.group(1) if m is not None else None + + +def transform_result(res): + """Transform results to get required information + + Processes the outputs of ls_file_collection to get relative path, + size in bytes, and md5 checksum. For annex keys which are not + present, the checksum will be taken from the annex key, if + possible. Keys match the sfb1451 tabby specification. + + """ + path = res["item"].relative_to(res["collection"]).as_posix() + size = res["annexsize"] if res["type"] == "annexed file" else res["size"] + checksum = res.get("hash-md5") + + if checksum is None and res["type"] == "annexed file": + checksum = match_md5(res["annexkey"]) + + return {"path[POSIX]": path, "size[bytes]": size, "checksum[md5]": checksum} + + +parser = ArgumentParser() +parser.add_argument("dataset", type=Path, help="Dataset for which files will be listed") +parser.add_argument("outfile", type=Path, help="Name of the tsv file to write") +args = parser.parse_args() + +fc = ls_file_collection( + "annexworktree", + args.dataset, + hash="md5", + result_renderer="disabled", + result_xfm=transform_result, + return_type="generator", +) + +with args.outfile.open("w", encoding="utf-8", newline="") as tsvfile: + fieldnames = ["path[POSIX]", "size[bytes]", "checksum[md5]"] + writer = DictWriter(tsvfile, delimiter="\t", fieldnames=fieldnames) + writer.writeheader() + for row in fc: + writer.writerow(row) ``` -> [!INFO] -> Another (more performant) alternative is to use the -> [ls_file_collection](http://docs.datalad.org/projects/next/en/latest/generated/datalad.api.ls_file_collection.html) -> function from DataLad-next, and transform its output. - ### 3. Publish the metadata For manual deposition, copy and send all tabby records to the INF -- 2.52.0 From 51980b122be9b66cc7815a2cc5245f913566cf0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Wed, 18 Jun 2025 19:27:42 +0200 Subject: [PATCH 06/25] Add a walkthrough for forgejo with encryption --- content/walkthroughs/forgejo-encryption.md | 191 +++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 content/walkthroughs/forgejo-encryption.md diff --git a/content/walkthroughs/forgejo-encryption.md b/content/walkthroughs/forgejo-encryption.md new file mode 100644 index 0000000..f5913de --- /dev/null +++ b/content/walkthroughs/forgejo-encryption.md @@ -0,0 +1,191 @@ ++++ +date = '2025-06-17T18:45:00+02:00' +draft = false +title = 'Forgejo with encryption' ++++ + +This walkthrough demonstrates how to encrypt annexed file content in Forgejo, +a bare Git repository, or another Git hosting platform with annex support. + +## Introduction + +When storing DataLad datasets on third party infrastructure, Git +remotes and git-annex special remotes are often configured pointing to +different services. However, some repository hosting services +(e.g. G-node GIN, or some instances of Forgejo), as well as bare Git +repositories[^1] on systems with git-annex installed, support storing +annex objects alongside the Git repository, which makes them very +convenient in terms of configuration and usage. Throughout the +article, we will refer to remotes supporting both Git and annex +content as "git+annex". For background, see DataLad handbook's chapter +[Beyond shared +infrastructure](https://handbook.datalad.org/en/latest/basics/101-138-sharethirdparty.html). + +Some of the SFB1451 datasets may require using data encryption at rest +to guarantee data privacy or access control. Git-annex [supports +encryption](https://git-annex.branchable.com/encryption/) and it can +be very easy to set up. However, encryption is configured for *special +remotes* and as such it would not apply to such "git+annex" remotes[^2]. + +This has been addressed in [git-annex v.10.20250520](https://git-annex.branchable.com/news/version_10.20250520/) +by the addition of the [mask special remote](https://git-annex.branchable.com/special_remotes/mask/). + +With the mask special remote, setting up encrypted storage for +git+annex comes down to the following steps: + +- set up a Git remote, +- set up a mask special remote with encryption enabled, +- make sure that annexed contents are pushed only to the latter. + +The model use case for this workflow is publishing datasets on Forgejo +instances with git-annex support. [Forgejo](https://forgejo.org/) is +a "self-hosted lightweight software forge", which can be used to set +up and maintain a platform similar to GitHub or GitLab. +[Forgejo-aneksajo](https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo) +is a lightly modified version of Forgejo which adds git-annex support. +While no public instances are readily available, and the SFB1451 does +not (currently) maintain its own instance, it is nevertheless very +relevant for the SFB1451 community because Forgejo-aneksajo can be +deployed by individual groups (see [Collaborative infrastructure for a +lab: Forgejo](https://blog.datalad.org/posts/lab-infra-forgejo/) for +more background information). + +## Prerequisites + +- git-annex v.10.20250520 or newer +- gpg, and a generated private/public key +- DataLad-next (optional but highly recommended) + +## Walkthrough + +### Initial set-up and publishing + +Publishing a dataset requires having a dataset in the first place. +In this example we create a dataset with one file in Git (README) and one file annexed (an image). + +``` +datalad create foo +cd foo +echo "Hello world" > README +datalad download https://images.pexels.com/photos/21533358/pexels-photo-21533358.jpeg +datalad save --to-git -m "Add a readme" README +datalad save -m "Add a photo of a penguin" pexels-photo-21533358.jpeg +``` + +Add a Git remote pointing to a valid address -- for example, an empty Forgejo-aneksajo repository, +or (for easier testing) a local bare git repository. + +{{< tabs >}} +{{% tab title="forgejo" %}} +``` +git remote add origin https://hub.example.com/user/dataset.git +``` +{{% /tab %}} +{{% tab title="local" %}} +``` +git init --bare /tmp/foo.git +git remote add origin /tmp/foo.git +``` +{{% /tab %}} +{{< /tabs >}} + +Push (only the Git part -- we do not want to send unencrypted data) +to ensure that the remote repository is initialized with annex UUID. + +``` +datalad push --to origin --data nothing +``` + +At this point, the remote should be recognized by Git-annex, and have an annex UUID. +You can check this with `git annex info origin`. +If the UUID is present, the mask remote can be set up[^3]. +Replace the value of the `keyid` parameter with the ID (either e-mail or the numeric key ID) of your GPG key. + +``` +git annex initremote encrypted-origin type=mask remote=origin encryption=hybrid keyid=me@example.com +``` + +With the remote configuration, the annexed content and the Git content can be pushed *separately* to these remotes. + +``` +datalad push --to encrypted-origin +datalad push --data nothing --to origin +``` + +That's it! The repository is a regular repository, but the annexed content can not be accessed. +The web interface will show the file tree, but report that the files are unavailable. +Cloning the repository is possible, but getting the annexed content requires possession of a private GPG key +for which the encryption has been enabled[^4]. + +### Automating + +Remembering to push to the mask remote first, and to use `--data nothing` with the unencrypted remote +can be tedious and error prone. +However, both of these behaviors can be automated. +Git-annex wanted mechanism can be used to state that no data are wanted by the unencrypted remote. +DataLad's publication dependency can be used to say that the encrypted remote should be pushed automatically +when a push to origin is requested. + +``` +git annex wanted origin "exclude=*" +datalad siblings configure -s origin --publish-depends encrypted-origin +``` + +With this, it is sufficient to `datalad push --to origin`[^5]. + +This can be taken even further. +For example, only *some* annexed files can be pushed encrypted. +It is advisable to configure the encrypted and regular remotes with opposite settings to avoid duplicating the storage. +For example, consider a situation where raw images are stored encrypted, but jpegs are not: + +``` +git annex wanted origin include=*jpeg +git annex wanted encrypted-origin include=*arw +``` + +## Looking under the hood + +Although we used Forgejo as an example (due to its potential relevance), +the walkthrough is valid for any "git+annex" repository. Using a local +bare Git repository is a good way to "look under the hood" and see that +the annex keys in the bare repository are indeed stored encrypted, +with HMAC hashing of the filenames. + +The example below shows the annex objects directory with three files, +two encrypted and one unencrypted. + +``` +❱ tree ../foo.git/annex/objects +../foo.git/annex/objects +├── 39e +│   └── bcf +│   └── GPGHMACSHA1--2c4b73cff92c9d186065f338551f8c5c8ec94b5b +│   └── GPGHMACSHA1--2c4b73cff92c9d186065f338551f8c5c8ec94b5b +├── 727 +│   └── f52 +│   └── GPGHMACSHA1--fbcb8f7931cb90c8fa9930368bfae671f36fec6b +│   └── GPGHMACSHA1--fbcb8f7931cb90c8fa9930368bfae671f36fec6b +└── a45 + └── ab5 + └── SHA256E-s2273030--fbeaece1f05264c39781adeac159ba33fbc1f4d2aea273106600bc4e3e3aa6c7.jpeg + └── SHA256E-s2273030--fbeaece1f05264c39781adeac159ba33fbc1f4d2aea273106600bc4e3e3aa6c7.jpeg +``` + +In Forgejo, the layout would be identical. + + +[^1]: Bare Git repository is a repository which does not contain a working directory. + It can be cloned from and pushed to. + See [Git on the server](https://git-scm.com/book/en/v2/Git-on-the-Server-Getting-Git-on-a-Server) in the Pro Git book. +[^2]: Git annex does have a [git special remote](https://git-annex.branchable.com/special_remotes/git/), + but its purpose is to record a Git remote URL in the repository (normally, Git remote URL is a local configuration) + and optionally autoenable it. It does not provide a way to set up encryption. +[^3]: In Forgejo-aneksajo versions prior to 11.0-0-git-annex2, the UUID was not always assigned immediately; + see discussion in [forgejo-aneksajo/issues/22](https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/issues/22). + A workaround was to perform any action (including viewing the page) in the web interface. + If the UUID is empty, visit the repository page in a browser (or refresh it) and try `git annex info origin` again. +[^4]: In the hybrid encryption scheme, data is encrypted using a symmetric cipher, + and that cipher is asymmetrically encrypted (to selected GPG keys) and stored in the Git repository. + This means that new recipient keys can be added without having to re-encrypt the data. + See the documentation for [git annex encryption](https://git-annex.branchable.com/encryption/). +[^5]: or `git annex push` which by default pushes to all available remotes -- 2.52.0 From 766f3ae277db93ffd124a949fe092f1a5d8cbb61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Thu, 24 Jul 2025 12:19:34 +0200 Subject: [PATCH 07/25] Reorder pages in the Z03 chapter It makes more sense to place the general introduction first. --- content/z03/digital-signatures/_index.md | 2 +- content/z03/encrypted-workflow.md | 2 +- content/z03/local-copies.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/content/z03/digital-signatures/_index.md b/content/z03/digital-signatures/_index.md index f2880dc..f57f207 100644 --- a/content/z03/digital-signatures/_index.md +++ b/content/z03/digital-signatures/_index.md @@ -2,7 +2,7 @@ date = '2025-02-12T18:01:06+01:00' draft = false title = 'Digital signatures' -weight = 1 +weight = 2 +++ The data entry form supports the submission of a digital signature for diff --git a/content/z03/encrypted-workflow.md b/content/z03/encrypted-workflow.md index e2e9b60..f4416f2 100644 --- a/content/z03/encrypted-workflow.md +++ b/content/z03/encrypted-workflow.md @@ -2,7 +2,7 @@ date = '2025-02-12T18:01:59+01:00' draft = false title = 'Encrypted workflow' -weight = 3 +weight = 1 +++ We are using DataLad to track and move around the motor assessment diff --git a/content/z03/local-copies.md b/content/z03/local-copies.md index 9a58820..4eb8800 100644 --- a/content/z03/local-copies.md +++ b/content/z03/local-copies.md @@ -2,7 +2,7 @@ date = '2025-02-12T18:01:47+01:00' draft = false title = 'Local copies' -weight = 2 +weight = 3 +++ The contents of all form fields (except signature) can be saved -- 2.52.0 From 0d04ed56da55724b9f9d4bbefa9c662fcf296d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Thu, 24 Jul 2025 13:46:46 +0200 Subject: [PATCH 08/25] Add detailed instructions for Z03 data access This uses pre-written iffmd document which was previously shared with Z03 via e-mail. --- content/z03/access-data.md | 269 +++++++++++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 content/z03/access-data.md diff --git a/content/z03/access-data.md b/content/z03/access-data.md new file mode 100644 index 0000000..c300e59 --- /dev/null +++ b/content/z03/access-data.md @@ -0,0 +1,269 @@ ++++ +date = '2025-02-12T18:01:59+01:00' +draft = false +title = 'Accessing motor assesment data collection on Windows' +weight = 4 ++++ + +This note will explain in detail how to set-up your local copy of the aggregated motor assesment data, and help deal with potential configuration-dependent obstacles. + +Overview of the encrypted workflow and prerequisites are explained in the [encrypted workflow](z3-encrypted-workflow/) page. It is highly recommended that you read it first. + +The note assumes that you are working on Windows and using cmd shell. It also assumes that you have access to the shared Sciebo folder, and the data is being encrypted to your GPG key. + +## Before you begin + +### GPG + +If you install Git (which is a requirement for using DataLad), it comes with its own `gpg` executable. If you install gpg4win (which is the solution recommended for digital signatures in our docs on [Input signing](digital-signatures) it naturally comes with a separate `gpg` executable. This means it is possible to have two GPG programs... which store their data (e.g. keys) in completely different locations on your computer. + +What's more, in this case, git-annex operations will default to using the one which came with Git, while user operations (be it via Kleopatra GUI or the command line) will more than likely[^1] use the one which came with gpg4win. This can become confusing. Fortunately, git-annex behaviour is configurable. The goal here is to have both the user and git-annex interact with the same `gpg` executable. + +If you do not have gpg4win: you probably do not need to do anything. + +If you have gpg4win and used it to generate (or import) your private key, you need to configure Git and git-annex to use this one. + +First, find out the path to the gpg executable. The path components would tell you whether it came with Git or gpg4win - the latter is more likely. + +``` +where gpg +``` + +Then (if it is gpg4win) tell Git to use this one. Git-annex obeys the same setting. Replace `PATH/TO/GPG` with the path reported by the previous command. + +``` +git config gpg.program PATH/TO/GPG +``` + + +### GPG-agent + +Operations which use your secret key (like downloading encrypted data with git-annex or listing your secret keys with `gpg --list-secret-keys`) require interactions with a program called `gpg-agent` [^2]. Normally, `gpg-agent` should be running in the background, and there is nothing to do. + +If you notice gpg-agent is not running (e.g. `get` fails with an error message which says that gpg-agent is not running, or you can not list your secret keys), you can start the gpg-agent with: + +``` +gpg-connect-agent reloadagent /bye +``` + +### Sciebo token (app password) + +Instead of using your pasword (the one you use to log in to Sciebo), it is recommended to generate a token for use with DataLad & git-annex. Tokens are like passwords, but they are generated by the website rather than created by the user. A token can be revoked when it is no longer used or there is a risk that it got exposed. + +To generate a token: + +- log in to Sciebo +- click your username on the upper right and go to settings +- select Security +- scroll down to App passwords / tokens +- enter a token name and click "Create new app passcode" +- copy (and store safely) the username and token + +The token name can be anything - it is only for your information, and will be displayed in the list of tokens, with the date of last use and an option to delete (revoke) the token. + + +### Save credentials for DataLad in advance + +We have seen that DataLad's prompt for password can behave weirdly in cmd, in that it does not show any indication of characters being typed or pasted. To reduce errors, we recommend saving a DataLad credential [^3] upfront, and verifying that it is saved correctly. + +To save the credential, use the `datalad credentials set` command, replacing the placeholders marked with angle brackets (`< >`): + +``` +datalad credentials set type=user_password user= realm= +``` + +e.g. + +``` +datalad credentials set sciebo type=user_password user=m.szczepanik@fz-juelich.de realm=https://fz-juelich.sciebo.de/sciebo +``` + +(note: the realm needs to end with `/sciebo`). + +You will be prompted for the password. Type or paste and hit enter. You will be prompted to enter the password again. + +To see what was saved, run the following (with a name given in the previous step): + +``` +datalad -f json_pp credentials get sciebo +``` + +The output will look like this. +``` +{ + "action": "credentials", + "cred_realm": "https://fz-juelich.sciebo.de/sciebo", + "cred_secret": "[YOUR TOKEN SHOULD BE HERE]", + "cred_type": "user_password", + "cred_user": "m.szczepanik@fz-juelich.de", + "name": "sciebo", + "status": "ok" +} +``` + +If something seems wrong, repeat the `datalad credentials set` step, adjusting as necessary. + +You can use `datalad credentials get` whenever you need to remind yourself of the token. + +## WebDAV URLs + +When using DataLad & git-annex, you will connect to Sciebo through a protocol called WebDAV. For this, you need to know your WebDAV URL, which is slightly different than the URL you use to visit Sciebo's web interface. + +### Base WebDAV URL + +To find your WebDAV address: + +- go to Sciebo (file view) +- click ⚙️ Settings in the lower left corner +- a "drawer" will slide up, showing your WebDAV URL + +The WebDAV URL will look like this: + +``` +https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/ +``` + +### WebDAV URLs to folders + +The URL above is pointing to your home directory. To access a specific folder, you will need to add some path components manually. + +Most likely, a folder named `Z03` will be shared with you. It will contain some folders; two which are relevant are: `motor_export_repo`, and `motor_export_data`. + +The folder path relative to your home directory will depend on whether you move the shared folder or not. Assuming that `Z03` is in your home directory on Sciebo, the relative paths will be `Z03/motor_export_repo` and `Z03/motor_export_data`, respectively. + +For use with DataLad, you will need to put the base URL and the relative paths together. Using the examples below: + +- repo URL (for clone): `https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Z03/motor_export_repo` +- data URL (for initremote) `https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Z03/motor_export_data` + + +## Initial setup + +### Clone the dataset + +The clone URL will be composed of two parts: + +- constant part (tells Datalad what we are cloning): `datalad-annex::?type=webdav&encryption=none&exporttree=no&url=` +- your repo URL (see above) + +To clone, substitute `` with your repo URL and `` with the target folder (do not include the angle brackets `<>`): +``` +datalad clone "datalad-annex::?type=webdav&encryption=none&exporttree=no&url=" motor_export +``` + +e.g. +``` +datalad clone "datalad-annex::?type=webdav&encryption=none&exporttree=no&url=https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Z03/motor_export_repo" motor_export +``` + +Then change your working directory to go into the dataset: +``` +cd motor_export +``` + +Note: it is important to use double quotes (`""`) as cmd distinguishes between single and double quotes. + +Note: do not worry about the constant part containing `encryption=none`. The repository ("Git part") of the dataset is stored unencrypted, but the file contents ("annex part") are stored encrypted in Sciebo. This step deals only with the repository. + +### Initialize sciebo remote + +Git-annex special remotes represent methods of accessing data, and special remote configuration contains things like access URL or encryption type to be used. + +One weaknes of the Sciebo setup is that WebDAV URLs contain user names and instance names, so will be different for each user (even though the shared folder is, in principle, the same). The dataset already has a sciebo remote, but it points to FZJ's sciebo and Michał Szczepanik's user account. There is no easy way to reconfigure it, other than each user initializing a new remote pointing to *their* sciebo URL. + +The trick is to initialize a special remote, and mark it *same as* the one above (meaning that we expect to reach exactly the same place, just through different means). + +This is by far the most involved part of the setup. + +#### Let git-annex know your token + +DataLad does not have a relevant interface, so we need to run this operation directly with `git-annex`. This means DataLad would not supply the credentials which we stored in the previous step, and we need to provide them via environment variables. Set the environment variables with the `set` command (substitute your Sciebo username and token): + +``` +set WEBDAV_USERNAME=m.szczepanik@fz-juelich.de +set WEBDAV_PASSWORD=XXXXX-XXXXX-XXXXX-XXXXX +``` + +Important: do not put the password in quotes (`""`), because `set` command treats them literally. + +Tip: If you don't have your token, retrieve it using DataLad: `datalad -f json_pp credentials get sciebo`. + +Tip: you can check value of an environment variable with `echo`, e.g. `echo %WEBDAV_USERNAME%`. + +#### (optional): check available remotes +Each special remote has its own ID and assigned name, which can be used interchangeably with git-annex commands. To list remotes known to the dataset, run: + +``` +git annex info +``` + +Pending future changes, the remote we want to enable is going to be named sciebo, i.e. it will be this one: +``` +325dd4e6-e71b-4512-bce6-0eb574b6ed15 -- sciebo-storage +``` + +#### Initialize your own private remote + +As explained above, you will need to initialize a special remote for yourself, and mark it "same as" the one above. + +Here's the command broken up into multiple lines with a caret (`^`) for readability: + +``` +git annex initremote my-sciebo-data^ + --private^ + --sameas sciebo-storage^ + type=webdav^ + url="https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Projects/Z03/motor_export_data" +``` + +And here it is in one line + +``` +git annex initremote my-sciebo-data --private --sameas sciebo-storage type=webdav url="https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Projects/Z03/motor_export_data" +``` + +Substitute the `url` with your data URL (the one which ends with `motor_export_data`)! + +#### Troubleshooting + +- If the error message complains about credentials (bearer authentication), most likely the credentials (environment variables) are incorrect. +- If the error message complains about gpg-agent not running, it is either not running or git-annex is using a different gpg than intended. See sections on gpg and gpg-agent above. +- If there is seemingly no error but data can not be downloaded, then probably the URL given to initremote was incorrect. Make sure that the URL points to `motor_export_data` (not `repo`). + + +## Data access + +{{% notice style="note" %}} +This assumes your working directory is in the dataset (that you `cd`'ed into the dataset). +{{% /notice %}} + +To download data, use: + +``` +datalad get output +``` + +To remove your local copy of the data, use `drop` which does the opposite of `get`: + +``` +datalad drop output +``` + +To obtain the latest version of the data, use: + +``` +datalad update --how merge +datalad get output +``` + +To see when the data (according to your local state) last changed, use `git log` (up/down arrows scroll, `Q` quits): + +``` +git log +``` + + + +[^1]: It would, at least in theory, be possible to modify the [PATH variable](https://en.wikipedia.org/wiki/PATH_(variable)) so that running `gpg` would use the Git-bundled gpg instead of the gpg4win gpg. +[^2]: You call `datalad get`, DataLad calls `git annex get`; git-annex calls `gpg --decrypt`, and gpg asks gpg-agent for a matching secret key. +[^3]: The credential name and username are stored in global (i.e. user) Git configuration, the password is stored in Windows' secret storage. -- 2.52.0 From f34d672ed5d56c91570d516b12617954b694add4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Fri, 25 Jul 2025 19:12:04 +0200 Subject: [PATCH 09/25] Change Z03 data access description to git-remote-annex --- content/z03/access-data.md | 122 +++++++++++++++---------------------- 1 file changed, 48 insertions(+), 74 deletions(-) diff --git a/content/z03/access-data.md b/content/z03/access-data.md index c300e59..d2a49d4 100644 --- a/content/z03/access-data.md +++ b/content/z03/access-data.md @@ -62,49 +62,6 @@ To generate a token: The token name can be anything - it is only for your information, and will be displayed in the list of tokens, with the date of last use and an option to delete (revoke) the token. -### Save credentials for DataLad in advance - -We have seen that DataLad's prompt for password can behave weirdly in cmd, in that it does not show any indication of characters being typed or pasted. To reduce errors, we recommend saving a DataLad credential [^3] upfront, and verifying that it is saved correctly. - -To save the credential, use the `datalad credentials set` command, replacing the placeholders marked with angle brackets (`< >`): - -``` -datalad credentials set type=user_password user= realm= -``` - -e.g. - -``` -datalad credentials set sciebo type=user_password user=m.szczepanik@fz-juelich.de realm=https://fz-juelich.sciebo.de/sciebo -``` - -(note: the realm needs to end with `/sciebo`). - -You will be prompted for the password. Type or paste and hit enter. You will be prompted to enter the password again. - -To see what was saved, run the following (with a name given in the previous step): - -``` -datalad -f json_pp credentials get sciebo -``` - -The output will look like this. -``` -{ - "action": "credentials", - "cred_realm": "https://fz-juelich.sciebo.de/sciebo", - "cred_secret": "[YOUR TOKEN SHOULD BE HERE]", - "cred_type": "user_password", - "cred_user": "m.szczepanik@fz-juelich.de", - "name": "sciebo", - "status": "ok" -} -``` - -If something seems wrong, repeat the `datalad credentials set` step, adjusting as necessary. - -You can use `datalad credentials get` whenever you need to remind yourself of the token. - ## WebDAV URLs When using DataLad & git-annex, you will connect to Sciebo through a protocol called WebDAV. For this, you need to know your WebDAV URL, which is slightly different than the URL you use to visit Sciebo's web interface. @@ -139,21 +96,41 @@ For use with DataLad, you will need to put the base URL and the relative paths t ## Initial setup +#### Let git-annex know your token + +Git-annex needs WebDAV credentials to access the stored repository and data. +It expects them to be provided via environment variables. +Set the environment variables with the `set` command (substitute your Sciebo username and token): + +``` +set WEBDAV_USERNAME=m.szczepanik@fz-juelich.de +set WEBDAV_PASSWORD=XXXXX-XXXXX-XXXXX-XXXXX +``` + +Important: do not put the password in quotes (`""`), because `set` command treats them literally. + +Tip: you can check value of an environment variable with `echo`, e.g. `echo %WEBDAV_USERNAME%`. + +These password will be cached in a owner-read-only file in the clone, +so the environment variables should not be needed for updates. + ### Clone the dataset The clone URL will be composed of two parts: -- constant part (tells Datalad what we are cloning): `datalad-annex::?type=webdav&encryption=none&exporttree=no&url=` -- your repo URL (see above) +- constant part (what we are cloning and how): `"annex::5e2afb7c-2985-454d-a12e-4ec5dcfb3623?encryption=none&type=webdav&url=` +- your repo URL (see above -- where we are cloning from) + +Additionally, we need to specify the branch name to check out (git-remote-annex has no default branch). To clone, substitute `` with your repo URL and `` with the target folder (do not include the angle brackets `<>`): ``` -datalad clone "datalad-annex::?type=webdav&encryption=none&exporttree=no&url=" motor_export +git clone --branch master "annex::5e2afb7c-2985-454d-a12e-4ec5dcfb3623?encryption=none&type=webdav&url=" ``` e.g. ``` -datalad clone "datalad-annex::?type=webdav&encryption=none&exporttree=no&url=https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Z03/motor_export_repo" motor_export +git clone --branch master "annex::5e2afb7c-2985-454d-a12e-4ec5dcfb3623?encryption=none&type=webdav&url=https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Projects/Z03/motor_export_annexrepo" motor_export ``` Then change your working directory to go into the dataset: @@ -165,7 +142,7 @@ Note: it is important to use double quotes (`""`) as cmd distinguishes between s Note: do not worry about the constant part containing `encryption=none`. The repository ("Git part") of the dataset is stored unencrypted, but the file contents ("annex part") are stored encrypted in Sciebo. This step deals only with the repository. -### Initialize sciebo remote +### Initialize sciebo remote(s) Git-annex special remotes represent methods of accessing data, and special remote configuration contains things like access URL or encryption type to be used. @@ -173,22 +150,10 @@ One weaknes of the Sciebo setup is that WebDAV URLs contain user names and insta The trick is to initialize a special remote, and mark it *same as* the one above (meaning that we expect to reach exactly the same place, just through different means). +Because the Git repository is *also* stored in a git-annex special remote, we need to set up *two* such remotes. + This is by far the most involved part of the setup. -#### Let git-annex know your token - -DataLad does not have a relevant interface, so we need to run this operation directly with `git-annex`. This means DataLad would not supply the credentials which we stored in the previous step, and we need to provide them via environment variables. Set the environment variables with the `set` command (substitute your Sciebo username and token): - -``` -set WEBDAV_USERNAME=m.szczepanik@fz-juelich.de -set WEBDAV_PASSWORD=XXXXX-XXXXX-XXXXX-XXXXX -``` - -Important: do not put the password in quotes (`""`), because `set` command treats them literally. - -Tip: If you don't have your token, retrieve it using DataLad: `datalad -f json_pp credentials get sciebo`. - -Tip: you can check value of an environment variable with `echo`, e.g. `echo %WEBDAV_USERNAME%`. #### (optional): check available remotes Each special remote has its own ID and assigned name, which can be used interchangeably with git-annex commands. To list remotes known to the dataset, run: @@ -197,32 +162,41 @@ Each special remote has its own ID and assigned name, which can be used intercha git annex info ``` -Pending future changes, the remote we want to enable is going to be named sciebo, i.e. it will be this one: +Pending future changes, the remotes we want to enable are going to be named sciebo-storage and sciebo-annexrepo, +i.e. it will be these two: + ``` 325dd4e6-e71b-4512-bce6-0eb574b6ed15 -- sciebo-storage +5e2afb7c-2985-454d-a12e-4ec5dcfb3623 -- sciebo-annexrepo [origin] ``` -#### Initialize your own private remote +#### Initialize your own private remotes As explained above, you will need to initialize a special remote for yourself, and mark it "same as" the one above. -Here's the command broken up into multiple lines with a caret (`^`) for readability: +Here's the command broken up into multiple lines with a caret (`^`) for readability. First, `-annexrepo`: ``` -git annex initremote my-sciebo-data^ +git annex initremote --with-url my-sciebo-annexrepo ^ + --private^ + --sameas sciebo-annexrepo^ + type=webdav^ + url="" +``` + +Substitute `` with your repo URL (the one which ends with `motor_export_annexrepo`)! + +And then, `-storage`: + +``` +git annex initremote my-sciebo-storage^ --private^ --sameas sciebo-storage^ type=webdav^ - url="https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Projects/Z03/motor_export_data" + url="" ``` -And here it is in one line - -``` -git annex initremote my-sciebo-data --private --sameas sciebo-storage type=webdav url="https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Projects/Z03/motor_export_data" -``` - -Substitute the `url` with your data URL (the one which ends with `motor_export_data`)! +Substitute the `` with your data URL (the one which ends with `motor_export_data`)! #### Troubleshooting -- 2.52.0 From 633b3e9de4d2d0b33cdff7c69de15cac976e05af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Fri, 1 Aug 2025 19:01:04 +0200 Subject: [PATCH 10/25] Change Z03 data access instructions to clone from jugit Encrypting the data and storing them in access-restricted Sciebo folder allowed us to make the repository public, thus simplifying the access and update procedure. --- content/z03/access-data.md | 81 ++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 48 deletions(-) diff --git a/content/z03/access-data.md b/content/z03/access-data.md index d2a49d4..5c5ea2d 100644 --- a/content/z03/access-data.md +++ b/content/z03/access-data.md @@ -77,21 +77,24 @@ To find your WebDAV address: The WebDAV URL will look like this: ``` -https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/ +https://fz-juelich.sciebo.de/remote.php/dav/files/j.doe%40fz-juelich.de/ ``` ### WebDAV URLs to folders The URL above is pointing to your home directory. To access a specific folder, you will need to add some path components manually. -Most likely, a folder named `Z03` will be shared with you. It will contain some folders; two which are relevant are: `motor_export_repo`, and `motor_export_data`. +Most likely, a folder named `Z03` will be shared with you. It will contain some folders; `motor_export_data` is the relevant one. The folder path relative to your home directory will depend on whether you move the shared folder or not. Assuming that `Z03` is in your home directory on Sciebo, the relative paths will be `Z03/motor_export_repo` and `Z03/motor_export_data`, respectively. -For use with DataLad, you will need to put the base URL and the relative paths together. Using the examples below: +For use with DataLad, you will need to put the base URL and the relative paths together. Using the example above, +and assuming that the shared folder is in the top-level directory, the data folder URL (for initremote) is: -- repo URL (for clone): `https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Z03/motor_export_repo` -- data URL (for initremote) `https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Z03/motor_export_data` + +``` +https://fz-juelich.sciebo.de/remote.php/dav/files/j.doe%40fz-juelich.de/Z03/motor_export_data +``` ## Initial setup @@ -103,34 +106,25 @@ It expects them to be provided via environment variables. Set the environment variables with the `set` command (substitute your Sciebo username and token): ``` -set WEBDAV_USERNAME=m.szczepanik@fz-juelich.de +set WEBDAV_USERNAME=j.doe@fz-juelich.de set WEBDAV_PASSWORD=XXXXX-XXXXX-XXXXX-XXXXX ``` Important: do not put the password in quotes (`""`), because `set` command treats them literally. -Tip: you can check value of an environment variable with `echo`, e.g. `echo %WEBDAV_USERNAME%`. +Tip: you can check the value of an environment variable with `echo`, e.g. `echo %WEBDAV_USERNAME%`. -These password will be cached in a owner-read-only file in the clone, -so the environment variables should not be needed for updates. +The environment variables persist until you close your cmd. +However, these credentials will later be cached in a owner-read-only file in the clone, +so re-entering the credentials should not be needed for subsequent updates. ### Clone the dataset -The clone URL will be composed of two parts: +Clone the (lightweight) dataset from JuGit (using the URL below) into a directory with a name of your choice +(here: `motor_export`): -- constant part (what we are cloning and how): `"annex::5e2afb7c-2985-454d-a12e-4ec5dcfb3623?encryption=none&type=webdav&url=` -- your repo URL (see above -- where we are cloning from) - -Additionally, we need to specify the branch name to check out (git-remote-annex has no default branch). - -To clone, substitute `` with your repo URL and `` with the target folder (do not include the angle brackets `<>`): ``` -git clone --branch master "annex::5e2afb7c-2985-454d-a12e-4ec5dcfb3623?encryption=none&type=webdav&url=" -``` - -e.g. -``` -git clone --branch master "annex::5e2afb7c-2985-454d-a12e-4ec5dcfb3623?encryption=none&type=webdav&url=https://fz-juelich.sciebo.de/remote.php/dav/files/m.szczepanik%40fz-juelich.de/Projects/Z03/motor_export_annexrepo" motor_export +datalad clone https://jugit.fz-juelich.de/m.szczepanik/motor-assesment.git motor_export ``` Then change your working directory to go into the dataset: @@ -138,11 +132,7 @@ Then change your working directory to go into the dataset: cd motor_export ``` -Note: it is important to use double quotes (`""`) as cmd distinguishes between single and double quotes. - -Note: do not worry about the constant part containing `encryption=none`. The repository ("Git part") of the dataset is stored unencrypted, but the file contents ("annex part") are stored encrypted in Sciebo. This step deals only with the repository. - -### Initialize sciebo remote(s) +### Initialize sciebo remote Git-annex special remotes represent methods of accessing data, and special remote configuration contains things like access URL or encryption type to be used. @@ -150,11 +140,8 @@ One weaknes of the Sciebo setup is that WebDAV URLs contain user names and insta The trick is to initialize a special remote, and mark it *same as* the one above (meaning that we expect to reach exactly the same place, just through different means). -Because the Git repository is *also* stored in a git-annex special remote, we need to set up *two* such remotes. - This is by far the most involved part of the setup. - #### (optional): check available remotes Each special remote has its own ID and assigned name, which can be used interchangeably with git-annex commands. To list remotes known to the dataset, run: @@ -162,31 +149,18 @@ Each special remote has its own ID and assigned name, which can be used intercha git annex info ``` -Pending future changes, the remotes we want to enable are going to be named sciebo-storage and sciebo-annexrepo, -i.e. it will be these two: +Pending future changes, the remote we want to enable is going to be named sciebo-storage, +i.e. it will be this one: ``` 325dd4e6-e71b-4512-bce6-0eb574b6ed15 -- sciebo-storage -5e2afb7c-2985-454d-a12e-4ec5dcfb3623 -- sciebo-annexrepo [origin] ``` -#### Initialize your own private remotes +#### Initialize your own private special remote As explained above, you will need to initialize a special remote for yourself, and mark it "same as" the one above. -Here's the command broken up into multiple lines with a caret (`^`) for readability. First, `-annexrepo`: - -``` -git annex initremote --with-url my-sciebo-annexrepo ^ - --private^ - --sameas sciebo-annexrepo^ - type=webdav^ - url="" -``` - -Substitute `` with your repo URL (the one which ends with `motor_export_annexrepo`)! - -And then, `-storage`: +Here's the command broken up into multiple lines with a caret (`^`) for readability. ``` git annex initremote my-sciebo-storage^ @@ -196,8 +170,20 @@ git annex initremote my-sciebo-storage^ url="" ``` +And this is the same as a one-liner: + +``` +git annex initremote my-sciebo-storage --private --sameas sciebo-storage type=webdav url="" +``` + Substitute the `` with your data URL (the one which ends with `motor_export_data`)! +{{% notice style="note" %}} +In the example above, we enclosed the URL with double quotes (`"...")`. +Keep in mind that cmd distinguishes single- and double quotes and treats them differently! +Find out more in https://ss64.com/nt/syntax-esc.html +{{% /notice %}} + #### Troubleshooting - If the error message complains about credentials (bearer authentication), most likely the credentials (environment variables) are incorrect. @@ -237,7 +223,6 @@ git log ``` - [^1]: It would, at least in theory, be possible to modify the [PATH variable](https://en.wikipedia.org/wiki/PATH_(variable)) so that running `gpg` would use the Git-bundled gpg instead of the gpg4win gpg. [^2]: You call `datalad get`, DataLad calls `git annex get`; git-annex calls `gpg --decrypt`, and gpg asks gpg-agent for a matching secret key. [^3]: The credential name and username are stored in global (i.e. user) Git configuration, the password is stored in Windows' secret storage. -- 2.52.0 From cefaae9adb9ab9e747d7fa4c2600cbf4fd0c069e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Fri, 1 Aug 2025 20:30:56 +0200 Subject: [PATCH 11/25] Add mermaid graph explaining JuGit plus Sciebo setup. --- content/z03/access-data.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/content/z03/access-data.md b/content/z03/access-data.md index 5c5ea2d..20b7ca6 100644 --- a/content/z03/access-data.md +++ b/content/z03/access-data.md @@ -97,6 +97,31 @@ https://fz-juelich.sciebo.de/remote.php/dav/files/j.doe%40fz-juelich.de/Z03/moto ``` +## Distributed storage + +There are two components to the dataset: + +- The Git repository (which we can call a "lightweight dataset") contains information about content identity and availability, but does not contain the actual data. +- The content managed by git-annex ("annexed data"), i.e. the actual data. + +For technical reasons they are stored and shared separately. +In the chart below, a square box symbolizes a Git remote, and cylinders symbolize Git-annex special remotes. + +```mermaid +flowchart TD + A(Your local dataset clone) + A ---o |clone, update| B["**JuGit** (public)"] + A ---o |initremote, get| C[("**Sciebo** + (restricted, encrypted)")] + A ---o |"🚫"| D[("**INM-7 storage** + (restricted, encrypted)")] +``` + +Retrieving data requires interacting with both the Git remote and the Git-annex special remote (key commands are shown over the lines). +Because no data are stored in JuGit, the repository can be made public, and fetching information from JuGit requires no credentials. +However, access to the data stored in Sciebo requires not only credentials, but also posession of a GPG key for which the data are encrypted. +Note that the Git repository also indexes data stored in INM7-storage, but that special remote is not accessible outside INM-7. + ## Initial setup #### Let git-annex know your token -- 2.52.0 From 76332539710123be4695ea0047bdfeb451f76771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Tue, 5 Aug 2025 11:49:32 +0200 Subject: [PATCH 12/25] Replace callouts with shortcodes for Hugo 0.131 compatibility This commit replaces markdown blockquote callouts with shortcodes provided by the Relearn theme. The callouts are supported starting with Hugo 0.132, but the psyinf-hugo-site-build action runs on Hugo 0.131. https://gohugo.io/render-hooks/blockquotes/ The relearn theme supports both styles: > Callout syntax has limited features as it does not provide all > of the below parameter. Nevertheless, it is widely available in > other Markdown parsers like GitHub or Obsidian and therefore is > the recommend syntax for generating portable Markdown. https://mcshelby.github.io/hugo-theme-relearn/shortcodes/notice/index.html For this reason, this commit could / should be reverted once the psyinf site build action is updated to a later Hugo release. --- content/catalog/datalad-based-deposition.md | 6 +- content/datalad/extensions/_index.md | 9 +- content/walkthroughs/publication-catalog.md | 99 +++++++++++---------- 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/content/catalog/datalad-based-deposition.md b/content/catalog/datalad-based-deposition.md index 545b09b..f272b6a 100644 --- a/content/catalog/datalad-based-deposition.md +++ b/content/catalog/datalad-based-deposition.md @@ -77,9 +77,9 @@ the added metadata records. ### 4. Publish the update -> [!important] -> This step assumes that the project has already been published -> (has a configured sibling). +{{% notice style="important" %}} +This step assumes that the project has already been published (has a configured sibling). +{{% /notice %}} Then, push the new state of the dataset to an existing sibling: diff --git a/content/datalad/extensions/_index.md b/content/datalad/extensions/_index.md index a64f5fb..d6bd3b2 100644 --- a/content/datalad/extensions/_index.md +++ b/content/datalad/extensions/_index.md @@ -16,10 +16,11 @@ SFB: ![DataLad graphical user interface](gooey_pushed.png) -> [!Important] -> Currently, the DataLad-Gooey extension is *de facto* unmaintained. -> However, it was an important proof of principle, and we expect it to -> make a comeback in the future; likely in a completely new guise. +{{% notice style="important" %}} +Currently, the DataLad-Gooey extension is *de facto* unmaintained. +However, it was an important proof of principle, and we expect it to +make a comeback in the future; likely in a completely new guise. +{{% /notice %}} The DataLad-Gooey (pronounce \"GUI\") package provides a graphical user interface for DataLad. It is specifically aiming at making key data diff --git a/content/walkthroughs/publication-catalog.md b/content/walkthroughs/publication-catalog.md index d5d215d..47e76bd 100644 --- a/content/walkthroughs/publication-catalog.md +++ b/content/walkthroughs/publication-catalog.md @@ -4,24 +4,27 @@ draft = false title = 'Publication Catalog' +++ -> [!CAUTION] -> The workflow described here remains a valid description for dataset -> publication on Sciebo. However, as a means for submitting datasets to -> the SFB1451 catalog, it has been deprecated in favour of the procedure -> described in -> [Register data in the data catalog]({{% relref "/catalog/register-data" %}}). -> -> If publications are to be shown in the catalog, information about them -> needs to be included directly in the dataset (metadata) files, -> preferably using standard metadata formats (e.g as -> [citation file format references field](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#references)) -> or as a [datalad-tabby publications table]({{% relref "/catalog/register-data/#table-publications" %}}). -> All datasets with -> project descriptions and publication metadata created according to the -> workflow description below (either by INF or by respective projects) -> have been republished to GitHub as the -> [sfb1451/all-projects](https://github.com/sfb1451/all-projects) -> superdataset. +{{% notice style="important" %}} + +The workflow described here remains a valid description for dataset +publication on Sciebo. However, as a means for submitting datasets to +the SFB1451 catalog, it has been deprecated in favour of the procedure +described in +[Register data in the data catalog]({{% relref "/catalog/register-data" %}}). + +If publications are to be shown in the catalog, information about them +needs to be included directly in the dataset (metadata) files, +preferably using standard metadata formats (e.g as +[citation file format references field](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#references)) +or as a [datalad-tabby publications table]({{% relref "/catalog/register-data/#table-publications" %}}). +All datasets with +project descriptions and publication metadata created according to the +workflow description below (either by INF or by respective projects) +have been republished to GitHub as the +[sfb1451/all-projects](https://github.com/sfb1451/all-projects) +superdataset. + +{{% /notice %}} ## Publication Catalog from Sciebo: a walkthrough @@ -76,10 +79,11 @@ it: cd my-project datalad run-procedure cfg_text2git -> [!INFO] -> The `run-procedure cfg_text2git` will configure DataLad so that text -> files will not be annexed -- this is a useful setting for the files we -> will be adding. +{{% notice style="info" %}} +The `run-procedure cfg_text2git` will configure DataLad so that text +files will not be annexed -- this is a useful setting for the files we +will be adding. +{{% /notice %}} ### Create a citation file @@ -131,10 +135,11 @@ DataLad: datalad save -m "Add citation file" CITATION.cff -> [!INFO] -> Citation File Format is a standard for plain text files with human- and -> machine-readable information for datasets (and software). You can read -> more on the [CFF website](https://citation-file-format.github.io/). +{{% notice style="info" %}} +Citation File Format is a standard for plain text files with human- and +machine-readable information for datasets (and software). You can read +more on the [CFF website](https://citation-file-format.github.io/). +{{% /notice %}} ### Add bibliographic information to the dataset @@ -173,29 +178,29 @@ If this is the first time you are using DataLad with Sciebo, you will be prompted for your credentials. -> [!INFO] -> Sciebo uses your entire e-mail address (name@example.com) as its -> username. +{{% notice style="info" %}} +Sciebo uses your entire e-mail address (name@example.com) as its username. +{{% /notice %}} -> [!INFO] -> The `--dataset` option lets us explicitly specify the dataset for which -> we configure a sibling. The `--name` option sets the name which we will -> use later when publishing. The `--mode filetree` option enables filetree -> mode, meaning that the sibling will have the same, human-readable, file -> tree layout as the folder on your drive. -> -> The WebDAV url can be obtained from Sciebo\'s web UI, by clicking at -> \"Settings\" in the lower left. This URL points to the user\'s home -> directory, and any subfolders must be appended \"by hand\". Nonexisting -> subfolders will be created. The URLs will be different for different -> instances and users. For example, with FZJ\'s instance, the full URL to -> the dataset folder would look like this (`` and `` are -> placeholders): -> -> ``` -> https://fz-juelich.sciebo.de/remote.php/dav/files///pub-dataset -> ``` +{{% notice style="info" %}} +The `--dataset` option lets us explicitly specify the dataset for which +we configure a sibling. The `--name` option sets the name which we will +use later when publishing. The `--mode filetree` option enables filetree +mode, meaning that the sibling will have the same, human-readable, file +tree layout as the folder on your drive. +The WebDAV url can be obtained from Sciebo\'s web UI, by clicking at +\"Settings\" in the lower left. This URL points to the user\'s home +directory, and any subfolders must be appended \"by hand\". Nonexisting +subfolders will be created. The URLs will be different for different +instances and users. For example, with FZJ\'s instance, the full URL to +the dataset folder would look like this (`` and `` are +placeholders): + +``` +https://fz-juelich.sciebo.de/remote.php/dav/files///pub-dataset +``` +{{% /notice %}} ### Publish to Sciebo -- 2.52.0 From 1fc62e247d5a0641efa1910594c724e3a7f0ba88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Tue, 5 Aug 2025 11:59:10 +0200 Subject: [PATCH 13/25] Update contributing page: new URL, shorter git submodule command This adds a link to hub.psychoinformatics.de and a terse info that "patches are welcome". The instruction to obtain the submodule is shortened from two Git commands (submodule init, submodule update) to just one, `submodule update --init`. --- content/about/contributing.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/content/about/contributing.md b/content/about/contributing.md index d738514..6dc5400 100644 --- a/content/about/contributing.md +++ b/content/about/contributing.md @@ -6,8 +6,8 @@ menu = "shortcuts" This documentation website is built with the [hugo](https://gohugo.io/) static website generator. Content of the pages is written in [markdown](https://daringfireball.net/projects/markdown/). -The repository is hosted on [JuGit](https://jugit.fz-juelich.de/). -The project visibility is currently set to private. +The repository is hosted on the [Psychoinformatics Lab Hub](https://hub.psychoinformatics.de/sfb1451/rdm.sfb1451.de) (Forgejo instance). +Patches are welcome. ## Setup @@ -15,8 +15,7 @@ After cloning the repository, you need to obtain the website theme. The theme has been added as a git submodule. ``` -git submodiule init -git submodule update +git submodule update --init ``` To build the site locally and output a URL to open in a browser to test it, run: -- 2.52.0 From 18bcfb3adf8f9148ba29cf227a9b23d29ce58726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Tue, 5 Aug 2025 13:23:18 +0200 Subject: [PATCH 14/25] Fix workflow syntax Fixes an error in if-expression: the entire if is a string, so only single quotes are available for string delimiter. All online examples use single quotes. --- .forgejo/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index ba9575d..4a714b2 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -12,7 +12,7 @@ on: jobs: deploy: - if: github.repository_owner == "www" + if: github.repository_owner == 'www' name: Build site with hugo and deploy on success runs-on: psyinf-hugo-site-build container: -- 2.52.0 From 9501e84122296ac083f891204895f25545f8163b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Tue, 5 Aug 2025 14:22:11 +0200 Subject: [PATCH 15/25] Restructure extensions, add more deprecation notices The DataLad-next extension is moved to the front. Its description is shortened to remove mentions of WebDAV, because now git-remote-annex can fulfill that role and the corresponding functionality in -next is slotted for removal: https://github.com/datalad/datalad-next/issues/785#issuecomment-3106625346 The XNAT and Redcap extension now get info boxes saying that they are unmaintained. Although no formal decision has been made in that regard, the notes reflect the reality. --- content/datalad/extensions/_index.md | 42 ++++++++++++---------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/content/datalad/extensions/_index.md b/content/datalad/extensions/_index.md index d6bd3b2..26fc116 100644 --- a/content/datalad/extensions/_index.md +++ b/content/datalad/extensions/_index.md @@ -12,6 +12,15 @@ extensions](https://github.com/datalad/datalad-extensions/) GitHub page. Development of the following extensions was directly influenced by the SFB: +## DataLad-NEXT {#sec-datalad-next} + +DataLad-NEXT is a proving ground for broadly applicable novel +functionality. It offers additional functionality, better performance, +and improved user experience. + +Read more in [DataLad-NEXT github page](https://github.com/datalad/datalad-next) +and in [DataLad-NEXT documentation](https://docs.datalad.org/projects/next/en/latest/). + ## DataLad-Gooey (GUI) {#sec-datalad-gooey} ![DataLad graphical user interface](gooey_pushed.png) @@ -40,6 +49,10 @@ documentation](https://docs.datalad.org/projects/gooey/en/latest/index.html). ## DataLad-XNAT +{{% notice style="info" %}} +The DataLad-XNAT extension is currently unmaintained. +{{% /notice %}} + [XNAT](https://www.xnat.org/) is an open source imaging informatics platform developed by the Neuroinformatics Research Group at Washington University. It facilitates common management, productivity, and quality @@ -65,34 +78,15 @@ You can read more in the [DataLad-XNAT github page](https://github.com/datalad/datalad-xnat) and in [datalad-XNAT documentation](http://docs.datalad.org/projects/datalad-xnat/en/latest/). -## DataLad-NEXT {#sec-datalad-next} - -DataLad-NEXT is a proving ground for broadly applicable novel -functionality. - -Most important for SFB is the much improved ability to deposit DataLad -datasets on any WebDAV server. This includes UKK\'s Dracoon system, as -well as Sciebo, EUDAT\'s B2DROP, and self-hosted OwnCloud or NextCloud -instances. - -The exported content can be made browsable like normal content for -non-DataLad consumers, but also remain a full dataset that can be cloned -and collaborated on. While it won\'t be as fast or flexible as with more -specialized services for Git or data hosting, this is a great approach -for dataset deposition, where only one site is uploading dataset and -file content updates, and the data consumers can choose between using -DataLad or point-and-click interfaces. See the [WebDAV -walkthrough]({{% relref "/walkthroughs/webdav/" %}}) page for a detailed overview. - -Read more in [DataLad-NEXT github -page](https://github.com/datalad/datalad-next) and in [DataLad-NEXT -documentation](https://docs.datalad.org/projects/next/en/latest/). - ## DataLad-REDCap {#sec-datalad-redcap} +{{% notice style="info" %}} +The DataLad-REDCap extension is currently unmaintained. +{{% /notice %}} + DataLad-REDCap extension provides convenience commands for exporting data from [REDCap](https://project-redcap.org/) into DataLad datasets. -It is yet unreleased, but you can install a working development version +It has not been released, but you can install a working development version from [DataLad-REDCap Github](https://github.com/datalad/datalad-redcap) and read [DataLad-RedCAP documentation](http://docs.datalad.org/projects/redcap/en/latest/?badge=latest). -- 2.52.0 From fa661ebc1fe4df338294eacac2e1c7b893546d5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Tue, 5 Aug 2025 15:44:01 +0200 Subject: [PATCH 16/25] Fix typos Based on a manual run of codespell. --- content/activities/activity-log.md | 2 +- content/catalog/register-data.md | 2 +- content/data-publications/_index.md | 2 +- content/z03/_index.md | 4 ++-- content/z03/access-data.md | 8 ++++---- content/z03/digital-signatures/_index.md | 2 +- content/z03/encrypted-workflow.md | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/content/activities/activity-log.md b/content/activities/activity-log.md index 6783045..bfa1a5c 100644 --- a/content/activities/activity-log.md +++ b/content/activities/activity-log.md @@ -107,7 +107,7 @@ user interface (GUI) for DataLad, is released. **2022-08-19** PI Workshop on research data management. An opportunity to present an overview of existing systems for decentralised data -management and discuss related chalenges. +management and discuss related challenges. **2022-07-25** User interface update for Z03 data entry portal (human motor assessment centre). diff --git a/content/catalog/register-data.md b/content/catalog/register-data.md index 2e32f21..a3b2948 100644 --- a/content/catalog/register-data.md +++ b/content/catalog/register-data.md @@ -278,7 +278,7 @@ Some additional grant information can be retrieved online during processing of records. This is currently done for projects available in GEPRIS (DFG) and CORDIS (EU). For DFG grants other than CRC1451 and its subprojects, enter \"DFG\" in the funder field and only the numeric -identifier in the identifier field. For European Commision grants, enter +identifier in the identifier field. For European Commission grants, enter the URL to their CORDIS page. ### Table [publications] diff --git a/content/data-publications/_index.md b/content/data-publications/_index.md index 8a5c675..c6e6aef 100644 --- a/content/data-publications/_index.md +++ b/content/data-publications/_index.md @@ -36,4 +36,4 @@ individual data publication solution employed by CRC members. `addurls`. See for more information. This approach is also applicable to any data file that - are published as attachements to journal articles. + are published as attachments to journal articles. diff --git a/content/z03/_index.md b/content/z03/_index.md index 52d1491..14ae6e3 100644 --- a/content/z03/_index.md +++ b/content/z03/_index.md @@ -1,5 +1,5 @@ +++ -title = "Z03 Motor Assesment Center" +title = "Z03 Motor Assessment Center" type = "chapter" weight = 2 +++ @@ -14,7 +14,7 @@ information is: - user: data-entry-engineer - password: Lop8-waSe-1307-Adkv -The data entry form supports the submission of a digitial signature +The data entry form supports the submission of a digital signature for each data record to verify submissions and submitter identities. The raw dataset consists of participant records stored in individual diff --git a/content/z03/access-data.md b/content/z03/access-data.md index 20b7ca6..7df6744 100644 --- a/content/z03/access-data.md +++ b/content/z03/access-data.md @@ -1,11 +1,11 @@ +++ date = '2025-02-12T18:01:59+01:00' draft = false -title = 'Accessing motor assesment data collection on Windows' +title = 'Accessing motor assessment data collection on Windows' weight = 4 +++ -This note will explain in detail how to set-up your local copy of the aggregated motor assesment data, and help deal with potential configuration-dependent obstacles. +This note will explain in detail how to set-up your local copy of the aggregated motor assessment data, and help deal with potential configuration-dependent obstacles. Overview of the encrypted workflow and prerequisites are explained in the [encrypted workflow](z3-encrypted-workflow/) page. It is highly recommended that you read it first. @@ -48,7 +48,7 @@ gpg-connect-agent reloadagent /bye ### Sciebo token (app password) -Instead of using your pasword (the one you use to log in to Sciebo), it is recommended to generate a token for use with DataLad & git-annex. Tokens are like passwords, but they are generated by the website rather than created by the user. A token can be revoked when it is no longer used or there is a risk that it got exposed. +Instead of using your password (the one you use to log in to Sciebo), it is recommended to generate a token for use with DataLad & git-annex. Tokens are like passwords, but they are generated by the website rather than created by the user. A token can be revoked when it is no longer used or there is a risk that it got exposed. To generate a token: @@ -119,7 +119,7 @@ flowchart TD Retrieving data requires interacting with both the Git remote and the Git-annex special remote (key commands are shown over the lines). Because no data are stored in JuGit, the repository can be made public, and fetching information from JuGit requires no credentials. -However, access to the data stored in Sciebo requires not only credentials, but also posession of a GPG key for which the data are encrypted. +However, access to the data stored in Sciebo requires not only credentials, but also possession of a GPG key for which the data are encrypted. Note that the Git repository also indexes data stored in INM7-storage, but that special remote is not accessible outside INM-7. ## Initial setup diff --git a/content/z03/digital-signatures/_index.md b/content/z03/digital-signatures/_index.md index f57f207..fe55e68 100644 --- a/content/z03/digital-signatures/_index.md +++ b/content/z03/digital-signatures/_index.md @@ -97,7 +97,7 @@ rights. ### Preparation -1. Open a terminal window by starting Applications -\> Utitities -\> +1. Open a terminal window by starting Applications -\> Utilities -\> Terminal.app. 2. Execute the command: diff --git a/content/z03/encrypted-workflow.md b/content/z03/encrypted-workflow.md index f4416f2..103c316 100644 --- a/content/z03/encrypted-workflow.md +++ b/content/z03/encrypted-workflow.md @@ -16,7 +16,7 @@ to download data, decryption is handled automatically. ## GPG - signing and encryption -To get access to the motor assesment collection and / or to have their +To get access to the motor assessment collection and / or to have their additions marked as \"trusted\", a Z03 member needs to: 1. create a GPG key pair (start with the signing instructions linked -- 2.52.0 From b7f38e8aa1548dffca1dccc366caea00a9418463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Tue, 5 Aug 2025 15:31:30 +0200 Subject: [PATCH 17/25] Add "Read-only public Sciebo shares" walkthrough This walk-through only contains a short preamble and links to https://blog.datalad.org/posts/annex-nextcloud/ for details. This partially overlaps with the WebDAV walkthrough, which could get rewritten or even deprecated. However, the old WebDAV walkthrough contains valuable information: it discusses e.g. filetree (export) mode. It would make most sense to change the WebDAV walkthrough to use git-remote-annex (but keeping the copy vs export comparison). This can be done at a later point (although not too late, because DataLad's functionality is slotted for removal: https://github.com/datalad/datalad-next/issues/785#issuecomment-3106625346). --- .../walkthroughs/read-only-sciebo-share.md | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 content/walkthroughs/read-only-sciebo-share.md diff --git a/content/walkthroughs/read-only-sciebo-share.md b/content/walkthroughs/read-only-sciebo-share.md new file mode 100644 index 0000000..e1416b7 --- /dev/null +++ b/content/walkthroughs/read-only-sciebo-share.md @@ -0,0 +1,27 @@ ++++ +date = '2025-08-05T15:00:00+02:00' +draft = false +title = 'Read-only public Sciebo shares' ++++ + +[Sciebo](https://hochschulcloud.nrw/) is accessible through WebDAV, which means that +git-annex's [webdav special remote](https://git-annex.branchable.com/special_remotes/webdav/) +can be used to synchronize data with Sciebo. This was explored in the +[WebDav walkthrough](webdav). + +However, git-annex used to require write access for enabling a webdav special remote, +which made it impractical for sharing data with broader audiences. This changed in +git-annex version 10.20250115. + +Additionally, the ability to push a git repository to a git-annex special remote +and clone from a special remote, previously provided in DataLad-next, is now built-in +into git-annex, as +[git-remote-annex](https://git-annex.branchable.com/git-remote-annex/). +This feature was added in git-annex version 10.20240531. +See git-annex tips: +[storing a git repository on any special remote](https://git-annex.branchable.com/tips/storing_a_git_repository_on_any_special_remote/). + +This means that it is possible to create an "all-in-one" (git + annex) deposit in Sciebo. +A walkthrough with explanations is provided in the +[Putting new git-annex features to use with Nextcloud](https://blog.datalad.org/posts/annex-nextcloud/) +post on the DataLad blog. -- 2.52.0 From 2d48f0a4daac937124581e57f3054304acc45d6b Mon Sep 17 00:00:00 2001 From: "j.goddard" Date: Mon, 12 Jan 2026 14:47:38 +0100 Subject: [PATCH 18/25] generalize OH date/time and 'zoom' reference --- content/datalad/help.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/content/datalad/help.md b/content/datalad/help.md index 1b121ed..8333ea0 100644 --- a/content/datalad/help.md +++ b/content/datalad/help.md @@ -12,15 +12,17 @@ to use the following channels. ## Virtual DataLad office hours -We are having a virtual office hour every Tuesday at 16:00. This is a -great opportunity to discuss problems and solutions with the members of -the [Psychoinformatics Group](https://www.psychoinformatics.de) at FZJ. +We are having a virtual office hour every week. +This is a great opportunity to discuss problems and solutions with the +members of the [Psychoinformatics Group](https://www.psychoinformatics.de) at FZJ. +Check the matrix chat room for the most up-to-date meeting time and joining +link. To join: - Follow [this link to the public matrix chat room](https://matrix.to/#/!NaMjKIhMXhSicFdxAj:matrix.org) and click continue. -- The link to the zoom meeting will be in the channel description. +- The link to the online meeting will be in the channel description. - We use the matrix chat room to communicate any changes of plans if they happen, or to discuss topics in advance. Joining isn\'t necessary, but you\'re invited to pick one of the available methods -- 2.52.0 From 88262005ecbbc59f1170cd676f05993a58c59772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Wed, 16 Sep 2026 19:17:41 +0200 Subject: [PATCH 19/25] Update Z03 storage graph with Cerebra This adds cerebra & cerebra-mask remotes to the schematic representation of motor assesment data storage locations. The JuGit + Sciebo pair is surrounded by a box labelled "deprecated". This is a half-step; the document needs to be rewritten to describe Cerebra access (it should become simpler, as the WebDAV caveats will go away). --- content/z03/access-data.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/content/z03/access-data.md b/content/z03/access-data.md index 7df6744..244c577 100644 --- a/content/z03/access-data.md +++ b/content/z03/access-data.md @@ -104,17 +104,26 @@ There are two components to the dataset: - The Git repository (which we can call a "lightweight dataset") contains information about content identity and availability, but does not contain the actual data. - The content managed by git-annex ("annexed data"), i.e. the actual data. -For technical reasons they are stored and shared separately. +For technical reasons, they can be stored and shared together (as is the case with Cerebra) or separately (JuGit + Sciebo). +The chart below illustrates the Git and git-annex repositories which can be configured for a local clone to access. In the chart below, a square box symbolizes a Git remote, and cylinders symbolize Git-annex special remotes. ```mermaid flowchart TD A(Your local dataset clone) - A ---o |clone, update| B["**JuGit** (public)"] - A ---o |initremote, get| C[("**Sciebo** - (restricted, encrypted)")] - A ---o |"🚫"| D[("**INM-7 storage** - (restricted, encrypted)")] + AA["**Cerebra** (restricted)"] + AB[("**Cerebra-mask** (restricted, encrypted)")] + subgraph Deprecated + B["**JuGit** (public)"] + C[("**Sciebo** (restricted, encrypted)")] + end + D[("**INM-7 storage** (restricted, encrypted)")] + AA --- AB + A ---o |clone, update| AA + A ---o |enableremote, get| AB + A ---o |clone, update| B + A ---o |initremote, get| C + A ---o |"🚫"| D ``` Retrieving data requires interacting with both the Git remote and the Git-annex special remote (key commands are shown over the lines). -- 2.52.0 From 6838f116e9bfe2916276b2da7ea44b92a67952a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Thu, 17 Sep 2026 18:21:47 +0200 Subject: [PATCH 20/25] Rewrite Z03 docs to focus on Cerebra This gets rid of all the Sciebo/WebDAV quirks, but still keeps some mentions of JuGit + Sciebo (valid until we fully phase it out). --- content/z03/access-data.md | 269 ++++++++++++------------------ content/z03/encrypted-workflow.md | 18 ++ 2 files changed, 126 insertions(+), 161 deletions(-) diff --git a/content/z03/access-data.md b/content/z03/access-data.md index 244c577..8e4934e 100644 --- a/content/z03/access-data.md +++ b/content/z03/access-data.md @@ -1,7 +1,7 @@ +++ date = '2025-02-12T18:01:59+01:00' draft = false -title = 'Accessing motor assessment data collection on Windows' +title = 'Accessing motor assessment data collection' weight = 4 +++ @@ -9,11 +9,86 @@ This note will explain in detail how to set-up your local copy of the aggregated Overview of the encrypted workflow and prerequisites are explained in the [encrypted workflow](z3-encrypted-workflow/) page. It is highly recommended that you read it first. -The note assumes that you are working on Windows and using cmd shell. It also assumes that you have access to the shared Sciebo folder, and the data is being encrypted to your GPG key. +The note assumes that you have access to a shared copy of the dataset, and the data is being encrypted to your GPG key. + +## Distributed storage + +There are two components to the dataset: + +- The Git repository (which we can call a "lightweight dataset") contains information about content identity and availability, but does not contain the actual data. +- The content managed by git-annex ("annexed data"), i.e. the actual data. + +Each can be stored in / retrieved from multiple locations. +Depending on the platform used, the Git repository and annexed data can be stored and shared together (as is the case with Cerebra) or separately (JuGit + Sciebo). +The chart below illustrates the Git and git-annex repositories which can be configured as "remotes" for a local clone. +In the chart below, a square box symbolizes a Git remote, and cylinders symbolize Git-annex special remotes. +Retrieving data requires interacting with both kinds of remotes, and the main operations are shown over the lines. + +```mermaid +flowchart TD + A(Your local dataset clone) + AA["**Cerebra** (restricted)"] + AB[("**Cerebra-mask** (restricted, encrypted)")] + subgraph Deprecated + B["**JuGit** (public)"] + C[("**Sciebo** (restricted, encrypted)")] + end + D[("**INM-7 storage** (restricted, encrypted)")] + AA --- AB + A ---o |clone, update| AA + A ---o |enableremote, get| AB + A ---o |clone, update| B + A ---o |initremote, get| C + A ---o |"🚫"| D +``` + +Note that Cerebra is a special case: it could store the Git repository and its annex in a single remote. However, using encryption requires us to set up a [mask special remote](https://git-annex.branchable.com/special_remotes/mask/) for the annex, which shows separately in the list of annex remotes, but uses the same transport (think: the same credentials) as the Git remote. + +Because no data are stored in JuGit, the repository can be made public, and fetching information from JuGit requires no credentials. +However, access to the data stored in Sciebo or Cerebra-mask requires not only credentials, but also possession of a GPG key for which the data are encrypted. +Note that the Git repository also indexes data stored in INM7-storage, but that special remote is not accessible outside INM-7. + +The JuGit + Sciebo setup continues to be maintained for a transition period, but will be phased out in favour of Cerebra. + ## Before you begin -### GPG +### Cerebra token + +Instead of using your password (the one you use to log in to Cerebra), for DataLad (Git, git-annex) operations you should be using a token. Tokens are like passwords, but they are generated by the website rather than created by the user. A token can be revoked when it is no longer used or there is a risk that it got exposed. + +To generate a token: + +- Visit Cerebra and log in. +- Click your avatar in the top-right corner, and navigate to Settings → Applications. +- Under *Access tokens* click *New access token* and fill in the form: + - give the token a recognizable name; + - choose either "All" or "Specific repositories" for *Repository and organization access* (if you choose the latter, don't forget to select the repositories); + - under *Select permissions*, enable "repository: Read and write"; + - click *Generate token*. +- The token appears as a long string. Copy the generated value, and store it safely. It disappears after refreshing. + +Treat your token like a password. Do not share it with anyone. Once generated, the token can be used until revoked. + +### Git credential helper + +To avoid having to enter the token multiple times, we recommend using a Git credential helper. + +The easiest one to configure is [git-credential-cache](https://git-scm.com/docs/git-credential-cache), which temporarily (default timeout: 15 minutes) stores passwords in memory. It can be made default by setting a global Git configuration): + +``` +git config --global credential.helper cache +``` + +You may also configure a credential helper with persistent storage, for example one which interacts with your preferred password manager. You can also use OAuth (redirects to browser login and creates a temporary bearer token). Credential management is a matter of personal taste, so we can not document all possibilities here. Instead, see Git documentation about [credential helpers](https://git-scm.com/doc/credential-helpers) for a list of available options [^3]. + +### GPG (Windows) + +{{% notice style="info" %}} +This section is Windows-specific, and assumes you are using cmd shell. + +On GNU/Linux (and likely MacOS), no additional configuration is needed. +{{% /notice %}} If you install Git (which is a requirement for using DataLad), it comes with its own `gpg` executable. If you install gpg4win (which is the solution recommended for digital signatures in our docs on [Input signing](digital-signatures) it naturally comes with a separate `gpg` executable. This means it is possible to have two GPG programs... which store their data (e.g. keys) in completely different locations on your computer. @@ -35,130 +110,15 @@ Then (if it is gpg4win) tell Git to use this one. Git-annex obeys the same setti git config gpg.program PATH/TO/GPG ``` - -### GPG-agent - -Operations which use your secret key (like downloading encrypted data with git-annex or listing your secret keys with `gpg --list-secret-keys`) require interactions with a program called `gpg-agent` [^2]. Normally, `gpg-agent` should be running in the background, and there is nothing to do. - -If you notice gpg-agent is not running (e.g. `get` fails with an error message which says that gpg-agent is not running, or you can not list your secret keys), you can start the gpg-agent with: - -``` -gpg-connect-agent reloadagent /bye -``` - -### Sciebo token (app password) - -Instead of using your password (the one you use to log in to Sciebo), it is recommended to generate a token for use with DataLad & git-annex. Tokens are like passwords, but they are generated by the website rather than created by the user. A token can be revoked when it is no longer used or there is a risk that it got exposed. - -To generate a token: - -- log in to Sciebo -- click your username on the upper right and go to settings -- select Security -- scroll down to App passwords / tokens -- enter a token name and click "Create new app passcode" -- copy (and store safely) the username and token - -The token name can be anything - it is only for your information, and will be displayed in the list of tokens, with the date of last use and an option to delete (revoke) the token. - - -## WebDAV URLs - -When using DataLad & git-annex, you will connect to Sciebo through a protocol called WebDAV. For this, you need to know your WebDAV URL, which is slightly different than the URL you use to visit Sciebo's web interface. - -### Base WebDAV URL - -To find your WebDAV address: - -- go to Sciebo (file view) -- click ⚙️ Settings in the lower left corner -- a "drawer" will slide up, showing your WebDAV URL - -The WebDAV URL will look like this: - -``` -https://fz-juelich.sciebo.de/remote.php/dav/files/j.doe%40fz-juelich.de/ -``` - -### WebDAV URLs to folders - -The URL above is pointing to your home directory. To access a specific folder, you will need to add some path components manually. - -Most likely, a folder named `Z03` will be shared with you. It will contain some folders; `motor_export_data` is the relevant one. - -The folder path relative to your home directory will depend on whether you move the shared folder or not. Assuming that `Z03` is in your home directory on Sciebo, the relative paths will be `Z03/motor_export_repo` and `Z03/motor_export_data`, respectively. - -For use with DataLad, you will need to put the base URL and the relative paths together. Using the example above, -and assuming that the shared folder is in the top-level directory, the data folder URL (for initremote) is: - - -``` -https://fz-juelich.sciebo.de/remote.php/dav/files/j.doe%40fz-juelich.de/Z03/motor_export_data -``` - - -## Distributed storage - -There are two components to the dataset: - -- The Git repository (which we can call a "lightweight dataset") contains information about content identity and availability, but does not contain the actual data. -- The content managed by git-annex ("annexed data"), i.e. the actual data. - -For technical reasons, they can be stored and shared together (as is the case with Cerebra) or separately (JuGit + Sciebo). -The chart below illustrates the Git and git-annex repositories which can be configured for a local clone to access. -In the chart below, a square box symbolizes a Git remote, and cylinders symbolize Git-annex special remotes. - -```mermaid -flowchart TD - A(Your local dataset clone) - AA["**Cerebra** (restricted)"] - AB[("**Cerebra-mask** (restricted, encrypted)")] - subgraph Deprecated - B["**JuGit** (public)"] - C[("**Sciebo** (restricted, encrypted)")] - end - D[("**INM-7 storage** (restricted, encrypted)")] - AA --- AB - A ---o |clone, update| AA - A ---o |enableremote, get| AB - A ---o |clone, update| B - A ---o |initremote, get| C - A ---o |"🚫"| D -``` - -Retrieving data requires interacting with both the Git remote and the Git-annex special remote (key commands are shown over the lines). -Because no data are stored in JuGit, the repository can be made public, and fetching information from JuGit requires no credentials. -However, access to the data stored in Sciebo requires not only credentials, but also possession of a GPG key for which the data are encrypted. -Note that the Git repository also indexes data stored in INM7-storage, but that special remote is not accessible outside INM-7. - ## Initial setup -#### Let git-annex know your token - -Git-annex needs WebDAV credentials to access the stored repository and data. -It expects them to be provided via environment variables. -Set the environment variables with the `set` command (substitute your Sciebo username and token): - -``` -set WEBDAV_USERNAME=j.doe@fz-juelich.de -set WEBDAV_PASSWORD=XXXXX-XXXXX-XXXXX-XXXXX -``` - -Important: do not put the password in quotes (`""`), because `set` command treats them literally. - -Tip: you can check the value of an environment variable with `echo`, e.g. `echo %WEBDAV_USERNAME%`. - -The environment variables persist until you close your cmd. -However, these credentials will later be cached in a owner-read-only file in the clone, -so re-entering the credentials should not be needed for subsequent updates. - ### Clone the dataset -Clone the (lightweight) dataset from JuGit (using the URL below) into a directory with a name of your choice +Clone the (lightweight) dataset from Cerebra (using the URL below) into a directory with a name of your choice (here: `motor_export`): ``` -datalad clone https://jugit.fz-juelich.de/m.szczepanik/motor-assesment.git motor_export +datalad clone https://cerebra.fz-juelich.de/sfb1451-z03-internal/motor-assessment motor_export ``` Then change your working directory to go into the dataset: @@ -166,64 +126,38 @@ Then change your working directory to go into the dataset: cd motor_export ``` -### Initialize sciebo remote +### Enable the mask special remote -Git-annex special remotes represent methods of accessing data, and special remote configuration contains things like access URL or encryption type to be used. +Git-annex special remotes represent methods of transporting data. In this particular case, a *mask special remote* was set up to store encrypted data alongside the repository in Cerebra. Enabling it (and transferring data) requires having a matching GPG key. For this reason, it is not enabled automatically. -One weaknes of the Sciebo setup is that WebDAV URLs contain user names and instance names, so will be different for each user (even though the shared folder is, in principle, the same). The dataset already has a sciebo remote, but it points to FZJ's sciebo and Michał Szczepanik's user account. There is no easy way to reconfigure it, other than each user initializing a new remote pointing to *their* sciebo URL. - -The trick is to initialize a special remote, and mark it *same as* the one above (meaning that we expect to reach exactly the same place, just through different means). - -This is by far the most involved part of the setup. - -#### (optional): check available remotes -Each special remote has its own ID and assigned name, which can be used interchangeably with git-annex commands. To list remotes known to the dataset, run: +Optionally, you can start by checking the list of available special remotes. +Each special remote has its own ID and assigned name, which can be used interchangeably inside git-annex commands. To list remotes known to the dataset, run: ``` git annex info ``` -Pending future changes, the remote we want to enable is going to be named sciebo-storage, -i.e. it will be this one: +Pending future changes, the remote we want to enable is going to be named cerebra-storage, +and it should appear in the list of *semitrusted repositories* like this: ``` -325dd4e6-e71b-4512-bce6-0eb574b6ed15 -- sciebo-storage +f095040a-18b7-4648-849e-f154c53ce976 -- cerebra-mask ``` -#### Initialize your own private special remote - -As explained above, you will need to initialize a special remote for yourself, and mark it "same as" the one above. - -Here's the command broken up into multiple lines with a caret (`^`) for readability. +To enable the cerebra-mask special remote, run: ``` -git annex initremote my-sciebo-storage^ - --private^ - --sameas sciebo-storage^ - type=webdav^ - url="" +git annex enableremote cerebra-mask ``` -And this is the same as a one-liner: +You should see the following output: -``` -git annex initremote my-sciebo-storage --private --sameas sciebo-storage type=webdav url="" +``` output +enableremote cerebra-mask (encryption update) (to gpg keys: [abbreviated]) ok +(recording state in git...) ``` -Substitute the `` with your data URL (the one which ends with `motor_export_data`)! - -{{% notice style="note" %}} -In the example above, we enclosed the URL with double quotes (`"...")`. -Keep in mind that cmd distinguishes single- and double quotes and treats them differently! -Find out more in https://ss64.com/nt/syntax-esc.html -{{% /notice %}} - -#### Troubleshooting - -- If the error message complains about credentials (bearer authentication), most likely the credentials (environment variables) are incorrect. -- If the error message complains about gpg-agent not running, it is either not running or git-annex is using a different gpg than intended. See sections on gpg and gpg-agent above. -- If there is seemingly no error but data can not be downloaded, then probably the URL given to initremote was incorrect. Make sure that the URL points to `motor_export_data` (not `repo`). - +If that didn't work, see the *Troubleshooting* section below. ## Data access @@ -257,6 +191,19 @@ git log ``` +## Troubleshooting + +Operations which use your secret key (like downloading encrypted data with git-annex or listing your secret keys with `gpg --list-secret-keys`) require interactions with a program called `gpg-agent` [^2]. Normally, `gpg-agent` should be running in the background, and there is nothing to do. + +If you notice gpg-agent is not running (e.g. `enableremote` or `get` fails with an error message which says that gpg-agent is not running, or you can not list your secret keys), you can start the gpg-agent with: + +``` +gpg-connect-agent reloadagent /bye +``` + +If an operation (e.g. `enableremote` or `get`) fails with an error message which says that a private key is not available (but you know from running `gpg --list-secret-keys` that your private key is configured), or the message says gpg-agent is not running (but you know it is) it is possible that you have more than one gpg program installed, and Git-annex uses a different one that you would expect. See the *GPG (Windows)* part of the *Before you begin* section above. + + [^1]: It would, at least in theory, be possible to modify the [PATH variable](https://en.wikipedia.org/wiki/PATH_(variable)) so that running `gpg` would use the Git-bundled gpg instead of the gpg4win gpg. [^2]: You call `datalad get`, DataLad calls `git annex get`; git-annex calls `gpg --decrypt`, and gpg asks gpg-agent for a matching secret key. -[^3]: The credential name and username are stored in global (i.e. user) Git configuration, the password is stored in Windows' secret storage. +[^3]: For a more technical introduction to what Git credential helpers do, see this blog post: [Git The Stupid Password Store](https://tylercipriani.com/blog/2024/07/31/git-as-a-password-prompt/) diff --git a/content/z03/encrypted-workflow.md b/content/z03/encrypted-workflow.md index 103c316..acc8d0b 100644 --- a/content/z03/encrypted-workflow.md +++ b/content/z03/encrypted-workflow.md @@ -31,8 +31,26 @@ an e-mail signed with a trusted GPG or S/MIME signature). INF their GPG key fingerprint (which, ideally, they would have confirmed in person), and their intended access rights. +## Cerebra - data engineering platform + +We are using [Cerebra](https://cerebra.fz-juelich.de/) to exchange +encrypted data. A Cerebra account is required for access. Repositories +are grouped in an organization, in which Z03 members have administrator +privileges and can manage user permissions. Both the organization and +repositories are private, and are only accessible to designated users. + +Cerebra is a data engineering platform operated by INM-7 and hosted on +JSC Cloud at FZJ. It is powered by +[Forgejo-aneksajo](https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/wiki), +a fork of [Forgejo](https://forgejo.org/) which adds git-annex +support. + ## Sciebo - cloud storage +{{% notice style="note" %}} +With the availability of Cerebra, the use of Sciebo will be phased out. +{{% /notice %}} + We are using Sciebo () to exchange encrypted data. Z03 members would likely already have accounts via UzK/UKK () or Jülich -- 2.52.0 From f7835f54f6d72b9cf0222a583fef9cdc416e36e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Fri, 13 Nov 2026 13:40:55 +0100 Subject: [PATCH 21/25] Update deploy workflow: new runner, git-pages --- .forgejo/workflows/deploy.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index 4a714b2..11cee0a 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -1,10 +1,5 @@ name: Deploy on webserver -defaults: - run: - # hugo container only has `sh` - shell: sh - on: push: branches: @@ -12,13 +7,8 @@ on: jobs: deploy: - if: github.repository_owner == 'www' name: Build site with hugo and deploy on success - runs-on: psyinf-hugo-site-build - container: - volumes: - # for depositing the built site directly - - /home/www/srv:/www:rw + runs-on: debian-13 steps: - name: Checkout uses: https://hub.datalad.org/forgejo/datalad-clone-action@v1 @@ -26,8 +16,20 @@ jobs: recursive: true get_file_content: false + - name: Setup Hugo + uses: https://github.com/peaceiris/actions-hugo@v3 + with: + hugo-version: '0.131.0' + extended: true + - name: Generate site run: hugo - name: Deploy - run: cp -r public /www/_rdm.sfb1451.de && rm -rf /www/rdm.sfb1451.de ; mv /www/_rdm.sfb1451.de /www/rdm.sfb1451.de + uses: actions/git-pages@v2 + with: + site: https://rdm.sfb1451.de/ + server: hub.psychoinformatics.de + password: ${{ secrets.DNS_CHALLENGE_PASSWORD }} + token: ${{ forgejo.token }} + source: public/ -- 2.52.0 From 760c18cd778fd8951da5780d090c865124923eaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Fri, 13 Nov 2026 13:58:25 +0100 Subject: [PATCH 22/25] Fix deploy: password/token mutually exclusive --- .forgejo/workflows/deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index 11cee0a..41667e6 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -31,5 +31,4 @@ jobs: site: https://rdm.sfb1451.de/ server: hub.psychoinformatics.de password: ${{ secrets.DNS_CHALLENGE_PASSWORD }} - token: ${{ forgejo.token }} source: public/ -- 2.52.0 From 4ffc2d3e0dfd09e4644dc86270b87e71a56330cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Fri, 13 Nov 2026 16:26:28 +0100 Subject: [PATCH 23/25] Update relearn theme to 9.5.1 --- themes/hugo-theme-relearn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/themes/hugo-theme-relearn b/themes/hugo-theme-relearn index adb82b0..289a038 160000 --- a/themes/hugo-theme-relearn +++ b/themes/hugo-theme-relearn @@ -1 +1 @@ -Subproject commit adb82b040f7cda6de4d78892bb580a62f248d5ef +Subproject commit 289a03821aae0c5fc5dcd823f7bb000fd2b5d857 -- 2.52.0 From 5f2fd4547f0158d60fddda6abbdb2dc1b138502b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Fri, 13 Nov 2026 16:36:58 +0100 Subject: [PATCH 24/25] Bump Hugo version; act on deprecations Relearn v9.5 requires hugo 0.166.0 or newer. We shall bump to latest available as of today, which is 0.169.0. Hugo deprecated `languageCode` (now called `locale`) and `_build` (now called `build`). The relearn theme deprecated the `notice` shortcode in favour of the new `callout` shortcode. The new version allows using Markdown callouts, which are IMO nicer. --- .forgejo/workflows/deploy.yml | 2 +- content/about/_index.md | 2 +- content/catalog/datalad-based-deposition.md | 3 +- content/datalad/extensions/_index.md | 19 ++-- content/walkthroughs/publication-catalog.md | 100 +++++++++----------- content/z03/access-data.md | 14 ++- content/z03/encrypted-workflow.md | 5 +- hugo.toml | 2 +- 8 files changed, 67 insertions(+), 80 deletions(-) diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index 41667e6..ea35087 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -19,7 +19,7 @@ jobs: - name: Setup Hugo uses: https://github.com/peaceiris/actions-hugo@v3 with: - hugo-version: '0.131.0' + hugo-version: '0.169.0' extended: true - name: Generate site diff --git a/content/about/_index.md b/content/about/_index.md index 50cdedb..7438e3d 100644 --- a/content/about/_index.md +++ b/content/about/_index.md @@ -1,7 +1,7 @@ +++ title = 'Extras' -[_build] +[build] list = 'never' publishResources = true render = 'always' diff --git a/content/catalog/datalad-based-deposition.md b/content/catalog/datalad-based-deposition.md index f272b6a..5afe4f2 100644 --- a/content/catalog/datalad-based-deposition.md +++ b/content/catalog/datalad-based-deposition.md @@ -77,9 +77,8 @@ the added metadata records. ### 4. Publish the update -{{% notice style="important" %}} +> [!important] This step assumes that the project has already been published (has a configured sibling). -{{% /notice %}} Then, push the new state of the dataset to an existing sibling: diff --git a/content/datalad/extensions/_index.md b/content/datalad/extensions/_index.md index 26fc116..6f35593 100644 --- a/content/datalad/extensions/_index.md +++ b/content/datalad/extensions/_index.md @@ -25,11 +25,10 @@ and in [DataLad-NEXT documentation](https://docs.datalad.org/projects/next/en/la ![DataLad graphical user interface](gooey_pushed.png) -{{% notice style="important" %}} -Currently, the DataLad-Gooey extension is *de facto* unmaintained. -However, it was an important proof of principle, and we expect it to -make a comeback in the future; likely in a completely new guise. -{{% /notice %}} +> [!important] +> Currently, the DataLad-Gooey extension is *de facto* unmaintained. +> However, it was an important proof of principle, and we expect it to +> make a comeback in the future; likely in a completely new guise. The DataLad-Gooey (pronounce \"GUI\") package provides a graphical user interface for DataLad. It is specifically aiming at making key data @@ -49,9 +48,8 @@ documentation](https://docs.datalad.org/projects/gooey/en/latest/index.html). ## DataLad-XNAT -{{% notice style="info" %}} -The DataLad-XNAT extension is currently unmaintained. -{{% /notice %}} +> [!info] +> The DataLad-XNAT extension is currently unmaintained. [XNAT](https://www.xnat.org/) is an open source imaging informatics platform developed by the Neuroinformatics Research Group at Washington @@ -80,9 +78,8 @@ documentation](http://docs.datalad.org/projects/datalad-xnat/en/latest/). ## DataLad-REDCap {#sec-datalad-redcap} -{{% notice style="info" %}} -The DataLad-REDCap extension is currently unmaintained. -{{% /notice %}} +> [!info] +> The DataLad-REDCap extension is currently unmaintained. DataLad-REDCap extension provides convenience commands for exporting data from [REDCap](https://project-redcap.org/) into DataLad datasets. diff --git a/content/walkthroughs/publication-catalog.md b/content/walkthroughs/publication-catalog.md index 47e76bd..dc3ec35 100644 --- a/content/walkthroughs/publication-catalog.md +++ b/content/walkthroughs/publication-catalog.md @@ -4,27 +4,23 @@ draft = false title = 'Publication Catalog' +++ -{{% notice style="important" %}} - -The workflow described here remains a valid description for dataset -publication on Sciebo. However, as a means for submitting datasets to -the SFB1451 catalog, it has been deprecated in favour of the procedure -described in -[Register data in the data catalog]({{% relref "/catalog/register-data" %}}). - -If publications are to be shown in the catalog, information about them -needs to be included directly in the dataset (metadata) files, -preferably using standard metadata formats (e.g as -[citation file format references field](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#references)) -or as a [datalad-tabby publications table]({{% relref "/catalog/register-data/#table-publications" %}}). -All datasets with -project descriptions and publication metadata created according to the -workflow description below (either by INF or by respective projects) -have been republished to GitHub as the -[sfb1451/all-projects](https://github.com/sfb1451/all-projects) -superdataset. - -{{% /notice %}} +> [!important] +> The workflow described here remains a valid description for dataset +> publication on Sciebo. However, as a means for submitting datasets to +> the SFB1451 catalog, it has been deprecated in favour of the procedure +> described in [Register data in the data catalog]({{% relref "/catalog/register-data" %}}). +> +> If publications are to be shown in the catalog, information about them +> needs to be included directly in the dataset (metadata) files, +> preferably using standard metadata formats (e.g as +> [citation file format references field](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#references)) +> or as a [datalad-tabby publications table]({{% relref "/catalog/register-data/#table-publications" %}}). +> All datasets with +> project descriptions and publication metadata created according to the +> workflow description below (either by INF or by respective projects) +> have been republished to GitHub as the +> [sfb1451/all-projects](https://github.com/sfb1451/all-projects) +> superdataset. ## Publication Catalog from Sciebo: a walkthrough @@ -79,11 +75,11 @@ it: cd my-project datalad run-procedure cfg_text2git -{{% notice style="info" %}} -The `run-procedure cfg_text2git` will configure DataLad so that text -files will not be annexed -- this is a useful setting for the files we -will be adding. -{{% /notice %}} +> [!info] +> The `run-procedure cfg_text2git` will configure DataLad so that text +> files will not be annexed -- this is a useful setting for the files we +> will be adding. + ### Create a citation file @@ -135,11 +131,11 @@ DataLad: datalad save -m "Add citation file" CITATION.cff -{{% notice style="info" %}} -Citation File Format is a standard for plain text files with human- and -machine-readable information for datasets (and software). You can read -more on the [CFF website](https://citation-file-format.github.io/). -{{% /notice %}} +> [!info] +> Citation File Format is a standard for plain text files with human- and +> machine-readable information for datasets (and software). You can read +> more on the [CFF website](https://citation-file-format.github.io/). + ### Add bibliographic information to the dataset @@ -178,29 +174,27 @@ If this is the first time you are using DataLad with Sciebo, you will be prompted for your credentials. -{{% notice style="info" %}} -Sciebo uses your entire e-mail address (name@example.com) as its username. -{{% /notice %}} +> [!info] +> Sciebo uses your entire e-mail address (name@example.com) as its username. -{{% notice style="info" %}} -The `--dataset` option lets us explicitly specify the dataset for which -we configure a sibling. The `--name` option sets the name which we will -use later when publishing. The `--mode filetree` option enables filetree -mode, meaning that the sibling will have the same, human-readable, file -tree layout as the folder on your drive. - -The WebDAV url can be obtained from Sciebo\'s web UI, by clicking at -\"Settings\" in the lower left. This URL points to the user\'s home -directory, and any subfolders must be appended \"by hand\". Nonexisting -subfolders will be created. The URLs will be different for different -instances and users. For example, with FZJ\'s instance, the full URL to -the dataset folder would look like this (`` and `` are -placeholders): - -``` -https://fz-juelich.sciebo.de/remote.php/dav/files///pub-dataset -``` -{{% /notice %}} +> [!info] +> The `--dataset` option lets us explicitly specify the dataset for which +> we configure a sibling. The `--name` option sets the name which we will +> use later when publishing. The `--mode filetree` option enables filetree +> mode, meaning that the sibling will have the same, human-readable, file +> tree layout as the folder on your drive. +> +> The WebDAV url can be obtained from Sciebo\'s web UI, by clicking at +> \"Settings\" in the lower left. This URL points to the user\'s home +> directory, and any subfolders must be appended \"by hand\". Nonexisting +> subfolders will be created. The URLs will be different for different +> instances and users. For example, with FZJ\'s instance, the full URL to +> the dataset folder would look like this (`` and `` are +> placeholders): +> +> ``` +> https://fz-juelich.sciebo.de/remote.php/dav/files///pub-dataset +> ``` ### Publish to Sciebo diff --git a/content/z03/access-data.md b/content/z03/access-data.md index 8e4934e..01cfc9b 100644 --- a/content/z03/access-data.md +++ b/content/z03/access-data.md @@ -84,11 +84,10 @@ You may also configure a credential helper with persistent storage, for example ### GPG (Windows) -{{% notice style="info" %}} -This section is Windows-specific, and assumes you are using cmd shell. - -On GNU/Linux (and likely MacOS), no additional configuration is needed. -{{% /notice %}} +> [!info] +> This section is Windows-specific, and assumes you are using cmd shell. +> +> On GNU/Linux (and likely MacOS), no additional configuration is needed. If you install Git (which is a requirement for using DataLad), it comes with its own `gpg` executable. If you install gpg4win (which is the solution recommended for digital signatures in our docs on [Input signing](digital-signatures) it naturally comes with a separate `gpg` executable. This means it is possible to have two GPG programs... which store their data (e.g. keys) in completely different locations on your computer. @@ -161,9 +160,8 @@ If that didn't work, see the *Troubleshooting* section below. ## Data access -{{% notice style="note" %}} -This assumes your working directory is in the dataset (that you `cd`'ed into the dataset). -{{% /notice %}} +> [!note] +> This assumes your working directory is in the dataset (that you `cd`'ed into the dataset). To download data, use: diff --git a/content/z03/encrypted-workflow.md b/content/z03/encrypted-workflow.md index acc8d0b..936695c 100644 --- a/content/z03/encrypted-workflow.md +++ b/content/z03/encrypted-workflow.md @@ -47,9 +47,8 @@ support. ## Sciebo - cloud storage -{{% notice style="note" %}} -With the availability of Cerebra, the use of Sciebo will be phased out. -{{% /notice %}} +> [!note] +> With the availability of Cerebra, the use of Sciebo will be phased out. We are using Sciebo () to exchange encrypted data. Z03 members would likely already have accounts via diff --git a/hugo.toml b/hugo.toml index 2fe0b4e..9d218e6 100644 --- a/hugo.toml +++ b/hugo.toml @@ -1,5 +1,5 @@ baseURL = 'https://rdm.sfb1451.de/' -languageCode = 'en-us' +locale = 'en-us' title = 'SFB1451 RDM docs' theme= 'hugo-theme-relearn' -- 2.52.0 From d0b2b81596957304a89fed1d571bd80214a5085d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Fri, 13 Nov 2026 17:19:41 +0100 Subject: [PATCH 25/25] Speed up CI build with shallow theme clone The entire relearn theme repository provides for a massive checkout (about 1 min 30 s during recent builds), so we abandon the recursive datalad checkout action and initialize the submodule with depth 1. There is no annex in this repository, so we don't need more steps. --- .forgejo/workflows/deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index ea35087..1157023 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -11,10 +11,10 @@ jobs: runs-on: debian-13 steps: - name: Checkout - uses: https://hub.datalad.org/forgejo/datalad-clone-action@v1 - with: - recursive: true - get_file_content: false + uses: https://hub.psychoinformatics.de/actions/taiki-e-checkout-action@v1 + + - name: Fetch theme + run: git submodule update --init --depth 1 -- themes/hugo-theme-relearn - name: Setup Hugo uses: https://github.com/peaceiris/actions-hugo@v3 -- 2.52.0