diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml deleted file mode 100644 index 1157023..0000000 --- a/.forgejo/workflows/deploy.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Deploy on webserver - -on: - push: - branches: - - trunk - -jobs: - deploy: - name: Build site with hugo and deploy on success - runs-on: debian-13 - steps: - - name: Checkout - uses: https://hub.psychoinformatics.de/actions/taiki-e-checkout-action@v1 - - - name: Fetch theme - run: git submodule update --init --depth 1 -- themes/hugo-theme-relearn - - - name: Setup Hugo - uses: https://github.com/peaceiris/actions-hugo@v3 - with: - hugo-version: '0.169.0' - extended: true - - - name: Generate site - run: hugo - - - name: Deploy - uses: actions/git-pages@v2 - with: - site: https://rdm.sfb1451.de/ - server: hub.psychoinformatics.de - password: ${{ secrets.DNS_CHALLENGE_PASSWORD }} - source: public/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 80895ee..b25a25c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ default: image: - name: hugomods/hugo:git-0.145.0 + name: hugomods/hugo:git entrypoint: [""] variables: diff --git a/content/about/_index.md b/content/about/_index.md index 7438e3d..50cdedb 100644 --- a/content/about/_index.md +++ b/content/about/_index.md @@ -1,7 +1,7 @@ +++ title = 'Extras' -[build] +[_build] list = 'never' publishResources = true render = 'always' diff --git a/content/about/acknowledgements.md b/content/about/acknowledgements.md deleted file mode 100644 index dda7ce2..0000000 --- a/content/about/acknowledgements.md +++ /dev/null @@ -1,11 +0,0 @@ -+++ -date = '2025-04-28T21:34:35+02:00' -draft = true -title = 'Acknowledgements' -menu = 'shortcuts' -+++ - -This documentation was developed with funding from the Deutsche -Forschungsgemeinschaft (DFG, German Research Foundation) under grant -SFB 1451 ([431549029](https://gepris.dfg.de/gepris/projekt/431549029), -INF project). diff --git a/content/about/contributing.md b/content/about/contributing.md index 6dc5400..d738514 100644 --- a/content/about/contributing.md +++ b/content/about/contributing.md @@ -6,8 +6,8 @@ menu = "shortcuts" This documentation website is built with the [hugo](https://gohugo.io/) static website generator. Content of the pages is written in [markdown](https://daringfireball.net/projects/markdown/). -The repository is hosted on the [Psychoinformatics Lab Hub](https://hub.psychoinformatics.de/sfb1451/rdm.sfb1451.de) (Forgejo instance). -Patches are welcome. +The repository is hosted on [JuGit](https://jugit.fz-juelich.de/). +The project visibility is currently set to private. ## Setup @@ -15,7 +15,8 @@ After cloning the repository, you need to obtain the website theme. The theme has been added as a git submodule. ``` -git submodule update --init +git submodiule init +git submodule update ``` To build the site locally and output a URL to open in a browser to test it, run: diff --git a/content/activities/activity-log.md b/content/activities/activity-log.md index bfa1a5c..6783045 100644 --- a/content/activities/activity-log.md +++ b/content/activities/activity-log.md @@ -107,7 +107,7 @@ user interface (GUI) for DataLad, is released. **2022-08-19** PI Workshop on research data management. An opportunity to present an overview of existing systems for decentralised data -management and discuss related challenges. +management and discuss related chalenges. **2022-07-25** User interface update for Z03 data entry portal (human motor assessment centre). diff --git a/content/catalog/datalad-based-deposition.md b/content/catalog/datalad-based-deposition.md index 5afe4f2..6126477 100644 --- a/content/catalog/datalad-based-deposition.md +++ b/content/catalog/datalad-based-deposition.md @@ -78,7 +78,8 @@ the added metadata records. ### 4. Publish the update > [!important] -This step assumes that the project has already been published (has a configured sibling). +> This step assumes that the project has already been published +> (has a configured sibling). Then, push the new state of the dataset to an existing sibling: @@ -112,88 +113,31 @@ steps 1 through 3. ### 2. \[Optional\] Create the `files` table -The [files table]({{% relref "register-data#table-files" %}}) is an -optional addition to the tabby record and lists one or more files that -form part of the DataLad dataset. Such a file listing can be easily -generated based on the output of -[ls_file_collection](https://docs.datalad.org/projects/next/en/latest/generated/datalad.api.ls_file_collection.html) -provided by Datalad-next. Below, we present an example implementation, -which you can also use directly. +The [files +table]({{% relref "register-data#table-files" %}}) is +an optional addition to the tabby record and lists one or more files +that form part of the DataLad dataset. Such a file listing can be easily +generated based on the output of `datalad status` - see the +[status2tabby.py](https://github.com/sfb1451/tabby-utils/blob/main/status2tabby.py) +script, which uses the `datalad-next` extension, for an example +implementation that you can also use directly: -Note that files which are present will be checksummed; for annexed -files which are not present locally the checksum will be taken from -the annex key. This assumes that DataLad's default backend (MD5E) is -used; adjust the code for different checksums if necessary. +``` None +usage: status2tabby.py [-h] dataset outfile +positional arguments: + dataset Dataset for which files will be listed + outfile Name of tsv file to write -```python {lineNos=true, style=catppuccin-latte} -"""Generate a TSV file listing the files in a dataset - -Usage: - - python this_file.py dataset outfile - -""" - -from argparse import ArgumentParser -from csv import DictWriter -from pathlib import Path -import re -from datalad.api import ls_file_collection - - -def match_md5(key): - """Report MD5 checksum contained in annex key - - Assumes MD5E backend (DataLad's default). - https://git-annex.branchable.com/internals/key_format/ - - """ - m = re.match(r"MD5E-s\d+--([a-f0-9]{32})", key) - return m.group(1) if m is not None else None - - -def transform_result(res): - """Transform results to get required information - - Processes the outputs of ls_file_collection to get relative path, - size in bytes, and md5 checksum. For annex keys which are not - present, the checksum will be taken from the annex key, if - possible. Keys match the sfb1451 tabby specification. - - """ - path = res["item"].relative_to(res["collection"]).as_posix() - size = res["annexsize"] if res["type"] == "annexed file" else res["size"] - checksum = res.get("hash-md5") - - if checksum is None and res["type"] == "annexed file": - checksum = match_md5(res["annexkey"]) - - return {"path[POSIX]": path, "size[bytes]": size, "checksum[md5]": checksum} - - -parser = ArgumentParser() -parser.add_argument("dataset", type=Path, help="Dataset for which files will be listed") -parser.add_argument("outfile", type=Path, help="Name of the tsv file to write") -args = parser.parse_args() - -fc = ls_file_collection( - "annexworktree", - args.dataset, - hash="md5", - result_renderer="disabled", - result_xfm=transform_result, - return_type="generator", -) - -with args.outfile.open("w", encoding="utf-8", newline="") as tsvfile: - fieldnames = ["path[POSIX]", "size[bytes]", "checksum[md5]"] - writer = DictWriter(tsvfile, delimiter="\t", fieldnames=fieldnames) - writer.writeheader() - for row in fc: - writer.writerow(row) +options: + -h, --help show this help message and exit ``` +> [!INFO] +> Another (more performant) alternative is to use the +> [ls_file_collection](http://docs.datalad.org/projects/next/en/latest/generated/datalad.api.ls_file_collection.html) +> function from DataLad-next, and transform its output. + ### 3. Publish the metadata For manual deposition, copy and send all tabby records to the INF diff --git a/content/catalog/register-data.md b/content/catalog/register-data.md index a3b2948..2e32f21 100644 --- a/content/catalog/register-data.md +++ b/content/catalog/register-data.md @@ -278,7 +278,7 @@ Some additional grant information can be retrieved online during processing of records. This is currently done for projects available in GEPRIS (DFG) and CORDIS (EU). For DFG grants other than CRC1451 and its subprojects, enter \"DFG\" in the funder field and only the numeric -identifier in the identifier field. For European Commission grants, enter +identifier in the identifier field. For European Commision grants, enter the URL to their CORDIS page. ### Table [publications] diff --git a/content/data-publications/_index.md b/content/data-publications/_index.md index c6e6aef..8a5c675 100644 --- a/content/data-publications/_index.md +++ b/content/data-publications/_index.md @@ -36,4 +36,4 @@ individual data publication solution employed by CRC members. `addurls`. See for more information. This approach is also applicable to any data file that - are published as attachments to journal articles. + are published as attachements to journal articles. diff --git a/content/datalad/extensions/_index.md b/content/datalad/extensions/_index.md index 6f35593..a64f5fb 100644 --- a/content/datalad/extensions/_index.md +++ b/content/datalad/extensions/_index.md @@ -12,20 +12,11 @@ extensions](https://github.com/datalad/datalad-extensions/) GitHub page. Development of the following extensions was directly influenced by the SFB: -## DataLad-NEXT {#sec-datalad-next} - -DataLad-NEXT is a proving ground for broadly applicable novel -functionality. It offers additional functionality, better performance, -and improved user experience. - -Read more in [DataLad-NEXT github page](https://github.com/datalad/datalad-next) -and in [DataLad-NEXT documentation](https://docs.datalad.org/projects/next/en/latest/). - ## DataLad-Gooey (GUI) {#sec-datalad-gooey} ![DataLad graphical user interface](gooey_pushed.png) -> [!important] +> [!Important] > Currently, the DataLad-Gooey extension is *de facto* unmaintained. > However, it was an important proof of principle, and we expect it to > make a comeback in the future; likely in a completely new guise. @@ -48,9 +39,6 @@ documentation](https://docs.datalad.org/projects/gooey/en/latest/index.html). ## DataLad-XNAT -> [!info] -> The DataLad-XNAT extension is currently unmaintained. - [XNAT](https://www.xnat.org/) is an open source imaging informatics platform developed by the Neuroinformatics Research Group at Washington University. It facilitates common management, productivity, and quality @@ -76,14 +64,34 @@ You can read more in the [DataLad-XNAT github page](https://github.com/datalad/datalad-xnat) and in [datalad-XNAT documentation](http://docs.datalad.org/projects/datalad-xnat/en/latest/). -## DataLad-REDCap {#sec-datalad-redcap} +## DataLad-NEXT {#sec-datalad-next} -> [!info] -> The DataLad-REDCap extension is currently unmaintained. +DataLad-NEXT is a proving ground for broadly applicable novel +functionality. + +Most important for SFB is the much improved ability to deposit DataLad +datasets on any WebDAV server. This includes UKK\'s Dracoon system, as +well as Sciebo, EUDAT\'s B2DROP, and self-hosted OwnCloud or NextCloud +instances. + +The exported content can be made browsable like normal content for +non-DataLad consumers, but also remain a full dataset that can be cloned +and collaborated on. While it won\'t be as fast or flexible as with more +specialized services for Git or data hosting, this is a great approach +for dataset deposition, where only one site is uploading dataset and +file content updates, and the data consumers can choose between using +DataLad or point-and-click interfaces. See the [WebDAV +walkthrough]({{% relref "/walkthroughs/webdav/" %}}) page for a detailed overview. + +Read more in [DataLad-NEXT github +page](https://github.com/datalad/datalad-next) and in [DataLad-NEXT +documentation](https://docs.datalad.org/projects/next/en/latest/). + +## DataLad-REDCap {#sec-datalad-redcap} DataLad-REDCap extension provides convenience commands for exporting data from [REDCap](https://project-redcap.org/) into DataLad datasets. -It has not been released, but you can install a working development version +It is yet unreleased, but you can install a working development version from [DataLad-REDCap Github](https://github.com/datalad/datalad-redcap) and read [DataLad-RedCAP documentation](http://docs.datalad.org/projects/redcap/en/latest/?badge=latest). diff --git a/content/datalad/help.md b/content/datalad/help.md index 8333ea0..1b121ed 100644 --- a/content/datalad/help.md +++ b/content/datalad/help.md @@ -12,17 +12,15 @@ to use the following channels. ## Virtual DataLad office hours -We are having a virtual office hour every week. -This is a great opportunity to discuss problems and solutions with the -members of the [Psychoinformatics Group](https://www.psychoinformatics.de) at FZJ. -Check the matrix chat room for the most up-to-date meeting time and joining -link. +We are having a virtual office hour every Tuesday at 16:00. This is a +great opportunity to discuss problems and solutions with the members of +the [Psychoinformatics Group](https://www.psychoinformatics.de) at FZJ. To join: - Follow [this link to the public matrix chat room](https://matrix.to/#/!NaMjKIhMXhSicFdxAj:matrix.org) and click continue. -- The link to the online meeting will be in the channel description. +- The link to the zoom meeting will be in the channel description. - We use the matrix chat room to communicate any changes of plans if they happen, or to discuss topics in advance. Joining isn\'t necessary, but you\'re invited to pick one of the available methods diff --git a/content/walkthroughs/forgejo-encryption.md b/content/walkthroughs/forgejo-encryption.md deleted file mode 100644 index f5913de..0000000 --- a/content/walkthroughs/forgejo-encryption.md +++ /dev/null @@ -1,191 +0,0 @@ -+++ -date = '2025-06-17T18:45:00+02:00' -draft = false -title = 'Forgejo with encryption' -+++ - -This walkthrough demonstrates how to encrypt annexed file content in Forgejo, -a bare Git repository, or another Git hosting platform with annex support. - -## Introduction - -When storing DataLad datasets on third party infrastructure, Git -remotes and git-annex special remotes are often configured pointing to -different services. However, some repository hosting services -(e.g. G-node GIN, or some instances of Forgejo), as well as bare Git -repositories[^1] on systems with git-annex installed, support storing -annex objects alongside the Git repository, which makes them very -convenient in terms of configuration and usage. Throughout the -article, we will refer to remotes supporting both Git and annex -content as "git+annex". For background, see DataLad handbook's chapter -[Beyond shared -infrastructure](https://handbook.datalad.org/en/latest/basics/101-138-sharethirdparty.html). - -Some of the SFB1451 datasets may require using data encryption at rest -to guarantee data privacy or access control. Git-annex [supports -encryption](https://git-annex.branchable.com/encryption/) and it can -be very easy to set up. However, encryption is configured for *special -remotes* and as such it would not apply to such "git+annex" remotes[^2]. - -This has been addressed in [git-annex v.10.20250520](https://git-annex.branchable.com/news/version_10.20250520/) -by the addition of the [mask special remote](https://git-annex.branchable.com/special_remotes/mask/). - -With the mask special remote, setting up encrypted storage for -git+annex comes down to the following steps: - -- set up a Git remote, -- set up a mask special remote with encryption enabled, -- make sure that annexed contents are pushed only to the latter. - -The model use case for this workflow is publishing datasets on Forgejo -instances with git-annex support. [Forgejo](https://forgejo.org/) is -a "self-hosted lightweight software forge", which can be used to set -up and maintain a platform similar to GitHub or GitLab. -[Forgejo-aneksajo](https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo) -is a lightly modified version of Forgejo which adds git-annex support. -While no public instances are readily available, and the SFB1451 does -not (currently) maintain its own instance, it is nevertheless very -relevant for the SFB1451 community because Forgejo-aneksajo can be -deployed by individual groups (see [Collaborative infrastructure for a -lab: Forgejo](https://blog.datalad.org/posts/lab-infra-forgejo/) for -more background information). - -## Prerequisites - -- git-annex v.10.20250520 or newer -- gpg, and a generated private/public key -- DataLad-next (optional but highly recommended) - -## Walkthrough - -### Initial set-up and publishing - -Publishing a dataset requires having a dataset in the first place. -In this example we create a dataset with one file in Git (README) and one file annexed (an image). - -``` -datalad create foo -cd foo -echo "Hello world" > README -datalad download https://images.pexels.com/photos/21533358/pexels-photo-21533358.jpeg -datalad save --to-git -m "Add a readme" README -datalad save -m "Add a photo of a penguin" pexels-photo-21533358.jpeg -``` - -Add a Git remote pointing to a valid address -- for example, an empty Forgejo-aneksajo repository, -or (for easier testing) a local bare git repository. - -{{< tabs >}} -{{% tab title="forgejo" %}} -``` -git remote add origin https://hub.example.com/user/dataset.git -``` -{{% /tab %}} -{{% tab title="local" %}} -``` -git init --bare /tmp/foo.git -git remote add origin /tmp/foo.git -``` -{{% /tab %}} -{{< /tabs >}} - -Push (only the Git part -- we do not want to send unencrypted data) -to ensure that the remote repository is initialized with annex UUID. - -``` -datalad push --to origin --data nothing -``` - -At this point, the remote should be recognized by Git-annex, and have an annex UUID. -You can check this with `git annex info origin`. -If the UUID is present, the mask remote can be set up[^3]. -Replace the value of the `keyid` parameter with the ID (either e-mail or the numeric key ID) of your GPG key. - -``` -git annex initremote encrypted-origin type=mask remote=origin encryption=hybrid keyid=me@example.com -``` - -With the remote configuration, the annexed content and the Git content can be pushed *separately* to these remotes. - -``` -datalad push --to encrypted-origin -datalad push --data nothing --to origin -``` - -That's it! The repository is a regular repository, but the annexed content can not be accessed. -The web interface will show the file tree, but report that the files are unavailable. -Cloning the repository is possible, but getting the annexed content requires possession of a private GPG key -for which the encryption has been enabled[^4]. - -### Automating - -Remembering to push to the mask remote first, and to use `--data nothing` with the unencrypted remote -can be tedious and error prone. -However, both of these behaviors can be automated. -Git-annex wanted mechanism can be used to state that no data are wanted by the unencrypted remote. -DataLad's publication dependency can be used to say that the encrypted remote should be pushed automatically -when a push to origin is requested. - -``` -git annex wanted origin "exclude=*" -datalad siblings configure -s origin --publish-depends encrypted-origin -``` - -With this, it is sufficient to `datalad push --to origin`[^5]. - -This can be taken even further. -For example, only *some* annexed files can be pushed encrypted. -It is advisable to configure the encrypted and regular remotes with opposite settings to avoid duplicating the storage. -For example, consider a situation where raw images are stored encrypted, but jpegs are not: - -``` -git annex wanted origin include=*jpeg -git annex wanted encrypted-origin include=*arw -``` - -## Looking under the hood - -Although we used Forgejo as an example (due to its potential relevance), -the walkthrough is valid for any "git+annex" repository. Using a local -bare Git repository is a good way to "look under the hood" and see that -the annex keys in the bare repository are indeed stored encrypted, -with HMAC hashing of the filenames. - -The example below shows the annex objects directory with three files, -two encrypted and one unencrypted. - -``` -❱ tree ../foo.git/annex/objects -../foo.git/annex/objects -├── 39e -│   └── bcf -│   └── GPGHMACSHA1--2c4b73cff92c9d186065f338551f8c5c8ec94b5b -│   └── GPGHMACSHA1--2c4b73cff92c9d186065f338551f8c5c8ec94b5b -├── 727 -│   └── f52 -│   └── GPGHMACSHA1--fbcb8f7931cb90c8fa9930368bfae671f36fec6b -│   └── GPGHMACSHA1--fbcb8f7931cb90c8fa9930368bfae671f36fec6b -└── a45 - └── ab5 - └── SHA256E-s2273030--fbeaece1f05264c39781adeac159ba33fbc1f4d2aea273106600bc4e3e3aa6c7.jpeg - └── SHA256E-s2273030--fbeaece1f05264c39781adeac159ba33fbc1f4d2aea273106600bc4e3e3aa6c7.jpeg -``` - -In Forgejo, the layout would be identical. - - -[^1]: Bare Git repository is a repository which does not contain a working directory. - It can be cloned from and pushed to. - See [Git on the server](https://git-scm.com/book/en/v2/Git-on-the-Server-Getting-Git-on-a-Server) in the Pro Git book. -[^2]: Git annex does have a [git special remote](https://git-annex.branchable.com/special_remotes/git/), - but its purpose is to record a Git remote URL in the repository (normally, Git remote URL is a local configuration) - and optionally autoenable it. It does not provide a way to set up encryption. -[^3]: In Forgejo-aneksajo versions prior to 11.0-0-git-annex2, the UUID was not always assigned immediately; - see discussion in [forgejo-aneksajo/issues/22](https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/issues/22). - A workaround was to perform any action (including viewing the page) in the web interface. - If the UUID is empty, visit the repository page in a browser (or refresh it) and try `git annex info origin` again. -[^4]: In the hybrid encryption scheme, data is encrypted using a symmetric cipher, - and that cipher is asymmetrically encrypted (to selected GPG keys) and stored in the Git repository. - This means that new recipient keys can be added without having to re-encrypt the data. - See the documentation for [git annex encryption](https://git-annex.branchable.com/encryption/). -[^5]: or `git annex push` which by default pushes to all available remotes diff --git a/content/walkthroughs/publication-catalog.md b/content/walkthroughs/publication-catalog.md index dc3ec35..d5d215d 100644 --- a/content/walkthroughs/publication-catalog.md +++ b/content/walkthroughs/publication-catalog.md @@ -4,12 +4,13 @@ draft = false title = 'Publication Catalog' +++ -> [!important] +> [!CAUTION] > The workflow described here remains a valid description for dataset > publication on Sciebo. However, as a means for submitting datasets to > the SFB1451 catalog, it has been deprecated in favour of the procedure -> described in [Register data in the data catalog]({{% relref "/catalog/register-data" %}}). -> +> described in +> [Register data in the data catalog]({{% relref "/catalog/register-data" %}}). +> > If publications are to be shown in the catalog, information about them > needs to be included directly in the dataset (metadata) files, > preferably using standard metadata formats (e.g as @@ -75,12 +76,11 @@ it: cd my-project datalad run-procedure cfg_text2git -> [!info] +> [!INFO] > The `run-procedure cfg_text2git` will configure DataLad so that text > files will not be annexed -- this is a useful setting for the files we > will be adding. - ### Create a citation file Create a `CITATION.cff` file with the following fields: cff-version, @@ -131,12 +131,11 @@ DataLad: datalad save -m "Add citation file" CITATION.cff -> [!info] +> [!INFO] > Citation File Format is a standard for plain text files with human- and > machine-readable information for datasets (and software). You can read > more on the [CFF website](https://citation-file-format.github.io/). - ### Add bibliographic information to the dataset Next, add your project\'s publication information to the dataset. This @@ -174,10 +173,11 @@ If this is the first time you are using DataLad with Sciebo, you will be prompted for your credentials. -> [!info] -> Sciebo uses your entire e-mail address (name@example.com) as its username. +> [!INFO] +> Sciebo uses your entire e-mail address (name@example.com) as its +> username. -> [!info] +> [!INFO] > The `--dataset` option lets us explicitly specify the dataset for which > we configure a sibling. The `--name` option sets the name which we will > use later when publishing. The `--mode filetree` option enables filetree @@ -191,11 +191,12 @@ prompted for your credentials. > instances and users. For example, with FZJ\'s instance, the full URL to > the dataset folder would look like this (`` and `` are > placeholders): -> +> > ``` > https://fz-juelich.sciebo.de/remote.php/dav/files///pub-dataset > ``` + ### Publish to Sciebo To publish this dataset to Sciebo use: diff --git a/content/walkthroughs/read-only-sciebo-share.md b/content/walkthroughs/read-only-sciebo-share.md deleted file mode 100644 index e1416b7..0000000 --- a/content/walkthroughs/read-only-sciebo-share.md +++ /dev/null @@ -1,27 +0,0 @@ -+++ -date = '2025-08-05T15:00:00+02:00' -draft = false -title = 'Read-only public Sciebo shares' -+++ - -[Sciebo](https://hochschulcloud.nrw/) is accessible through WebDAV, which means that -git-annex's [webdav special remote](https://git-annex.branchable.com/special_remotes/webdav/) -can be used to synchronize data with Sciebo. This was explored in the -[WebDav walkthrough](webdav). - -However, git-annex used to require write access for enabling a webdav special remote, -which made it impractical for sharing data with broader audiences. This changed in -git-annex version 10.20250115. - -Additionally, the ability to push a git repository to a git-annex special remote -and clone from a special remote, previously provided in DataLad-next, is now built-in -into git-annex, as -[git-remote-annex](https://git-annex.branchable.com/git-remote-annex/). -This feature was added in git-annex version 10.20240531. -See git-annex tips: -[storing a git repository on any special remote](https://git-annex.branchable.com/tips/storing_a_git_repository_on_any_special_remote/). - -This means that it is possible to create an "all-in-one" (git + annex) deposit in Sciebo. -A walkthrough with explanations is provided in the -[Putting new git-annex features to use with Nextcloud](https://blog.datalad.org/posts/annex-nextcloud/) -post on the DataLad blog. diff --git a/content/z03/_index.md b/content/z03/_index.md index 14ae6e3..52d1491 100644 --- a/content/z03/_index.md +++ b/content/z03/_index.md @@ -1,5 +1,5 @@ +++ -title = "Z03 Motor Assessment Center" +title = "Z03 Motor Assesment Center" type = "chapter" weight = 2 +++ @@ -14,7 +14,7 @@ information is: - user: data-entry-engineer - password: Lop8-waSe-1307-Adkv -The data entry form supports the submission of a digital signature +The data entry form supports the submission of a digitial signature for each data record to verify submissions and submitter identities. The raw dataset consists of participant records stored in individual diff --git a/content/z03/access-data.md b/content/z03/access-data.md deleted file mode 100644 index 01cfc9b..0000000 --- a/content/z03/access-data.md +++ /dev/null @@ -1,207 +0,0 @@ -+++ -date = '2025-02-12T18:01:59+01:00' -draft = false -title = 'Accessing motor assessment data collection' -weight = 4 -+++ - -This note will explain in detail how to set-up your local copy of the aggregated motor assessment data, and help deal with potential configuration-dependent obstacles. - -Overview of the encrypted workflow and prerequisites are explained in the [encrypted workflow](z3-encrypted-workflow/) page. It is highly recommended that you read it first. - -The note assumes that you have access to a shared copy of the dataset, and the data is being encrypted to your GPG key. - -## Distributed storage - -There are two components to the dataset: - -- The Git repository (which we can call a "lightweight dataset") contains information about content identity and availability, but does not contain the actual data. -- The content managed by git-annex ("annexed data"), i.e. the actual data. - -Each can be stored in / retrieved from multiple locations. -Depending on the platform used, the Git repository and annexed data can be stored and shared together (as is the case with Cerebra) or separately (JuGit + Sciebo). -The chart below illustrates the Git and git-annex repositories which can be configured as "remotes" for a local clone. -In the chart below, a square box symbolizes a Git remote, and cylinders symbolize Git-annex special remotes. -Retrieving data requires interacting with both kinds of remotes, and the main operations are shown over the lines. - -```mermaid -flowchart TD - A(Your local dataset clone) - AA["**Cerebra** (restricted)"] - AB[("**Cerebra-mask** (restricted, encrypted)")] - subgraph Deprecated - B["**JuGit** (public)"] - C[("**Sciebo** (restricted, encrypted)")] - end - D[("**INM-7 storage** (restricted, encrypted)")] - AA --- AB - A ---o |clone, update| AA - A ---o |enableremote, get| AB - A ---o |clone, update| B - A ---o |initremote, get| C - A ---o |"🚫"| D -``` - -Note that Cerebra is a special case: it could store the Git repository and its annex in a single remote. However, using encryption requires us to set up a [mask special remote](https://git-annex.branchable.com/special_remotes/mask/) for the annex, which shows separately in the list of annex remotes, but uses the same transport (think: the same credentials) as the Git remote. - -Because no data are stored in JuGit, the repository can be made public, and fetching information from JuGit requires no credentials. -However, access to the data stored in Sciebo or Cerebra-mask requires not only credentials, but also possession of a GPG key for which the data are encrypted. -Note that the Git repository also indexes data stored in INM7-storage, but that special remote is not accessible outside INM-7. - -The JuGit + Sciebo setup continues to be maintained for a transition period, but will be phased out in favour of Cerebra. - - -## Before you begin - -### Cerebra token - -Instead of using your password (the one you use to log in to Cerebra), for DataLad (Git, git-annex) operations you should be using a token. Tokens are like passwords, but they are generated by the website rather than created by the user. A token can be revoked when it is no longer used or there is a risk that it got exposed. - -To generate a token: - -- Visit Cerebra and log in. -- Click your avatar in the top-right corner, and navigate to Settings → Applications. -- Under *Access tokens* click *New access token* and fill in the form: - - give the token a recognizable name; - - choose either "All" or "Specific repositories" for *Repository and organization access* (if you choose the latter, don't forget to select the repositories); - - under *Select permissions*, enable "repository: Read and write"; - - click *Generate token*. -- The token appears as a long string. Copy the generated value, and store it safely. It disappears after refreshing. - -Treat your token like a password. Do not share it with anyone. Once generated, the token can be used until revoked. - -### Git credential helper - -To avoid having to enter the token multiple times, we recommend using a Git credential helper. - -The easiest one to configure is [git-credential-cache](https://git-scm.com/docs/git-credential-cache), which temporarily (default timeout: 15 minutes) stores passwords in memory. It can be made default by setting a global Git configuration): - -``` -git config --global credential.helper cache -``` - -You may also configure a credential helper with persistent storage, for example one which interacts with your preferred password manager. You can also use OAuth (redirects to browser login and creates a temporary bearer token). Credential management is a matter of personal taste, so we can not document all possibilities here. Instead, see Git documentation about [credential helpers](https://git-scm.com/doc/credential-helpers) for a list of available options [^3]. - -### GPG (Windows) - -> [!info] -> This section is Windows-specific, and assumes you are using cmd shell. -> -> On GNU/Linux (and likely MacOS), no additional configuration is needed. - -If you install Git (which is a requirement for using DataLad), it comes with its own `gpg` executable. If you install gpg4win (which is the solution recommended for digital signatures in our docs on [Input signing](digital-signatures) it naturally comes with a separate `gpg` executable. This means it is possible to have two GPG programs... which store their data (e.g. keys) in completely different locations on your computer. - -What's more, in this case, git-annex operations will default to using the one which came with Git, while user operations (be it via Kleopatra GUI or the command line) will more than likely[^1] use the one which came with gpg4win. This can become confusing. Fortunately, git-annex behaviour is configurable. The goal here is to have both the user and git-annex interact with the same `gpg` executable. - -If you do not have gpg4win: you probably do not need to do anything. - -If you have gpg4win and used it to generate (or import) your private key, you need to configure Git and git-annex to use this one. - -First, find out the path to the gpg executable. The path components would tell you whether it came with Git or gpg4win - the latter is more likely. - -``` -where gpg -``` - -Then (if it is gpg4win) tell Git to use this one. Git-annex obeys the same setting. Replace `PATH/TO/GPG` with the path reported by the previous command. - -``` -git config gpg.program PATH/TO/GPG -``` - -## Initial setup - -### Clone the dataset - -Clone the (lightweight) dataset from Cerebra (using the URL below) into a directory with a name of your choice -(here: `motor_export`): - -``` -datalad clone https://cerebra.fz-juelich.de/sfb1451-z03-internal/motor-assessment motor_export -``` - -Then change your working directory to go into the dataset: -``` -cd motor_export -``` - -### Enable the mask special remote - -Git-annex special remotes represent methods of transporting data. In this particular case, a *mask special remote* was set up to store encrypted data alongside the repository in Cerebra. Enabling it (and transferring data) requires having a matching GPG key. For this reason, it is not enabled automatically. - -Optionally, you can start by checking the list of available special remotes. -Each special remote has its own ID and assigned name, which can be used interchangeably inside git-annex commands. To list remotes known to the dataset, run: - -``` -git annex info -``` - -Pending future changes, the remote we want to enable is going to be named cerebra-storage, -and it should appear in the list of *semitrusted repositories* like this: - -``` -f095040a-18b7-4648-849e-f154c53ce976 -- cerebra-mask -``` - -To enable the cerebra-mask special remote, run: - -``` -git annex enableremote cerebra-mask -``` - -You should see the following output: - -``` output -enableremote cerebra-mask (encryption update) (to gpg keys: [abbreviated]) ok -(recording state in git...) -``` - -If that didn't work, see the *Troubleshooting* section below. - -## Data access - -> [!note] -> This assumes your working directory is in the dataset (that you `cd`'ed into the dataset). - -To download data, use: - -``` -datalad get output -``` - -To remove your local copy of the data, use `drop` which does the opposite of `get`: - -``` -datalad drop output -``` - -To obtain the latest version of the data, use: - -``` -datalad update --how merge -datalad get output -``` - -To see when the data (according to your local state) last changed, use `git log` (up/down arrows scroll, `Q` quits): - -``` -git log -``` - - -## Troubleshooting - -Operations which use your secret key (like downloading encrypted data with git-annex or listing your secret keys with `gpg --list-secret-keys`) require interactions with a program called `gpg-agent` [^2]. Normally, `gpg-agent` should be running in the background, and there is nothing to do. - -If you notice gpg-agent is not running (e.g. `enableremote` or `get` fails with an error message which says that gpg-agent is not running, or you can not list your secret keys), you can start the gpg-agent with: - -``` -gpg-connect-agent reloadagent /bye -``` - -If an operation (e.g. `enableremote` or `get`) fails with an error message which says that a private key is not available (but you know from running `gpg --list-secret-keys` that your private key is configured), or the message says gpg-agent is not running (but you know it is) it is possible that you have more than one gpg program installed, and Git-annex uses a different one that you would expect. See the *GPG (Windows)* part of the *Before you begin* section above. - - -[^1]: It would, at least in theory, be possible to modify the [PATH variable](https://en.wikipedia.org/wiki/PATH_(variable)) so that running `gpg` would use the Git-bundled gpg instead of the gpg4win gpg. -[^2]: You call `datalad get`, DataLad calls `git annex get`; git-annex calls `gpg --decrypt`, and gpg asks gpg-agent for a matching secret key. -[^3]: For a more technical introduction to what Git credential helpers do, see this blog post: [Git The Stupid Password Store](https://tylercipriani.com/blog/2024/07/31/git-as-a-password-prompt/) diff --git a/content/z03/digital-signatures/_index.md b/content/z03/digital-signatures/_index.md index fe55e68..f2880dc 100644 --- a/content/z03/digital-signatures/_index.md +++ b/content/z03/digital-signatures/_index.md @@ -2,7 +2,7 @@ date = '2025-02-12T18:01:06+01:00' draft = false title = 'Digital signatures' -weight = 2 +weight = 1 +++ The data entry form supports the submission of a digital signature for @@ -97,7 +97,7 @@ rights. ### Preparation -1. Open a terminal window by starting Applications -\> Utilities -\> +1. Open a terminal window by starting Applications -\> Utitities -\> Terminal.app. 2. Execute the command: diff --git a/content/z03/encrypted-workflow.md b/content/z03/encrypted-workflow.md index 936695c..e2e9b60 100644 --- a/content/z03/encrypted-workflow.md +++ b/content/z03/encrypted-workflow.md @@ -2,7 +2,7 @@ date = '2025-02-12T18:01:59+01:00' draft = false title = 'Encrypted workflow' -weight = 1 +weight = 3 +++ We are using DataLad to track and move around the motor assessment @@ -16,7 +16,7 @@ to download data, decryption is handled automatically. ## GPG - signing and encryption -To get access to the motor assessment collection and / or to have their +To get access to the motor assesment collection and / or to have their additions marked as \"trusted\", a Z03 member needs to: 1. create a GPG key pair (start with the signing instructions linked @@ -31,25 +31,8 @@ an e-mail signed with a trusted GPG or S/MIME signature). INF their GPG key fingerprint (which, ideally, they would have confirmed in person), and their intended access rights. -## Cerebra - data engineering platform - -We are using [Cerebra](https://cerebra.fz-juelich.de/) to exchange -encrypted data. A Cerebra account is required for access. Repositories -are grouped in an organization, in which Z03 members have administrator -privileges and can manage user permissions. Both the organization and -repositories are private, and are only accessible to designated users. - -Cerebra is a data engineering platform operated by INM-7 and hosted on -JSC Cloud at FZJ. It is powered by -[Forgejo-aneksajo](https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/wiki), -a fork of [Forgejo](https://forgejo.org/) which adds git-annex -support. - ## Sciebo - cloud storage -> [!note] -> With the availability of Cerebra, the use of Sciebo will be phased out. - We are using Sciebo () to exchange encrypted data. Z03 members would likely already have accounts via UzK/UKK () or Jülich diff --git a/content/z03/local-copies.md b/content/z03/local-copies.md index 4eb8800..9a58820 100644 --- a/content/z03/local-copies.md +++ b/content/z03/local-copies.md @@ -2,7 +2,7 @@ date = '2025-02-12T18:01:47+01:00' draft = false title = 'Local copies' -weight = 3 +weight = 2 +++ The contents of all form fields (except signature) can be saved diff --git a/hugo.toml b/hugo.toml index 9d218e6..2fe0b4e 100644 --- a/hugo.toml +++ b/hugo.toml @@ -1,5 +1,5 @@ baseURL = 'https://rdm.sfb1451.de/' -locale = 'en-us' +languageCode = 'en-us' title = 'SFB1451 RDM docs' theme= 'hugo-theme-relearn' diff --git a/themes/hugo-theme-relearn b/themes/hugo-theme-relearn index 289a038..adb82b0 160000 --- a/themes/hugo-theme-relearn +++ b/themes/hugo-theme-relearn @@ -1 +1 @@ -Subproject commit 289a03821aae0c5fc5dcd823f7bb000fd2b5d857 +Subproject commit adb82b040f7cda6de4d78892bb580a62f248d5ef