Merge branch 'linkchecker' into 'develop'

add linkchecker

See merge request R3/howto-cards!286

.ci/generateIndex.py

 import os, re
 from os import path
 from natsort import natsorted
+from pathlib import Path
 
 def line_prepender(filename, line):
     with open(filename, 'r+') as f:
@@ -100,6 +101,7 @@ sections = natsorted(sections)
 # Index contains the generated content, init it with an empty container
 index = ''
 index += '\n<div class="index-box-container">\n'
+whiteList = ''
 
 localIndexArr = [[]] * len(sections)
 for folder in cardDirs:
@@ -157,6 +159,12 @@ for folder in cardDirs:
                                 header += "  - /" + folder + "/cards/" + shortcut + "\n"
                                 header += "---"
 
+                                # add autogenerated links to whitelist
+                                whiteList += permalink + "\n"
+                                whiteList += "/?" + shortcut + "\n"
+                                whiteList += "/cards/" +shortcut + "\n"
+                                whiteList += "/" + folder + "/cards/" + shortcut + "\n"
+
                                 # add the header properly speaking
                                 line_prepender(fileName, header)
 
@@ -197,7 +205,7 @@ for s in sections:
     index += build_section_end()
     k += 1
 
-# Close the container
+# close the container
 index += "\n</div>"
 
 ## add link to return to main index
@@ -226,3 +234,13 @@ with open(indexFile, 'w') as file:
   file.write(filedata)
 
 print("\n > New index generated and saved in " + indexFile)
+
+# write link whitelist out
+whiteListFile = ".ci/whitelist.txt"
+if Path(whiteListFile).exists():
+    with open(whiteListFile, 'r') as file :
+        for line in file:
+            whiteList += line
+
+with open(whiteListFile, 'w') as file:
+    file.write(whiteList)

.ci/whitelist.txt

+/privacy-policy
+/search
+https://www.dev47apps.com/
+https://www.dev47apps.com/droidcam/connect/
+https://cerbere.uni.lu/
+http://iptel.uni.lux/
+javascript:%20showBanner();
+javascript:showBanner();
+media/7zip-encryption-windows.mp4?width=400
+media/Get_MD5_checksum_windows.mp4

.gitignore

@@ -42,4 +42,6 @@ handbook/
 internal/handbook
 internal/handbook-additional
 internal/handbook-annexes
-!internal/covid-19/exit-strategy/*.pdf
+!internal/covid-19/exit-strategy/*.pdf
\ No newline at end of file
+checkFolder
+.cache/

.gitlab-ci.yml

+
+
 stages:
   - prepare
   - save
   - build
+  - check
   - generate
   - deploy
   - trigger
@@ -26,6 +29,7 @@ prepare:index:
     - python .ci/generateIndex.py
     - mkdir .tmp
     - cp cards.md .tmp/.
+    - cp .ci/whitelist.txt .tmp/.
   artifacts:
     expire_in: 1 day
     paths:
@@ -77,27 +81,49 @@ build:pages:
     expire_in: 1 day
     paths:
       - build
+      - .tmp
   rules:
     - if: $CI_COMMIT_REF_NAME
-    - if: $CI_MERGE_REQUEST_ID
   before_script:
     - apt-get -qq update
     - apt-get install -y -qq git-lfs
     - gem install bundler:$BUNDLER_VERSION && bundle install
   script:
     # Generate the configuration for forks (will use Gitlab Pages on personal namespaces)
-    - 'echo "url: https://$CI_PROJECT_NAMESPACE.$CI_PAGES_DOMAIN" >> .ci/_config_gitlab_pages.yml'
-    - 'echo "baseurl: /$CI_PROJECT_NAME" >> .ci/_config_gitlab_pages.yml'
-
-    # If there is no config for the current branch, use the one for Gitlab Pages
-    - 'if [ ! -f ".ci/_config_$CI_COMMIT_REF_NAME.yml" ]; then mv .ci/_config_gitlab_pages.yml .ci/_config_$CI_COMMIT_REF_NAME.yml; fi'
-
+    - |
+      if [ $CI_MERGE_REQUEST_ID ]; then
+          export CI_COMMIT_REF_NAME="develop";
+      else
+          echo "url: https://$CI_PROJECT_NAMESPACE.$CI_PAGES_DOMAIN" >> .ci/_config_gitlab_pages.yml;
+          echo "baseurl: /$CI_PROJECT_NAME" >> .ci/_config_gitlab_pages.yml;
+          # If there is no config for the current branch, use the one for Gitlab Pages
+          if [ ! -f ".ci/_config_$CI_COMMIT_REF_NAME.yml" ]; then
+              mv .ci/_config_gitlab_pages.yml .ci/_config_$CI_COMMIT_REF_NAME.yml;
+          fi
+      fi
     # Display, which configuration is used
-    - 'echo "Configuration: " && cat ".ci/_config_$CI_COMMIT_REF_NAME.yml"'
-
+    - |
+      echo "Configuration: " && cat ".ci/_config_$CI_COMMIT_REF_NAME.yml"
     # Run Jekyll with custom configuration
     - bundle exec jekyll build -d build --config "_config.yml,.ci/_config_$CI_COMMIT_REF_NAME.yml"
 
+# check
+# ------------------------------------------------------------------------------------
+
+check:links:
+   stage: check
+   image: $CI_REGISTRY/r3/apps/tailorbird/linkchecker
+   rules:
+     - if: $CI_COMMIT_REF_NAME
+     - if: $CI_MERGE_REQUEST_ID
+   allow_failure: true
+   before_script:
+     - cp .tmp/whitelist.txt build/.
+     - cp -r build /check
+     - cd /
+   script:
+     - python link_check.py
+
 # generate
 # ------------------------------------------------------------------------------------
 
@@ -163,4 +189,4 @@ trigger:
   tags:
     - privileged
   script:
-    - curl --silent --output /dev/null -X POST -F token=$INTERNAL_TRIGGER_TOKEN -F ref=$CI_COMMIT_BRANCH $INTERNAL_REPO
+    - curl --silent --output /dev/null -X POST -F token=$INTERNAL_TRIGGER_TOKEN -F ref=$CI_COMMIT_BRANCH $INTERNAL_REPO
\ No newline at end of file

_config.yml

@@ -19,8 +19,8 @@ description: >- # this means to ignore newlines until "baseurl:"
   This page is an index for lab cards  that are intended to provide practical guidance  in  implementing Data Management, Data Protection and IT setup.
 
 # URL settings (the most difficult part, please refer to the guide)
-baseurl: "/"  # the subpath of your site, e.g. /gitlab-repository-name
-url: "https://localhost"  # the base hostname & protocol for your site, e.g. http://gitlab-namespace-name.pages.uni.lu/
+baseurl: "/howto-cards"  # the subpath of your site, e.g. /gitlab-repository-name
+url: "https://howto.lcsb.uni.lu"  # the base hostname & protocol for your site, e.g. http://gitlab-namespace-name.pages.uni.lu/
 
 # Banner settings
 banner: howto-card  # When you have custom images, change this setting's value to the name of the folder containing them

external/access/lums-passwords/lums-passwords.md

@@ -31,7 +31,7 @@ In case you are not a member of the university, please ask your collaborator at
 
 ## (Re-)Activating LUMS account upon receiving credentials or after password reset
 
-The password you will receive from the system administrators is temporary and **it will expire after one month**, so you have to reset it as soon as possible. It is usually sent as a link to [PrivateBin](https://howto.lcsb.uni.lu/?passwords). **This link is valid for one week and is deleted after you see it once**, thus you need to keep it open until you have followed these steps to reset your password:
+The password you will receive from the system administrators is temporary and **it will expire after one month**, so you have to reset it as soon as possible. It is usually sent as a link to [PrivateBin]({{ '/?access:passwords' | relative_url }}). **This link is valid for one week and is deleted after you see it once**, thus you need to keep it open until you have followed these steps to reset your password:
 
   1. Go to [lums.uni.lu](https://lums.uni.lu) and login with your LUMS username and the temporary password you received.<br>**NOTE:** If you experience any issues at this step, try to use *Chrome* browser for this procedure.<br/><img src="img/lums_login.png" height="200px"><br/>
   Should the login look different from this, e.g. like a pop-up, click on **x** or **Cancel** to close it.
@@ -43,7 +43,7 @@ The password you will receive from the system administrators is temporary and **
   5. Enter a new password of your choice in the field **"New Password"** and repeat the same password in the field **"Verify Password"**. This is the actual password you need to use from now on, so make sure to remember it.
   6. Click on **"Reset Password and Login"**.
 
-It is a good practice to use a password manager which will not only help you to keep track of your passwords so you will never forget it, but it will also help you to generate passwords which are very safe (see [HowTo card](https://howto.lcsb.uni.lu/?passwords) on password management for more detail).
+It is a good practice to use a password manager which will not only help you to keep track of your passwords so you will never forget it, but it will also help you to generate passwords which are very safe (see [HowTo card]({{ '/?access:passwords' | relative_url }}) on password management for more detail).
 
 ## Changing password for LUMS accounts
 If you want to change your password for the LUMS account, you can:

external/contribute/git-clients/git-clients.md

@@ -8,7 +8,7 @@ redirect_from:
 ---
 # Git clients
 
-Installation instructions are provided [here](https://howto.lcsb.uni.lu/?contribute:install-git).
+Installation instructions are provided [here]({{ '/?contribute:install-git' | relative_url }}).
 
 ## GUI clients
 

external/contribute/install-git/install-git.md

@@ -9,7 +9,7 @@ redirect_from:
 
 # Installation of Git
 
-Several git clients are presented [here](https://howto.lcsb.uni.lu/?contribute:git-clients).
+Several git clients are presented [here]({{ '/?contribute:git-clients' | relative_url }}).
 
 For most of the git clients, `git` is actually required to be installed on the system. You can find installers for Windows, MacOS and Linux on the [Git webpage](https://git-scm.com/downloads).
 
@@ -17,4 +17,4 @@ If you are using Visual Studio Code, and if you want to contribute to a reposito
 ```bash
 ssh git@gitlab.lcsb.uni.lu -p8022
 ```
-Then, press Enter and type `yes`. This will add the git-r3lab server to the known hosts on your system.
+Then, press Enter and type `yes`. This will add the Gitlab server to the known hosts on your system.
\ No newline at end of file

external/contribute/markdown/markdown.md

@@ -132,4 +132,4 @@ Markdown file can be written in any text editor. There are many editors supporti
   * ByWord
 * Online
   * [GitHub pages](https://pages.github.com/)
-  * [stackedit.io](stackedit.io)
+  * [stackedit.io](https://stackedit.io)

external/contribute/vscode/vscode.md

@@ -59,7 +59,7 @@ For each how-to card, it is strongly recommended to create a new branch.
 
 ## Step 4: Write the procedure in Markdown
 
-If you need more details on how to do this, refer to the [corresponding howto-card](.../write-markdown).
+If you need more details on how to do this, refer to the [corresponding howto-card](/?contribute:markdown).
 
 ## Step 5: Commit your changes
 

external/contribute/web-ide/web-ide.md

@@ -13,7 +13,10 @@ This shows how to use Gitlab Web IDE to add or edit a howto-card:
 
 Before you start, please make sure that you have the correct access rights to edit the repository.
 
-*Note: If you do not have the rights to edit the repository, you can either ask for permissions by contacting the repository maintainers, or use your own copy of the repository (called a `fork`). Please follow [these instructions](https://docs.gitlab.com/ee/user/project/repository/forking_workflow.html#creating-a-fork) to obtain your copy (fork). You can then [mirror the fork automatically](/cards/contribute:mirror-fork) to keep in sync.*
+*Note: If you do not have the rights to edit the repository, you can either ask for permissions by contacting
+the repository maintainers, or use your own copy of the repository (called a `fork`).
+Please follow [these instructions](https://docs.gitlab.com/ee/user/project/repository/forking_workflow.html#creating-a-fork) to
+obtain your copy (fork). You can then [mirror the fork automatically]({{ '/?contribute:mirror-fork' | relative_url }}) to keep in sync.*
 
 A clear difference between editing a file in a git-tracked repository compared to a live document is that you can make an edit directly, and then suggesting the change. You can make this edit on your own version (called a `branch`), and then request to merge your changes into the main repository (through opening a `merge request`).
 

external/exchange-channels/asperaweb/asperaweb.md

@@ -85,7 +85,7 @@ You can use the official [IBM Aspera Diagnostic Tool](https://test-connect.asper
 
 ### **Using Microsoft Edge browser**
 
-Microsoft Edge browser requires to download and install [IBM Aspera Connect for Edge](https://www.microsoft.com/store/productId/9N6XL57H8BMG).
+Microsoft Edge browser requires to download and install [IBM Aspera Connect for Edge](https://microsoftedge.microsoft.com/addons/detail/ibm-aspera-connect/kbffkbiljjejklcpnfmoiaehplhcifki).
 
 ### **UDP/TCP port and firewall**
 

external/exchange-channels/owncloud/owncloud.md

@@ -44,7 +44,7 @@ When sharing research data, one should observe the following guidance using Ownc
 
 ## Installing Owncloud Desktop Client
 
-**Important:** OwnCloud is fetching information from LUMS, therefore changes like a password reset (see How-to cards [Changing password for LUMS accounts](https://howto.lcsb.uni.lu/?lums-passwords)) should be performed **only** via LUMS!
+**Important:** OwnCloud is fetching information from LUMS, therefore changes like a password reset (see How-to cards [Changing password for LUMS accounts]({{ '/?lums-passwords' | relative_url }}) should be performed **only** via LUMS!
 
 1. Please try following link to download the owncloud desktop client depending on your operating system (this is an example for Windows, similar steps for other OS) -
 

external/general/remote-working/remote-working.md

@@ -11,7 +11,7 @@ redirect_from:
 In case you need to work remotely, there are several tools and services that you can use to
 stay connected and continue your work.
 
-An important reference is the [list of important links](https://howto.lcsb.uni.lu/?general:links).
+An important reference is the [list of important links]({{ '/?general:links' | relative_url }}).
 
 <p style="color:grey"><b>Note:</b> If the link does not work, please refer to the section <b>Connecting to the university network</b> since you are most probably not connected to the university internal network.</p>
 
@@ -20,12 +20,12 @@ There is also a set of [tutorials](https://remote.uni.lu/) prepared by the unive
 ## Connecting to the university network
 
 If you want to access the university infrastructure, such as the servers or storage, you can do so by connecting
-via [Virtual Private Network (VPN)](http://vpn.uni.lu). All details are given [here](https://howto.lcsb.uni.lu/?access:vpn-cerbere-access).
+via [Virtual Private Network (VPN)](http://vpn.uni.lu). All details are given [here]({{ '/?access:vpn-cerbere-access' | relative_url }}).
 
 ## Instant messaging
 
 One of the recommended tools to communicate is Slack. Each research group can sign up for free and
-create their own Slack instance on [slack.com](www.slack.com).
+create their own Slack instance on [slack.com](https://www.slack.com).
 
 ## Video conferencing and virtual meetings
 
@@ -43,7 +43,7 @@ In case you want to do video conferences, but do not have a webcam, you can use
 
 For Android phones you can use [DroidCam Wireless Webcam](http://www.dev47apps.com/). It comes with a desktop client for Windows and Linux, but also works on MacOS without the client.
 
-On Apple iPhones (and Android as well) you can use [EpocCam](http://www.kinoni.com/). This requires you to install drivers on your computer as well.
+On Apple iPhones (and Android as well) you can use [EpocCam](https://www.elgato.com/en/epoccam). This requires you to install drivers on your computer as well.
 
 Both apps support connecting to your computer either via WiFi or USB. EpocCam supports USB connection only for iOS, though. With DroidCam, you need to [install ADB, enable developer options and USB debugging](http://www.dev47apps.com/droidcam/connect/), if you want to connect your phone via USB.
 
@@ -65,8 +65,8 @@ You can use the following tools to take notes or writing manuscripts together:
 
 When on-the-go or working remotely, the [LCSB ownCloud](https://owncloud.lcsb.uni.lu) is the best way to share and work collaboratively on files.
 
-More details on how to set up ownCloud are provided [here](https://howto.lcsb.uni.lu/?exchange-channels:owncloud). You can share your files and folders
-directly with your collaborators by following [these instructions](https://howto.lcsb.uni.lu/?exchange-channels:owncloud:share-files).
+More details on how to set up ownCloud are provided [here]({{ '/?exchange-channels:owncloud' | relative_url }}). You can share your files and folders
+directly with your collaborators by following [these instructions]({{ '/?exchange-channels:owncloud:share-files' | relative_url }}).
 
 ## Development of code
 

external/integrity/encryption/cloud/cloud.md

@@ -40,7 +40,7 @@ Download and install `7zip` from [here](https://www.7-zip.org/)
 3. Enter a password and make a copy of the encryption password :key:
 4. Click save to save the encrypted zip file
 
-![](./media/7zip-encryption-windows.mp4?width=400)
+![](media/7zip-encryption-windows.mp4?width=400)
 
 ## Calculate the MD5 checksum with `Simple Checksum Calculator`
 
@@ -49,7 +49,7 @@ Download and install `7zip` from [here](https://www.7-zip.org/)
 3. Click on the file icon box in the left panel and load the encrypted zip file
 4. Make a copy of the MD5 checksum key :label:
 
-![](./media/Get_MD5_checksum_windows.mp4)
+![](media/Get_MD5_checksum_windows.mp4)
 
 ## Data upload
 

external/integrity/encryption/disk/encryption_disk.md

@@ -9,7 +9,7 @@ redirect_from:
 # Encrypting the Startup Disk for Your Laptop/Desktop
 Encrypting an entire hard disk is an effective protective measure against computer theft and loss. In this lab card we provide instructions for switching on disk encryption on macOS and Windows platforms.
 
-**IMPORTANT NOTICE:** One important requirement of using Encryption is that you must  manage your Encryption Passwords/Keys. Failing to do so will mean **loosing your data indefinitely**. In this [link](https://howto.lcsb.uni.lu/?access:passwords) we list  tools that can be used for password management. **Please make sure you have arrangements for password management before starting the encryption of data**.
+**IMPORTANT NOTICE:** One important requirement of using Encryption is that you must  manage your Encryption Passwords/Keys. Failing to do so will mean **loosing your data indefinitely**. In this [link]({{ '/?access:passwords' | relative_url }}) we list  tools that can be used for password management. **Please make sure you have arrangements for password management before starting the encryption of data**.
 
 
 ## macOS

external/integrity/encryption/file/encryption_file.md

@@ -9,7 +9,7 @@ redirect_from:
 # Encrypting Files and Folders
 Encryption is an effective measure to protect sensitive data. In this lab card we provide instructions for file/folder encryption on platforms commonly used by LCSB staff.
 
-**IMPORTANT NOTICE:** One important requirement of using Encryption is that you must  manage your Encryption Passwords/Keys. Failing to do so will mean **loosing your data indefinitely**. In this [link](https://howto.lcsb.uni.lu/?access:passwords) we list  tools that can be used for password management. **Please make sure you have arrangements for password management before starting the encryption of data**.
+**IMPORTANT NOTICE:** One important requirement of using Encryption is that you must  manage your Encryption Passwords/Keys. Failing to do so will mean **loosing your data indefinitely**. In this [link]({{ '/?access:passwords' | relative_url }}) we list  tools that can be used for password management. **Please make sure you have arrangements for password management before starting the encryption of data**.
 
 ## macOS
 The built-in mechanism for file-level encryption on a Mac is  Encrypted Disk Images (*.dmg* files). In order to create a disk image:

external/integrity/naming/file_naming.md

@@ -16,7 +16,7 @@ Good file name follows three basic principles:
 * human readable
 * plays well with default ordering
 
-If you are looking for information on how to organize and structure your folders, you may find this [dedicated card](https://howto.lcsb.uni.lu/?integrity:organization) helpful.
+If you are looking for information on how to organize and structure your folders, you may find this [dedicated card]({{ '/?integrity:organization' | relative_url }}) helpful.
 
 ## Machine readable
 
@@ -56,7 +56,7 @@ Beware of typos and avoid using multiple names varying in small ways unless it h
 ## Exploiting machine readable names
 
 You may already have a lot of files collected for your project or you have received big dataset from one of your collaborators. Then you might think about organizing and renaming them to be compliant with your new or existing naming policy.
-If the names are consistent and you don't want to loose time renaming them by hand, you may try to use dedicated tools (e.g. [PSRenamer](https://www.powersurgepub.com/products/psrenamer/index.html)) or simple commands in your command line (**rename** for Mac and Linux, **ren** for Windows).
+If the names are consistent and you don't want to loose time renaming them by hand, you may try to use dedicated tools (e.g. [PSRenamer](https://github.com/hbowie/psrenamer) or simple commands in your command line (**rename** for Mac and Linux, **ren** for Windows).
 
 Once your skills develop, you will be able to use machines and machine readable file names to perform advanced operations on them, e.g. search using regular expression.
 Imagine folder with thousands of files. Running simple R command

external/integrity/organization/organization.md

@@ -21,7 +21,7 @@ able to find back the files easily.
 A key rule is to name the folders with straightforward but explicit names, and to include the decision tree of where to put a given file inside the folder structure. In fact, the directory tree should in some sense
 reflect the decision tree.
 
-More details on naming files and folders are provided in the dedicated [file naming card](https://howto.lcsb.uni.lu/?integrity:naming).
+More details on naming files and folders are provided in the dedicated [file naming card]({{ '/?integrity:naming' | relative_url }}).
 
 ## Suggestions
 

external/integrity/spreadsheets/spreadsheets.md

@@ -42,7 +42,7 @@ For reproducibility purposes, the collected data should be always exported from
 
 ### Table
 
-- Keep header column names machine readable. You can follow the same best practices as for file naming (see our [Card on file naming](../naming/file_naming.md)).
+- Keep header column names machine readable. You can follow the same best practices as for file naming (see our [Card on file naming]({{ '/?integrity:naming' | relative_url }})).
 - Keep values in columns atomic.
 - Use primary keys - values in one particular column should be unique for the whole table. This will allow you to create unique references pointing to one and only one observation/record.
 - Do not insert empty rows or columns which would split the table in two.