generateIndex.py

import os, re
from os import path
from natsort import natsorted
from pathlib import Path

def line_prepender(filename, line):
    with open(filename, 'r+') as f:
        content = f.read()
        f.seek(0, 0)
        f.write(line.rstrip('\r\n') + '\n' + content)

def build_link(title, href):

    # strip the number of the title
    if ('handbook' in href):
        title = re.sub('[0-9.]', '', title).strip()

    # add relative url
    href = "{{ '" + href + "' | relative_url }}"

    return f'\t\t\t<li><a href="{href}">{title}</a></li>\n'

def root_handbook(href):
    # remove sym link reference in href
    if ('handbook-additional' in href):
        href = href.replace('handbook-additional', 'handbook').strip()
    elif ('handbook-annexes' in href):
        href = href.replace('handbook-annexes', 'handbook').strip()

    return href

def build_section_start(title, shortcut):

    title = title.replace("Gdpr", "GDPR")
    title = title.replace("Handbook additional", "PI/Supervisor specifics")
    title = title.replace("Handbook annexes", "Annexes")
    title = title.replace("Covid 19", "COVID-19")
    title = title.replace("Ppc", "PPC")

    return f'\n\t<div class="index-box noborderbox" id="{shortcut}-card">\n\t\t<h3>{title}</h3>\n\t\t<ul>\n'

def build_section_end():
    return "\t\t</ul>\n\t</div>"

def save_order(localroot, root, filename):
    order = ""
    os.chdir(root)

    with open(filename, 'r') as f:
        for line in f:
            # check for the start of the section
            #print(line)
            if line[0:6] == "order:":
                order = line[7:]
                break

    # change back to the local root
    os.chdir(localroot)

    return order.rstrip()

def save_legacy_from(localroot, root, filename):
    legacy_from = []
    os.chdir(root)

    count = 0
    legacy_from_flag = False
    with open(filename, 'r') as f:
        for line in f:
            count += 1
            # check for the start of the section
            if line[0:12] == "legacy_from:":
                legacy_from_flag = True

            # append lines from the legacy section
            if legacy_from_flag:
                legacy_from.append(line)

            # check for the end of the header
            if legacy_from_flag and line[0:3] == "---":
               legacy_from_flag = False
               break

    # change back to the local root
    os.chdir(localroot)

    return legacy_from

def remove_header(localroot, root, filename):
    nfirstlines = []

    os.chdir(root)

    # count the number of lines
    count = 0
    n = 0
    headerCheck = False
    with open(filename, 'r') as f:
        for line in f:
            count += 1

            # check if the header is actually a header
            if count > 1 and line[0:3] == "---":
                headerCheck = True
                n = count

    # remove the header
    if headerCheck:
        with open(filename) as f, open("tmp"+filename, "w") as out:
            for _ in range(n):
                nfirstlines.append(next(f))
            for line in f:
                out.write(line)

        os.remove(filename)
        os.rename("tmp"+filename, filename)
        print(" - Old header removed.")

    # change back to the local root
    os.chdir(localroot)

    return n

# loop through the entire internal tree
localroot = os.getcwd()


# generate the index properly speaking
cardDirs = ["internal", "external", "policies"]
sections = []

# determine first the directories
for direct in cardDirs:
    if path.isdir(direct):
        dirs = os.listdir(direct)
        dirs = natsorted(dirs)

        for d in dirs:
            if d[0] != ".":
                sections.append(d)

sections = list(set(sections))
sections = natsorted(sections)

# Index contains the generated content, init it with an empty container
index = ''
index += '\n<div class="index-box-container">\n'
whiteList = ''

localIndexArr = [[]] * len(sections)
for folder in cardDirs:

    # check if folder exists
    if path.isdir(folder):
        dirs = os.listdir(folder)
        dirs = natsorted(dirs)

        for d in dirs:
            if d[0] != ".":
                # set the header of the section
                #index += "\n### " + d.replace("-", " ").capitalize() + "\n"


                # get the index of the section
                indexS = sections.index(d)

                #print(indexS)
                if len(localIndexArr[indexS]) == 0:
                    localIndexArr[indexS] = ["\n"]

                orderArr = []
                # walk through the folders with all the cards
                for root, dirs, files in os.walk(folder+"/"+d):
                    for file in files:
                        if file.endswith(".md"):
                            fileName = os.path.join(root, file)

                            # ignore subsections (.md files that start with _)
                            if file[0] != "_":
                                print(" > Generating header for: " + fileName)

                                # save order and legacy section
                                order = save_order(localroot, root, file)
                                legacy_from = save_legacy_from(localroot, root, file)

                                # remove the previous header
                                n = remove_header(localroot, root, file)

                                # generate a permalink
                                permalink = "/" + root + "/"

                                # generate the shortcut
                                shortcut = re.sub(folder, '', root)

                                # remove the first /
                                shortcut = shortcut[1:]

                                # replace the / with a :
                                shortcut = re.sub('/', ':', shortcut)

                                # define the header for each card
                                header = "---\n"
                                if len(order) > 0:
                                    header += "order: " + str(order) + "\n"
                                    orderArr.append(int(order))
                                header += "layout: page\n"
                                header += "permalink: " + permalink + "\n"
                                header += "shortcut: " + root_handbook(shortcut) + "\n"
                                header += "redirect_from:\n"
                                header += "  - /cards/" + shortcut + "\n"
                                header += "  - /" + folder + "/cards/" + shortcut + "\n"

                                # special case of handbook
                                if "handbook-annexes" in permalink or "handbook-additional" in permalink:
                                    header += "  - /cards/" + root_handbook(shortcut) + "\n"
                                    header += "  - /" + folder + "/cards/" + root_handbook(shortcut) + "\n"

                                # include the legacy section
                                if len(legacy_from) > 0:
                                    for item in legacy_from:
                                        header += str(item)
                                else:
                                    header += "---"

                                # add autogenerated links to whitelist
                                whiteList += permalink + "\n"
                                whiteList += "/?" + shortcut + "\n"
                                whiteList += "/cards/" +shortcut + "\n"
                                whiteList += "/" + folder + "/cards/" + shortcut + "\n"
                                if "handbook-annexes" in permalink or "handbook-additional" in permalink:
                                    whiteList += "  - /cards/" + root_handbook(shortcut) + "\n"
                                    whiteList += "  - /" + folder + "/cards/" + root_handbook(shortcut) + "\n"

                                # add the header properly speaking
                                line_prepender(fileName, header)

                                # open file and get the title after the header
                                count = 0
                                title = ""
                                bp = n + 1

                                with open(fileName, 'r') as f:
                                    for line in f:
                                        count += 1
                                        if count == bp:
                                            if len(line) > 2:
                                                title = line
                                                break
                                            else:
                                                bp += 1

                                # remove first and last chars
                                title = title.rstrip("\n\r")
                                title = title[2:]

                                localIndexArr[indexS].append(build_link(title, root))

                                # output
                                print(" + New header added.")
                                print("-----------------------")

                if len(orderArr) > 0:
                    X = localIndexArr[indexS][1:]
                    Y = orderArr
                    localIndexArr[indexS] = [x for _, x in sorted(zip(Y, X))]
                else:
                    # join all subcategories to the index
                    localIndexArr[indexS] = natsorted(localIndexArr[indexS])

print(localIndexArr)

# determine the index
k = 0
for s in sections:
    index += build_section_start(s.replace("-", " ").capitalize(), s)
    index += ''.join(localIndexArr[k])
    index += build_section_end()
    k += 1

# close the container
index += "\n</div>"

## add link to return to main index
index += """<br><center><a href="{{ '/' | relative_url }}">go back</a></center>"""
index += """<br><center><a href="{{ '/cards' | relative_url }}">Overview of all HowTo cards</a></center>"""

# output the index
#print(index)
# Read in the file
indexFile = "cards.md"
filedata = ""
with open(indexFile, 'r') as file :
    for line in file:
        filedata += line

        # stop reading once the index place holder has been reached
        if re.search("<!-- index -->", line):
            filedata += "[[ index ]]"
            break

# Replace the target string
filedata = filedata.replace('[[ index ]]', index)

# Write the file out again
with open(indexFile, 'w') as file:
  file.write(filedata)

print("\n > New index generated and saved in " + indexFile)

# write link whitelist out
whiteListFile = ".ci/whitelist.txt"
if Path(whiteListFile).exists():
    with open(whiteListFile, 'r') as file :
        for line in file:
            whiteList += line

with open(whiteListFile, 'w') as file:
    file.write(whiteList)