Snakefile

import os
import shutil
import gzip
import json
from copy import deepcopy


def dict_merge(a, b):
    """
    Deep merge 2 dicts together
    """
    if not isinstance(b, dict):
        return b
    result = deepcopy(a)
    for k, v in b.items():
        if k in result and isinstance(result[k], dict):
            result[k] = dict_merge(result[k], v)
        else:
            result[k] = deepcopy(v)
    return result

# default configuration file
configfile:
    "src/config.imp.json"

# default executable for snakmake
shell.executable("bash")

# custom configuration file
CUSTOM_CONFIG_PATH = "conf/userconfig.imp.json"

# merge 2 configurations files together
if os.path.exists(CUSTOM_CONFIG_PATH):
    with open(CUSTOM_CONFIG_PATH, 'r') as rhandle:
        data = json.load(rhandle)
        config = dict_merge(config, data)


# some parameters
SRCDIR = os.environ.get("SRCDIR", config['imp_src'])

# get parameters from the command line
DATADIR = os.environ.get("DATADIR", config['datadir'])
OUTPUTDIR = os.environ.get("OUTPUTDIR", config['outputdir'])
MG = os.environ.get("MG", config['raws']['Metagenomics']).split()
MT = os.environ.get("MT", config['raws']['Metatranscriptomics']).split()
SAMPLE = os.environ.get("SAMPLE", config['sample'])
DBPATH = os.environ.get("DBPATH", config['db_path'])
if not os.path.exists(DBPATH):
    os.makedirs(DBPATH)

# Get general parameters
THREADS = os.environ.get("THREADS", config['threads'])
MEMTOTAL = os.environ.get("MEMTOTAL", config['memory_total_gb'])
MEMCORE = os.environ.get("MEMCORE", config['memory_per_core_gb'])

# temporary directory will be stored inside the OUTPUTDIR directory
TMPDIR = os.environ.get("TMPDIR", config['tmp_dir'])


def prepare_environment(stepname):
    """
    Prepare the output directories and logs.
    stepname: the name of the pipeline step
    return: the step master directory, the step log
    """
    out = os.path.join(OUTPUTDIR, stepname)
    # mkdirs
    if not os.path.exists(out):
        os.makedirs(out)
    elif not os.path.isdir(out):
        raise OSError("//[IMP] Output is not a directory: %s" % out)
    if not os.path.exists(TMPDIR):
        os.makedirs(TMPDIR)
    bench = os.path.join(out, 'benchmarks')
    if not os.path.exists(bench):
        os.makedirs(bench)

    return out, os.path.join(out, '%s.log' % stepname)


# INCLUDES RULES
include:
    "rules/Util.rules"
include:
    "rules/Preprocessing/master.rules"
include:
    "rules/Assembly/master.rules"
include:
    "rules/Analysis/master.rules"


# locate source directory and name scripts
src = lambda p: os.path.join(SRCDIR, p)


rule ALL:
    input:
        preprocessing_output_files(),
        assembly_output_files(),
        analysis_output_files()

    shell:
        "echo 'DONE'"

rule MODULE_LOAD_TEST:
    shell:
        """
        IMPPRL="{config[preload][test]}"; if [[ -n $IMPPRL ]]; then $IMPPRL; fi
        """