diff --git a/2022/2022-06-08_JuliaForNewcomers/slides/bootstrap.md b/2022/2022-06-08_JuliaForNewcomers/slides/bootstrap.md index bd735cbe6b0f0e1d7f0cfbd34ef9d9c443307566..0e59d80b7ef6dafb339880332a631ac8ac150dbc 100644 --- a/2022/2022-06-08_JuliaForNewcomers/slides/bootstrap.md +++ b/2022/2022-06-08_JuliaForNewcomers/slides/bootstrap.md @@ -1,5 +1,6 @@ <div class=leader> - Bootstrapping Julia +🪓🪚🪛🔧🔨<br> +Bootstrapping Julia </div> @@ -63,20 +64,8 @@ include("mylibrary.jl") - Load a package, add its exports to the global namespace -using UnicodePlots - -- Load a package without exports - ```julia -import UnicodePlots -``` - -- Trick: load package exports to a custom namespace - -```julia -module Plt - using UnicodePlots -end +using UnicodePlots ``` @@ -95,69 +84,68 @@ end ] remove UnicodePlots ``` -- Enter a local project with separate package versions -```julia -] activate path/to/project -``` -- Install dependencies of the local project +# 💡 How to write a standalone program? + +*Your scripts should communicate well with the environment!* +(that means, among other, you) ```julia -] instantiate -``` +#!/usr/bin/env julia -(Project data is stored in `Project.toml`, `Manifest.toml`) +function process_file(filename) + @info "Processing $filename..." + + # ... do something ... + if error_detected + @error "something terrible has happened" + exit(1) + end +end -# Workflow: Testing in REPL +for file in ARGS + process_file(file) +end +``` -- Code in REPL -- Paste pieces of code back and forth to editor/IDE - - VS Code etc.: `Ctrl`+`Enter` - - Linuxes: magic middleclick -- A script is eventually materialized - - ...or picked from history in `.julia/logs/` :) +Correct processing of commandline arguments makes your scripts *repurposable* +and *configurable*. -# Workflow: Write a good standalone script -*Your scripts should communicate well with the environment!* (that means, among other, you) +# 💡 Workflow: Make a local environment for your script -```julia -#!/usr/bin/env julia -global_param = get(ENV, "MY_SETTING", "default") +- Enter a local project with separate package versions -function process_file(fn::String) - println("working on $fn...") - #... - if error_detected - @error "something terrible has happened" fn - exit(1) - end -end +```julia +] activate path/to/project +``` -process_file.(ARGS) -exit(0) +- Install dependencies of the local project + +```julia +] instantiate ``` +- Execute a script with the project environment +```sh +$ julia --project=path/to/project script.jl +``` -# Workflow: What makes your script sustainable? +(Project data is stored in `Project.toml`, `Manifest.toml`.) -Main UNIX facilities: -- Commandline arguments tell the script where to do the work (make it *repurposable*) -- Environment lets you customize stuff that doesn't easily fit into arguments (makes it *reconfigurable*) -- Proper success & error reporting tells the other programs that something broke (makes the pipeline *robust*) -- `#!` (aka "shabang") converts your script to a normal program (makes the user (you) much happier) <div class=leader> - PAUSE +☕🧋🧃<br> +PAUSE </div> Let's have *10 minutes* for a coffee or something. diff --git a/2022/2022-06-08_JuliaForNewcomers/slides/distributed.md b/2022/2022-06-08_JuliaForNewcomers/slides/distributed.md index a1266a214b04add54143fd38c1481911afbdf34c..28081c21258434f005d58afedbbe9c58ee6ccc12 100644 --- a/2022/2022-06-08_JuliaForNewcomers/slides/distributed.md +++ b/2022/2022-06-08_JuliaForNewcomers/slides/distributed.md @@ -1,6 +1,7 @@ <div class=leader> - Parallel Julia on HPCs +🚀🚀🚀<br> +Parallel Julia </div> @@ -13,16 +14,6 @@ -# What does ULHPC look like? - -<center> -<img src="slides/img/iris.png" width="30%"> -<br> -<tt>hpc-docs.uni.lu/systems/iris</tt> -</center> - - - # Basic parallel processing **Using `Threads`:** @@ -42,34 +33,93 @@ end ```julia using Distributed addprocs(N) -newVector = pmap(function, oldVector) +newVector = pmap(myFunction, myVector) ``` We will use the `Distributed` approach. +# Managing your workers + +```julia +using Distributed +addprocs(4) + +myid() +workers() +``` + +Running commands on workers: +```julia +@spawnat 3 @info "Message from worker" + +@spawnat :any myid() +``` + +Getting results from workers: +```julia +job = @spawnat :any begin sleep(10); return 123+321; end + +fetch(job) +``` + +Cleaning up: +```julia +rmprocs(workers()) +``` + + + +# Processing lots of data items in parallel + +```julia +datafiles = ["file$i.csv" for i=1:20] + +@everywhere function process_file(name) + println("Processing file $name") + # ... do something ... +end + +@sync for f in datafiles + @async @spawnat :any process_file(f) +end +``` + + + +# Gathering results from workers + +```julia +items = collect(1:1000) + +@everywhere compute_item(i) = 123 + 321*i + +pmap(compute_item, items) +``` + +💡💡💡 Doing manually with `@spawnat`: +```julia +futures = [@spawnat :any compute_item(item) for item in items] + +fetch.(futures) +``` + + + # How to design for parallelization? -- *Divide software into completely independent parts* - - avoid shared writeable state (to allow reentrancy) - - avoid global variables (to allow separation from the "mother" process) - - avoid complicated intexing in arrays (to allow slicing) - - avoid tiny computation steps (to allow high-yield computation) -- *Design for utilization of the high-level looping primitives* - - use `map` - - use `reduce` or `mapreduce` - - parallelize programs using `pmap` and `dmapreduce` (DistributedData.jl) -- Decompose more advanced programs into *tasks with dependencies* - - Dagger.jl - - `make -jN` may be a surprisingly good tool for parallelization! +**Recommended way:** *Utilize the high-level looping primitives!* + - use `map`, parallelize by just switching to `pmap` + - use `reduce` or `mapreduce`, parallelize by just switching to `dmapreduce` (DistributedData.jl) -# Parallel → distributed processing +# 💡 Parallel → distributed processing -You need a working `ssh` -connection to the server, ideally with keys: +It is very easy to organize *multiple computers* to work for you! + +You need a working `ssh` connection: ```sh user@pc1 $ ssh server1 @@ -88,9 +138,26 @@ julia> addprocs([("server1", 10), ("pc2", 2)]) -# Making a HPC-compatible script +<div class=leader> +💻 🇱🇺 🧮 💿<br> +Utilizing ULHPC 💡 +</div> + + + +# What does ULHPC look like? + +<center> +<img src="slides/img/iris.png" width="30%"> +<br> +<tt>hpc-docs.uni.lu/systems/iris</tt> +</center> + + + +# Making a HPC-compatible Julia script -Main problems: +Main challenges: 1. discover the available resources 2. spawn worker processes at the right place @@ -128,7 +195,8 @@ You start the script using: <div class=leader> - Questions? +🫠🎠🈠ðŸ‡<br> +Questions? </div> Lets do some hands-on problem solving (expected around 15 minutes) diff --git a/2022/2022-06-08_JuliaForNewcomers/slides/img/whyjulia.png b/2022/2022-06-08_JuliaForNewcomers/slides/img/whyjulia.png new file mode 100644 index 0000000000000000000000000000000000000000..294defa465bd40b6bf0dad117070cfdbd797c692 Binary files /dev/null and b/2022/2022-06-08_JuliaForNewcomers/slides/img/whyjulia.png differ diff --git a/2022/2022-06-08_JuliaForNewcomers/slides/index.md b/2022/2022-06-08_JuliaForNewcomers/slides/index.md index 77e313d899bf0ec810b089a58fccfdd4abca0c3a..c01bff73bf5bbccc44f440d0507202a2df0ba53b 100644 --- a/2022/2022-06-08_JuliaForNewcomers/slides/index.md +++ b/2022/2022-06-08_JuliaForNewcomers/slides/index.md @@ -25,6 +25,6 @@ .reveal pre code {border: 0; font-size: 18pt; line-height:27pt;} em {color: #e02;} li {margin-bottom: 1ex;} - div.leader {font-size:400%; font-weight:bold; margin: 1em;} + div.leader {font-size:400%; line-height:120%; font-weight:bold; margin: 1em;} section {padding-bottom: 10em;} </style> diff --git a/2022/2022-06-08_JuliaForNewcomers/slides/intro.md b/2022/2022-06-08_JuliaForNewcomers/slides/intro.md index 64053affab21012a36544ad0f05474ebeded1520..5afbd21d0bd1f78922bd29246fa585afe4e4e873 100644 --- a/2022/2022-06-08_JuliaForNewcomers/slides/intro.md +++ b/2022/2022-06-08_JuliaForNewcomers/slides/intro.md @@ -10,27 +10,12 @@ *Is Julia ecosystem ready for my needs?* -- Likely. If not, extending the packages is unbelievably easy. +- Likely. If not, extending the packages is super easy. - Base includes most of the functionality of Matlab, R and Python with numpy, and many useful bits of C++ -# How to lose performance? +# Why Julia? -Type `a+1` in a typical interpreted language. - -Computer has to do this: - -1. Check if `a` exists in the available variables -2. Find the address of `a` -3. Check if `a` is an actual object or null -4. Find if there is `__add__` in the object, get its address -5. Find if `__add__` is a function with 2 parameters -6. Load the value of `a` -7. Call the function, push call stack -8. Find if 1 is an integer and can be added -9. Check if `a` has a primitive representation (ie. not a big-int) -10. Run the `add` instruction (this takes 1 CPU cycle!) -11. Pop call stack -12. Save the result to the place where the runtime can work with it +<center><img src="slides/img/whyjulia.png" width="80%"></center> diff --git a/2022/2022-06-08_JuliaForNewcomers/slides/io.md b/2022/2022-06-08_JuliaForNewcomers/slides/io.md index c6a35f9d06a0316461c05cd00aa7771e819df823..897c148e90e27570f6b84278cccea3c5ee26466e 100644 --- a/2022/2022-06-08_JuliaForNewcomers/slides/io.md +++ b/2022/2022-06-08_JuliaForNewcomers/slides/io.md @@ -1,6 +1,7 @@ <div class=leader> - Working with data +📘 💽 📈<br> +Working with data </div> @@ -139,16 +140,18 @@ DataFrame(XLSX.gettable(x["Results sheet"])...) # Plotting <center> -<img src="slides/img/unicodeplot.png" height="80%" /> +<img src="slides/img/unicodeplot.png" width="40%" /> </center> # Usual plotting packages -- `UnicodePlots.jl` (useful in terminal) -- `Plots.jl` (matplotlib workalike) +- `UnicodePlots.jl` (useful in terminal, https://github.com/JuliaPlots/UnicodePlots.jl) +- `Plots.jl` (matplotlib workalike, works with Plotly) - `GLMakie.jl` (interactive plots) -- `CairoMakie` (PDF export of Makie plots) +- `CairoMakie.jl` (PDF export of Makie plots) Native `ggplot` and `cowplot` ports are in development. + +Gallery available: https://makie.juliaplots.org diff --git a/2022/2022-06-08_JuliaForNewcomers/slides/language.md b/2022/2022-06-08_JuliaForNewcomers/slides/language.md index e5d1d904362c79a913b7b51b17543dc22df1a0e1..6c7777fe62cf9f2d5564765ee9f32787b201fea5 100644 --- a/2022/2022-06-08_JuliaForNewcomers/slides/language.md +++ b/2022/2022-06-08_JuliaForNewcomers/slides/language.md @@ -1,6 +1,7 @@ <div class=leader> - Julia language primer +🔵 🔴 🟢 🟣<br> +Julia language primer </div> @@ -39,7 +40,7 @@ false, true - `Vector{Int}` ```julia -1, 2, 5, 10 +[1, 2, 5, 10] ``` - `Matrix{Float64}` @@ -61,35 +62,6 @@ false, true -# Supertype hierarchy - -Types possess a single supertype, which allows you to easily group -multiple types under e.g. `Real`, `Function`, `Type`, `Any`, ... - -<pre style="font-size: 90%; line-height:120%;"><code class="language-julia hljs"> -julia> Int -Int64 - -julia> Int.super -Signed - -julia> Int.super.super -Integer - -julia> Int.super.super.super -Real - -julia> Int.super.super.super.super -Number - -julia> Int.super.super.super.super.super -Any - -</code></pre> -Useful for restricting your functions to work on reasonable subsets of inputs. - - - # Basic functionality and expectable stuff - Math: `+`, `-`, `*`, `/`, `^`, ... @@ -139,18 +111,20 @@ else # optional end ``` -- Onesided shell-like shortcuts: +- Shorter inline condition: ```julia -a<0 && (a = 0) - -isfinite(a) || throw_infinite_a_error() +myfunction( index<=10 ? array[index] : default_value ) ``` -- Shorter inline condition: +- 💡 Useful shell-like shortcuts: ```julia -myfunction( index<=10 ? array[index] : default_value ) +a < 0 && (a = 0) +a > 10 && (a = 10) + + +isfinite(a) || @error "a is infinite, program will crash!" ``` @@ -207,7 +181,7 @@ Example: -# Structured cycles! +# 💡 Structured cycles! Using functional-style loops is *much less error-prone* to indexing errors. @@ -246,35 +220,77 @@ julia> join(sort([c for word in ["the result is 123", "what's happening?", "stuf -# Control flow: subroutines (functions and methods) +# Control flow: subroutines (functions) -- Multi-line function definition +- Multi-line function definition ```julia -function f(a,b) +function combine(a,b) return a + b end ``` -- "Mathematical" definition +- "Mathematical" neater definition + +```julia +combine(a,b) = a + b +``` + +- 💡 Definition with types specified (prevents errors, allows optimizations!) ```julia -f(a,b) = a + b +function combine(a::Int, b::Int)::Int + return a + b +end ``` -- Definition with types specified (creates a *method* of a function) + + +# 💡💡 Control flow: subroutine overloading (methods) + +- A method for combining integers ```julia -f(a::Int, b::Int)::Int = a + b +combine(a::Int, b::Int)::Int = a + b ``` -- Overloading (adds another *method* to the function) +- A method of the "same function" for combining strings ```julia -f(a::Complex, b::Complex)::Complex = complex(a.re+b.re, a.im+b.im) +combine(a::String, b::String)::String = "$a and $b" ``` -(Upon calling the function, Julia picks the *most specific* method.) + + +# 💡💡💡 Supertype hierarchy + +Types possess a single supertype, which allows you to easily group +multiple types under e.g. `Real`, `Function`, `Type`, `Any`, ... + +This creates *groups of types* that are useful for restricting your functions to work on the most reasonable subsets of inputs. + +<pre style="font-size: 80%; line-height:120%;"><code class="language-julia hljs"> +julia> Int +Int64 + +julia> Int.super +Signed + +julia> Int.super.super +Integer + +julia> Int.super.super.super +Real + +julia> Int.super.super.super.super +Number + +julia> Int.super.super.super.super.super +Any + +</code></pre> + +(Upon calling the function, Julia picks the *most specific* available method.) @@ -290,12 +306,12 @@ end f(123, extra=321) ``` -- Managing arguments en masse +- 💡 Managing arguments en masse ```julia euclidean(x; kwargs...) = sqrt.(sum(x.^2; kwargs...)) -max(args...) = maximum(args) +max_squared(args...) = maximum(args .^ 2) ``` @@ -318,7 +334,7 @@ x = [1,2,3,4] x' .* x ``` -Internally handled by `broadcast()`. +💡 The "magic dot" is a shortcut for calling `broadcast(...)`. @@ -335,7 +351,7 @@ person["age"] indexof(v::Vector) = Dict(v .=> eachindex(v)) ``` -- Sets are value-less dictionaries (i.e., the elements are unique keys) +- 💡 Sets are key-only containers (keys are _unique_) ```julia julia> x=Set([1,2,3,2,1]);