diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..348f835 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,39 @@ +name: Run tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + + workflow_dispatch: + +jobs: + run-tests: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.allow-fail }} + strategy: + matrix: + rust: [stable] + julia: ['1.10', '1.11', '1.12'] + allow-fail: [false] + steps: + - uses: actions/checkout@v4 + + - name: Setup Julia environment + uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.julia }} + + - name: Setup Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + + - name: Run tests + run: | + julia -e "import Pkg; Pkg.add(url=\"https://github.com/Taaitaaiger/JlrsCore.jl\", rev=\"5fee2463f10a856cb3c43c538c18609036d26072\")" + export JLRS_JULIA_DIR="$(dirname $(dirname $(which julia)))" + export LD_LIBRARY_PATH="${JLRS_JULIA_DIR}/lib:${LD_LIBRARY_PATH}" + ./generate_tests.py + ./run_tests.sh diff --git a/.gitignore b/.gitignore index 7585238..206d8c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ book +test_cases \ No newline at end of file diff --git a/generate_tests.py b/generate_tests.py new file mode 100755 index 0000000..c0883ea --- /dev/null +++ b/generate_tests.py @@ -0,0 +1,469 @@ +#! /usr/bin/env python3 + +import glob +import os +import subprocess +from pathlib import Path +import shutil +import logging +import sys + + +logger = logging.getLogger(__name__) + +try: + JLRS_PATH = sys.argv[1] +except: + JLRS_PATH = None + + +def cargo_toml_bin_template(jlrs_path): + if jlrs_path is not None: + jlrs_path = f'path = "{jlrs_path}", ' + else: + jlrs_path = "" + + return f"""[package] +name = "julia_app" +version = "0.1.0" +edition = "2024" + +[features] + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" + +[dependencies] +jlrs = {{version = "0.22", {jlrs_path}features = ["full", "ccall"]}}""" + + +def cargo_toml_lib_template(name, jlrs_path): + if jlrs_path is not None: + jlrs_path = f'path = "{jlrs_path}", ' + else: + jlrs_path = "" + + return f"""[package] +name = "{name}" +version = "0.1.0" +edition = "2024" + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" + +[features] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +jlrs = {{ {jlrs_path}version = "0.22", features = ["jlrs-derive", "ccall", "complex"] }}""" + + +def bin_fragment(bin_name): + return f""" +[[bin]] +name = "{bin_name}" +path = "src/{bin_name}.rs" +""" + + +def default_module_fragment(test_lib_path): + return f"""module JuliaModuleTutorial +using JlrsCore.Wrap + +@wrapmodule("{test_lib_path}", :julia_module_tutorial_init_fn) + +function __init__() + @initjlrs +end +end + +""" + + +class TestCase: + def __init__(self, file_name): + self.rust_lines = [] + self.file_name = file_name + self.generated = False + + def generate_name(self, idx): + clean_name = ( + self.file_name.removeprefix("src/").removesuffix(".md").replace("/", "-") + ) + + name_offset = clean_name.find("-") + 1 + test_name = clean_name[name_offset:] + f"-{idx}" + self.name = test_name.replace("-", "_") + + def append(self, line): + self.rust_lines.append(line) + + +class DocTest(TestCase): + def __init__(self, file_name): + super().__init__(file_name) + + def generate(self, idx): + self.generate_name(idx) + logger.info(f"Generating bin test-case {self.name}") + + with open(f"src/{self.name}.rs", "w") as main_rs: + main_rs.writelines(self.rust_lines) + + with open("Cargo.toml", "a") as cargo_toml: + content = bin_fragment(self.name) + cargo_toml.write(content) + + self.generated = True + + +def indented(prefix_len, lines): + line_prefix = prefix_len * " " + + formatted_lines = [f"{line_prefix}{lines[0]}"] + for line in lines[1:]: + formatted_lines.append(f"{line_prefix}{line}") + + return formatted_lines + + +def adjust_spaces(lst, offset, n): + prefix = " " * n + return [prefix + line[offset:] for line in lst] + + +class LibTest(TestCase): + def __init__(self, file_name): + super().__init__(file_name) + self.raw_julia_lines = [] + self.generated_julia_lines = [] + self.parsing_jl = False + + def set_parsing_julia(self): + self.parsing_jl = True + + def append(self, line): + if self.parsing_jl: + self.raw_julia_lines.append(line) + else: + self.rust_lines.append(line) + + def _generate_call(self, cmd_lines): + if len(cmd_lines) == 1: + self.generated_julia_lines += [f'println("Call: {cmd_lines[0][:-1]}")\n'] + else: + joined = "".join(adjust_spaces(cmd_lines[1:], 7, 6)) + self.generated_julia_lines += [f'println("Call: {cmd_lines[0][:-1]}\n{joined[:-1]}")\n'] + + def _generate_result_assignment(self): + pass + + def _generate_result(self, cmd_lines, result_lines, is_assignment, var_name): + if len(cmd_lines) == 1: + if is_assignment: + self.generated_julia_lines += [ + 'print("Result: ")\n', + "try\n", + f" global {cmd_lines[0][:-1]}\n", + f" show({var_name})\n", + "catch e\n", + " showerror(stdout, e, stacktrace(catch_backtrace()))\n", + "end\n", + "println()\n", + ] + else: + if "print" in cmd_lines[0]: + show = f" {cmd_lines[0][:-1]}\n" + else: + show = f" show({cmd_lines[0][:-1]})\n" + + self.generated_julia_lines += [ + 'print("Result: ")\n', + "try\n", + show, + "catch e\n", + " showerror(stdout, e, stacktrace(catch_backtrace()))\n", + "end\n", + "println()\n", + ] + else: + if is_assignment: + joined = "".join(adjust_spaces(cmd_lines[1:], 7, 8)) + self.generated_julia_lines += [ + 'print("Result: ")\n', + "try\n", + f" global {cmd_lines[0][:-1]}\n{joined[:-1]}\n", + f" show({var_name})\n", + "catch e\n", + " showerror(stdout, e, stacktrace(catch_backtrace()))\n", + "end\n", + "println()\n", + ] + else: + joined = "".join(adjust_spaces(cmd_lines[1:], 7, 8)) + + if "print" in cmd_lines[0] or "print" in joined: + show = [ + f" {cmd_lines[0][:-1]}\n{joined[:-1]}", + ] + else: + show = [ + " show(\n", + f" {cmd_lines[0][:-1]}\n{joined[:-1]}", + " )", + ] + + self.generated_julia_lines += ( + ['print("Result: ")\n', "try\n"] + + show + + [ + "catch e\n", + " showerror(stdout, e, stacktrace(catch_backtrace()))\n", + "end\n", + "println()\n", + ] + ) + + + def _generate_expected(self, result_lines): + if len(result_lines) == 1: + self.generated_julia_lines += [f'println("Expected: {result_lines[0][:-1]}")\n'] + else: + joined = "".join(adjust_spaces(result_lines[1:], 0, 0)) + self.generated_julia_lines += [f'println("Expected: {result_lines[0]}{joined[:-1]}")\n'] + + + def generate_julia_module_case(self, cmd_lines, result_lines): + is_assignment = False + var_name = None + if "=" in cmd_lines[0]: + var_name = cmd_lines[0].split("=")[0].strip() + is_assignment = True + + self._generate_call(cmd_lines) + self._generate_result(cmd_lines, result_lines, is_assignment, var_name) + self._generate_expected(result_lines) + + return self.generated_julia_lines + + def prepare_julia_src(self): + test_lines = [] + + module_lines = None + cmd_lines = None + result_lines = None + parsing_help = False + + for line in self.raw_julia_lines: + if module_lines is not None and len(test_lines) == 0: + if line.startswith(" "): + module_lines.append(line) + continue + + if cmd_lines is not None and result_lines is None: + if not line.startswith(" "): + result_lines = [] + else: + cmd_lines.append(line) + continue + + if result_lines is not None: + if line.startswith("help?>"): + parsing_help = True + continue + elif line.startswith("julia> "): + prep = self.generate_julia_module_case(cmd_lines, result_lines) + test_lines += prep + parsing_help = False + cmd_lines = None + result_lines = None + elif not parsing_help: + result_lines.append(line) + continue + + if line.startswith("julia> "): + if "julia> module JuliaModuleTutorial ... end" in line: + continue + elif "module JuliaModuleTutorial" in line: + module_lines = [line.removeprefix("julia> ")] + continue + + stripped = line.removeprefix("julia> ") + assert cmd_lines is None + cmd_lines = [stripped] + + if result_lines is not None: + test_lines += self.generate_julia_module_case(cmd_lines, result_lines) + + if module_lines is None: + self.raw_julia_lines = [default_module_fragment(f"target/debug/lib{self.name}")] + else: + joined = "".join(adjust_spaces(module_lines[1:], 7, 0)).replace( + "libjulia_module_tutorial", f"lib{self.name}" + ) + self.raw_julia_lines = [f"{module_lines[0][:-1]}\n{joined[:-1]}\n\n"] + + self.raw_julia_lines += test_lines + return True + + def generate(self, idx): + self.generate_name(idx) + logger.info(f"Generating lib test-case {self.name}") + + args = ["cargo", "new", "--lib", self.name] + subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + os.chdir(self.name) + with open("src/lib.rs", "w") as lib_rs: + lib_rs.writelines(self.rust_lines) + + with open("Cargo.toml", "w") as cargo_toml: + content = cargo_toml_lib_template(self.name, JLRS_PATH) + cargo_toml.write(content) + + if self.raw_julia_lines and self.prepare_julia_src(): + with open("ModuleTest.jl", "w+") as module_test_jl: + module_test_jl.writelines(self.raw_julia_lines) + + os.chdir("..") + self.generated = True + + +def prepare_test_dir(): + logger.info(f"Preparing test case directory") + path = Path("./test_cases") + if path.exists(): + logger.debug(f"Removing old test case directory") + shutil.rmtree(path) + + +def extract_tests_from_file(file_name, file_path): + logger.debug(f"Read {file_path}") + with open(file_path) as f: + lines = f.readlines() + + file_tests = [] + test_case = None + parsing_test = False + + for line in lines: + if line.startswith(""): + test_case = DocTest(file_name) + + logger.debug(f"Found test start") + if parsing_test: + logger.error(f"Previous test parsed incompletely") + assert not parsing_test + + parsing_test = True + continue + + if line.startswith(""): + logger.debug(f"Found test end") + if not parsing_test: + logger.error(f"Found test end while not parsing test") + assert parsing_test + + parsing_test = False + file_tests.append(test_case) + continue + + if line.startswith(""): + test_case = LibTest(file_name) + + logger.debug(f"Found libtest start") + if parsing_test: + logger.error(f"Previous test parsed incompletely") + assert not parsing_test + + parsing_test = True + continue + + if line.startswith(""): + logger.debug(f"Found libtest end") + if not parsing_test: + logger.error(f"Found test end while not parsing test") + assert parsing_test + + file_tests.append(test_case) + parsing_test = False + continue + + if line.startswith(""): + logger.debug(f"Found libtest jl end") + if parsing_test: + logger.error(f"Previous test parsed incompletely") + assert not parsing_test + + parsing_test = True + test_case.set_parsing_julia() + continue + + if line.startswith(""): + logger.debug(f"Found libtest jl end") + if not parsing_test: + logger.error(f"Found test end while not parsing test") + assert parsing_test + + parsing_test = False + continue + + if parsing_test and not line.startswith("```"): + test_case.append(line) + + if parsing_test: + logger.error(f"Did not find test end while parsing test") + assert not parsing_test + + return file_tests + + +def create_test_crate(name, cargo_toml_content): + logger.info(f"Create test crate {name}") + subprocess.run( + ["cargo", "new", name], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + os.chdir(name) + + with open("Cargo.toml", "w") as cargo_toml: + cargo_toml.write(cargo_toml_content) + + os.chdir("..") + + +def main(): + prepare_test_dir() + cargo_toml = cargo_toml_bin_template(JLRS_PATH) + + docfiles = sorted(glob.glob("src/**/*.md", recursive=True)) + abs_paths = [Path(file_name).resolve() for file_name in docfiles] + + create_test_crate("test_cases", cargo_toml) + + os.chdir("test_cases") + + test_cases = [] + for file_name, abs_path in zip(docfiles, abs_paths): + logger.info(f"Check {file_name} for tests") + extracted_tests = extract_tests_from_file(file_name, abs_path) + + try: + for idx, test_case in enumerate(extracted_tests): + test_case.generate(idx) + test_cases.append(test_case) + except Exception as e: + logger.error(f"Cannot generate test case {e}") + assert False + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000..ea129b1 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + + + +scriptdir=$(dirname -- "$(realpath -- "$0")") +current_dir=$PWD + +cd $scriptdir + +if [ ! -d test_cases ]; then + echo >&2 -e "\033[1;31mERROR: Test cases have not been generated yet, execute generate_tests.py\033[0m" + exit 1 +fi + +if [ -z "$JLRS_JULIA_DIR" -o ! -f "$JLRS_JULIA_DIR/bin/julia" ]; then + if [ -z "$(which julia)" ]; then + # Julia is neither installed nor provided via JLRS_JULIA_PATH + echo >&2 -e "\033[1;31mERROR: \033[0;31mjulia executable not found\033[0m" + exit 1 + elif [ ! -z "$(julia --help 2>&1 | grep juliaup)" ]; then + # Julia is on PATH, but juliaup is used. + echo >&2 -e "\033[1;31mERROR: \033[0;31musing juliaup is not supported, set JLRS_JULIA_DIR\033[0m" + exit 1 + else + # Julia is on PATH, set as preferred version + jlrs_julia_dir=$(dirname $(dirname $(which julia))) + export JLRS_JULIA_DIR=$jlrs_julia_dir + fi +fi + +export LD_LIBRARY_PATH=$JLRS_JULIA_DIR/lib:$LD_LIBRARY_PATH + +cd test_cases +test_cases=$(cargo read-manifest | jq '.targets[].name') + +echo -e "\033[1;34mINFO: \033[0;34mBuild binary tests\033[0m" +cargo build --all >/dev/null 2>&1 + +echo -e "\033[1;34mINFO: \033[0;34mRun binary tests\033[0m" +n_failed=0 +for case in $test_cases; do + case=${case:1:-1} + echo -e "\033[1;34mINFO: \033[0;34mRun $case\033[0m" + script --flush --quiet --return /tmp/test_output --command "cargo run --bin $case 2>&1" 2>&1 >/dev/null + if [ $? -eq 0 ]; then + echo >&2 -e "\033[1;34mINFO: \033[0;34mTest $case succeeded\033[0m" + else + echo >&2 -e "\033[1;31mERROR: \033[0;31mTest $case failed\033[0m" + out=$(sed '$d' /tmp/test_output | sed '$d' | sed '1d') + echo -e "$out" + failed_tests[$n_failed]="$case (bin)" + n_failed=$((n_failed+1)) + fi +done + +# Remove libtester before searching Cargo.toml files +if [ -d libtester ]; then + rm -rf libtester +fi + +lib_tomls=$(ls */Cargo.toml) + +# Separate testing crate to avoid repeatedly compiling everything from scratch +cargo new --lib libtester 2>/dev/null +if [ $? -ne 0 ]; then + echo >&2 -e "\033[1;31mERROR: \033[0;31mFailed to create libtester-crate\033[0m" + exit 1 +fi + +cd libtester + +for lib_toml in $lib_tomls; do + lib_dir=${lib_toml::-11} + echo -e "\033[1;34mINFO: \033[0;34mBuild $lib_dir\033[0m" + cp ../$lib_dir/Cargo.toml . + cp ../${lib_dir}/src/lib.rs src + script --flush --quiet --return /tmp/test_output --command "cargo build 2>&1" 2>&1 >/dev/null + + if [ $? -eq 0 ]; then + echo >&2 -e "\033[1;34mINFO: \033[0;34mSuccessfully built library test $lib_dir\033[0m" + else + echo >&2 -e "\033[1;31mERROR: \033[0;31mFailed to build library test $lib_dir\033[0m" + out=$(sed '$d' /tmp/test_output | sed '$d' | sed '1d') + failed_tests[$n_failed]="$lib_dir (lib)" + n_failed=$((n_failed+1)) + echo -e "$out" + continue + fi + + if [ -f ../$lib_dir/ModuleTest.jl ]; then + cp ../$lib_dir/ModuleTest.jl . + echo >&2 -e "\033[1;34mINFO: \033[0;34mRun library test module in $lib_dir\033[0m" + $JLRS_JULIA_DIR/bin/julia ModuleTest.jl + + if [ $? -eq 0 ]; then + echo >&2 -e "\033[1;34mINFO: \033[0;34mLibrary test $lib_dir succeeded\033[0m" + else + failed_tests[$n_failed]="$lib_dir (lib)" + n_failed=$((n_failed+1)) + echo >&2 -e "\033[1;31mERROR: \033[0;31mLibrary test $lib_dir failed\033[0m" + fi + fi +done + +if [ $n_failed -ne 0 ]; then + echo >&2 -e "\033[1;31mERROR: \033[0;31mFailed tests:\033[0m" + last=$((n_failed-1)) + for i in $(seq 0 $last); do + echo >&2 -e " \033[0;31m${failed_tests[$i]}\033[0m" + done + + exit 1 +fi + diff --git a/src/01-dependencies/julia.md b/src/01-dependencies/julia.md index 06d239c..fc0a861 100644 --- a/src/01-dependencies/julia.md +++ b/src/01-dependencies/julia.md @@ -1,23 +1,25 @@ # Julia -jlrs currently supports Julia 1.6 up to and including Julia 1.11. Using the most recent stable version is recommended. While juliaup can be used, manually installing Julia is recommended. The reason is that to compile jlrs successfully, the path to the Julia's header files and library must be known and this can be tricky to achieve with juliaup. +jlrs currently supports Julia 1.10 up to and including Julia 1.12. Using the most recent stable version is recommended. If you use `juliaup` to manage your Julia installations, you should install [`jlrs-launcher`]. The reason is that to compile jlrs successfully, the path to the Julia's header files and library must be known and this can be tricky to achieve with `juliaup`. By using this launcher application, `juliaup`'s logic is used to find the location of the necessary files and propagated to the launched application. -There are several platform-dependent ways to make these paths known: +There are several platform-dependent ways to make these paths known if Julia is installed manually: #### Linux -If `julia` is on your `PATH` at `/path/to/bin/julia`, the main header file is expected to live at `/path/to/include/julia/julia.h` and the library at `/path/to/lib/libjulia.so`. If you do not want to add `julia` to your `PATH`, you can set the `JULIA_DIR` environment variable instead. If `JULIA_DIR=/path/to`, the headers and library must live at the previously mentioned paths. +If `julia` is on your `PATH` at `/path/to/bin/julia`, the main header file is expected to live at `/path/to/include/julia/julia.h` and the library at `/path/to/lib/libjulia.so`. If you do not want to add `julia` to your `PATH`, you can set the `JLRS_JULIA_DIR` environment variable instead. If `JLRS_JULIA_DIR=/path/to`, the headers and library must live at the previously mentioned paths. The directory that contains `libjulia.so` must be on the library search path. If this is not the case and the library lives at `/path/to/lib/libjulia.so`, you must add `/path/to/lib/` to the `LD_LIBRARY_PATH` environment variable. #### Windows -If `julia` is on your `Path` at `X:\path\to\bin\julia.exe`, the main header file is expected to live at `X:\path\to\include\julia\julia.h` and the library at `X:\path\to\bin\libjulia.dll`. You can set the `JULIA_DIR` environment variable instead. If `JULIA_DIR=X:\path\to`, the headers and library must live at the previously mentioned paths. +If `julia` is on your `Path` at `X:\path\to\bin\julia.exe`, the main header file is expected to live at `X:\path\to\include\julia\julia.h` and the library at `X:\path\to\bin\libjulia.dll`. You can set the `JLRS_JULIA_DIR` environment variable instead. If `JLRS_JULIA_DIR=X:\path\to`, the headers and library must live at the previously mentioned paths. The directory that contains `libjulia.dll` must be on your `Path` at runtime if Julia is embedded. #### MacOS -If `julia` is on your `PATH` at `/path/to/bin/julia`, the main header file is expected to live at `/path/to/include/julia/julia.h` and the library at `/path/to/lib/libjulia.dylib`. If you do not want to add `julia` to your `PATH`, you can set the `JULIA_DIR` environment variable instead. If `JULIA_DIR=/path/to`, the headers and library must live at the previously mentioned paths. +If `julia` is on your `PATH` at `/path/to/bin/julia`, the main header file is expected to live at `/path/to/include/julia/julia.h` and the library at `/path/to/lib/libjulia.dylib`. If you do not want to add `julia` to your `PATH`, you can set the `JLRS_JULIA_DIR` environment variable instead. If `JLRS_JULIA_DIR=/path/to`, the headers and library must live at the previously mentioned paths. The directory that contains `libjulia.dylib` must be on the library search path. If this is not the case and the library lives at `/path/to/lib/libjulia.dylib`, you must add `/path/to/lib/` to the `DYLD_LIBRARY_PATH` environment variable. + +[`jlrs-launcher`]: https://github.com/Taaitaaiger/jlrs-launcher \ No newline at end of file diff --git a/src/01-dependencies/rust.md b/src/01-dependencies/rust.md index e891428..f61d5b8 100644 --- a/src/01-dependencies/rust.md +++ b/src/01-dependencies/rust.md @@ -1,6 +1,6 @@ # Rust -The minimum supported Rust version (MSRV) is currently 1.77, but some features may require a more recent version. The MSRV can be bumped in minor releases of jlrs.[^1] +The minimum supported Rust version (MSRV) is currently 1.85, but some features may require a more recent version. The MSRV can be bumped in minor releases of jlrs.[^1] Note for Windows users: only the GNU toolchain is supported for dynamic libraries, applications that embed Julia can use either the GNU or MSVC toolchain. diff --git a/src/03-basics/basics.md b/src/02-basics/basics.md similarity index 100% rename from src/03-basics/basics.md rename to src/02-basics/basics.md diff --git a/src/03-basics/casting-unboxing-and-accessing-julia-data.md b/src/02-basics/casting-unboxing-and-accessing-julia-data.md similarity index 95% rename from src/03-basics/casting-unboxing-and-accessing-julia-data.md rename to src/02-basics/casting-unboxing-and-accessing-julia-data.md index 08432f3..8cf89c2 100644 --- a/src/03-basics/casting-unboxing-and-accessing-julia-data.md +++ b/src/02-basics/casting-unboxing-and-accessing-julia-data.md @@ -4,6 +4,7 @@ So far, the only Julia function we've called is `println`, which isn't particula A `Value` is an instance of some Julia type, managed by the GC. If there's a more specific managed type for that Julia type, we can convert the `Value` by casting it with `Value::cast`. + ```rust,ignore use jlrs::prelude::*; @@ -20,9 +21,11 @@ fn main() { }); } ``` + Managed types aren't the only types that map between Rust and Julia. There are many types where the layout in Rust matches the layout of the managed data, including most primitive types. These types implement the `Unbox` trait which lets us extract the data from the `Value` with `Value::unbox`. + ```rust,ignore use jlrs::prelude::*; @@ -36,9 +39,11 @@ fn main() { }); } ``` + If there's no appropriate type that implements `Unbox` or `Managed`, we can access the fields of a `Value` manually. + ```rust,ignore use jlrs::prelude::*; @@ -65,7 +70,7 @@ fn main() { // Safety: the constructor of CustomType is safe to call let inst = unsafe { custom_type - .call0(&mut frame) + .call(&mut frame, []) .expect("cannot call constructor of CustomType") }; @@ -86,6 +91,7 @@ fn main() { }); } ``` + There's a lot going on in this example, but a lot of it is just setup code. We first evaluate some Julia code that defines `CustomType`. Constructors in Julia are just functions linked to a type, so we can call `CustomType`'s constructor by calling the result of the code we've evaluated. Finally, we get to the point and use `Value::get_field` to access the fields before unboxing their content.[^1] The second field is unboxed as a `Bool`, not a `bool`. The Julia `Char` type similarly maps to jlrs's `Char` type. These types exist to avoid any potential mismatches between Rust and Julia. diff --git a/src/03-basics/julia-data-and-functions.md b/src/02-basics/julia-data-and-functions.md similarity index 91% rename from src/03-basics/julia-data-and-functions.md rename to src/02-basics/julia-data-and-functions.md index bc740bf..9e5aed2 100644 --- a/src/03-basics/julia-data-and-functions.md +++ b/src/02-basics/julia-data-and-functions.md @@ -4,6 +4,7 @@ In the previous section we printed `"Hello, World!"` from Julia by evaluating `p What we really want to do is call Julia functions with arbitrary arguments. Let's start with `println(1)`. + ```rust,ignore use jlrs::prelude::*; @@ -17,10 +18,11 @@ fn main() { .expect("println not found in Base"); // Safety: calling println with an integer is safe - unsafe { println_fn.call1(&mut frame, one).expect("println threw an exception") }; + unsafe { println_fn.call(&mut frame, [one]).expect("println threw an exception") }; }); } ``` + The capacity of the frame is set to `3` because `&mut frame` is used three times to root managed data. @@ -28,7 +30,7 @@ The first use of the frame happens in the call to `Value::new`, which converts d Most functions are globals in a module, `println` is defined in the `Base` module. Julia modules can be accessed via the `Module` type, which is a managed type just like `Value`. The functions `Module::base` and `Module::main` provide access to the `Base` and `Main` modules respectively. These functions take an immutable reference to a frame to prevent them from existing outside a scope, but they don't need to be rooted and this doesn't count as a use of the frame. Globals in Julia modules can be accessed with `Module::global`, we use the frame a second time when we call this method to root its result.[^1] -Finally we call `println_fn` with the frame and one argument. This is the third and last use of the frame. Any `Value` is potentially callable, the `Call` trait provides methods to call them with any number of arguments. Specialized methods like `Call::call1` exist to call functions with 3 or fewer arguments, `Call::call` accepts an arbitrary number of arguments. Every argument must be a `Value`. +Finally we call `println_fn` with the frame and one argument. This is the third and last use of the frame. Any `Value` is potentially callable, the `Call` trait provides methods to call them with any number of arguments. Every argument must be a `Value`. Calling Julia functions is unsafe for mostly the same reason as evaluating Julia code is, nothing prevents us from calling `unsafe_load` with a wild pointer. Other risks involve thread-safety and mutably aliasing data that is directly accessed from Rust, which can't be statically prevented. In practice, most Julia code is as safe to call from Rust as it is from Julia. @@ -36,6 +38,7 @@ One thing that should be noted is that while calling a function is more efficien All of that said, we didn't want to print `1`, we wanted to print `Hello, World!`. If we tried the most obvious thing and replaced `1usize` in the code above with `"Hello, World!"`, we'd see that this would fail to compile because `&str` doesn't implement `IntoJulia`. We need to use another managed type, `JuliaString`, which maps to Julia's `String` type. + ```rust,ignore use jlrs::prelude::*; @@ -49,10 +52,11 @@ fn main() { .expect("println not found in Base"); // Safety: calling println with a string is safe - unsafe { println_fn.call1(&mut frame, s).expect("println threw an exception") }; + unsafe { println_fn.call(&mut frame, [s]).expect("println threw an exception") }; }); } ``` + So far we've encountered three managed types, `Value`, `Module`, and `JuliaString`, we'll see several more in the future. All managed types implement the `Managed` trait and have at least one lifetime that encodes their scope, the method `Managed::as_value` can be used to convert managed data to a `Value`. diff --git a/src/03-basics/loading-packages-and-other-custom-code.md b/src/02-basics/loading-packages-and-other-custom-code.md similarity index 79% rename from src/03-basics/loading-packages-and-other-custom-code.md rename to src/02-basics/loading-packages-and-other-custom-code.md index 1cd7fd4..2ea66a2 100644 --- a/src/03-basics/loading-packages-and-other-custom-code.md +++ b/src/02-basics/loading-packages-and-other-custom-code.md @@ -2,8 +2,9 @@ Everything we've done so far has involved standard functionality that's available directly in jlrs and Julia, at worst we've had to evaluate some code to define a custom type. While it's nice that we can use this essential functionality, it's reasonable that we also want to make use of packages. -Any package that has been installed for the targeted version of Julia can be loaded with `LocalHandle::using`.[^1] +Any package that has been installed for the targeted version of Julia can be loaded with `Runtime::using`.[^1] + ```rust,ignore use jlrs::prelude::*; @@ -26,12 +27,13 @@ fn main() { }); } ``` + The function `dot` isn't defined in the `Main` module until we've called `handle.using("LinearAlgebra")`, which internally just evaluates `using LinearAlgebra`. To restrict our imports, we have to construct a `using` or `import` statement manually and evaluate it with `Value::eval_string`. Every package we load must have been installed in advance. Unlike the REPL, trying to use a package that hasn't been installed doesn't lead to a prompt to install it, it just fails. After a package has been loaded, its root module can be accessed with `Module::package_root_module`. -Including a file with custom Julia code works similarly; any file can be loaded and evaluated with `LocalHandle::include`, which calls `Main.include` with the provided path. This works well for local development, but figuring out the correct path to the file when we distribute our code can become problematic. In this case it's better to include the content of the file with the `include_str!` macro and evaluate it with `Value::eval_string`. +Including a file with custom Julia code works similarly; any file can be loaded and evaluated with `Runtime::include`, which calls `Main.include` with the provided path. This works well for local development, but figuring out the correct path to the file when we distribute our code can become problematic. In this case it's better to include the content of the file with the `include_str!` macro and evaluate it with `Value::eval_string`. [^1]: As long as we don't mess with the [`JULIA_DEPOT_PATH` environment variable] diff --git a/src/02-basics/project-setup.md b/src/02-basics/project-setup.md new file mode 100644 index 0000000..9c40390 --- /dev/null +++ b/src/02-basics/project-setup.md @@ -0,0 +1,71 @@ +# Project setup + +We first create a new binary package with `cargo`: + +```bash +cargo new julia_app --bin +``` + +Open `Cargo.toml`, add jlrs as a dependency and enable the `local_rt` feature. We abort on panics[^1]: + +```toml +[package] +name = "julia_app" +version = "0.1.0" +edition = "2024" + +[features] + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" + +[dependencies] +jlrs = {version = "0.22", features = ["local-rt"]} +``` + +If Julia has been installed and we've configured our environment according to the steps in the [dependency chapter], building and running should succeed: + +```bash +cargo build +``` + +If you use `juliaup` and `jlrs-launcher`, the following command must be used: + +```bash +jlrs-launcher run cargo build +``` + +The Julia version can be specified: + +```bash +jlrs-launcher run +1.11 cargo build +``` + +It's important to set the `-rdynamic` linker flag when we embed Julia, Julia will perform badly otherwise.[^2] This flag can be set on the command line with the `RUSTFLAGS` environment variable: + +`RUSTFLAGS="-Clink-args=-rdynamic" cargo build` + +It's also possible to set this flag with a `config.toml` file in [one of the supported directories] for [supported platforms]: + +```toml +[target.x86_64-unknown-linux-gnu] +rustflags = [ "-C", "link-args=-rdynamic" ] + +[target.aarch64-unknown-linux-musl] +rustflags = [ "-C", "link-args=-rdynamic" ] + +# ...etc +``` + +[dependency chapter]: ../01-dependencies/julia.md + +[^1]: In certain circumstances panicking can cause soundness issues, so it's better to abort. + +[^2]: The nitty-gritty reason is that there's some thread-local data that Julia uses constantly. To effectively access this data, it must be defined in an application so the most performant TLS model can be used. By setting the `-rdynamic` linker flag, `libjulia` can find and make use of the definition in our application. If this flag hasn't been set Julia will fall back to a slower TLS model, which has signifant, negative performance implications. + +[one of the supported directories]: https://doc.rust-lang.org/cargo/reference/config.html + +[supported platforms]: https://doc.rust-lang.org/rustc/platform-support.html diff --git a/src/03-basics/scopes-and-evaluating-julia-code.md b/src/02-basics/scopes-and-evaluating-julia-code.md similarity index 91% rename from src/03-basics/scopes-and-evaluating-julia-code.md rename to src/02-basics/scopes-and-evaluating-julia-code.md index 19374b0..8eb45dc 100644 --- a/src/03-basics/scopes-and-evaluating-julia-code.md +++ b/src/02-basics/scopes-and-evaluating-julia-code.md @@ -10,6 +10,7 @@ println("Hello world!") 2. Create a scope. 3. Evaluate the code inside the scope. + ```rust,ignore use jlrs::prelude::*; @@ -24,6 +25,7 @@ fn main() { }); } ``` + Let's go through this code step-by-step. @@ -39,7 +41,7 @@ The handle lets us call into Julia from the current thread, the runtime shuts do handle.local_scope::<_, 1>(|mut frame| { /*snip*/ }); ``` -Before we can call into Julia we have to create a scope by calling `LocalHandle::local_scope` first. This method takes a constant generic integer and a closure that provides access to a frame. The frame is used to prevent data that is managed by Julia's garbage collector, or GC, from being freed while we're using it from Rust. This is called rooting. We'll call such data managed data. +Before we can call into Julia we have to create a scope by calling `LocalHandle::local_scope` first. This method takes a constant generic integer and a closure that provides access to a frame; the other generic is the return type of the closure. The frame is used to prevent data that is managed by Julia's garbage collector, or GC, from being freed while we're using it from Rust. This is called rooting. We'll call such data managed data. An important question to ask is: when can the GC be triggered? The rough answer is whenever managed data is allocated. If the GC is triggered from some thread, it will wait until all threads that can call into Julia have reached a safepoint. Because we're only using a single thread, there are no other threads that need to reach a safepoint and the GC can run immediately, we'll leave it at that for now. diff --git a/src/02-version-features/version-features.md b/src/02-version-features/version-features.md deleted file mode 100644 index 498302f..0000000 --- a/src/02-version-features/version-features.md +++ /dev/null @@ -1,23 +0,0 @@ -# Version features - -If we add jlrs as a dependency and try to compile our crate, we'll see that this fails even after following the instructions in the previous chapter. The reason is that there's an issue we need to deal with: the Julia C API is not stable and each new version tends to introduce a few minor, but backwards-incompatible, changes. jlrs strives to handle these incompatibilities internally as much as possible, but this requires enabling a feature to select the targeted version of Julia. - -Features that select the targeted version of Julia are called version features. They are admittedly kind of a hack because version features are not additive; we must enable exactly one, and it must match the version of Julia that is used. If multiple version features, no version features, or an incorrect version feature is used, compilation will fail. - -The following version features currently exist: - -- `julia-1-6` -- `julia-1-7` -- `julia-1-8` -- `julia-1-9` -- `julia-1-10` -- `julia-1-11` - -It's recommended to "reexport" these version features, and enable the correct one at compile time. - -```toml -[features] -julia-1-6 = ["jlrs/julia-1-6"] -julia-1-7 = ["jlrs/julia-1-7"] -# etc... -``` diff --git a/src/03-basics/project-setup.md b/src/03-basics/project-setup.md deleted file mode 100644 index 12300d6..0000000 --- a/src/03-basics/project-setup.md +++ /dev/null @@ -1,68 +0,0 @@ -# Project setup - -We first create a new binary package with `cargo`: - -```bash -cargo new julia_app --bin -``` - -Open `Cargo.toml`, add jlrs as a dependency and enable the `local_rt` feature. We abort on panics[^1], and reexport the version features: - -```toml -[package] -name = "julia_app" -version = "0.1.0" -edition = "2021" - -[features] -julia-1-6 = ["jlrs/julia-1-6"] -julia-1-7 = ["jlrs/julia-1-7"] -julia-1-8 = ["jlrs/julia-1-8"] -julia-1-9 = ["jlrs/julia-1-9"] -julia-1-10 = ["jlrs/julia-1-10"] -julia-1-11 = ["jlrs/julia-1-11"] - -[profile.dev] -panic = "abort" - -[profile.release] -panic = "abort" - -[dependencies] -jlrs = {version = "0.21", features = ["local-rt"]} -``` - -If we tried to build our application without enabling a version feature, we'd see the following error: - -```text -error: A Julia version must be selected by enabling exactly one of the following version features: - julia-1-6 - julia-1-7 - julia-1-8 - julia-1-9 - julia-1-10 - julia-1-11 -``` - -If Julia 1.10 has been installed and we've configured our environment according to the steps in the [dependency chapter], building and running should succeed after enabling the `julia-1-10` feature: - -```bash -cargo build --features julia-1-10 -``` - -It's important to set the `-rdynamic` linker flag when we embed Julia on Linux, Julia will perform badly otherwise.[^2] This flag can be set on the command line with the `RUSTFLAGS` environment variable: - -`RUSTFLAGS="-Clink-args=-rdynamic" cargo build --features julia-1-10` - -It's also possible to set this flag with a `config.toml` file in the project's root directory: - -```toml -[target.linux] -rustflags = [ "-C", "link-args=-rdynamic" ] -``` - -[dependency chapter]: ../01-dependencies/julia.md - -[^1]: In certain circumstances panicking can cause soundness issues, so it's better to abort. - -[^2]: The nitty-gritty reason is that there's some thread-local data that Julia uses constantly. To effectively access this data, it must be defined in an application so the most performant TLS model can be used. By setting the `-rdynamic` linker flag, `libjulia` can find and make use of the definition in our application. If this flag hasn't been set Julia will fall back to a slower TLS model, which has signifant, negative performance implications. This is only important on Linux, macOS and Windows users can ignore this entirely because the concept of TLS models don't exists on these platforms. diff --git a/src/04-memory-management.md/dynamic-targets.md b/src/03-memory-management/dynamic-targets.md similarity index 82% rename from src/04-memory-management.md/dynamic-targets.md rename to src/03-memory-management/dynamic-targets.md index 5f39984..6d56ba9 100644 --- a/src/04-memory-management.md/dynamic-targets.md +++ b/src/03-memory-management/dynamic-targets.md @@ -2,8 +2,9 @@ A `GcFrame` is a dynamically-sized alternative for `LocalGcFrame`. With a `GcFrame` we avoid having to count how many slots we'll need. -We'll first need to set up a dynamic stack. This is a matter of calling `WithStack::with_stack`, the `WithStack` trait is implemented for `LocalHandle`. Like `LocalGcFrame`, there are two secondary targets which reserve a slot, `Output` and `ReusableSlot`. They behave exactly the same as their local counterparts do. +We'll first need to set up a dynamic stack. This is a matter of calling `WithStack::with_stack`, the `WithStack` trait is implemented for `LocalHandle`. Like `LocalGcFrame`, `Output`s and `ReusableSlot`s can be created. + ```rust,ignore use jlrs::prelude::*; @@ -11,7 +12,7 @@ fn add<'target, Tgt>(target: Tgt, a: u8, b: u8) -> ValueResult<'target, 'static, where Tgt: Target<'target>, { - target.with_local_scope::<_, _, 3>(|target, mut frame| { + target.with_local_scope::<_, 3>(|target, mut frame| { let a = Value::new(&mut frame, a); let b = Value::new(&mut frame, b); let func = Module::base(&frame) @@ -19,7 +20,7 @@ where .expect("+ not found in Base"); // Safety: calling + is safe - unsafe { func.call2(target, a, b) } + unsafe { func.call(target, [a, b]) } }) } @@ -66,8 +67,9 @@ fn main() { }) } ``` + -While a dynamic scope can be nested like a local scope can, this can only be done by calling `GcFrame::scope`. Due to requiring a stack, it's not possible to let an arbitrary target create a new dynamic scope.[^1] Allocating and resizing this stack is relatively expensive, and threading it through our application can be complicated, so it's best to stick with local scopes. +While a dynamic scope can be nested like a local scope can, this can only be done by calling `Scope::scope`. Due to requiring a stack, it's not possible to let an arbitrary target create a new dynamic scope.[^1] Allocating and resizing this stack is relatively expensive, and threading it through our application can be complicated, so it's preferable to stick with local scopes. There's one more dynamic target: `AsyncGcFrame`. It's a `GcFrame` with some additional async capabilities, we'll take a closer look when the async runtime is introduced. diff --git a/src/04-memory-management.md/local-targets.md b/src/03-memory-management/local-targets.md similarity index 78% rename from src/04-memory-management.md/local-targets.md rename to src/03-memory-management/local-targets.md index 1479c7f..5a65da9 100644 --- a/src/04-memory-management.md/local-targets.md +++ b/src/03-memory-management/local-targets.md @@ -2,8 +2,9 @@ The frame we've use so far is a `LocalGcFrame`. It's called local because all roots are stored locally on the stack, which is why we need to know its size at compile time. -Every time we root data by using a mutable reference to a `LocalGcFrame` we consume one of its slots. It's also possible to reserve a slot as a `LocalOutput` or `LocalReusableSlot`, they can be created by calling `LocalGcFrame::local_output` and `LocalGcFrame::local_reusable_slot`. These methods consume a slot. The main difference between the two is that `LocalReusableSlot` is a bit more permissive with the lifetime of the result at the cost of returning an unrooted reference. They're useful if we need to return multiple instances of managed data from a scope, or want to reuse a slot inside one. +Every time we root data by using a mutable reference to a `LocalGcFrame` we consume one of its slots. It's also possible to reserve a slot as an `Output` or `ReusableSlot`, they can be created by calling `LocalGcFrame::output` and `LocalGcFrame::reusable_slot`. These methods consume a slot. The main difference between the two is that `ReusableSlot` is a bit more permissive with the lifetime of the result at the cost of returning unrooted data. They're useful if we need to return multiple instances of managed data from a scope, or want to reuse a slot inside one. + ```rust,ignore use jlrs::prelude::*; @@ -11,7 +12,7 @@ fn add<'target, Tgt>(target: Tgt, a: u8, b: u8) -> ValueResult<'target, 'static, where Tgt: Target<'target>, { - target.with_local_scope::<_, _, 3>(|target, mut frame| { + target.with_local_scope::<_, 3>(|target, mut frame| { let a = Value::new(&mut frame, a); let b = Value::new(&mut frame, b); let func = Module::base(&frame) @@ -19,7 +20,7 @@ where .expect("+ not found in Base"); // Safety: calling + is safe - unsafe { func.call2(target, a, b) } + unsafe { func.call(target, [a, b]) } }) } @@ -27,8 +28,8 @@ fn main() { let handle = Builder::new().start_local().expect("cannot init Julia"); handle.local_scope::<_, 2>(|mut frame| { - let mut output = frame.local_output(); - let mut reusable_slot = frame.local_reusable_slot(); + let mut output = frame.output(); + let mut reusable_slot = frame.reusable_slot(); { // This result can be used until the next time `(&mut) output` is used @@ -64,9 +65,11 @@ fn main() { }); } ``` + -An `UnsizedLocalGcFrame` is similar to a `LocalGcFrame`, the major difference is that its size isn't required to be known at runtime. If the size of the frame is statically known, use `LocalGcFrame`. +An `UnsizedLocalGcFrame` is similar to a `LocalGcFrame`, the major difference is that its size isn't required to be known at compile time. If the size of the frame is statically known, use `LocalGcFrame`. + ```rust,ignore use jlrs::prelude::*; @@ -74,7 +77,7 @@ fn add<'target, Tgt>(target: Tgt, a: u8, b: u8) -> ValueResult<'target, 'static, where Tgt: Target<'target>, { - target.with_local_scope::<_, _, 3>(|target, mut frame| { + target.with_local_scope::<_, 3>(|target, mut frame| { let a = Value::new(&mut frame, a); let b = Value::new(&mut frame, b); let func = Module::base(&frame) @@ -82,7 +85,7 @@ where .expect("+ not found in Base"); // Safety: calling + is safe - unsafe { func.call2(target, a, b) } + unsafe { func.call(target, [a, b]) } }) } @@ -90,8 +93,8 @@ fn main() { let handle = Builder::new().start_local().expect("cannot init Julia"); handle.unsized_local_scope(2, |mut frame| { - let mut output = frame.local_output(); - let mut reusable_slot = frame.local_reusable_slot(); + let mut output = frame.output(); + let mut reusable_slot = frame.reusable_slot(); { let result = add(&mut output, 1, 2).expect("could not add numbers"); @@ -121,3 +124,4 @@ fn main() { }) } ``` + diff --git a/src/03-memory-management/memory-management.md b/src/03-memory-management/memory-management.md new file mode 100644 index 0000000..800fd46 --- /dev/null +++ b/src/03-memory-management/memory-management.md @@ -0,0 +1,23 @@ +# Targets + +In the previous chapter we've seen that we can only interact with Julia inside a scope, where we can use a frame to root managed data. If we look at the signature of any method we've called with a frame, we see that these methods are generic and can take an instance of any type that implements the `Target` trait. Their return type also depends on this target type. + +Take the signature of `Value::eval_string`, for example: + +```rust,ignore +pub unsafe fn eval_string<'target, C, Tgt>( + target: Tgt, + cmd: C, +) -> ValueResult<'target, 'static, Tgt> +where + Tgt: Target<'target>, + C: AsRef, +``` + +Any type that implements `Target` is called a target. There are two things a target encodes: whether the result is rooted, and what lifetime restrictions apply to it. + +If we call `Value::eval_string` with `&mut frame`, `ValueResult` is `Result`. `&frame` also implement `Target`, if we call `Value::eval_string` with it the result is left unrooted, and `ValueResult` is `Result`. We say that `&mut frame` is a rooting target, and `&frame` is a weak target. + +The difference between `Value` and `WeakValue` is that `Value` is guaranteed to be rooted, `WeakValue` isn't. It's unsafe to use a `WeakValue` in any meaningful way. Distinguishing between rooted and unrooted data at the type level helps avoid accidental use of unrooted data and running into use-after-free issues, which can be hard to debug. Every managed type has a `Weak` alias. We'll call `Weak` types and their instances unrooted data. + +The `Result` alias is used with functions that catch exceptions, otherwise `ValueData` is used instead; `ValueResult` is defined as `Result`. Every managed type has a `Result` and `Data` alias. diff --git a/src/04-memory-management.md/target-types.md b/src/03-memory-management/target-types.md similarity index 68% rename from src/04-memory-management.md/target-types.md rename to src/03-memory-management/target-types.md index d481290..26e6c6f 100644 --- a/src/04-memory-management.md/target-types.md +++ b/src/03-memory-management/target-types.md @@ -8,10 +8,6 @@ No target types have been named yet, even frames have only been called just that | `&mut LocalGcFrame<'target>` | Yes | | `UnsizedLocalGcFrame<'target>` | Yes | | `&mut UnsizedLocalGcFrame<'target>` | Yes | -| `LocalOutput<'target>` | Yes | -| `&'target mut LocalOutput` | Yes | -| `LocalReusableSlot<'target>` | Yes | -| `&mut LocalReusableSlot<'target>` | Yes[^1] | | `GcFrame<'target>` | Yes | | `&mut GcFrame<'target>` | Yes | | `Output<'target>` | Yes | @@ -26,6 +22,6 @@ No target types have been named yet, even frames have only been called just that | `ActiveHandle<'target>` | No | | `&Tgt where Tgt: Target<'target>` | No | -These targets belong to three different groups: local targets, dynamic targets, and non-rooting targets. +These targets belong to three different groups: local targets, dynamic targets, and weak targets. -[^1]: While a mutable reference to a `(Local)ReusableSlot` roots the data, it assigns the scope's lifetime to the result which allows the result to live until we leave the scope. This slot can be reused, though, so the data is not guaranteed to remain rooted for the entire `'target` lifetime. For this reason an unrooted reference is returned. +[^1]: While a mutable reference to a `ReusableSlot` roots the data, it assigns the scope's lifetime to the result which allows the result to live until we leave the scope. This slot can be reused, though, so the data is not guaranteed to remain rooted for the entire `'target` lifetime. For this reason unrooted data is returned. diff --git a/src/04-memory-management.md/using-targets.md b/src/03-memory-management/using-targets.md similarity index 69% rename from src/04-memory-management.md/using-targets.md rename to src/03-memory-management/using-targets.md index a39f186..41ab41e 100644 --- a/src/04-memory-management.md/using-targets.md +++ b/src/03-memory-management/using-targets.md @@ -1,9 +1,10 @@ # Using targets and nested scopes -Functions that take a target do so by value, which means the target can only be used once.[^1] If we call such a function with `&mut frame`, only one slot of that frame will be used to root the result. This keeps counting the number of slots we need as easy as possible because we only need to count the number of times the frame is used as a target inside the closure. +Functions that take a target do so by value, which means the target can only be used once.[^1] [^2] If we call such a function with `&mut frame`, only one slot of that frame will be used to root the result. This keeps counting the number of slots we need as easy as possible because we only need to count the number of times the frame is used as a target inside the closure. This does raise an obvious question: what if the function that takes a target needs to root more than one value? The answer is that targets let us create a nested scope. + ```rust,ignore use jlrs::prelude::*; @@ -11,7 +12,7 @@ fn add<'target, Tgt>(target: Tgt, a: u8, b: u8) -> ValueResult<'target, 'static, where Tgt: Target<'target>, { - target.with_local_scope::<_, _, 3>(|target, mut frame| { + target.with_local_scope::<_, 3>(|target, mut frame| { let a = Value::new(&mut frame, a); let b = Value::new(&mut frame, b); let func = Module::base(&frame) @@ -19,7 +20,7 @@ where .expect("+ not found in Base"); // Safety: calling + is safe. - unsafe { func.call2(target, a, b) } + unsafe { func.call(target, [a, b]) } }) } @@ -33,9 +34,12 @@ fn main() { }); } ``` + This approach helps avoid rooting managed data longer than necessary. After calling `add`, only its result is rooted. The temporary values we created in that function are no longer rooted because we've left its scope. It's strongly recommended to avoid writing functions that take a specific target type, and always take a target generically. -[^1]: Some functions take a target by immutable reference and return rooted data. This data is guaranteed to be globally rooted, and the operation won't consume the target. \ No newline at end of file +[^1]: Some functions take a target by immutable reference and return rooted data. This data is guaranteed to be globally rooted, and the operation won't consume the target. + +[^2]: We'll see later that any target can be converted to a weak target by taking it as a reference. This leaves the original target in place. diff --git a/src/03-memory-management/weak-targets.md b/src/03-memory-management/weak-targets.md new file mode 100644 index 0000000..0e4335b --- /dev/null +++ b/src/03-memory-management/weak-targets.md @@ -0,0 +1,7 @@ +# Weak targets + +We don't always need to root managed data. If we never use the result of a function, or if we can guarantee it's globally rooted, it's perfectly fine to leave it unrooted. Keeping unnecessary data alive only leads to additional GC overhead. In this case we want to use a weak target. + +Any target can be used as a weak target by using it behind an immutable reference. There is also `Unrooted`, which can be created by calling `Target::unrooted` or `Managed::unrooted_target`. It's useful if it's not possible to use a reference to an existing target. When an unrooted target is created with the first method, it inherits the `'target` lifetime of the target, with the second it inherits the managed data's `'scope` lifetime. + +There are several other weak targets mentioned in the table, which are all handle types, most of which we haven't seen yet. They're relatively unimportant, they're treated as targets because they can only exist when it's safe to call into Julia, introduce a useful `'target` lifetime, and targets can create new scopes. diff --git a/src/04-memory-management.md/memory-management.md b/src/04-memory-management.md/memory-management.md deleted file mode 100644 index eeb4880..0000000 --- a/src/04-memory-management.md/memory-management.md +++ /dev/null @@ -1,19 +0,0 @@ -# Targets - -In the previous chapter we've seen that we can only interact with Julia inside a scope, where we can use a frame to root managed data. If we look at the signature of any method we've called with a frame, we see that these methods are generic and can take an instance of any type that implements the `Target` trait. Their return type also depends on this target type. - -Take the signature of `Call::call0`, for example: - -```rust,ignore -unsafe fn call0<'target, Tgt>(self, target: Tgt) -> ValueResult<'target, 'data, Tgt> - where - Tgt: Target<'target>; -``` - -Any type that implements `Target` is called a target. There are two things a target encodes: whether the result is rooted, and what lifetime restrictions apply to it. - -If we call `call0` with `&mut frame`, `ValueResult` is `Result`. `&frame` also implement `Target`, if we call `call0` with it the result is left unrooted, and `ValueResult` is `Result`. We say that `&mut frame` is a rooting target, and `&frame` is a non-rooting target. - -The difference between `Value` and `ValueRef` is that `Value` is guaranteed to be rooted, `ValueRef` isn't. It's unsafe to use a `ValueRef` in any meaningful way. Distinguishing between rooted and unrooted data at the type level helps avoid accidental use of unrooted data and running into use-after-free issues, which can be hard to debug. Every managed type has a `Ref` alias, we'll call instances of these types unrooted references [to managed data]. - -The `Result` alias is used with functions that catch exceptions, otherwise `ValueData` is used instead; `ValueResult` is defined as `Result`. Every managed type has a `Result` and `Data` alias. diff --git a/src/04-memory-management.md/non-rooting-targets.md b/src/04-memory-management.md/non-rooting-targets.md deleted file mode 100644 index 417da22..0000000 --- a/src/04-memory-management.md/non-rooting-targets.md +++ /dev/null @@ -1,7 +0,0 @@ -# Non-rooting targets - -We don't always need to root managed data. If we never use the result of a function, or if we can guarantee it's globally rooted, it's perfectly fine to leave it unrooted. Keeping unnecessary data alive only leads to additional GC overhead. In this case we want to use a non-rooting target. - -Any target can be used as a non-rooting target by using it behind an immutable reference. There is also `Unrooted`, which can be created by calling `Target::unrooted` or `Managed::unrooted_target`. It's useful if it's not possible to use a reference to an existing target. When an unrooted target is created with the first method, it inherits the `'target` lifetime of the target, with the second it inherits the managed data's `'scope` lifetime. - -There are several other non-rooting targets mentioned in the table, which are all handle types, most of which we haven't seen yet. They're relatively unimportant, they're treated as targets because they can only exist when it's safe to call into Julia, introduce a useful `'target` lifetime, and targets can create new scopes. diff --git a/src/05-types-and-layouts/generics.md b/src/04-types-and-layouts/generics.md similarity index 100% rename from src/05-types-and-layouts/generics.md rename to src/04-types-and-layouts/generics.md diff --git a/src/05-types-and-layouts/inline-and-non-inline-layouts.md b/src/04-types-and-layouts/inline-and-non-inline-layouts.md similarity index 66% rename from src/05-types-and-layouts/inline-and-non-inline-layouts.md rename to src/04-types-and-layouts/inline-and-non-inline-layouts.md index 0bfb8b4..4e953db 100644 --- a/src/05-types-and-layouts/inline-and-non-inline-layouts.md +++ b/src/04-types-and-layouts/inline-and-non-inline-layouts.md @@ -26,7 +26,7 @@ struct Outer<'scope, 'data> { } ``` -An unrooted reference is used instead of a managed type to represent non-inlined fields to account for mutability, which can render the field's old value unreachable. Managed types and unrooted references make use of the `Option` niche optimization to guarantee `Option` has the same size as a pointer.[^1] We'll say that instances of `Outer` reference managed data. +Unrooted data is used to represent non-inlined fields to account for mutability, which can render the field's old value unreachable. Managed types and unrooted data make use of the `Option` niche optimization to guarantee `Option` has the same size as a pointer.[^1] We'll say that instances of `Outer` reference managed data. Because mutable types aren't inlined, `Inner` can only implement `ValidLayout`, not `ValidField`. Immutable types are normally inlined, so `Outer` can implement both traits. The layouts identify single types, so both types can implement `ConstructType`. diff --git a/src/05-types-and-layouts/isbits-layouts.md b/src/04-types-and-layouts/isbits-layouts.md similarity index 100% rename from src/05-types-and-layouts/isbits-layouts.md rename to src/04-types-and-layouts/isbits-layouts.md diff --git a/src/05-types-and-layouts/types-and-layouts.md b/src/04-types-and-layouts/types-and-layouts.md similarity index 98% rename from src/05-types-and-layouts/types-and-layouts.md rename to src/04-types-and-layouts/types-and-layouts.md index 59abb15..d4a5d9d 100644 --- a/src/05-types-and-layouts/types-and-layouts.md +++ b/src/04-types-and-layouts/types-and-layouts.md @@ -4,6 +4,7 @@ We've already seen a few different types in action, but we haven't really covere Every `Value` has a type, or `DataType`, which we can access at runtime. + ```rust,ignore use jlrs::prelude::*; @@ -17,6 +18,7 @@ fn main() { }) } ``` + This example prints `Float32`, the `DataType` of a 32-bits floating point number in Julia. Internally, a `Value` is a pointer to some memory managed by Julia, and its `DataType` determines the layout of the memory it's pointing to. diff --git a/src/05-types-and-layouts/union-fields.md b/src/04-types-and-layouts/union-fields.md similarity index 100% rename from src/05-types-and-layouts/union-fields.md rename to src/04-types-and-layouts/union-fields.md diff --git a/src/06-arrays/access-arrays.md b/src/05-arrays/access-arrays.md similarity index 98% rename from src/06-arrays/access-arrays.md rename to src/05-arrays/access-arrays.md index c4bed60..94f5263 100644 --- a/src/06-arrays/access-arrays.md +++ b/src/05-arrays/access-arrays.md @@ -6,6 +6,7 @@ A quick note on safety: never access an array that's already accessed mutably, e It's possible to completely ignore the layout of the elements with an `IndeterminateAccessor`. It can be created with the `ArrayBase::indeterminate_data` method. It implements the `Accessor` trait which provides a `get_value` method which returns the element as a `Value`. Unlike Julia, array indexing starts at 0. + ```rust,ignore use jlrs::prelude::*; @@ -33,6 +34,7 @@ fn main() { }); } ``` + We've seen in the previous chapter that there are three ways a field of a composite type can be stored: it can be stored inline, as a reference to managed data, or as an inlined union. An array element is stored as if it were a field of a composite type with one minor exception, there's a difference between how inlined unions are stored in composite types and arrays.[^1] @@ -42,6 +44,7 @@ If a `Typed(Ranked)Array` is used the correct accessor might be inferred from th All these accessor types implement `Accessor`, and additionally provide a `get` function to access an element at some index. Excluding the `BitsUnionAccessor`, they also implement `Index`. These implementations accept the same multidimensional indices as the functions that create new arrays do. The `as_slice` and `into_slice` methods provided by the indexable types let us ignore the multidimensionality and access the data as a slice in column-major order. + ```rust,ignore use jlrs::prelude::*; @@ -135,5 +138,6 @@ fn main() { }); } ``` + [^1]: In composite types, the data and the tag that identifies its type are stored adjacently, in an array the flags are collectively stored after the data. diff --git a/src/06-arrays/arrays.md b/src/05-arrays/arrays.md similarity index 70% rename from src/06-arrays/arrays.md rename to src/05-arrays/arrays.md index d020564..5a131c9 100644 --- a/src/06-arrays/arrays.md +++ b/src/05-arrays/arrays.md @@ -4,20 +4,20 @@ So far we've only worked with relatively simple types, now it's time to look at This special handling involves a single base type, `ArrayBase`, and aliases for the four possible cases: - - `Array == ArrayBase` - - `TypedArray == ArrayBase` - - `RankedArray == ArrayBase` - - `TypedRankedArray == ArrayBase` +- `Array == ArrayBase` +- `TypedArray == ArrayBase` +- `RankedArray == ArrayBase` +- `TypedRankedArray == ArrayBase` As can be seen in this list it's possible to ignore the element type and rank of an array. A known element type must implement `ConstructType`, a known rank is greater than or equal to 0. There are a few additional specialized type aliases: - - `Vector == RankedArray<1>` - - `TypedVector == TypedRankedArray` - - `VectorAny == TypedVector` - - `Matrix == RankedArray<2>` - - `TypedMatrix == TypedRankedArray` +- `Vector == RankedArray<1>` +- `TypedVector == TypedRankedArray` +- `VectorAny == TypedVector` +- `Matrix == RankedArray<2>` +- `TypedMatrix == TypedRankedArray` The elements of Julia arrays are stored in column-major order, which is also known as "F" or "Fortran" order. The sequence `1, 2, 3, 4, 5, 6` maps to the following 2 x 3 matrix: diff --git a/src/06-arrays/create-arrays.md b/src/05-arrays/create-arrays.md similarity index 95% rename from src/06-arrays/create-arrays.md rename to src/05-arrays/create-arrays.md index 2beabf8..b42e0aa 100644 --- a/src/06-arrays/create-arrays.md +++ b/src/05-arrays/create-arrays.md @@ -4,6 +4,7 @@ Functions that create new arrays can mostly be divided into two classes: `Typed( In addition to the element type, these functions take the desired dimensions of the array as an argument. Up to rank 4, tuples of `usize` can be used to express these dimensions. It's also possible to use `[usize; N]`, `&[usize; N]`, and `&[usize]`. If the rank of the array and the dimensions are known at compile time and they don't match, the code will fail to compile. + ```rust,ignore use jlrs::prelude::*; @@ -11,7 +12,7 @@ fn main() { let handle = Builder::new().start_local().expect("cannot init Julia"); handle.local_scope::<_, 2>(|mut frame| { - let arr1 = TypedArray::::new(&mut frame, (2, 2)) + let arr1 = TypedArray::::new(&mut frame, [2, 2]) .expect("invalid size"); assert_eq!(arr1.rank(), 2); @@ -23,11 +24,13 @@ fn main() { }) } ``` + The `new(_for)` functions return an array whose elements haven't been initialized.[^1] It's also possible to wrap an existing `Vec` or slice with `from_vec(_for)` and `from_slice(_for)`. These functions require that the elements are laid out correctly for an array whose element type is `T`. If the layout of the elements is `U`, this layout must be correct for `T`. This connection is expressed with the `HasLayout` trait, which connects a type constructor with its layout type. They're the same type as long as no type parameters have been elided. The `from_vec(_for)` functions take ownership of a `Vec`, which is dropped when the array is freed by the GC. The `from_slice(_for)` functions borrow their data from Rust instead. `Value` and `Array` have a second lifetime called `'data`. This lifetime is set to the lifetime of the borrow to prevent this array from being accessed after the borrow ends. Be aware that Julia is unaware of this lifetime, so there's nothing that prevents us from keeping the array alive by assigning it to a global variable or sending it to some background thread. It's your responsibility to guarantee this doesn't happen, which is one of the reasons why the methods to call Julia functions are unsafe. + ```rust,ignore use jlrs::prelude::*; @@ -36,7 +39,7 @@ fn main() { handle.local_scope::<_, 2>(|mut frame| { let data = vec![1.0f64, 2., 3., 4.]; - let arr = TypedArray::::from_vec(&mut frame, data, (2, 2)) + let arr = TypedArray::::from_vec(&mut frame, data, [2, 2]) .expect("incompatible type and layout") .expect("invalid size"); assert_eq!(arr.rank(), 2); @@ -52,7 +55,7 @@ fn main() { handle.local_scope::<_, 2>(|mut frame| { let mut data = vec![1.0f64, 2., 3., 4.]; - let arr = TypedArray::::from_slice(&mut frame, &mut data, (2, 2)) + let arr = TypedArray::::from_slice(&mut frame, &mut data, [2, 2]) .expect("incompatible type and layout") .expect("invalid size"); assert_eq!(arr.rank(), 2); @@ -67,9 +70,11 @@ fn main() { }) } ``` + The functions `from_slice_cloned(_for)` and `from_slice_copied(_for)` use `new(_for)` to allocate the array, then clone or copy the elements from a given slice to this array. These functions avoid the finalizer of `from_vec(_for)` and the lifetime limitations of `from_slice(_for)`, at the cost of cloning or copying the elements. + ```rust,ignore use jlrs::prelude::*; @@ -78,7 +83,7 @@ fn main() { handle.local_scope::<_, 2>(|mut frame| { let data = [1.0f64, 2., 3., 4.]; - let arr = TypedArray::::from_slice_cloned(&mut frame, &data, (2, 2)) + let arr = TypedArray::::from_slice_cloned(&mut frame, &data, [2, 2]) .expect("incompatible type and layout") .expect("invalid size"); assert_eq!(arr.rank(), 2); @@ -93,7 +98,7 @@ fn main() { handle.local_scope::<_, 2>(|mut frame| { let data = [1.0f64, 2., 3., 4.]; - let arr = TypedArray::::from_slice_copied(&mut frame, &data, (2, 2)) + let arr = TypedArray::::from_slice_copied(&mut frame, &data, [2, 2]) .expect("incompatible type and layout") .expect("invalid size"); assert_eq!(arr.rank(), 2); @@ -107,9 +112,11 @@ fn main() { }); } ``` + Finally, there are two specialized functions. `TypedVector::::new_any` allocates a vector that can hold elements of any type. `TypedVector::::from_bytes` can convert anything that can be referenced as a slice of bytes to a `TypedVector`, it's similar to `TypedVector::::from_slice_copied`. + ```rust,ignore use jlrs::prelude::*; @@ -132,5 +139,6 @@ fn main() { }); } ``` + [^1]: If the elements reference other managed data, the array storage will be initialized to 0. diff --git a/src/06-arrays/mutate-arrays.md b/src/05-arrays/mutate-arrays.md similarity index 99% rename from src/06-arrays/mutate-arrays.md rename to src/05-arrays/mutate-arrays.md index 7f3ebf9..ff8a84d 100644 --- a/src/06-arrays/mutate-arrays.md +++ b/src/05-arrays/mutate-arrays.md @@ -8,6 +8,7 @@ Mutating managed data from Rust is generally unsafe in jlrs. The reason essentia The array types implement `Copy` so it's trivial to create two mutable accessors to the same array, or multiple mutable and immutable accessor in general. It's your responsibility to ensure this doesn't happen. It's possible to avoid this issue to a degree by tracking the array, which we'll cover later in this chapter. + ```rust,ignore use jlrs::prelude::*; @@ -145,3 +146,4 @@ fn main() { }); } ``` + diff --git a/src/06-arrays/ndarray.md b/src/05-arrays/ndarray.md similarity index 97% rename from src/06-arrays/ndarray.md rename to src/05-arrays/ndarray.md index 77a1a2e..2247df5 100644 --- a/src/06-arrays/ndarray.md +++ b/src/05-arrays/ndarray.md @@ -2,6 +2,7 @@ `BitsAccessor`, `InlineAccessor`, and `BitsAccessorMut` are compatible with ndarray via the `NdArrayView` and `NdArrayViewMut` traits. This requires enabling jlrs's `jlrs-ndarray` feature. + ```rust,ignore use jlrs::{ convert::ndarray::{NdArrayView, NdArrayViewMut}, @@ -61,3 +62,4 @@ fn main() { }); } ``` + diff --git a/src/06-arrays/track-arrays.md b/src/05-arrays/track-arrays.md similarity index 97% rename from src/06-arrays/track-arrays.md rename to src/05-arrays/track-arrays.md index 70f878b..b2c9530 100644 --- a/src/06-arrays/track-arrays.md +++ b/src/05-arrays/track-arrays.md @@ -4,6 +4,7 @@ It's trivial to create multiple mutable accessors to the same array. A band-aid Overall, tracking can make accessing arrays safer as long as it's used consistently, but it's unaware of accesses in Julia code. + ```rust,ignore use jlrs::prelude::*; @@ -45,3 +46,4 @@ fn main() { }); } ``` + diff --git a/src/07-exception-handling/exception-handling.md b/src/06-exception-handling/exception-handling.md similarity index 91% rename from src/07-exception-handling/exception-handling.md rename to src/06-exception-handling/exception-handling.md index d7a1a71..3348ee0 100644 --- a/src/07-exception-handling/exception-handling.md +++ b/src/06-exception-handling/exception-handling.md @@ -6,6 +6,7 @@ The function that doesn't catch the exception is always unsafe. Julia exceptions If an exception is thrown and there is no handler available, Julia aborts the process. + ```rust,ignore use jlrs::{catch::catch_exceptions, prelude::*}; @@ -16,15 +17,17 @@ fn main() { handle.local_scope::<_, 1>(|mut frame| unsafe { catch_exceptions( || { - TypedArray::::new_unchecked(&mut frame, (usize::MAX, usize::MAX)); + TypedArray::::new_unchecked(&mut frame, [usize::MAX, usize::MAX]); }, |e| { + let e = e.value(); println!("caught exception: {e:?}") }, ).expect_err("allocated ridiculously-sized array successfully"); }); } ``` + This example should print `caught exception: ArgumentError("invalid Array dimensions")`. diff --git a/src/07-exception-handling/parachutes.md b/src/06-exception-handling/parachutes.md similarity index 87% rename from src/07-exception-handling/parachutes.md rename to src/06-exception-handling/parachutes.md index ca3a331..c07ad71 100644 --- a/src/07-exception-handling/parachutes.md +++ b/src/06-exception-handling/parachutes.md @@ -4,6 +4,7 @@ If we can't avoid data that must be dropped, it might be possible to attach a pa A parachute can be attached by calling `AttachParachute::attach_parachute`, this trait is implemented for any type that is `Sized + Send + Sync + 'static`. The resulting `WithParachute` derefences to the original type, the parachute can be removed by calling `WithParachute::remove_parachute`. + ```rust,ignore use jlrs::{catch::catch_exceptions, data::managed::parachute::AttachParachute, prelude::*}; @@ -16,19 +17,23 @@ fn main() { unsafe { catch_exceptions( || { - let dims = (usize::MAX, usize::MAX); + let dims = [usize::MAX, usize::MAX]; let vec = vec![1usize]; let mut with_parachute = vec.attach_parachute(&mut frame); let arr = TypedArray::::new_unchecked(&mut frame, dims); with_parachute.push(2); arr }, - |e| println!("caught exception: {e:?}"), + |e| { + let e = e.value(); + println!("caught exception: {e:?}") + }, ) } .expect_err("allocated ridiculously-sized array successfully"); }); } ``` + We've attached a parachute to `vec` so it's fine that the next line throws an exception. The GC will eventually take care of dropping it for us. diff --git a/src/08-bindings-and-derivable-traits/bindings-and-derivable-traits.md b/src/07-bindings-and-derivable-traits/bindings-and-derivable-traits.md similarity index 100% rename from src/08-bindings-and-derivable-traits/bindings-and-derivable-traits.md rename to src/07-bindings-and-derivable-traits/bindings-and-derivable-traits.md diff --git a/src/08-bindings-and-derivable-traits/customizing-bindings.md b/src/07-bindings-and-derivable-traits/customizing-bindings.md similarity index 100% rename from src/08-bindings-and-derivable-traits/customizing-bindings.md rename to src/07-bindings-and-derivable-traits/customizing-bindings.md diff --git a/src/08-bindings-and-derivable-traits/generating-bindings.md b/src/07-bindings-and-derivable-traits/generating-bindings.md similarity index 100% rename from src/08-bindings-and-derivable-traits/generating-bindings.md rename to src/07-bindings-and-derivable-traits/generating-bindings.md diff --git a/src/09-multithreaded-runtime/garbage-collection-locks-and-other-blocking-functions.md b/src/08-multithreaded-runtime/garbage-collection-locks-and-other-blocking-functions.md similarity index 68% rename from src/09-multithreaded-runtime/garbage-collection-locks-and-other-blocking-functions.md rename to src/08-multithreaded-runtime/garbage-collection-locks-and-other-blocking-functions.md index cb82dcd..2022b6d 100644 --- a/src/09-multithreaded-runtime/garbage-collection-locks-and-other-blocking-functions.md +++ b/src/08-multithreaded-runtime/garbage-collection-locks-and-other-blocking-functions.md @@ -15,6 +15,7 @@ These GC-safe alternatives are adapted from similarly-named types found in parki A similar issue arises if we call arbitrary long-running code that doesn't all into Julia: it doesn't reach a safepoint, if the GC needs to run it needs to wait until this operation has completed. Since the operation doesn't need to call into Julia, it's safe to execute it in a GC-safe block. We can use the `gc_safe` function to do so, it's unsound to interact with Julia any way inside a GC-safe block. + ```rust,ignore use std::{thread, time::Duration}; @@ -25,34 +26,22 @@ fn long_running_op() { } fn main() { - let (mut mt_handle, thread_handle) = Builder::new().spawn_mt().expect("cannot init Julia"); - let mut mt_handle2 = mt_handle.clone(); - - let t1 = thread::spawn(move || { - mt_handle.with(|handle| { - handle.local_scope::<_, 1>(|mut frame| { - // Safety: long_running_op doesn't interact with Julia - unsafe { gc_safe(long_running_op) }; - - // Safety: we're just printing a string - unsafe { Value::eval_string(&mut frame, "println(\"Hello from thread 1\")") } - .expect("caught exception"); + Builder::new().start_mt(|mt_handle| { + let t1 = mt_handle.spawn(move |mut mt_handle| { + mt_handle.with(|handle| { + handle.local_scope::<_, 1>(|mut frame| { + // Safety: long_running_op doesn't interact with Julia + unsafe { gc_safe(long_running_op) }; + + // Safety: we're just printing a string + unsafe { Value::eval_string(&mut frame, "println(\"Hello from thread 1\")") } + .expect("caught exception"); + }) }) - }) - }); - - let t2 = thread::spawn(move || { - mt_handle2.with(|handle| { - handle.local_scope::<_, 1>(|mut frame| { - // Safety: we're just printing a string - unsafe { Value::eval_string(&mut frame, "println(\"Hello from thread 2\")") } - .expect("caught exception"); - }) - }) - }); + }); - t1.join().expect("thread 1 panicked"); - t2.join().expect("thread 2 panicked"); - thread_handle.join().expect("runtime thread panicked") + t1.join().expect("thread 1 panicked"); + }).expect("cannot init Julia"); } ``` + diff --git a/src/08-multithreaded-runtime/multithreaded-runtime.md b/src/08-multithreaded-runtime/multithreaded-runtime.md new file mode 100644 index 0000000..6c3e755 --- /dev/null +++ b/src/08-multithreaded-runtime/multithreaded-runtime.md @@ -0,0 +1,40 @@ +# Multithreaded runtime + +In all examples so far we've used the local runtime, which is limited to a single thread. The multithreaded runtime can be used from any thread, this feature requires enabling the `multi-rt` feature. + +Using the multithreaded runtime instead of the local runtime is mostly a matter of starting the runtime differently. + + +```rust,ignore +use jlrs::{prelude::*, runtime::builder::Builder}; + +fn main() { + Builder::new().start_mt(|mt_handle| { + let t1 = mt_handle.spawn(move |mut mt_handle| { + mt_handle.with(|handle| { + handle.local_scope::<_, 1>(|mut frame| { + // Safety: we're just printing a string + unsafe { Value::eval_string(&mut frame, "println(\"Hello from thread 1\")") } + .expect("caught exception"); + }) + }) + }); + + let t2 = mt_handle.spawn(move |mut mt_handle| { + mt_handle.with(|handle| { + handle.local_scope::<_, 1>(|mut frame| { + // Safety: we're just printing a string + unsafe { Value::eval_string(&mut frame, "println(\"Hello from thread 2\")") } + .expect("caught exception"); + }) + }) + }); + + t1.join().expect("thread 1 panicked"); + t2.join().expect("thread 2 panicked"); + }).expect("cannot init Julia"); +} +``` + + +Julia is initialized on the current thread when `start_mt` is called, the closure is called on a new thread. This method returns an `MtHandle` that we can use to call into Julia. The `MtHandle` can be cloned, we can create new scoped threads with `MtHandle::spawn`, and by calling `MtHandle::with` the thread is temporarily put into a state where it can create scopes and call into Julia. The runtime thread shuts down when all `MtHandle`s have been dropped. diff --git a/src/10-async-runtime/async-runtime.md b/src/09-async-runtime/async-runtime.md similarity index 85% rename from src/10-async-runtime/async-runtime.md rename to src/09-async-runtime/async-runtime.md index 2f2d186..e3dbc8d 100644 --- a/src/10-async-runtime/async-runtime.md +++ b/src/09-async-runtime/async-runtime.md @@ -1,7 +1,8 @@ # Async runtime -The async runtime lets us run Julia on a background thread, its handle lets us send tasks to this thread. We need to enable the `async-rt` feature to use it. Some tasks support async operations, so we'll also need an async executor. A tokio-based executor is available when the `tokio-rt` feature is enabled, this feature automatically enables `async-rt` as well. +The async runtime lets us run Julia on a background thread, its handle lets us send tasks to this thread. We need to enable the `async-rt` feature to use it. Some tasks support async operations, so we'll also need an async executor. A tokio-based executor is available when the `tokio-rt` feature is enabled, this feature automatically enables `async-rt` as well. The async runtime requires using at least Rust 1.85. + ```rust,ignore use jlrs::prelude::*; @@ -16,6 +17,7 @@ fn main() { thread_handle.join().expect("runtime thread panicked") } ``` + We've configured Julia to use 4 threads.[^1] The builder is upgraded to an `AsyncBuilder` by providing it with the necessary configuration. Here we've configure the runtime to use the tokio-based executor without the I/O driver, and to support 3 concurrent tasks.[^2] If you want to use this driver, enable the `tokio-net` feature and change the argument of `Tokio::new` to `true`. @@ -23,6 +25,6 @@ By default, an unbounded channel is used to let the handles communicate with the After spawning the runtime, we get an `AsyncHandle` that we can use to interact with the runtime thread and a `JoinHandle` to that thread. The runtime thread shuts down when all `AsyncHandle`s have been dropped. It's also possible to manually shut down the runtime by calling `AsyncHandle::close`. -[^1]: Since Julia 1.9 these threads belong to Julia's default thread pool, we can also configure the number of interactive threads with `(Async)Builder::n_interactive_threads`. +[^1]: These threads belong to Julia's default thread pool, we can also configure the number of interactive threads with `(Async)Builder::n_interactive_threads`. [^2]: Multiple tasks that support async operations can be executed concurrently, the runtime can switch to another task while waiting for an async operation to complete. Tasks are not executed in parallel, all tasks are executed on the single runtime thread. diff --git a/src/10-async-runtime/async-tasks.md b/src/09-async-runtime/async-tasks.md similarity index 54% rename from src/10-async-runtime/async-tasks.md rename to src/09-async-runtime/async-tasks.md index 51a07d9..fe97460 100644 --- a/src/10-async-runtime/async-tasks.md +++ b/src/09-async-runtime/async-tasks.md @@ -2,8 +2,58 @@ Async tasks can call async functions, and while awaiting an async function the runtime can switch to another async task. It's possible to call any Julia function as a new Julia task with the methods of the `CallAsync` trait and await its completion. For this to be effective Julia must be configured to use multiple threads. -To create an async task we'll need to implement the `AsyncTask` trait. Let's implement a simple task that adds two numbers. +The easiest way to use async tasks is with an async closure. Let's implement a simple task that adds two numbers. + +```rust,ignore +use jlrs::prelude::*; + +fn main() { + let (async_handle, thread_handle) = Builder::new() + .n_threads(4) + .async_runtime(Tokio::<3>::new(false)) + .spawn() + .expect("cannot init Julia"); + + let a = 1.0; + let b = 2.0; + let recv = async_handle + .task(async move |mut frame: AsyncGcFrame| { + let v1 = Value::new(&mut frame, a); + let v2 = Value::new(&mut frame, b); + let add_fn = Module::base(&frame) + .global(&mut frame, "+") + .expect("cannot find Base.+"); + + // Safety: we're just adding two floating-point numbers + unsafe { add_fn.call_async(&mut frame, [v1, v2]) } + .await + .expect("caught an exception") + .unbox::() + .expect("cannot unbox as f64") + }) + .try_dispatch() + .expect("runtime has shut down"); + + let res = recv.blocking_recv().expect("cannot receive result"); + + assert_eq!(res, 3.0); + + std::mem::drop(async_handle); + thread_handle.join().expect("runtime thread panicked") +} +``` + + +This is very similar to the closures we've used with scopes so far, the major difference as that it's an async and that it takes an `AsyncGcFrame` that we haven't used before. + +An `AsyncGcFrame` is a `GcFrame` that provides some extra features. In particular, the methods of the `CallAsync` trait, e.g. `call_async`, don't take an arbitrary target but must be called with a mutable reference to an `AsyncGcFrame`. These methods execute a function as a new Julia task in a way that lets us await its completion, the runtime thread can switch to other tasks while it's waiting for this task to be completed. + +Dispatching an async task to the runtime is very similar to dispatching a blocking task, we just need to replace `AsyncHandle::blocking_task` with `AsyncHandle::task`. + +We can also use the `AsyncTask` trait. Let's express the previous example with an `AsyncTask`. + + ```rust,ignore use jlrs::prelude::*; @@ -12,11 +62,10 @@ struct AdditionTask { b: f64, } -#[async_trait(?Send)] impl AsyncTask for AdditionTask { type Output = f64; - async fn run<'frame>(&mut self, mut frame: AsyncGcFrame<'frame>) -> Self::Output { + async fn run<'frame>(self, mut frame: AsyncGcFrame<'frame>) -> Self::Output { let v1 = Value::new(&mut frame, self.a); let v2 = Value::new(&mut frame, self.b); let add_fn = Module::base(&frame) @@ -54,9 +103,4 @@ fn main() { thread_handle.join().expect("runtime thread panicked") } ``` - -The trait implementation is marked with `#[async_trait(?Send)]` because the future returned by `AsyncTask::run` can't be sent to another thread but must be executed on the runtime thread. This method is very similar to the closures we've used with scopes so far, the major difference as that it's an async method and that it takes an `AsyncGcFrame` that we haven't used before. - -An `AsyncGcFrame` is a `GcFrame` that provides some extra features. In particular, the methods of the `CallAsync` trait, e.g. `call_async`, don't take an arbitrary target but must be called with a mutable reference to an `AsyncGcFrame`. These methods execute a function as a new Julia task in a way that lets us await its completion, the runtime thread can switch to other tasks while it's waiting for this task to be completed. - -Dispatching an async task to the runtime is very similar to dispatching a blocking task, we just need to replace `AsyncHandle::blocking_task` with `AsyncHandle::task`. + diff --git a/src/10-async-runtime/blocking-tasks.md b/src/09-async-runtime/blocking-tasks.md similarity index 97% rename from src/10-async-runtime/blocking-tasks.md rename to src/09-async-runtime/blocking-tasks.md index 019b97e..3560087 100644 --- a/src/10-async-runtime/blocking-tasks.md +++ b/src/09-async-runtime/blocking-tasks.md @@ -2,6 +2,7 @@ Blocking tasks are the simplest kind of task, they're closures that take a `GcFrame` which are sent to the runtime thread and executed in a dynamic scope. As their name implies, when a blocking task is executed the runtime thread is blocked until the task has completed. + ```rust,ignore use jlrs::prelude::*; @@ -30,6 +31,7 @@ fn main() { thread_handle.join().expect("runtime thread panicked") } ``` + Sending a task is a two-step process. The `AsyncHandle::blocking_task` method returns an instance of `Dispatch`, which provides sync and async methods to dispatch the task. If the backing channel is full, `Dispatch::try_dispatch` fails but returns itself as an `Err` to allow retrying later. diff --git a/src/10-async-runtime/combining-the-multithreaded-and-async-runtimes.md b/src/09-async-runtime/combining-the-multithreaded-and-async-runtimes.md similarity index 50% rename from src/10-async-runtime/combining-the-multithreaded-and-async-runtimes.md rename to src/09-async-runtime/combining-the-multithreaded-and-async-runtimes.md index dc76024..ba3959a 100644 --- a/src/10-async-runtime/combining-the-multithreaded-and-async-runtimes.md +++ b/src/09-async-runtime/combining-the-multithreaded-and-async-runtimes.md @@ -2,50 +2,47 @@ It's possible to combine the functionality of the multithreaded and async runtimes. -The `AsyncBuilder` provides `start_mt` and `spawn_mt` methods that let us use both an `MtHandle` and an `AsyncHandle` when both the `async-rt` and `multi-rt` features have been enabled. The `AsyncHandle` lets us send tasks to the main runtime thread, which is useful if we have some code that we must be called from that thread. +The `AsyncBuilder` provides a `start_mt` method when both the `async-rt` and `multi-rt` features have been enabled that let us use both an `MtHandle` and an `AsyncHandle`. The `AsyncHandle` lets us send tasks to the main runtime thread, which is useful if we have some code that we must be called from that thread. + ```rust,ignore use jlrs::prelude::*; fn main() { - let (mt_handle, async_handle, thread_handle) = Builder::new() - .n_threads(4) + Builder::new() .async_runtime(Tokio::<3>::new(false)) - .spawn_mt() - .expect("cannot init Julia"); - - std::mem::drop(async_handle); - std::mem::drop(mt_handle); - thread_handle.join().expect("runtime thread panicked") + .start_mt(|_mt_handle, _async_handle| { + // Interact with Julia + }) + .unwrap(); } ``` + -We can also create thread pools where each worker thread can call into Julia and runs an async runtime.[^1] We can configure and create new pools with `MtHandle::pool_builder`. When a pool is spawned, an `AsyncHandle` to the pool is returned. Taslks are sent to this pool instead of a specific thread, it can be handled by any of its workers. If a worker dies due to a panic a new worker is automatically spawned.[^2] +We can also create thread pools where each worker thread can call into Julia and runs an async runtime.[^1] We can configure and create new pools with `MtHandle::pool_builder`. When a pool is spawned, an `AsyncHandle` to the pool is returned. Tasks are sent to this pool instead of a specific thread, it can be handled by any of its workers. If a worker dies due to a panic a new worker is automatically spawned.[^2] Workers can be dynamically added and removed with `AsyncHandle::try_add_worker` and `AsyncHandle::try_remove_worker`. The pool shuts down when all workers have been removed, all handles have been dropped, or if its closed explicitly. It's not possible to add workers to the async runtime itself, only to pools. + ```rust,ignore use jlrs::prelude::*; fn main() { - let (mt_handle, thread_handle) = Builder::new() - .n_threads(4) - .spawn_mt() - .expect("cannot init Julia"); - - let pool_handle = mt_handle - .pool_builder(Tokio::<3>::new(false)) - .n_workers(3.try_into().unwrap()) - .spawn(); - - assert!(pool_handle.try_add_worker()); - assert!(pool_handle.try_remove_worker()); - - std::mem::drop(pool_handle); - std::mem::drop(mt_handle); - thread_handle.join().expect("runtime thread panicked") + Builder::new() + .async_runtime(Tokio::<3>::new(false)) + .start_mt(|mt_handle, _async_handle| { + let pool_handle = mt_handle + .pool_builder(Tokio::<3>::new(false)) + .n_workers(3.try_into().unwrap()) + .spawn(); + + assert!(pool_handle.try_add_worker()); + assert!(pool_handle.try_remove_worker()); + }) + .unwrap(); } ``` + One additional advantage that pools have over the async runtime thread is that the latency is typically much lower. If we don't need code to run on the main thread specifically, it's more effective to use the multithreaded runtime and create a pool instead. diff --git a/src/10-async-runtime/persistent-tasks.md b/src/09-async-runtime/persistent-tasks.md similarity index 90% rename from src/10-async-runtime/persistent-tasks.md rename to src/09-async-runtime/persistent-tasks.md index d92da93..bda8ccb 100644 --- a/src/10-async-runtime/persistent-tasks.md +++ b/src/09-async-runtime/persistent-tasks.md @@ -4,6 +4,7 @@ Persistent tasks let us set up a task that we can send messages to indepently of To create a persistent task we'll need to implement the `PersistentTask` trait. Let's implement a task that accumulates a sum of floating point numbers. + ```rust,ignore use jlrs::prelude::*; @@ -11,7 +12,6 @@ struct AccumulatorTask { init_value: f64, } -#[async_trait(?Send)] impl PersistentTask for AccumulatorTask { type State<'state> = Value<'state, 'static>; type Input = f64; @@ -21,12 +21,12 @@ impl PersistentTask for AccumulatorTask { &mut self, frame: AsyncGcFrame<'frame>, ) -> JlrsResult> { - frame.with_local_scope::<_, _, 2>(|mut async_frame, mut local_frame| { + frame.with_local_scope::<_, 2>(|mut async_frame, mut local_frame| { let ref_ctor = Module::base(&local_frame).global(&mut local_frame, "Ref")?; let init_v = Value::new(&mut local_frame, self.init_value); // Safety: we're just calling the constructor of `Ref`, which is safe. - let state = unsafe { ref_ctor.call1(&mut async_frame, init_v) }.into_jlrs_result()?; + let state = unsafe { ref_ctor.call(&mut async_frame, [init_v]) }?; Ok(state) }) } @@ -41,8 +41,7 @@ impl PersistentTask for AccumulatorTask { let setindex_func = Module::base(&frame).global(&mut frame, "setindex!")?; // Safety: Calling getindex with state is equivalent to calling `state[]`. - let current_sum = unsafe { getindex_func.call1(&mut frame, *state) } - .into_jlrs_result()? + let current_sum = unsafe { getindex_func.call(&mut frame, [*state]) }? .unbox::()?; let new_sum = current_sum + input; @@ -50,7 +49,7 @@ impl PersistentTask for AccumulatorTask { // Safety: Calling setindex! with state and new_value is equivalent to calling // `state[] = new_value`. - unsafe { setindex_func.call2(&mut frame, *state, new_value) }.into_jlrs_result()?; + unsafe { setindex_func.call(&mut frame, [*state, new_value]) }?; Ok(new_sum) } @@ -88,6 +87,7 @@ fn main() { thread_handle.join().expect("runtime thread panicked") } ``` + When the persistent task is started by the async runtime, the `init` method is called to initialize the state of the task. In this case the state is an instance of `Ref{Float64}`, We can't use a `Float64` directly because `Float64` isn't a mutable type. Any data rooted in the async frame provided to `init` function remains rooted until the task has shut down, a local scope is used to root temporary data so we only need to root the state in the async frame. diff --git a/src/09-multithreaded-runtime/multithreaded-runtime.md b/src/09-multithreaded-runtime/multithreaded-runtime.md deleted file mode 100644 index 8223468..0000000 --- a/src/09-multithreaded-runtime/multithreaded-runtime.md +++ /dev/null @@ -1,86 +0,0 @@ -# Multithreaded runtime - -In all examples so far we've used the local runtime, which is limited to a single thread. The multithreaded runtime can be used from any thread, this feature requires using at least Julia 1.9 and enabling the `multi-rt` feature. - -Using the multithreaded runtime instead of the local runtime is mostly a matter of starting the runtime differently. - -```rust,ignore -use std::thread; - -use jlrs::{prelude::*, runtime::builder::Builder}; - -fn main() { - let (mut mt_handle, thread_handle) = Builder::new().spawn_mt().expect("cannot init Julia"); - let mut mt_handle2 = mt_handle.clone(); - - let t1 = thread::spawn(move || { - mt_handle.with(|handle| { - handle.local_scope::<_, 1>(|mut frame| { - // Safety: we're just printing a string - unsafe { Value::eval_string(&mut frame, "println(\"Hello from thread 1\")") } - .expect("caught exception"); - }) - }) - }); - - let t2 = thread::spawn(move || { - mt_handle2.with(|handle| { - handle.local_scope::<_, 1>(|mut frame| { - // Safety: we're just printing a string - unsafe { Value::eval_string(&mut frame, "println(\"Hello from thread 2\")") } - .expect("caught exception"); - }) - }) - }); - - t1.join().expect("thread 1 panicked"); - t2.join().expect("thread 2 panicked"); - thread_handle.join().expect("runtime thread panicked") -} -``` - -Julia is initialized on a background thread when `spawn_mt` is called. This method returns an `MtHandle` that we can use to call into Julia and a handle to the runtime thread. The `MtHandle` can be cloned and sent to other threads, by calling `MtHandle::with` the thread is temporarily put into a state where it can create scopes and call into Julia. The runtime thread shuts down when all `MtHandle`s have been dropped. - -Instead of spawning the runtime thread, we can also initialize Julia on the current thread and spawn a new thread that can use an `MtHandle`: - -```rust,ignore -use std::thread; - -use jlrs::{ - prelude::*, - runtime::{builder::Builder, handle::mt_handle::MtHandle}, -}; - -fn main_inner(mut mt_handle: MtHandle) { - let mut mt_handle2 = mt_handle.clone(); - - let t1 = thread::spawn(move || { - mt_handle.with(|handle| { - handle.local_scope::<_, 1>(|mut frame| { - // Safety: we're just printing a string - unsafe { Value::eval_string(&mut frame, "println(\"Hello from thread 1\")") } - .expect("caught exception"); - }) - }) - }); - - let t2 = thread::spawn(move || { - mt_handle2.with(|handle| { - handle.local_scope::<_, 1>(|mut frame| { - // Safety: we're just printing a string - unsafe { Value::eval_string(&mut frame, "println(\"Hello from thread 2\")") } - .expect("caught exception"); - }) - }) - }); - - t1.join().expect("thread 1 panicked"); - t2.join().expect("thread 2 panicked"); -} - -fn main() { - Builder::new().start_mt(main_inner).expect("cannot init Julia"); -} -``` - -This is useful if we interact with code in Julia that is picky about being called from the main application thread, e.g. code involving Qt. diff --git a/src/11-ccall-basics/argument-types/argument-types.md b/src/10-ccall-basics/argument-types/argument-types.md similarity index 100% rename from src/11-ccall-basics/argument-types/argument-types.md rename to src/10-ccall-basics/argument-types/argument-types.md diff --git a/src/11-ccall-basics/argument-types/arrays.md b/src/10-ccall-basics/argument-types/arrays.md similarity index 100% rename from src/11-ccall-basics/argument-types/arrays.md rename to src/10-ccall-basics/argument-types/arrays.md diff --git a/src/11-ccall-basics/ccall-basics.md b/src/10-ccall-basics/ccall-basics.md similarity index 95% rename from src/11-ccall-basics/ccall-basics.md rename to src/10-ccall-basics/ccall-basics.md index be79678..37f8275 100644 --- a/src/11-ccall-basics/ccall-basics.md +++ b/src/10-ccall-basics/ccall-basics.md @@ -6,6 +6,7 @@ The intent of this chapter is to cover some essential information about `ccall`. To get started with calling into Rust from Julia, we're going to look at a final embedding example first before creating our first dynamic library. We'll expose a function pointer to Julia and call it with `ccall`. + ```rust,ignore use std::ffi::c_void; @@ -35,7 +36,7 @@ fn main() { // Safety: Immutable types are passed and returned by value, so `add` // has the correct signature for the `ccall` in `call_rust`. All // `add` does is add `a` and `b`, which is perfectly safe. - let res = unsafe { func.call3(&mut frame, ptr, a, b) } + let res = unsafe { func.call(&mut frame, [ptr, a, b]) } .expect("an exception occurred") .unbox::() .expect("not an f64"); @@ -44,6 +45,7 @@ fn main() { }); } ``` + All this example does is call `add`, which adds two numbers and returns the result. We can convert this function to `Value` by converting it to a void pointer first. It's not possible to call `ccall` directly from Rust because the return and argument types must be statically known, so we create a function that `ccall`s the function pointer with the given arguments by evaluating its definition. diff --git a/src/11-ccall-basics/custom-types.md b/src/10-ccall-basics/custom-types.md similarity index 100% rename from src/11-ccall-basics/custom-types.md rename to src/10-ccall-basics/custom-types.md diff --git a/src/11-ccall-basics/dynamic-libraries.md b/src/10-ccall-basics/dynamic-libraries.md similarity index 85% rename from src/11-ccall-basics/dynamic-libraries.md rename to src/10-ccall-basics/dynamic-libraries.md index b21f714..5abe551 100644 --- a/src/11-ccall-basics/dynamic-libraries.md +++ b/src/10-ccall-basics/dynamic-libraries.md @@ -14,7 +14,7 @@ We need to change the crate type to `cdylib`, this can be configured in `Cargo.t [package] name = "julia_lib" version = "0.1.0" -edition = "2021" +edition = "2024" [profile.release] panic = "abort" @@ -33,13 +33,13 @@ We don't need to add jlrs as a dependency, we'll discuss the advantages and disa Replace the content of `lib.rs` with the following code: ```rust,ignore -#[no_mangle] +#[unsafe(no_mangle)]` pub unsafe extern "C" fn add(a: f64, b: f64) -> f64 { a + b } ``` -The function is annotated with `#[no_mangle]` to prevent the name from being mangled. After building with `cargo build` we can find the library in `target/debug`. On Linux it will be named `libjulia_lib.so`, on macOS `libjulia_lib.dylib`, and on Windows `libjulia_lib.dll`. Let's use it! +The function is annotated with `#[unsafe(no_mangle)]` to prevent the name from being mangled. After building with `cargo build` we can find the library in `target/debug`. On Linux it will be named `libjulia_lib.so`, on macOS `libjulia_lib.dylib`, and on Windows `libjulia_lib.dll`. Let's use it! Open the Julia REPL in `julia_lib`'s root directory and evaluate the following code: diff --git a/src/11-ccall-basics/return-type.md b/src/10-ccall-basics/return-type.md similarity index 100% rename from src/11-ccall-basics/return-type.md rename to src/10-ccall-basics/return-type.md diff --git a/src/11-ccall-basics/yggdrasil.md b/src/10-ccall-basics/yggdrasil.md similarity index 100% rename from src/11-ccall-basics/yggdrasil.md rename to src/10-ccall-basics/yggdrasil.md diff --git a/src/12-julia-module/constants/constants.md b/src/11-julia-module/constants/constants.md similarity index 86% rename from src/12-julia-module/constants/constants.md rename to src/11-julia-module/constants/constants.md index 29cc9e9..57dd6b9 100644 --- a/src/12-julia-module/constants/constants.md +++ b/src/11-julia-module/constants/constants.md @@ -2,6 +2,7 @@ The simplest thing we can export from Rust to Julia is a constant. New constants can be created from static and constant items whose type implements `IntoJulia`. + ```rust,ignore use jlrs::prelude::*; @@ -15,9 +16,11 @@ julia_module! { const STATIC_U8: u8; } ``` + If we compile this code and wrap it, we can access these constants: + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -28,9 +31,11 @@ julia> JuliaModuleTutorial.CONST_U8 julia> JuliaModuleTutorial.STATIC_U8 0x02 ``` + It's possible to rename a constant by putting `as NEW_NAME` at the end of the declaration. They can also be documented, Julia doctests are supported. + ```rust,ignore use jlrs::prelude::*; @@ -45,7 +50,9 @@ julia_module! { const CONST_U8: u8 as CONST_UINT8; } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -58,3 +65,4 @@ help?> JuliaModuleTutorial.CONST_UINT8 An exported constant. ``` + \ No newline at end of file diff --git a/src/12-julia-module/functions/array-arguments.md b/src/11-julia-module/functions/array-arguments.md similarity index 86% rename from src/12-julia-module/functions/array-arguments.md rename to src/11-julia-module/functions/array-arguments.md index 40de1e5..91060cf 100644 --- a/src/12-julia-module/functions/array-arguments.md +++ b/src/11-julia-module/functions/array-arguments.md @@ -4,12 +4,15 @@ Without jlrs we had to convert arrays to a pointer to their first element if we Any of the aliases of `ArrayBase` can be used as an argument type, they enforce the obvious restrictions: `Array` only enforces that the argument is an array, `TypedArray` puts restrictions on the element type, `RankedArray` on the rank, and `TypedRankedArray` on both. Other aliases like `Vector` are expressed in terms of these aliases so they can also be used as argument types. + ```rust,ignore use jlrs::prelude::*; // Safety: the array must not be mutated from another thread unsafe fn sum_array(array: TypedArray) -> f64 { - array.bits_data().as_slice().iter().sum() + unsafe { + array.bits_data().as_slice().iter().sum() + } } julia_module! { @@ -18,7 +21,9 @@ julia_module! { fn sum_array(array: TypedArray) -> f64; } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -29,3 +34,4 @@ julia> JuliaModuleTutorial.sum_array([1.0 2.0]) julia> JuliaModuleTutorial.sum_array([1.0; 2.0]) 3.0 ``` + diff --git a/src/12-julia-module/functions/ccall-ref.md b/src/11-julia-module/functions/ccall-ref.md similarity index 95% rename from src/12-julia-module/functions/ccall-ref.md rename to src/11-julia-module/functions/ccall-ref.md index c106f0e..f273b84 100644 --- a/src/12-julia-module/functions/ccall-ref.md +++ b/src/11-julia-module/functions/ccall-ref.md @@ -6,6 +6,7 @@ When `CCallRef` is used as an argument type, the generated function restricts If `CCallRefRet` is used as a return type, `ccall` returns it as `Ref{T}` and the function as `T`. The main advantage returning `CCallRefRet` has over `TypedValueRet` is that using `CCallRefRet` produces more type-stable code. + ```rust,ignore use jlrs::{ data::managed::{ @@ -32,7 +33,9 @@ julia_module! { fn add(a: CCallRef, b: CCallRef) -> CCallRefRet; } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -40,5 +43,6 @@ Main.JuliaModuleTutorial julia> JuliaModuleTutorial.add(1.0, 2.0) 3.0 ``` + [^1]: While both are pointers to the same layout, managed data is guaranteed to be preceded in memory by a tag that identifies its type. This tag isn't guaranteed to be present when an argument is passed by reference. diff --git a/src/12-julia-module/functions/functions.md b/src/11-julia-module/functions/functions.md similarity index 91% rename from src/12-julia-module/functions/functions.md rename to src/11-julia-module/functions/functions.md index 3d4cbc3..176f534 100644 --- a/src/12-julia-module/functions/functions.md +++ b/src/11-julia-module/functions/functions.md @@ -6,6 +6,7 @@ These traits should not be implemented manually. jlrs provides implementations f Exporting a function is a matter copying and pasting its signature: + ```rust,ignore use jlrs::prelude::*; @@ -19,7 +20,9 @@ julia_module! { fn add(a: f64, b: f64) -> f64; } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -30,11 +33,13 @@ julia> JuliaModuleTutorial.add(1.0, 2.0) julia> JuliaModuleTutorial.add(1, 2.0) ERROR: MethodError: no method matching add(::Int64, ::Float64) ``` + We don't have to mark our function as `extern "C"`, the `julia_module!` macro generates an `extern "C"` wrapper function for every exported function. These wrappers only exist inside the `julia_module_tutorial_init_fn` so we don't need to worry about name conflicts. Like constants, exported functions can be renamed and documented. + ```rust,ignore use jlrs::prelude::*; @@ -49,7 +54,9 @@ julia_module! { fn add(a: f64, b: f64) -> f64 as add!; } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -60,5 +67,6 @@ julia> JuliaModuleTutorial.add!(1.0, 2.0) help?> JuliaModuleTutorial.add! add!(::Float64, ::Float64)::Float64 ``` + [^1]: i.e., ConstructType is also derived. diff --git a/src/12-julia-module/functions/gc-safety.md b/src/11-julia-module/functions/gc-safety.md similarity index 96% rename from src/12-julia-module/functions/gc-safety.md rename to src/11-julia-module/functions/gc-safety.md index 8e205ca..3e2146b 100644 --- a/src/12-julia-module/functions/gc-safety.md +++ b/src/11-julia-module/functions/gc-safety.md @@ -6,6 +6,7 @@ A thread that can call into Julia is normally in a GC-unsafe state, the unsafe h If an exported function doesn't need to call into Julia at all, we can ensure it's called in a GC-safe state by annotating the export with `#[gc_safe]`. To simulate a long-running function we're going to sleep for a few seconds. + ```rust,ignore use std::{thread::sleep, time::Duration}; @@ -23,9 +24,11 @@ julia_module! { fn add(a: f64, b: f64) -> f64; } ``` + We can manually create gc-safe blocks. + ```rust,ignore use std::{thread::sleep, time::Duration}; @@ -57,9 +60,11 @@ julia_module! { fn some_operation(len: usize) -> TypedVectorRet; } ``` + It's possible to revert to a GC-unsafe state in a GC-safe block by inserting a GC-unsafe block with `gc_unsafe`. + ```rust,ignore use std::{thread::sleep, time::Duration}; @@ -92,3 +97,4 @@ julia_module! { fn some_operation(len: usize) -> TypedVectorRet; } ``` + diff --git a/src/12-julia-module/functions/managed-arguments.md b/src/11-julia-module/functions/managed-arguments.md similarity index 90% rename from src/12-julia-module/functions/managed-arguments.md rename to src/11-julia-module/functions/managed-arguments.md index 262b7db..57bafe1 100644 --- a/src/12-julia-module/functions/managed-arguments.md +++ b/src/11-julia-module/functions/managed-arguments.md @@ -2,6 +2,7 @@ We're not limited to just using immutable types as function arguments, managed types also implement `CCallArg`. The function can just as easily take a `Module` or `Value` as an argument. If the argument type is `Value`, that argument's type is left unspecified in the generated function signature and passed to `ccall` as `Any`. + ```rust,ignore use jlrs::prelude::*; @@ -21,7 +22,9 @@ julia_module! { fn print_value(value: Value); } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -32,3 +35,4 @@ julia> JuliaModuleTutorial.print_module_name(JuliaModuleTutorial) julia> JuliaModuleTutorial.print_value(JuliaModuleTutorial) Main.JuliaModuleTutorial ``` + diff --git a/src/12-julia-module/functions/returning-managed-data.md b/src/11-julia-module/functions/returning-managed-data.md similarity index 72% rename from src/12-julia-module/functions/returning-managed-data.md rename to src/11-julia-module/functions/returning-managed-data.md index bb12891..d28d9bb 100644 --- a/src/12-julia-module/functions/returning-managed-data.md +++ b/src/11-julia-module/functions/returning-managed-data.md @@ -6,10 +6,11 @@ To create a scope we'll need a handle, introducing: `WeakHandle`. A `WeakHandle` While this at least gives us a way to create scopes, we still need to solve the other problem: how do we return managed data from a scope? -Every managed type in jlrs has a `'scope` lifetime, to return managed data from the scope we'll need to erase this lifetime. jlrs takes the rootedness guarantee of managed types seriously, so we can't simply adjust the lifetime of such data directly. `Ref`-types don't guarantee that the data is rooted for its `'scope` lifetime, so we're free to relax it to `'static`, which solves our issue. We call this leaking managed data. +Every managed type in jlrs has a `'scope` lifetime, to return managed data from the scope we'll need to erase this lifetime. jlrs takes the rootedness guarantee of managed types seriously, so we can't simply adjust the lifetime of such data directly. `Weak` types don't guarantee that the data is rooted for its `'scope` lifetime, so we're free to relax it to `'static`, which solves our issue. We call this leaking managed data. -In short, to return managed data we'll need to convert it to a `Ref` with static lifetimes first. All managed types have a `Ret` alias, which is the `Ref` alias with static lifetimes. These `Ret`-aliases implement `CCallReturn`. Converting managed data to a `Ret` type is a matter of calling `Managed::leak`. +In short, to return managed data we'll need to convert it to a `Weak` type with static lifetimes first. All managed types have a `Ret` alias, which is the `Weak` alias with static lifetimes. These `Ret`-aliases implement `CCallReturn`. Converting managed data to a `Ret` type is a matter of calling `Managed::leak`. + ```rust,ignore use jlrs::{ data::managed::value::typed::{TypedValue, TypedValueRet}, @@ -30,7 +31,9 @@ julia_module! { fn add(a: f64, b: f64) -> TypedValueRet; } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -38,11 +41,13 @@ Main.JuliaModuleTutorial julia> JuliaModuleTutorial.add(1.0, 2.0) 3.0 ``` + -We didn't have to create a scope because a `WeakHandle` is a non-rooting target itself. We can skip rooting the data because we call no other functions that could hit a safepoint before returning from `add`. The `weak_handle!` macro must be used in combination with `match` or `if let`, we can't `unwrap` or `expect` it. +We didn't have to create a scope because a `WeakHandle` is a weak target itself. We can skip rooting the data because we call no other functions that could hit a safepoint before returning from `add`. The `weak_handle!` macro must be used in combination with `match` or `if let`, we can't `unwrap` or `expect` it. We can return arrays the same way, all `ArrayBase` aliases have a `Ret`-alias. + ```rust,ignore use jlrs::{data::managed::array::TypedMatrixRet, prelude::*, weak_handle}; @@ -65,7 +70,9 @@ julia_module! { fn new_matrix(rows: usize, cols: usize) -> TypedMatrixRet; } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -77,3 +84,4 @@ julia> JuliaModuleTutorial.new_matrix(UInt(4), UInt(2)) 0.0 0.0 0.0 0.0 ``` + diff --git a/src/12-julia-module/functions/throwing-exceptions.md b/src/11-julia-module/functions/throwing-exceptions.md similarity index 93% rename from src/12-julia-module/functions/throwing-exceptions.md rename to src/11-julia-module/functions/throwing-exceptions.md index f1e6dab..d0543be 100644 --- a/src/12-julia-module/functions/throwing-exceptions.md +++ b/src/11-julia-module/functions/throwing-exceptions.md @@ -2,6 +2,7 @@ Throwing an exception is a matter of returning either `Result` or `JlrsResult`. If the error variant is returned it's thrown as an exception, otherwise the result is unwrapped and returned to Julia. + ```rust,ignore use jlrs::{data::managed::value::ValueRet, prelude::*, weak_handle}; @@ -25,7 +26,9 @@ julia_module! { fn throws_exception() -> Result<(), ValueRet>; } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -36,5 +39,6 @@ Stacktrace: [1] top-level scope @ REPL[2]:1 ``` + Many methods in jlrs have a name ending in `unchecked`, these methods don't catch exceptions. If such a method is called and an exception is thrown, there must be no pending drops because control flow will directly jump back to Julia. It's recommended to always catch exceptions and rethrow them as in the example. diff --git a/src/12-julia-module/functions/typed-layouts.md b/src/11-julia-module/functions/typed-layouts.md similarity index 95% rename from src/12-julia-module/functions/typed-layouts.md rename to src/11-julia-module/functions/typed-layouts.md index 69e96d2..fb27672 100644 --- a/src/12-julia-module/functions/typed-layouts.md +++ b/src/11-julia-module/functions/typed-layouts.md @@ -2,6 +2,7 @@ If the layout of an immutable type has one or more elided type parameters, the layout doesn't map to a single Julia type and can't implement `ConstructType`. This prevents us from using it as an argument type, despite the fact that it could be passed by value. Just like `TypedValue` let us annotate a `Value` with its type constructor, we can use `TypedLayout` to annotate a layout with its type constructor. + ```rust,ignore use jlrs::{ data::{layout::typed_layout::TypedLayout, types::construct_type::ConstantBool}, @@ -41,7 +42,9 @@ julia_module! { fn get_inner(he: TypedLayout) -> i32; } ``` + + ```julia julia> module JuliaModuleTutorial using JlrsCore.Wrap @@ -67,3 +70,4 @@ ERROR: MethodError: no method matching get_inner(::Main.JuliaModuleTutorial.HasE Closest candidates are: get_inner(::Main.JuliaModuleTutorial.HasElided{true}) ``` + diff --git a/src/12-julia-module/functions/typed-values.md b/src/11-julia-module/functions/typed-values.md similarity index 91% rename from src/12-julia-module/functions/typed-values.md rename to src/11-julia-module/functions/typed-values.md index 9d27fde..359a59c 100644 --- a/src/12-julia-module/functions/typed-values.md +++ b/src/11-julia-module/functions/typed-values.md @@ -2,6 +2,7 @@ While it's nice that we can use `Value` to handle argument types that don't implement `CCallArg`, it's annoying that this doesn't introduce any restrictions on that argument. A `TypedValue` is a `Value` that has been annotated with its type constructor. When we use it as an argument type, the generated function will restrict that argument to that type object and pass it to `ccall` as `Any`. + ```rust,ignore use jlrs::{data::managed::value::typed::TypedValue, prelude::*}; @@ -18,7 +19,9 @@ julia_module! { fn add(a: TypedValue, b: TypedValue) -> f64; } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -32,3 +35,4 @@ ERROR: MethodError: no method matching add(::Int64, ::Float64) Closest candidates are: add(::Float64, ::Float64) ``` + diff --git a/src/12-julia-module/generic-functions/generic-functions.md b/src/11-julia-module/generic-functions/generic-functions.md similarity index 89% rename from src/12-julia-module/generic-functions/generic-functions.md rename to src/11-julia-module/generic-functions/generic-functions.md index 8f735da..5b4e975 100644 --- a/src/12-julia-module/generic-functions/generic-functions.md +++ b/src/11-julia-module/generic-functions/generic-functions.md @@ -2,6 +2,7 @@ All functions in Julia are generic, we can add new methods as long as the argument types are different from existing methods. If a generic function in Rust takes an argument `T`, we can export it multiple times with different types. + ```rust,ignore use jlrs::prelude::*; @@ -17,7 +18,9 @@ julia_module! { } } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -37,9 +40,11 @@ Closest candidates are: return_first_arg(::Int64, ::Int64) @ Main.JuliaModuleTutorial none:0 ``` + It's not necessary to use this for-loop construction, it's also valid to repeat the export with the generic types filled in. + ```rust,ignore use jlrs::prelude::*; @@ -54,9 +59,11 @@ julia_module! { fn return_first_arg(a: f64, b: f64) -> f64; } ``` + A type parameter may appear in arbitrary positions, the next example requires enabling the `complex` feature. + ```rust,ignore use jlrs::{data::layout::complex::Complex, prelude::*}; @@ -72,7 +79,9 @@ julia_module! { } } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -83,3 +92,4 @@ julia> JuliaModuleTutorial.real_part(ComplexF32(1.0, 2.0)) julia> JuliaModuleTutorial.real_part(ComplexF64(1.0, 2.0)) 1.0 ``` + diff --git a/src/12-julia-module/generic-functions/type-environment.md b/src/11-julia-module/generic-functions/type-environment.md similarity index 96% rename from src/12-julia-module/generic-functions/type-environment.md rename to src/11-julia-module/generic-functions/type-environment.md index 0e5af36..86797a5 100644 --- a/src/12-julia-module/generic-functions/type-environment.md +++ b/src/11-julia-module/generic-functions/type-environment.md @@ -4,6 +4,7 @@ In every function we've exported so far, pretty much all argument and return typ We can use the `tvar!` macro to create a type parameter, this macro only supports single-character names. To create a type parameter `C`, we use `tvar!('C')`. The environment can be created with the `tvars!` macro, which must contain all used parameters in a valid order. The types in the signature must not include any bounds, bounds must only be used in the environment. To create the typevar `C` with an upper bound, we use `tvar!('C'; UpperBoundType)` where `UpperBoundType` is the type constructor of the upper bound. Rust macro's don't like `<` in this position so the name and bounds are seperated with a semicolon instead of `<:`. + ```rust,ignore use jlrs::{ data::{ @@ -33,7 +34,9 @@ julia_module! { fn print_args(_array: TypedValue, _data: TypedValue) use GenericEnv; } ``` + + ```julia julia> module JuliaModuleTutorial ... end Main.JuliaModuleTutorial @@ -59,6 +62,7 @@ Closest candidates are: print_args(::A, ::T) where {T<:AbstractFloat, N, A<:AbstractArray{T, N}} @ Main.JuliaModuleTutorial none:0 ``` + To rename a function that uses a type environment, we have to put `as {{name}}` before `use {{EnvType}}`. diff --git a/src/12-julia-module/julia-module.md b/src/11-julia-module/julia-module.md similarity index 81% rename from src/12-julia-module/julia-module.md rename to src/11-julia-module/julia-module.md index 45890a9..07d5131 100644 --- a/src/12-julia-module/julia-module.md +++ b/src/11-julia-module/julia-module.md @@ -1,6 +1,6 @@ # `julia_module!` -In the previous chapter we've created a dynamic library that exposed Rust code to Julia without using jlrs. Using jlrs provides many additional features, including better support for custom types, code generation, and integration with Julia code. The main disadvantage is that our library won't be compatible with different versions of Julia, but is compiled for the specific version selected with a version feature. +In the previous chapter we've created a dynamic library that exposed Rust code to Julia without using jlrs. Using jlrs provides many additional features, including better support for custom types, code generation, and integration with Julia code. The main disadvantage is that our library won't be compatible with different versions of Julia, but is compiled for the specific version. In this chapter we'll use the `julia_module!` macro to export constants, types and functions to Julia. To use this make we have to enable the `jlrs-derive` and `ccall` features. @@ -8,7 +8,7 @@ In this chapter we'll use the `julia_module!` macro to export constants, types a [package] name = "julia_module_tutorial" version = "0.1.0" -edition = "2021" +edition = "2024" [profile.dev] panic = "abort" @@ -17,18 +17,12 @@ panic = "abort" panic = "abort" [features] -julia-1-6 = ["jlrs/julia-1-6"] -julia-1-7 = ["jlrs/julia-1-7"] -julia-1-8 = ["jlrs/julia-1-8"] -julia-1-9 = ["jlrs/julia-1-9"] -julia-1-10 = ["jlrs/julia-1-10"] -julia-1-11 = ["jlrs/julia-1-11"] [lib] crate-type = ["cdylib"] [dependencies] -jlrs = { version = "0.21", features = ["jlrs-derive", "ccall"] } +jlrs = { version = "0.22", features = ["jlrs-derive", "ccall"] } ``` It's important that we don't enable any runtime features like `local-rt` when we build a dynamic library. diff --git a/src/11-julia-module/opaque-and-foreign-types/foreign-type.md b/src/11-julia-module/opaque-and-foreign-types/foreign-type.md new file mode 100644 index 0000000..7071234 --- /dev/null +++ b/src/11-julia-module/opaque-and-foreign-types/foreign-type.md @@ -0,0 +1,103 @@ +# `ForeignType` + +Foreign types are a special kind of opaque type which may contain references to managed data, but can't have type parameters. Instead of `OpaqueType` directly, we'll need to derive `ForeignType`. The only attibute that still applies is `super_type`. + +Like `OpaqueType`, a type can only implement `ForeignType` if it is `'static` and implements `Send` and `Sync`. This is problematic, since managed data has lifetimes and doesn't implement those traits. To work around the first issue, we need to use `Weak` data because it allows the `'scope`-lifetime to be erased. To work around the second, we'll need to manually implement `Send` and `Sync`. As long as the Rust parts of the type implement these traits, it should be fine to assume they can be implemented; all managed data is hidden behind `Weak`, which lets us ignore the issue until we actually try to access it because these considerations are already a part of the safety contract of accessing weakly-referenced managed data. + +## Mark functions + +Foreign types can reference managed data because they have custom mark functions. When the GC encounters an instance of a foreign type during the mark phase, this custom function is called to allow the GC to find those live references. If the type of a field implements the `Mark` trait, that field can simply be annotated with `#[jlrs((mark)]` to include it in the generated mark function. This trait is implemented for all `Weak` types, `Option`, and arrays and `Vec`s of types that implement `Mark` themselves. + +If a field has a type that references Julia data but doesn't implement `Mark`, e.g. `HashMap`, we'll need to implement a custom marking function manually and annotate the field with `#[jlrs(mark_with = custom_mark_fn)]`. This custom function for `T` must have this signature: `unsafe fn custom_mark_fn(&T, ptls: PTls, parent: &P) -> usize`. This is the same signature as `Mark::mark`; unlike `Mark`, custom fuctions can be implemented even for types defined in other crates. + +To implement a custom mark function correctly, we must mark every instance of Julia data referenced by that type. In the case of `HashMap`, this means iterating over all values in the map and calling `Mark::mark` on every one with the provided `ptls` and `parent`, and returning the sum of their return values. + +## Write barriers + +A custom mark function isn't the only thing we need to maintain GC invariants, we'll use the word object to refer to an instance of a managed type. The GC has two generations, young and old. A newly allocated object is young, if it survives a collection cycle it becomes old. The GC can do a full collection cycle and look at both generations, or an incremental one and just look at the young generation. If a young object is only referenced by an old one, we hit a snag: an incremental run only looks at young objects, so it should never see that reference in an old object and free it. To prevent this from happening, we have to insert a write barrier whenever we start referencing an object that might be young. Two cases where a write barrier must be inserted are setting a field to another object, and adding an object to a collection. + + +```rust,ignore +use std::collections::HashMap; + +use jlrs::{ + data::{managed::value::{typed::{TypedValue, TypedValueRet}, ValueRet}, types::foreign_type::{mark::Mark, ForeignType}}, prelude::*, weak_handle +}; + +// We can introduce additional generics as long as they can be inferred. +unsafe fn mark_map( + data: &HashMap<(), M>, + ptls: jlrs::memory::PTls, + parent: &P, +) -> usize { + data.values().map(|v| unsafe { v.mark(ptls, parent) }).sum() +} + +#[derive(ForeignType)] +pub struct ForeignThing { + #[jlrs(mark)] + a: WeakValue<'static, 'static>, + #[jlrs(mark_with = mark_map)] + b: HashMap<(), WeakValue<'static, 'static>>, +} + +unsafe impl Send for ForeignThing {} +unsafe impl Sync for ForeignThing {} + +impl ForeignThing { + pub fn new(value: Value<'_, 'static>) -> TypedValueRet { + match weak_handle!() { + Ok(handle) => { + TypedValue::new( + handle, + ForeignThing { + a: value.leak(), + b: HashMap::default(), + }, + ) + .leak() + }, + Err(_) => panic!("not called from Julia"), + } + } + + pub fn get(&self) -> ValueRet { + unsafe { self.a.assume_owned().leak() } + } + + pub fn set(&mut self, value: Value) { + unsafe { + self.a = value.assume_owned().leak(); + self.write_barrier(self.a, self); + } + } +} + +julia_module! { + become julia_module_tutorial_init_fn; + + struct ForeignThing; + in ForeignThing fn new(value: Value<'_, 'static>) -> TypedValueRet as ForeignThing; + in ForeignThing fn get(&self) -> ValueRet; + in ForeignThing fn set(&mut self, value: Value); +} +``` + + + +```julia +julia> module JuliaModuleTutorial ... end +Main.JuliaModuleTutorial + +julia> v = JuliaModuleTutorial.ForeignThing(Float32(3.0)) +Main.JuliaModuleTutorial.ForeignThing() + +julia> JuliaModuleTutorial.get(v) +3.0 + +julia> JuliaModuleTutorial.set(v, Float32(4.0)) + +julia> JuliaModuleTutorial.get(v) +4.0 +``` + diff --git a/src/12-julia-module/opaque-and-foreign-types/opaque-and-foreign-types.md b/src/11-julia-module/opaque-and-foreign-types/opaque-and-foreign-types.md similarity index 60% rename from src/12-julia-module/opaque-and-foreign-types/opaque-and-foreign-types.md rename to src/11-julia-module/opaque-and-foreign-types/opaque-and-foreign-types.md index b36cece..dad4f73 100644 --- a/src/12-julia-module/opaque-and-foreign-types/opaque-and-foreign-types.md +++ b/src/11-julia-module/opaque-and-foreign-types/opaque-and-foreign-types.md @@ -2,4 +2,4 @@ In the previous section we've seen how an exported function can take and return data, which was always backed by some type that exists in Julia. In the previous chapter, when we avoided using jlrs entirely, we saw that if we wanted to expose custom types we had to hide them behind void pointers. Because we're no longer avoiding jlrs, we can create new types for our custom types and use them in the exported API. -We'll see that we can distinguish between two kinds of custom types, opaque and foreign types. Opaque types are opaque to Julia and can't reference managed data. Foreign types can reference managed data, we'll need to implement a custom mark function so the GC can find those references. +We can distinguish between two kinds of custom types, opaque and foreign types. Opaque types can't reference managed data, foreign types can. Both are implemented with derive macros. These types, their methods, and associated functions, can be exported. diff --git a/src/11-julia-module/opaque-and-foreign-types/opaque-type.md b/src/11-julia-module/opaque-and-foreign-types/opaque-type.md new file mode 100644 index 0000000..5bdbf40 --- /dev/null +++ b/src/11-julia-module/opaque-and-foreign-types/opaque-type.md @@ -0,0 +1,9 @@ +# `OpaqueType` + +The `OpaqueType` trait is the simplest way to expose a Rust type to Julia. When an opaque type is exported a new mutable type in Julia is created that can contain an instance of that type, Julia is unaware of the internal layout of the type. + +Be aware `OpaqueType` is an unsafe trait. We have to initialize the type before we can use, which is handled by exporting it, and must not access its contents outside of our library. An opaque type must not reference managed data in any way: the layout of this type is unknown to Julia, so the GC would be unable to find those references. It can't contain any references to Rust data either; it must be `'static`, and implement `Send` and `Sync`. Rust doesn't have a stable ABI and libraries can be built with different versions of Rust, so it's unsound to access opaque data outside the library that has exported it. + +Types that implement `OpaqueType` automatically implement the following traits: `IntoJulia`, `ConstructType`, `Typecheck`, and `ValidLayout`. If the type implements `Clone`, `Unbox` is also implemented. + +We can derive `OpaqueType` if a type meets these constraints. Let's look at a few examples. diff --git a/src/11-julia-module/opaque-and-foreign-types/opaque-type/other-attributes.md b/src/11-julia-module/opaque-and-foreign-types/opaque-type/other-attributes.md new file mode 100644 index 0000000..131bcec --- /dev/null +++ b/src/11-julia-module/opaque-and-foreign-types/opaque-type/other-attributes.md @@ -0,0 +1,7 @@ +# Other attributes + +There are two attributes that we haven't covered yet: `super_type` and `bounds`. These attributes can be used to set the super type and bound on the type parameters in Julia. + +The first works similarly to `key`. The syntax is `#[jlrs(super_type = "path::to::Type")]`, the type must implement `ConstructType` and must not use any of the type parameters. The latter restriction may be dropped in the future. + +The second is admittedly of limited use, because the only variants that can be used are those that have been explicitly exported. Its syntax is very similar to Julia; just replace the upper (and/or lower) bound with the path to a constructible type: `#[jlrs(bounds = "T <: path::to::Type, ...")]`. diff --git a/src/11-julia-module/opaque-and-foreign-types/opaque-type/with-generics.md b/src/11-julia-module/opaque-and-foreign-types/opaque-type/with-generics.md new file mode 100644 index 0000000..6fded4e --- /dev/null +++ b/src/11-julia-module/opaque-and-foreign-types/opaque-type/with-generics.md @@ -0,0 +1,74 @@ +# With generics + +Opaque types can have generics, which are exposed as type parameters to Julia. In the best-case scenario, we don't have to deal with bounds on the struct type. + +Opaque types must be `'static` and implement `Send` and `Sync`, so any generic must obey these constraints as well. They must also implement `ConstructType` because this type information is exposed to Julia. + +We need to export an opaque type with parameters with every supported type parameter. The same is true for its methods, a separate function is generated for every combination of types. To do so effectively, we can loop through an array of types. + +It can be useful to expose aliases for specific exported types. This alias can only be used as a constructor if that method is exposed again under the alias's name. + + +```rust,ignore +use std::fmt::Debug; + +use jlrs::{ + data::{ + managed::{ccall_ref::CCallRefRet, value::typed::TypedValue}, + types::construct_type::ConstructType, + }, + prelude::*, + weak_handle, +}; + +#[derive(Debug, OpaqueType)] +struct Opaque { + _a: T, +} + +impl Opaque { + fn new(a: T) -> CCallRefRet> { + match weak_handle!() { + Ok(handle) => CCallRefRet::new(TypedValue::new(handle, Opaque { _a: a }).leak()), + Err(_) => panic!("not called from Julia"), + } + } + + fn print(&self) { + println!("{:?}", self) + } +} + +julia_module! { + become julia_module_tutorial_init_fn; + + for T in [f32, f64] { + struct Opaque; + in Opaque fn new(a: T) -> CCallRefRet> as Opaque; + in Opaque fn print(&self); + }; + + type OpaqueF32 = Opaque; + in Opaque fn new(a: f32) -> CCallRefRet> as OpaqueF32; +} +``` + + + +```julia +julia> module JuliaModuleTutorial ... end +Main.JuliaModuleTutorial + +julia> v = JuliaModuleTutorial.Opaque(Float32(3.0)) +Main.JuliaModuleTutorial.Opaque{Float32}() + +julia> v = JuliaModuleTutorial.Opaque(Float64(3.0)) +Main.JuliaModuleTutorial.Opaque{Float64}() + +julia> v = JuliaModuleTutorial.OpaqueF32(Float32(3.0)) +Main.JuliaModuleTutorial.Opaque{Float32}() + +julia> JuliaModuleTutorial.print(v) +Opaque { _a: 3.0 } +``` + diff --git a/src/11-julia-module/opaque-and-foreign-types/opaque-type/with-restrictions.md b/src/11-julia-module/opaque-and-foreign-types/opaque-type/with-restrictions.md new file mode 100644 index 0000000..54f7b94 --- /dev/null +++ b/src/11-julia-module/opaque-and-foreign-types/opaque-type/with-restrictions.md @@ -0,0 +1,75 @@ +# With restrictions + +In the previous section we've seen that we can derive `OpaqueType` for types with generics. There's a limitation, though: if a generic of the type is bounded by a trait which isn't implemented for `()`, deriving the trait will fail. + +Every opaque type has an associated `Key` type, which must be unique and the same for the entire family of types (i.e. it must not depend on any of the generics). It's normally generated by replacing all generics with `()`. We have to provide a custom `Key` if this default type is rejected due to bounds on the type, it can be set with the `#[jlrs(key = "path::to::Type")]` attribute. The key type must implement `Any`. In practice, it's best to use one of the exported variants or a custom zero-sized type. + + +```rust,ignore +use std::fmt::Debug; + +use jlrs::{ + data::{ + managed::{ccall_ref::CCallRefRet, value::typed::TypedValue}, + types::construct_type::ConstructType, + }, + prelude::*, + weak_handle, +}; + +pub trait IsFloat {} +impl IsFloat for f32 {} +impl IsFloat for f64 {} + +#[derive(Debug, OpaqueType)] +#[jlrs(key = "Opaque")] +struct Opaque { + _a: T, +} + +impl Opaque { + fn new(a: T) -> CCallRefRet> { + match weak_handle!() { + Ok(handle) => CCallRefRet::new(TypedValue::new(handle, Opaque { _a: a }).leak()), + Err(_) => panic!("not called from Julia"), + } + } + + fn print(&self) { + println!("{:?}", self) + } +} + +julia_module! { + become julia_module_tutorial_init_fn; + + for T in [f32, f64] { + struct Opaque; + in Opaque fn new(a: T) -> CCallRefRet> as Opaque; + in Opaque fn print(&self); + }; + + type OpaqueF32 = Opaque; + in Opaque fn new(a: f32) -> CCallRefRet> as OpaqueF32; +} +``` + + + +```julia +julia> module JuliaModuleTutorial ... end +Main.JuliaModuleTutorial + +julia> v = JuliaModuleTutorial.Opaque(Float32(3.0)) +Main.JuliaModuleTutorial.Opaque{Float32}() + +julia> v = JuliaModuleTutorial.Opaque(Float64(3.0)) +Main.JuliaModuleTutorial.Opaque{Float64}() + +julia> v = JuliaModuleTutorial.OpaqueF32(Float32(3.0)) +Main.JuliaModuleTutorial.Opaque{Float32}() + +julia> JuliaModuleTutorial.print(v) +Opaque { _a: 3.0 } +``` + diff --git a/src/11-julia-module/opaque-and-foreign-types/opaque-type/without-generics.md b/src/11-julia-module/opaque-and-foreign-types/opaque-type/without-generics.md new file mode 100644 index 0000000..fceaa3f --- /dev/null +++ b/src/11-julia-module/opaque-and-foreign-types/opaque-type/without-generics.md @@ -0,0 +1,59 @@ +# Without generics + +As mentioned before, `OpaqueType` can be derived if a type doesn't contain any references to either Rust or Julia data, and implements `Send` and `Sync`. A type that implements this trait must be exported with `struct {{Name}}`. Methods and associated functions can be exported with `in {{Type}} {{Signature}}`. + +When an exported method is called from Julia, an instance of the opaque type must be used as the first argument. This data is tracked before it is converted to a reference, which guarantees that mutable aliasing is prevented. It's possible to opt out of tracking by annotating the export with `#[untracked_self]`, which is safe to use if no methods which take `&mut self` are exported. `#[gc_safe]` is also supported. + +To create a constructor, we can export an (associated) function and rename it to the name of the type. The constructor must return either a `CCallRefRet`, a `TypedValueRet`, or a `ValueRet`; opaque types implement `IntoJulia`, so they can be converted with `(Typed)Value::new`. A finalizer that drops the data is automatically registered. + + +```rust,ignore +use jlrs::{ + data::managed::{ccall_ref::CCallRefRet, value::typed::TypedValue}, + prelude::*, + weak_handle, +}; + +#[derive(Debug, OpaqueType)] +struct OpaqueInt { + _a: i32, +} + +impl OpaqueInt { + fn new(a: i32) -> CCallRefRet { + match weak_handle!() { + Ok(handle) => CCallRefRet::new(TypedValue::new(handle, OpaqueInt { _a: a }).leak()), + Err(_) => panic!("not called from Julia"), + } + } + + fn print(&self) { + println!("{:?}", self) + } +} + +julia_module! { + become julia_module_tutorial_init_fn; + + struct OpaqueInt; + + in OpaqueInt fn new(a: i32) -> CCallRefRet as OpaqueInt; + + #[untracked_self] + in OpaqueInt fn print(&self); +} +``` + + + +```julia +julia> module JuliaModuleTutorial ... end +Main.JuliaModuleTutorial + +julia> v = JuliaModuleTutorial.OpaqueInt(Int32(3)) +Main.JuliaModuleTutorial.OpaqueInt() + +julia> JuliaModuleTutorial.print(v) +OpaqueInt { _a: 3 } +``` + diff --git a/src/11-julia-module/type-aliases/type-aliases.md b/src/11-julia-module/type-aliases/type-aliases.md new file mode 100644 index 0000000..253cd34 --- /dev/null +++ b/src/11-julia-module/type-aliases/type-aliases.md @@ -0,0 +1,64 @@ +# Type aliases + +Sometimes we don't want to rename a type but create additional aliases for it, This is particularly useful with parametric opaque types whose constructor can't infer its parameters from the arguments. + +The syntax is `type {{Name}} = {{TypeConstructor}}`. The alias doesn't inherit any constructors, they must be defined for every alias separately. + + +```rust,ignore +use std::fmt::Debug; + +use jlrs::{ + data::{ + managed::{ccall_ref::CCallRefRet, value::typed::TypedValue}, + types::construct_type::ConstructType, + }, + prelude::*, + weak_handle, +}; + +#[derive(Debug, OpaqueType)] +struct Opaque { + _a: T, +} + +impl Opaque { + fn new(a: T) -> CCallRefRet> { + match weak_handle!() { + Ok(handle) => CCallRefRet::new(TypedValue::new(handle, Opaque { _a: a }).leak()), + Err(_) => panic!("not called from Julia"), + } + } + + fn print(&self) { + println!("{:?}", self) + } +} + +julia_module! { + become julia_module_tutorial_init_fn; + + for T in [f32, f64] { + struct Opaque; + in Opaque fn new(a: T) -> CCallRefRet> as Opaque; + in Opaque fn print(&self); + }; + + type OpaqueF32 = Opaque; + in Opaque fn new(a: f32) -> CCallRefRet> as OpaqueF32; +} +``` + + + +```julia +julia> module JuliaModuleTutorial ... end +Main.JuliaModuleTutorial + +julia> v = JuliaModuleTutorial.OpaqueF32(Float32(3.0)) +Main.JuliaModuleTutorial.Opaque{Float32}() + +julia> JuliaModuleTutorial.print(v) +Opaque { _a: 3.0 } +``` + diff --git a/src/12-julia-module/yggdrasil-and-jlrs/yggdrasil-and-jlrs.md b/src/11-julia-module/yggdrasil-and-jlrs/yggdrasil-and-jlrs.md similarity index 62% rename from src/12-julia-module/yggdrasil-and-jlrs/yggdrasil-and-jlrs.md rename to src/11-julia-module/yggdrasil-and-jlrs/yggdrasil-and-jlrs.md index ca8e797..a976026 100644 --- a/src/12-julia-module/yggdrasil-and-jlrs/yggdrasil-and-jlrs.md +++ b/src/11-julia-module/yggdrasil-and-jlrs/yggdrasil-and-jlrs.md @@ -1,19 +1,6 @@ # Yggdrasil and jlrs -In the previous chapter we saw how we could write a recipe to build a Rust crate and contribute it to Yggdrasil to distribute it as a JLL package. When the crate we want to build depends on jlrs, we have to deal with a complication: we need to build the library against every version of Julia that we want to support, and enable the correct version feature at compile time. We'll also need to enable the `yggdrasil` feature. This requires a few adjustments to the recipe. - -We're going to assume the crate re-exposes the version and `yggdrasil` features: - -```toml -[features] -julia-1-6 = ["jlrs/julia-1-6"] -julia-1-7 = ["jlrs/julia-1-7"] -julia-1-8 = ["jlrs/julia-1-8"] -julia-1-9 = ["jlrs/julia-1-9"] -julia-1-10 = ["jlrs/julia-1-10"] -julia-1-11 = ["jlrs/julia-1-11"] -yggdrasil = ["jlrs/yggdrasil"] -``` +In the previous chapter we saw how we could write a recipe to build a Rust crate and contribute it to Yggdrasil to distribute it as a JLL package. When the crate we want to build depends on jlrs, we have to deal with a complication: we need to build the library against every version of Julia that we want to support. This requires a few adjustments to the recipe. The recipe should look as follows: @@ -29,7 +16,7 @@ delete!(Pkg.Types.get_last_stdlibs(v"1.6.3"), uuid) name = "{{crate_name}}" version = v"0.1.0" -julia_versions = [v"1.6.3", v"1.7", v"1.8", v"1.9", v"1.10", v"1.11"] +julia_versions = [v"1.10", v"1.11", v"1.12"] # Collection of sources required to complete build sources = [ @@ -40,23 +27,7 @@ sources = [ # Bash recipe for building across all platforms script = raw""" cd $WORKSPACE/srcdir/{{crate_name}} - -# This program prints the version feature that must be passed to `cargo build` -# Adapted from ../../G/GAP/build_tarballs.jl -# HACK: determine Julia version -cat > version.c < -#include "julia/julia_version.h" -int main(int argc, char**argv) -{ - printf("julia-%d-%d", JULIA_VERSION_MAJOR, JULIA_VERSION_MINOR); - return 0; -} -EOF -${CC_BUILD} -I${includedir} -Wall version.c -o julia_version -julia_version=$(./julia_version) - -cargo build --features yggdrasil,${julia_version} --release --verbose +cargo build --release --verbose install_license LICENSE install -Dvm 0755 "target/${rust_target}/release/"*{{crate_name}}".${dlext}" "${libdir}/lib{{crate_name}}.${dlext}" """ @@ -81,15 +52,13 @@ dependencies = [ # Build the tarballs. build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; - preferred_gcc_version=v"10", julia_compat="1.6", compilers=[:c, :rust]) + preferred_gcc_version=v"10", julia_compat="1.10", compilers=[:c, :rust]) ``` The main differences with the recipe for a crate that doesn't depend on jlrs are: - The workaround for issue [#2942]. - The supported versions of Julia are set. -- A small executable that prints the version feature we need to enable is built and executed as part of the build script. -- `libjulia/common.jl` is included. - Supported platforms are acquired via `libjulia_platforms`, not `supported_platforms`. - `libjulia_jll` is added to the dependencies as a build dependency. diff --git a/src/12-julia-module/generic-functions/parametric-opaque-types.md b/src/12-julia-module/generic-functions/parametric-opaque-types.md deleted file mode 100644 index 6fa2934..0000000 --- a/src/12-julia-module/generic-functions/parametric-opaque-types.md +++ /dev/null @@ -1,110 +0,0 @@ -# Parametric opaque types - -The `OpaqueType` and `ForeignType` traits create new Julia types without any type parameters, so we can't use these traits when the type has one or more parameters that we want to expose to Julia. Instead, we'll need to implement the `ParametricBase` and `ParametricVariant` traits. - -`ParametricBase` describes the type when its parameters haven't been set to an explicit type. We have to provide a `Key` type which doesn't depend on any of the generics, and the names of all type parameters our Julia type will have. `ParametricVariant` describes a specific variant of the parameteric type and we must provide type constructors for all generics. A parametric opaque type must be exported with every combination of generics that we want to use. - -```rust,ignore -use jlrs::{ - data::{ - managed::value::typed::{TypedValue, TypedValueRet}, - types::{ - construct_type::ConstructType, - foreign_type::{ParametricBase, ParametricVariant}, - }, - }, - impl_type_parameters, impl_variant_parameters, - prelude::*, - weak_handle, -}; - -pub struct ParametricOpaque { - a: T, - b: U, -} - -impl ParametricOpaque -where - T: 'static + Send + Sync + Copy + ConstructType, - U: 'static + Send + Sync + Copy + ConstructType, -{ - fn new(a: T, b: U) -> TypedValueRet> { - match weak_handle!() { - Ok(handle) => { - let data = ParametricOpaque { a, b }; - TypedValue::new(handle, data).leak() - } - Err(_) => panic!("not called from Julia"), - } - } - - fn get_a(&self) -> T { - self.a - } - - fn set_b(&mut self, b: U) -> U { - let old = self.b; - self.b = b; - old - } -} - -// Safety: we've correctly mapped the generics to type parameters -unsafe impl ParametricBase for ParametricOpaque -where - T: 'static + Send + Sync + Copy + ConstructType, - U: 'static + Send + Sync + Copy + ConstructType, -{ - type Key = ParametricOpaque<(), ()>; - impl_type_parameters!('T', 'U'); -} - -// Safety: we've correctly mapped the generics to variant parameters -unsafe impl ParametricVariant for ParametricOpaque -where - T: 'static + Send + Sync + Copy + ConstructType, - U: 'static + Send + Sync + Copy + ConstructType, -{ - impl_variant_parameters!(T, U); -} - -julia_module! { - become julia_module_tutorial_init_fn; - - for T in [f32, f64] { - for U in [f32, f64] { - struct ParametricOpaque; - - in ParametricOpaque fn new(a: T, b: U) -> TypedValueRet> as ParametricOpaque; - - in ParametricOpaque fn get_a(&self) -> T; - in ParametricOpaque fn set_b(&mut self, b: U) -> U; - } - } -} -``` - -```julia -julia> module JuliaModuleTutorial ... end -Main.JuliaModuleTutorial - -julia> typeof(JuliaModuleTutorial.ParametricOpaque) -UnionAll - -julia> v = JuliaModuleTutorial.ParametricOpaque(1.0, float(2.0)) -Main.JuliaModuleTutorial.ParametricOpaque{Float64, Float64}() - -julia> JuliaModuleTutorial.get_a(v) -1.0 - -julia> methods(JuliaModuleTutorial.set_b) -# 4 methods for generic function "set_b" from Main.JuliaModuleTutorial: - [1] set_b(arg1::Main.JuliaModuleTutorial.ParametricOpaque{Float64, Float64}, arg2::Float64) - @ none:0 - [2] set_b(arg1::Main.JuliaModuleTutorial.ParametricOpaque{Float64, Float32}, arg2::Float32) - @ none:0 - [3] set_b(arg1::Main.JuliaModuleTutorial.ParametricOpaque{Float32, Float64}, arg2::Float64) - @ none:0 - [4] set_b(arg1::Main.JuliaModuleTutorial.ParametricOpaque{Float32, Float32}, arg2::Float32) - @ none:0 -``` diff --git a/src/12-julia-module/opaque-and-foreign-types/foreign-type.md b/src/12-julia-module/opaque-and-foreign-types/foreign-type.md deleted file mode 100644 index dc4a932..0000000 --- a/src/12-julia-module/opaque-and-foreign-types/foreign-type.md +++ /dev/null @@ -1,104 +0,0 @@ -# `ForeignType` - -Foreign types are very similar to opaque types, the main difference is that a foreign type can contain references to managed data. Instead of `OpaqueType` we'll need to implement `ForeignType`. When we implement this trait, we have to provide a mark function to let the GC find these references. - -Like `OpaqueType`, implementations of `ForeignType` must be thread-safe. A foreign type may only be used in the library that defines it. Fields that reference managed data must use `Ret`-aliases because the `'scope` lifetime has to be erased. One thing that's important to keep in mind is that we whenever we change what managed data is referenced by a field, we must insert a write barrier after this mutation. See [this footnote] for more information. - -To implement the associated `mark` function[^1] we'll need to use `mark_queue_obj` and `mark_queue_objarray` to mark every reference to managed data. We need to sum the result of every call to `mark_queue_obj` and return this sum; `mark_queue_objarray` can be used to mark a slice of references to managed data, this operation doesn't affect the sum. - -```rust,ignore -use jlrs::{ - data::{ - managed::value::{ - typed::{TypedValue, TypedValueRet}, - ValueRet, - }, - memory::PTls, - types::foreign_type::ForeignType, - }, - memory::gc::{mark_queue_obj, write_barrier}, - prelude::*, - weak_handle, -}; - -pub struct ForeignWrapper { - a: ValueRet, - b: ValueRet, -} - -// Safety: Tracking `self` guarantees access to a `ForeignWrapper` is thread-safe. -unsafe impl Send for ForeignWrapper {} -unsafe impl Sync for ForeignWrapper {} - -unsafe impl ForeignType for ForeignWrapper { - fn mark(ptls: PTls, data: &Self) -> usize { - // Safety: We mark all referenced managed data. - unsafe { - let mut n_marked = 0; - n_marked += mark_queue_obj(ptls, data.a) as usize; - n_marked += mark_queue_obj(ptls, data.b) as usize; - n_marked - } - } -} - -impl ForeignWrapper { - fn new(a: Value<'_, 'static>, b: Value<'_, 'static>) -> TypedValueRet { - match weak_handle!() { - Ok(handle) => { - let data = ForeignWrapper { - a: a.leak(), - b: b.leak(), - }; - TypedValue::new(handle, data).leak() - } - Err(_) => panic!("not called from Julia"), - } - } - - fn set_a(&mut self, a: Value<'_, 'static>) { - // Safety: we insert a write barrier after mutating the field - unsafe { - self.a = a.leak(); - write_barrier(self, a); - } - } - - fn get_a(&self) -> ValueRet { - self.a - } -} - -julia_module! { - become julia_module_tutorial_init_fn; - - struct ForeignWrapper; - - in ForeignWrapper fn new(a: Value<'_, 'static>, b: Value<'_, 'static>) - -> TypedValueRet as ForeignWrapper; - - in ForeignWrapper fn set_a(&mut self, a: Value<'_, 'static>); - - in ForeignWrapper fn get_a(&self) -> ValueRet; -} -``` - -```julia -julia> module JuliaModuleTutorial ... end -Main.JuliaModuleTutorial - -julia> x = JuliaModuleTutorial.ForeignWrapper(1, 2) -Main.JuliaModuleTutorial.ForeignWrapper() - -julia> JuliaModuleTutorial.get_a(x) -1 - -julia> JuliaModuleTutorial.set_a(x, 4) - -julia> JuliaModuleTutorial.get_a(x) -4 -``` - -[this footnote]: ../../11-ccall-basics/argument-types/argument-types.md#1 - -[^1]: Yes, the signature of `mark` is odd. It takes `PTls` as its first argument for consistency with `mark_queue_*` and other functions in the Julia C API which take `PTls` explicitly. diff --git a/src/12-julia-module/opaque-and-foreign-types/opaque-type.md b/src/12-julia-module/opaque-and-foreign-types/opaque-type.md deleted file mode 100644 index a29dd9c..0000000 --- a/src/12-julia-module/opaque-and-foreign-types/opaque-type.md +++ /dev/null @@ -1,64 +0,0 @@ -# `OpaqueType` - -The `OpaqueType` trait is the simplest way to expose a Rust type to Julia, for most intents and purposes it's just a marker trait. It's an unsafe trait because we have to initialize the type before we can use it, but this will be handled by exporting it. An opaque type can't reference managed data in any way: the layout of this type is unknown to Julia, so the GC would be unable to find those references. Besides not referencing any Julia data, it can't contain any references to Rust data either and must be thread-safe[^1]. An opaque type may only be used by the library that defines it. - -Any type that implements `OpaqueType` can be exported by adding `struct {{Type}}` to `julia_module!`. When the initialization-function is called, a new mutable type with that name is created in the wrapping module. - -A type just by itself isn't useful, if we tried to export it we'd find the type in our module, but we'd be unable to do anything with it. We can export an opaque type's associated functions and methods almost as easily as we can export other functions, the only additional thing we need to do is prefix the export with `in {{Type}}`. Methods can take `&self` and `&mut self`, if the type implements `Clone` it can also take `self`. The `self` argument is tracked before it's dereferenced to prevent mutable aliasing, it's possible to opt out of this by annotating the method with `#[untracked_self]`. - -```rust,ignore -use jlrs::{ - data::{ - managed::{ccall_ref::CCallRefRet, value::typed::TypedValue}, - types::foreign_type::OpaqueType, - }, - prelude::*, - weak_handle, -}; - -#[derive(Debug)] -struct OpaqueInt { - _a: i32, -} - -unsafe impl OpaqueType for OpaqueInt {} - -impl OpaqueInt { - fn new(a: i32) -> CCallRefRet { - match weak_handle!() { - Ok(handle) => CCallRefRet::new(TypedValue::new(handle, OpaqueInt { _a: a }).leak()), - Err(_) => panic!("not called from Julia"), - } - } - - fn print(&self) { - println!("{:?}", self) - } -} - -julia_module! { - become julia_module_tutorial_init_fn; - - struct OpaqueInt; - - in OpaqueInt fn new(a: i32) -> CCallRefRet as OpaqueInt; - - #[untracked_self] - in OpaqueInt fn print(&self); -} -``` - -```julia -julia> module JuliaModuleTutorial ... end -Main.JuliaModuleTutorial - -julia> v = JuliaModuleTutorial.OpaqueInt(Int32(3)) -Main.JuliaModuleTutorial.OpaqueInt() - -julia> JuliaModuleTutorial.print(v) -OpaqueInt { _a: 3 } -``` - -Note that `OpaqueInt::new` has been renamed to `OpaqueInt` to serve as a constructor. We don't need to track `self` when we call `print` because we never create a mutable reference to `self`. - -[^1]: I.e., the type must be `'static`, `Send` and `Sync`. diff --git a/src/12-julia-module/type-aliases/type-aliases.md b/src/12-julia-module/type-aliases/type-aliases.md deleted file mode 100644 index 5412563..0000000 --- a/src/12-julia-module/type-aliases/type-aliases.md +++ /dev/null @@ -1,98 +0,0 @@ -# Type aliases - -Sometimes we don't want to rename a type but create additional aliases for it, This is particularly useful with parametric opaque types whose constructor can't infer its parameters from the arguments. - -The syntax is `type {{Name}} = {{TypeConstructor}}`. The alias doesn't inherit any constructors, they must be defined for every alias separately. - -```rust,ignore -use std::marker::PhantomData; - -use jlrs::{ - data::{ - managed::{ccall_ref::CCallRefRet, value::typed::TypedValue}, - types::{ - construct_type::ConstructType, - foreign_type::{ParametricBase, ParametricVariant}, - }, - }, - impl_type_parameters, impl_variant_parameters, - prelude::*, - weak_handle, -}; - -pub struct HasParam { - data: isize, - _param: PhantomData, -} - -impl HasParam -where - T: 'static + Send + Sync + ConstructType, -{ - fn new(data: isize) -> CCallRefRet> { - match weak_handle!() { - Ok(handle) => { - let data = HasParam { - data, - _param: PhantomData, - }; - CCallRefRet::new(TypedValue::new(handle, data).leak()) - } - Err(_) => panic!("not called from Julia"), - } - } - - fn data(&self) -> isize { - self.data - } -} - -// Safety: we've correctly mapped the generics to type parameters -unsafe impl ParametricBase for HasParam -where - T: 'static + Send + Sync + ConstructType, -{ - type Key = HasParam<()>; - impl_type_parameters!('T'); -} - -// Safety: we've correctly mapped the generics to variant parameters -unsafe impl ParametricVariant for HasParam -where - T: 'static + Send + Sync + ConstructType, -{ - impl_variant_parameters!(T); -} - -julia_module! { - become julia_module_tutorial_init_fn; - - for T in [f32, f64] { - struct HasParam; - in HasParam fn data(&self) -> isize; - }; - - type HasParam32 = HasParam; - in HasParam fn new(data: isize) -> CCallRefRet> as HasParam32; - - type HasParam64 = HasParam; - in HasParam fn new(data: isize) -> CCallRefRet> as HasParam64; -} -``` - -```julia -julia> module JuliaModuleTutorial ... end -Main.JuliaModuleTutorial - -julia> d = JuliaModuleTutorial.HasParam32(1) -Main.JuliaModuleTutorial.HasParam{Float32}() - -julia> JuliaModuleTutorial.data(d) -1 - -julia> d = JuliaModuleTutorial.HasParam64(2) -Main.JuliaModuleTutorial.HasParam{Float64}() - -julia> JuliaModuleTutorial.data(d) -2 -``` diff --git a/src/13-keyword-arguments/keyword-arguments.md b/src/12-keyword-arguments/keyword-arguments.md similarity index 76% rename from src/13-keyword-arguments/keyword-arguments.md rename to src/12-keyword-arguments/keyword-arguments.md index 2804f13..064d0cf 100644 --- a/src/13-keyword-arguments/keyword-arguments.md +++ b/src/12-keyword-arguments/keyword-arguments.md @@ -6,6 +6,7 @@ Calling a function with custom keyword arguments involves a few small steps: 2. Provide those arguments to the function we want to call with `ProvideKeyword::provide_keywords`. 3. Call the resulting `WithKeywords` instance with the positional arguments; `WithKeywords` implements `Call`. + ```rust,ignore use jlrs::prelude::*; @@ -14,27 +15,28 @@ fn main() { handle.local_scope::<_, 8>(|mut frame| { unsafe { - let func = Value::eval_string(&mut frame, "add(a, b; c=3.0, d=4.0, e=5.0) = a + b + c + d + e") - .expect("an exception occurred"); + let func = Value::eval_string( + &mut frame, + "add(a, b; c=3.0, d=4.0, e=5.0) = a + b + c + d + e" + ) + .expect("an exception occurred"); let a = Value::new(&mut frame, 1.0); let b = Value::new(&mut frame, 2.0); let c = Value::new(&mut frame, 5.0); let d = Value::new(&mut frame, 1.0); - let kwargs = named_tuple!(&mut frame, "c" => c, "d" => d); + let kwargs = named_tuple!(&mut frame, "c" => c, "d" => d) + .expect("invalid keyword arguments"); - let res = func.call2(&mut frame, a, b) + let res = func.call(&mut frame, [a, b]) .expect("caught exception") .unbox::() .expect("not an f64"); assert_eq!(res, 15.0); - let func_with_kwargs = func - .provide_keywords(kwargs) - .expect("invalid keyword arguments"); - - let res = func_with_kwargs.call2(&mut frame, a, b) + let res = func + .call_kw(&mut frame, [a, b], kwargs) .expect("caught exception") .unbox::() .expect("not an f64"); @@ -44,3 +46,4 @@ fn main() { }); } ``` + diff --git a/src/14-safety/safety.md b/src/13-safety/safety.md similarity index 100% rename from src/14-safety/safety.md rename to src/13-safety/safety.md diff --git a/src/15-when-to-leave-things-unrooted/when-to-leave-things-unrooted.md b/src/14-when-to-leave-things-unrooted/when-to-leave-things-unrooted.md similarity index 79% rename from src/15-when-to-leave-things-unrooted/when-to-leave-things-unrooted.md rename to src/14-when-to-leave-things-unrooted/when-to-leave-things-unrooted.md index bea9ec2..4bbe0c7 100644 --- a/src/15-when-to-leave-things-unrooted/when-to-leave-things-unrooted.md +++ b/src/14-when-to-leave-things-unrooted/when-to-leave-things-unrooted.md @@ -1,13 +1,14 @@ # When to leave things unrooted -In this tutorial we've mostly used rooting targets to ensure the managed data we created would remain valid as long as we didn't leave the scope it was tied to. In many cases, though, we don't need to root managed data and can use a non-rooting target without running into any problems. +In this tutorial we've mostly used rooting targets to ensure the managed data we created would remain valid as long as we didn't leave the scope it was tied to. In many cases, though, we don't need to root managed data and can use a weak target without running into any problems. -Some data is globally rooted, most importantly constants defined in modules. When we access such constant data with `Module::get_global`, we can safely skip rooting it and convert the `Ref`-type to a managed type if we want to use it. If the data is global but not constant, it's safe to use it without rooting it if we can guarantee its value never changes as long as we use it from Rust. +Some data is globally rooted, most importantly constants defined in modules. When we access such constant data with `Module::get_global`, we can safely skip rooting it and convert the `Weak` type to a managed type if we want to use it. If the data is global but not constant, it's safe to use it without rooting it if we can guarantee its value never changes as long as we use it from Rust. If a function returns an instance of a zero-sized type like `nothing` we don't need to root the result; there's only one, globally-rooted instance of a zero-sized type. Symbols and instances of booleans and 8-bit integers are also globally rooted. Finally, it's safe to leave data unrooted if we can guarantee the GC won't run until we're done using the data. The GC can be triggered whenever new managed data is allocated.[^1] If the GC determines it needs to run, every thread will be suspended when it reaches a safepoint. The GC runs when all threads have been suspended. If we don't call into Julia while we access the data, we won't hit a safepoint so we can leave it unrooted. + ```rust,ignore use jlrs::prelude::*; @@ -25,7 +26,7 @@ fn main() { let a = Value::new(&mut frame, 1.0); // Safety: We're just calling println with a Float64 argument - let res = unsafe { func.call1(&frame, a).expect("caught exception") }; + let res = unsafe { func.call(&frame, [a]).expect("caught exception") }; // Safety: println returns nothing, which is globally rooted let res = unsafe { res.as_value() }; @@ -33,5 +34,6 @@ fn main() { }); } ``` + [^1]: The GC can also be triggered manually with `Gc::gc_collect`, all targets implement this trait. diff --git a/src/16-caching-julia-data/caching-julia-data.md b/src/15-caching-julia-data/caching-julia-data.md similarity index 91% rename from src/16-caching-julia-data/caching-julia-data.md rename to src/15-caching-julia-data/caching-julia-data.md index 967ab61..21b3e7b 100644 --- a/src/16-caching-julia-data/caching-julia-data.md +++ b/src/15-caching-julia-data/caching-julia-data.md @@ -2,6 +2,7 @@ Accessing data in a module can be expensive, especially if we need to access it often. These accesses can be cached with a `StaticRef`, which can be defined with the `define_static_ref!` macro and accessed with the `static_ref!` macro. + ```rust,ignore use jlrs::{define_static_ref, prelude::*, static_ref}; @@ -15,7 +16,7 @@ fn main() { let v2 = Value::new(&mut frame, 2.0f64); let add_func = static_ref!(ADD_FUNCTION, &frame); - let res = unsafe { add_func.call2(&mut frame, v1, v2) } + let res = unsafe { add_func.call(&mut frame, [v1, v2]) } .expect("caught an exception") .unbox::() .expect("wrong type"); @@ -24,9 +25,11 @@ fn main() { }) } ``` + It's possible to combine these two operations with `inline_static_ref!`, this is useful if we only need to use the data in a single function or want to expose a separate function to access it. + ```rust,ignore use jlrs::{inline_static_ref, prelude::*}; @@ -46,7 +49,7 @@ fn main() { let v2 = Value::new(&mut frame, 2.0f64); let add_func = add_function(&frame); - let res = unsafe { add_func.call2(&mut frame, v1, v2) } + let res = unsafe { add_func.call(&mut frame, [v1, v2]) } .expect("caught an exception") .unbox::() .expect("wrong type"); @@ -55,6 +58,7 @@ fn main() { }) } ``` + A `StaticRef` is thread-safe: it's just an atomic pointer internally, which is initialized when it's first accessed. Any thread that can call into Julia can access it, if multiple threads try to access this data before it has been initialazed, all these threads will try to initialize it. The data is globally rooted so we don't need to root it ourselves.[^1] diff --git a/src/17-cross-language-lto/cross-language-lto.md b/src/16-cross-language-lto/cross-language-lto.md similarity index 68% rename from src/17-cross-language-lto/cross-language-lto.md rename to src/16-cross-language-lto/cross-language-lto.md index 73003f7..114fe75 100644 --- a/src/17-cross-language-lto/cross-language-lto.md +++ b/src/16-cross-language-lto/cross-language-lto.md @@ -2,11 +2,11 @@ At the core of jlrs lives a small static library written in C. This library serves a few purposes: - - It hides implementation details of Julia's C API. - - It exposes functionality implemented in terms of macros and static inline functions. - - It provides work-arounds for backwards-incompatible changes. +- It hides implementation details of Julia's C API. +- It exposes functionality implemented in terms of macros and static inline functions. +- It provides work-arounds for backwards-incompatible changes. -Many operations are delegated to this library, which tend to be very cheap compared to the overhead of calling a function. Because the library is written in C, these functions will never be inlined. +Many operations are delegated to this library, these operation tend to be very cheap compared to the overhead of calling a function. Because the library is written in C, these functions will never be inlined. If we use `clang` to build this library, we can enable cross-language LTO with the `lto` feature if `clang` and `rustc` use the same major LLVM version. We can query what version of clang we need to use with `rustc -vV`. @@ -26,7 +26,7 @@ The relevant information is in the final line: LLVM 18 is used, so we need to us ```bash RUSTFLAGS="-Clinker-plugin-lto -Clinker=clang-18 -Clink-arg=-fuse-ld=lld -Clink-args=-rdynamic" \ CC=clang-18 \ -cargo build --release --features {{julia_version}} +cargo build --release --features jlrs/lto ``` Cross-language LTO has only been tested on Linux, it can be enabled for applications and dynamic libraries. It has no effect on the performance of Julia code, only on Rust code that calls into the intermediate library. diff --git a/src/18-testing-applications/testing-applications.md b/src/17-testing-applications/testing-applications.md similarity index 100% rename from src/18-testing-applications/testing-applications.md rename to src/17-testing-applications/testing-applications.md diff --git a/src/19-testing-libraries/testing-libraries.md b/src/18-testing-libraries/testing-libraries.md similarity index 100% rename from src/19-testing-libraries/testing-libraries.md rename to src/18-testing-libraries/testing-libraries.md diff --git a/src/SUMMARY.md b/src/SUMMARY.md index f167ed1..baa1357 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -9,91 +9,92 @@ - [Rust](./01-dependencies/rust.md) - [C](./01-dependencies/c.md) -- [Version features](./02-version-features/version-features.md) - -- [Basics](./03-basics/basics.md) - - [Project setup](./03-basics/project-setup.md) - - [Scopes and evaluating Julia code](./03-basics/scopes-and-evaluating-julia-code.md) - - [Managed data and functions](./03-basics/julia-data-and-functions.md) - - [Casting, unboxing and accessing managed data](./03-basics/casting-unboxing-and-accessing-julia-data.md) - - [Loading packages and other custom code](./03-basics/loading-packages-and-other-custom-code.md) +- [Basics](./02-basics/basics.md) + - [Project setup](./02-basics/project-setup.md) + - [Scopes and evaluating Julia code](./02-basics/scopes-and-evaluating-julia-code.md) + - [Managed data and functions](./02-basics/julia-data-and-functions.md) + - [Casting, unboxing and accessing managed data](./02-basics/casting-unboxing-and-accessing-julia-data.md) + - [Loading packages and other custom code](./02-basics/loading-packages-and-other-custom-code.md) # Getting familiar -- [Targets](./04-memory-management.md/memory-management.md) - - [Using targets and nested scopes](./04-memory-management.md/using-targets.md) - - [Target types](./04-memory-management.md/target-types.md) - - [Local targets](./04-memory-management.md/local-targets.md) - - [Dynamic targets](./04-memory-management.md/dynamic-targets.md) - - [Non-rooting targets](./04-memory-management.md/non-rooting-targets.md) - -- [Types and layouts](./05-types-and-layouts/types-and-layouts.md) - - [`isbits` layouts](./05-types-and-layouts/isbits-layouts.md) - - [Inline and non-inline layouts](./05-types-and-layouts/inline-and-non-inline-layouts.md) - - [Union fields](./05-types-and-layouts/union-fields.md) - - [Generics](./05-types-and-layouts/generics.md) - -- [Arrays](./06-arrays/arrays.md) - - [Creating arrays](./06-arrays/create-arrays.md) - - [Accessing arrays](./06-arrays/access-arrays.md) - - [Mutating arrays](./06-arrays/mutate-arrays.md) - - [`ndarray`](./06-arrays/ndarray.md) - - [Tracking arrays](./06-arrays/track-arrays.md) - -- [Exception handling](./07-exception-handling/exception-handling.md) - - [Parachutes](./07-exception-handling/parachutes.md) - -- [Bindings and derivable traits](./08-bindings-and-derivable-traits/bindings-and-derivable-traits.md) - - [Generating bindings](./08-bindings-and-derivable-traits/generating-bindings.md) - - [Customizing bindings](./08-bindings-and-derivable-traits/customizing-bindings.md) +- [Targets](./03-memory-management/memory-management) + - [Using targets and nested scopes](./03-memory-management/using-targets.md) + - [Target types](./03-memory-management/target-types.md) + - [Local targets](./03-memory-management/local-targets.md) + - [Dynamic targets](./03-memory-management/dynamic-targets.md) + - [Weak targets](./03-memory-management/weak-targets.md) + +- [Types and layouts](./04-types-and-layouts/types-and-layouts.md) + - [`isbits` layouts](./04-types-and-layouts/isbits-layouts.md) + - [Inline and non-inline layouts](./04-types-and-layouts/inline-and-non-inline-layouts.md) + - [Union fields](./04-types-and-layouts/union-fields.md) + - [Generics](./04-types-and-layouts/generics.md) + +- [Arrays](./05-arrays/arrays.md) + - [Creating arrays](./05-arrays/create-arrays.md) + - [Accessing arrays](./05-arrays/access-arrays.md) + - [Mutating arrays](./05-arrays/mutate-arrays.md) + - [`ndarray`](./05-arrays/ndarray.md) + - [Tracking arrays](./05-arrays/track-arrays.md) + +- [Exception handling](./06-exception-handling/exception-handling.md) + - [Parachutes](./06-exception-handling/parachutes.md) + +- [Bindings and derivable traits](./07-bindings-and-derivable-traits/bindings-and-derivable-traits.md) + - [Generating bindings](./07-bindings-and-derivable-traits/generating-bindings.md) + - [Customizing bindings](./07-bindings-and-derivable-traits/customizing-bindings.md) # Other runtimes -- [Multithreaded runtime](./09-multithreaded-runtime/multithreaded-runtime.md) - - [Garbage collection, locks, and other blocking functions](./09-multithreaded-runtime/garbage-collection-locks-and-other-blocking-functions.md) +- [Multithreaded runtime](./08-multithreaded-runtime/multithreaded-runtime.md) + - [Garbage collection, locks, and other blocking functions](./08-multithreaded-runtime/garbage-collection-locks-and-other-blocking-functions.md) -- [Async runtime](./10-async-runtime/async-runtime.md) - - [Blocking tasks](./10-async-runtime/blocking-tasks.md) - - [Async tasks](./10-async-runtime/async-tasks.md) - - [Persistent tasks](./10-async-runtime/persistent-tasks.md) - - [Combining the multithreaded and async runtimes](./10-async-runtime/combining-the-multithreaded-and-async-runtimes.md) +- [Async runtime](./09-async-runtime/async-runtime.md) + - [Blocking tasks](./09-async-runtime/blocking-tasks.md) + - [Async tasks](./09-async-runtime/async-tasks.md) + - [Persistent tasks](./09-async-runtime/persistent-tasks.md) + - [Combining the multithreaded and async runtimes](./09-async-runtime/combining-the-multithreaded-and-async-runtimes.md) # Dynamic libraries -- [ccall basics](./11-ccall-basics/ccall-basics.md) - - [Argument types](./11-ccall-basics/argument-types/argument-types.md) - - [Arrays](./11-ccall-basics/argument-types/arrays.md) - - [Return type](./11-ccall-basics/return-type.md) - - [Dynamic libraries](./11-ccall-basics/dynamic-libraries.md) - - [Custom types](./11-ccall-basics/custom-types.md) - - [Yggdrasil](./11-ccall-basics/yggdrasil.md) - -- [`julia_module!`](./12-julia-module/julia-module.md) - - [Constants](./12-julia-module/constants/constants.md) - - [Functions](./12-julia-module/functions/functions.md) - - [Managed arguments](./12-julia-module/functions/managed-arguments.md) - - [Array arguments](./12-julia-module/functions/array-arguments.md) - - [Typed values](./12-julia-module/functions/typed-values.md) - - [Typed layouts](./12-julia-module/functions/typed-layouts.md) - - [Returning managed data](./12-julia-module/functions/returning-managed-data.md) - - [`CCallRef`](./12-julia-module/functions/ccall-ref.md) - - [Throwing exceptions](./12-julia-module/functions/throwing-exceptions.md) - - [GC-safety](./12-julia-module/functions/gc-safety.md) - - [Opaque and foreign types](./12-julia-module/opaque-and-foreign-types/opaque-and-foreign-types.md) - - [`OpaqueType`](./12-julia-module/opaque-and-foreign-types/opaque-type.md) - - [`ForeignType`](./12-julia-module/opaque-and-foreign-types/foreign-type.md) - - [Generic functions](./12-julia-module/generic-functions/generic-functions.md) - - [Parametric opaque types](./12-julia-module/generic-functions/parametric-opaque-types.md) - - [Type environment](./12-julia-module/generic-functions/type-environment.md) - - [`Type aliases`](./12-julia-module/type-aliases/type-aliases.md) - - [Yggdrasil and jlrs](./12-julia-module/yggdrasil-and-jlrs/yggdrasil-and-jlrs.md) +- [ccall basics](./10-ccall-basics/ccall-basics.md) + - [Argument types](./10-ccall-basics/argument-types/argument-types.md) + - [Arrays](./10-ccall-basics/argument-types/arrays.md) + - [Return type](./10-ccall-basics/return-type.md) + - [Dynamic libraries](./10-ccall-basics/dynamic-libraries.md) + - [Custom types](./10-ccall-basics/custom-types.md) + - [Yggdrasil](./10-ccall-basics/yggdrasil.md) + +- [`julia_module!`](./11-julia-module/julia-module.md) + - [Constants](./11-julia-module/constants/constants.md) + - [Functions](./11-julia-module/functions/functions.md) + - [Managed arguments](./11-julia-module/functions/managed-arguments.md) + - [Array arguments](./11-julia-module/functions/array-arguments.md) + - [Typed values](./11-julia-module/functions/typed-values.md) + - [Typed layouts](./11-julia-module/functions/typed-layouts.md) + - [Returning managed data](./11-julia-module/functions/returning-managed-data.md) + - [`CCallRef`](./11-julia-module/functions/ccall-ref.md) + - [Throwing exceptions](./11-julia-module/functions/throwing-exceptions.md) + - [GC-safety](./11-julia-module/functions/gc-safety.md) + - [Opaque and foreign types](./11-julia-module/opaque-and-foreign-types/opaque-and-foreign-types.md) + - [`OpaqueType`](./11-julia-module/opaque-and-foreign-types/opaque-type.md) + - [Without generics](./11-julia-module/opaque-and-foreign-types/opaque-type/without-generics.md) + - [With generics](./11-julia-module/opaque-and-foreign-types/opaque-type/with-generics.md) + - [With restrictions](./11-julia-module/opaque-and-foreign-types/opaque-type/with-restrictions.md) + - [Other attributes](./11-julia-module/opaque-and-foreign-types/opaque-type/other-attributes.md) + - [`ForeignType`](./11-julia-module/opaque-and-foreign-types/foreign-type.md) + - [Generic functions](./11-julia-module/generic-functions/generic-functions.md) + - [Type environment](./11-julia-module/generic-functions/type-environment.md) + - [`Type aliases`](./11-julia-module/type-aliases/type-aliases.md) + - [Yggdrasil and jlrs](./11-julia-module/yggdrasil-and-jlrs/yggdrasil-and-jlrs.md) # Other topics -- [Keyword arguments](./13-keyword-arguments/keyword-arguments.md) -- [Safety](./14-safety/safety.md) -- [When to leave things unrooted](./15-when-to-leave-things-unrooted/when-to-leave-things-unrooted.md) -- [Caching Julia data](./16-caching-julia-data/caching-julia-data.md) -- [Cross-language LTO](./17-cross-language-lto/cross-language-lto.md) -- [Testing applications](./18-testing-applications/testing-applications.md) -- [Testing libraries](./19-testing-libraries/testing-libraries.md) +- [Keyword arguments](./12-keyword-arguments/keyword-arguments.md) +- [Safety](./13-safety/safety.md) +- [When to leave things unrooted](./14-when-to-leave-things-unrooted/when-to-leave-things-unrooted.md) +- [Caching Julia data](./15-caching-julia-data/caching-julia-data.md) +- [Cross-language LTO](./16-cross-language-lto/cross-language-lto.md) +- [Testing applications](./17-testing-applications/testing-applications.md) +- [Testing libraries](./18-testing-libraries/testing-libraries.md)