From 65f2b626456514afc765e2ae0e5cd48d33733d36 Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Wed, 9 Apr 2025 00:28:50 -0400 Subject: [PATCH] Add trampoline TCO emulation for unsupported cases --- include/lauf/asm/type.h | 9 ++++- include/lauf/config.h | 6 +-- include/lauf/runtime/builtin.h | 72 ++++++++++++++++++++++++++-------- src/CMakeLists.txt | 6 +++ src/lauf/asm/builder.cpp | 22 +++++++++-- src/lauf/asm/type.cpp | 19 ++++----- src/lauf/lib/bits.cpp | 7 ++-- src/lauf/lib/fiber.cpp | 9 ++--- src/lauf/lib/heap.cpp | 17 ++++---- src/lauf/lib/int.cpp | 48 ++++++++++++----------- src/lauf/lib/limits.cpp | 7 ++-- src/lauf/lib/memory.cpp | 29 +++++++------- src/lauf/lib/test.cpp | 16 ++++---- src/lauf/runtime/process.cpp | 15 +++++++ src/lauf/vm_execute.cpp | 71 +++++++++++++++++---------------- src/lauf/vm_execute.hpp | 43 ++++++++++++++------ 16 files changed, 247 insertions(+), 149 deletions(-) diff --git a/include/lauf/asm/type.h b/include/lauf/asm/type.h index 59366d36..b59e064d 100644 --- a/include/lauf/asm/type.h +++ b/include/lauf/asm/type.h @@ -13,9 +13,17 @@ typedef union lauf_runtime_value lauf_runtime_value; typedef struct lauf_runtime_process lauf_runtime_process; typedef struct lauf_runtime_stack_frame lauf_runtime_stack_frame; +#if LAUF_HAS_TAIL_CALL_ELIMINATION typedef bool lauf_runtime_builtin_impl(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, lauf_runtime_stack_frame* frame_ptr, lauf_runtime_process* process); +#else +typedef union lauf_tail_call_result lauf_tail_call_result; +typedef lauf_tail_call_result lauf_runtime_builtin_impl(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process); +#endif /// The layout of a type. typedef struct lauf_asm_layout @@ -72,4 +80,3 @@ extern const lauf_asm_type lauf_asm_type_value; LAUF_HEADER_END #endif // LAUF_ASM_TYPE_H_INCLUDED - diff --git a/include/lauf/config.h b/include/lauf/config.h index b2461a63..ce324cf2 100644 --- a/include/lauf/config.h +++ b/include/lauf/config.h @@ -59,10 +59,8 @@ typedef uint64_t lauf_uint; # define LAUF_CONFIG_DISPATCH_JUMP_TABLE 1 #endif -#if defined(__has_cpp_attribute) -# if __has_cpp_attribute(clang::musttail) -# define LAUF_HAS_TAIL_CALL_ELIMINATION 1 -# endif +#ifndef LAUF_HAS_TAIL_CALL_ELIMINATION +# define LAUF_HAS_TAIL_CALL_ELIMINATION 1 #endif //=== warnings ===// diff --git a/include/lauf/runtime/builtin.h b/include/lauf/runtime/builtin.h index db8b6675..5841367e 100644 --- a/include/lauf/runtime/builtin.h +++ b/include/lauf/runtime/builtin.h @@ -47,16 +47,23 @@ typedef enum lauf_runtime_builtin_flags LAUF_RUNTIME_BUILTIN_ALWAYS_PANIC = 1 << 4, } lauf_runtime_builtin_flags; +#if LAUF_HAS_TAIL_CALL_ELIMINATION +# define LAUF_BUILTIN_RETURN_TYPE bool +#else +typedef union lauf_tail_call_result lauf_tail_call_result; +# define LAUF_BUILTIN_RETURN_TYPE lauf_tail_call_result +#endif + /// Must be tail-called when a buitlin finishes succesfully. -LAUF_RUNTIME_BUILTIN_IMPL bool lauf_runtime_builtin_dispatch(const lauf_asm_inst* ip, - lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process); +LAUF_RUNTIME_BUILTIN_IMPL LAUF_BUILTIN_RETURN_TYPE lauf_runtime_builtin_dispatch( + const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process); /// The signature of the implementation of a builtin. -typedef bool lauf_runtime_builtin_impl(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process); +typedef LAUF_BUILTIN_RETURN_TYPE lauf_runtime_builtin_impl(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process); /// A builtin function. typedef struct lauf_runtime_builtin @@ -74,16 +81,47 @@ typedef struct lauf_runtime_builtin const lauf_runtime_builtin* next; } lauf_runtime_builtin; -#define LAUF_RUNTIME_BUILTIN(ConstantName, InputCount, OutputCount, Flags, Name, Next) \ - static bool ConstantName##_impl(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, \ - lauf_runtime_stack_frame* frame_ptr, \ - lauf_runtime_process* process); \ - const lauf_runtime_builtin ConstantName \ - = {&ConstantName##_impl, InputCount, OutputCount, Flags, Name, Next}; \ - LAUF_RUNTIME_BUILTIN_IMPL static bool ConstantName##_impl(const lauf_asm_inst* ip, \ - lauf_runtime_value* vstack_ptr, \ - lauf_runtime_stack_frame* frame_ptr, \ - lauf_runtime_process* process) +#if !LAUF_HAS_TAIL_CALL_ELIMINATION +typedef union lauf_tail_call_result +{ + struct + { + bool is_function; + lauf_runtime_builtin_impl* next_func; + const lauf_asm_inst* cur_ip; + lauf_runtime_value* cur_stack_ptr; + lauf_runtime_stack_frame* cur_frame_ptr; + lauf_runtime_process* cur_process; + } function; + struct + { + bool is_function; + bool value; + } value; +} lauf_tail_call_result; + +# define LAUF_BUILTIN_RETURN(...) \ + return \ + { \ + .value \ + { \ + false, __VA_ARGS__ \ + } \ + } +#else +# define LAUF_BUILTIN_RETURN(...) return __VA_ARGS__ +#endif + +#define LAUF_RUNTIME_BUILTIN(ConstantName, InputCount, OutputCount, Flags, Name, Next) \ + static LAUF_BUILTIN_RETURN_TYPE ConstantName##_impl(const lauf_asm_inst* ip, \ + lauf_runtime_value* vstack_ptr, \ + lauf_runtime_stack_frame* frame_ptr, \ + lauf_runtime_process* process); \ + const lauf_runtime_builtin ConstantName \ + = {&ConstantName##_impl, InputCount, OutputCount, Flags, Name, Next}; \ + LAUF_RUNTIME_BUILTIN_IMPL static LAUF_BUILTIN_RETURN_TYPE \ + ConstantName##_impl(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, \ + lauf_runtime_stack_frame* frame_ptr, lauf_runtime_process* process) #define LAUF_RUNTIME_BUILTIN_DISPATCH \ LAUF_TAIL_CALL return lauf_runtime_builtin_dispatch(ip, vstack_ptr, frame_ptr, process) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1545095f..c849d42b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -42,6 +42,12 @@ endif() # They would record all previously executed instructions in the call stack. target_compile_options(lauf_core PRIVATE -fomit-frame-pointer) +if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_definitions(lauf_core PUBLIC LAUF_HAS_TAIL_CALL_ELIMINATION=0) + endif() +endif() + target_sources(lauf_core PUBLIC ${include_dir}/config.h ${include_dir}/reader.h diff --git a/src/lauf/asm/builder.cpp b/src/lauf/asm/builder.cpp index ad24f401..ec3f970b 100644 --- a/src/lauf/asm/builder.cpp +++ b/src/lauf/asm/builder.cpp @@ -839,12 +839,26 @@ void lauf_asm_inst_call_builtin(lauf_asm_builder* b, lauf_runtime_builtin_functi && (callee.flags & LAUF_RUNTIME_BUILTIN_CONSTANT_FOLD) != 0) { assert(vstack_ptr == vstack + UINT8_MAX); - lauf_asm_inst code[3] = {LAUF_BUILD_INST_NONE(nop), - LAUF_BUILD_INST_SIGNATURE(call_builtin_sig, callee.input_count, - callee.output_count, callee.flags), - LAUF_BUILD_INST_NONE(exit)}; + lauf_asm_inst code[3] = {LAUF_BUILD_INST_NONE(nop), + LAUF_BUILD_INST_SIGNATURE(call_builtin_sig, callee.input_count, + callee.output_count, callee.flags), + LAUF_BUILD_INST_NONE(exit)}; + +#if LAUF_HAS_TAIL_CALL_ELIMINATION [[maybe_unused]] auto success = callee.impl(code, vstack_ptr - callee.input_count, nullptr, nullptr); +#else + LAUF_BUILTIN_RETURN_TYPE tail_call_result + = callee.impl(code, vstack_ptr - callee.input_count, nullptr, nullptr); + while (tail_call_result.function.is_function) + { + tail_call_result + = tail_call_result.function.next_func(tail_call_result.function.cur_ip, + tail_call_result.function.cur_stack_ptr, + nullptr, nullptr); + } + [[maybe_unused]] bool success = tail_call_result.value.value; +#endif if (success) { // Pop the input values as the call would. diff --git a/src/lauf/asm/type.cpp b/src/lauf/asm/type.cpp index 7fd44c94..c966a55c 100644 --- a/src/lauf/asm/type.cpp +++ b/src/lauf/asm/type.cpp @@ -3,6 +3,7 @@ #include +#include #include #include #include @@ -25,17 +26,17 @@ lauf_asm_layout lauf_asm_aggregate_layout(const lauf_asm_layout* member_layouts, size += lauf::align_offset(size, member_layouts[i].alignment); size += member_layouts[i].size; - if (member_layouts[i].alignment > alignment) - alignment = member_layouts[i].alignment; + alignment = std::max(member_layouts[i].alignment, alignment); } return {size, alignment}; } namespace { -LAUF_RUNTIME_BUILTIN_IMPL bool load_value(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +LAUF_RUNTIME_BUILTIN_IMPL LAUF_BUILTIN_RETURN_TYPE load_value(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) { vstack_ptr[1] = *static_cast(vstack_ptr[1].as_native_ptr); ++vstack_ptr; @@ -43,9 +44,10 @@ LAUF_RUNTIME_BUILTIN_IMPL bool load_value(const lauf_asm_inst* ip, lauf_runtime_ LAUF_RUNTIME_BUILTIN_DISPATCH; } -LAUF_RUNTIME_BUILTIN_IMPL bool store_value(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +LAUF_RUNTIME_BUILTIN_IMPL LAUF_BUILTIN_RETURN_TYPE store_value(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) { *static_cast(vstack_ptr[1].as_native_ptr) = vstack_ptr[2]; vstack_ptr += 3; @@ -60,4 +62,3 @@ const lauf_asm_type lauf_asm_type_value = {LAUF_ASM_NATIVE_LAYOUT_OF(lauf_runtim &store_value, "lauf.Value", nullptr}; - diff --git a/src/lauf/lib/bits.cpp b/src/lauf/lib/bits.cpp index 08e7f111..89c10e3f 100644 --- a/src/lauf/lib/bits.cpp +++ b/src/lauf/lib/bits.cpp @@ -52,7 +52,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_bits_shl, 2, 1, panic_flags, "shl", &lauf_lib_bits ++vstack_ptr; if (LAUF_UNLIKELY(n >= sizeof(lauf_uint) * CHAR_BIT)) - return lauf_runtime_panic(process, "shift amount too big"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "shift amount too big")); vstack_ptr[0].as_uint = x << n; LAUF_RUNTIME_BUILTIN_DISPATCH; @@ -65,7 +65,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_bits_ushr, 2, 1, panic_flags, "ushr", &lauf_lib_bi ++vstack_ptr; if (LAUF_UNLIKELY(n >= sizeof(lauf_uint) * CHAR_BIT)) - return lauf_runtime_panic(process, "shift amount too big"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "shift amount too big")); vstack_ptr[0].as_uint = x >> n; LAUF_RUNTIME_BUILTIN_DISPATCH; @@ -78,7 +78,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_bits_sshr, 2, 1, panic_flags, "sshr", &lauf_lib_bi ++vstack_ptr; if (LAUF_UNLIKELY(n >= sizeof(lauf_sint) * CHAR_BIT)) - return lauf_runtime_panic(process, "shift amount too big"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "shift amount too big")); static_assert(-1 >> 1 == -1, "compiler does not implement arithmetic right shift"); vstack_ptr[0].as_sint = x >> n; @@ -86,4 +86,3 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_bits_sshr, 2, 1, panic_flags, "sshr", &lauf_lib_bi } const lauf_runtime_builtin_library lauf_lib_bits = {"lauf.bits", &lauf_lib_bits_sshr, nullptr}; - diff --git a/src/lauf/lib/fiber.cpp b/src/lauf/lib/fiber.cpp index 5fed8bcf..52622fa6 100644 --- a/src/lauf/lib/fiber.cpp +++ b/src/lauf/lib/fiber.cpp @@ -14,7 +14,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_fiber_create, 1, 1, LAUF_RUNTIME_BUILTIN_DEFAULT, auto fn = lauf_runtime_get_function_ptr_any(process, address); if (LAUF_UNLIKELY(fn == nullptr)) - return lauf_runtime_panic(process, "invalid function address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid function address")); auto fiber = lauf_runtime_create_fiber(process, fn); vstack_ptr[0].as_address = lauf_runtime_get_fiber_handle(fiber); @@ -30,10 +30,10 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_fiber_destroy, 1, 0, LAUF_RUNTIME_BUILTIN_DEFAULT, auto fiber = lauf_runtime_get_fiber_ptr(process, handle); if (LAUF_UNLIKELY(fiber == nullptr)) - return lauf_runtime_panic(process, "invalid fiber handle"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid fiber handle")); if (LAUF_UNLIKELY(!lauf_runtime_destroy_fiber(process, fiber))) - return false; + LAUF_BUILTIN_RETURN(false); LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -70,7 +70,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_fiber_done, 1, 1, LAUF_RUNTIME_BUILTIN_DEFAULT, "d auto handle = vstack_ptr[0].as_address; auto fiber = lauf_runtime_get_fiber_ptr(process, handle); if (LAUF_UNLIKELY(fiber == nullptr)) - return lauf_runtime_panic(process, "invalid fiber handle"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid fiber handle")); auto status = lauf_runtime_get_fiber_status(fiber); vstack_ptr[0].as_uint = status == LAUF_RUNTIME_FIBER_DONE ? 1 : 0; @@ -79,4 +79,3 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_fiber_done, 1, 1, LAUF_RUNTIME_BUILTIN_DEFAULT, "d } const lauf_runtime_builtin_library lauf_lib_fiber = {"lauf.fiber", &lauf_lib_fiber_done, nullptr}; - diff --git a/src/lauf/lib/heap.cpp b/src/lauf/lib/heap.cpp index c7f21af8..3c223f77 100644 --- a/src/lauf/lib/heap.cpp +++ b/src/lauf/lib/heap.cpp @@ -19,7 +19,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_heap_alloc, 2, 1, LAUF_RUNTIME_BUILTIN_DEFAULT, "a auto allocator = lauf_vm_get_allocator(lauf_runtime_get_vm(process)); auto memory = allocator.heap_alloc(allocator.user_data, size, alignment); if (memory == nullptr) - return lauf_runtime_panic(process, "out of memory"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "out of memory")); auto address = lauf_runtime_add_heap_allocation(process, memory, size); @@ -53,7 +53,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_heap_free, 1, 0, LAUF_RUNTIME_BUILTIN_DEFAULT, "fr lauf_runtime_allocation alloc; if (!lauf_runtime_get_allocation(process, address, &alloc) || !lauf_runtime_leak_heap_allocation(process, address)) - return lauf_runtime_panic(process, "invalid heap address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid heap address")); auto allocator = lauf_vm_get_allocator(lauf_runtime_get_vm(process)); allocator.free_alloc(allocator.user_data, alloc.ptr, alloc.size); @@ -69,14 +69,14 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_heap_transfer_local, 1, 1, LAUF_RUNTIME_BUILTIN_DE lauf_runtime_allocation alloc; if (!lauf_runtime_get_allocation(process, address, &alloc) || (alloc.permission & LAUF_RUNTIME_PERM_READ) == 0) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); if (alloc.source == LAUF_RUNTIME_LOCAL_ALLOCATION) { auto allocator = lauf_vm_get_allocator(lauf_runtime_get_vm(process)); auto memory = allocator.heap_alloc(allocator.user_data, alloc.size, alignof(void*)); if (memory == nullptr) - return lauf_runtime_panic(process, "out of memory"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "out of memory")); std::memcpy(memory, alloc.ptr, alloc.size); @@ -104,7 +104,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_heap_declare_reachable, 1, 0, LAUF_RUNTIME_BUILTIN ++vstack_ptr; if (!lauf_runtime_declare_reachable(process, addr)) - return lauf_runtime_panic(process, "invalid heap address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid heap address")); LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -116,7 +116,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_heap_undeclare_reachable, 1, 0, LAUF_RUNTIME_BUILT ++vstack_ptr; if (!lauf_runtime_undeclare_reachable(process, addr)) - return lauf_runtime_panic(process, "invalid heap address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid heap address")); LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -128,7 +128,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_heap_declare_weak, 1, 0, LAUF_RUNTIME_BUILTIN_VM_D ++vstack_ptr; if (!lauf_runtime_declare_weak(process, addr)) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -140,11 +140,10 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_heap_undeclare_weak, 1, 0, LAUF_RUNTIME_BUILTIN_VM ++vstack_ptr; if (!lauf_runtime_undeclare_weak(process, addr)) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); LAUF_RUNTIME_BUILTIN_DISPATCH; } const lauf_runtime_builtin_library lauf_lib_heap = {"lauf.heap", &lauf_lib_heap_undeclare_weak, nullptr}; - diff --git a/src/lauf/lib/int.cpp b/src/lauf/lib/int.cpp index b69ea3a3..572e3746 100644 --- a/src/lauf/lib/int.cpp +++ b/src/lauf/lib/int.cpp @@ -67,7 +67,7 @@ LAUF_RUNTIME_BUILTIN(sadd_panic, 2, 1, panic_flags, "sadd_panic", &sadd_sat) auto overflow = __builtin_add_overflow(vstack_ptr[1].as_sint, vstack_ptr[0].as_sint, &vstack_ptr[1].as_sint); if (overflow) - return lauf_runtime_panic(process, "integer overflow"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "integer overflow")); ++vstack_ptr; LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -109,7 +109,7 @@ LAUF_RUNTIME_BUILTIN(ssub_panic, 2, 1, panic_flags, "ssub_panic", &ssub_sat) auto overflow = __builtin_sub_overflow(vstack_ptr[1].as_sint, vstack_ptr[0].as_sint, &vstack_ptr[1].as_sint); if (overflow) - return lauf_runtime_panic(process, "integer overflow"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "integer overflow")); ++vstack_ptr; LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -152,7 +152,7 @@ LAUF_RUNTIME_BUILTIN(smul_panic, 2, 1, panic_flags, "smul_panic", &smul_sat) auto overflow = __builtin_mul_overflow(vstack_ptr[1].as_sint, vstack_ptr[0].as_sint, &vstack_ptr[1].as_sint); if (overflow) - return lauf_runtime_panic(process, "integer overflow"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "integer overflow")); ++vstack_ptr; LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -189,7 +189,7 @@ LAUF_RUNTIME_BUILTIN(uadd_panic, 2, 1, panic_flags, "uadd_panic", &uadd_sat) auto overflow = __builtin_add_overflow(vstack_ptr[1].as_uint, vstack_ptr[0].as_uint, &vstack_ptr[1].as_uint); if (overflow) - return lauf_runtime_panic(process, "integer overflow"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "integer overflow")); ++vstack_ptr; LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -226,7 +226,7 @@ LAUF_RUNTIME_BUILTIN(usub_panic, 2, 1, panic_flags, "usub_panic", &usub_sat) auto overflow = __builtin_sub_overflow(vstack_ptr[1].as_uint, vstack_ptr[0].as_uint, &vstack_ptr[1].as_uint); if (overflow) - return lauf_runtime_panic(process, "integer overflow"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "integer overflow")); ++vstack_ptr; LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -263,7 +263,7 @@ LAUF_RUNTIME_BUILTIN(umul_panic, 2, 1, panic_flags, "umul_panic", &umul_sat) auto overflow = __builtin_mul_overflow(vstack_ptr[1].as_uint, vstack_ptr[0].as_uint, &vstack_ptr[1].as_uint); if (overflow) - return lauf_runtime_panic(process, "integer overflow"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "integer overflow")); ++vstack_ptr; LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -278,7 +278,7 @@ LAUF_RUNTIME_BUILTIN(sdiv_flag, 2, 2, panic_flags, "sdiv_flag", &umul_panic) auto lhs = vstack_ptr[1].as_sint; auto rhs = vstack_ptr[0].as_sint; if (rhs == 0) - return lauf_runtime_panic(process, "division by zero"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "division by zero")); if (lhs == INT64_MIN && rhs == -1) { @@ -298,7 +298,7 @@ LAUF_RUNTIME_BUILTIN(sdiv_wrap, 2, 1, panic_flags, "sdiv_wrap", &sdiv_flag) auto lhs = vstack_ptr[1].as_sint; auto rhs = vstack_ptr[0].as_sint; if (rhs == 0) - return lauf_runtime_panic(process, "division by zero"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "division by zero")); ++vstack_ptr; if (lhs == INT64_MIN && rhs == -1) @@ -313,7 +313,7 @@ LAUF_RUNTIME_BUILTIN(sdiv_sat, 2, 1, panic_flags, "sdiv_sat", &sdiv_wrap) auto lhs = vstack_ptr[1].as_sint; auto rhs = vstack_ptr[0].as_sint; if (rhs == 0) - return lauf_runtime_panic(process, "division by zero"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "division by zero")); ++vstack_ptr; if (lhs == INT64_MIN && rhs == -1) @@ -328,11 +328,11 @@ LAUF_RUNTIME_BUILTIN(sdiv_panic, 2, 1, panic_flags, "sdiv_panic", &sdiv_sat) auto lhs = vstack_ptr[1].as_sint; auto rhs = vstack_ptr[0].as_sint; if (rhs == 0) - return lauf_runtime_panic(process, "division by zero"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "division by zero")); ++vstack_ptr; if (lhs == INT64_MIN && rhs == -1) - return lauf_runtime_panic(process, "integer overflow"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "integer overflow")); else vstack_ptr[0].as_sint = lhs / rhs; @@ -347,7 +347,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_int_udiv, 2, 1, panic_flags, "udiv", &sdiv_panic) auto lhs = vstack_ptr[1].as_uint; auto rhs = vstack_ptr[0].as_uint; if (rhs == 0) - return lauf_runtime_panic(process, "division by zero"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "division by zero")); ++vstack_ptr; vstack_ptr[0].as_uint = lhs / rhs; @@ -361,7 +361,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_int_srem, 2, 1, panic_flags, "srem", &lauf_lib_int ++vstack_ptr; if (rhs == 0) - return lauf_runtime_panic(process, "division by zero"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "division by zero")); else if (lhs == INT64_MIN && rhs == -1) // lhs % rhs is actually UB for those values! vstack_ptr[0].as_sint = 0; @@ -378,7 +378,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_int_urem, 2, 1, panic_flags, "urem", &lauf_lib_int ++vstack_ptr; if (rhs == 0) - return lauf_runtime_panic(process, "division by zero"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "division by zero")); else vstack_ptr[0].as_uint = lhs % rhs; @@ -424,7 +424,7 @@ LAUF_RUNTIME_BUILTIN(stou_sat, 1, 1, no_panic_flags, "stou_sat", &stou_wrap) LAUF_RUNTIME_BUILTIN(stou_panic, 1, 1, panic_flags, "stou_panic", &stou_sat) { if (vstack_ptr[0].as_sint < 0) - return lauf_runtime_panic(process, "integer overflow"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "integer overflow")); LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -448,7 +448,7 @@ LAUF_RUNTIME_BUILTIN(utos_sat, 1, 1, no_panic_flags, "utos_sat", &utos_wrap) LAUF_RUNTIME_BUILTIN(utos_panic, 1, 1, panic_flags, "utos_panic", &utos_sat) { if (vstack_ptr[0].as_uint > INT64_MAX) - return lauf_runtime_panic(process, "integer overflow"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "integer overflow")); LAUF_RUNTIME_BUILTIN_DISPATCH; } } // namespace @@ -502,7 +502,7 @@ LAUF_RUNTIME_BUILTIN(sabs_panic, 1, 1, panic_flags, "sabs_panic", &sabs_sat) { auto input = vstack_ptr[0].as_sint; if (input == INT64_MIN) - return lauf_runtime_panic(process, "integer overflow"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "integer overflow")); else if (input < 0) vstack_ptr[0].as_sint = -input; @@ -530,9 +530,10 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_int_uabs, 1, 1, no_panic_flags, "uabs", &sabs_pani namespace { template -LAUF_RUNTIME_BUILTIN_IMPL bool load_int(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +LAUF_RUNTIME_BUILTIN_IMPL LAUF_BUILTIN_RETURN_TYPE load_int(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) { auto value = *static_cast(vstack_ptr[1].as_native_ptr); @@ -549,9 +550,10 @@ LAUF_RUNTIME_BUILTIN_IMPL bool load_int(const lauf_asm_inst* ip, lauf_runtime_va } template -LAUF_RUNTIME_BUILTIN_IMPL bool store_int(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +LAUF_RUNTIME_BUILTIN_IMPL LAUF_BUILTIN_RETURN_TYPE store_int(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) { auto dest = static_cast(vstack_ptr[1].as_native_ptr); diff --git a/src/lauf/lib/limits.cpp b/src/lauf/lib/limits.cpp index 3b3942f0..ea09d511 100644 --- a/src/lauf/lib/limits.cpp +++ b/src/lauf/lib/limits.cpp @@ -12,10 +12,10 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_limits_set_step_limit, 1, 0, LAUF_RUNTIME_BUILTIN_ { auto new_limit = vstack_ptr[0].as_uint; if (new_limit == 0) - return lauf_runtime_panic(process, "cannot remove step limit"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "cannot remove step limit")); if (!lauf_runtime_set_step_limit(process, new_limit)) - return lauf_runtime_panic(process, "cannot increase step limit"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "cannot increase step limit")); ++vstack_ptr; LAUF_RUNTIME_BUILTIN_DISPATCH; @@ -25,7 +25,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_limits_step, 0, 0, LAUF_RUNTIME_BUILTIN_VM_DIRECTI &lauf_lib_limits_set_step_limit) { if (!lauf_runtime_increment_step(process)) - return lauf_runtime_panic(process, "step limit exceeded"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "step limit exceeded")); // Note that if the panic recovers (via `lauf.test.assert_panic`), the process now // has an unlimited step limit. @@ -34,4 +34,3 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_limits_step, 0, 0, LAUF_RUNTIME_BUILTIN_VM_DIRECTI const lauf_runtime_builtin_library lauf_lib_limits = {"lauf.limits", &lauf_lib_limits_step, nullptr}; - diff --git a/src/lauf/lib/memory.cpp b/src/lauf/lib/memory.cpp index 2fe4bd02..0fcb3c59 100644 --- a/src/lauf/lib/memory.cpp +++ b/src/lauf/lib/memory.cpp @@ -17,7 +17,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_memory_poison, 1, 0, LAUF_RUNTIME_BUILTIN_VM_DIREC ++vstack_ptr; if (!lauf_runtime_poison_allocation(process, address)) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -29,7 +29,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_memory_unpoison, 1, 0, LAUF_RUNTIME_BUILTIN_VM_DIR ++vstack_ptr; if (!lauf_runtime_unpoison_allocation(process, address)) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -42,7 +42,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_memory_split, 1, 2, LAUF_RUNTIME_BUILTIN_DEFAULT, --vstack_ptr; if (!lauf_runtime_split_allocation(process, addr, &vstack_ptr[1].as_address, &vstack_ptr[0].as_address)) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -55,7 +55,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_memory_merge, 2, 1, LAUF_RUNTIME_BUILTIN_DEFAULT, ++vstack_ptr; if (!lauf_runtime_merge_allocation(process, addr1, addr2)) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); LAUF_RUNTIME_BUILTIN_DISPATCH; } @@ -67,7 +67,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_memory_addr_to_int, 1, 2, LAUF_RUNTIME_BUILTIN_DEF auto ptr = lauf_runtime_get_const_ptr(process, addr, {0, 1}); if (ptr == nullptr) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); auto provenance = lauf_runtime_address{addr.allocation, addr.generation, 0}; --vstack_ptr; @@ -83,7 +83,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_memory_int_to_addr, 2, 1, LAUF_RUNTIME_BUILTIN_DEF auto ptr = reinterpret_cast(vstack_ptr[0].as_uint); // NOLINT if (!lauf_runtime_get_address(process, &provenance, ptr)) - return lauf_runtime_panic(process, "invalid int for int_to_addr"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid int for int_to_addr")); ++vstack_ptr; vstack_ptr[0].as_address = provenance; @@ -136,7 +136,7 @@ LAUF_RUNTIME_BUILTIN(addr_add_panic, 2, 1, LAUF_RUNTIME_BUILTIN_DEFAULT, "addr_a auto new_offset = addr_offset(addr, offset); if (!validate_addr_offset(process, addr, new_offset)) - return false; + LAUF_BUILTIN_RETURN(false); ++vstack_ptr; vstack_ptr[0].as_address = {addr.allocation, addr.generation, new_offset}; @@ -150,7 +150,7 @@ LAUF_RUNTIME_BUILTIN(addr_add_panic_strict, 2, 1, LAUF_RUNTIME_BUILTIN_DEFAULT, auto new_offset = addr_offset(addr, offset); if (!validate_addr_offset(process, addr, new_offset)) - return false; + LAUF_BUILTIN_RETURN(false); ++vstack_ptr; vstack_ptr[0].as_address = {addr.allocation, addr.generation, new_offset}; @@ -177,7 +177,7 @@ LAUF_RUNTIME_BUILTIN(addr_sub_panic, 2, 1, LAUF_RUNTIME_BUILTIN_DEFAULT, "addr_s auto new_offset = addr_offset(addr, -offset); if (!validate_addr_offset(process, addr, new_offset)) - return false; + LAUF_BUILTIN_RETURN(false); ++vstack_ptr; vstack_ptr[0].as_address = {addr.allocation, addr.generation, new_offset}; @@ -191,7 +191,7 @@ LAUF_RUNTIME_BUILTIN(addr_sub_panic_strict, 2, 1, LAUF_RUNTIME_BUILTIN_DEFAULT, auto new_offset = addr_offset(addr, -offset); if (!validate_addr_offset(process, addr, new_offset)) - return false; + LAUF_BUILTIN_RETURN(false); ++vstack_ptr; vstack_ptr[0].as_address = {addr.allocation, addr.generation, new_offset}; @@ -234,7 +234,8 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_memory_addr_distance, 2, 1, LAUF_RUNTIME_BUILTIN_D auto rhs = vstack_ptr[0].as_address; if (lhs.allocation != rhs.allocation || lhs.generation != rhs.generation) - return lauf_runtime_panic(process, "addresses are from different allocations"); + LAUF_BUILTIN_RETURN( + lauf_runtime_panic(process, "addresses are from different allocations")); auto distance = lauf_sint(lhs.offset) - lauf_sint(rhs.offset); @@ -253,7 +254,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_memory_copy, 3, 0, LAUF_RUNTIME_BUILTIN_DEFAULT, " auto dest_ptr = lauf_runtime_get_mut_ptr(process, dest, {count, 1}); auto src_ptr = lauf_runtime_get_const_ptr(process, src, {count, 1}); if (dest_ptr == nullptr || src_ptr == nullptr) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); std::memmove(dest_ptr, src_ptr, count); @@ -270,7 +271,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_memory_fill, 3, 0, LAUF_RUNTIME_BUILTIN_DEFAULT, " auto dest_ptr = lauf_runtime_get_mut_ptr(process, dest, {count, 1}); if (dest_ptr == nullptr) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); std::memset(dest_ptr, static_cast(byte), count); @@ -288,7 +289,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_memory_cmp, 3, 1, LAUF_RUNTIME_BUILTIN_DEFAULT, "c auto lhs_ptr = lauf_runtime_get_const_ptr(process, lhs, {count, 1}); auto rhs_ptr = lauf_runtime_get_const_ptr(process, rhs, {count, 1}); if (lhs_ptr == nullptr || rhs_ptr == nullptr) - return lauf_runtime_panic(process, "invalid address"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid address")); auto result = std::memcmp(lhs_ptr, rhs_ptr, count); diff --git a/src/lauf/lib/test.cpp b/src/lauf/lib/test.cpp index 928f1be7..488596b6 100644 --- a/src/lauf/lib/test.cpp +++ b/src/lauf/lib/test.cpp @@ -31,7 +31,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_test_unreachable, 0, 0, (void)ip; (void)vstack_ptr; (void)frame_ptr; - return lauf_runtime_panic(process, "unreachable code reached"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "unreachable code reached")); } LAUF_RUNTIME_BUILTIN(lauf_lib_test_assert, 1, 0, LAUF_RUNTIME_BUILTIN_NO_PROCESS, "assert", @@ -43,7 +43,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_test_assert, 1, 0, LAUF_RUNTIME_BUILTIN_NO_PROCESS if (value != 0) LAUF_RUNTIME_BUILTIN_DISPATCH; else - return lauf_runtime_panic(process, "assert failed"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "assert failed")); } LAUF_RUNTIME_BUILTIN(lauf_lib_test_assert_eq, 2, 0, LAUF_RUNTIME_BUILTIN_NO_PROCESS, "assert_eq", @@ -56,7 +56,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_test_assert_eq, 2, 0, LAUF_RUNTIME_BUILTIN_NO_PROC if (lhs == rhs) LAUF_RUNTIME_BUILTIN_DISPATCH; else - return lauf_runtime_panic(process, "assert_eq failed"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "assert_eq failed")); } LAUF_RUNTIME_BUILTIN(lauf_lib_test_assert_panic, 2, 0, LAUF_RUNTIME_BUILTIN_DEFAULT, "assert_panic", @@ -67,7 +67,7 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_test_assert_panic, 2, 0, LAUF_RUNTIME_BUILTIN_DEFA vstack_ptr += 2; if (fn == nullptr) - return lauf_runtime_panic(process, "invalid function"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "invalid function")); // We temporarily replace the panic handler with one that simply remembers the message. auto vm = lauf_runtime_get_vm(process); @@ -81,15 +81,15 @@ LAUF_RUNTIME_BUILTIN(lauf_lib_test_assert_panic, 2, 0, LAUF_RUNTIME_BUILTIN_DEFA lauf_vm_set_panic_handler(vm, handler); if (did_not_panic) - return lauf_runtime_panic(process, "assert_panic failed: no panic"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "assert_panic failed: no panic")); else if (expected_msg == nullptr && panic_msg != nullptr) - return lauf_runtime_panic(process, "assert_panic failed: did not expect message"); + LAUF_BUILTIN_RETURN( + lauf_runtime_panic(process, "assert_panic failed: did not expect message")); else if (expected_msg != nullptr && std::strcmp(expected_msg, panic_msg) != 0) - return lauf_runtime_panic(process, "assert_panic failed: different message"); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, "assert_panic failed: different message")); else LAUF_RUNTIME_BUILTIN_DISPATCH; } const lauf_runtime_builtin_library lauf_lib_test = {"lauf.test", &lauf_lib_test_assert_panic, nullptr}; - diff --git a/src/lauf/runtime/process.cpp b/src/lauf/runtime/process.cpp index 8290fafd..650212cd 100644 --- a/src/lauf/runtime/process.cpp +++ b/src/lauf/runtime/process.cpp @@ -225,8 +225,23 @@ bool lauf_runtime_resume(lauf_runtime_process* process, lauf_runtime_fiber* fibe vstack_ptr[0] = input[i]; } +#if LAUF_HAS_TAIL_CALL_ELIMINATION auto success = lauf::execute(fiber->suspension_point.ip + 1, fiber->suspension_point.vstack_ptr, fiber->suspension_point.frame_ptr, process); +#else + LAUF_BUILTIN_RETURN_TYPE tail_call_result + = lauf::execute(fiber->suspension_point.ip + 1, fiber->suspension_point.vstack_ptr, + fiber->suspension_point.frame_ptr, process); + while (tail_call_result.function.is_function) + { + tail_call_result + = tail_call_result.function.next_func(tail_call_result.function.cur_ip, + tail_call_result.function.cur_stack_ptr, + tail_call_result.function.cur_frame_ptr, + tail_call_result.function.cur_process); + } + [[maybe_unused]] bool success = tail_call_result.value.value; +#endif if (LAUF_LIKELY(success)) { // fiber could have changed, so reset back to the current fiber. diff --git a/src/lauf/vm_execute.cpp b/src/lauf/vm_execute.cpp index fc033ed6..89081ce3 100644 --- a/src/lauf/vm_execute.cpp +++ b/src/lauf/vm_execute.cpp @@ -13,10 +13,9 @@ //=== execute ===// #define LAUF_ASM_INST(Name, Type) \ - LAUF_NOINLINE static bool execute_##Name(const lauf_asm_inst* ip, \ - lauf_runtime_value* vstack_ptr, \ - lauf_runtime_stack_frame* frame_ptr, \ - lauf_runtime_process* process); + LAUF_NOINLINE static LAUF_BUILTIN_RETURN_TYPE \ + execute_##Name(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, \ + lauf_runtime_stack_frame* frame_ptr, lauf_runtime_process* process); #include #undef LAUF_ASM_INST @@ -30,9 +29,10 @@ lauf_runtime_builtin_impl* const lauf::_vm_dispatch_table[] = { #else -LAUF_FORCE_INLINE bool lauf::_vm_dispatch(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +LAUF_FORCE_INLINE LAUF_BUILTIN_RETURN_TYPE lauf::_vm_dispatch(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) { // Note that we're using a switch here, which the compiler could also lower to a jump table. // If jump tables are disabled using CMake, it also adds the corresponding optimization flag to @@ -53,10 +53,9 @@ LAUF_FORCE_INLINE bool lauf::_vm_dispatch(const lauf_asm_inst* ip, lauf_runtime_ #if !defined(__clang__) && (defined(__GNUC__) || defined(__GNUG__)) # undef lauf_runtime_builtin_dispatch -LAUF_RUNTIME_BUILTIN_IMPL bool lauf_runtime_builtin_dispatch(const lauf_asm_inst* ip, - lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +LAUF_RUNTIME_BUILTIN_IMPL LAUF_BUILTIN_RETURN_TYPE lauf_runtime_builtin_dispatch( + const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) { return lauf_runtime_builtin_dispatch_inline(ip, vstack_ptr, frame_ptr, process); } @@ -68,20 +67,21 @@ LAUF_RUNTIME_BUILTIN_IMPL bool lauf_runtime_builtin_dispatch(const lauf_asm_inst // That way the the hot path contains no function calls, so the compiler doesn't spill stuff. namespace { -LAUF_NOINLINE bool do_panic(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, lauf_runtime_process* process) +LAUF_NOINLINE LAUF_BUILTIN_RETURN_TYPE do_panic(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) { auto msg = reinterpret_cast(vstack_ptr); process->regs = {ip, vstack_ptr, frame_ptr}; - return lauf_runtime_panic(process, msg); + LAUF_BUILTIN_RETURN(lauf_runtime_panic(process, msg)); } #define LAUF_DO_PANIC(Msg) \ LAUF_TAIL_CALL return do_panic(ip, (lauf_runtime_value*)(Msg), frame_ptr, process) -LAUF_NOINLINE bool allocate_more_vstack_space(const lauf_asm_inst* ip, - lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +LAUF_NOINLINE LAUF_BUILTIN_RETURN_TYPE + allocate_more_vstack_space(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, lauf_runtime_process* process) { process->cur_fiber->vstack.grow(process->vm->page_allocator, vstack_ptr); if (LAUF_UNLIKELY(process->cur_fiber->vstack.capacity() > process->vm->max_vstack_size)) @@ -90,10 +90,9 @@ LAUF_NOINLINE bool allocate_more_vstack_space(const lauf_asm_inst* ip, LAUF_VM_DISPATCH; } -LAUF_NOINLINE bool allocate_more_cstack_space(const lauf_asm_inst* ip, - lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +LAUF_NOINLINE LAUF_BUILTIN_RETURN_TYPE + allocate_more_cstack_space(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, lauf_runtime_process* process) { process->cur_fiber->cstack.grow(process->vm->page_allocator, frame_ptr); if (LAUF_UNLIKELY(process->cur_fiber->cstack.capacity() > process->vm->max_cstack_size)) @@ -119,9 +118,10 @@ LAUF_NOINLINE bool allocate_more_cstack_space(const lauf_asm_inst* ip, ip = (Callee)->insts; \ } -LAUF_NOINLINE bool call_undefined_function(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +LAUF_NOINLINE LAUF_BUILTIN_RETURN_TYPE call_undefined_function(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) { auto callee = lauf::uncompress_pointer_offset(frame_ptr->function, ip->call.offset); @@ -155,7 +155,7 @@ LAUF_NOINLINE bool call_undefined_function(const lauf_asm_inst* ip, lauf_runtime // Call the function. if (!definition->native.fn(definition->native.user_data, process, input, vstack_ptr)) - return false; + LAUF_BUILTIN_RETURN(false); // We need to reverse the order of output arguments. for (auto lhs = vstack_ptr, rhs = vstack_ptr + callee->sig.output_count - 1; lhs < rhs; @@ -173,9 +173,10 @@ LAUF_NOINLINE bool call_undefined_function(const lauf_asm_inst* ip, lauf_runtime } } -LAUF_NOINLINE bool grow_allocation_array(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +LAUF_NOINLINE LAUF_BUILTIN_RETURN_TYPE grow_allocation_array(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) { process->memory.grow(process->vm->page_allocator); LAUF_VM_DISPATCH; @@ -199,8 +200,10 @@ LAUF_FORCE_INLINE std::size_t get_global_allocation_idx(lauf_runtime_stack_frame } // namespace #define LAUF_VM_EXECUTE(Name) \ - bool execute_##Name(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, \ - lauf_runtime_stack_frame* frame_ptr, lauf_runtime_process* process) + LAUF_BUILTIN_RETURN_TYPE execute_##Name(const lauf_asm_inst* ip, \ + lauf_runtime_value* vstack_ptr, \ + lauf_runtime_stack_frame* frame_ptr, \ + lauf_runtime_process* process) //=== control flow ===// LAUF_VM_EXECUTE(nop) @@ -288,7 +291,7 @@ LAUF_VM_EXECUTE(exit) if (LAUF_UNLIKELY(process == nullptr)) // During constant folding, we don't have a process, so check first. // We also don't have fibers, so just return. - return true; + LAUF_BUILTIN_RETURN(true); auto cur_fiber = process->cur_fiber; auto new_fiber = lauf::get_fiber(process, cur_fiber->parent); @@ -300,7 +303,7 @@ LAUF_VM_EXECUTE(exit) // We don't have a parent (anymore?), return to lauf_runtime_call(). // We don't reset cur_fiber. process->regs = {nullptr, nullptr, nullptr}; - return true; + LAUF_BUILTIN_RETURN(true); } else { @@ -452,7 +455,7 @@ LAUF_VM_EXECUTE(fiber_suspend) // We're suspending the main fiber, so return instead. cur_fiber->suspend({ip, vstack_ptr, frame_ptr}, ip->fiber_suspend.output_count); // We don't reset process->cur_fiber, so we know which fiber was suspended last. - return true; + LAUF_BUILTIN_RETURN(true); } else { diff --git a/src/lauf/vm_execute.hpp b/src/lauf/vm_execute.hpp index ca5b4a7b..091df827 100644 --- a/src/lauf/vm_execute.hpp +++ b/src/lauf/vm_execute.hpp @@ -14,14 +14,27 @@ namespace lauf extern lauf_runtime_builtin_impl* const _vm_dispatch_table[]; -# define LAUF_VM_DISPATCH \ - LAUF_TAIL_CALL return lauf::_vm_dispatch_table[int(ip->op())](ip, vstack_ptr, frame_ptr, \ - process) +# if LAUF_HAS_TAIL_CALL_ELIMINATION +# define LAUF_VM_DISPATCH \ + LAUF_TAIL_CALL return lauf::_vm_dispatch_table[int(ip->op())](ip, vstack_ptr, \ + frame_ptr, process) +# else +# define LAUF_VM_DISPATCH \ + return \ + { \ + .function \ + { \ + true, lauf::_vm_dispatch_table[int(ip->op())], ip, vstack_ptr, frame_ptr, \ + process \ + } \ + } +# endif #else -bool _vm_dispatch(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, lauf_runtime_process* process); +LAUF_BUILTIN_RETURN_TYPE _vm_dispatch(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process); # define LAUF_VM_DISPATCH \ LAUF_TAIL_CALL return lauf::_vm_dispatch(ip, vstack_ptr, frame_ptr, process) @@ -50,8 +63,9 @@ constexpr lauf_asm_inst trampoline_code[3] = { }(), }; -inline bool execute(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, lauf_runtime_process* process) +inline LAUF_BUILTIN_RETURN_TYPE execute(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) { LAUF_VM_DISPATCH; } @@ -60,13 +74,16 @@ inline bool execute(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, // Clang supports inline of previously non-inlined function // GCC does #if !defined(__clang__) && (defined(__GNUC__) || defined(__GNUG__)) -extern "C" [[gnu::always_inline]] inline bool lauf_runtime_builtin_dispatch_inline( - const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +extern "C" [[gnu::always_inline]] inline LAUF_BUILTIN_RETURN_TYPE + lauf_runtime_builtin_dispatch_inline(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) #else -inline bool lauf_runtime_builtin_dispatch(const lauf_asm_inst* ip, lauf_runtime_value* vstack_ptr, - lauf_runtime_stack_frame* frame_ptr, - lauf_runtime_process* process) +inline LAUF_BUILTIN_RETURN_TYPE lauf_runtime_builtin_dispatch(const lauf_asm_inst* ip, + lauf_runtime_value* vstack_ptr, + lauf_runtime_stack_frame* frame_ptr, + lauf_runtime_process* process) + #endif { assert(ip[1].op() == lauf::asm_op::call_builtin_sig);