From dcd15f342437f8063ddc5c0805abb32502930ee4 Mon Sep 17 00:00:00 2001 From: Leonardo de Moura Date: Wed, 21 Aug 2019 21:04:42 -0700 Subject: [PATCH] refactor(runtime): C backend --- src/CMakeLists.txt | 3 +- src/kernel/local_ctx.cpp | 4 +- src/library/module.cpp | 4 +- src/runtime/alloc.cpp | 53 +- src/runtime/alloc.h | 2 - src/runtime/apply.cpp | 902 ++++++++-------- src/runtime/compact.cpp | 119 ++- src/runtime/compact.h | 2 +- src/runtime/debug.h | 1 - src/runtime/io.cpp | 48 +- src/runtime/lean.h | 79 +- src/runtime/mpq.h | 1 + src/runtime/mpz.h | 8 +- src/runtime/object.cpp | 714 ++++++------- src/runtime/object.h | 1695 ++++++------------------------ src/runtime/optional.h | 1 + src/runtime/serializer.cpp | 69 +- src/runtime/serializer.h | 7 +- src/tests/util/CMakeLists.txt | 49 +- src/tests/util/object.cpp | 7 +- src/util/CMakeLists.txt | 2 +- src/util/object_ref.h | 5 +- src/util/option_declarations.cpp | 9 +- src/util/pair_ref.h | 1 + 24 files changed, 1335 insertions(+), 2450 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2e37c98a8c..b073d0dac8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -530,8 +530,7 @@ function(add_exec_test name tgt) endif() endfunction() -# C++ unit tests are disabled for now since they rely on `format` which relies on `Lean` generated code. -# add_subdirectory(tests/util) +add_subdirectory(tests/util) # Include style check if (NOT(${CMAKE_SYSTEM_NAME} MATCHES "Windows") AND PYTHONINTERP_FOUND) diff --git a/src/kernel/local_ctx.cpp b/src/kernel/local_ctx.cpp index 205ae31ab8..70836052d1 100644 --- a/src/kernel/local_ctx.cpp +++ b/src/kernel/local_ctx.cpp @@ -61,7 +61,7 @@ local_decl local_ctx::mk_local_decl(name const & n, name const & un, expr const object * p = lean_local_ctx_mk_let_decl(raw(), n.to_obj_arg(), un.to_obj_arg(), type.to_obj_arg(), value.to_obj_arg()); local_decl decl(cnstr_get(p, 0)); m_obj = cnstr_get(p, 1); - free_heap_obj(p); + lean_free_heap_obj(p); return decl; } @@ -69,7 +69,7 @@ local_decl local_ctx::mk_local_decl(name const & n, name const & un, expr const object * p = lean_local_ctx_mk_local_decl(raw(), n.to_obj_arg(), un.to_obj_arg(), type.to_obj_arg(), static_cast(bi)); local_decl decl(cnstr_get(p, 0)); m_obj = cnstr_get(p, 1); - free_heap_obj(p); + lean_free_heap_obj(p); return decl; } diff --git a/src/library/module.cpp b/src/library/module.cpp index f887459d28..733619085f 100644 --- a/src/library/module.cpp +++ b/src/library/module.cpp @@ -155,7 +155,7 @@ extern "C" object * lean_serialize_modifications(object * mod_list, object * w) std::string bytes = out.str(); object * r = alloc_sarray(1, bytes.size(), bytes.size()); - memcpy(sarray_cptr(r), bytes.data(), bytes.size()); + memcpy(sarray_cptr(r), bytes.data(), bytes.size()); return set_io_result(w, r); } catch (exception & ex) { @@ -165,7 +165,7 @@ extern "C" object * lean_serialize_modifications(object * mod_list, object * w) extern "C" object * lean_perform_serialized_modifications(object * env0, object * bytes, object * w) { environment env(env0); - std::string code(sarray_cptr(bytes), sarray_size(bytes)); + std::string code((char*)sarray_cptr(bytes), sarray_size(bytes)); dec_ref(bytes); try { std::istringstream in(code, std::ios_base::binary); diff --git a/src/runtime/alloc.cpp b/src/runtime/alloc.cpp index e989e1ed58..bbe918b33a 100644 --- a/src/runtime/alloc.cpp +++ b/src/runtime/alloc.cpp @@ -7,12 +7,9 @@ Author: Leonardo de Moura #include #include "runtime/thread.h" #include "runtime/debug.h" -#include "runtime/compiler_hints.h" -#include "runtime/object.h" #include "runtime/alloc.h" +#include "runtime/lean.h" -#define LEAN_OBJECT_SIZE_DELTA 8 -#define LEAN_MAX_SMALL_OBJECT_SIZE 512 #define LEAN_PAGE_SIZE 8192 // 8 Kb #define LEAN_SEGMENT_SIZE 8*1024*1024 // 8 Mb #define LEAN_NUM_SLOTS (LEAN_MAX_SMALL_OBJECT_SIZE / LEAN_OBJECT_SIZE_DELTA) @@ -73,12 +70,8 @@ struct page { void push_free_obj(void * o); }; -inline size_t align(size_t v, size_t a) { - return (v / a)*a + a * (v % a != 0); -} - inline char * align_ptr(char * p, size_t a) { - return reinterpret_cast(align(reinterpret_cast(p), a)); + return reinterpret_cast(lean_align(reinterpret_cast(p), a)); } struct segment { @@ -149,12 +142,6 @@ LEAN_THREAD_GLOBAL_PTR(page *, g_curr_pages); LEAN_THREAD_PTR(heap, g_heap); static heap_manager * g_heap_manager = nullptr; -inline unsigned get_slot_idx(unsigned obj_size) { - lean_assert(obj_size > 0); - lean_assert(align(obj_size, LEAN_OBJECT_SIZE_DELTA) == obj_size); - return obj_size / LEAN_OBJECT_SIZE_DELTA - 1; -} - inline void set_next_obj(void * obj, void * next) { *reinterpret_cast(obj) = next; } @@ -300,7 +287,7 @@ void heap::alloc_segment() { } static page * alloc_page(heap * h, unsigned obj_size) { - lean_assert(align(obj_size, LEAN_OBJECT_SIZE_DELTA) == obj_size); + lean_assert(lean_align(obj_size, LEAN_OBJECT_SIZE_DELTA) == obj_size); segment * s = h->m_curr_segment; LEAN_RUNTIME_STAT_CODE(g_num_pages++); page * p = new (s->m_next_page_mem) page(); @@ -309,7 +296,7 @@ static page * alloc_page(heap * h, unsigned obj_size) { /* s is full, we need to allocate a new one. */ h->alloc_segment(); } - unsigned slot_idx = get_slot_idx(obj_size); + unsigned slot_idx = lean_get_slot_idx(obj_size); p->m_header.m_heap = h; page_list_insert(h->m_curr_page[slot_idx], p); p->m_header.m_slot_idx = slot_idx; @@ -372,17 +359,7 @@ void init_thread_heap() { init_heap(false); } -void * alloc(size_t sz) { - sz = align(sz, LEAN_OBJECT_SIZE_DELTA); - LEAN_RUNTIME_STAT_CODE(g_num_alloc++); - if (LEAN_UNLIKELY(sz > LEAN_MAX_SMALL_OBJECT_SIZE)) { - void * r = malloc(sz); - if (r == nullptr) throw std::bad_alloc(); - return r; - } - lean_assert(g_heap); - LEAN_RUNTIME_STAT_CODE(g_num_small_alloc++); - unsigned slot_idx = get_slot_idx(sz); +extern "C" void * lean_alloc_small(unsigned sz, unsigned slot_idx) { page * p = g_heap->m_curr_page[slot_idx]; void * r = p->m_header.m_free_list; if (LEAN_UNLIKELY(r == nullptr)) { @@ -402,6 +379,20 @@ void * alloc(size_t sz) { return r; } +void * alloc(size_t sz) { + sz = lean_align(sz, LEAN_OBJECT_SIZE_DELTA); + LEAN_RUNTIME_STAT_CODE(g_num_alloc++); + if (LEAN_UNLIKELY(sz > LEAN_MAX_SMALL_OBJECT_SIZE)) { + void * r = malloc(sz); + if (r == nullptr) lean_panic_out_of_memory(); + return r; + } + lean_assert(g_heap); + LEAN_RUNTIME_STAT_CODE(g_num_small_alloc++); + unsigned slot_idx = lean_get_slot_idx(sz); + return lean_alloc_small(sz, slot_idx); +} + static inline void dealloc_small_core(void * o) { LEAN_RUNTIME_STAT_CODE(g_num_small_dealloc++); if (LEAN_UNLIKELY(g_heap == nullptr)) { @@ -424,18 +415,18 @@ static inline void dealloc_small_core(void * o) { void dealloc(void * o, size_t sz) { LEAN_RUNTIME_STAT_CODE(g_num_dealloc++); - sz = align(sz, LEAN_OBJECT_SIZE_DELTA); + sz = lean_align(sz, LEAN_OBJECT_SIZE_DELTA); if (LEAN_UNLIKELY(sz > LEAN_MAX_SMALL_OBJECT_SIZE)) { return free(o); } dealloc_small_core(o); } -void dealloc_small(void * o) { +extern "C" void lean_free_small(void * o) { dealloc_small_core(o); } -unsigned get_small_object_size(void * o) { +extern "C" unsigned lean_small_mem_size(void * o) { page * p = get_page_of(o); return p->m_header.m_obj_size; } diff --git a/src/runtime/alloc.h b/src/runtime/alloc.h index 216200ffb8..75276dff44 100644 --- a/src/runtime/alloc.h +++ b/src/runtime/alloc.h @@ -11,8 +11,6 @@ namespace lean { void init_thread_heap(); void * alloc(size_t sz); void dealloc(void * o, size_t sz); -void dealloc_small(void * o); -unsigned get_small_object_size(void * o); void initialize_alloc(); void finalize_alloc(); } diff --git a/src/runtime/apply.cpp b/src/runtime/apply.cpp index d261bc7b78..7041ad9374 100644 --- a/src/runtime/apply.cpp +++ b/src/runtime/apply.cpp @@ -8,28 +8,28 @@ Author: Leonardo de Moura // Generated using script: ../../gen/apply.lean #include "runtime/apply.h" namespace lean { -#define obj object -#define fx(i) closure_arg_cptr(f)[i] +#define obj lean_object +#define fx(i) lean_closure_arg_cptr(f)[i] static obj* fix_args(obj* f, unsigned n, obj*const* as) { - unsigned arity = closure_arity(f); - unsigned fixed = closure_num_fixed(f); + unsigned arity = lean_closure_arity(f); + unsigned fixed = lean_closure_num_fixed(f); unsigned new_fixed = fixed + n; lean_assert(new_fixed < arity); - obj * r = alloc_closure(closure_fun(f), arity, new_fixed); - obj ** source = closure_arg_cptr(f); - obj ** target = closure_arg_cptr(r); - if (!is_exclusive(f)) { + obj * r = lean_alloc_closure(lean_closure_fun(f), arity, new_fixed); + obj ** source = lean_closure_arg_cptr(f); + obj ** target = lean_closure_arg_cptr(r); + if (!lean_is_exclusive(f)) { for (unsigned i = 0; i < fixed; i++, source++, target++) { *target = *source; - inc(*target); + lean_inc(*target); } dec_ref(f); } else { for (unsigned i = 0; i < fixed; i++, source++, target++) { *target = *source; } - free_closure_obj(f); + lean_free_small_object(f); } for (unsigned i = 0; i < n; i++, as++, target++) { *target = *as; @@ -41,39 +41,39 @@ static inline obj* fix_args(obj* f, std::initializer_list const & l) { return fix_args(f, l.size(), l.begin()); } typedef obj* (*fn1)(obj*); // NOLINT -#define FN1(f) reinterpret_cast(closure_fun(f)) +#define FN1(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn2)(obj*, obj*); // NOLINT -#define FN2(f) reinterpret_cast(closure_fun(f)) +#define FN2(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn3)(obj*, obj*, obj*); // NOLINT -#define FN3(f) reinterpret_cast(closure_fun(f)) +#define FN3(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn4)(obj*, obj*, obj*, obj*); // NOLINT -#define FN4(f) reinterpret_cast(closure_fun(f)) +#define FN4(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn5)(obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN5(f) reinterpret_cast(closure_fun(f)) +#define FN5(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn6)(obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN6(f) reinterpret_cast(closure_fun(f)) +#define FN6(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn7)(obj*, obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN7(f) reinterpret_cast(closure_fun(f)) +#define FN7(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn8)(obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN8(f) reinterpret_cast(closure_fun(f)) +#define FN8(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn9)(obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN9(f) reinterpret_cast(closure_fun(f)) +#define FN9(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn10)(obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN10(f) reinterpret_cast(closure_fun(f)) +#define FN10(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn11)(obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN11(f) reinterpret_cast(closure_fun(f)) +#define FN11(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn12)(obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN12(f) reinterpret_cast(closure_fun(f)) +#define FN12(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn13)(obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN13(f) reinterpret_cast(closure_fun(f)) +#define FN13(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn14)(obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN14(f) reinterpret_cast(closure_fun(f)) +#define FN14(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn15)(obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN15(f) reinterpret_cast(closure_fun(f)) +#define FN15(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fn16)(obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*, obj*); // NOLINT -#define FN16(f) reinterpret_cast(closure_fun(f)) +#define FN16(f) reinterpret_cast(lean_closure_fun(f)) typedef obj* (*fnn)(obj**); // NOLINT -#define FNN(f) reinterpret_cast(closure_fun(f)) +#define FNN(f) reinterpret_cast(lean_closure_fun(f)) static obj* curry(obj* f, unsigned n, obj** as) { switch (n) { case 0: lean_unreachable(); @@ -96,56 +96,56 @@ case 16: return FN16(f)(as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], default: return FNN(f)(as); } } -obj* apply_n(obj*, unsigned, obj**); -obj* apply_1(obj* f, obj* a1) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_n(obj*, unsigned, obj**); +extern "C" obj* lean_apply_1(obj* f, obj* a1) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 1) { if (is_exclusive(f)) { switch (arity) { - case 1: { obj* r = FN1(f)(a1); free_closure_obj(f); return r; } - case 2: { obj* r = FN2(f)(fx(0), a1); free_closure_obj(f); return r; } - case 3: { obj* r = FN3(f)(fx(0), fx(1), a1); free_closure_obj(f); return r; } - case 4: { obj* r = FN4(f)(fx(0), fx(1), fx(2), a1); free_closure_obj(f); return r; } - case 5: { obj* r = FN5(f)(fx(0), fx(1), fx(2), fx(3), a1); free_closure_obj(f); return r; } - case 6: { obj* r = FN6(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1); free_closure_obj(f); return r; } - case 7: { obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1); free_closure_obj(f); return r; } - case 8: { obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1); free_closure_obj(f); return r; } - case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1); free_closure_obj(f); return r; } - case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1); free_closure_obj(f); return r; } - case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), a1); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), fx(14), a1); free_closure_obj(f); return r; } + case 1: { obj* r = FN1(f)(a1); lean_free_small_object(f); return r; } + case 2: { obj* r = FN2(f)(fx(0), a1); lean_free_small_object(f); return r; } + case 3: { obj* r = FN3(f)(fx(0), fx(1), a1); lean_free_small_object(f); return r; } + case 4: { obj* r = FN4(f)(fx(0), fx(1), fx(2), a1); lean_free_small_object(f); return r; } + case 5: { obj* r = FN5(f)(fx(0), fx(1), fx(2), fx(3), a1); lean_free_small_object(f); return r; } + case 6: { obj* r = FN6(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1); lean_free_small_object(f); return r; } + case 7: { obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1); lean_free_small_object(f); return r; } + case 8: { obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1); lean_free_small_object(f); return r; } + case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1); lean_free_small_object(f); return r; } + case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1); lean_free_small_object(f); return r; } + case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), a1); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), fx(14), a1); lean_free_small_object(f); return r; } } } switch (arity) { - case 1: { obj* r = FN1(f)(a1); dec_ref(f); return r; } - case 2: { inc(fx(0)); obj* r = FN2(f)(fx(0), a1); dec_ref(f); return r; } - case 3: { inc(fx(0)); inc(fx(1)); obj* r = FN3(f)(fx(0), fx(1), a1); dec_ref(f); return r; } - case 4: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN4(f)(fx(0), fx(1), fx(2), a1); dec_ref(f); return r; } - case 5: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN5(f)(fx(0), fx(1), fx(2), fx(3), a1); dec_ref(f); return r; } - case 6: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN6(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1); dec_ref(f); return r; } - case 7: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1); dec_ref(f); return r; } - case 8: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1); dec_ref(f); return r; } - case 9: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1); dec_ref(f); return r; } - case 10: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1); dec_ref(f); return r; } - case 11: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1); dec_ref(f); return r; } - case 12: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); inc(fx(11)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); inc(fx(11)); inc(fx(12)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); inc(fx(11)); inc(fx(12)); inc(fx(13)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), a1); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); inc(fx(11)); inc(fx(12)); inc(fx(13)); inc(fx(14)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), fx(14), a1); dec_ref(f); return r; } + case 1: { obj* r = FN1(f)(a1); lean_dec_ref(f); return r; } + case 2: { lean_inc(fx(0)); obj* r = FN2(f)(fx(0), a1); lean_dec_ref(f); return r; } + case 3: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN3(f)(fx(0), fx(1), a1); lean_dec_ref(f); return r; } + case 4: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN4(f)(fx(0), fx(1), fx(2), a1); lean_dec_ref(f); return r; } + case 5: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN5(f)(fx(0), fx(1), fx(2), fx(3), a1); lean_dec_ref(f); return r; } + case 6: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN6(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1); lean_dec_ref(f); return r; } + case 7: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1); lean_dec_ref(f); return r; } + case 8: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1); lean_dec_ref(f); return r; } + case 9: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1); lean_dec_ref(f); return r; } + case 10: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1); lean_dec_ref(f); return r; } + case 11: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); lean_inc(fx(11)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); lean_inc(fx(11)); lean_inc(fx(12)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); lean_inc(fx(11)); lean_inc(fx(12)); lean_inc(fx(13)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), a1); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); lean_inc(fx(11)); lean_inc(fx(12)); lean_inc(fx(13)); lean_inc(fx(14)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), fx(14), a1); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[1] = { a1 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 1; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 1) { @@ -155,753 +155,753 @@ if (arity == fixed + 1) { return fix_args(f, {a1}); } } -obj* apply_2(obj* f, obj* a1, obj* a2) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_2(obj* f, obj* a1, obj* a2) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 2) { if (is_exclusive(f)) { switch (arity) { - case 2: { obj* r = FN2(f)(a1, a2); free_closure_obj(f); return r; } - case 3: { obj* r = FN3(f)(fx(0), a1, a2); free_closure_obj(f); return r; } - case 4: { obj* r = FN4(f)(fx(0), fx(1), a1, a2); free_closure_obj(f); return r; } - case 5: { obj* r = FN5(f)(fx(0), fx(1), fx(2), a1, a2); free_closure_obj(f); return r; } - case 6: { obj* r = FN6(f)(fx(0), fx(1), fx(2), fx(3), a1, a2); free_closure_obj(f); return r; } - case 7: { obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2); free_closure_obj(f); return r; } - case 8: { obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2); free_closure_obj(f); return r; } - case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2); free_closure_obj(f); return r; } - case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2); free_closure_obj(f); return r; } - case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1, a2); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), a1, a2); free_closure_obj(f); return r; } + case 2: { obj* r = FN2(f)(a1, a2); lean_free_small_object(f); return r; } + case 3: { obj* r = FN3(f)(fx(0), a1, a2); lean_free_small_object(f); return r; } + case 4: { obj* r = FN4(f)(fx(0), fx(1), a1, a2); lean_free_small_object(f); return r; } + case 5: { obj* r = FN5(f)(fx(0), fx(1), fx(2), a1, a2); lean_free_small_object(f); return r; } + case 6: { obj* r = FN6(f)(fx(0), fx(1), fx(2), fx(3), a1, a2); lean_free_small_object(f); return r; } + case 7: { obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2); lean_free_small_object(f); return r; } + case 8: { obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2); lean_free_small_object(f); return r; } + case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2); lean_free_small_object(f); return r; } + case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2); lean_free_small_object(f); return r; } + case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1, a2); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), a1, a2); lean_free_small_object(f); return r; } } } switch (arity) { - case 2: { obj* r = FN2(f)(a1, a2); dec_ref(f); return r; } - case 3: { inc(fx(0)); obj* r = FN3(f)(fx(0), a1, a2); dec_ref(f); return r; } - case 4: { inc(fx(0)); inc(fx(1)); obj* r = FN4(f)(fx(0), fx(1), a1, a2); dec_ref(f); return r; } - case 5: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN5(f)(fx(0), fx(1), fx(2), a1, a2); dec_ref(f); return r; } - case 6: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN6(f)(fx(0), fx(1), fx(2), fx(3), a1, a2); dec_ref(f); return r; } - case 7: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2); dec_ref(f); return r; } - case 8: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2); dec_ref(f); return r; } - case 9: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2); dec_ref(f); return r; } - case 10: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2); dec_ref(f); return r; } - case 11: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2); dec_ref(f); return r; } - case 12: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); inc(fx(11)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); inc(fx(11)); inc(fx(12)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1, a2); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); inc(fx(11)); inc(fx(12)); inc(fx(13)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), a1, a2); dec_ref(f); return r; } + case 2: { obj* r = FN2(f)(a1, a2); lean_dec_ref(f); return r; } + case 3: { lean_inc(fx(0)); obj* r = FN3(f)(fx(0), a1, a2); lean_dec_ref(f); return r; } + case 4: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN4(f)(fx(0), fx(1), a1, a2); lean_dec_ref(f); return r; } + case 5: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN5(f)(fx(0), fx(1), fx(2), a1, a2); lean_dec_ref(f); return r; } + case 6: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN6(f)(fx(0), fx(1), fx(2), fx(3), a1, a2); lean_dec_ref(f); return r; } + case 7: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2); lean_dec_ref(f); return r; } + case 8: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2); lean_dec_ref(f); return r; } + case 9: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2); lean_dec_ref(f); return r; } + case 10: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2); lean_dec_ref(f); return r; } + case 11: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); lean_inc(fx(11)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); lean_inc(fx(11)); lean_inc(fx(12)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1, a2); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); lean_inc(fx(11)); lean_inc(fx(12)); lean_inc(fx(13)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), fx(13), a1, a2); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[2] = { a1, a2 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 2; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 2) { obj * as[2] = { a1, a2 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 2+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 2+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2}); } } -obj* apply_3(obj* f, obj* a1, obj* a2, obj* a3) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_3(obj* f, obj* a1, obj* a2, obj* a3) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 3) { if (is_exclusive(f)) { switch (arity) { - case 3: { obj* r = FN3(f)(a1, a2, a3); free_closure_obj(f); return r; } - case 4: { obj* r = FN4(f)(fx(0), a1, a2, a3); free_closure_obj(f); return r; } - case 5: { obj* r = FN5(f)(fx(0), fx(1), a1, a2, a3); free_closure_obj(f); return r; } - case 6: { obj* r = FN6(f)(fx(0), fx(1), fx(2), a1, a2, a3); free_closure_obj(f); return r; } - case 7: { obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3); free_closure_obj(f); return r; } - case 8: { obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3); free_closure_obj(f); return r; } - case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3); free_closure_obj(f); return r; } - case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3); free_closure_obj(f); return r; } - case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2, a3); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1, a2, a3); free_closure_obj(f); return r; } + case 3: { obj* r = FN3(f)(a1, a2, a3); lean_free_small_object(f); return r; } + case 4: { obj* r = FN4(f)(fx(0), a1, a2, a3); lean_free_small_object(f); return r; } + case 5: { obj* r = FN5(f)(fx(0), fx(1), a1, a2, a3); lean_free_small_object(f); return r; } + case 6: { obj* r = FN6(f)(fx(0), fx(1), fx(2), a1, a2, a3); lean_free_small_object(f); return r; } + case 7: { obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3); lean_free_small_object(f); return r; } + case 8: { obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3); lean_free_small_object(f); return r; } + case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3); lean_free_small_object(f); return r; } + case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3); lean_free_small_object(f); return r; } + case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2, a3); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1, a2, a3); lean_free_small_object(f); return r; } } } switch (arity) { - case 3: { obj* r = FN3(f)(a1, a2, a3); dec_ref(f); return r; } - case 4: { inc(fx(0)); obj* r = FN4(f)(fx(0), a1, a2, a3); dec_ref(f); return r; } - case 5: { inc(fx(0)); inc(fx(1)); obj* r = FN5(f)(fx(0), fx(1), a1, a2, a3); dec_ref(f); return r; } - case 6: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN6(f)(fx(0), fx(1), fx(2), a1, a2, a3); dec_ref(f); return r; } - case 7: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3); dec_ref(f); return r; } - case 8: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3); dec_ref(f); return r; } - case 9: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3); dec_ref(f); return r; } - case 10: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3); dec_ref(f); return r; } - case 11: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3); dec_ref(f); return r; } - case 12: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); inc(fx(11)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2, a3); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); inc(fx(11)); inc(fx(12)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1, a2, a3); dec_ref(f); return r; } + case 3: { obj* r = FN3(f)(a1, a2, a3); lean_dec_ref(f); return r; } + case 4: { lean_inc(fx(0)); obj* r = FN4(f)(fx(0), a1, a2, a3); lean_dec_ref(f); return r; } + case 5: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN5(f)(fx(0), fx(1), a1, a2, a3); lean_dec_ref(f); return r; } + case 6: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN6(f)(fx(0), fx(1), fx(2), a1, a2, a3); lean_dec_ref(f); return r; } + case 7: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN7(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3); lean_dec_ref(f); return r; } + case 8: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3); lean_dec_ref(f); return r; } + case 9: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3); lean_dec_ref(f); return r; } + case 10: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3); lean_dec_ref(f); return r; } + case 11: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); lean_inc(fx(11)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2, a3); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); lean_inc(fx(11)); lean_inc(fx(12)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), fx(12), a1, a2, a3); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[3] = { a1, a2, a3 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 3; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 3) { obj * as[3] = { a1, a2, a3 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 3+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 3+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3}); } } -obj* apply_4(obj* f, obj* a1, obj* a2, obj* a3, obj* a4) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_4(obj* f, obj* a1, obj* a2, obj* a3, obj* a4) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 4) { if (is_exclusive(f)) { switch (arity) { - case 4: { obj* r = FN4(f)(a1, a2, a3, a4); free_closure_obj(f); return r; } - case 5: { obj* r = FN5(f)(fx(0), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 6: { obj* r = FN6(f)(fx(0), fx(1), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 7: { obj* r = FN7(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 8: { obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3, a4); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2, a3, a4); free_closure_obj(f); return r; } + case 4: { obj* r = FN4(f)(a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 5: { obj* r = FN5(f)(fx(0), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 6: { obj* r = FN6(f)(fx(0), fx(1), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 7: { obj* r = FN7(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 8: { obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3, a4); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2, a3, a4); lean_free_small_object(f); return r; } } } switch (arity) { - case 4: { obj* r = FN4(f)(a1, a2, a3, a4); dec_ref(f); return r; } - case 5: { inc(fx(0)); obj* r = FN5(f)(fx(0), a1, a2, a3, a4); dec_ref(f); return r; } - case 6: { inc(fx(0)); inc(fx(1)); obj* r = FN6(f)(fx(0), fx(1), a1, a2, a3, a4); dec_ref(f); return r; } - case 7: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN7(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4); dec_ref(f); return r; } - case 8: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4); dec_ref(f); return r; } - case 9: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4); dec_ref(f); return r; } - case 10: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4); dec_ref(f); return r; } - case 11: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4); dec_ref(f); return r; } - case 12: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3, a4); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); inc(fx(11)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2, a3, a4); dec_ref(f); return r; } + case 4: { obj* r = FN4(f)(a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 5: { lean_inc(fx(0)); obj* r = FN5(f)(fx(0), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 6: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN6(f)(fx(0), fx(1), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 7: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN7(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 8: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN8(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 9: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 10: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 11: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3, a4); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); lean_inc(fx(11)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), fx(11), a1, a2, a3, a4); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[4] = { a1, a2, a3, a4 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 4; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 4) { obj * as[4] = { a1, a2, a3, a4 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 4+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 4+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4}); } } -obj* apply_5(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_5(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 5) { if (is_exclusive(f)) { switch (arity) { - case 5: { obj* r = FN5(f)(a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 6: { obj* r = FN6(f)(fx(0), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 7: { obj* r = FN7(f)(fx(0), fx(1), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 8: { obj* r = FN8(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3, a4, a5); free_closure_obj(f); return r; } + case 5: { obj* r = FN5(f)(a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 6: { obj* r = FN6(f)(fx(0), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 7: { obj* r = FN7(f)(fx(0), fx(1), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 8: { obj* r = FN8(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3, a4, a5); lean_free_small_object(f); return r; } } } switch (arity) { - case 5: { obj* r = FN5(f)(a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 6: { inc(fx(0)); obj* r = FN6(f)(fx(0), a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 7: { inc(fx(0)); inc(fx(1)); obj* r = FN7(f)(fx(0), fx(1), a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 8: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN8(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 9: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 10: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 11: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 12: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4, a5); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); inc(fx(10)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3, a4, a5); dec_ref(f); return r; } + case 5: { obj* r = FN5(f)(a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 6: { lean_inc(fx(0)); obj* r = FN6(f)(fx(0), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 7: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN7(f)(fx(0), fx(1), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 8: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN8(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 9: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN9(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 10: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 11: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); lean_inc(fx(10)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), fx(10), a1, a2, a3, a4, a5); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[5] = { a1, a2, a3, a4, a5 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 5; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 5) { obj * as[5] = { a1, a2, a3, a4, a5 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 5+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 5+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5}); } } -obj* apply_6(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_6(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 6) { if (is_exclusive(f)) { switch (arity) { - case 6: { obj* r = FN6(f)(a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } - case 7: { obj* r = FN7(f)(fx(0), a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } - case 8: { obj* r = FN8(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } - case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } - case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } - case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4, a5, a6); free_closure_obj(f); return r; } + case 6: { obj* r = FN6(f)(a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } + case 7: { obj* r = FN7(f)(fx(0), a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } + case 8: { obj* r = FN8(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } + case 9: { obj* r = FN9(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } + case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } + case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4, a5, a6); lean_free_small_object(f); return r; } } } switch (arity) { - case 6: { obj* r = FN6(f)(a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } - case 7: { inc(fx(0)); obj* r = FN7(f)(fx(0), a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } - case 8: { inc(fx(0)); inc(fx(1)); obj* r = FN8(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } - case 9: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN9(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } - case 10: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } - case 11: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } - case 12: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); inc(fx(9)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4, a5, a6); dec_ref(f); return r; } + case 6: { obj* r = FN6(f)(a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } + case 7: { lean_inc(fx(0)); obj* r = FN7(f)(fx(0), a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } + case 8: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN8(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } + case 9: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN9(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } + case 10: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN10(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } + case 11: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); lean_inc(fx(9)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), fx(9), a1, a2, a3, a4, a5, a6); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[6] = { a1, a2, a3, a4, a5, a6 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 6; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 6) { obj * as[6] = { a1, a2, a3, a4, a5, a6 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 6+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 6+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6}); } } -obj* apply_7(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_7(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 7) { if (is_exclusive(f)) { switch (arity) { - case 7: { obj* r = FN7(f)(a1, a2, a3, a4, a5, a6, a7); free_closure_obj(f); return r; } - case 8: { obj* r = FN8(f)(fx(0), a1, a2, a3, a4, a5, a6, a7); free_closure_obj(f); return r; } - case 9: { obj* r = FN9(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7); free_closure_obj(f); return r; } - case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7); free_closure_obj(f); return r; } - case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6, a7); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5, a6, a7); free_closure_obj(f); return r; } + case 7: { obj* r = FN7(f)(a1, a2, a3, a4, a5, a6, a7); lean_free_small_object(f); return r; } + case 8: { obj* r = FN8(f)(fx(0), a1, a2, a3, a4, a5, a6, a7); lean_free_small_object(f); return r; } + case 9: { obj* r = FN9(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7); lean_free_small_object(f); return r; } + case 10: { obj* r = FN10(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7); lean_free_small_object(f); return r; } + case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6, a7); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5, a6, a7); lean_free_small_object(f); return r; } } } switch (arity) { - case 7: { obj* r = FN7(f)(a1, a2, a3, a4, a5, a6, a7); dec_ref(f); return r; } - case 8: { inc(fx(0)); obj* r = FN8(f)(fx(0), a1, a2, a3, a4, a5, a6, a7); dec_ref(f); return r; } - case 9: { inc(fx(0)); inc(fx(1)); obj* r = FN9(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7); dec_ref(f); return r; } - case 10: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN10(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7); dec_ref(f); return r; } - case 11: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7); dec_ref(f); return r; } - case 12: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6, a7); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); inc(fx(8)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5, a6, a7); dec_ref(f); return r; } + case 7: { obj* r = FN7(f)(a1, a2, a3, a4, a5, a6, a7); lean_dec_ref(f); return r; } + case 8: { lean_inc(fx(0)); obj* r = FN8(f)(fx(0), a1, a2, a3, a4, a5, a6, a7); lean_dec_ref(f); return r; } + case 9: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN9(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7); lean_dec_ref(f); return r; } + case 10: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN10(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7); lean_dec_ref(f); return r; } + case 11: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN11(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6, a7); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); lean_inc(fx(8)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), fx(8), a1, a2, a3, a4, a5, a6, a7); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[7] = { a1, a2, a3, a4, a5, a6, a7 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 7; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 7) { obj * as[7] = { a1, a2, a3, a4, a5, a6, a7 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 7+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 7+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6, a7}); } } -obj* apply_8(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_8(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 8) { if (is_exclusive(f)) { switch (arity) { - case 8: { obj* r = FN8(f)(a1, a2, a3, a4, a5, a6, a7, a8); free_closure_obj(f); return r; } - case 9: { obj* r = FN9(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8); free_closure_obj(f); return r; } - case 10: { obj* r = FN10(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8); free_closure_obj(f); return r; } - case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7, a8); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6, a7, a8); free_closure_obj(f); return r; } + case 8: { obj* r = FN8(f)(a1, a2, a3, a4, a5, a6, a7, a8); lean_free_small_object(f); return r; } + case 9: { obj* r = FN9(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8); lean_free_small_object(f); return r; } + case 10: { obj* r = FN10(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8); lean_free_small_object(f); return r; } + case 11: { obj* r = FN11(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7, a8); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6, a7, a8); lean_free_small_object(f); return r; } } } switch (arity) { - case 8: { obj* r = FN8(f)(a1, a2, a3, a4, a5, a6, a7, a8); dec_ref(f); return r; } - case 9: { inc(fx(0)); obj* r = FN9(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8); dec_ref(f); return r; } - case 10: { inc(fx(0)); inc(fx(1)); obj* r = FN10(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8); dec_ref(f); return r; } - case 11: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN11(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8); dec_ref(f); return r; } - case 12: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7, a8); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); inc(fx(7)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6, a7, a8); dec_ref(f); return r; } + case 8: { obj* r = FN8(f)(a1, a2, a3, a4, a5, a6, a7, a8); lean_dec_ref(f); return r; } + case 9: { lean_inc(fx(0)); obj* r = FN9(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8); lean_dec_ref(f); return r; } + case 10: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN10(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8); lean_dec_ref(f); return r; } + case 11: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN11(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN12(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7, a8); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); lean_inc(fx(7)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), fx(7), a1, a2, a3, a4, a5, a6, a7, a8); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[8] = { a1, a2, a3, a4, a5, a6, a7, a8 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 8; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 8) { obj * as[8] = { a1, a2, a3, a4, a5, a6, a7, a8 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 8+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 8+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6, a7, a8}); } } -obj* apply_9(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_9(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 9) { if (is_exclusive(f)) { switch (arity) { - case 9: { obj* r = FN9(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9); free_closure_obj(f); return r; } - case 10: { obj* r = FN10(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9); free_closure_obj(f); return r; } - case 11: { obj* r = FN11(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8, a9); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7, a8, a9); free_closure_obj(f); return r; } + case 9: { obj* r = FN9(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_free_small_object(f); return r; } + case 10: { obj* r = FN10(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_free_small_object(f); return r; } + case 11: { obj* r = FN11(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_free_small_object(f); return r; } } } switch (arity) { - case 9: { obj* r = FN9(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9); dec_ref(f); return r; } - case 10: { inc(fx(0)); obj* r = FN10(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9); dec_ref(f); return r; } - case 11: { inc(fx(0)); inc(fx(1)); obj* r = FN11(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9); dec_ref(f); return r; } - case 12: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN12(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8, a9); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); inc(fx(6)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7, a8, a9); dec_ref(f); return r; } + case 9: { obj* r = FN9(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_dec_ref(f); return r; } + case 10: { lean_inc(fx(0)); obj* r = FN10(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_dec_ref(f); return r; } + case 11: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN11(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN12(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN13(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); lean_inc(fx(6)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), fx(6), a1, a2, a3, a4, a5, a6, a7, a8, a9); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[9] = { a1, a2, a3, a4, a5, a6, a7, a8, a9 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 9; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 9) { obj * as[9] = { a1, a2, a3, a4, a5, a6, a7, a8, a9 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 9+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 9+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6, a7, a8, a9}); } } -obj* apply_10(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_10(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 10) { if (is_exclusive(f)) { switch (arity) { - case 10: { obj* r = FN10(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); free_closure_obj(f); return r; } - case 11: { obj* r = FN11(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); free_closure_obj(f); return r; } + case 10: { obj* r = FN10(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_free_small_object(f); return r; } + case 11: { obj* r = FN11(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_free_small_object(f); return r; } } } switch (arity) { - case 10: { obj* r = FN10(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); dec_ref(f); return r; } - case 11: { inc(fx(0)); obj* r = FN11(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); dec_ref(f); return r; } - case 12: { inc(fx(0)); inc(fx(1)); obj* r = FN12(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN13(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); inc(fx(5)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); dec_ref(f); return r; } + case 10: { obj* r = FN10(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_dec_ref(f); return r; } + case 11: { lean_inc(fx(0)); obj* r = FN11(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN12(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN13(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN14(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); lean_inc(fx(5)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), fx(5), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[10] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 10; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 10) { obj * as[10] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 10+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 10+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6, a7, a8, a9, a10}); } } -obj* apply_11(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_11(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 11) { if (is_exclusive(f)) { switch (arity) { - case 11: { obj* r = FN11(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); free_closure_obj(f); return r; } - case 12: { obj* r = FN12(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); free_closure_obj(f); return r; } + case 11: { obj* r = FN11(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_free_small_object(f); return r; } + case 12: { obj* r = FN12(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_free_small_object(f); return r; } } } switch (arity) { - case 11: { obj* r = FN11(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); dec_ref(f); return r; } - case 12: { inc(fx(0)); obj* r = FN12(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); dec_ref(f); return r; } - case 13: { inc(fx(0)); inc(fx(1)); obj* r = FN13(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN14(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); inc(fx(4)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); dec_ref(f); return r; } + case 11: { obj* r = FN11(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_dec_ref(f); return r; } + case 12: { lean_inc(fx(0)); obj* r = FN12(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN13(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN14(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN15(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); lean_inc(fx(4)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), fx(4), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[11] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 11; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 11) { obj * as[11] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 11+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 11+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11}); } } -obj* apply_12(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11, obj* a12) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_12(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11, obj* a12) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 12) { if (is_exclusive(f)) { switch (arity) { - case 12: { obj* r = FN12(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); free_closure_obj(f); return r; } - case 13: { obj* r = FN13(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); free_closure_obj(f); return r; } + case 12: { obj* r = FN12(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); lean_free_small_object(f); return r; } + case 13: { obj* r = FN13(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); lean_free_small_object(f); return r; } } } switch (arity) { - case 12: { obj* r = FN12(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); dec_ref(f); return r; } - case 13: { inc(fx(0)); obj* r = FN13(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); dec_ref(f); return r; } - case 14: { inc(fx(0)); inc(fx(1)); obj* r = FN14(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN15(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); inc(fx(3)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); dec_ref(f); return r; } + case 12: { obj* r = FN12(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); lean_dec_ref(f); return r; } + case 13: { lean_inc(fx(0)); obj* r = FN13(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN14(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN15(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); lean_inc(fx(3)); obj* r = FN16(f)(fx(0), fx(1), fx(2), fx(3), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[12] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 12; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 12) { obj * as[12] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 12+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 12+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12}); } } -obj* apply_13(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11, obj* a12, obj* a13) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_13(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11, obj* a12, obj* a13) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 13) { if (is_exclusive(f)) { switch (arity) { - case 13: { obj* r = FN13(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); free_closure_obj(f); return r; } - case 14: { obj* r = FN14(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); free_closure_obj(f); return r; } + case 13: { obj* r = FN13(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); lean_free_small_object(f); return r; } + case 14: { obj* r = FN14(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); lean_free_small_object(f); return r; } } } switch (arity) { - case 13: { obj* r = FN13(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); dec_ref(f); return r; } - case 14: { inc(fx(0)); obj* r = FN14(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); dec_ref(f); return r; } - case 15: { inc(fx(0)); inc(fx(1)); obj* r = FN15(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); inc(fx(2)); obj* r = FN16(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); dec_ref(f); return r; } + case 13: { obj* r = FN13(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); lean_dec_ref(f); return r; } + case 14: { lean_inc(fx(0)); obj* r = FN14(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN15(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); lean_inc(fx(2)); obj* r = FN16(f)(fx(0), fx(1), fx(2), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[13] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 13; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 13) { obj * as[13] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 13+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 13+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13}); } } -obj* apply_14(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11, obj* a12, obj* a13, obj* a14) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_14(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11, obj* a12, obj* a13, obj* a14) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 14) { if (is_exclusive(f)) { switch (arity) { - case 14: { obj* r = FN14(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); free_closure_obj(f); return r; } - case 15: { obj* r = FN15(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); free_closure_obj(f); return r; } + case 14: { obj* r = FN14(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); lean_free_small_object(f); return r; } + case 15: { obj* r = FN15(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); lean_free_small_object(f); return r; } } } switch (arity) { - case 14: { obj* r = FN14(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); dec_ref(f); return r; } - case 15: { inc(fx(0)); obj* r = FN15(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); dec_ref(f); return r; } - case 16: { inc(fx(0)); inc(fx(1)); obj* r = FN16(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); dec_ref(f); return r; } + case 14: { obj* r = FN14(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); lean_dec_ref(f); return r; } + case 15: { lean_inc(fx(0)); obj* r = FN15(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); lean_inc(fx(1)); obj* r = FN16(f)(fx(0), fx(1), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[14] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 14; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 14) { obj * as[14] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 14+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 14+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14}); } } -obj* apply_15(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11, obj* a12, obj* a13, obj* a14, obj* a15) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_15(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11, obj* a12, obj* a13, obj* a14, obj* a15) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 15) { if (is_exclusive(f)) { switch (arity) { - case 15: { obj* r = FN15(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15); free_closure_obj(f); return r; } - case 16: { obj* r = FN16(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15); free_closure_obj(f); return r; } + case 15: { obj* r = FN15(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15); lean_free_small_object(f); return r; } + case 16: { obj* r = FN16(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15); lean_free_small_object(f); return r; } } } switch (arity) { - case 15: { obj* r = FN15(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15); dec_ref(f); return r; } - case 16: { inc(fx(0)); obj* r = FN16(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15); dec_ref(f); return r; } + case 15: { obj* r = FN15(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15); lean_dec_ref(f); return r; } + case 16: { lean_inc(fx(0)); obj* r = FN16(f)(fx(0), a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[15] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 15; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 15) { obj * as[15] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 15+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 15+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15}); } } -obj* apply_16(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11, obj* a12, obj* a13, obj* a14, obj* a15, obj* a16) { -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +extern "C" obj* lean_apply_16(obj* f, obj* a1, obj* a2, obj* a3, obj* a4, obj* a5, obj* a6, obj* a7, obj* a8, obj* a9, obj* a10, obj* a11, obj* a12, obj* a13, obj* a14, obj* a15, obj* a16) { +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + 16) { if (is_exclusive(f)) { switch (arity) { - case 16: { obj* r = FN16(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16); free_closure_obj(f); return r; } + case 16: { obj* r = FN16(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16); lean_free_small_object(f); return r; } } } switch (arity) { - case 16: { obj* r = FN16(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16); dec_ref(f); return r; } + case 16: { obj* r = FN16(f)(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16); lean_dec_ref(f); return r; } default: lean_assert(arity > 16); obj * as[16] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < 16; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } } else if (arity < fixed + 16) { obj * as[16] = { a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16 }; obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = curry(f, arity, args); - dec_ref(f); - return apply_n(new_f, 16+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, 16+fixed-arity, as+arity-fixed); } else { return fix_args(f, {a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16}); } } -obj* apply_m(obj* f, unsigned n, obj** as) { +extern "C" obj* lean_apply_m(obj* f, unsigned n, obj** as) { lean_assert(n > 16); -unsigned arity = closure_arity(f); -unsigned fixed = closure_num_fixed(f); +unsigned arity = lean_closure_arity(f); +unsigned fixed = lean_closure_num_fixed(f); if (arity == fixed + n) { obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < n; i++) args[fixed+i] = as[i]; obj * r = FNN(f)(args); - dec_ref(f); + lean_dec_ref(f); return r; } else if (arity < fixed + n) { obj ** args = static_cast(LEAN_ALLOCA(arity*sizeof(obj*))); // NOLINT - for (unsigned i = 0; i < fixed; i++) { inc(fx(i)); args[i] = fx(i); } + for (unsigned i = 0; i < fixed; i++) { lean_inc(fx(i)); args[i] = fx(i); } for (unsigned i = 0; i < arity-fixed; i++) args[fixed+i] = as[i]; obj * new_f = FNN(f)(args); - dec_ref(f); - return apply_n(new_f, n+fixed-arity, as+arity-fixed); + lean_dec_ref(f); + return lean_apply_n(new_f, n+fixed-arity, as+arity-fixed); } else { return fix_args(f, n, as); } } -obj* apply_n(obj* f, unsigned n, obj** as) { +extern "C" obj* lean_apply_n(obj* f, unsigned n, obj** as) { switch (n) { case 0: lean_unreachable(); -case 1: return apply_1(f, as[0]); -case 2: return apply_2(f, as[0], as[1]); -case 3: return apply_3(f, as[0], as[1], as[2]); -case 4: return apply_4(f, as[0], as[1], as[2], as[3]); -case 5: return apply_5(f, as[0], as[1], as[2], as[3], as[4]); -case 6: return apply_6(f, as[0], as[1], as[2], as[3], as[4], as[5]); -case 7: return apply_7(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6]); -case 8: return apply_8(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7]); -case 9: return apply_9(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8]); -case 10: return apply_10(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9]); -case 11: return apply_11(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10]); -case 12: return apply_12(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10], as[11]); -case 13: return apply_13(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10], as[11], as[12]); -case 14: return apply_14(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10], as[11], as[12], as[13]); -case 15: return apply_15(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10], as[11], as[12], as[13], as[14]); -case 16: return apply_16(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10], as[11], as[12], as[13], as[14], as[15]); -default: return apply_m(f, n, as); +case 1: return lean_apply_1(f, as[0]); +case 2: return lean_apply_2(f, as[0], as[1]); +case 3: return lean_apply_3(f, as[0], as[1], as[2]); +case 4: return lean_apply_4(f, as[0], as[1], as[2], as[3]); +case 5: return lean_apply_5(f, as[0], as[1], as[2], as[3], as[4]); +case 6: return lean_apply_6(f, as[0], as[1], as[2], as[3], as[4], as[5]); +case 7: return lean_apply_7(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6]); +case 8: return lean_apply_8(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7]); +case 9: return lean_apply_9(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8]); +case 10: return lean_apply_10(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9]); +case 11: return lean_apply_11(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10]); +case 12: return lean_apply_12(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10], as[11]); +case 13: return lean_apply_13(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10], as[11], as[12]); +case 14: return lean_apply_14(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10], as[11], as[12], as[13]); +case 15: return lean_apply_15(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10], as[11], as[12], as[13], as[14]); +case 16: return lean_apply_16(f, as[0], as[1], as[2], as[3], as[4], as[5], as[6], as[7], as[8], as[9], as[10], as[11], as[12], as[13], as[14], as[15]); +default: return lean_apply_m(f, n, as); } } } diff --git a/src/runtime/compact.cpp b/src/runtime/compact.cpp index 839b3533bd..afbe948ac9 100644 --- a/src/runtime/compact.cpp +++ b/src/runtime/compact.cpp @@ -12,23 +12,18 @@ Author: Leonardo de Moura #define LEAN_COMPACTOR_INIT_SZ 1024*1024 namespace lean { -#define TERMINATOR_ID (static_cast(object_kind::External) + 1) - /* Special object that terminates the data block constructing the object graph rooted in `m_value`. We use this object to ensure `m_value` is correctly aligned. In the past, we would allocate a chunk of memory `p` of size `sizeof(object) + sizeof(object*)`, and then write at `p + sizeof(object)`. This is incorrect because `sizeof(object)` is not a multiple of the word size. */ -struct terminator_object : public object { - object * m_value; - - explicit terminator_object(object * value) : object(object_kind::External, object_memory_kind::Region), m_value(value) { - // not an actual `object_kind`, so write to the field directly - m_kind = TERMINATOR_ID; - } +struct terminator_object { + lean_object m_header; + lean_object * m_value; }; + object_compactor::object_compactor(): m_begin(malloc(LEAN_COMPACTOR_INIT_SZ)), m_end(m_begin), @@ -75,16 +70,17 @@ object_offset object_compactor::to_offset(object * o) { } void object_compactor::insert_terminator(object * o) { - void * mem = alloc(sizeof(terminator_object)); - new (mem) terminator_object(to_offset(o)); + terminator_object * t = (terminator_object*) alloc(sizeof(terminator_object)); + lean_set_other_header((lean_object*)t, LeanReserved, 0); + t->m_value = to_offset(o); } object * object_compactor::copy_object(object * o) { - size_t sz = obj_byte_size(o); + size_t sz = lean_object_byte_size(o); void * mem = alloc(sz); memcpy(mem, o, sz); object * r = static_cast(mem); - r->m_mem_kind = static_cast(object_memory_kind::Region); + lean_set_other_mem_kind(r); save(o, r); return r; } @@ -121,12 +117,15 @@ bool object_compactor::insert_array(object * o) { } if (missing_children) return false; - void * mem = alloc(sizeof(array_object) + sz * sizeof(object *)); - object * new_o = new (mem) array_object(sz, sz, object_memory_kind::Region); + + lean_array_object * new_o = (lean_array_object*)alloc(sizeof(lean_array_object) + sizeof(void*)*sz); + lean_set_other_header((lean_object*)new_o, LeanArray, 0); + new_o->m_size = sz; + new_o->m_capacity = sz; for (size_t i = 0; i < sz; i++) { - array_set(new_o, i, offsets[i]); + array_set((lean_object*)new_o, i, offsets[i]); } - save(o, new_o); + save(o, (lean_object*)new_o); return true; } @@ -136,17 +135,17 @@ bool object_compactor::insert_thunk(object * o) { if (c == g_null_offset) return false; object * r = copy_object(o); - to_thunk(r)->m_value = c; + lean_to_thunk(r)->m_value = c; return true; } bool object_compactor::insert_ref(object * o) { - object * v = to_ref(o)->m_value; + object * v = lean_to_ref(o)->m_value; object_offset c = to_offset(v); if (c == g_null_offset) return false; object * r = copy_object(o); - to_ref(r)->m_value = c; + lean_to_ref(r)->m_value = c; return true; } @@ -164,9 +163,11 @@ bool object_compactor::insert_task(object * o) { To cope with this issue, we always save tasks as thunks, and rely on the fact that all task API accepts thunks as arguments even when multi-threading is enabled. */ - void * mem = alloc(sizeof(thunk_object)); - object * new_o = new (mem) thunk_object(c, true, object_memory_kind::Region); - save(o, new_o); + lean_thunk_object * new_o = (lean_thunk_object*)alloc(sizeof(lean_thunk_object)); + lean_set_other_header((lean_object*)new_o, LeanThunk, 0); + new_o->m_value = c; + new_o->m_closure = (lean_object*)0; + save(o, (lean_object*)new_o); return true; } @@ -177,8 +178,9 @@ void object_compactor::insert_mpz(object * o) { into an mpz number. So, we use std::max to make sure we have enough space for both. */ size_t extra_space = std::max(s.size() + 1, sizeof(mpz_object*)); void * mem = alloc(sizeof(mpz_object) + extra_space); - object * new_o = new (mem) object(object_kind::MPZ, object_memory_kind::Region); - save(o, new_o); + mpz_object * new_o = new (mem) mpz_object(); + lean_set_other_header((lean_object*)new_o, LeanMPZ, 0); + save(o, (lean_object*)new_o); void * data = reinterpret_cast(new_o) + sizeof(mpz_object); memcpy(data, s.c_str(), s.size() + 1); } @@ -195,17 +197,18 @@ void object_compactor::operator()(object * o) { } lean_assert(!is_scalar(curr)); bool r = true; - switch (get_kind(curr)) { - case object_kind::Constructor: r = insert_constructor(curr); break; - case object_kind::Closure: throw exception("closures cannot be compacted"); - case object_kind::Array: r = insert_array(curr); break; - case object_kind::ScalarArray: copy_object(curr); break; - case object_kind::String: copy_object(curr); break; - case object_kind::MPZ: insert_mpz(curr); break; - case object_kind::Thunk: r = insert_thunk(curr); break; - case object_kind::Task: r = insert_task(curr); break; - case object_kind::Ref: r = insert_ref(curr); break; - case object_kind::External: throw exception("external objects cannot be compacted"); + switch (lean_ptr_tag(curr)) { + case LeanClosure: throw exception("closures cannot be compacted"); + case LeanArray: r = insert_array(curr); break; + case LeanScalarArray: copy_object(curr); break; + case LeanString: copy_object(curr); break; + case LeanMPZ: insert_mpz(curr); break; + case LeanThunk: r = insert_thunk(curr); break; + case LeanTask: r = insert_task(curr); break; + case LeanRef: r = insert_ref(curr); break; + case LeanExternal: throw exception("external objects cannot be compacted"); + case LeanReserved: lean_unreachable(); + default: r = insert_constructor(curr); break; } if (r) m_todo.pop_back(); } @@ -256,7 +259,7 @@ inline void compacted_region::fix_constructor(object * o) { for (; it != end; it++) { *it = fix_object_ptr(*it); } - move(cnstr_byte_size(o)); + move(lean_ctor_byte_size(o)); } inline void compacted_region::fix_array(object * o) { @@ -265,17 +268,17 @@ inline void compacted_region::fix_array(object * o) { for (; it != end; it++) { *it = fix_object_ptr(*it); } - move(array_byte_size(o)); + move(lean_array_byte_size(o)); } inline void compacted_region::fix_thunk(object * o) { - to_thunk(o)->m_value = fix_object_ptr(to_thunk(o)->m_value); - move(sizeof(thunk_object)); + lean_to_thunk(o)->m_value = fix_object_ptr(lean_to_thunk(o)->m_value); + move(sizeof(lean_thunk_object)); } inline void compacted_region::fix_ref(object * o) { - to_ref(o)->m_value = fix_object_ptr(to_ref(o)->m_value); - move(sizeof(ref_object)); + lean_to_ref(o)->m_value = fix_object_ptr(lean_to_ref(o)->m_value); + move(sizeof(lean_ref_object)); } void compacted_region::fix_mpz(object * o) { @@ -290,10 +293,10 @@ void compacted_region::fix_mpz(object * o) { sz++; } /* use string to initialize memory */ - new (&(to_mpz(o)->m_value)) mpz(s.c_str()); + new (&(((mpz_object*)o)->m_value)) mpz(s.c_str()); /* update m_nested_mpzs list */ *reinterpret_cast(m_next) = m_nested_mpzs; - m_nested_mpzs = to_mpz(o); + m_nested_mpzs = (mpz_object*)o; /* consume region after mpz_object */ sz++; // string delimiter if (sz < sizeof(mpz_object*)) @@ -307,23 +310,23 @@ object * compacted_region::read() { while (true) { lean_assert(static_cast(m_next) + sizeof(object) <= m_end); object * curr = reinterpret_cast(m_next); - if (curr->m_kind == TERMINATOR_ID) { + switch (lean_ptr_tag(curr)) { + case LeanClosure: lean_unreachable(); + case LeanArray: fix_array(curr); break; + case LeanScalarArray: move(lean_sarray_byte_size(curr)); break; + case LeanString: move(lean_string_byte_size(curr)); break; + case LeanMPZ: fix_mpz(curr); break; + case LeanThunk: fix_thunk(curr); break; + case LeanRef: fix_ref(curr); break; + case LeanTask: lean_unreachable(); + case LeanExternal: lean_unreachable(); + case LeanReserved: { object * r = reinterpret_cast(m_next)->m_value; move(sizeof(terminator_object)); return fix_object_ptr(r); - } else { - switch (get_kind(curr)) { - case object_kind::Constructor: fix_constructor(curr); break; - case object_kind::Closure: lean_unreachable(); - case object_kind::Array: fix_array(curr); break; - case object_kind::ScalarArray: move(sarray_byte_size(curr)); break; - case object_kind::String: move(string_byte_size(curr)); break; - case object_kind::MPZ: fix_mpz(curr); break; - case object_kind::Thunk: fix_thunk(curr); break; - case object_kind::Ref: fix_ref(curr); break; - case object_kind::Task: lean_unreachable(); - case object_kind::External: lean_unreachable(); - } + } + default: + fix_constructor(curr); break; } } } diff --git a/src/runtime/compact.h b/src/runtime/compact.h index 5b693d2091..3ac4ef4b8e 100644 --- a/src/runtime/compact.h +++ b/src/runtime/compact.h @@ -11,7 +11,7 @@ Author: Leonardo de Moura #include "runtime/object.h" namespace lean { -typedef object * object_offset; +typedef lean_object * object_offset; class object_compactor { std::unordered_map, std::equal_to> m_obj_table; diff --git a/src/runtime/debug.h b/src/runtime/debug.h index f9ded2e1fd..d3d857a66f 100644 --- a/src/runtime/debug.h +++ b/src/runtime/debug.h @@ -8,7 +8,6 @@ Author: Leonardo de Moura #include #include #include "runtime/exception.h" -#include "runtime/compiler_hints.h" #ifndef __has_builtin #define __has_builtin(x) 0 diff --git a/src/runtime/io.cpp b/src/runtime/io.cpp index 67e931c643..3d096a8f50 100644 --- a/src/runtime/io.cpp +++ b/src/runtime/io.cpp @@ -14,12 +14,14 @@ Author: Leonardo de Moura #endif #include #include +#include #include #include #include #include #include #include "runtime/object.h" +#include "runtime/thread.h" #include "runtime/allocprof.h" #ifdef _MSC_VER @@ -31,7 +33,7 @@ Author: Leonardo de Moura namespace lean { static obj_res const REAL_WORLD = box(0); -void io_result_show_error(b_obj_arg r) { +extern "C" void lean_io_result_show_error(b_obj_arg r) { std::cerr << "uncaught exception: " << string_cstr(io_result_get_error(r)) << std::endl; } @@ -76,7 +78,7 @@ obj_res set_io_error(obj_arg r, std::string const & msg) { } static bool g_initializing = true; -void io_mark_end_initialization() { +extern "C" void lean_io_mark_end_initialization() { g_initializing = false; } @@ -254,15 +256,17 @@ extern "C" obj_res lean_io_app_dir(obj_arg r) { // ======================================= // IO ref primitives -obj_res io_mk_ref(obj_arg a, obj_arg r) { - object * ref = new (alloc_heap_object(sizeof(ref_object))) ref_object(a); - return set_io_result(r, ref); +extern "C" obj_res lean_io_mk_ref(obj_arg a, obj_arg r) { + lean_ref_object * o = (lean_ref_object*)lean_alloc_small_object(sizeof(lean_ref_object)); + lean_set_st_header((lean_object*)o, LeanRef, 0); + o->m_value = a; + return set_io_result(r, (lean_object*)o); } static object * g_io_error_nullptr_read = nullptr; static inline atomic * mt_ref_val_addr(object * o) { - return reinterpret_cast *>(&(to_ref(o)->m_value)); + return reinterpret_cast *>(&(lean_to_ref(o)->m_value)); } /* @@ -276,13 +280,9 @@ static inline atomic * mt_ref_val_addr(object * o) { object as we do for multi-threaded `IO.Ref`s. It makes sense since the global `IO.Ref` may be used to communicate data between threads. */ -static inline bool ref_maybe_mt(b_obj_arg ref) { - return - ref->m_mem_kind == static_cast(object_memory_kind::MTHeap) || - ref->m_mem_kind == static_cast(object_memory_kind::Persistent); -} +static inline bool ref_maybe_mt(b_obj_arg ref) { return lean_is_mt(ref) || lean_is_persistent(ref); } -obj_res io_ref_get(b_obj_arg ref, obj_arg r) { +extern "C" obj_res lean_io_ref_get(b_obj_arg ref, obj_arg r) { if (ref_maybe_mt(ref)) { atomic * val_addr = mt_ref_val_addr(ref); object * val = val_addr->exchange(nullptr); @@ -296,7 +296,7 @@ obj_res io_ref_get(b_obj_arg ref, obj_arg r) { } return set_io_result(r, val); } else { - object * val = to_ref(ref)->m_value; + object * val = lean_to_ref(ref)->m_value; if (val == nullptr) return set_io_error(r, g_io_error_nullptr_read); inc(val); @@ -306,7 +306,7 @@ obj_res io_ref_get(b_obj_arg ref, obj_arg r) { static_assert(sizeof(atomic) == sizeof(unsigned short), "`atomic` and `unsigned short` must have the same size"); // NOLINT -obj_res io_ref_reset(b_obj_arg ref, obj_arg r) { +extern "C" obj_res lean_io_ref_reset(b_obj_arg ref, obj_arg r) { if (ref_maybe_mt(ref)) { atomic * val_addr = mt_ref_val_addr(ref); object * old_a = val_addr->exchange(nullptr); @@ -314,14 +314,14 @@ obj_res io_ref_reset(b_obj_arg ref, obj_arg r) { dec(old_a); return r; } else { - if (to_ref(ref)->m_value != nullptr) - dec(to_ref(ref)->m_value); - to_ref(ref)->m_value = nullptr; + if (lean_to_ref(ref)->m_value != nullptr) + dec(lean_to_ref(ref)->m_value); + lean_to_ref(ref)->m_value = nullptr; return r; } } -obj_res io_ref_set(b_obj_arg ref, obj_arg a, obj_arg r) { +extern "C" obj_res lean_io_ref_set(b_obj_arg ref, obj_arg a, obj_arg r) { if (ref_maybe_mt(ref)) { /* We must mark `a` as multi-threaded if `ref` is marked as multi-threaded. Reason: our runtime relies on the fact that a single-threaded object @@ -333,14 +333,14 @@ obj_res io_ref_set(b_obj_arg ref, obj_arg a, obj_arg r) { dec(old_a); return r; } else { - if (to_ref(ref)->m_value != nullptr) - dec(to_ref(ref)->m_value); - to_ref(ref)->m_value = a; + if (lean_to_ref(ref)->m_value != nullptr) + dec(lean_to_ref(ref)->m_value); + lean_to_ref(ref)->m_value = a; return r; } } -obj_res io_ref_swap(b_obj_arg ref, obj_arg a, obj_arg r) { +extern "C" obj_res lean_io_ref_swap(b_obj_arg ref, obj_arg a, obj_arg r) { if (ref_maybe_mt(ref)) { /* See io_ref_write */ mark_mt(a); @@ -350,10 +350,10 @@ obj_res io_ref_swap(b_obj_arg ref, obj_arg a, obj_arg r) { return set_io_error(r, g_io_error_nullptr_read); return set_io_result(r, old_a); } else { - object * old_a = to_ref(ref)->m_value; + object * old_a = lean_to_ref(ref)->m_value; if (old_a == nullptr) return set_io_error(r, g_io_error_nullptr_read); - to_ref(ref)->m_value = a; + lean_to_ref(ref)->m_value = a; return set_io_result(r, old_a); } } diff --git a/src/runtime/lean.h b/src/runtime/lean.h index 585131bc03..bc76b092dc 100644 --- a/src/runtime/lean.h +++ b/src/runtime/lean.h @@ -15,6 +15,9 @@ Author: Leonardo de Moura #include #endif +#define LEAN_SMALL_ALLOCATOR +#define LEAN_COMPRESSED_OBJECT_HEADER + #ifdef __cplusplus #include #define _Atomic(t) std::atomic @@ -25,7 +28,9 @@ extern "C" { #define LEAN_USING_STD #endif -#define LEAN_CLOSURE_MAX_ARGS 16 +#define LEAN_CLOSURE_MAX_ARGS 16 +#define LEAN_OBJECT_SIZE_DELTA 8 +#define LEAN_MAX_SMALL_OBJECT_SIZE 512 #ifdef _MSC_VER #define LEAN_ALLOCA(s) _alloca(s) @@ -43,6 +48,12 @@ extern "C" { #define LEAN_ALWAYS_INLINE #endif +#ifdef LEAN_RUNTIME_STATS +#define LEAN_RUNTIME_STAT_CODE(c) c +#else +#define LEAN_RUNTIME_STAT_CODE(c) +#endif + #define LEAN_BYTE(Var, Index) *(((uint8_t*)&Var)+Index) #define LeanMaxCtorTag 244 @@ -218,8 +229,52 @@ static inline size_t lean_unbox(lean_object * o) { return (size_t)(o) >> 1; } void lean_panic_out_of_memory(); -void * lean_alloc_heap_object(size_t sz); -void lean_free_heap_obj(lean_object * o); +static inline size_t lean_align(size_t v, size_t a) { + return (v / a)*a + a * (v % a != 0); +} + +static inline unsigned lean_get_slot_idx(unsigned sz) { + assert(sz > 0); + assert(lean_align(sz, LEAN_OBJECT_SIZE_DELTA) == sz); + return sz / LEAN_OBJECT_SIZE_DELTA - 1; +} + +void * lean_alloc_small(unsigned sz, unsigned slot_idx); +void lean_free_small(void * p); +unsigned lean_small_mem_size(void * p); + +static inline lean_object * lean_alloc_small_object(unsigned sz) { +#ifdef LEAN_SMALL_ALLOCATOR + sz = lean_align(sz, LEAN_OBJECT_SIZE_DELTA); + unsigned slot_idx = lean_get_slot_idx(sz); + assert(sz <= LEAN_MAX_SMALL_OBJECT_SIZE); + return (lean_object*)lean_alloc_small(sz, slot_idx); +#else + void * mem = malloc(sizeof(size_t) + sz); + if (mem == 0) lean_panic_out_of_memory(); + *(size_t*)mem = sz; + return (lean_object*)((size_t*)mem + 1); +#endif +} + +static inline unsigned lean_small_object_size(lean_object * o) { +#ifdef LEAN_SMALL_ALLOCATOR + return lean_small_mem_size(o); +#else + return *((size_t*)o - 1); +#endif +} + +static inline void lean_free_small_object(lean_object * o) { +#ifdef LEAN_SMALL_ALLOCATOR + lean_free_small(o); +#else + free((size_t*)o - 1); +#endif +} + +lean_object * lean_alloc_object(size_t sz); +void lean_free_object(lean_object * o); /* The object size may be slightly bigger for constructor objects. The runtime does not track the size of the scalar size area. @@ -260,7 +315,7 @@ static inline bool lean_is_persistent(lean_object * o) { #endif } -static inline bool lean_is_heap_obj(lean_object * o) { +static inline bool lean_is_heap_object(lean_object * o) { return lean_is_st(o) || lean_is_mt(o); } @@ -485,7 +540,7 @@ static inline uint8_t * lean_ctor_scalar_cptr(lean_object * o) { static inline lean_object * lean_alloc_ctor(unsigned tag, unsigned num_objs, unsigned scalar_sz) { assert(tag <= LeanMaxCtorTag && num_objs < 256 && scalar_sz < 1024); - lean_object * o = (lean_object*)lean_alloc_heap_object(sizeof(lean_ctor_object) + sizeof(void*)*num_objs + scalar_sz); + lean_object * o = lean_alloc_small_object(sizeof(lean_ctor_object) + sizeof(void*)*num_objs + scalar_sz); lean_set_st_header(o, tag, num_objs); return o; } @@ -575,7 +630,7 @@ static inline lean_object ** lean_closure_arg_cptr(lean_object * o) { return lea static inline lean_obj_res lean_alloc_closure(void * fun, unsigned arity, unsigned num_fixed) { assert(arity > 0); assert(num_fixed < arity); - lean_closure_object * o = (lean_closure_object*)lean_alloc_heap_object(sizeof(lean_closure_object) + sizeof(void*)*num_fixed); + lean_closure_object * o = (lean_closure_object*)lean_alloc_small_object(sizeof(lean_closure_object) + sizeof(void*)*num_fixed); lean_set_st_header((lean_object*)o, LeanClosure, 0); o->m_fun = fun; o->m_arity = arity; @@ -622,7 +677,7 @@ lean_obj_res lean_fixpoint6(lean_obj_arg rec, lean_obj_arg a1, lean_obj_arg a2, /* Arrays of objects (low level API) */ static inline lean_obj_res lean_alloc_array(size_t size, size_t capacity) { - lean_array_object * o = (lean_array_object*)lean_alloc_heap_object(sizeof(lean_array_object) + sizeof(void*)*capacity); + lean_array_object * o = (lean_array_object*)lean_alloc_object(sizeof(lean_array_object) + sizeof(void*)*capacity); lean_set_st_header((lean_object*)o, LeanArray, 0); o->m_size = size; o->m_capacity = capacity; @@ -771,7 +826,7 @@ lean_object * lean_mk_array(lean_obj_arg n, lean_obj_arg v); /* Array of scalars */ static inline lean_obj_res lean_alloc_sarray(unsigned elem_size, size_t size, size_t capacity) { - lean_sarray_object * o = (lean_sarray_object*)lean_alloc_heap_object(sizeof(lean_sarray_object) + elem_size*capacity); + lean_sarray_object * o = (lean_sarray_object*)lean_alloc_object(sizeof(lean_sarray_object) + elem_size*capacity); lean_set_st_header((lean_object*)o, LeanScalarArray, elem_size); o->m_size = size; o->m_capacity = capacity; @@ -847,7 +902,7 @@ static inline lean_obj_res lean_byte_array_set(lean_obj_arg a, b_lean_obj_arg i, /* Strings */ static inline lean_obj_res lean_alloc_string(size_t size, size_t capacity, size_t len) { - lean_string_object * o = (lean_string_object*)lean_alloc_heap_object(sizeof(lean_string_object) + capacity); + lean_string_object * o = (lean_string_object*)lean_alloc_object(sizeof(lean_string_object) + capacity); lean_set_st_header((lean_object*)o, LeanString, 0); o->m_size = size; o->m_capacity = capacity; @@ -892,7 +947,7 @@ size_t lean_string_hash(b_lean_obj_arg); /* Thunks */ static inline lean_obj_res lean_mk_thunk(lean_obj_arg c) { - lean_thunk_object * o = (lean_thunk_object*)lean_alloc_heap_object(sizeof(lean_thunk_object)); + lean_thunk_object * o = (lean_thunk_object*)lean_alloc_small_object(sizeof(lean_thunk_object)); lean_set_st_header((lean_object*)o, LeanThunk, 0); o->m_value = (lean_object*)0; o->m_closure = c; @@ -901,7 +956,7 @@ static inline lean_obj_res lean_mk_thunk(lean_obj_arg c) { /* Thunk.pure : A -> Thunk A */ static inline lean_obj_res lean_thunk_pure(lean_obj_arg v) { - lean_thunk_object * o = (lean_thunk_object*)lean_alloc_heap_object(sizeof(lean_thunk_object)); + lean_thunk_object * o = (lean_thunk_object*)lean_alloc_small_object(sizeof(lean_thunk_object)); lean_set_st_header((lean_object*)o, LeanThunk, 0); o->m_value = v; o->m_closure = (lean_object*)0; @@ -957,7 +1012,7 @@ b_lean_obj_res lean_io_wait_any_core(b_lean_obj_arg task_list); /* External objects */ static inline lean_object * lean_alloc_external(lean_external_class * cls, void * data) { - lean_external_object * o = (lean_external_object*)lean_alloc_heap_object(sizeof(lean_external_object)); + lean_external_object * o = (lean_external_object*)lean_alloc_small_object(sizeof(lean_external_object)); lean_set_st_header((lean_object*)o, LeanExternal, 0); o->m_class = cls; o->m_data = data; diff --git a/src/runtime/mpq.h b/src/runtime/mpq.h index 0241a9d108..e86e265b7f 100644 --- a/src/runtime/mpq.h +++ b/src/runtime/mpq.h @@ -6,6 +6,7 @@ Author: Leonardo de Moura */ #pragma once #include "runtime/mpz.h" +#include "runtime/serializer.h" namespace lean { /** diff --git a/src/runtime/mpz.h b/src/runtime/mpz.h index 5f8fd04c23..7df1dd44fe 100644 --- a/src/runtime/mpz.h +++ b/src/runtime/mpz.h @@ -10,9 +10,12 @@ Author: Leonardo de Moura #include #include #include +#include "runtime/int64.h" #include "runtime/debug.h" -#include "runtime/serializer.h" +#include "runtime/lean.h" + namespace lean { + class mpq; /** \brief Wrapper for GMP integers */ class mpz { @@ -275,7 +278,4 @@ public: struct mpz_cmp_fn { int operator()(mpz const & v1, mpz const & v2) const { return cmp(v1, v2); } }; - -inline serializer & operator<<(serializer & s, mpz const & n) { s.write_mpz(n); return s; } -inline deserializer & operator>>(deserializer & d, mpz & n) { n = d.read_mpz(); return d; } } diff --git a/src/runtime/object.cpp b/src/runtime/object.cpp index ab39c4b5f3..aa763d2337 100644 --- a/src/runtime/object.cpp +++ b/src/runtime/object.cpp @@ -4,251 +4,169 @@ Released under Apache 2.0 license as described in the file LICENSE. Author: Leonardo de Moura */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "runtime/stackinfo.h" #include "runtime/object.h" -#include "runtime/utf8.h" -#include "runtime/apply.h" -#include "runtime/flet.h" -#include "runtime/interrupt.h" -#include "runtime/hash.h" +#include "runtime/thread.h" #include "runtime/alloc.h" +#include "runtime/debug.h" +#include "util/buffer.h" // move to runtime #define LEAN_MAX_PRIO 8 namespace lean { -// ======================================= -// External objects - -struct external_object_class { - external_object_finalize_proc m_finalize; - external_object_foreach_proc m_foreach; -}; - -static std::vector * g_ext_classes; -static mutex * g_ext_classes_mutex; - -external_object_class * register_external_object_class(external_object_finalize_proc p1, external_object_foreach_proc p2) { - unique_lock lock(*g_ext_classes_mutex); - external_object_class * cls = new external_object_class{p1, p2}; - g_ext_classes->push_back(cls); - return cls; +extern "C" void lean_panic_out_of_memory() { + std::cerr << "out of memory\n"; + std::exit(1); } -// ======================================= -// Object - -size_t obj_byte_size(object * o) { - switch (get_kind(o)) { - case object_kind::Constructor: return cnstr_byte_size(o); - case object_kind::Closure: return closure_byte_size(o); - case object_kind::Array: return array_byte_size(o); - case object_kind::ScalarArray: return sarray_byte_size(o); - case object_kind::String: return string_byte_size(o); - case object_kind::MPZ: return sizeof(mpz_object); - case object_kind::Thunk: return sizeof(thunk_object); - case object_kind::Task: return sizeof(task_object); - case object_kind::Ref: return sizeof(ref_object); - case object_kind::External: lean_unreachable(); +extern "C" size_t lean_object_byte_size(lean_object * o) { + switch (lean_ptr_tag(o)) { + case LeanArray: return lean_array_byte_size(o); + case LeanScalarArray: return lean_sarray_byte_size(o); + case LeanString: return lean_string_byte_size(o); + default: return lean_small_object_size(o); } - lean_unreachable(); } -size_t obj_header_size(object * o) { - switch (get_kind(o)) { - case object_kind::Constructor: return sizeof(constructor_object); - case object_kind::Closure: return sizeof(closure_object); - case object_kind::Array: return sizeof(array_object); - case object_kind::ScalarArray: return sizeof(sarray_object); - case object_kind::String: return sizeof(string_object); - case object_kind::MPZ: return sizeof(mpz_object); - case object_kind::Thunk: return sizeof(thunk_object); - case object_kind::Task: return sizeof(task_object); - case object_kind::Ref: return sizeof(ref_object); - case object_kind::External: lean_unreachable(); - } - lean_unreachable(); +static inline lean_object * get_next(lean_object * o) { +#ifdef LEAN_COMPRESSED_OBJECT_HEADER + size_t header = o->m_header; + LEAN_BYTE(header, 6) = 0; + LEAN_BYTE(header, 7) = 0; + return (lean_object*)(header); +#else + return (lean_object*)(o->m_rc); +#endif } -/* We use the RC memory to implement a linked list of lean objects to be deleted. - This hack is safe because rc_type is uintptr_t. */ +static inline void set_next(lean_object * o, lean_object * n) { +#ifdef LEAN_COMPRESSED_OBJECT_HEADER + size_t new_header = (size_t)n; + LEAN_BYTE(new_header, 6) = LEAN_BYTE(o->m_header, 6); + LEAN_BYTE(new_header, 7) = LEAN_BYTE(o->m_header, 7); + o->m_header = new_header; +#else + *(lean_object*)(o->m_rc) = n; +#endif +} -static_assert(sizeof(atomic) == sizeof(object*), "unexpected atomic size, the object GC assumes these two types have the same size"); // NOLINT +static inline void push_back(lean_object * & todo, lean_object * v) { + set_next(v, todo); + todo = v; +} + +static inline lean_object * pop_back(lean_object * & todo) { + lean_object * r = todo; + todo = get_next(todo); + return r; +} + +static inline void dec_ref(lean_object * o, lean_object* & todo) { + if (lean_dec_ref_core(o)) + push_back(todo, o); +} + +static inline void dec(lean_object * o, lean_object* & todo) { + if (!lean_is_scalar(o) && lean_dec_ref_core(o)) + push_back(todo, o); +} #ifdef LEAN_LAZY_RC LEAN_THREAD_PTR(object, g_to_free); #endif -inline object * get_next(object * o) { - return *reinterpret_cast(st_rc_addr(o)); -} - -inline void set_next(object * o, object * n) { - *reinterpret_cast(st_rc_addr(o)) = n; -} - -inline void push_back(object * & todo, object * v) { - set_next(v, todo); - todo = v; -} - -inline object * pop_back(object * & todo) { - object * r = todo; - todo = get_next(todo); - return r; -} - -inline void dec_ref(object * o, object* & todo) { - if (dec_ref_core(o)) - push_back(todo, o); -} - -inline void dec(object * o, object* & todo) { - if (!is_scalar(o) && dec_ref_core(o)) - push_back(todo, o); -} - -void deactivate_task(task_object * t); +static void lean_del_core(object * o, object * & todo); +extern "C" lean_object * lean_alloc_object(size_t sz) { +#ifdef LEAN_LAZY_RC + if (g_to_free) { + object * o = pop_back(g_to_free); + lean_del_core(o, g_to_free); + } +#endif #ifdef LEAN_SMALL_ALLOCATOR -static inline void free_heap_obj_core(object * o, size_t sz) { + return (lean_object*)alloc(sz); #else -static inline void free_heap_obj_core(object * o) { -#endif -#ifdef LEAN_FAKE_FREE - // Set kinds to invalid values, which should trap any further accesses in debug mode. - lean_always_assert(o->m_kind != 42); - o->m_kind = 42; - o->m_mem_kind = 42; -#else -#ifdef LEAN_SMALL_ALLOCATOR - dealloc(reinterpret_cast(o) - sizeof(rc_type), sz); -#else - free(reinterpret_cast(o) - sizeof(rc_type)); -#endif + void * r = malloc(sz); + if (r == nullptr) lean_panic_out_of_memory(); + return (lean_object*)r; #endif } -#ifdef LEAN_SMALL_ALLOCATOR -#define FREE_OBJ(o, sz) free_heap_obj_core(o, sz) +static void deactivate_task(lean_task_object * t); + +static inline void lean_set_tag(b_lean_obj_arg o, uint8_t new_tag) { + assert(new_tag <= LeanMaxCtorTag); +#ifdef LEAN_COMPRESSED_OBJECT_HEADER + LEAN_BYTE(o->m_header, 7) = new_tag; #else -#define FREE_OBJ(o, sz) free_heap_obj_core(o) + o->m_tag = new_tag; #endif - -void free_heap_obj(object * o) { - FREE_OBJ(o, obj_byte_size(o) + sizeof(rc_type)); } -static inline void free_cnstr_obj(object * o) { - FREE_OBJ(o, cnstr_byte_size(o) + sizeof(rc_type)); -} - -static inline void free_closure_obj_core(object * o) { - FREE_OBJ(o, closure_byte_size(o) + sizeof(rc_type)); -} - -void free_closure_obj(object * o) { - return free_closure_obj_core(o); -} - -static inline void free_array_obj(object * o) { - FREE_OBJ(o, array_byte_size(o) + sizeof(rc_type)); -} - -static inline void free_sarray_obj(object * o) { - FREE_OBJ(o, sarray_byte_size(o) + sizeof(rc_type)); -} - -static inline void free_string_obj(object * o) { - FREE_OBJ(o, string_byte_size(o) + sizeof(rc_type)); -} - -inline void free_mpz_obj(object * o) { - FREE_OBJ(o, sizeof(mpz_object) + sizeof(rc_type)); -} - -static inline void free_thunk_obj(object * o) { - FREE_OBJ(o, sizeof(thunk_object) + sizeof(rc_type)); -} - -static inline void free_ref_obj(object * o) { - FREE_OBJ(o, sizeof(ref_object) + sizeof(rc_type)); -} - -static inline void free_task_obj(object * o) { - FREE_OBJ(o, sizeof(task_object) + sizeof(rc_type)); -} - -static inline void free_external_obj(object * o) { - FREE_OBJ(o, sizeof(external_object) + sizeof(rc_type)); -} - -static void del_core(object * o, object * & todo) { - lean_assert(is_heap_obj(o)); - LEAN_RUNTIME_STAT_CODE(g_num_del++); - switch (get_kind(o)) { - case object_kind::Constructor: { - object ** it = cnstr_obj_cptr(o); - object ** end = it + cnstr_num_objs(o); +static void lean_del_core(object * o, object * & todo) { + lean_assert(lean_is_heap_object(o)); + uint8 tag = lean_ptr_tag(o); + if (tag <= LeanMaxCtorTag) { + object ** it = lean_ctor_obj_cptr(o); + object ** end = it + lean_ctor_num_objs(o); for (; it != end; ++it) dec(*it, todo); - free_cnstr_obj(o); - break; + lean_free_small(o); + return; } - case object_kind::Closure: { - object ** it = closure_arg_cptr(o); - object ** end = it + closure_num_fixed(o); + switch (tag) { + case LeanClosure: { + object ** it = lean_closure_arg_cptr(o); + object ** end = it + lean_closure_num_fixed(o); for (; it != end; ++it) dec(*it, todo); - free_closure_obj_core(o); + lean_free_small(o); break; } - case object_kind::Array: { - object ** it = array_cptr(o); - object ** end = it + array_size(o); + case LeanArray: { + object ** it = lean_array_cptr(o); + object ** end = it + lean_array_size(o); for (; it != end; ++it) dec(*it, todo); - free_array_obj(o); + dealloc(o, lean_array_byte_size(o)); break; } - case object_kind::ScalarArray: - free_sarray_obj(o); break; - case object_kind::String: - free_string_obj(o); break; - case object_kind::MPZ: - dealloc_mpz(o); break; - case object_kind::Thunk: - if (object * c = to_thunk(o)->m_closure) dec(c, todo); - if (object * v = to_thunk(o)->m_value) dec(v, todo); - free_thunk_obj(o); + case LeanScalarArray: + dealloc(o, lean_sarray_byte_size(o)); break; - case object_kind::Ref: - if (object * v = to_ref(o)->m_value) dec(v, todo); - free_ref_obj(o); + case LeanString: + dealloc(o, lean_string_byte_size(o)); break; - case object_kind::Task: - deactivate_task(to_task(o)); + case LeanMPZ: + to_mpz(o)->m_value.~mpz(); + lean_free_small(o); break; - case object_kind::External: - to_external(o)->m_class->m_finalize(to_external(o)->m_data); - free_external_obj(o); + case LeanThunk: + if (object * c = lean_to_thunk(o)->m_closure) dec(c, todo); + if (object * v = lean_to_thunk(o)->m_value) dec(v, todo); + lean_free_small(o); break; + case LeanRef: + if (object * v = lean_to_ref(o)->m_value) dec(v, todo); + lean_free_small(o); + break; + case LeanTask: + deactivate_task(lean_to_task(o)); + break; + case LeanExternal: + lean_to_external(o)->m_class->m_finalize(lean_to_external(o)->m_data); + lean_free_small(o); + break; + default: + lean_unreachable(); } } -void del(object * o) { +extern "C" void lean_del(object * o) { #ifdef LEAN_LAZY_RC push_back(g_to_free, o); #else object * todo = nullptr; while (true) { - lean_assert(is_heap_obj(o)); - del_core(o, todo); + lean_assert(lean_is_heap_object(o)); + lean_del_core(o, todo); if (todo == nullptr) return; o = pop_back(todo); @@ -256,23 +174,6 @@ void del(object * o) { #endif } -void * alloc_heap_object(size_t sz) { -#ifdef LEAN_LAZY_RC - if (g_to_free) { - object * o = pop_back(g_to_free); - del_core(o, g_to_free); - } -#endif -#ifdef LEAN_SMALL_ALLOCATOR - void * r = alloc(sizeof(rc_type) + sz); -#else - void * r = malloc(sizeof(rc_type) + sz); - if (r == nullptr) throw std::bad_alloc(); -#endif - *static_cast(r) = 1; - return static_cast(r) + sizeof(rc_type); -} - // ======================================= // Arrays static object * g_array_empty = nullptr; @@ -281,6 +182,7 @@ object * array_mk_empty() { return g_array_empty; } + // ======================================= // Closures @@ -288,15 +190,15 @@ typedef object * (*lean_cfun2)(object *, object *); // NOLINT typedef object * (*lean_cfun3)(object *, object *, object *); // NOLINT static obj_res mk_closure_2_1(lean_cfun2 fn, obj_arg a) { - object * c = alloc_closure(fn, 1); - closure_set(c, 0, a); + object * c = lean_alloc_closure((void*)fn, 2, 1); + lean_closure_set(c, 0, a); return c; } static obj_res mk_closure_3_2(lean_cfun3 fn, obj_arg a1, obj_arg a2) { - object * c = alloc_closure(fn, 2); - closure_set(c, 0, a1); - closure_set(c, 1, a2); + object * c = lean_alloc_closure((void*)fn, 3, 2); + lean_closure_set(c, 0, a1); + lean_closure_set(c, 1, a2); return c; } @@ -304,11 +206,11 @@ static obj_res mk_closure_3_2(lean_cfun3 fn, obj_arg a1, obj_arg a2) { // Thunks static obj_res mk_thunk_3_2(lean_cfun3 fn, obj_arg a1, obj_arg a2) { - return mk_thunk(mk_closure_3_2(fn, a1, a2)); + return lean_mk_thunk(mk_closure_3_2(fn, a1, a2)); } -b_obj_res thunk_get_core(b_obj_arg t) { - object * c = to_thunk(t)->m_closure.exchange(nullptr); +extern "C" b_obj_res lean_thunk_get_core(b_obj_arg t) { + object * c = lean_to_thunk(t)->m_closure.exchange(nullptr); if (c != nullptr) { /* Recall that a closure uses the standard calling convention. `thunk_get` "consumes" the result `r` by storing it at `to_thunk(t)->m_value`. @@ -317,48 +219,225 @@ b_obj_res thunk_get_core(b_obj_arg t) { to be object stored in the constructor object. Recall that `apply_1` also consumes `c`'s RC. */ - object * r = apply_1(c, box(0)); + object * r = lean_apply_1(c, lean_box(0)); lean_assert(r != nullptr); /* Closure must return a valid lean object */ - lean_assert(to_thunk(t)->m_value == nullptr); - to_thunk(t)->m_value = r; + lean_assert(lean_to_thunk(t)->m_value == nullptr); + lean_to_thunk(t)->m_value = r; return r; } else { lean_assert(c == nullptr); /* There is another thread executing the closure. We keep waiting for the m_value to be set by another thread. */ - while (!to_thunk(t)->m_value) { + while (!lean_to_thunk(t)->m_value) { this_thread::yield(); } - return to_thunk(t)->m_value; + return lean_to_thunk(t)->m_value; } } static obj_res thunk_map_fn_closure(obj_arg f, obj_arg t, obj_arg /* u */) { - b_obj_res v = thunk_get(t); - inc(v); - obj_res r = apply_1(f, v); - dec(v); + b_obj_res v = lean_thunk_get(t); + lean_inc(v); + obj_res r = lean_apply_1(f, v); + lean_dec(v); return r; } -obj_res thunk_map(obj_arg f, obj_arg t) { - lean_assert(is_closure(f)); - lean_assert(is_thunk(t)); +extern "C" obj_res lean_thunk_map(obj_arg f, obj_arg t) { + lean_assert(lean_is_closure(f)); + lean_assert(lean_is_thunk(t)); return mk_thunk_3_2(thunk_map_fn_closure, f, t); } static obj_res thunk_bind_fn_closure(obj_arg x, obj_arg f, obj_arg /* u */) { - b_obj_res v = thunk_get(x); - inc(v); - obj_res r = apply_1(f, v); - dec(x); + b_obj_res v = lean_thunk_get(x); + lean_inc(v); + obj_res r = lean_apply_1(f, v); + lean_dec(x); return r; } -obj_res thunk_bind(obj_arg x, obj_arg f) { +extern "C" obj_res lean_thunk_bind(obj_arg x, obj_arg f) { return mk_thunk_3_2(thunk_bind_fn_closure, x, f); } +// ======================================= +// Fixpoint + +static inline object * ptr_to_weak_ptr(object * p) { + return reinterpret_cast(reinterpret_cast(p) | 1); +} + +static inline object * weak_ptr_to_ptr(object * w) { + return reinterpret_cast((reinterpret_cast(w) >> 1) << 1); +} + +obj_res fixpoint_aux(obj_arg rec, obj_arg weak_k, obj_arg a) { + object * k = weak_ptr_to_ptr(weak_k); + lean_inc(k); + return lean_apply_2(rec, k, a); +} + +extern "C" obj_res lean_fixpoint(obj_arg rec, obj_arg a) { + object * k = lean_alloc_closure((void*)fixpoint_aux, 3, 2); + lean_inc(rec); + lean_closure_set(k, 0, rec); + lean_closure_set(k, 1, ptr_to_weak_ptr(k)); + object * r = lean_apply_2(rec, k, a); + return r; +} + +obj_res fixpoint_aux2(obj_arg rec, obj_arg weak_k, obj_arg a1, obj_arg a2) { + object * k = weak_ptr_to_ptr(weak_k); + lean_inc(k); + return lean_apply_3(rec, k, a1, a2); +} + +extern "C" obj_res lean_fixpoint2(obj_arg rec, obj_arg a1, obj_arg a2) { + object * k = lean_alloc_closure((void*)fixpoint_aux2, 4, 2); + lean_inc(rec); + lean_closure_set(k, 0, rec); + lean_closure_set(k, 1, ptr_to_weak_ptr(k)); + object * r = lean_apply_3(rec, k, a1, a2); + return r; +} + +obj_res fixpoint_aux3(obj_arg rec, obj_arg weak_k, obj_arg a1, obj_arg a2, obj_arg a3) { + object * k = weak_ptr_to_ptr(weak_k); + lean_inc(k); + return lean_apply_4(rec, k, a1, a2, a3); +} + +extern "C" obj_res lean_fixpoint3(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3) { + object * k = lean_alloc_closure((void*)fixpoint_aux3, 5, 2); + lean_inc(rec); + lean_closure_set(k, 0, rec); + lean_closure_set(k, 1, ptr_to_weak_ptr(k)); + object * r = lean_apply_4(rec, k, a1, a2, a3); + return r; +} + +obj_res fixpoint_aux4(obj_arg rec, obj_arg weak_k, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4) { + object * k = weak_ptr_to_ptr(weak_k); + lean_inc(k); + return lean_apply_5(rec, k, a1, a2, a3, a4); +} + +extern "C" obj_res lean_fixpoint4(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4) { + object * k = lean_alloc_closure((void*)fixpoint_aux4, 6, 2); + lean_inc(rec); + lean_closure_set(k, 0, rec); + lean_closure_set(k, 1, ptr_to_weak_ptr(k)); + object * r = lean_apply_5(rec, k, a1, a2, a3, a4); + return r; +} + +obj_res fixpoint_aux5(obj_arg rec, obj_arg weak_k, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5) { + object * k = weak_ptr_to_ptr(weak_k); + lean_inc(k); + return lean_apply_6(rec, k, a1, a2, a3, a4, a5); +} + +extern "C" obj_res lean_fixpoint5(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5) { + object * k = lean_alloc_closure((void*)fixpoint_aux5, 7, 2); + lean_inc(rec); + lean_closure_set(k, 0, rec); + lean_closure_set(k, 1, ptr_to_weak_ptr(k)); + object * r = lean_apply_6(rec, k, a1, a2, a3, a4, a5); + return r; +} + +obj_res fixpoint_aux6(obj_arg rec, obj_arg weak_k, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5, obj_arg a6) { + object * k = weak_ptr_to_ptr(weak_k); + lean_inc(k); + return lean_apply_7(rec, k, a1, a2, a3, a4, a5, a6); +} + +extern "C" obj_res lean_fixpoint6(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5, obj_arg a6) { + object * k = lean_alloc_closure((void*)fixpoint_aux6, 8, 2); + lean_inc(rec); + lean_closure_set(k, 0, rec); + lean_closure_set(k, 1, ptr_to_weak_ptr(k)); + object * r = lean_apply_7(rec, k, a1, a2, a3, a4, a5, a6); + return r; +} + +// ======================================= +// Mark Persistent + +extern "C" void lean_mark_persistent(object * o); + +static obj_res mark_persistent_fn(obj_arg o) { + lean_mark_persistent(o); + return box(0); +} + +extern "C" void lean_mark_persistent(object * o) { + buffer todo; + todo.push_back(o); + while (!todo.empty()) { + object * o = todo.back(); + todo.pop_back(); + if (!lean_is_scalar(o) && lean_is_heap_object(o)) { +#ifdef LEAN_COMPRESSED_OBJECT_HEADER + o->m_header &= ~((1ull << LEAN_ST_BIT) | (1ull << LEAN_MT_BIT)); + o->m_header |= (1ull << LEAN_PERSISTENT_BIT); +#else + o->m_mem_kind = LEAN_PERSISTENT_MEM_KIND; +#endif + switch (lean_ptr_tag(o)) { + case LeanScalarArray: + case LeanString: + case LeanMPZ: + break; + case LeanExternal: { + object * fn = lean_alloc_closure(reinterpret_cast(mark_persistent_fn), 1, 0); + lean_to_external(o)->m_class->m_foreach(lean_to_external(o)->m_data, fn); + lean_dec(fn); + break; + } + case LeanTask: + todo.push_back(lean_task_get(o)); + break; + case LeanClosure: { + object ** it = lean_closure_arg_cptr(o); + object ** end = it + lean_closure_num_fixed(o); + for (; it != end; ++it) todo.push_back(*it); + break; + } + case LeanArray: { + object ** it = lean_array_cptr(o); + object ** end = it + lean_array_size(o); + for (; it != end; ++it) todo.push_back(*it); + break; + } + case LeanThunk: + if (object * c = lean_to_thunk(o)->m_closure) todo.push_back(c); + if (object * v = lean_to_thunk(o)->m_value) todo.push_back(v); + break; + case LeanRef: + if (object * v = lean_to_ref(o)->m_value) todo.push_back(v); + break; + case LeanReserved: + lean_unreachable(); + break; + default: { + object ** it = lean_ctor_obj_cptr(o); + object ** end = it + lean_ctor_num_objs(o); + for (; it != end; ++it) todo.push_back(*it); + break; + }} + } + } +} + +} + +#if 0 +namespace lean { + + + // ======================================= // Tasks @@ -841,64 +920,6 @@ b_obj_res io_wait_any_core(b_obj_arg task_list) { return g_task_manager->wait_any(task_list); } -void mark_persistent(object * o); -static obj_res mark_persistent_fn(obj_arg o) { - mark_persistent(o); - return box(0); -} - -void mark_persistent(object * o) { - buffer todo; - todo.push_back(o); - while (!todo.empty()) { - object * o = todo.back(); - todo.pop_back(); - if (!is_scalar(o) && is_heap_obj(o)) { - o->m_mem_kind = static_cast(object_memory_kind::Persistent); - switch (get_kind(o)) { - case object_kind::ScalarArray: - case object_kind::String: - case object_kind::MPZ: - break; - case object_kind::External: { - object * fn = alloc_closure(reinterpret_cast(mark_persistent_fn), 1, 0); - to_external(o)->m_class->m_foreach(to_external(o)->m_data, fn); - dec(fn); - break; - } - case object_kind::Task: - todo.push_back(task_get(o)); - break; - case object_kind::Constructor: { - object ** it = cnstr_obj_cptr(o); - object ** end = it + cnstr_num_objs(o); - for (; it != end; ++it) todo.push_back(*it); - break; - } - case object_kind::Closure: { - object ** it = closure_arg_cptr(o); - object ** end = it + closure_num_fixed(o); - for (; it != end; ++it) todo.push_back(*it); - break; - } - case object_kind::Array: { - object ** it = array_cptr(o); - object ** end = it + array_size(o); - for (; it != end; ++it) todo.push_back(*it); - break; - } - case object_kind::Thunk: - if (object * c = to_thunk(o)->m_closure) todo.push_back(c); - if (object * v = to_thunk(o)->m_value) todo.push_back(v); - break; - case object_kind::Ref: - if (object * v = to_ref(o)->m_value) todo.push_back(v); - break; - } - } - } -} - // ======================================= // Natural numbers @@ -1605,106 +1626,6 @@ object * array_push(obj_arg a, obj_arg v) { return r; } -// ======================================= -// fixpoint - -static inline object * ptr_to_weak_ptr(object * p) { - return reinterpret_cast(reinterpret_cast(p) | 1); -} - -static inline object * weak_ptr_to_ptr(object * w) { - return reinterpret_cast((reinterpret_cast(w) >> 1) << 1); -} - -obj_res fixpoint_aux(obj_arg rec, obj_arg weak_k, obj_arg a) { - object * k = weak_ptr_to_ptr(weak_k); - inc(k); - return apply_2(rec, k, a); -} - -obj_res fixpoint(obj_arg rec, obj_arg a) { - object * k = alloc_closure(fixpoint_aux, 2); - inc(rec); - closure_set(k, 0, rec); - closure_set(k, 1, ptr_to_weak_ptr(k)); - object * r = apply_2(rec, k, a); - return r; -} - -obj_res fixpoint_aux2(obj_arg rec, obj_arg weak_k, obj_arg a1, obj_arg a2) { - object * k = weak_ptr_to_ptr(weak_k); - inc(k); - return apply_3(rec, k, a1, a2); -} - -obj_res fixpoint2(obj_arg rec, obj_arg a1, obj_arg a2) { - object * k = alloc_closure(fixpoint_aux2, 2); - inc(rec); - closure_set(k, 0, rec); - closure_set(k, 1, ptr_to_weak_ptr(k)); - object * r = apply_3(rec, k, a1, a2); - return r; -} - -obj_res fixpoint_aux3(obj_arg rec, obj_arg weak_k, obj_arg a1, obj_arg a2, obj_arg a3) { - object * k = weak_ptr_to_ptr(weak_k); - inc(k); - return apply_4(rec, k, a1, a2, a3); -} - -obj_res fixpoint3(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3) { - object * k = alloc_closure(fixpoint_aux3, 2); - inc(rec); - closure_set(k, 0, rec); - closure_set(k, 1, ptr_to_weak_ptr(k)); - object * r = apply_4(rec, k, a1, a2, a3); - return r; -} - -obj_res fixpoint_aux4(obj_arg rec, obj_arg weak_k, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4) { - object * k = weak_ptr_to_ptr(weak_k); - inc(k); - return apply_5(rec, k, a1, a2, a3, a4); -} - -obj_res fixpoint4(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4) { - object * k = alloc_closure(fixpoint_aux4, 2); - inc(rec); - closure_set(k, 0, rec); - closure_set(k, 1, ptr_to_weak_ptr(k)); - object * r = apply_5(rec, k, a1, a2, a3, a4); - return r; -} - -obj_res fixpoint_aux5(obj_arg rec, obj_arg weak_k, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5) { - object * k = weak_ptr_to_ptr(weak_k); - inc(k); - return apply_6(rec, k, a1, a2, a3, a4, a5); -} - -obj_res fixpoint5(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5) { - object * k = alloc_closure(fixpoint_aux5, 2); - inc(rec); - closure_set(k, 0, rec); - closure_set(k, 1, ptr_to_weak_ptr(k)); - object * r = apply_6(rec, k, a1, a2, a3, a4, a5); - return r; -} - -obj_res fixpoint_aux6(obj_arg rec, obj_arg weak_k, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5, obj_arg a6) { - object * k = weak_ptr_to_ptr(weak_k); - inc(k); - return apply_7(rec, k, a1, a2, a3, a4, a5, a6); -} - -obj_res fixpoint6(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5, obj_arg a6) { - object * k = alloc_closure(fixpoint_aux6, 2); - inc(rec); - closure_set(k, 0, rec); - closure_set(k, 1, ptr_to_weak_ptr(k)); - object * r = apply_7(rec, k, a1, a2, a3, a4, a5, a6); - return r; -} // ======================================= // Runtime info @@ -1812,3 +1733,4 @@ void finalize_object() { extern "C" void lean_dbg_print_str(lean::object* o) { lean::dbg_print_str(o); } extern "C" void lean_dbg_print_num(lean::object* o) { lean::dbg_print_num(o); } +#endif diff --git a/src/runtime/object.h b/src/runtime/object.h index 01fa9cb8b2..40f15d542f 100644 --- a/src/runtime/object.h +++ b/src/runtime/object.h @@ -5,639 +5,112 @@ Released under Apache 2.0 license as described in the file LICENSE. Author: Leonardo de Moura */ #pragma once -#include -#include -#include -#include -#if !defined(__APPLE__) -#include -#endif -#include "config.h" // NOLINT -#include "runtime/compiler_hints.h" +#include "runtime/lean.h" #include "runtime/mpz.h" -#include "runtime/int64.h" -#include "runtime/thread.h" - -#ifdef _MSC_VER -#define LEAN_ALLOCA(s) ::_alloca(s) -#else -#define LEAN_ALLOCA(s) ::alloca(s) -#endif namespace lean { -typedef unsigned char uint8; -typedef unsigned short uint16; -typedef unsigned uint32; -typedef unsigned long long uint64; -typedef size_t usize; +typedef uint8_t uint8; +typedef uint16_t uint16; +typedef uint32_t uint32; +typedef uint64_t uint64; +typedef size_t usize; -#ifdef LEAN_RUNTIME_STATS -#define LEAN_RUNTIME_STAT_CODE(c) c -extern atomic g_num_ctor; -extern atomic g_num_closure; -extern atomic g_num_string; -extern atomic g_num_array; -extern atomic g_num_thunk; -extern atomic g_num_task; -extern atomic g_num_ext; -extern atomic g_num_st_inc; -extern atomic g_num_mt_inc; -extern atomic g_num_st_dec; -extern atomic g_num_mt_dec; -extern atomic g_num_del; -#else -#define LEAN_RUNTIME_STAT_CODE(c) -#endif +typedef lean_object object; +typedef object * obj_arg; +typedef object * b_obj_arg; +typedef object * u_obj_arg; +typedef object * obj_res; +typedef object * b_obj_res; -/* Objects can be stored in 5 different kinds of memory: - - `MTHeap`: multi-threaded heap, the reference counter (RC) is updated using atomic operations. - All objects reachable from an object in the `MTHeap` are in `MTHeap`, `Persistent` or `Region`. - - `STHeap`: single-threaded heap, the RC is faster to update. - All objects reachable from an object in the `STHeap` are not in the `Stack`. - - `Persistent`: RC is never updated, this kind of object is never garbage collected. - All objects reachable from a persistent object are not in the `Stack`. - - `Stack`: object does not have a RC and is allocated on the system stack. - - `Region`: object does not have a RC, and is stored in a compacted region. - All objects reachable from an object in a compacted region are also in the same compacted region. -*/ -enum class object_memory_kind { MTHeap = 0, STHeap, Persistent, Stack, Region }; - -enum class object_kind { Constructor, Closure, Array, ScalarArray, - String, MPZ, Thunk, Task, Ref, External }; - -/* Objects are initially allocated as STHeap. When we create a task, we change it to MTHeap. */ -constexpr object_memory_kind c_init_mem_kind = object_memory_kind::STHeap; - -/* The reference counter is a uintptr_t, because at deletion time, we use this field to implement - a linked list of objects to be deleted. */ -typedef uintptr_t rc_type; - -/* Base class for all runtime objects. - - \remark If m_mem_kind == STHeap/MTHeap, then we store the reference counter before the object. */ -struct object { - unsigned m_kind:8; - unsigned m_mem_kind:8; - object(object_kind k, object_memory_kind m = c_init_mem_kind): - m_kind(static_cast(k)), m_mem_kind(static_cast(m)) {} +struct mpz_object { + lean_object m_header; + mpz m_value; }; -/* We can represent inductive datatypes that have: - 1) At most 2^16 constructors - 2) At most 2^16 - 1 object fields per constructor - 3) At most 2^16 - 1 bytes for scalar/unboxed fields +typedef lean_external_class external_object_class; +typedef lean_external_finalize_proc external_object_finalize_proc; +typedef lean_external_foreach_proc external_object_foreach_proc; - We only need m_scalar_size for implementing sanity checks at runtime. - - Header size: 12 bytes in 32 bit machines and 16 bytes in 64 bit machines. */ -struct constructor_object : public object { - unsigned short m_tag; - unsigned short m_num_objs; - unsigned short m_scalar_size; - constructor_object(unsigned tag, unsigned num_objs, unsigned scalar_sz, object_memory_kind m = c_init_mem_kind): - object(object_kind::Constructor, m), m_tag(tag), m_num_objs(num_objs), m_scalar_size(scalar_sz) {} -}; - -/* -In our runtime, a Lean function consume the reference counter (RC) of its argument or not. -We say this behavior is part of the "calling convention" for the function. We say an argument uses: - -1- "standard" calling convention if it consumes/decrements the RC. - In this calling convention each argument should be viewed as a resource that is consumed by the function. - This is roughly equivalent to `S && a` in C++, where `S` is a smart pointer, and `a` is the argument. - When this calling convention is used for an argument `x`, then it is safe to perform destructive updates to - `x` if its RC is 1. - -2- "borrowed" calling convention if it doesn't consume/decrement the RC, and it is the responsability of the caller - to decrement the RC. - This is roughly equivalent to `S const & a` in C++, where `S` is a smart pointer, and `a` is the argument. - -For returning objects, we also have two conventions - -1- "standard" result. The caller is responsible for consuming the RC of the result. - This is roughly equivalent to returning a smart point `S` by value in C++. - -2- "borrowed" result. The caller is not responsible for decreasing the RC. - This is roughly equivalent to returning a smart point reference `S const &` in C++. - -Functions stored in closures use the "standard" calling convention. -*/ - -/* The following typedef's are used to document the calling convention for the primitives. */ -typedef object * obj_arg; /* Standard object argument. */ -typedef object * b_obj_arg; /* Borrowed object argument. */ -typedef object * u_obj_arg; /* Unique (aka non shared) object argument. */ -typedef object * obj_res; /* Standard object result. */ -typedef object * b_obj_res; /* Borrowed object result. */ - -/* Array of objects. - Header size: 16 bytes in 32 bit machines and 32 bytes in 64 bit machines. */ -struct array_object : public object { - size_t m_size; - size_t m_capacity; - array_object(size_t sz, size_t c, object_memory_kind m = c_init_mem_kind): - object(object_kind::Array, m), m_size(sz), m_capacity(c) {} -}; - -/* Array of scalar values. - We support arrays with upto 2^64 elements in 64 bit machines. - - The field m_elem_size is only needed for implementing sanity checks at runtime. - Header size: 16 bytes in 32 bit machines and 32 bytes in 64 bit machines. */ -struct sarray_object : public object { - unsigned m_elem_size:16; // in bytes - size_t m_size; - size_t m_capacity; - sarray_object(unsigned esz, size_t sz, size_t c, object_memory_kind m = c_init_mem_kind): - object(object_kind::ScalarArray, m), m_elem_size(esz), m_size(sz), m_capacity(c) {} -}; - -struct string_object : public object { - size_t m_size; - size_t m_capacity; - size_t m_length; // UTF8 length - string_object(size_t sz, size_t c, size_t len, object_memory_kind m = c_init_mem_kind): - object(object_kind::String, m), m_size(sz), m_capacity(c), m_length(len) {} -}; - -/* Note that `m_fun` is a pointer to a C function. - The `apply` operator performs a cast operation. It casts m_fun to a C function pointer of the right arity. - - Header size: 16 bytes in 32 bit machines and 24 bytes in 64 bit machines. - - Note that this structure may also be used to simulate closures built - from bytecodes. We just store an extra argument: the virtual machine - function descriptor. We store in m_fun a pointer to a C function - that extracts the function descriptor and then invokes the VM. */ -struct closure_object : public object { - unsigned m_arity:16; // number of arguments expected by m_fun. - unsigned m_num_fixed:16; // number of arguments that have been already fixed. - void * m_fun; - closure_object(void * f, unsigned arity, unsigned n, object_memory_kind m = c_init_mem_kind): - object(object_kind::Closure, m), m_arity(arity), m_num_fixed(n), m_fun(f) {} -}; - -struct mpz_object : public object { - mpz m_value; - mpz_object(mpz const & v, object_memory_kind m = c_init_mem_kind): - object(object_kind::MPZ, m), m_value(v) {} -}; - -struct ref_object : public object { - object * m_value; - ref_object(object * v, object_memory_kind m = c_init_mem_kind): - object(object_kind::Ref, m), m_value(v) { - }; -}; - -struct thunk_object : public object { - atomic m_value; - atomic m_closure; - thunk_object(object * c, bool is_value = false, object_memory_kind m = c_init_mem_kind); -}; - -struct task_object : public object { - /* Data required for executing a task. It is released as soon as - the task terminates. */ - struct imp { - object * m_closure; - task_object * m_head_dep{nullptr}; /* head of the reverse dependency list of this task. */ - task_object * m_next_dep{nullptr}; /* next element in the reverse dependency list. Each task can be in at most one reverse dependency list. */ - unsigned m_prio; - bool m_interrupted{false}; - bool m_deleted{false}; - imp(object * c, unsigned prio):m_closure(c), m_prio(prio) {} - }; - atomic m_value; - imp * m_imp; - task_object(object * c, unsigned prio); - task_object(object * v); -}; - -struct external_object_class; -typedef void (*external_object_finalize_proc)(void *); -typedef void (*external_object_foreach_proc)(void *, b_obj_arg); -external_object_class * register_external_object_class(external_object_finalize_proc, external_object_foreach_proc); - -/* Object for wrapping external data. - For example, we use it to wrap the Lean environment object. */ -struct external_object : public object { - external_object_class * m_class; - void * m_data; - explicit external_object(external_object_class * cls, void * data, object_memory_kind m = c_init_mem_kind): - object(object_kind::External, m), m_class(cls), m_data(data) {} -}; - -static_assert(sizeof(size_t) == sizeof(void*), "runtime assumes that size_t and void* have the same size, we do not plan to support old platforms where this is not true"); - -inline bool is_null(object * o) { return o == nullptr; } -inline bool is_scalar(object * o) { return (reinterpret_cast(o) & 1) == 1; } -inline object * box(size_t n) { return reinterpret_cast((static_cast(n) << 1) | 1); } -inline size_t unbox(object * o) { return reinterpret_cast(o) >> 1; } - -/* Generic Lean object delete operation. - - The generic delete must be used when we are compiling: - 1- Polymorphic code. - 2- Code using `any` type. - The `any` type is introduced when we translate Lean expression into the Core language based on System-F. - - We are planning to generate delete operations for non-polymorphic code. - They can be faster because: - 1- They do not need to test whether nested pointers are boxed scalars or not. - 2- They do not need to test the kind. - 3- They can unfold the loop that decrements the reference counters for nested objects. - - \pre !is_scalar(o); */ -void del(object * o); - -static_assert(sizeof(atomic) == sizeof(rc_type), "atomic and rc_type must have the same size"); // NOLINT - -void * alloc_heap_object(size_t sz); -void free_heap_obj(object * o); -void free_mpz_obj(object * o); -void free_closure_obj(object * o); - -inline atomic * mt_rc_addr(object * o) { - return reinterpret_cast *>(reinterpret_cast(o) - sizeof(rc_type)); +inline external_object_class * register_external_object_class(external_object_finalize_proc p1, external_object_foreach_proc p2) { + return lean_register_external_class(p1, p2); } -inline rc_type * st_rc_addr(object * o) { - return reinterpret_cast(reinterpret_cast(o) - sizeof(rc_type)); -} +inline bool is_scalar(object * o) { return lean_is_scalar(o); } +inline object * box(size_t n) { return lean_box(n); } +inline size_t unbox(object * o) { return lean_unbox(o); } -inline rc_type & st_rc_ref(object * o) { - return *st_rc_addr(o); -} - -inline bool is_mt_heap_obj(object * o) { -#ifdef LEAN_MULTI_THREAD - return o->m_mem_kind == static_cast(object_memory_kind::MTHeap); -#else - return false; -#endif -} -inline bool is_st_heap_obj(object * o) { return o->m_mem_kind == static_cast(object_memory_kind::STHeap); } +inline bool is_mt_heap_obj(object * o) { return lean_is_mt(o); } +inline bool is_st_heap_obj(object * o) { return lean_is_st(o); } inline bool is_heap_obj(object * o) { return is_st_heap_obj(o) || is_mt_heap_obj(o); } -void mark_mt(object * o); +inline void mark_mt(object * o) { lean_mark_mt(o); } +inline bool is_shared(object * o) { return lean_is_shared(o); } +inline bool is_exclusive(object * o) { return lean_is_exclusive(o); } +inline void inc_ref(object * o) { lean_inc_ref(o); } +inline void inc_ref(object * o, size_t n) { lean_inc_ref_n(o, n); } +inline bool dec_ref_core(object * o) { return lean_dec_ref_core(o); } +inline void dec_ref(object * o) { lean_dec_ref(o); } +inline void inc(object * o) { lean_inc(o); } +inline void inc(object * o, size_t n) { lean_inc_n(o, n); } +inline void dec(object * o) { lean_dec(o); } +inline void free_heap_obj(object * o) { lean_free_object(o); } -inline rc_type get_rc(object * o) { - lean_assert(!is_scalar(o)); - if (LEAN_LIKELY(is_st_heap_obj(o))) { - return st_rc_ref(o); - } else { - lean_assert(is_mt_heap_obj(o)); - return atomic_load_explicit(mt_rc_addr(o), memory_order_acquire); - } -} +inline bool is_cnstr(object * o) { return lean_is_ctor(o); } +inline bool is_closure(object * o) { return lean_is_closure(o); } +inline bool is_array(object * o) { return lean_is_array(o); } +inline bool is_sarray(object * o) { return lean_is_sarray(o); } +inline bool is_string(object * o) { return lean_is_string(o); } +inline bool is_mpz(object * o) { return lean_is_mpz(o); } +inline bool is_thunk(object * o) { return lean_is_thunk(o); } +inline bool is_task(object * o) { return lean_is_task(o); } +inline bool is_external(object * o) { return lean_is_external(o); } +inline bool is_ref(object * o) { return lean_is_ref(o); } -inline bool is_shared(object * o) { return get_rc(o) > 1; } -inline bool is_exclusive(object * o) { return is_heap_obj(o) && !is_shared(o); } +inline void dealloc(object * o) { return lean_dealloc(o); } -inline void inc_ref(object * o) { - if (LEAN_LIKELY(is_st_heap_obj(o))) { - LEAN_RUNTIME_STAT_CODE(g_num_st_inc++); - st_rc_ref(o)++; - } else if (is_mt_heap_obj(o)) { - LEAN_RUNTIME_STAT_CODE(g_num_mt_inc++); - atomic_fetch_add_explicit(mt_rc_addr(o), static_cast(1), memory_order_relaxed); - } -} +inline void mark_persistent(object * o) { return lean_mark_persistent(o); } -inline void inc_ref(object * o, rc_type n) { - if (LEAN_LIKELY(is_st_heap_obj(o))) { - LEAN_RUNTIME_STAT_CODE(g_num_st_inc++); - st_rc_ref(o) += n; - } else if (is_mt_heap_obj(o)) { - LEAN_RUNTIME_STAT_CODE(g_num_mt_inc++); - atomic_fetch_add_explicit(mt_rc_addr(o), static_cast(n), memory_order_relaxed); - } -} - -inline void dec_shared_ref(object * o) { - lean_assert(is_shared(o)); - if (LEAN_LIKELY(is_st_heap_obj(o))) { - LEAN_RUNTIME_STAT_CODE(g_num_st_dec++); - st_rc_ref(o)--; - } else if (is_mt_heap_obj(o)) { - LEAN_RUNTIME_STAT_CODE(g_num_mt_dec++); - atomic_fetch_sub_explicit(mt_rc_addr(o), static_cast(1), memory_order_acq_rel); - } -} - -inline bool dec_ref_core(object * o) { - if (LEAN_LIKELY(is_st_heap_obj(o))) { - lean_assert(get_rc(o) > 0); - LEAN_RUNTIME_STAT_CODE(g_num_st_dec++); - st_rc_ref(o)--; - return st_rc_ref(o) == 0; - } else if (is_mt_heap_obj(o)) { - lean_assert(get_rc(o) > 0); - LEAN_RUNTIME_STAT_CODE(g_num_mt_dec++); - return atomic_fetch_sub_explicit(mt_rc_addr(o), static_cast(1), memory_order_acq_rel) == 1; - } else { - return false; - } -} - -inline void dec_ref(object * o) { if (dec_ref_core(o)) del(o); } -inline void inc(object * o) { if (!is_scalar(o)) inc_ref(o); } -inline void inc(object * o, rc_type n) { if (!is_scalar(o)) inc_ref(o, n); } -inline void dec(object * o) { if (!is_scalar(o)) dec_ref(o); } +inline unsigned obj_tag(b_obj_arg o) { return lean_obj_tag(o); } // ======================================= -// Testers +// Constructors -inline object_kind get_kind(object * o) { return static_cast(o->m_kind); } -inline bool is_cnstr(object * o) { return get_kind(o) == object_kind::Constructor; } -inline bool is_closure(object * o) { return get_kind(o) == object_kind::Closure; } -inline bool is_array(object * o) { return get_kind(o) == object_kind::Array; } -inline bool is_sarray(object * o) { return get_kind(o) == object_kind::ScalarArray; } -inline bool is_string(object * o) { return get_kind(o) == object_kind::String; } -inline bool is_mpz(object * o) { return get_kind(o) == object_kind::MPZ; } -inline bool is_thunk(object * o) { return get_kind(o) == object_kind::Thunk; } -inline bool is_task(object * o) { return get_kind(o) == object_kind::Task; } -inline bool is_external(object * o) { return get_kind(o) == object_kind::External; } -inline bool is_ref(object * o) { return get_kind(o) == object_kind::Ref; } - -// ======================================= -// Casting - -inline constructor_object * to_cnstr(object * o) { lean_assert(is_cnstr(o)); return static_cast(o); } -inline closure_object * to_closure(object * o) { lean_assert(is_closure(o)); return static_cast(o); } -inline array_object * to_array(object * o) { lean_assert(is_array(o)); return static_cast(o); } -inline sarray_object * to_sarray(object * o) { lean_assert(is_sarray(o)); return static_cast(o); } -inline string_object * to_string(object * o) { lean_assert(is_string(o)); return static_cast(o); } -inline mpz_object * to_mpz(object * o) { lean_assert(is_mpz(o)); return static_cast(o); } -inline thunk_object * to_thunk(object * o) { lean_assert(is_thunk(o)); return static_cast(o); } -inline task_object * to_task(object * o) { lean_assert(is_task(o)); return static_cast(o); } -inline ref_object * to_ref(object * o) { lean_assert(is_ref(o)); return static_cast(o); } -inline external_object * to_external(object * o) { lean_assert(is_external(o)); return static_cast(o); } - -/* The memory associated with all Lean objects but `mpz_object` and `external_object` can be deallocated using `free`. - All fields in these objects are integral types, but the reference counter. - However, `std::atomic` has a trivial destructor. - In the C++ reference manual (http://en.cppreference.com/w/cpp/atomic/atomic), we find the following sentence: - - "Additionally, the resulting std::atomic specialization has standard layout, a trivial default constructor, - and a trivial destructor." */ -inline void dealloc_mpz(object * o) { to_mpz(o)->~mpz_object(); free_mpz_obj(o); } -inline void dealloc(object * o) { - lean_assert(is_heap_obj(o)); - switch (get_kind(o)) { - case object_kind::MPZ: dealloc_mpz(o); break; - case object_kind::Task: lean_unreachable(); // only the task manager can deallocate tasks. - default: free_heap_obj(o); break; - } -} - -// ======================================= -// Object auxiliary functions - -/* Size of the object in bytes. This function is used for debugging purposes. - \pre !is_scalar(o) */ -size_t obj_byte_size(object * o); - -/* Size of the object header in bytes. This function is used for debugging purposes. - \pre !is_scalar(o) */ -size_t obj_header_size(object * o); - -/* Retrieves data of type `T` stored offset bytes inside of `o` */ -template -inline T obj_data(object * o, size_t offset) { - lean_assert(obj_header_size(o) <= offset); - lean_assert(offset + sizeof(T) <= obj_byte_size(o)); - return *(reinterpret_cast(reinterpret_cast(o) + offset)); -} - -/* Set object data of type T */ -template -inline void obj_set_data(object * o, size_t offset, T v) { - lean_assert(!is_heap_obj(o) || !is_shared(o)); - lean_assert(obj_header_size(o) <= offset); - lean_assert(offset + sizeof(T) <= obj_byte_size(o)); - *(reinterpret_cast(reinterpret_cast(o) + offset)) = v; -} - -/* Mark all objects reachable from `o` as persistent */ -void mark_persistent(object * o); - -// ======================================= -// Constructor auxiliary functions - -inline unsigned cnstr_num_objs(object * o) { return to_cnstr(o)->m_num_objs; } -inline unsigned cnstr_scalar_size(object * o) { return to_cnstr(o)->m_scalar_size; } -inline size_t cnstr_byte_size(unsigned num_objs, unsigned scalar_sz) { return sizeof(constructor_object) + num_objs*sizeof(object*) + scalar_sz; } // NOLINT -inline size_t cnstr_byte_size(object * o) { return cnstr_byte_size(cnstr_num_objs(o), cnstr_scalar_size(o)); } -inline object ** cnstr_obj_cptr(object * o) { - lean_assert(is_cnstr(o)); - return reinterpret_cast(reinterpret_cast(o) + sizeof(constructor_object)); -} -inline unsigned char * cnstr_scalar_cptr(object * o) { - lean_assert(is_cnstr(o)); - return reinterpret_cast(reinterpret_cast(o) + sizeof(constructor_object) + cnstr_num_objs(o)*sizeof(object*)); // NOLINT -} - -// ======================================= -// Closure auxiliary functions - -inline void * closure_fun(object * o) { return to_closure(o)->m_fun; } -inline unsigned closure_arity(object * o) { return to_closure(o)->m_arity; } -inline unsigned closure_num_fixed(object * o) { return to_closure(o)->m_num_fixed; } -inline size_t closure_byte_size(unsigned num_fixed) { return sizeof(closure_object) + num_fixed*sizeof(object*); } // NOLINT -inline size_t closure_byte_size(object * o) { return closure_byte_size(closure_num_fixed(o)); } -inline object ** closure_arg_cptr(object * o) { - lean_assert(is_closure(o)); - return reinterpret_cast(reinterpret_cast(o) + sizeof(closure_object)); -} - -// ======================================= -// Thunk auxiliary functions - -inline thunk_object::thunk_object(object * c, bool is_value, object_memory_kind m): - object(object_kind::Thunk, m) { - if (is_value) { - m_closure = nullptr; - m_value = c; - } else { - lean_assert(is_closure(c)); - m_closure = c; - m_value = nullptr; - } -} -object * apply_1(object * f, object * a1); -/* Expensive version of thunk_get which tries to execute the nested closure */ -object * thunk_get_core(object * t); - -// ======================================= -// Array auxiliary functions - -inline size_t array_capacity(object * o) { return to_array(o)->m_capacity; } -inline size_t array_byte_size(size_t capacity) { return sizeof(array_object) + capacity*sizeof(object*); } // NOLINT -inline size_t array_byte_size(object * o) { return array_byte_size(array_capacity(o)); } -inline object ** array_cptr(object * o) { - lean_assert(is_array(o)); - return reinterpret_cast(reinterpret_cast(o) + sizeof(array_object)); -} - -// ======================================= -// Array of scalars auxiliary functions - -inline unsigned sarray_elem_size(object * o) { return to_sarray(o)->m_elem_size; } -inline size_t sarray_capacity(object * o) { return to_sarray(o)->m_capacity; } -inline size_t sarray_byte_size(size_t capacity, unsigned elem_size) { return sizeof(sarray_object) + capacity*elem_size; } // NOLINT -inline size_t sarray_byte_size(object * o) { return sarray_byte_size(sarray_capacity(o), sarray_elem_size(o)); } -template -T * sarray_cptr_core(object * o) { lean_assert(is_sarray(o)); return reinterpret_cast(reinterpret_cast(o) + sizeof(sarray_object)); } -template -T * sarray_cptr(object * o) { lean_assert(sarray_elem_size(o) == sizeof(T)); return sarray_cptr_core(o); } - -// ======================================= -// String auxiliary functions - -inline size_t string_capacity(object * o) { return to_string(o)->m_capacity; } -inline size_t string_byte_size(size_t capacity) { return sizeof(string_object) + capacity; } // NOLINT -inline size_t string_byte_size(object * o) { return string_byte_size(string_capacity(o)); } - -// ======================================= -// MPZ auxiliary function - -inline object * alloc_mpz(mpz const & m) { return new (alloc_heap_object(sizeof(mpz_object))) mpz_object(m); } - -// ======================================= -// External objects - -inline object * alloc_external(external_object_class * cls, void * data) { - LEAN_RUNTIME_STAT_CODE(g_num_ext++); - return new (alloc_heap_object(sizeof(external_object))) external_object(cls, data); -} - -inline external_object_class * external_class(object * o) { return to_external(o)->m_class; } -inline void * external_data(object * o) { return to_external(o)->m_data; } - -// ======================================= -// Natural numbers auxiliary functions - -#define LEAN_MAX_SMALL_NAT (std::numeric_limits::max() >> 1) // NOLINT -inline object * mk_nat_obj_core(mpz const & m) { - lean_assert(m > LEAN_MAX_SMALL_NAT); - return alloc_mpz(m); -} -object * nat_big_add(object * a1, object * a2); -object * nat_big_sub(object * a1, object * a2); -object * nat_big_mul(object * a1, object * a2); -object * nat_big_div(object * a1, object * a2); -object * nat_big_mod(object * a1, object * a2); -bool nat_big_eq(object * a1, object * a2); -bool nat_big_le(object * a1, object * a2); -bool nat_big_lt(object * a1, object * a2); -object * nat_big_land(object * a1, object * a2); -object * nat_big_lor(object * a1, object * a2); -object * nat_big_xor(object * a1, object * a2); - -// ======================================= -// Integers auxiliary functions - -#define LEAN_MAX_SMALL_INT (sizeof(void*) == 8 ? std::numeric_limits::max() : (1 << 30)) // NOLINT -#define LEAN_MIN_SMALL_INT (sizeof(void*) == 8 ? std::numeric_limits::min() : -(1 << 30)) // NOLINT -inline object * mk_int_obj_core(mpz const & m) { - lean_assert(m < LEAN_MIN_SMALL_INT || m > LEAN_MAX_SMALL_INT); - return alloc_mpz(m); -} -object * int_big_add(object * a1, object * a2); -object * int_big_sub(object * a1, object * a2); -object * int_big_mul(object * a1, object * a2); -object * int_big_div(object * a1, object * a2); -object * int_big_mod(object * a1, object * a2); -bool int_big_eq(object * a1, object * a2); -bool int_big_le(object * a1, object * a2); -bool int_big_lt(object * a1, object * a2); - -// ======================================= -// Constructor objects -inline obj_res alloc_cnstr(unsigned tag, unsigned num_objs, unsigned scalar_sz) { - LEAN_RUNTIME_STAT_CODE(g_num_ctor++); - lean_assert(tag < 65536 && num_objs < 65536 && scalar_sz < 65536); - return new (alloc_heap_object(cnstr_byte_size(num_objs, scalar_sz))) constructor_object(tag, num_objs, scalar_sz); // NOLINT -} -inline unsigned cnstr_tag(b_obj_arg o) { return to_cnstr(o)->m_tag; } -inline void cnstr_set_tag(b_obj_arg o, unsigned tag) { to_cnstr(o)->m_tag = tag; } -/* Access constructor object field `i` */ -inline b_obj_res cnstr_get(b_obj_arg o, unsigned i) { - lean_assert(i < cnstr_num_objs(o)); - return obj_data(o, sizeof(constructor_object) + sizeof(object*)*i); // NOLINT -} -/* Update constructor field `i` */ -inline void cnstr_set(u_obj_arg o, unsigned i, obj_arg v) { - lean_assert(!is_heap_obj(o) || !is_shared(o)); - lean_assert(i < cnstr_num_objs(o)); - obj_set_data(o, sizeof(constructor_object) + sizeof(object*)*i, v); // NOLINT -} -/* Release field `i`, that is, decrement its reference counter, and then set it to box(0) */ -inline void cnstr_release(u_obj_arg o, unsigned i) { - lean_assert(!is_heap_obj(o) || !is_shared(o)); - lean_assert(i < cnstr_num_objs(o)); - object ** field_ptr = reinterpret_cast(reinterpret_cast(o) + sizeof(constructor_object) + sizeof(object*)*i); - dec(*field_ptr); - *field_ptr = box(0); -} -inline usize cnstr_get_usize(b_obj_arg o, unsigned i) { - return obj_data(o, sizeof(constructor_object) + i*sizeof(void*)); -} -inline void cnstr_set_usize(b_obj_arg o, unsigned i, usize v) { - obj_set_data(o, sizeof(constructor_object) + i*sizeof(void*), v); -} -/* Access scalar data at the given offset. */ -inline uint8 cnstr_get_uint8(b_obj_arg o, unsigned offset) { - return obj_data(o, sizeof(constructor_object) + offset); -} -inline uint16 cnstr_get_uint16(b_obj_arg o, unsigned offset) { - return obj_data(o, sizeof(constructor_object) + offset); -} -inline uint32 cnstr_get_uint32(b_obj_arg o, unsigned offset) { - return obj_data(o, sizeof(constructor_object) + offset); -} -inline uint64 cnstr_get_uint64(b_obj_arg o, unsigned offset) { - return obj_data(o, sizeof(constructor_object) + offset); -} -inline void cnstr_set_uint8(b_obj_arg o, unsigned offset, uint8 v) { - obj_set_data(o, sizeof(constructor_object) + offset, v); -} -inline void cnstr_set_uint16(b_obj_arg o, unsigned offset, uint16 v) { - obj_set_data(o, sizeof(constructor_object) + offset, v); -} -inline void cnstr_set_uint32(b_obj_arg o, unsigned offset, uint32 v) { - obj_set_data(o, sizeof(constructor_object) + offset, v); -} -inline void cnstr_set_uint64(b_obj_arg o, unsigned offset, uint64 v) { - obj_set_data(o, sizeof(constructor_object) + offset, v); -} +inline unsigned cnstr_num_objs(object * o) { return lean_ctor_num_objs(o); } +inline object ** cnstr_obj_cptr(object * o) { return lean_ctor_obj_cptr(o); } +inline uint8 * cnstr_scalar_cptr(object * o) { return lean_ctor_scalar_cptr(o); } +inline obj_res alloc_cnstr(unsigned tag, unsigned num_objs, unsigned scalar_sz) { return lean_alloc_ctor(tag, num_objs, scalar_sz); } +inline unsigned cnstr_tag(b_obj_arg o) { return lean_ptr_tag(o); } +inline void cnstr_set_tag(b_obj_arg o, unsigned tag) { lean_ctor_set_tag(o, tag); } +inline b_obj_res cnstr_get(b_obj_arg o, unsigned i) { return lean_ctor_get(o, i); } +inline void cnstr_set(u_obj_arg o, unsigned i, obj_arg v) { lean_ctor_set(o, i, v); } +inline void cnstr_release(u_obj_arg o, unsigned i) { lean_ctor_release(o, i); } +inline usize cnstr_get_usize(b_obj_arg o, unsigned i) { return lean_ctor_get_usize(o, i); } +inline void cnstr_set_usize(b_obj_arg o, unsigned i, usize v) { lean_ctor_set_usize(o, i, v); } +inline uint8 cnstr_get_uint8(b_obj_arg o, unsigned offset) { return lean_ctor_get_uint8(o, offset); } +inline uint16 cnstr_get_uint16(b_obj_arg o, unsigned offset) { return lean_ctor_get_uint16(o, offset); } +inline uint32 cnstr_get_uint32(b_obj_arg o, unsigned offset) { return lean_ctor_get_uint32(o, offset); } +inline uint64 cnstr_get_uint64(b_obj_arg o, unsigned offset) { return lean_ctor_get_uint64(o, offset); } +inline void cnstr_set_uint8(b_obj_arg o, unsigned offset, uint8 v) { lean_ctor_set_uint8(o, offset, v); } +inline void cnstr_set_uint16(b_obj_arg o, unsigned offset, uint16 v) { lean_ctor_set_uint16(o, offset, v); } +inline void cnstr_set_uint32(b_obj_arg o, unsigned offset, uint32 v) { lean_ctor_set_uint32(o, offset, v); } +inline void cnstr_set_uint64(b_obj_arg o, unsigned offset, uint64 v) { lean_ctor_set_uint64(o, offset, v); } template inline T cnstr_get_scalar(b_obj_arg o, unsigned offset) { - return obj_data(o, sizeof(constructor_object) + offset); + return *((T*)((uint8_t*)(lean_ctor_obj_cptr(o)) + offset)); } template inline void cnstr_set_scalar(b_obj_arg o, unsigned offset, T v) { - obj_set_data(o, sizeof(constructor_object) + offset, v); + *((T*)((uint8_t*)(lean_ctor_obj_cptr(o)) + offset)) = v; } -inline unsigned obj_tag(b_obj_arg o) { if (is_scalar(o)) return unbox(o); else return cnstr_tag(o); } - // ======================================= // Closures -inline obj_res alloc_closure(void * fun, unsigned arity, unsigned num_fixed) { - LEAN_RUNTIME_STAT_CODE(g_num_closure++); - lean_assert(arity > 0); - lean_assert(num_fixed < arity); - return new (alloc_heap_object(closure_byte_size(num_fixed))) closure_object(fun, arity, num_fixed); // NOLINT -} -inline b_obj_res closure_get(b_obj_arg o, unsigned i) { - lean_assert(i < closure_num_fixed(o)); - return obj_data(o, sizeof(closure_object) + sizeof(object*)*i); // NOLINT -} -inline void closure_set(u_obj_arg o, unsigned i, obj_arg a) { - lean_assert(i < closure_num_fixed(o)); - obj_set_data(o, sizeof(closure_object) + sizeof(object*)*i, a); // NOLINT -} +void free_closure_obj(object * o); +inline void * closure_fun(object * o) { return lean_closure_fun(o); } +inline unsigned closure_arity(object * o) { return lean_closure_arity(o); } +inline unsigned closure_num_fixed(object * o) { return lean_closure_num_fixed(o); } +inline object ** closure_arg_cptr(object * o) { return lean_closure_arg_cptr(o); } +inline obj_res alloc_closure(void * fun, unsigned arity, unsigned num_fixed) { return lean_alloc_closure(fun, arity, num_fixed); } +inline b_obj_res closure_get(b_obj_arg o, unsigned i) { return lean_closure_get(o, i); } +inline void closure_set(u_obj_arg o, unsigned i, obj_arg a) { lean_closure_set(o, i, a); } inline obj_res alloc_closure(object*(*fun)(object *), unsigned num_fixed) { return alloc_closure(reinterpret_cast(fun), 1, num_fixed); } @@ -662,93 +135,119 @@ inline obj_res alloc_closure(object*(*fun)(object *, object *, object *, object inline obj_res alloc_closure(object*(*fun)(object *, object *, object *, object *, object *, object *, object *, object *), unsigned num_fixed) { return alloc_closure(reinterpret_cast(fun), 8, num_fixed); } +object * apply_1(object * f, object * a1) { return lean_apply_1(f, a1); } // ======================================= // Fixpoint -obj_res fixpoint(obj_arg rec, obj_arg a); -obj_res fixpoint2(obj_arg rec, obj_arg a1, obj_arg a2); -obj_res fixpoint3(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3); -obj_res fixpoint4(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4); -obj_res fixpoint5(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5); -obj_res fixpoint6(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5, obj_arg a6); - -// ======================================= -// Array of objects -inline obj_res alloc_array(size_t size, size_t capacity) { - LEAN_RUNTIME_STAT_CODE(g_num_array++); - return new (alloc_heap_object(array_byte_size(capacity))) array_object(size, capacity); // NOLINT +inline obj_res fixpoint(obj_arg rec, obj_arg a) { return lean_fixpoint(rec, a); } +inline obj_res fixpoint2(obj_arg rec, obj_arg a1, obj_arg a2) { return lean_fixpoint2(rec, a1, a2); } +inline obj_res fixpoint3(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3) { return lean_fixpoint3(rec, a1, a2, a3); } +inline obj_res fixpoint4(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4) { return lean_fixpoint4(rec, a1, a2, a3, a4); } +inline obj_res fixpoint5(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5) { + return lean_fixpoint5(rec, a1, a2, a3, a4, a5); } -/* Return empty array. Result is a persistent object, no RC operations are needed. */ -object * array_mk_empty(); -inline size_t array_size(b_obj_arg o) { return to_array(o)->m_size; } -inline void array_set_size(u_obj_arg o, size_t sz) { - lean_assert(is_array(o)); - lean_assert(!is_heap_obj(o) || !is_shared(o)); - lean_assert(sz <= array_capacity(o)); - to_array(o)->m_size = sz; -} -inline b_obj_res array_get(b_obj_arg o, size_t i) { - lean_assert(i < array_size(o)); - return obj_data(o, sizeof(array_object) + sizeof(object*)*i); // NOLINT -} -inline void array_set(u_obj_arg o, size_t i, obj_arg v) { - lean_assert(!is_heap_obj(o) || !is_shared(o)); - lean_assert(i < array_size(o)); - obj_set_data(o, sizeof(array_object) + sizeof(object*)*i, v); // NOLINT -} - -// ======================================= -// Array of scalars - -inline obj_res alloc_sarray(unsigned elem_size, size_t size, size_t capacity) { - return new (alloc_heap_object(sarray_byte_size(capacity, elem_size))) sarray_object(elem_size, size, capacity); // NOLINT -} -inline size_t sarray_size(b_obj_arg o) { return to_sarray(o)->m_size; } -inline void sarray_set_size(u_obj_arg o, size_t sz) { - lean_assert(is_sarray(o)); - lean_assert(!is_heap_obj(o) || !is_shared(o)); - lean_assert(sz <= sarray_capacity(o)); - to_sarray(o)->m_size = sz; -} -template T sarray_get(b_obj_arg o, size_t i) { return sarray_cptr(o)[i]; } -template void sarray_set(u_obj_arg o, size_t i, T v) { - obj_set_data(o, sizeof(sarray_object) + sizeof(T)*i, v); +inline obj_res fixpoint6(obj_arg rec, obj_arg a1, obj_arg a2, obj_arg a3, obj_arg a4, obj_arg a5, obj_arg a6) { + return lean_fixpoint6(rec, a1, a2, a3, a4, a5, a6); } // ======================================= // MPZ -inline mpz const & mpz_value(b_obj_arg o) { return to_mpz(o)->m_value; } +object * alloc_mpz(mpz const &); +inline mpz_object * to_mpz(object * o) { lean_assert(is_mpz(o)); return (mpz_object*)o; } + +// ======================================= +// Array of objects + +inline size_t array_capacity(object * o) { return lean_array_capacity(o); } +inline object ** array_cptr(object * o) { return lean_array_cptr(o); } +inline obj_res alloc_array(size_t size, size_t capacity) { return lean_alloc_array(size, capacity); } +object * array_mk_empty(); +inline size_t array_size(b_obj_arg o) { return lean_array_size(o); } +inline void array_set_size(u_obj_arg o, size_t sz) { lean_array_set_size(o, sz); } +inline b_obj_res array_get(b_obj_arg o, size_t i) { return lean_array_get_core(o, i); } +inline void array_set(u_obj_arg o, size_t i, obj_arg v) { lean_array_set_core(o, i, v); } +inline object * array_sz(obj_arg a) { return lean_array_sz(a); } +inline object * array_get_size(b_obj_arg a) { return lean_array_get_size(a); } +inline object * mk_empty_array() { return lean_mk_empty_array(); } +inline object * mk_empty_array(b_obj_arg capacity) { return lean_mk_empty_array_with_capacity(capacity); } +inline object * array_uget(b_obj_arg a, usize i) { return lean_array_uget(a, i); } +inline obj_res array_fget(b_obj_arg a, b_obj_arg i) { return lean_array_fget(a, i); } +inline object * array_get(obj_arg def_val, b_obj_arg a, b_obj_arg i) { return lean_array_get(def_val, a, i); } +inline obj_res copy_array(obj_arg a, bool expand = false) { return lean_copy_expand_array(a, expand); } +inline object * array_uset(obj_arg a, usize i, obj_arg v) { return lean_array_uset(a, i, v); } +inline object * array_fset(obj_arg a, b_obj_arg i, obj_arg v) { return lean_array_fset(a, i, v); } +inline object * array_set(obj_arg a, b_obj_arg i, obj_arg v) { return lean_array_set(a, i, v); } +inline object * array_pop(obj_arg a) { return lean_array_pop(a); } +inline object * array_uswap(obj_arg a, usize i, usize j) { return lean_array_uswap(a, i, j); } +inline object * array_fswap(obj_arg a, b_obj_arg i, b_obj_arg j) { return lean_array_fswap(a, i, j); } +inline object * array_swap(obj_arg a, b_obj_arg i, b_obj_arg j) { return lean_array_swap(a, i, j); } +inline object * array_push(obj_arg a, obj_arg v) { return lean_array_push(a, v); } +inline object * mk_array(obj_arg n, obj_arg v) { return lean_mk_array(n, v); } + +// ======================================= +// Array of scalars + +inline obj_res alloc_sarray(unsigned elem_size, size_t size, size_t capacity) { return lean_alloc_sarray(elem_size, size, capacity); } +inline size_t sarray_size(b_obj_arg o) { return lean_sarray_size(o); } +inline void sarray_set_size(u_obj_arg o, size_t sz) { lean_sarray_set_size(o, sz); } +inline unsigned sarray_elem_size(object * o) { return lean_sarray_elem_size(o); } +inline size_t sarray_capacity(object * o) { return lean_sarray_capacity(o); } +inline uint8 * sarray_cptr(object * o) { return lean_sarray_cptr(o); } + +// ======================================= +// ByteArray + +inline obj_res byte_array_mk(obj_arg a) { return lean_byte_array_mk(a); } +inline obj_res byte_array_data(obj_arg a) { return lean_byte_array_data(a); } +inline obj_res copy_byte_array(obj_arg a) { return lean_copy_byte_array(a); } +inline obj_res mk_empty_byte_array(b_obj_arg capacity) { return lean_mk_empty_byte_array(capacity); } +inline obj_res byte_array_size(b_obj_arg a) { return lean_byte_array_size(a); } +inline uint8 byte_array_get(b_obj_arg a, b_obj_arg i) { return lean_byte_array_get(a, i); } +inline obj_res byte_array_push(obj_arg a, uint8 b) { return lean_byte_array_push(a, b); } +inline obj_res byte_array_set(obj_arg a, b_obj_arg i, uint8 b) { return lean_byte_array_set(a, i, b); } + +// ======================================= +// String + +inline size_t string_capacity(object * o) { return lean_string_capacity(o); } +inline uint32 char_default_value() { return lean_char_default_value(); } +inline obj_res alloc_string(size_t size, size_t capacity, size_t len) { return lean_alloc_string(size, capacity, len); } +inline obj_res mk_string(char const * s) { return lean_mk_string(s); } +obj_res mk_string(std::string const & s); +std::string string_to_std(b_obj_arg o); +inline char const * string_cstr(b_obj_arg o) { return lean_string_cstr(o); } +inline size_t string_size(b_obj_arg o) { return lean_string_size(o); } +inline size_t string_len(b_obj_arg o) { return lean_string_len(o); } +inline obj_res string_push(obj_arg s, uint32 c) { return lean_string_push(s, c); } +inline obj_res string_append(obj_arg s1, b_obj_arg s2) { return lean_string_append(s1, s2); } +inline obj_res string_length(b_obj_arg s) { return lean_string_length(s); } +inline obj_res string_mk(obj_arg cs) { return lean_string_mk(cs); } +inline obj_res string_data(obj_arg s) { return lean_string_data(s); } +inline uint32 string_utf8_get(b_obj_arg s, b_obj_arg i) { return lean_string_utf8_get(s, i); } +inline obj_res string_utf8_next(b_obj_arg s, b_obj_arg i) { return lean_string_utf8_next(s, i); } +inline obj_res string_utf8_prev(b_obj_arg s, b_obj_arg i) { return lean_string_utf8_prev(s, i); } +inline obj_res string_utf8_set(obj_arg s, b_obj_arg i, uint32 c) { return lean_string_utf8_set(s, i, c); } +inline uint8 string_utf8_at_end(b_obj_arg s, b_obj_arg i) { return lean_string_utf8_at_end(s, i); } +inline obj_res string_utf8_extract(b_obj_arg s, b_obj_arg b, b_obj_arg e) { return lean_string_utf8_extract(s, b, e); } +inline bool string_eq(b_obj_arg s1, b_obj_arg s2) { return lean_string_eq(s1, s2); } +bool string_eq(b_obj_arg s1, char const * s2); +inline bool string_ne(b_obj_arg s1, b_obj_arg s2) { return lean_string_ne(s1, s2); } +inline bool string_lt(b_obj_arg s1, b_obj_arg s2) { return lean_string_lt(s1, s2); } +inline uint8 string_dec_eq(b_obj_arg s1, b_obj_arg s2) { return string_eq(s1, s2); } +inline uint8 string_dec_lt(b_obj_arg s1, b_obj_arg s2) { return string_lt(s1, s2); } +inline usize string_hash(b_obj_arg s) { return lean_string_hash(s); } // ======================================= // Thunks -inline obj_res mk_thunk(obj_arg c) { - LEAN_RUNTIME_STAT_CODE(g_num_thunk++); - return new (alloc_heap_object(sizeof(thunk_object))) thunk_object(c, false); // NOLINT -} - -/* thunk.pure : A -> thunk A */ -inline obj_res thunk_pure(obj_arg v) { - return new (alloc_heap_object(sizeof(thunk_object))) thunk_object(v, true); // NOLINT -} - -inline b_obj_res thunk_get(b_obj_arg t) { - if (object * r = to_thunk(t)->m_value) - return r; - return thunk_get_core(t); -} - -/* Primitive for implementing the IR instruction for thunk.get : thunk A -> A */ -inline obj_res thunk_get_own(b_obj_arg t) { - object * r = thunk_get(t); - inc(r); - return r; -} - -obj_res thunk_map(obj_arg f, obj_arg t); -obj_res thunk_bind(obj_arg x, obj_arg f); +inline obj_res mk_thunk(obj_arg c) { return lean_mk_thunk(c); } +inline obj_res thunk_pure(obj_arg v) { return lean_thunk_pure(v); } +inline b_obj_res thunk_get(b_obj_arg t) { return lean_thunk_get(t); } +inline obj_res thunk_get_own(b_obj_arg t) { return lean_thunk_get_own(t); } +inline obj_res thunk_map(obj_arg f, obj_arg t) { return lean_thunk_map(f, t); } +inline obj_res thunk_bind(obj_arg x, obj_arg f) { return lean_thunk_bind(x, f); } // ======================================= // Tasks @@ -761,406 +260,27 @@ public: ~scoped_task_manager(); }; -/* Convert a closure (unit -> A) into a task A */ -obj_res mk_task(obj_arg c, unsigned prio = 0); -/* Convert a value `a : A` into `task A` */ -obj_res task_pure(obj_arg a); -/* task.bind (x : task A) (f : A -> task B) : task B */ -obj_res task_bind(obj_arg x, obj_arg f, unsigned prio = 0); -/* task.map (f : A -> B) (t : task A) : task B */ -obj_res task_map(obj_arg f, obj_arg t, unsigned prio = 0); -/* task.get (t : task A) : A */ -b_obj_res task_get(b_obj_arg t); +inline obj_res mk_task(obj_arg c, unsigned prio = 0) { return lean_mk_task_with_prio(c, prio); } +inline obj_res task_pure(obj_arg a) { return lean_task_pure(a); } +inline obj_res task_bind(obj_arg x, obj_arg f, unsigned prio = 0) { return lean_task_bind_with_prio(x, f, prio); } +inline obj_res task_map(obj_arg f, obj_arg t, unsigned prio = 0) { return lean_task_map_with_prio(f, t, prio); } +inline b_obj_res task_get(b_obj_arg t) { return lean_task_get(t); } /* primitive for implementing `io.check_interrupt : io bool` */ -bool io_check_interrupt_core(); +inline bool io_check_interrupt_core() { return lean_io_check_interrupt_core(); } /* primitive for implementing `io.request_interrupt : task A -> io unit` */ -void io_request_interrupt_core(b_obj_arg t); +inline void io_request_interrupt_core(b_obj_arg t) { return lean_io_request_interrupt_core(t); } /* primitive for implementing `io.has_finished : task A -> io unit` */ -bool io_has_finished_core(b_obj_arg t); +inline bool io_has_finished_core(b_obj_arg t) { return lean_io_has_finished_core(t); } /* primitive for implementing `io.wait_any : list (task A) -> io (task A) */ -b_obj_res io_wait_any_core(b_obj_arg task_list); +inline b_obj_res io_wait_any_core(b_obj_arg task_list) { return lean_io_wait_any_core(task_list); } // ======================================= -// Natural numbers +// External -inline obj_res mk_nat_obj(mpz const & m) { - if (m.is_size_t() && m.get_size_t() <= LEAN_MAX_SMALL_NAT) - return box(m.get_size_t()); - else - return mk_nat_obj_core(m); -} - -inline obj_res usize_to_nat(usize n) { - if (n <= LEAN_MAX_SMALL_NAT) { - return box(n); - } else { - return mk_nat_obj_core(mpz(n)); - } -} - -inline obj_res mk_nat_obj(unsigned n) { - return usize_to_nat(static_cast(n)); -} - -inline obj_res uint64_to_nat(uint64 n) { - if (LEAN_LIKELY(n <= LEAN_MAX_SMALL_NAT)) { - return box(n); - } else { - return mk_nat_obj_core(mpz(n)); - } -} - -inline obj_res nat_succ(b_obj_arg a) { - if (LEAN_LIKELY(is_scalar(a))) { - return uint64_to_nat(unbox(a) + 1); - } else { - return mk_nat_obj_core(mpz_value(a) + 1); - } -} - -inline obj_res nat_add(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - usize r = unbox(a1) + unbox(a2); - if (r <= LEAN_MAX_SMALL_NAT) - return box(r); - else - return mk_nat_obj_core(mpz(r)); - } else { - return nat_big_add(a1, a2); - } -} - -inline obj_res nat_sub(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - usize n1 = unbox(a1); - usize n2 = unbox(a2); - if (n1 < n2) - return box(0); - else - return box(n1 - n2); - } else { - return nat_big_sub(a1, a2); - } -} - -inline obj_res nat_mul(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - usize n1 = unbox(a1); - if (n1 == 0) - return a1; - usize n2 = unbox(a2); - usize r = n1*n2; - if (r <= LEAN_MAX_SMALL_NAT && r / n1 == n2) - return box(r); - else - return mk_nat_obj_core(mpz(n1)*mpz(n2)); - } else { - return nat_big_mul(a1, a2); - } -} - -inline obj_res nat_div(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - usize n1 = unbox(a1); - usize n2 = unbox(a2); - if (n2 == 0) - return box(0); - else - return box(n1 / n2); - } else { - return nat_big_div(a1, a2); - } -} - -inline obj_res nat_mod(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - usize n1 = unbox(a1); - usize n2 = unbox(a2); - if (n2 == 0) - return box(0); - else - return box(n1 % n2); - } else { - return nat_big_mod(a1, a2); - } -} - -inline bool nat_eq(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return a1 == a2; - } else { - return nat_big_eq(a1, a2); - } -} - -inline uint8 nat_dec_eq(b_obj_arg a1, b_obj_arg a2) { return nat_eq(a1, a2); } - -inline bool nat_ne(b_obj_arg a1, b_obj_arg a2) { - return !nat_eq(a1, a2); -} - -inline bool nat_le(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return a1 <= a2; - } else { - return nat_big_le(a1, a2); - } -} - -inline uint8 nat_dec_le(b_obj_arg a1, b_obj_arg a2) { return nat_le(a1, a2); } - -inline bool nat_lt(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return a1 < a2; - } else { - return nat_big_lt(a1, a2); - } -} - -inline uint8 nat_dec_lt(b_obj_arg a1, b_obj_arg a2) { return nat_lt(a1, a2); } - -inline obj_res nat_land(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return reinterpret_cast(reinterpret_cast(a1) & reinterpret_cast(a2)); - } else { - return nat_big_land(a1, a2); - } -} - -inline obj_res nat_lor(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return reinterpret_cast(reinterpret_cast(a1) | reinterpret_cast(a2)); - } else { - return nat_big_lor(a1, a2); - } -} - -inline obj_res nat_lxor(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return box(unbox(a1) ^ unbox(a2)); - } else { - return nat_big_xor(a1, a2); - } -} - -// ======================================= -// Integers - -inline obj_res mk_int_obj(mpz const & m) { - if (m < LEAN_MIN_SMALL_INT || m > LEAN_MAX_SMALL_INT) - return mk_int_obj_core(m); - else - return box(static_cast(m.get_int())); -} - -inline obj_res mk_int_obj(int n) { - if (sizeof(void*) == 8) { // NOLINT - return box(static_cast(n)); - } else if (LEAN_MIN_SMALL_INT <= n && n <= LEAN_MAX_SMALL_INT) { - return box(static_cast(n)); - } else { - return alloc_mpz(mpz(n)); - } -} - -inline obj_res mk_int_obj(int64 n) { - if (LEAN_LIKELY(LEAN_MIN_SMALL_INT <= n && n <= LEAN_MAX_SMALL_INT)) { - return box(static_cast(static_cast(n))); - } else { - return mk_int_obj_core(mpz(n)); - } -} - -inline int64 int2int64(b_obj_arg a) { - lean_assert(is_scalar(a)); - if (sizeof(void*) == 8) { // NOLINT - return static_cast(static_cast(unbox(a))); - } else { - return static_cast(reinterpret_cast(a)) >> 1; - } -} - -inline int int2int(b_obj_arg a) { - lean_assert(is_scalar(a)); - if (sizeof(void*) == 8) { // NOLINT - return static_cast(static_cast(unbox(a))); - } else { - return static_cast(reinterpret_cast(a)) >> 1; - } -} - -inline obj_res nat2int(obj_arg a) { - if (is_scalar(a)) { - usize v = unbox(a); - if (v <= LEAN_MAX_SMALL_INT) { - return a; - } else { - return alloc_mpz(mpz(v)); - } - } else { - return a; - } -} - -inline obj_res int_neg(b_obj_arg a) { - if (LEAN_LIKELY(is_scalar(a))) { - return mk_int_obj(-int2int64(a)); - } else { - return mk_int_obj(neg(mpz_value(a))); - } -} - -inline obj_res int_neg_succ_of_nat(obj_arg a) { - obj_res s = nat_succ(a); - obj_res i = nat2int(s); - obj_res r = int_neg(i); - dec(s); dec(i); dec(a); - return r; -} - -inline obj_res int_add(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return mk_int_obj(int2int64(a1) + int2int64(a2)); - } else { - return int_big_add(a1, a2); - } -} - -inline obj_res int_sub(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return mk_int_obj(int2int64(a1) - int2int64(a2)); - } else { - return int_big_sub(a1, a2); - } -} - -inline obj_res int_mul(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return mk_int_obj(int2int64(a1) * int2int64(a2)); - } else { - return int_big_mul(a1, a2); - } -} - -inline obj_res int_div(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - int v1 = int2int(a1); - int v2 = int2int(a2); - if (v2 == 0) - return box(0); - else - return mk_int_obj(v1 / v2); - } else { - return int_big_div(a1, a2); - } -} - -inline obj_res int_mod(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - int v1 = int2int(a1); - int v2 = int2int(a2); - if (v2 == 0) - return box(0); - else - return mk_int_obj(v1 % v2); - } else { - return int_big_mod(a1, a2); - } -} - -inline bool int_eq(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return a1 == a2; - } else { - return int_big_eq(a1, a2); - } -} - -inline bool int_ne(b_obj_arg a1, b_obj_arg a2) { - return !int_eq(a1, a2); -} - -inline bool int_le(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return int2int(a1) <= int2int(a2); - } else { - return int_big_le(a1, a2); - } -} - -inline bool int_lt(b_obj_arg a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a1) && is_scalar(a2))) { - return int2int(a1) < int2int(a2); - } else { - return int_big_lt(a1, a2); - } -} - -inline obj_res nat_abs(b_obj_arg i) { - if (int_lt(i, mk_int_obj(0))) { - return int_neg(i); - } else { - inc(i); - return i; - } -} - -inline uint8 int_dec_eq(b_obj_arg a1, b_obj_arg a2) { return int_eq(a1, a2); } - -inline uint8 int_dec_le(b_obj_arg a1, b_obj_arg a2) { return int_le(a1, a2); } - -inline uint8 int_dec_lt(b_obj_arg a1, b_obj_arg a2) { return int_lt(a1, a2); } - -inline uint8 int_dec_nonneg(b_obj_arg a) { - if (LEAN_LIKELY(is_scalar(a))) { - return int2int(a) >= 0; - } else { - return mpz_value(a) >= 0; - } -} - -inline obj_res box_uint32(unsigned v) { - if (sizeof(void*) == 4) { - // 32-bit implementation - obj_res r = alloc_cnstr(0, 0, sizeof(unsigned)); - cnstr_set_scalar(r, 0, v); - return r; - } else { - // 64-bit implementation - return box(v); - } -} - -inline unsigned unbox_uint32(b_obj_arg o) { - if (sizeof(void*) == 4) { - // 32-bit implementation - return cnstr_get_scalar(o, 0); - } else { - // 64-bit implementation - return unbox(o); - } -} - -inline obj_res box_uint64(unsigned long long v) { - obj_res r = alloc_cnstr(0, 0, sizeof(unsigned long long)); - cnstr_set_scalar(r, 0, v); - return r; -} - -inline unsigned long long unbox_uint64(b_obj_arg o) { - return cnstr_get_scalar(o, 0); -} - -inline obj_res box_size_t(size_t v) { - obj_res r = alloc_cnstr(0, 0, sizeof(size_t)); - cnstr_set_scalar(r, 0, v); - return r; -} - -inline size_t unbox_size_t(b_obj_arg o) { - return cnstr_get_scalar(o, 0); -} +inline object * alloc_external(external_object_class * cls, void * data) { return lean_alloc_external(cls, data); } +inline external_object_class * external_class(object * o) { return lean_get_external_class(o); } +inline void * external_data(object * o) { return lean_get_external_data(o); } // ======================================= // Option @@ -1169,340 +289,167 @@ inline obj_res mk_option_none() { return box(0); } inline obj_res mk_option_some(obj_arg v) { obj_res r = alloc_cnstr(1, 1, 0); cnstr_set(r, 0, v); return r; } // ======================================= -// String +// Natural numbers -/* instance : inhabited char := ⟨'A'⟩ */ -inline uint32 char_default_value() { return 'A'; } -inline obj_res alloc_string(size_t size, size_t capacity, size_t len) { - LEAN_RUNTIME_STAT_CODE(g_num_string++); - return new (alloc_heap_object(string_byte_size(capacity))) string_object(size, capacity, len); // NOLINT +inline mpz const & mpz_value(b_obj_arg o) { return ((mpz_object*)o)->m_value; } +object * mk_nat_obj_core(mpz const & m); +inline obj_res mk_nat_obj(mpz const & m) { + if (m.is_size_t() && m.get_size_t() <= LEAN_MAX_SMALL_NAT) + return box(m.get_size_t()); + else + return mk_nat_obj_core(m); } -obj_res mk_string(char const * s); -obj_res mk_string(std::string const & s); -std::string string_to_std(b_obj_arg o); -inline char const * string_cstr(b_obj_arg o) { lean_assert(is_string(o)); return reinterpret_cast(o) + sizeof(string_object); } -inline size_t string_size(b_obj_arg o) { return to_string(o)->m_size; } -inline size_t string_len(b_obj_arg o) { return to_string(o)->m_length; } -obj_res string_push(obj_arg s, uint32 c); -obj_res string_append(obj_arg s1, b_obj_arg s2); -inline obj_res string_length(b_obj_arg s) { return usize_to_nat(string_len(s)); } // TODO(Leo): improve -obj_res string_mk(obj_arg cs); -obj_res string_data(obj_arg s); - -uint32 string_utf8_get(b_obj_arg s, b_obj_arg i); -obj_res string_utf8_next(b_obj_arg s, b_obj_arg i); -obj_res string_utf8_prev(b_obj_arg s, b_obj_arg i); -obj_res string_utf8_set(obj_arg s, b_obj_arg i, uint32 c); -inline uint8 string_utf8_at_end(b_obj_arg s, b_obj_arg i) { return !is_scalar(i) || unbox(i) >= string_size(s) - 1; } -obj_res string_utf8_extract(b_obj_arg s, b_obj_arg b, b_obj_arg e); -inline obj_res string_utf8_byte_size(b_obj_arg s) { return usize_to_nat(string_size(s) - 1); } - -inline bool string_eq(b_obj_arg s1, b_obj_arg s2) { return s1 == s2 || (string_size(s1) == string_size(s2) && std::memcmp(string_cstr(s1), string_cstr(s2), string_size(s1)) == 0); } -bool string_eq(b_obj_arg s1, char const * s2); -inline bool string_ne(b_obj_arg s1, b_obj_arg s2) { return !string_eq(s1, s2); } -bool string_lt(b_obj_arg s1, b_obj_arg s2); -inline uint8 string_dec_eq(b_obj_arg s1, b_obj_arg s2) { return string_eq(s1, s2); } -inline uint8 string_dec_lt(b_obj_arg s1, b_obj_arg s2) { return string_lt(s1, s2); } -usize string_hash(b_obj_arg); +inline obj_res usize_to_nat(usize n) { return lean_usize_to_nat(n); } +inline obj_res mk_nat_obj(unsigned n) { return lean_unsigned_to_nat(n); } +inline obj_res uint64_to_nat(uint64 n) { return lean_uint64_to_nat(n); } +inline obj_res nat_succ(b_obj_arg a) { return lean_nat_succ(a); } +inline obj_res nat_add(b_obj_arg a1, b_obj_arg a2) { return lean_nat_add(a1, a2); } +inline obj_res nat_sub(b_obj_arg a1, b_obj_arg a2) { return lean_nat_sub(a1, a2); } +inline obj_res nat_mul(b_obj_arg a1, b_obj_arg a2) { return lean_nat_mul(a1, a2); } +inline obj_res nat_div(b_obj_arg a1, b_obj_arg a2) { return lean_nat_div(a1, a2); } +inline obj_res nat_mod(b_obj_arg a1, b_obj_arg a2) { return lean_nat_mod(a1, a2); } +inline bool nat_eq(b_obj_arg a1, b_obj_arg a2) { return lean_nat_eq(a1, a2); } +inline uint8 nat_dec_eq(b_obj_arg a1, b_obj_arg a2) { return lean_nat_dec_eq(a1, a2); } +inline bool nat_ne(b_obj_arg a1, b_obj_arg a2) { return lean_nat_ne(a1, a2); } +inline bool nat_le(b_obj_arg a1, b_obj_arg a2) { return lean_nat_le(a1, a2); } +inline uint8 nat_dec_le(b_obj_arg a1, b_obj_arg a2) { return lean_nat_dec_le(a1, a2); } +inline bool nat_lt(b_obj_arg a1, b_obj_arg a2) { return lean_nat_lt(a1, a2); } +inline uint8 nat_dec_lt(b_obj_arg a1, b_obj_arg a2) { return lean_nat_dec_lt(a1, a2); } +inline obj_res nat_land(b_obj_arg a1, b_obj_arg a2) { return lean_nat_land(a1, a2); } +inline obj_res nat_lor(b_obj_arg a1, b_obj_arg a2) { return lean_nat_lor(a1, a2); } +inline obj_res nat_lxor(b_obj_arg a1, b_obj_arg a2) { return lean_nat_lxor(a1, a2); } // ======================================= -// ByteArray - -obj_res byte_array_mk(obj_arg a); -obj_res byte_array_data(obj_arg a); -obj_res copy_byte_array(obj_arg a); - -inline obj_res mk_empty_byte_array(b_obj_arg capacity) { - if (!is_scalar(capacity)) throw std::bad_alloc(); // we will run out of memory - usize cap = unbox(capacity); - return alloc_sarray(1, 0, cap); +// Integers +object * mk_int_obj_core(mpz const & m); +inline obj_res mk_int_obj(mpz const & m) { + if (m < LEAN_MIN_SMALL_INT || m > LEAN_MAX_SMALL_INT) + return mk_int_obj_core(m); + else + return box(static_cast(m.get_int())); } +inline obj_res mk_int_obj(int n) { return lean_int_to_int(n); } +inline obj_res mk_int_obj(int64 n) { return lean_int64_to_int(n); } +inline obj_res nat2int(obj_arg a) { return lean_nat_to_int(a); } +inline obj_res int_neg(b_obj_arg a) { return lean_int_neg(a); } +inline obj_res int_neg_succ_of_nat(obj_arg a) { return lean_int_neg_succ_of_nat(a); } +inline obj_res int_add(b_obj_arg a1, b_obj_arg a2) { return lean_int_add(a1, a2); } +inline obj_res int_sub(b_obj_arg a1, b_obj_arg a2) { return lean_int_sub(a1, a2); } +inline obj_res int_mul(b_obj_arg a1, b_obj_arg a2) { return lean_int_mul(a1, a2); } +inline obj_res int_div(b_obj_arg a1, b_obj_arg a2) { return lean_int_div(a1, a2); } +inline obj_res int_mod(b_obj_arg a1, b_obj_arg a2) { return lean_int_mod(a1, a2); } +inline bool int_eq(b_obj_arg a1, b_obj_arg a2) { return lean_int_eq(a1, a2); } +inline bool int_ne(b_obj_arg a1, b_obj_arg a2) { return lean_int_ne(a1, a2); } +inline bool int_le(b_obj_arg a1, b_obj_arg a2) { return lean_int_le(a1, a2); } +inline bool int_lt(b_obj_arg a1, b_obj_arg a2) { return lean_int_lt(a1, a2); } +inline obj_res nat_abs(b_obj_arg i) { return lean_nat_abs(i); } +inline uint8 int_dec_eq(b_obj_arg a1, b_obj_arg a2) { return lean_int_dec_eq(a1, a2); } +inline uint8 int_dec_le(b_obj_arg a1, b_obj_arg a2) { return lean_int_dec_le(a1, a2); } +inline uint8 int_dec_lt(b_obj_arg a1, b_obj_arg a2) { return lean_int_dec_lt(a1, a2); } +inline uint8 int_dec_nonneg(b_obj_arg a) { return lean_int_dec_nonneg(a); } -inline obj_res byte_array_size(b_obj_arg a) { - return box(sarray_size(a)); -} +// ======================================= +// Boxing -inline uint8 byte_array_get(b_obj_arg a, b_obj_arg i) { - if (is_scalar(i)) { - usize idx = unbox(i); - return idx < sarray_size(a) ? sarray_get(a, idx) : 0; - } else { - /* The index must be out of bounds. Otherwise we would be out of memory. */ - return 0; - } -} - -obj_res byte_array_push(obj_arg a, uint8 b); - -inline obj_res byte_array_set(obj_arg a, b_obj_arg i, uint8 b) { - if (!is_scalar(i)) return a; - usize idx = unbox(i); - if (idx >= sarray_size(a)) return a; - obj_res r; - if (is_exclusive(a)) r = a; - else r = copy_byte_array(a); - uint8 * it = sarray_cptr(r) + idx; - *it = b; - return r; -} +inline obj_res box_uint32(unsigned v) { return lean_box_uint32(v); } +inline unsigned unbox_uint32(b_obj_arg o) { return lean_unbox_uint32(o); } +inline obj_res box_uint64(unsigned long long v) { return lean_box_uint64(v); } +inline unsigned long long unbox_uint64(b_obj_arg o) { return lean_unbox_uint64(o); } +inline obj_res box_size_t(size_t v) { return lean_box_usize(v); } +inline size_t unbox_size_t(b_obj_arg o) { return lean_unbox_usize(o); } // ======================================= // uint8 -uint8 uint8_of_big_nat(b_obj_arg a); -inline uint8 uint8_of_nat(b_obj_arg a) { return is_scalar(a) ? static_cast(unbox(a)) : uint8_of_big_nat(a); } -inline obj_res uint8_to_nat(uint8 a) { return usize_to_nat(static_cast(a)); } -inline uint8 uint8_add(uint8 a1, uint8 a2) { return a1+a2; } -inline uint8 uint8_sub(uint8 a1, uint8 a2) { return a1-a2; } -inline uint8 uint8_mul(uint8 a1, uint8 a2) { return a1*a2; } -inline uint8 uint8_div(uint8 a1, uint8 a2) { return a2 == 0 ? 0 : a1/a2; } -inline uint8 uint8_mod(uint8 a1, uint8 a2) { return a2 == 0 ? 0 : a1%a2; } -inline uint8 uint8_modn(uint8 a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a2))) { - unsigned n2 = unbox(a2); - return n2 == 0 ? 0 : a1 % n2; - } else { - return a1; - } -} -inline uint8 uint8_dec_eq(uint8 a1, uint8 a2) { return a1 == a2; } -inline uint8 uint8_dec_lt(uint8 a1, uint8 a2) { return a1 < a2; } -inline uint8 uint8_dec_le(uint8 a1, uint8 a2) { return a1 <= a2; } +inline uint8 uint8_of_nat(b_obj_arg a) { return lean_uint8_of_nat(a); } +inline obj_res uint8_to_nat(uint8 a) { return lean_uint8_to_nat(a); } +inline uint8 uint8_add(uint8 a1, uint8 a2) { return lean_uint8_add(a1, a2); } +inline uint8 uint8_sub(uint8 a1, uint8 a2) { return lean_uint8_sub(a1, a2); } +inline uint8 uint8_mul(uint8 a1, uint8 a2) { return lean_uint8_mul(a1, a2); } +inline uint8 uint8_div(uint8 a1, uint8 a2) { return lean_uint8_div(a1, a2); } +inline uint8 uint8_mod(uint8 a1, uint8 a2) { return lean_uint8_mod(a1, a2); } +inline uint8 uint8_modn(uint8 a1, b_obj_arg a2) { return lean_uint8_modn(a1, a2); } +inline uint8 uint8_dec_eq(uint8 a1, uint8 a2) { return lean_uint8_dec_eq(a1, a2); } +inline uint8 uint8_dec_lt(uint8 a1, uint8 a2) { return lean_uint8_dec_lt(a1, a2); } +inline uint8 uint8_dec_le(uint8 a1, uint8 a2) { return lean_uint8_dec_le(a1, a2); } // ======================================= // uint16 -uint16 uint16_of_big_nat(b_obj_arg a); -inline uint16 uint16_of_nat(b_obj_arg a) { return is_scalar(a) ? static_cast(unbox(a)) : uint16_of_big_nat(a); } -inline obj_res uint16_to_nat(uint16 a) { return usize_to_nat(static_cast(a)); } -inline uint16 uint16_add(uint16 a1, uint16 a2) { return a1+a2; } -inline uint16 uint16_sub(uint16 a1, uint16 a2) { return a1-a2; } -inline uint16 uint16_mul(uint16 a1, uint16 a2) { return a1*a2; } -inline uint16 uint16_div(uint16 a1, uint16 a2) { return a2 == 0 ? 0 : a1/a2; } -inline uint16 uint16_mod(uint16 a1, uint16 a2) { return a2 == 0 ? 0 : a1%a2; } -inline uint16 uint16_modn(uint16 a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a2))) { - unsigned n2 = unbox(a2); - return n2 == 0 ? 0 : a1 % n2; - } else { - return a1; - } -} -inline uint8 uint16_dec_eq(uint16 a1, uint16 a2) { return a1 == a2; } -inline uint8 uint16_dec_lt(uint16 a1, uint16 a2) { return a1 < a2; } -inline uint8 uint16_dec_le(uint16 a1, uint16 a2) { return a1 <= a2; } +inline uint16 uint16_of_nat(b_obj_arg a) { return lean_uint16_of_nat(a); } +inline obj_res uint16_to_nat(uint16 a) { return lean_uint16_to_nat(a); } +inline uint16 uint16_add(uint16 a1, uint16 a2) { return lean_uint16_add(a1, a2); } +inline uint16 uint16_sub(uint16 a1, uint16 a2) { return lean_uint16_sub(a1, a2); } +inline uint16 uint16_mul(uint16 a1, uint16 a2) { return lean_uint16_mul(a1, a2); } +inline uint16 uint16_div(uint16 a1, uint16 a2) { return lean_uint16_div(a1, a2); } +inline uint16 uint16_mod(uint16 a1, uint16 a2) { return lean_uint16_mod(a1, a2); } +inline uint16 uint16_modn(uint16 a1, b_obj_arg a2) { return lean_uint16_modn(a1, a2); } +inline uint16 uint16_dec_eq(uint16 a1, uint16 a2) { return lean_uint16_dec_eq(a1, a2); } +inline uint16 uint16_dec_lt(uint16 a1, uint16 a2) { return lean_uint16_dec_lt(a1, a2); } +inline uint16 uint16_dec_le(uint16 a1, uint16 a2) { return lean_uint16_dec_le(a1, a2); } // ======================================= // uint32 -uint32 uint32_of_big_nat(b_obj_arg a); -inline uint32 uint32_of_nat(b_obj_arg a) { return is_scalar(a) ? static_cast(unbox(a)) : uint32_of_big_nat(a); } -inline obj_res uint32_to_nat(uint32 a) { return usize_to_nat(static_cast(a)); } -inline uint32 uint32_add(uint32 a1, uint32 a2) { return a1+a2; } -inline uint32 uint32_sub(uint32 a1, uint32 a2) { return a1-a2; } -inline uint32 uint32_mul(uint32 a1, uint32 a2) { return a1*a2; } -inline uint32 uint32_div(uint32 a1, uint32 a2) { return a2 == 0 ? 0 : a1/a2; } -inline uint32 uint32_mod(uint32 a1, uint32 a2) { return a2 == 0 ? 0 : a1%a2; } -inline uint32 uint32_modn(uint32 a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a2))) { - usize n2 = unbox(a2); - return n2 == 0 ? 0 : a1 % n2; - } else if (sizeof(void*) == 4) { - // 32-bit - mpz const & m = mpz_value(a2); - return m.is_unsigned_int() ? a1 % m.get_unsigned_int() : a1; - } else { - // 64-bit - return a1; - } -} -inline uint8 uint32_dec_eq(uint32 a1, uint32 a2) { return a1 == a2; } -inline uint8 uint32_dec_lt(uint32 a1, uint32 a2) { return a1 < a2; } -inline uint8 uint32_dec_le(uint32 a1, uint32 a2) { return a1 <= a2; } +inline uint32 uint32_of_nat(b_obj_arg a) { return lean_uint32_of_nat(a); } +inline obj_res uint32_to_nat(uint32 a) { return lean_uint32_to_nat(a); } +inline uint32 uint32_add(uint32 a1, uint32 a2) { return lean_uint32_add(a1, a2); } +inline uint32 uint32_sub(uint32 a1, uint32 a2) { return lean_uint32_sub(a1, a2); } +inline uint32 uint32_mul(uint32 a1, uint32 a2) { return lean_uint32_mul(a1, a2); } +inline uint32 uint32_div(uint32 a1, uint32 a2) { return lean_uint32_div(a1, a2); } +inline uint32 uint32_mod(uint32 a1, uint32 a2) { return lean_uint32_mod(a1, a2); } +inline uint32 uint32_modn(uint32 a1, b_obj_arg a2) { return lean_uint32_modn(a1, a2); } +inline uint32 uint32_dec_eq(uint32 a1, uint32 a2) { return lean_uint32_dec_eq(a1, a2); } +inline uint32 uint32_dec_lt(uint32 a1, uint32 a2) { return lean_uint32_dec_lt(a1, a2); } +inline uint32 uint32_dec_le(uint32 a1, uint32 a2) { return lean_uint32_dec_le(a1, a2); } // ======================================= // uint64 -uint64 uint64_of_big_nat(b_obj_arg a); -inline uint64 uint64_of_nat(b_obj_arg a) { return is_scalar(a) ? static_cast(unbox(a)) : uint64_of_big_nat(a); } -obj_res uint64_to_nat(uint64 a); -inline uint64 uint64_add(uint64 a1, uint64 a2) { return a1+a2; } -inline uint64 uint64_sub(uint64 a1, uint64 a2) { return a1-a2; } -inline uint64 uint64_mul(uint64 a1, uint64 a2) { return a1*a2; } -inline uint64 uint64_div(uint64 a1, uint64 a2) { return a2 == 0 ? 0 : a1/a2; } -inline uint64 uint64_mod(uint64 a1, uint64 a2) { return a2 == 0 ? 0 : a1%a2; } -inline uint64 uint64_modn(uint64 a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a2))) { - usize n2 = unbox(a2); - return n2 == 0 ? 0 : a1 % n2; - } else { - // TODO(Leo) - return a1; - } -} -inline uint8 uint64_dec_eq(uint64 a1, uint64 a2) { return a1 == a2; } -inline uint8 uint64_dec_lt(uint64 a1, uint64 a2) { return a1 < a2; } -inline uint8 uint64_dec_le(uint64 a1, uint64 a2) { return a1 <= a2; } +inline uint64 uint64_of_nat(b_obj_arg a) { return lean_uint64_of_nat(a); } +inline uint64 uint64_add(uint64 a1, uint64 a2) { return lean_uint64_add(a1, a2); } +inline uint64 uint64_sub(uint64 a1, uint64 a2) { return lean_uint64_sub(a1, a2); } +inline uint64 uint64_mul(uint64 a1, uint64 a2) { return lean_uint64_mul(a1, a2); } +inline uint64 uint64_div(uint64 a1, uint64 a2) { return lean_uint64_div(a1, a2); } +inline uint64 uint64_mod(uint64 a1, uint64 a2) { return lean_uint64_mod(a1, a2); } +inline uint64 uint64_modn(uint64 a1, b_obj_arg a2) { return lean_uint64_modn(a1, a2); } +inline uint64 uint64_dec_eq(uint64 a1, uint64 a2) { return lean_uint64_dec_eq(a1, a2); } +inline uint64 uint64_dec_lt(uint64 a1, uint64 a2) { return lean_uint64_dec_lt(a1, a2); } +inline uint64 uint64_dec_le(uint64 a1, uint64 a2) { return lean_uint64_dec_le(a1, a2); } // ======================================= // usize -usize usize_of_big_nat(b_obj_arg a); -inline usize usize_of_nat(b_obj_arg a) { return is_scalar(a) ? unbox(a) : usize_of_big_nat(a); } -inline usize usize_add(usize a1, usize a2) { return a1+a2; } -inline usize usize_sub(usize a1, usize a2) { return a1-a2; } -inline usize usize_mul(usize a1, usize a2) { return a1*a2; } -inline usize usize_div(usize a1, usize a2) { return a2 == 0 ? 0 : a1/a2; } -inline usize usize_mod(usize a1, usize a2) { return a2 == 0 ? 0 : a1%a2; } -inline usize usize_modn(usize a1, b_obj_arg a2) { - if (LEAN_LIKELY(is_scalar(a2))) { - usize n2 = unbox(a2); - return n2 == 0 ? 0 : a1 % n2; - } else { - // TODO(Leo) - return a1; - } -} -inline uint8 usize_dec_eq(usize a1, usize a2) { return a1 == a2; } -inline uint8 usize_dec_lt(usize a1, usize a2) { return a1 < a2; } -inline uint8 usize_dec_le(usize a1, usize a2) { return a1 <= a2; } -usize usize_mix_hash(usize a1, usize a2); -// ======================================= -// array functions for generated code -inline object * array_sz(obj_arg a) { - object * r = box(array_size(a)); - dec(a); - return r; -} - -inline object * array_get_size(b_obj_arg a) { - return box(array_size(a)); -} - -inline object * mk_empty_array() { - return alloc_array(0, 0); -} - -inline object * mk_empty_array(b_obj_arg capacity) { - if (!is_scalar(capacity)) throw std::bad_alloc(); // we will run out of memory - usize cap = unbox(capacity); - return alloc_array(0, cap); -} - -inline object * array_uget(b_obj_arg a, usize i) { - object * r = array_get(a, i); inc(r); - return r; -} - -inline obj_res array_fget(b_obj_arg a, b_obj_arg i) { - return array_uget(a, unbox(i)); -} - -inline object * array_get(obj_arg def_val, b_obj_arg a, b_obj_arg i) { - if (is_scalar(i)) { - usize idx = unbox(i); - if (idx < array_size(a)) { - dec(def_val); - return array_uget(a, idx); - } else { - return def_val; - } - } else { - /* The index must be out of bounds because - i > LEAN_MAX_SMALL_NAT == std::numeric_limits::max() >> 1, - but each array entry is 8 bytes in 64-bit machines and 4 in 32-bit ones. - In both cases, we would be out-of-memory. */ - return def_val; - } -} - -obj_res copy_array(obj_arg a, bool expand = false); - -inline obj_res ensure_exclusive_array(obj_arg a) { - if (is_exclusive(a)) return a; - return copy_array(a); -} - -inline object * array_uset(obj_arg a, usize i, obj_arg v) { - object * r = ensure_exclusive_array(a); - object ** it = array_cptr(r) + i; - dec(*it); - *it = v; - return r; -} - -inline object * array_fset(obj_arg a, b_obj_arg i, obj_arg v) { - return array_uset(a, unbox(i), v); -} - -inline object * array_set(obj_arg a, b_obj_arg i, obj_arg v) { - if (is_scalar(i)) { - usize idx = unbox(i); - if (idx < array_size(a)) - return array_uset(a, idx, v); - } - dec(v); - return a; -} - -inline object * array_pop(obj_arg a) { - object * r = ensure_exclusive_array(a); - size_t & sz = to_array(r)->m_size; - if (sz == 0) return r; - sz--; - object ** last = array_cptr(r) + sz; - dec(*last); - return r; -} - -inline object * array_uswap(obj_arg a, usize i, usize j) { - object * r = ensure_exclusive_array(a); - object ** it = array_cptr(r); - object * v1 = it[i]; - it[i] = it[j]; - it[j] = v1; - return r; -} - -inline object * array_fswap(obj_arg a, b_obj_arg i, b_obj_arg j) { - return array_uswap(a, unbox(i), unbox(j)); -} - -inline object * array_swap(obj_arg a, b_obj_arg i, b_obj_arg j) { - if (!is_scalar(i) || !is_scalar(j)) return a; - usize ui = unbox(i); - usize uj = unbox(j); - usize sz = to_array(a)->m_size; - if (ui >= sz || uj >= sz) return a; - return array_uswap(a, ui, uj); -} - -object * array_push(obj_arg a, obj_arg v); -object * mk_array(obj_arg n, obj_arg v); +inline usize usize_of_nat(b_obj_arg a) { return lean_usize_of_nat(a); } +inline usize usize_add(usize a1, usize a2) { return lean_usize_add(a1, a2); } +inline usize usize_sub(usize a1, usize a2) { return lean_usize_sub(a1, a2); } +inline usize usize_mul(usize a1, usize a2) { return lean_usize_mul(a1, a2); } +inline usize usize_div(usize a1, usize a2) { return lean_usize_div(a1, a2); } +inline usize usize_mod(usize a1, usize a2) { return lean_usize_mod(a1, a2); } +inline usize usize_modn(usize a1, b_obj_arg a2) { return lean_usize_modn(a1, a2); } +inline usize usize_dec_eq(usize a1, usize a2) { return lean_usize_dec_eq(a1, a2); } +inline usize usize_dec_lt(usize a1, usize a2) { return lean_usize_dec_lt(a1, a2); } +inline usize usize_dec_le(usize a1, usize a2) { return lean_usize_dec_le(a1, a2); } +inline usize usize_mix_hash(usize a1, usize a2) { return lean_usize_mix_hash(a1, a2); } // ======================================= // debugging helper functions -object * dbg_trace(obj_arg s, obj_arg fn); -object * dbg_sleep(uint32 ms, obj_arg fn); -object * dbg_trace_if_shared(obj_arg s, obj_arg a); +inline object * dbg_trace(obj_arg s, obj_arg fn) { return lean_dbg_trace(s, fn); } +inline object * dbg_sleep(uint32 ms, obj_arg fn) { return lean_dbg_sleep(ms, fn); } +inline object * dbg_trace_if_shared(obj_arg s, obj_arg a) { return lean_dbg_trace_if_shared(s, a); } // ======================================= // IO helper functions -inline obj_res io_mk_world() { - object * r = alloc_cnstr(0, 2, 0); - cnstr_set(r, 0, box(0)); - cnstr_set(r, 1, box(0)); - return r; -} -inline bool io_result_is_ok(b_obj_arg r) { return cnstr_tag(r) == 0; } -inline bool io_result_is_error(b_obj_arg r) { return cnstr_tag(r) == 1; } -inline b_obj_res io_result_get_value(b_obj_arg r) { lean_assert(io_result_is_ok(r)); return cnstr_get(r, 0); } -inline b_obj_res io_result_get_error(b_obj_arg r) { lean_assert(io_result_is_error(r)); return cnstr_get(r, 0); } -void io_result_show_error(b_obj_arg r); -void io_mark_end_initialization(); +inline obj_res io_mk_world() { return lean_io_mk_world(); } +inline bool io_result_is_ok(b_obj_arg r) { return lean_io_result_is_ok(r); } +inline bool io_result_is_error(b_obj_arg r) { return lean_io_result_is_error(r); } +inline b_obj_res io_result_get_value(b_obj_arg r) { return lean_io_result_get_value(r); } +inline b_obj_res io_result_get_error(b_obj_arg r) { return lean_io_result_get_error(r); } +inline void io_result_show_error(b_obj_arg r) { return lean_io_result_show_error(r); } +inline void io_mark_end_initialization() { return lean_io_mark_end_initialization(); } // ======================================= // IO ref primitives -obj_res io_mk_ref(obj_arg, obj_arg); -obj_res io_ref_get(b_obj_arg, obj_arg); -obj_res io_ref_set(b_obj_arg, obj_arg, obj_arg); -obj_res io_ref_reset(b_obj_arg, obj_arg); -obj_res io_ref_swap(b_obj_arg, obj_arg, obj_arg); +inline obj_res io_mk_ref(obj_arg v, obj_arg w) { return lean_io_mk_ref(v, w); } +obj_res io_ref_get(b_obj_arg r, obj_arg w) { return lean_io_ref_get(r, w); } +obj_res io_ref_set(b_obj_arg r, obj_arg v, obj_arg w) { return lean_io_ref_set(r, v, w); } +obj_res io_ref_reset(b_obj_arg r, obj_arg w) { return lean_io_ref_reset(r, w); } +obj_res io_ref_swap(b_obj_arg r, obj_arg v, obj_arg w) { return lean_io_ref_swap(r, v, w); } // ======================================= // Module initialization/finalization diff --git a/src/runtime/optional.h b/src/runtime/optional.h index 6b096511c8..1846da2b80 100644 --- a/src/runtime/optional.h +++ b/src/runtime/optional.h @@ -5,6 +5,7 @@ Released under Apache 2.0 license as described in the file LICENSE. Author: Leonardo de Moura */ #pragma once +#include "runtime/lean.h" #include "runtime/debug.h" namespace lean { diff --git a/src/runtime/serializer.cpp b/src/runtime/serializer.cpp index b160334b11..224df41d04 100644 --- a/src/runtime/serializer.cpp +++ b/src/runtime/serializer.cpp @@ -75,7 +75,10 @@ void serializer::write_blob(std::string const & s) { void serializer::write_constructor(object * o) { lean_assert(is_cnstr(o)); unsigned num_objs = cnstr_num_objs(o); - unsigned scalar_sz = cnstr_scalar_size(o); + unsigned obj_size = lean_object_byte_size(o); + unsigned main_size = sizeof(lean_ctor_object) + sizeof(lean_object*)*num_objs; + lean_assert(obj_size >= main_size); + unsigned scalar_sz = obj_size - main_size; write_unsigned_short(cnstr_tag(o)); write_unsigned_short(num_objs); write_unsigned_short(scalar_sz); @@ -127,8 +130,8 @@ void serializer::write_scalar_array(object * o) { write_unsigned(esz); write_size_t(sz); size_t num_bytes = sz*esz; - unsigned char const * it = sarray_cptr_core(o); - unsigned char const * end = it + num_bytes; + uint8 const * it = sarray_cptr(o); + uint8 const * end = it + num_bytes; for (; it != end; ++it) m_out.put(*it); } @@ -167,19 +170,19 @@ void serializer::write_object(object * o) { m_out.put(1); write_unsigned(it->second); } else { - object_kind k = get_kind(o); + uint8 k = lean_ptr_tag(o); m_out.put(static_cast(k) + 2); switch (k) { - case object_kind::Constructor: write_constructor(o); break; - case object_kind::Closure: write_closure(o); break; - case object_kind::Task: write_task(o); break; - case object_kind::Thunk: write_thunk(o); break; - case object_kind::Array: write_array(o); break; - case object_kind::ScalarArray: write_scalar_array(o); break; - case object_kind::String: write_string_object(o); break; - case object_kind::MPZ: write_mpz(mpz_value(o)); break; - case object_kind::External: write_external(o); break; - default: lean_unreachable(); + case LeanClosure: write_closure(o); break; + case LeanTask: write_task(o); break; + case LeanThunk: write_thunk(o); break; + case LeanArray: write_array(o); break; + case LeanScalarArray: write_scalar_array(o); break; + case LeanString: write_string_object(o); break; + case LeanMPZ: write_mpz(mpz_value(o)); break; + case LeanExternal: write_external(o); break; + case LeanReserved: lean_unreachable(); break; + default: write_constructor(o); break; } inc_ref(o); m_obj_table.insert(std::make_pair(o, m_obj_table.size())); @@ -320,11 +323,11 @@ object * deserializer::read_array() { } object * deserializer::read_scalar_array() { - unsigned esz = read_unsigned(); - size_t sz = read_size_t(); - object * r = alloc_sarray(esz, sz, sz); - unsigned char * it = sarray_cptr_core(r); - unsigned char * end = it + sz*esz; + unsigned esz = read_unsigned(); + size_t sz = read_size_t(); + object * r = alloc_sarray(esz, sz, sz); + uint8 * it = sarray_cptr(r); + uint8 * end = it + sz*esz; for (; it != end; ++it) *it = m_in.get(); return r; @@ -355,19 +358,23 @@ object * deserializer::read_object() { throw corrupted_stream_exception(); return m_objs[i]; } else { - object_kind k = static_cast(c - 2); object * r; - switch (k) { - case object_kind::Constructor: r = read_constructor(); break; - case object_kind::Closure: r = read_closure(); break; - case object_kind::Thunk: r = read_thunk(); break; - case object_kind::Task: r = read_task(); break; - case object_kind::Array: r = read_array(); break; - case object_kind::ScalarArray: r = read_scalar_array(); break; - case object_kind::String: r = read_string_object(); break; - case object_kind::MPZ: r = alloc_mpz(read_mpz()); break; - case object_kind::External: r = read_external(); break; - default: throw corrupted_stream_exception(); + switch (c - 2) { + case LeanClosure: r = read_closure(); break; + case LeanThunk: r = read_thunk(); break; + case LeanTask: r = read_task(); break; + case LeanArray: r = read_array(); break; + case LeanScalarArray: r = read_scalar_array(); break; + case LeanString: r = read_string_object(); break; + case LeanMPZ: r = alloc_mpz(read_mpz()); break; + case LeanExternal: r = read_external(); break; + case LeanReserved: throw corrupted_stream_exception(); + default: + if (c - 2 < LeanMaxCtorTag) + r = read_constructor(); + else + throw corrupted_stream_exception(); + break; } m_objs.push_back(r); return r; diff --git a/src/runtime/serializer.h b/src/runtime/serializer.h index 30ed2cc1a0..c6c25a27f0 100644 --- a/src/runtime/serializer.h +++ b/src/runtime/serializer.h @@ -12,12 +12,10 @@ Author: Leonardo de Moura #include #include #include -#include "runtime/int64.h" +#include "runtime/object.h" #include "runtime/optional.h" namespace lean { -struct object; -class mpz; class serializer { std::ostream & m_out; std::unordered_map, std::equal_to> m_obj_table; @@ -138,4 +136,7 @@ deserializer & operator>>(deserializer & d, optional & a) { a = read_optional(d); return d; } + +inline serializer & operator<<(serializer & s, mpz const & n) { s.write_mpz(n); return s; } +inline deserializer & operator>>(deserializer & d, mpz & n) { n = d.read_mpz(); return d; } } diff --git a/src/tests/util/CMakeLists.txt b/src/tests/util/CMakeLists.txt index a3a01891db..44c5e6c6a3 100644 --- a/src/tests/util/CMakeLists.txt +++ b/src/tests/util/CMakeLists.txt @@ -4,48 +4,7 @@ add_exec_test(object "object") add_executable(compact compact.cpp $ $) target_link_libraries(compact ${EXTRA_LIBS}) add_exec_test(compact "compact") -add_executable(name name.cpp $ $) -target_link_libraries(name ${EXTRA_LIBS}) -add_exec_test(name "name") -add_executable(nat nat.cpp $ $) -target_link_libraries(nat ${EXTRA_LIBS}) -add_exec_test(nat "nat") -add_executable(buffer buffer.cpp $ $) -target_link_libraries(buffer ${EXTRA_LIBS}) -add_exec_test(buffer "buffer") -add_executable(list list.cpp $ $) -target_link_libraries(list ${EXTRA_LIBS}) -add_exec_test(list "list") -add_executable(rb_tree rb_tree.cpp $ $) -target_link_libraries(rb_tree ${EXTRA_LIBS}) -add_exec_test(rb_tree "rb_tree") -add_executable(rb_map rb_map.cpp $ $) -target_link_libraries(rb_map ${EXTRA_LIBS}) -add_exec_test(rb_map "rb_map") -add_executable(bit_tricks bit_tricks.cpp $ $) -target_link_libraries(bit_tricks ${EXTRA_LIBS}) -add_exec_test(bit_tricks "bit_tricks") -add_executable(hash hash.cpp $ $) -target_link_libraries(hash ${EXTRA_LIBS}) -add_exec_test(hash "hash") -add_executable(set set.cpp $ $) -target_link_libraries(set ${EXTRA_LIBS}) -add_exec_test(set "set") -add_executable(optional optional.cpp $ $) -target_link_libraries(optional ${EXTRA_LIBS}) -add_exec_test(optional "optional") -add_executable(stackinfo stackinfo.cpp $ $) -target_link_libraries(stackinfo ${EXTRA_LIBS}) -add_exec_test(stackinfo "stackinfo") -add_executable(serializer serializer.cpp $ $) -target_link_libraries(serializer ${EXTRA_LIBS}) -add_exec_test(serializer "serializer") -add_executable(trie trie.cpp $ $) -target_link_libraries(trie ${EXTRA_LIBS}) -add_exec_test(trie "trie") -add_executable(thread thread.cpp $ $) -target_link_libraries(thread ${EXTRA_LIBS}) -add_exec_test(thread "thread") -add_executable(bitap_fuzzy_search bitap_fuzzy_search.cpp $ $) -target_link_libraries(bitap_fuzzy_search ${EXTRA_LIBS}) -add_exec_test(bitap_fuzzy_search "bitap_fuzzy_search") +add_executable(testruntime runtime.cpp ${LEAN_SOURCE_DIR}/runtime/object.cpp ${LEAN_SOURCE_DIR}/runtime/alloc.cpp ${LEAN_SOURCE_DIR}/runtime/thread.cpp +${LEAN_SOURCE_DIR}/runtime/exception.cpp ${LEAN_SOURCE_DIR}/runtime/interrupt.cpp ${LEAN_SOURCE_DIR}/runtime/stackinfo.cpp ${LEAN_SOURCE_DIR}/runtime/memory.cpp ${LEAN_SOURCE_DIR}/runtime/debug.cpp ${LEAN_SOURCE_DIR}/runtime/apply.cpp) +target_link_libraries(testruntime ${EXTRA_LIBS}) +add_exec_test(testruntime "runtime") diff --git a/src/tests/util/object.cpp b/src/tests/util/object.cpp index 51465e8771..4f935d501a 100644 --- a/src/tests/util/object.cpp +++ b/src/tests/util/object.cpp @@ -11,6 +11,7 @@ Author: Leonardo de Moura #include "util/test.h" // <<< comment this list for performance experiments #include "util/timeit.h" #include "runtime/stackinfo.h" +#include "runtime/thread.h" #include "runtime/serializer.h" #include "runtime/sstream.h" #include "util/object_ref.h" @@ -433,7 +434,7 @@ bool contains(object * l, object * v) { object * h = cnstr_get(l, 0); object * t = cnstr_get(l, 1); if (!is_shared(l)) { - free_heap_obj(l); + lean_free_object(l); } else { inc(h); inc(t); @@ -482,7 +483,7 @@ bool contains_hybrid(object * l, object * v) { if (l_b) { t = mark_borrowed(t_obj); } else if (!is_shared(l_obj)) { - free_heap_obj(l_obj); + lean_free_object(l_obj); t = t_obj; } else { inc(h_obj); @@ -511,7 +512,7 @@ bool contains_fast_hybrid(object * l, object * v) { object * t = cnstr_get(l, 1); bool shared = is_shared(l); if (!shared) { - free_heap_obj(l); + lean_free_object(l); } else { inc(h); inc(t); diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 742bcb2b86..77936a42ba 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -1,5 +1,5 @@ add_library(util OBJECT object_ref.cpp name.cpp name_set.cpp fresh_name.cpp - escaped.cpp bit_tricks.cpp ascii.cpp shared_mutex.cpp + escaped.cpp bit_tricks.cpp ascii.cpp path.cpp lean_path.cpp lbool.cpp bitap_fuzzy_search.cpp init_module.cpp list_fn.cpp file_lock.cpp timeit.cpp timer.cpp parser_exception.cpp name_generator.cpp kvmap.cpp map_foreach.cpp diff --git a/src/util/object_ref.h b/src/util/object_ref.h index b033634bc3..684a2be948 100644 --- a/src/util/object_ref.h +++ b/src/util/object_ref.h @@ -9,6 +9,7 @@ Author: Leonardo de Moura #include #include "runtime/object.h" #include "runtime/optional.h" +#include "runtime/serializer.h" namespace lean { /* Smart point for Lean objects. It is useful for writing C++ code that manipulates Lean objects. */ @@ -17,7 +18,7 @@ protected: object * m_obj; public: object_ref():m_obj(box(0)) {} - explicit object_ref(obj_arg o):m_obj(o) { lean_assert(is_scalar(o) || !is_heap_obj(o) || get_rc(o) > 0); } + explicit object_ref(obj_arg o):m_obj(o) { lean_assert(is_scalar(o) || !is_heap_obj(o) || lean_nonzero_rc(o)); } object_ref(b_obj_arg o, bool):m_obj(o) { inc(o); } object_ref(object_ref const & s):m_obj(s.m_obj) { inc(m_obj); } object_ref(object_ref && s):m_obj(s.m_obj) { s.m_obj = box(0); } @@ -128,7 +129,7 @@ inline object_ref mk_cnstr(unsigned tag, object_ref const & o1, object_ref const inline object_ref const & cnstr_get_ref(object * o, unsigned i) { static_assert(sizeof(object_ref) == sizeof(object *), "unexpected object_ref size"); // NOLINT lean_assert(is_cnstr(o)); - return reinterpret_cast(reinterpret_cast(o) + sizeof(constructor_object))[i]; + return reinterpret_cast(lean_to_ctor(o)->m_objs)[i]; } inline object_ref const & cnstr_get_ref(object_ref const & ref, unsigned i) { diff --git a/src/util/option_declarations.cpp b/src/util/option_declarations.cpp index de2a5d0588..4ab0c5c309 100644 --- a/src/util/option_declarations.cpp +++ b/src/util/option_declarations.cpp @@ -4,7 +4,6 @@ Released under Apache 2.0 license as described in the file LICENSE. Author: Leonardo de Moura */ -#include "util/shared_mutex.h" #include "util/option_declarations.h" namespace lean { @@ -46,11 +45,11 @@ void option_declaration::display_value(std::ostream & out, options const & o) co } static option_declarations * g_option_declarations = nullptr; -static shared_mutex * g_option_declarations_guard = nullptr; +static mutex * g_option_declarations_guard = nullptr; void initialize_option_declarations() { g_option_declarations = new option_declarations(); - g_option_declarations_guard = new shared_mutex(); + g_option_declarations_guard = new mutex(); } void finalize_option_declarations() { @@ -61,14 +60,14 @@ void finalize_option_declarations() { option_declarations get_option_declarations() { option_declarations r; { - shared_lock lock(*g_option_declarations_guard); + unique_lock lock(*g_option_declarations_guard); r = *g_option_declarations; } return r; } void register_option(name const & n, data_value_kind k, char const * default_value, char const * description) { - exclusive_lock lock(*g_option_declarations_guard); + unique_lock lock(*g_option_declarations_guard); g_option_declarations->insert(n, option_declaration(n, k, default_value, description)); } } diff --git a/src/util/pair_ref.h b/src/util/pair_ref.h index 0d02391c58..531e26c47e 100644 --- a/src/util/pair_ref.h +++ b/src/util/pair_ref.h @@ -5,6 +5,7 @@ Released under Apache 2.0 license as described in the file LICENSE. Author: Leonardo de Moura */ #pragma once +#include "runtime/serializer.h" #include "util/object_ref.h" namespace lean {