unix: Add stackless config settings, for easy access.

I.e. in this mode, C stack will never be used to call a Python function,
but if there's no free heap for a call, it will be reported as
RuntimeError, not MemoryError, as expected.

py/bc.c

@@ -86,6 +86,9 @@ void mp_setup_code_state(mp_code_state *code_state, mp_obj_t self_in, mp_uint_t
     mp_obj_fun_bc_t *self = self_in;
     mp_uint_t n_state = code_state->n_state;
 
+    #if MICROPY_STACKLESS
+    code_state->prev = NULL;
+    #endif
     code_state->code_info = self->bytecode;
     code_state->sp = &code_state->state[0] - 1;
     code_state->exc_sp = (mp_exc_stack_t*)(code_state->state + n_state) - 1;

py/bc.h

@@ -46,6 +46,9 @@ typedef struct _mp_code_state {
     // bit 0 is saved currently_in_except_block value
     mp_exc_stack_t *exc_sp;
     mp_obj_dict_t *old_globals;
+    #if MICROPY_STACKLESS
+    struct _mp_code_state *prev;
+    #endif
     mp_uint_t n_state;
     // Variable-length
     mp_obj_t state[0];
@@ -56,6 +59,7 @@ typedef struct _mp_code_state {
 mp_uint_t mp_decode_uint(const byte **ptr);
 
 mp_vm_return_kind_t mp_execute_bytecode(mp_code_state *code_state, volatile mp_obj_t inject_exc);
+mp_code_state *mp_obj_fun_bc_prepare_codestate(mp_obj_t func, mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args);
 void mp_setup_code_state(mp_code_state *code_state, mp_obj_t self_in, mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args);
 void mp_bytecode_print(const void *descr, mp_uint_t n_total_args, const byte *code, mp_uint_t len);
 void mp_bytecode_print2(const byte *code, mp_uint_t len);

py/mpconfig.h

@@ -125,6 +125,19 @@
 #define MICROPY_QSTR_BYTES_IN_LEN (1)
 #endif
 
+// Avoid using C stack when making Python function calls. C stack still
+// may be used if there's no free heap.
+#ifndef MICROPY_STACKLESS
+#define MICROPY_STACKLESS (0)
+#endif
+
+// Never use C stack when making Python function calls. This may break
+// testsuite as will subtly change which exception is thrown in case
+// of too deep recursion and other similar cases.
+#ifndef MICROPY_STACKLESS_STRICT
+#define MICROPY_STACKLESS_STRICT (0)
+#endif
+
 /*****************************************************************************/
 /* Micro Python emitters                                                     */
 

py/objfun.c

@@ -142,6 +142,41 @@ STATIC void dump_args(const mp_obj_t *a, mp_uint_t sz) {
 // Set this to enable a simple stack overflow check.
 #define VM_DETECT_STACK_OVERFLOW (0)
 
+mp_code_state *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args) {
+    MP_STACK_CHECK();
+    mp_obj_fun_bc_t *self = self_in;
+
+    // skip code-info block
+    const byte *code_info = self->bytecode;
+    mp_uint_t code_info_size = mp_decode_uint(&code_info);
+    const byte *ip = self->bytecode + code_info_size;
+
+    // bytecode prelude: skip arg names
+    ip += (self->n_pos_args + self->n_kwonly_args) * sizeof(mp_obj_t);
+
+    // bytecode prelude: state size and exception stack size
+    mp_uint_t n_state = mp_decode_uint(&ip);
+    mp_uint_t n_exc_stack = mp_decode_uint(&ip);
+
+    // allocate state for locals and stack
+    mp_uint_t state_size = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t);
+    mp_code_state *code_state;
+    code_state = m_new_obj_var_maybe(mp_code_state, byte, state_size);
+    if (!code_state) {
+        return NULL;
+    }
+
+    code_state->n_state = n_state;
+    code_state->ip = ip;
+    mp_setup_code_state(code_state, self_in, n_args, n_kw, args);
+
+    // execute the byte code with the correct globals context
+    code_state->old_globals = mp_globals_get();
+    mp_globals_set(self->globals);
+
+    return code_state;
+}
+
 STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args) {
     MP_STACK_CHECK();
 

py/runtime.c

@@ -577,7 +577,7 @@ mp_obj_t mp_call_method_n_kw(mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *a
     return mp_call_function_n_kw(args[0], n_args + adjust, n_kw, args + 2 - adjust);
 }
 
-mp_obj_t mp_call_method_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp_obj_t *args) {
+void mp_call_prepare_args_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp_obj_t *args, call_args_t *out_args) {
     mp_obj_t fun = *args++;
     mp_obj_t self = MP_OBJ_NULL;
     if (have_self) {
@@ -715,8 +715,19 @@ mp_obj_t mp_call_method_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp
         }
     }
 
-    mp_obj_t res = mp_call_function_n_kw(fun, pos_args_len, (args2_len - pos_args_len) / 2, args2);
-    m_del(mp_obj_t, args2, args2_alloc);
+    out_args->fun = fun;
+    out_args->args = args2;
+    out_args->n_args = pos_args_len;
+    out_args->n_kw = (args2_len - pos_args_len) / 2;
+    out_args->n_alloc = args2_alloc;
+}
+
+mp_obj_t mp_call_method_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp_obj_t *args) {
+    call_args_t out_args;
+    mp_call_prepare_args_n_kw_var(have_self, n_args_n_kw, args, &out_args);
+
+    mp_obj_t res = mp_call_function_n_kw(out_args.fun, out_args.n_args, out_args.n_kw, out_args.args);
+    m_del(mp_obj_t, out_args.args, out_args.n_alloc);
 
     return res;
 }

py/runtime.h

@@ -97,6 +97,17 @@ mp_obj_t mp_call_function_n_kw(mp_obj_t fun, mp_uint_t n_args, mp_uint_t n_kw, c
 mp_obj_t mp_call_method_n_kw(mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args);
 mp_obj_t mp_call_method_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp_obj_t *args);
 
+typedef struct _call_args_t {
+    mp_obj_t fun;
+    mp_uint_t n_args, n_kw, n_alloc;
+    mp_obj_t *args;
+} call_args_t;
+
+// Takes arguments which are the most general mix of Python arg types, and
+// prepares argument array suitable for passing to ->call() method of a
+// function object (and mp_call_function_n_kw()).
+void mp_call_prepare_args_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp_obj_t *args, call_args_t *out_args);
+
 void mp_unpack_sequence(mp_obj_t seq, mp_uint_t num, mp_obj_t *items);
 void mp_unpack_ex(mp_obj_t seq, mp_uint_t num, mp_obj_t *items);
 mp_obj_t mp_store_map(mp_obj_t map, mp_obj_t key, mp_obj_t value);
@@ -119,6 +130,7 @@ void mp_import_all(mp_obj_t module);
 
 // Raise NotImplementedError with given message
 NORETURN void mp_not_implemented(const char *msg);
+NORETURN void mp_exc_recursion_depth(void);
 
 // helper functions for native/viper code
 mp_uint_t mp_convert_obj_to_native(mp_obj_t obj, mp_uint_t type);

py/stackctrl.c

@@ -27,6 +27,7 @@
 #include "py/mpstate.h"
 #include "py/nlr.h"
 #include "py/obj.h"
+#include "py/runtime.h"
 #include "py/stackctrl.h"
 
 void mp_stack_ctrl_init(void) {
@@ -46,10 +47,14 @@ void mp_stack_set_limit(mp_uint_t limit) {
     MP_STATE_VM(stack_limit) = limit;
 }
 
+void mp_exc_recursion_depth(void) {
+    nlr_raise(mp_obj_new_exception_arg1(&mp_type_RuntimeError,
+        MP_OBJ_NEW_QSTR(MP_QSTR_maximum_space_recursion_space_depth_space_exceeded)));
+}
+
 void mp_stack_check(void) {
     if (mp_stack_usage() >= MP_STATE_VM(stack_limit)) {
-        nlr_raise(mp_obj_new_exception_arg1(&mp_type_RuntimeError,
-            MP_OBJ_NEW_QSTR(MP_QSTR_maximum_space_recursion_space_depth_space_exceeded)));
+        mp_exc_recursion_depth();
     }
 }
 

py/vm.c

@@ -129,9 +129,12 @@ mp_vm_return_kind_t mp_execute_bytecode(mp_code_state *code_state, volatile mp_o
     // loop and the exception handler, leading to very obscure bugs.
     #define RAISE(o) do { nlr_pop(); nlr.ret_val = o; goto exception_handler; } while(0)
 
+#if MICROPY_STACKLESS
+run_code_state: ;
+#endif
     // Pointers which are constant for particular invocation of mp_execute_bytecode()
-    mp_obj_t *const fastn = &code_state->state[code_state->n_state - 1];
-    mp_exc_stack_t *const exc_stack = (mp_exc_stack_t*)(code_state->state + code_state->n_state);
+    mp_obj_t */*const*/ fastn = &code_state->state[code_state->n_state - 1];
+    mp_exc_stack_t */*const*/ exc_stack = (mp_exc_stack_t*)(code_state->state + code_state->n_state);
 
     // variables that are visible to the exception handler (declared volatile)
     volatile bool currently_in_except_block = MP_TAGPTR_TAG0(code_state->exc_sp); // 0 or 1, to detect nested exceptions
@@ -865,6 +868,26 @@ unwind_jump:;
                     // unum & 0xff == n_positional
                     // (unum >> 8) & 0xff == n_keyword
                     sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe);
+                    #if MICROPY_STACKLESS
+                    if (mp_obj_get_type(*sp) == &mp_type_fun_bc) {
+                        code_state->ip = ip;
+                        code_state->sp = sp;
+                        code_state->exc_sp = MP_TAGPTR_MAKE(exc_sp, currently_in_except_block);
+                        mp_code_state *new_state = mp_obj_fun_bc_prepare_codestate(*sp, unum & 0xff, (unum >> 8) & 0xff, sp + 1);
+                        if (new_state) {
+                            new_state->prev = code_state;
+                            code_state = new_state;
+                            nlr_pop();
+                            goto run_code_state;
+                        }
+                        #if MICROPY_STACKLESS_STRICT
+                        else {
+                        deep_recursion_error:
+                            mp_exc_recursion_depth();
+                        }
+                        #endif
+                    }
+                    #endif
                     SET_TOP(mp_call_function_n_kw(*sp, unum & 0xff, (unum >> 8) & 0xff, sp + 1));
                     DISPATCH();
                 }
@@ -877,6 +900,31 @@ unwind_jump:;
                     // We have folowing stack layout here:
                     // fun arg0 arg1 ... kw0 val0 kw1 val1 ... seq dict <- TOS
                     sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe) + 2;
+                    #if MICROPY_STACKLESS
+                    if (mp_obj_get_type(*sp) == &mp_type_fun_bc) {
+                        code_state->ip = ip;
+                        code_state->sp = sp;
+                        code_state->exc_sp = MP_TAGPTR_MAKE(exc_sp, currently_in_except_block);
+
+                        call_args_t out_args;
+                        mp_call_prepare_args_n_kw_var(false, unum, sp, &out_args);
+
+                        mp_code_state *new_state = mp_obj_fun_bc_prepare_codestate(out_args.fun,
+                            out_args.n_args, out_args.n_kw, out_args.args);
+                        m_del(mp_obj_t, out_args.args, out_args.n_alloc);
+                        if (new_state) {
+                            new_state->prev = code_state;
+                            code_state = new_state;
+                            nlr_pop();
+                            goto run_code_state;
+                        }
+                        #if MICROPY_STACKLESS_STRICT
+                        else {
+                            goto deep_recursion_error;
+                        }
+                        #endif
+                    }
+                    #endif
                     SET_TOP(mp_call_method_n_kw_var(false, unum, sp));
                     DISPATCH();
                 }
@@ -887,6 +935,30 @@ unwind_jump:;
                     // unum & 0xff == n_positional
                     // (unum >> 8) & 0xff == n_keyword
                     sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe) + 1;
+                    #if MICROPY_STACKLESS
+                    if (mp_obj_get_type(*sp) == &mp_type_fun_bc) {
+                        code_state->ip = ip;
+                        code_state->sp = sp;
+                        code_state->exc_sp = MP_TAGPTR_MAKE(exc_sp, currently_in_except_block);
+
+                        mp_uint_t n_args = unum & 0xff;
+                        mp_uint_t n_kw = (unum >> 8) & 0xff;
+                        int adjust = (sp[1] == NULL) ? 0 : 1;
+
+                        mp_code_state *new_state = mp_obj_fun_bc_prepare_codestate(*sp, n_args + adjust, n_kw, sp + 2 - adjust);
+                        if (new_state) {
+                            new_state->prev = code_state;
+                            code_state = new_state;
+                            nlr_pop();
+                            goto run_code_state;
+                        }
+                        #if MICROPY_STACKLESS_STRICT
+                        else {
+                            goto deep_recursion_error;
+                        }
+                        #endif
+                    }
+                    #endif
                     SET_TOP(mp_call_method_n_kw(unum & 0xff, (unum >> 8) & 0xff, sp));
                     DISPATCH();
                 }
@@ -899,6 +971,31 @@ unwind_jump:;
                     // We have folowing stack layout here:
                     // fun self arg0 arg1 ... kw0 val0 kw1 val1 ... seq dict <- TOS
                     sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe) + 3;
+                    #if MICROPY_STACKLESS
+                    if (mp_obj_get_type(*sp) == &mp_type_fun_bc) {
+                        code_state->ip = ip;
+                        code_state->sp = sp;
+                        code_state->exc_sp = MP_TAGPTR_MAKE(exc_sp, currently_in_except_block);
+
+                        call_args_t out_args;
+                        mp_call_prepare_args_n_kw_var(true, unum, sp, &out_args);
+
+                        mp_code_state *new_state = mp_obj_fun_bc_prepare_codestate(out_args.fun,
+                            out_args.n_args, out_args.n_kw, out_args.args);
+                        m_del(mp_obj_t, out_args.args, out_args.n_alloc);
+                        if (new_state) {
+                            new_state->prev = code_state;
+                            code_state = new_state;
+                            nlr_pop();
+                            goto run_code_state;
+                        }
+                        #if MICROPY_STACKLESS_STRICT
+                        else {
+                            goto deep_recursion_error;
+                        }
+                        #endif
+                    }
+                    #endif
                     SET_TOP(mp_call_method_n_kw_var(true, unum, sp));
                     DISPATCH();
                 }
@@ -925,6 +1022,15 @@ unwind_return:
                     nlr_pop();
                     code_state->sp = sp;
                     assert(exc_sp == exc_stack - 1);
+                    #if MICROPY_STACKLESS
+                    if (code_state->prev != NULL) {
+                        mp_obj_t res = *sp;
+                        mp_globals_set(code_state->old_globals);
+                        code_state = code_state->prev;
+                        *code_state->sp = res;
+                        goto run_code_state;
+                    }
+                    #endif
                     return MP_VM_RETURN_NORMAL;
 
                 ENTRY(MP_BC_RAISE_VARARGS): {
@@ -1122,6 +1228,9 @@ exception_handler:
                 goto outer_dispatch_loop; // continue with dispatch loop
             }
 
+#if MICROPY_STACKLESS
+unwind_loop:
+#endif
             // set file and line number that the exception occurred at
             // TODO: don't set traceback for exceptions re-raised by END_FINALLY.
             // But consider how to handle nested exceptions.
@@ -1185,6 +1294,18 @@ exception_handler:
                 PUSH(mp_obj_get_type(nlr.ret_val));
                 code_state->sp = sp;
 
+            #if MICROPY_STACKLESS
+            } else if (code_state->prev != NULL) {
+                mp_globals_set(code_state->old_globals);
+                code_state = code_state->prev;
+                fastn = &code_state->state[code_state->n_state - 1];
+                exc_stack = (mp_exc_stack_t*)(code_state->state + code_state->n_state);
+                // variables that are visible to the exception handler (declared volatile)
+                currently_in_except_block = MP_TAGPTR_TAG0(code_state->exc_sp); // 0 or 1, to detect nested exceptions
+                exc_sp = MP_TAGPTR_PTR(code_state->exc_sp); // stack grows up, exc_sp points to top of stack
+                goto unwind_loop;
+
+            #endif
             } else {
                 // propagate exception to higher level
                 // TODO what to do about ip and sp? they don't really make sense at this point

unix/mpconfigport.h

@@ -76,6 +76,9 @@
 #define MICROPY_PY_IO_FILEIO        (1)
 #define MICROPY_PY_GC_COLLECT_RETVAL (1)
 
+#define MICROPY_STACKLESS           (0)
+#define MICROPY_STACKLESS_STRICT    (0)
+
 #define MICROPY_PY_UCTYPES          (1)
 #define MICROPY_PY_UZLIB            (1)
 #define MICROPY_PY_UJSON            (1)