diff --git a/py/dynruntime.h b/py/dynruntime.h new file mode 100644 index 000000000..34cf6010c --- /dev/null +++ b/py/dynruntime.h @@ -0,0 +1,193 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2019 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_PY_DYNRUNTIME_H +#define MICROPY_INCLUDED_PY_DYNRUNTIME_H + +// This header file contains definitions to dynamically implement the static +// MicroPython runtime API defined in py/obj.h and py/runtime.h. + +#include "py/nativeglue.h" +#include "py/objstr.h" + +#undef MP_ROM_QSTR +#undef MP_OBJ_QSTR_VALUE +#undef MP_OBJ_NEW_QSTR +#undef mp_const_none +#undef mp_const_false +#undef mp_const_true +#undef mp_const_empty_tuple +#undef nlr_raise + +/******************************************************************************/ +// Memory allocation + +#define m_malloc(n) (m_malloc_dyn((n))) +#define m_free(ptr) (m_free_dyn((ptr))) +#define m_realloc(ptr, new_num_bytes) (m_realloc_dyn((ptr), (new_num_bytes))) + +static inline void *m_malloc_dyn(size_t n) { + // TODO won't raise on OOM + return mp_fun_table.realloc_(NULL, n, false); +} + +static inline void m_free_dyn(void *ptr) { + mp_fun_table.realloc_(ptr, 0, false); +} + +static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) { + // TODO won't raise on OOM + return mp_fun_table.realloc_(ptr, new_num_bytes, true); +} + +/******************************************************************************/ +// Printing + +#define mp_plat_print (*mp_fun_table.plat_print) +#define mp_printf(p, ...) (mp_fun_table.printf_((p), __VA_ARGS__)) +#define mp_vprintf(p, fmt, args) (mp_fun_table.vprintf_((p), (fmt), (args))) + +/******************************************************************************/ +// Types and objects + +#define MP_OBJ_NEW_QSTR(x) MP_OBJ_NEW_QSTR_ ## x + +#define mp_type_type (*mp_fun_table.type_type) +#define mp_type_str (*mp_fun_table.type_str) +#define mp_type_list (*mp_fun_table.type_list) +#define mp_type_EOFError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_EOFError))) +#define mp_type_IndexError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_IndexError))) +#define mp_type_KeyError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_KeyError))) +#define mp_type_NotImplementedError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_NotImplementedError))) +#define mp_type_RuntimeError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_RuntimeError))) +#define mp_type_TypeError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_TypeError))) +#define mp_type_ValueError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_ValueError))) + +#define mp_stream_read_obj (*mp_fun_table.stream_read_obj) +#define mp_stream_readinto_obj (*mp_fun_table.stream_readinto_obj) +#define mp_stream_unbuffered_readline_obj (*mp_fun_table.stream_unbuffered_readline_obj) +#define mp_stream_write_obj (*mp_fun_table.stream_write_obj) + +#define mp_const_none ((mp_obj_t)mp_fun_table.const_none) +#define mp_const_false ((mp_obj_t)mp_fun_table.const_false) +#define mp_const_true ((mp_obj_t)mp_fun_table.const_true) +#define mp_const_empty_tuple (mp_fun_table.new_tuple(0, NULL)) + +#define mp_obj_new_bool(b) ((b) ? (mp_obj_t)mp_fun_table.const_true : (mp_obj_t)mp_fun_table.const_false) +#define mp_obj_new_int(i) (mp_fun_table.native_to_obj(i, MP_NATIVE_TYPE_INT)) +#define mp_obj_new_str(data, len) (mp_fun_table.obj_new_str((data), (len))) +#define mp_obj_new_str_of_type(t, d, l) (mp_obj_new_str_of_type_dyn((t), (d), (l))) +#define mp_obj_new_bytes(data, len) (mp_fun_table.obj_new_bytes((data), (len))) +#define mp_obj_new_bytearray_by_ref(n, i) (mp_fun_table.obj_new_bytearray_by_ref((n), (i))) +#define mp_obj_new_tuple(n, items) (mp_fun_table.new_tuple((n), (items))) +#define mp_obj_new_list(n, items) (mp_fun_table.new_list((n), (items))) + +#define mp_obj_get_type(o) (mp_fun_table.obj_get_type((o))) +#define mp_obj_get_int(o) (mp_fun_table.native_from_obj(o, MP_NATIVE_TYPE_INT)) +#define mp_obj_str_get_str(s) ((void*)mp_fun_table.native_from_obj(s, MP_NATIVE_TYPE_PTR)) +#define mp_obj_str_get_data(o, len) (mp_obj_str_get_data_dyn((o), (len))) +#define mp_get_buffer_raise(o, bufinfo, fl) (mp_fun_table.get_buffer_raise((o), (bufinfo), (fl))) +#define mp_get_stream_raise(s, flags) (mp_fun_table.get_stream_raise((s), (flags))) + +#define mp_obj_list_append(list, item) (mp_fun_table.list_append((list), (item))) + +static inline mp_obj_t mp_obj_new_str_of_type_dyn(const mp_obj_type_t *type, const byte* data, size_t len) { + if (type == &mp_type_str) { + return mp_obj_new_str((const char*)data, len); + } else { + return mp_obj_new_bytes(data, len); + } +} + +static inline void *mp_obj_str_get_data_dyn(mp_obj_t o, size_t *l) { + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(o, &bufinfo, MP_BUFFER_READ); + *l = bufinfo.len; + return bufinfo.buf; +} + +/******************************************************************************/ +// General runtime functions + +#define mp_load_name(qst) (mp_fun_table.load_name(qst)) +#define mp_load_global(qst) (mp_fun_table.load_global(qst)) +#define mp_store_global(qst, obj) (mp_fun_table.store_global((qst), (obj))) +#define mp_unary_op(op, obj) (mp_fun_table.unary_op((op), (obj))) +#define mp_binary_op(op, lhs, rhs) (mp_fun_table.binary_op((op), (lhs), (rhs))) + +#define mp_make_function_from_raw_code(rc, def_args, def_kw_args) \ + (mp_fun_table.make_function_from_raw_code((rc), (def_args), (def_kw_args))) + +#define mp_call_function_n_kw(fun, n_args, n_kw, args) \ + (mp_fun_table.call_function_n_kw((fun), (n_args) | ((n_kw) << 8), args)) + +#define mp_arg_check_num(n_args, n_kw, n_args_min, n_args_max, takes_kw) \ + (mp_fun_table.arg_check_num_sig((n_args), (n_kw), MP_OBJ_FUN_MAKE_SIG((n_args_min), (n_args_max), (takes_kw)))) + +#define MP_DYNRUNTIME_INIT_ENTRY \ + mp_obj_t old_globals = mp_fun_table.swap_globals(self->globals); \ + mp_raw_code_t rc; \ + rc.kind = MP_CODE_NATIVE_VIPER; \ + rc.scope_flags = 0; \ + rc.const_table = (void*)self->const_table; \ + (void)rc; + +#define MP_DYNRUNTIME_INIT_EXIT \ + mp_fun_table.swap_globals(old_globals); \ + return mp_const_none; + +#define MP_DYNRUNTIME_MAKE_FUNCTION(f) \ + (mp_make_function_from_raw_code((rc.fun_data = (f), &rc), MP_OBJ_NULL, MP_OBJ_NULL)) + +/******************************************************************************/ +// Exceptions + +#define mp_obj_new_exception(o) ((mp_obj_t)(o)) // Assumes returned object will be raised, will create instance then +#define mp_obj_new_exception_arg1(e_type, arg) (mp_obj_new_exception_arg1_dyn((e_type), (arg))) + +#define nlr_raise(o) (mp_raise_dyn(o)) +#define mp_raise_msg(type, msg) (mp_fun_table.raise_msg((type), (msg))) +#define mp_raise_OSError(er) (mp_raise_OSError_dyn(er)) +#define mp_raise_NotImplementedError(msg) (mp_raise_msg(&mp_type_NotImplementedError, (msg))) +#define mp_raise_TypeError(msg) (mp_raise_msg(&mp_type_TypeError, (msg))) +#define mp_raise_ValueError(msg) (mp_raise_msg(&mp_type_ValueError, (msg))) + +static inline mp_obj_t mp_obj_new_exception_arg1_dyn(const mp_obj_type_t *exc_type, mp_obj_t arg) { + mp_obj_t args[1] = { arg }; + return mp_call_function_n_kw(MP_OBJ_FROM_PTR(exc_type), 1, 0, &args[0]); +} + +static NORETURN inline void mp_raise_dyn(mp_obj_t o) { + mp_fun_table.raise(o); + for (;;) { + } +} + +static inline void mp_raise_OSError_dyn(int er) { + mp_obj_t args[1] = { MP_OBJ_NEW_SMALL_INT(er) }; + nlr_raise(mp_call_function_n_kw(mp_load_global(MP_QSTR_OSError), 1, 0, &args[0])); +} + +#endif // MICROPY_INCLUDED_PY_DYNRUNTIME_H diff --git a/py/dynruntime.mk b/py/dynruntime.mk new file mode 100644 index 000000000..b01b80e0d --- /dev/null +++ b/py/dynruntime.mk @@ -0,0 +1,134 @@ +# Makefile fragment for generating native .mpy files from C source +# MPY_DIR must be set to the top of the MicroPython source tree + +BUILD ?= build + +ECHO = @echo +RM = /bin/rm +MKDIR = /bin/mkdir +PYTHON = python3 +MPY_CROSS = $(MPY_DIR)/mpy-cross/mpy-cross +MPY_TOOL = $(PYTHON) $(MPY_DIR)/tools/mpy-tool.py +MPY_LD = $(PYTHON) $(MPY_DIR)/tools/mpy_ld.py + +Q = @ +ifeq ("$(origin V)", "command line") +ifeq ($(V),1) +Q = +MPY_LD += '-vvv' +endif +endif + +ARCH_UPPER = $(shell echo $(ARCH) | tr '[:lower:]' '[:upper:]') +CONFIG_H = $(BUILD)/$(MOD).config.h + +CFLAGS += -I. -I$(MPY_DIR) +CFLAGS += -std=c99 +CFLAGS += -Os +CFLAGS += -Wall -Werror -DNDEBUG +CFLAGS += -DNO_QSTR +CFLAGS += -DMP_CONFIGFILE='<$(CONFIG_H)>' +CFLAGS += -fpic -fno-common +CFLAGS += -U _FORTIFY_SOURCE # prevent use of __*_chk libc functions +#CFLAGS += -fdata-sections -ffunction-sections + +MPY_CROSS_FLAGS += -march=$(ARCH) + +SRC_O += $(addprefix $(BUILD)/, $(patsubst %.c,%.o,$(filter %.c,$(SRC)))) +SRC_MPY += $(addprefix $(BUILD)/, $(patsubst %.py,%.mpy,$(filter %.py,$(SRC)))) + +################################################################################ +# Architecture configuration + +ifeq ($(ARCH),x86) + +# x86 +CROSS = +CFLAGS += -m32 -fno-stack-protector +MPY_CROSS_FLAGS += -mcache-lookup-bc + +else ifeq ($(ARCH),x64) + +# x64 +CROSS = +CFLAGS += -fno-stack-protector +MPY_CROSS_FLAGS += -mcache-lookup-bc + +else ifeq ($(ARCH),armv7m) + +# thumb +CROSS = arm-none-eabi- +CFLAGS += -mthumb -mcpu=cortex-m3 + +else ifeq ($(ARCH),armv7emsp) + +# thumb +CROSS = arm-none-eabi- +CFLAGS += -mthumb -mcpu=cortex-m4 +CFLAGS += -mfpu=fpv4-sp-d16 -mfloat-abi=hard + +else ifeq ($(ARCH),armv7emdp) + +# thumb +CROSS = arm-none-eabi- +CFLAGS += -mthumb -mcpu=cortex-m7 +CFLAGS += -mfpu=fpv5-d16 -mfloat-abi=hard + +else ifeq ($(ARCH),xtensa) + +# xtensa +CROSS = xtensa-lx106-elf- +CFLAGS += -mforce-l32 + +else ifeq ($(ARCH),xtensawin) + +# xtensawin +CROSS = xtensa-esp32-elf- +CFLAGS += + +else +$(error architecture '$(ARCH)' not supported) +endif + +CFLAGS += $(CFLAGS_EXTRA) + +################################################################################ +# Build rules + +.PHONY: all clean + +all: $(MOD).mpy + +clean: + $(RM) -rf $(BUILD) $(CLEAN_EXTRA) + +# Create build destination directories first +BUILD_DIRS = $(sort $(dir $(CONFIG_H) $(SRC_O) $(SRC_MPY))) +$(CONFIG_H) $(SRC_O) $(SRC_MPY): | $(BUILD_DIRS) +$(BUILD_DIRS): + $(Q)$(MKDIR) -p $@ + +# Preprocess all source files to generate $(CONFIG_H) +$(CONFIG_H): $(SRC) + $(ECHO) "GEN $@" + $(Q)$(MPY_LD) --arch $(ARCH) --preprocess -o $@ $^ + +# Build .o from .c source files +$(BUILD)/%.o: %.c $(CONFIG_H) Makefile + $(ECHO) "CC $<" + $(Q)$(CROSS)gcc $(CFLAGS) -o $@ -c $< + +# Build .mpy from .py source files +$(BUILD)/%.mpy: %.py + $(ECHO) "MPY $<" + $(Q)$(MPY_CROSS) $(MPY_CROSS_FLAGS) -o $@ $< + +# Build native .mpy from object files +$(BUILD)/$(MOD).native.mpy: $(SRC_O) + $(ECHO) "LINK $<" + $(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) -o $@ $^ + +# Build final .mpy from all intermediate .mpy files +$(MOD).mpy: $(BUILD)/$(MOD).native.mpy $(SRC_MPY) + $(ECHO) "GEN $@" + $(Q)$(MPY_TOOL) --merge -o $@ $^ diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py new file mode 100755 index 000000000..07105caac --- /dev/null +++ b/tools/mpy_ld.py @@ -0,0 +1,978 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2019 Damien P. George +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +""" +Link .o files to .mpy +""" + +import sys, os, struct, re +from elftools.elf import elffile + +sys.path.append(os.path.dirname(__file__) + '/../py') +import makeqstrdata as qstrutil + +# MicroPython constants +MPY_VERSION = 5 +MP_NATIVE_ARCH_X86 = 1 +MP_NATIVE_ARCH_X64 = 2 +MP_NATIVE_ARCH_ARMV7M = 5 +MP_NATIVE_ARCH_ARMV7EMSP = 7 +MP_NATIVE_ARCH_ARMV7EMDP = 8 +MP_NATIVE_ARCH_XTENSA = 9 +MP_NATIVE_ARCH_XTENSAWIN = 10 +MP_CODE_BYTECODE = 2 +MP_CODE_NATIVE_VIPER = 4 +MP_SCOPE_FLAG_VIPERRELOC = 0x10 +MP_SCOPE_FLAG_VIPERRODATA = 0x20 +MP_SCOPE_FLAG_VIPERBSS = 0x40 +MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE = 1 +MICROPY_PY_BUILTINS_STR_UNICODE = 2 +MP_SMALL_INT_BITS = 31 +QSTR_WINDOW_SIZE = 32 + +# ELF constants +R_386_32 = 1 +R_X86_64_64 = 1 +R_XTENSA_32 = 1 +R_386_PC32 = 2 +R_X86_64_PC32 = 2 +R_ARM_ABS32 = 2 +R_386_GOT32 = 3 +R_ARM_REL32 = 3 +R_386_PLT32 = 4 +R_X86_64_PLT32 = 4 +R_XTENSA_PLT = 6 +R_386_GOTOFF = 9 +R_386_GOTPC = 10 +R_ARM_THM_CALL = 10 +R_XTENSA_DIFF32 = 19 +R_XTENSA_SLOT0_OP = 20 +R_ARM_BASE_PREL = 25 # aka R_ARM_GOTPC +R_ARM_GOT_BREL = 26 # aka R_ARM_GOT32 +R_ARM_THM_JUMP24 = 30 +R_X86_64_REX_GOTPCRELX = 42 +R_386_GOT32X = 43 + +################################################################################ +# Architecture configuration + +def asm_jump_x86(entry): + return struct.pack('> 11 == 0 or b_off >> 11 == -1: + # Signed value fits in 12 bits + b0 = 0xe000 | (b_off >> 1 & 0x07ff) + b1 = 0 + else: + # Use large jump + b0 = 0xf000 | (b_off >> 12 & 0x07ff) + b1 = 0xb800 | (b_off >> 1 & 0x7ff) + return struct.pack('> 8) + +class ArchData: + def __init__(self, name, mpy_feature, qstr_entry_size, word_size, arch_got, asm_jump): + self.name = name + self.mpy_feature = mpy_feature + self.qstr_entry_size = qstr_entry_size + self.word_size = word_size + self.arch_got = arch_got + self.asm_jump = asm_jump + self.separate_rodata = name == 'EM_XTENSA' and qstr_entry_size == 4 + +ARCH_DATA = { + 'x86': ArchData( + 'EM_386', + MP_NATIVE_ARCH_X86 << 2 | MICROPY_PY_BUILTINS_STR_UNICODE | MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE, + 2, 4, (R_386_PC32, R_386_GOT32, R_386_GOT32X), asm_jump_x86, + ), + 'x64': ArchData( + 'EM_X86_64', + MP_NATIVE_ARCH_X64 << 2 | MICROPY_PY_BUILTINS_STR_UNICODE | MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE, + 2, 8, (R_X86_64_REX_GOTPCRELX,), asm_jump_x86, + ), + 'armv7m': ArchData( + 'EM_ARM', + MP_NATIVE_ARCH_ARMV7M << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, + 2, 4, (R_ARM_GOT_BREL,), asm_jump_arm, + ), + 'armv7emsp': ArchData( + 'EM_ARM', + MP_NATIVE_ARCH_ARMV7EMSP << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, + 2, 4, (R_ARM_GOT_BREL,), asm_jump_arm, + ), + 'armv7emdp': ArchData( + 'EM_ARM', + MP_NATIVE_ARCH_ARMV7EMDP << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, + 2, 4, (R_ARM_GOT_BREL,), asm_jump_arm, + ), + 'xtensa': ArchData( + 'EM_XTENSA', + MP_NATIVE_ARCH_XTENSA << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, + 2, 4, (R_XTENSA_32, R_XTENSA_PLT), asm_jump_xtensa, + ), + 'xtensawin': ArchData( + 'EM_XTENSA', + MP_NATIVE_ARCH_XTENSAWIN << 2 | MICROPY_PY_BUILTINS_STR_UNICODE, + 4, 4, (R_XTENSA_32, R_XTENSA_PLT), asm_jump_xtensa, + ), +} + +################################################################################ +# Helper functions + +def align_to(value, align): + return (value + align - 1) & ~(align - 1) + +def unpack_u24le(data, offset): + return data[offset] | data[offset + 1] << 8 | data[offset + 2] << 16 + +def pack_u24le(data, offset, value): + data[offset] = value & 0xff + data[offset + 1] = value >> 8 & 0xff + data[offset + 2] = value >> 16 & 0xff + +def xxd(text): + for i in range(0, len(text), 16): + print('{:08x}:'.format(i), end='') + for j in range(4): + off = i + j * 4 + if off < len(text): + d = int.from_bytes(text[off:off + 4], 'little') + print(' {:08x}'.format(d), end='') + print() + +# Smaller numbers are enabled first +LOG_LEVEL_1 = 1 +LOG_LEVEL_2 = 2 +LOG_LEVEL_3 = 3 +log_level = LOG_LEVEL_1 + +def log(level, msg): + if level <= log_level: + print(msg) + +################################################################################ +# Qstr extraction + +def extract_qstrs(source_files): + def read_qstrs(f): + with open(f) as f: + vals = set() + objs = set() + for line in f: + while line: + m = re.search(r'MP_OBJ_NEW_QSTR\((MP_QSTR_[A-Za-z0-9_]*)\)', line) + if m: + objs.add(m.group(1)) + else: + m = re.search(r'MP_QSTR_[A-Za-z0-9_]*', line) + if m: + vals.add(m.group()) + if m: + s = m.span() + line = line[:s[0]] + line[s[1]:] + else: + line = '' + return vals, objs + + static_qstrs = ['MP_QSTR_' + qstrutil.qstr_escape(q) for q in qstrutil.static_qstr_list] + + qstr_vals = set() + qstr_objs = set() + for f in source_files: + vals, objs = read_qstrs(f) + qstr_vals.update(vals) + qstr_objs.update(objs) + qstr_vals.difference_update(static_qstrs) + + return static_qstrs, qstr_vals, qstr_objs + +################################################################################ +# Linker + +class LinkError(Exception): + pass + +class Section: + def __init__(self, name, data, alignment, filename=None): + self.filename = filename + self.name = name + self.data = data + self.alignment = alignment + self.addr = 0 + self.reloc = [] + + @staticmethod + def from_elfsec(elfsec, filename): + assert elfsec.header.sh_addr == 0 + return Section(elfsec.name, elfsec.data(), elfsec.data_alignment, filename) + +class GOTEntry: + def __init__(self, name, sym, link_addr=0): + self.name = name + self.sym = sym + self.offset = None + self.link_addr = link_addr + + def isexternal(self): + return self.sec_name.startswith('.external') + + def istext(self): + return self.sec_name.startswith('.text') + + def isrodata(self): + return self.sec_name.startswith(('.rodata', '.data.rel.ro')) + + def isbss(self): + return self.sec_name.startswith('.bss') + +class LiteralEntry: + def __init__(self, value, offset): + self.value = value + self.offset = offset + +class LinkEnv: + def __init__(self, arch): + self.arch = ARCH_DATA[arch] + self.sections = [] # list of sections in order of output + self.literal_sections = [] # list of literal sections (xtensa only) + self.known_syms = {} # dict of symbols that are defined + self.unresolved_syms = [] # list of unresolved symbols + self.mpy_relocs = [] # list of relocations needed in the output .mpy file + + def check_arch(self, arch_name): + if arch_name != self.arch.name: + raise LinkError('incompatible arch') + + def print_sections(self): + log(LOG_LEVEL_2, 'sections:') + for sec in self.sections: + log(LOG_LEVEL_2, ' {:08x} {} size={}'.format(sec.addr, sec.name, len(sec.data))) + + def find_addr(self, name): + if name in self.known_syms: + s = self.known_syms[name] + return s.section.addr + s['st_value'] + raise LinkError('unknown symbol: {}'.format(name)) + +def build_got_generic(env): + env.got_entries = {} + for sec in env.sections: + for r in sec.reloc: + s = r.sym + if not (s.entry['st_info']['bind'] == 'STB_GLOBAL' and r['r_info_type'] in env.arch.arch_got): + continue + s_type = s.entry['st_info']['type'] + assert s_type in ('STT_NOTYPE', 'STT_FUNC', 'STT_OBJECT'), s_type + assert s.name + if s.name in env.got_entries: + continue + env.got_entries[s.name] = GOTEntry(s.name, s) + +def build_got_xtensa(env): + env.got_entries = {} + env.lit_entries = {} + env.xt_literals = {} + + # Extract the values from the literal table + for sec in env.literal_sections: + assert len(sec.data) % env.arch.word_size == 0 + + # Look through literal relocations to find any global pointers that should be GOT entries + for r in sec.reloc: + s = r.sym + s_type = s.entry['st_info']['type'] + assert s_type in ('STT_NOTYPE', 'STT_FUNC', 'STT_OBJECT', 'STT_SECTION'), s_type + assert r['r_info_type'] in env.arch.arch_got + assert r['r_offset'] % env.arch.word_size == 0 + # This entry is a global pointer + existing = struct.unpack_from(' {}+{:08x}'.format(g.offset, g.name, g.sec_name, g.link_addr)) + +def populate_lit(env): + log(LOG_LEVEL_2, 'LIT: {:08x}'.format(env.lit_section.addr)) + for lit_entry in env.lit_entries.values(): + value = lit_entry.value + log(LOG_LEVEL_2, ' {:08x} = {:08x}'.format(lit_entry.offset, value)) + o = env.lit_section.addr + lit_entry.offset + env.full_text[o:o + env.arch.word_size] = value.to_bytes(env.arch.word_size, 'little') + +def do_relocation_text(env, text_addr, r): + # Extract relevant info about symbol that's being relocated + s = r.sym + s_bind = s.entry['st_info']['bind'] + s_shndx = s.entry['st_shndx'] + s_type = s.entry['st_info']['type'] + r_offset = r['r_offset'] + text_addr + r_info_type = r['r_info_type'] + try: + # only for RELA sections + r_addend = r['r_addend'] + except KeyError: + r_addend = 0 + + # Default relocation type and name for logging + reloc_type = 'le32' + log_name = None + + if (env.arch.name == 'EM_386' and r_info_type in (R_386_PC32, R_386_PLT32) + or env.arch.name == 'EM_X86_64' and r_info_type in (R_X86_64_PC32, R_X86_64_PLT32) + or env.arch.name == 'EM_ARM' and r_info_type in (R_ARM_REL32, R_ARM_THM_CALL, R_ARM_THM_JUMP24) + or s_bind == 'STB_LOCAL' and env.arch.name == 'EM_XTENSA' and r_info_type == R_XTENSA_32 # not GOT + ): + # Standard relocation to fixed location within text/rodata + if hasattr(s, 'resolved'): + s = s.resolved + + sec = s.section + + if env.arch.separate_rodata and sec.name.startswith('.rodata'): + raise LinkError('fixed relocation to rodata with rodata referenced via GOT') + + if sec.name.startswith('.bss'): + raise LinkError('{}: fixed relocation to bss (bss variables can\'t be static)'.format(s.filename)) + + if sec.name.startswith('.external'): + raise LinkError('{}: fixed relocation to external symbol: {}'.format(s.filename, s.name)) + + addr = sec.addr + s['st_value'] + reloc = addr - r_offset + r_addend + + if r_info_type in (R_ARM_THM_CALL, R_ARM_THM_JUMP24): + # Both relocations have the same bit pattern to rewrite: + # R_ARM_THM_CALL: bl + # R_ARM_THM_JUMP24: b.w + reloc_type = 'thumb_b' + + elif (env.arch.name == 'EM_386' and r_info_type == R_386_GOTPC + or env.arch.name == 'EM_ARM' and r_info_type == R_ARM_BASE_PREL + ): + # Relocation to GOT address itself + assert s.name == '_GLOBAL_OFFSET_TABLE_' + addr = env.got_section.addr + reloc = addr - r_offset + r_addend + + elif (env.arch.name == 'EM_386' and r_info_type in (R_386_GOT32, R_386_GOT32X) + or env.arch.name == 'EM_ARM' and r_info_type == R_ARM_GOT_BREL + ): + # Relcation pointing to GOT + reloc = addr = env.got_entries[s.name].offset + + elif env.arch.name == 'EM_X86_64' and r_info_type == R_X86_64_REX_GOTPCRELX: + # Relcation pointing to GOT + got_entry = env.got_entries[s.name] + addr = env.got_section.addr + got_entry.offset + reloc = addr - r_offset + r_addend + + elif env.arch.name == 'EM_386' and r_info_type == R_386_GOTOFF: + # Relocation relative to GOT + addr = s.section.addr + s['st_value'] + reloc = addr - env.got_section.addr + r_addend + + elif env.arch.name == 'EM_XTENSA' and r_info_type == R_XTENSA_SLOT0_OP: + # Relocation pointing to GOT, xtensa specific + sec = s.section + if sec.name.startswith('.text'): + # it looks like R_XTENSA_SLOT0_OP into .text is already correctly relocated + return + assert sec.name.startswith('.literal'), sec.name + lit_idx = '{}+0x{:x}'.format(sec.filename, r_addend) + lit_ptr = env.xt_literals[lit_idx] + if isinstance(lit_ptr, str): + addr = env.got_section.addr + env.got_entries[lit_ptr].offset + log_name = 'GOT {}'.format(lit_ptr) + else: + addr = env.lit_section.addr + env.lit_entries[lit_ptr].offset + log_name = 'LIT' + reloc = addr - r_offset + reloc_type = 'xtensa_l32r' + + elif env.arch.name == 'EM_XTENSA' and r_info_type == R_XTENSA_DIFF32: + if s.section.name.startswith('.text'): + # it looks like R_XTENSA_DIFF32 into .text is already correctly relocated + return + assert 0 + + else: + # Unknown/unsupported relocation + assert 0, r_info_type + + # Write relocation + if reloc_type == 'le32': + existing, = struct.unpack_from('= 0x400000: # 2's complement + existing -= 0x800000 + new = existing + reloc + b_h = (b_h & 0xf800) | (new >> 12) & 0x7ff + b_l = (b_l & 0xf800) | (new >> 1) & 0x7ff + struct.pack_into('> 8 + l32r_imm16 = (l32r_imm16 + reloc >> 2) & 0xffff + l32r = l32r & 0xff | l32r_imm16 << 8 + pack_u24le(env.full_text, r_offset, l32r) + else: + assert 0, reloc_type + + # Log information about relocation + if log_name is None: + if s_type == 'STT_SECTION': + log_name = s.section.name + else: + log_name = s.name + log(LOG_LEVEL_3, ' {:08x} {} -> {:08x}'.format(r_offset, log_name, addr)) + +def do_relocation_data(env, text_addr, r): + s = r.sym + s_type = s.entry['st_info']['type'] + r_offset = r['r_offset'] + text_addr + r_info_type = r['r_info_type'] + try: + # only for RELA sections + r_addend = r['r_addend'] + except KeyError: + r_addend = 0 + + if (env.arch.name == 'EM_386' and r_info_type == R_386_32 + or env.arch.name == 'EM_X86_64' and r_info_type == R_X86_64_64 + or env.arch.name == 'EM_ARM' and r_info_type == R_ARM_ABS32 + or env.arch.name == 'EM_XTENSA' and r_info_type == R_XTENSA_32): + # Relocation in data.rel.ro to internal/external symbol + if env.arch.word_size == 4: + struct_type = ' {} {:08x}'.format(r_offset, log_name, addr)) + if env.arch.separate_rodata: + data = env.full_rodata + else: + data = env.full_text + existing, = struct.unpack_from(struct_type, data, r_offset) + if sec.name.startswith(('.text', '.rodata', '.data.rel.ro', '.bss')): + struct.pack_into(struct_type, data, r_offset, existing + addr) + kind = sec.name + elif sec.name == '.external.mp_fun_table': + assert addr == 0 + kind = s.mp_fun_table_offset + else: + assert 0, sec.name + if env.arch.separate_rodata: + base = '.rodata' + else: + base = '.text' + env.mpy_relocs.append((base, r_offset, kind)) + + else: + # Unknown/unsupported relocation + assert 0, r_info_type + +def load_object_file(env, felf): + with open(felf, 'rb') as f: + elf = elffile.ELFFile(f) + env.check_arch(elf['e_machine']) + + # Get symbol table + symtab = list(elf.get_section_by_name('.symtab').iter_symbols()) + + # Load needed sections from ELF file + sections_shndx = {} # maps elf shndx to Section object + for idx, s in enumerate(elf.iter_sections()): + if s.header.sh_type in ('SHT_PROGBITS', 'SHT_NOBITS'): + if s.data_size == 0: + # Ignore empty sections + pass + elif s.name.startswith(('.literal', '.text', '.rodata', '.data.rel.ro', '.bss')): + sec = Section.from_elfsec(s, felf) + sections_shndx[idx] = sec + if s.name.startswith('.literal'): + env.literal_sections.append(sec) + else: + env.sections.append(sec) + elif s.name.startswith('.data'): + raise LinkError('{}: {} non-empty'.format(felf, s.name)) + else: + # Ignore section + pass + elif s.header.sh_type in ('SHT_REL', 'SHT_RELA'): + shndx = s.header.sh_info + if shndx in sections_shndx: + sec = sections_shndx[shndx] + sec.reloc_name = s.name + sec.reloc = list(s.iter_relocations()) + for r in sec.reloc: + r.sym = symtab[r['r_info_sym']] + + # Link symbols to their sections, and update known and unresolved symbols + for sym in symtab: + sym.filename = felf + shndx = sym.entry['st_shndx'] + if shndx in sections_shndx: + # Symbol with associated section + sym.section = sections_shndx[shndx] + if sym['st_info']['bind'] == 'STB_GLOBAL': + # Defined global symbol + if sym.name in env.known_syms and not sym.name.startswith('__x86.get_pc_thunk.'): + raise LinkError('duplicate symbol: {}'.format(sym.name)) + env.known_syms[sym.name] = sym + elif sym.entry['st_shndx'] == 'SHN_UNDEF' and sym['st_info']['bind'] == 'STB_GLOBAL': + # Undefined global symbol, needs resolving + env.unresolved_syms.append(sym) + +def link_objects(env, native_qstr_vals_len, native_qstr_objs_len): + # Build GOT information + if env.arch.name == 'EM_XTENSA': + build_got_xtensa(env) + else: + build_got_generic(env) + + # Creat GOT section + got_size = len(env.got_entries) * env.arch.word_size + env.got_section = Section('GOT', bytearray(got_size), env.arch.word_size) + if env.arch.name == 'EM_XTENSA': + env.sections.insert(0, env.got_section) + else: + env.sections.append(env.got_section) + + # Create optional literal section + if env.arch.name == 'EM_XTENSA': + lit_size = len(env.lit_entries) * env.arch.word_size + env.lit_section = Section('LIT', bytearray(lit_size), env.arch.word_size) + env.sections.insert(1, env.lit_section) + + # Create section to contain mp_native_qstr_val_table + env.qstr_val_section = Section('.text.QSTR_VAL', bytearray(native_qstr_vals_len * env.arch.qstr_entry_size), env.arch.qstr_entry_size) + env.sections.append(env.qstr_val_section) + + # Create section to contain mp_native_qstr_obj_table + env.qstr_obj_section = Section('.text.QSTR_OBJ', bytearray(native_qstr_objs_len * env.arch.word_size), env.arch.word_size) + env.sections.append(env.qstr_obj_section) + + # Resolve unknown symbols + mp_fun_table_sec = Section('.external.mp_fun_table', b'', 0) + fun_table = {key: 63 + idx + for idx, key in enumerate([ + 'mp_type_type', + 'mp_type_str', + 'mp_type_list', + 'mp_type_dict', + 'mp_type_fun_builtin_0', + 'mp_type_fun_builtin_1', + 'mp_type_fun_builtin_2', + 'mp_type_fun_builtin_3', + 'mp_type_fun_builtin_var', + 'mp_stream_read_obj', + 'mp_stream_readinto_obj', + 'mp_stream_unbuffered_readline_obj', + 'mp_stream_write_obj', + ]) + } + for sym in env.unresolved_syms: + assert sym['st_value'] == 0 + if sym.name == '_GLOBAL_OFFSET_TABLE_': + pass + elif sym.name == 'mp_fun_table': + sym.section = Section('.external', b'', 0) + elif sym.name == 'mp_native_qstr_val_table': + sym.section = env.qstr_val_section + elif sym.name == 'mp_native_qstr_obj_table': + sym.section = env.qstr_obj_section + elif sym.name in env.known_syms: + sym.resolved = env.known_syms[sym.name] + else: + if sym.name in fun_table: + sym.section = mp_fun_table_sec + sym.mp_fun_table_offset = fun_table[sym.name] + else: + raise LinkError('{}: undefined symbol: {}'.format(sym.filename, sym.name)) + + # Align sections, assign their addresses, and create full_text + env.full_text = bytearray(env.arch.asm_jump(8)) # dummy, to be filled in later + env.full_rodata = bytearray(0) + env.full_bss = bytearray(0) + for sec in env.sections: + if env.arch.separate_rodata and sec.name.startswith(('.rodata', '.data.rel.ro')): + data = env.full_rodata + elif sec.name.startswith('.bss'): + data = env.full_bss + else: + data = env.full_text + sec.addr = align_to(len(data), sec.alignment) + data.extend(b'\x00' * (sec.addr - len(data))) + data.extend(sec.data) + + env.print_sections() + + populate_got(env) + if env.arch.name == 'EM_XTENSA': + populate_lit(env) + + # Fill in relocations + for sec in env.sections: + if not sec.reloc: + continue + log(LOG_LEVEL_3, '{}: {} relocations via {}:'.format(sec.filename, sec.name, sec.reloc_name)) + for r in sec.reloc: + if sec.name.startswith(('.text', '.rodata')): + do_relocation_text(env, sec.addr, r) + elif sec.name.startswith('.data.rel.ro'): + do_relocation_data(env, sec.addr, r) + else: + assert 0, sec.name + +################################################################################ +# .mpy output + +class MPYOutput: + def open(self, fname): + self.f = open(fname, 'wb') + self.prev_base = -1 + self.prev_offset = -1 + + def close(self): + self.f.close() + + def write_bytes(self, buf): + self.f.write(buf) + + def write_uint(self, val): + b = bytearray() + b.insert(0, val & 0x7f) + val >>= 7 + while val: + b.insert(0, 0x80 | (val & 0x7f)) + val >>= 7 + self.write_bytes(b) + + def write_qstr(self, s): + if s in qstrutil.static_qstr_list: + self.write_bytes(bytes([0, qstrutil.static_qstr_list.index(s) + 1])) + else: + s = bytes(s, 'ascii') + self.write_uint(len(s) << 1) + self.write_bytes(s) + + def write_obj(self, o): + if o is Ellipsis: + self.write_bytes(b'e') + return + else: + # Other Python types not implemented + assert 0 + + def write_reloc(self, base, offset, dest, n): + need_offset = not (base == self.prev_base and offset == self.prev_offset + 1) + self.prev_offset = offset + n - 1 + if dest <= 2: + dest = (dest << 1) | (n > 1) + else: + assert 6 <= dest <= 127 + assert n == 1 + dest = dest << 1 | need_offset + assert 0 <= dest <= 0xfe, dest + self.write_bytes(bytes([dest])) + if need_offset: + if base == '.text': + base = 0 + elif base == '.rodata': + base = 1 + self.write_uint(offset << 1 | base) + if n > 1: + self.write_uint(n) + +def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): + # Write jump instruction to start of text + jump = env.arch.asm_jump(entry_offset) + env.full_text[:len(jump)] = jump + + log(LOG_LEVEL_1, 'arch: {}'.format(env.arch.name)) + log(LOG_LEVEL_1, 'text size: {}'.format(len(env.full_text))) + if len(env.full_rodata): + log(LOG_LEVEL_1, 'rodata size: {}'.format(len(env.full_rodata))) + log(LOG_LEVEL_1, 'bss size: {}'.format(len(env.full_bss))) + log(LOG_LEVEL_1, 'GOT entries: {}'.format(len(env.got_entries))) + + #xxd(env.full_text) + + out = MPYOutput() + out.open(fmpy) + + # MPY: header + out.write_bytes(bytearray([ + ord('M'), + MPY_VERSION, + env.arch.mpy_feature, + MP_SMALL_INT_BITS, + QSTR_WINDOW_SIZE, + ])) + + # MPY: kind/len + out.write_uint(len(env.full_text) << 2 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE)) + + # MPY: machine code + out.write_bytes(env.full_text) + + # MPY: n_qstr_link (assumes little endian) + out.write_uint(len(native_qstr_vals) + len(native_qstr_objs)) + for q in range(len(native_qstr_vals)): + off = env.qstr_val_section.addr + q * env.arch.qstr_entry_size + out.write_uint(off << 2) + out.write_qstr(native_qstr_vals[q]) + for q in range(len(native_qstr_objs)): + off = env.qstr_obj_section.addr + q * env.arch.word_size + out.write_uint(off << 2 | 3) + out.write_qstr(native_qstr_objs[q]) + + # MPY: scope_flags + scope_flags = MP_SCOPE_FLAG_VIPERRELOC + if len(env.full_rodata): + scope_flags |= MP_SCOPE_FLAG_VIPERRODATA + if len(env.full_bss): + scope_flags |= MP_SCOPE_FLAG_VIPERBSS + out.write_uint(scope_flags) + + # MPY: n_obj + out.write_uint(bool(len(env.full_rodata)) + bool(len(env.full_bss))) + + # MPY: n_raw_code + out.write_uint(0) + + # MPY: optional bytes object with rodata + if len(env.full_rodata): + out.write_obj(Ellipsis) + if len(env.full_bss): + out.write_obj(Ellipsis) + + # MPY: rodata and/or bss + if len(env.full_rodata): + rodata_const_table_idx = 1 + out.write_uint(len(env.full_rodata)) + out.write_bytes(env.full_rodata) + if len(env.full_bss): + bss_const_table_idx = bool(env.full_rodata) + 1 + out.write_uint(len(env.full_bss)) + + # MPY: relocation information + prev_kind = None + for base, addr, kind in env.mpy_relocs: + if isinstance(kind, str) and kind.startswith('.text'): + kind = 0 + elif kind in ('.rodata', '.data.rel.ro'): + if env.arch.separate_rodata: + kind = rodata_const_table_idx + else: + kind = 0 + elif isinstance(kind, str) and kind.startswith('.bss'): + kind = bss_const_table_idx + elif kind == 'mp_fun_table': + kind = 6 + else: + kind = 7 + kind + assert addr % env.arch.word_size == 0, addr + offset = addr // env.arch.word_size + if kind == prev_kind and base == prev_base and offset == prev_offset + 1: + prev_n += 1 + prev_offset += 1 + else: + if prev_kind is not None: + out.write_reloc(prev_base, prev_offset - prev_n + 1, prev_kind, prev_n) + prev_kind = kind + prev_base = base + prev_offset = offset + prev_n = 1 + if prev_kind is not None: + out.write_reloc(prev_base, prev_offset - prev_n + 1, prev_kind, prev_n) + + # MPY: sentinel for end of relocations + out.write_bytes(b'\xff') + + out.close() + +################################################################################ +# main + +def do_preprocess(args): + if args.output is None: + assert args.files[0].endswith('.c') + args.output = args.files[0][:-1] + 'config.h' + static_qstrs, qstr_vals, qstr_objs = extract_qstrs(args.files) + with open(args.output, 'w') as f: + print('#include \n' + 'typedef uintptr_t mp_uint_t;\n' + 'typedef intptr_t mp_int_t;\n' + 'typedef uintptr_t mp_off_t;', file=f) + for i, q in enumerate(static_qstrs): + print('#define %s (%u)' % (q, i + 1), file=f) + for i, q in enumerate(sorted(qstr_vals)): + print('#define %s (mp_native_qstr_val_table[%d])' % (q, i), file=f) + for i, q in enumerate(sorted(qstr_objs)): + print('#define MP_OBJ_NEW_QSTR_%s ((mp_obj_t)mp_native_qstr_obj_table[%d])' % (q, i), file=f) + if args.arch == 'xtensawin': + qstr_type = 'uint32_t' # esp32 can only read 32-bit values from IRAM + else: + qstr_type = 'uint16_t' + print('extern const {} mp_native_qstr_val_table[];'.format(qstr_type), file=f) + print('extern const mp_uint_t mp_native_qstr_obj_table[];', file=f) + +def do_link(args): + if args.output is None: + assert args.files[0].endswith('.o') + args.output = args.files[0][:-1] + 'mpy' + native_qstr_vals = [] + native_qstr_objs = [] + if args.qstrs is not None: + with open(args.qstrs) as f: + for l in f: + m = re.match(r'#define MP_QSTR_([A-Za-z0-9_]*) \(mp_native_', l) + if m: + native_qstr_vals.append(m.group(1)) + else: + m = re.match(r'#define MP_OBJ_NEW_QSTR_MP_QSTR_([A-Za-z0-9_]*)', l) + if m: + native_qstr_objs.append(m.group(1)) + log(LOG_LEVEL_2, 'qstr vals: ' + ', '.join(native_qstr_vals)) + log(LOG_LEVEL_2, 'qstr objs: ' + ', '.join(native_qstr_objs)) + env = LinkEnv(args.arch) + try: + for file in args.files: + load_object_file(env, file) + link_objects(env, len(native_qstr_vals), len(native_qstr_objs)) + build_mpy(env, env.find_addr('mpy_init'), args.output, native_qstr_vals, native_qstr_objs) + except LinkError as er: + print('LinkError:', er.args[0]) + sys.exit(1) + +def main(): + import argparse + cmd_parser = argparse.ArgumentParser(description='Run scripts on the pyboard.') + cmd_parser.add_argument('--verbose', '-v', action='count', default=1, help='increase verbosity') + cmd_parser.add_argument('--arch', default='x64', help='architecture') + cmd_parser.add_argument('--preprocess', action='store_true', help='preprocess source files') + cmd_parser.add_argument('--qstrs', default=None, help='file defining additional qstrs') + cmd_parser.add_argument('--output', '-o', default=None, help='output .mpy file (default to input with .o->.mpy)') + cmd_parser.add_argument('files', nargs='+', help='input files') + args = cmd_parser.parse_args() + + global log_level + log_level = args.verbose + + if args.preprocess: + do_preprocess(args) + else: + do_link(args) + +if __name__ == '__main__': + main()