From 2069c563f9e944d8f45e524b425fff23208e8153 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 25 Jul 2019 13:15:54 +1000 Subject: [PATCH] py: Add support for matmul operator @ as per PEP 465. To make progress towards MicroPython supporting Python 3.5, adding the matmul operator is important because it's a really "low level" part of the language, being a new token and modifications to the grammar. It doesn't make sense to make it configurable because 1) it would make the grammar and lexer complicated/messy; 2) no other operators are configurable; 3) it's not a feature that can be "dynamically plugged in" via an import. And matmul can be useful as a general purpose user-defined operator, it doesn't have to be just for numpy use. Based on work done by Jim Mussared. --- py/grammar.h | 10 +++++----- py/lexer.c | 6 ++++-- py/lexer.h | 7 ++++--- py/objtype.c | 3 +++ py/parse.c | 14 +++++++------- py/runtime0.h | 7 +++++-- tests/cmdline/cmd_showbc.py.exp | 12 ++++++------ 7 files changed, 34 insertions(+), 25 deletions(-) diff --git a/py/grammar.h b/py/grammar.h index 5a5b682ac..ca9b5b379 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -55,7 +55,7 @@ DEF_RULE_NC(eval_input_2, and(1), tok(NEWLINE)) // varargslist: vfpdef ['=' test] (',' vfpdef ['=' test])* [',' ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef // vfpdef: NAME -DEF_RULE_NC(decorator, and(4), tok(DEL_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE)) +DEF_RULE_NC(decorator, and(4), tok(OP_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE)) DEF_RULE_NC(decorators, one_or_more, rule(decorator)) DEF_RULE(decorated, c(decorated), and_ident(2), rule(decorators), rule(decorated_body)) #if MICROPY_PY_ASYNC_AWAIT @@ -96,7 +96,7 @@ DEF_RULE(simple_stmt_2, c(generic_all_nodes), list_with_end, rule(small_stmt), t // small_stmt: expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt // expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*) // testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] -// augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=' +// augassign: '+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=' // # For normal assignments, additional restrictions enforced by the interpreter DEF_RULE_NC(small_stmt, or(8), rule(del_stmt), rule(pass_stmt), rule(flow_stmt), rule(import_stmt), rule(global_stmt), rule(nonlocal_stmt), rule(assert_stmt), rule(expr_stmt)) @@ -108,7 +108,7 @@ DEF_RULE_NC(expr_stmt_assign, and_ident(2), tok(DEL_EQUAL), rule(expr_stmt_6)) DEF_RULE_NC(expr_stmt_6, or(2), rule(yield_expr), rule(testlist_star_expr)) DEF_RULE(testlist_star_expr, c(generic_tuple), list_with_end, rule(testlist_star_expr_2), tok(DEL_COMMA)) DEF_RULE_NC(testlist_star_expr_2, or(2), rule(star_expr), rule(test)) -DEF_RULE_NC(augassign, or(12), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DEL_STAR_EQUAL), tok(DEL_SLASH_EQUAL), tok(DEL_PERCENT_EQUAL), tok(DEL_AMPERSAND_EQUAL), tok(DEL_PIPE_EQUAL), tok(DEL_CARET_EQUAL), tok(DEL_DBL_LESS_EQUAL), tok(DEL_DBL_MORE_EQUAL), tok(DEL_DBL_STAR_EQUAL), tok(DEL_DBL_SLASH_EQUAL)) +DEF_RULE_NC(augassign, or(13), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DEL_STAR_EQUAL), tok(DEL_AT_EQUAL), tok(DEL_SLASH_EQUAL), tok(DEL_PERCENT_EQUAL), tok(DEL_AMPERSAND_EQUAL), tok(DEL_PIPE_EQUAL), tok(DEL_CARET_EQUAL), tok(DEL_DBL_LESS_EQUAL), tok(DEL_DBL_MORE_EQUAL), tok(DEL_DBL_STAR_EQUAL), tok(DEL_DBL_SLASH_EQUAL)) // del_stmt: 'del' exprlist // pass_stmt: 'pass' @@ -226,7 +226,7 @@ DEF_RULE(lambdef_nocond, c(lambdef), and_blank(4), tok(KW_LAMBDA), opt_rule(vara // and_expr: shift_expr ('&' shift_expr)* // shift_expr: arith_expr (('<<'|'>>') arith_expr)* // arith_expr: term (('+'|'-') term)* -// term: factor (('*'|'/'|'%'|'//') factor)* +// term: factor (('*'|'@'|'/'|'%'|'//') factor)* // factor: ('+'|'-'|'~') factor | power // power: atom_expr ['**' factor] // atom_expr: 'await' atom trailer* | atom trailer* @@ -249,7 +249,7 @@ DEF_RULE_NC(shift_op, or(2), tok(OP_DBL_LESS), tok(OP_DBL_MORE)) DEF_RULE(arith_expr, c(term), list, rule(term), rule(arith_op)) DEF_RULE_NC(arith_op, or(2), tok(OP_PLUS), tok(OP_MINUS)) DEF_RULE(term, c(term), list, rule(factor), rule(term_op)) -DEF_RULE_NC(term_op, or(4), tok(OP_STAR), tok(OP_SLASH), tok(OP_PERCENT), tok(OP_DBL_SLASH)) +DEF_RULE_NC(term_op, or(5), tok(OP_STAR), tok(OP_AT), tok(OP_SLASH), tok(OP_PERCENT), tok(OP_DBL_SLASH)) DEF_RULE_NC(factor, or(2), rule(factor_2), rule(power)) DEF_RULE(factor_2, c(factor_2), and_ident(2), rule(factor_op), rule(factor)) DEF_RULE_NC(factor_op, or(3), tok(OP_PLUS), tok(OP_MINUS), tok(OP_TILDE)) diff --git a/py/lexer.c b/py/lexer.c index e161700b1..5f8adda91 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -174,7 +174,7 @@ STATIC void indent_pop(mp_lexer_t *lex) { // this means if the start of two ops are the same then they are equal til the last char STATIC const char *const tok_enc = - "()[]{},:;@~" // singles + "()[]{},:;~" // singles " >= >> >>= "*e=c*e=" // * *= ** **= @@ -185,6 +185,7 @@ STATIC const char *const tok_enc = "/e=c/e=" // / /= // //= "%e=" // % %= "^e=" // ^ ^= + "@e=" // @ @= "=e=" // = == "!."; // start of special cases: != . ... @@ -193,7 +194,7 @@ STATIC const uint8_t tok_enc_kind[] = { MP_TOKEN_DEL_PAREN_OPEN, MP_TOKEN_DEL_PAREN_CLOSE, MP_TOKEN_DEL_BRACKET_OPEN, MP_TOKEN_DEL_BRACKET_CLOSE, MP_TOKEN_DEL_BRACE_OPEN, MP_TOKEN_DEL_BRACE_CLOSE, - MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_DEL_AT, MP_TOKEN_OP_TILDE, + MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_OP_TILDE, MP_TOKEN_OP_LESS, MP_TOKEN_OP_LESS_EQUAL, MP_TOKEN_OP_DBL_LESS, MP_TOKEN_DEL_DBL_LESS_EQUAL, MP_TOKEN_OP_MORE, MP_TOKEN_OP_MORE_EQUAL, MP_TOKEN_OP_DBL_MORE, MP_TOKEN_DEL_DBL_MORE_EQUAL, @@ -205,6 +206,7 @@ STATIC const uint8_t tok_enc_kind[] = { MP_TOKEN_OP_SLASH, MP_TOKEN_DEL_SLASH_EQUAL, MP_TOKEN_OP_DBL_SLASH, MP_TOKEN_DEL_DBL_SLASH_EQUAL, MP_TOKEN_OP_PERCENT, MP_TOKEN_DEL_PERCENT_EQUAL, MP_TOKEN_OP_CARET, MP_TOKEN_DEL_CARET_EQUAL, + MP_TOKEN_OP_AT, MP_TOKEN_DEL_AT_EQUAL, MP_TOKEN_DEL_EQUAL, MP_TOKEN_OP_DBL_EQUAL, }; diff --git a/py/lexer.h b/py/lexer.h index 321185f10..34126a08f 100644 --- a/py/lexer.h +++ b/py/lexer.h @@ -106,7 +106,7 @@ typedef enum _mp_token_kind_t { MP_TOKEN_OP_MORE_EQUAL, MP_TOKEN_OP_NOT_EQUAL, - // Order of these 12 matches corresponding mp_binary_op_t operator + // Order of these 13 matches corresponding mp_binary_op_t operator MP_TOKEN_OP_PIPE, MP_TOKEN_OP_CARET, MP_TOKEN_OP_AMPERSAND, @@ -115,12 +115,13 @@ typedef enum _mp_token_kind_t { MP_TOKEN_OP_PLUS, MP_TOKEN_OP_MINUS, MP_TOKEN_OP_STAR, + MP_TOKEN_OP_AT, MP_TOKEN_OP_DBL_SLASH, MP_TOKEN_OP_SLASH, MP_TOKEN_OP_PERCENT, MP_TOKEN_OP_DBL_STAR, - // Order of these 12 matches corresponding mp_binary_op_t operator + // Order of these 13 matches corresponding mp_binary_op_t operator MP_TOKEN_DEL_PIPE_EQUAL, MP_TOKEN_DEL_CARET_EQUAL, MP_TOKEN_DEL_AMPERSAND_EQUAL, @@ -129,6 +130,7 @@ typedef enum _mp_token_kind_t { MP_TOKEN_DEL_PLUS_EQUAL, MP_TOKEN_DEL_MINUS_EQUAL, MP_TOKEN_DEL_STAR_EQUAL, + MP_TOKEN_DEL_AT_EQUAL, MP_TOKEN_DEL_DBL_SLASH_EQUAL, MP_TOKEN_DEL_SLASH_EQUAL, MP_TOKEN_DEL_PERCENT_EQUAL, @@ -144,7 +146,6 @@ typedef enum _mp_token_kind_t { MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_PERIOD, MP_TOKEN_DEL_SEMICOLON, - MP_TOKEN_DEL_AT, MP_TOKEN_DEL_EQUAL, MP_TOKEN_DEL_MINUS_MORE, } mp_token_kind_t; diff --git a/py/objtype.c b/py/objtype.c index 3e65a32f0..318c7b9a3 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -478,6 +478,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = { [MP_BINARY_OP_INPLACE_SUBTRACT] = MP_QSTR___isub__, #if MICROPY_PY_ALL_INPLACE_SPECIAL_METHODS [MP_BINARY_OP_INPLACE_MULTIPLY] = MP_QSTR___imul__, + [MP_BINARY_OP_INPLACE_MAT_MULTIPLY] = MP_QSTR___imatmul__, [MP_BINARY_OP_INPLACE_FLOOR_DIVIDE] = MP_QSTR___ifloordiv__, [MP_BINARY_OP_INPLACE_TRUE_DIVIDE] = MP_QSTR___itruediv__, [MP_BINARY_OP_INPLACE_MODULO] = MP_QSTR___imod__, @@ -493,6 +494,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = { [MP_BINARY_OP_SUBTRACT] = MP_QSTR___sub__, #if MICROPY_PY_ALL_SPECIAL_METHODS [MP_BINARY_OP_MULTIPLY] = MP_QSTR___mul__, + [MP_BINARY_OP_MAT_MULTIPLY] = MP_QSTR___matmul__, [MP_BINARY_OP_FLOOR_DIVIDE] = MP_QSTR___floordiv__, [MP_BINARY_OP_TRUE_DIVIDE] = MP_QSTR___truediv__, [MP_BINARY_OP_MODULO] = MP_QSTR___mod__, @@ -510,6 +512,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = { [MP_BINARY_OP_REVERSE_SUBTRACT] = MP_QSTR___rsub__, #if MICROPY_PY_ALL_SPECIAL_METHODS [MP_BINARY_OP_REVERSE_MULTIPLY] = MP_QSTR___rmul__, + [MP_BINARY_OP_REVERSE_MAT_MULTIPLY] = MP_QSTR___rmatmul__, [MP_BINARY_OP_REVERSE_FLOOR_DIVIDE] = MP_QSTR___rfloordiv__, [MP_BINARY_OP_REVERSE_TRUE_DIVIDE] = MP_QSTR___rtruediv__, [MP_BINARY_OP_REVERSE_MODULO] = MP_QSTR___rmod__, diff --git a/py/parse.c b/py/parse.c index b9b578545..254dbc7ba 100644 --- a/py/parse.c +++ b/py/parse.c @@ -135,8 +135,8 @@ STATIC const uint16_t rule_arg_combined_table[] = { #define RULE_EXPAND(x) x #define RULE_PADDING(rule, ...) RULE_PADDING2(rule, __VA_ARGS__, RULE_PADDING_IDS(rule)) #define RULE_PADDING2(rule, ...) RULE_EXPAND(RULE_PADDING3(rule, __VA_ARGS__)) -#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, ...) __VA_ARGS__ -#define RULE_PADDING_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, +#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) __VA_ARGS__ +#define RULE_PADDING_IDS(r) PAD13_##r, PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, // Use an enum to create constants specifying how much room a rule takes in rule_arg_combined_table enum { @@ -155,8 +155,8 @@ enum { // Macro to compute the start of a rule in rule_arg_combined_table #define RULE_ARG_OFFSET(rule, ...) RULE_ARG_OFFSET2(rule, __VA_ARGS__, RULE_ARG_OFFSET_IDS(rule)) #define RULE_ARG_OFFSET2(rule, ...) RULE_EXPAND(RULE_ARG_OFFSET3(rule, __VA_ARGS__)) -#define RULE_ARG_OFFSET3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) _13 -#define RULE_ARG_OFFSET_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, PAD0_##r, +#define RULE_ARG_OFFSET3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, ...) _14 +#define RULE_ARG_OFFSET_IDS(r) PAD13_##r, PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, PAD0_##r, // Use the above enum values to create a table of offsets for each rule's arg // data, which indexes rule_arg_combined_table. The offsets require 9 bits of @@ -632,7 +632,7 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { } else if (rule_id == RULE_shift_expr || rule_id == RULE_arith_expr || rule_id == RULE_term) { - // folding for binary ops: << >> + - * / % // + // folding for binary ops: << >> + - * @ / % // mp_parse_node_t pn = peek_result(parser, num_args - 1); if (!mp_parse_node_get_int_maybe(pn, &arg0)) { return false; @@ -644,8 +644,8 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { return false; } mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(peek_result(parser, i)); - if (tok == MP_TOKEN_OP_SLASH || tok == MP_TOKEN_OP_DBL_STAR) { - // Can't fold / or ** + if (tok == MP_TOKEN_OP_AT || tok == MP_TOKEN_OP_SLASH || tok == MP_TOKEN_OP_DBL_STAR) { + // Can't fold @ or / or ** return false; } mp_binary_op_t op = MP_BINARY_OP_LSHIFT + (tok - MP_TOKEN_OP_DBL_LESS); diff --git a/py/runtime0.h b/py/runtime0.h index e80e272ae..3ba089bb3 100644 --- a/py/runtime0.h +++ b/py/runtime0.h @@ -81,7 +81,7 @@ typedef enum { MP_BINARY_OP_IS, MP_BINARY_OP_EXCEPTION_MATCH, - // 12 inplace arithmetic operations; order matches corresponding mp_token_kind_t + // 13 inplace arithmetic operations; order matches corresponding mp_token_kind_t MP_BINARY_OP_INPLACE_OR, MP_BINARY_OP_INPLACE_XOR, MP_BINARY_OP_INPLACE_AND, @@ -90,12 +90,13 @@ typedef enum { MP_BINARY_OP_INPLACE_ADD, MP_BINARY_OP_INPLACE_SUBTRACT, MP_BINARY_OP_INPLACE_MULTIPLY, + MP_BINARY_OP_INPLACE_MAT_MULTIPLY, MP_BINARY_OP_INPLACE_FLOOR_DIVIDE, MP_BINARY_OP_INPLACE_TRUE_DIVIDE, MP_BINARY_OP_INPLACE_MODULO, MP_BINARY_OP_INPLACE_POWER, - // 12 normal arithmetic operations; order matches corresponding mp_token_kind_t + // 13 normal arithmetic operations; order matches corresponding mp_token_kind_t MP_BINARY_OP_OR, MP_BINARY_OP_XOR, MP_BINARY_OP_AND, @@ -104,6 +105,7 @@ typedef enum { MP_BINARY_OP_ADD, MP_BINARY_OP_SUBTRACT, MP_BINARY_OP_MULTIPLY, + MP_BINARY_OP_MAT_MULTIPLY, MP_BINARY_OP_FLOOR_DIVIDE, MP_BINARY_OP_TRUE_DIVIDE, MP_BINARY_OP_MODULO, @@ -123,6 +125,7 @@ typedef enum { MP_BINARY_OP_REVERSE_ADD, MP_BINARY_OP_REVERSE_SUBTRACT, MP_BINARY_OP_REVERSE_MULTIPLY, + MP_BINARY_OP_REVERSE_MAT_MULTIPLY, MP_BINARY_OP_REVERSE_FLOOR_DIVIDE, MP_BINARY_OP_REVERSE_TRUE_DIVIDE, MP_BINARY_OP_REVERSE_MODULO, diff --git a/tests/cmdline/cmd_showbc.py.exp b/tests/cmdline/cmd_showbc.py.exp index 36f040438..5e8f762ef 100644 --- a/tests/cmdline/cmd_showbc.py.exp +++ b/tests/cmdline/cmd_showbc.py.exp @@ -43,9 +43,9 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): bc=\\d\+ line=126 00 LOAD_CONST_NONE 01 LOAD_CONST_FALSE -02 BINARY_OP 26 __add__ +02 BINARY_OP 27 __add__ 03 LOAD_CONST_TRUE -04 BINARY_OP 26 __add__ +04 BINARY_OP 27 __add__ 05 STORE_FAST 0 06 LOAD_CONST_SMALL_INT 0 07 STORE_FAST 0 @@ -84,7 +84,7 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): \\d\+ STORE_FAST 7 \\d\+ LOAD_FAST 0 \\d\+ LOAD_DEREF 14 -\\d\+ BINARY_OP 26 __add__ +58 BINARY_OP 27 __add__ \\d\+ STORE_FAST 8 \\d\+ LOAD_FAST 0 \\d\+ UNARY_OP 1 @@ -363,7 +363,7 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): 42 STORE_FAST_N 19 44 LOAD_FAST 9 45 LOAD_FAST_N 19 -47 BINARY_OP 26 __add__ +47 BINARY_OP 27 __add__ 48 POP_TOP 49 LOAD_CONST_NONE 50 RETURN_VALUE @@ -520,7 +520,7 @@ arg names: * bc=\\d\+ line=113 00 LOAD_DEREF 0 02 LOAD_CONST_SMALL_INT 1 -03 BINARY_OP 26 __add__ +03 BINARY_OP 27 __add__ 04 STORE_FAST 1 05 LOAD_CONST_SMALL_INT 1 06 STORE_DEREF 0 @@ -539,7 +539,7 @@ arg names: * b bc=\\d\+ line=139 00 LOAD_FAST 1 01 LOAD_DEREF 0 -03 BINARY_OP 26 __add__ +03 BINARY_OP 27 __add__ 04 RETURN_VALUE mem: total=\\d\+, current=\\d\+, peak=\\d\+ stack: \\d\+ out of \\d\+