diff --git a/py/objstr.c b/py/objstr.c
index ddad7d3bd..cc3dda59e 100644
--- a/py/objstr.c
+++ b/py/objstr.c
@@ -347,8 +347,9 @@ mp_obj_t mp_obj_str_binary_op(mp_uint_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) {
         rhs_data = bufinfo.buf;
         rhs_len = bufinfo.len;
     } else {
-        // incompatible types
-        return MP_OBJ_NULL; // op not supported
+        // LHS is str and RHS has an incompatible type
+        // (except if operation is EQUAL, but that's handled by mp_obj_equal)
+        bad_implicit_conversion(rhs_in);
     }
 
     switch (op) {
diff --git a/tests/basics/containment.py b/tests/basics/containment.py
index bae366113..4c94a9bae 100644
--- a/tests/basics/containment.py
+++ b/tests/basics/containment.py
@@ -16,6 +16,17 @@ for needle in [haystack[:i+1] for i in range(len(haystack))]:
     print(haystack, "in", needle, "::", haystack in needle)
     print(haystack, "not in", needle, "::", haystack not in needle)
 
+# containment of bytes/ints in bytes
+print(b'' in b'123')
+print(b'0' in b'123', b'1' in b'123')
+print(48 in b'123', 49 in b'123')
+
+# containment of int in str is an error
+try:
+    1 in '123'
+except TypeError:
+    print('TypeError')
+
 # until here, the tests would work without the 'second attempt' iteration thing.
 
 for i in 1, 2: