diff --git a/selfdrive/tombstoned.py b/selfdrive/tombstoned.py old mode 100644 new mode 100755 index 55a09233..c98a373b --- a/selfdrive/tombstoned.py +++ b/selfdrive/tombstoned.py @@ -1,7 +1,6 @@ +#!/usr/bin/env python3 import os -import re import time -import datetime from raven import Client from raven.transport.http import HTTPTransport @@ -9,96 +8,41 @@ from raven.transport.http import HTTPTransport from selfdrive.version import version, dirty from selfdrive.swaglog import cloudlog +MAX_SIZE = 100000 * 10 # Normal size is 40-100k, allow up to 1M + + def get_tombstones(): + """Returns list of (c_time, filename) for all tombstones in /data/tombstones""" DIR_DATA = "/data/tombstones/" - return [(DIR_DATA + fn, int(os.stat(DIR_DATA + fn).st_ctime) ) + return [(DIR_DATA + fn, int(os.stat(DIR_DATA + fn).st_ctime)) for fn in os.listdir(DIR_DATA) if fn.startswith("tombstone")] + def report_tombstone(fn, client): - mtime = os.path.getmtime(fn) + f_size = os.path.getsize(fn) + if f_size > MAX_SIZE: + cloudlog.error(f"Tombstone {fn} too big, {f_size}. Skipping...") + with open(fn, encoding='ISO-8859-1') as f: - dat = f.read() + contents = f.read() - # see system/core/debuggerd/tombstone.cpp - parsed = re.match(r"[* ]*\n" - r"(?P
CM Version:[\s\S]*?ABI:.*\n)" - r"(?Ppid:.*\n)" - r"(?Psignal.*\n)?" - r"(?PAbort.*\n)?" - r"(?P\s+x0[\s\S]*?\n)\n" - r"(?:backtrace:\n" - r"(?P[\s\S]*?\n)\n" - r"stack:\n" - r"(?P[\s\S]*?\n)\n" - r")?", dat) - - logtail = re.search(r"--------- tail end of.*\n([\s\S]*?\n)---", dat) - logtail = logtail and logtail.group(1) - - if parsed: - parsedict = parsed.groupdict() - else: - parsedict = {} - - thread_line = parsedict.get('thread', '') - thread_parsed = re.match(r'pid: (?P\d+), tid: (?P\d+), name: (?P.*) >>> (?P.*) <<<', thread_line) - if thread_parsed: - thread_parseddict = thread_parsed.groupdict() - else: - thread_parseddict = {} - pid = thread_parseddict.get('pid', '') - tid = thread_parseddict.get('tid', '') - name = thread_parseddict.get('name', 'unknown') - cmd = thread_parseddict.get('cmd', 'unknown') - - signal_line = parsedict.get('signal', '') - signal_parsed = re.match(r'signal (?P.*?), code (?P.*?), fault addr (?P.*)\n', signal_line) - if signal_parsed: - signal_parseddict = signal_parsed.groupdict() - else: - signal_parseddict = {} - signal = signal_parseddict.get('signal', 'unknown') - code = signal_parseddict.get('code', 'unknown') - fault_addr = signal_parseddict.get('fault_addr', '') - - abort_line = parsedict.get('abort', '') - - if parsed: - message = 'Process {} ({}) got signal {} code {}'.format(name, cmd, signal, code) - if abort_line: - message += '\n'+abort_line - else: - message = fn+'\n'+dat[:1024] + # Get summary for sentry title + message = " ".join(contents.split('\n')[5:7]) + # Cut off pid/tid, since that varies per run + name_idx = message.find('name') + if name_idx >= 0: + message = message[name_idx:] + cloudlog.error({'tombstone': message}) client.captureMessage( message=message, - date=datetime.datetime.utcfromtimestamp(mtime), - data={ - 'logger':'tombstoned', - 'platform':'other', - }, sdk={'name': 'tombstoned', 'version': '0'}, extra={ - 'fault_addr': fault_addr, - 'abort_msg': abort_line, - 'pid': pid, - 'tid': tid, - 'name':'{} ({})'.format(name, cmd), 'tombstone_fn': fn, - 'header': parsedict.get('header'), - 'registers': parsedict.get('registers'), - 'backtrace': parsedict.get('backtrace'), - 'logtail': logtail, - }, - tags={ - 'name':'{} ({})'.format(name, cmd), - 'signal':signal, - 'code':code, - 'fault_addr':fault_addr, + 'tombstone': contents }, ) - cloudlog.error({'tombstone': message}) def main(): @@ -112,11 +56,12 @@ def main(): now_tombstones = set(get_tombstones()) for fn, ctime in (now_tombstones - initial_tombstones): - cloudlog.info("reporting new tombstone %s", fn) + cloudlog.info(f"reporting new tombstone {fn}") report_tombstone(fn, client) initial_tombstones = now_tombstones time.sleep(5) + if __name__ == "__main__": main()