Simplify tombstoned, should fix #1258
parent
106be23c4e
commit
d8d2d568dc
|
@ -1,7 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import datetime
|
||||
|
||||
from raven import Client
|
||||
from raven.transport.http import HTTPTransport
|
||||
|
@ -9,96 +8,41 @@ from raven.transport.http import HTTPTransport
|
|||
from selfdrive.version import version, dirty
|
||||
from selfdrive.swaglog import cloudlog
|
||||
|
||||
MAX_SIZE = 100000 * 10 # Normal size is 40-100k, allow up to 1M
|
||||
|
||||
|
||||
def get_tombstones():
|
||||
"""Returns list of (c_time, filename) for all tombstones in /data/tombstones"""
|
||||
DIR_DATA = "/data/tombstones/"
|
||||
return [(DIR_DATA + fn, int(os.stat(DIR_DATA + fn).st_ctime) )
|
||||
return [(DIR_DATA + fn, int(os.stat(DIR_DATA + fn).st_ctime))
|
||||
for fn in os.listdir(DIR_DATA) if fn.startswith("tombstone")]
|
||||
|
||||
|
||||
def report_tombstone(fn, client):
|
||||
mtime = os.path.getmtime(fn)
|
||||
f_size = os.path.getsize(fn)
|
||||
if f_size > MAX_SIZE:
|
||||
cloudlog.error(f"Tombstone {fn} too big, {f_size}. Skipping...")
|
||||
|
||||
with open(fn, encoding='ISO-8859-1') as f:
|
||||
dat = f.read()
|
||||
contents = f.read()
|
||||
|
||||
# see system/core/debuggerd/tombstone.cpp
|
||||
parsed = re.match(r"[* ]*\n"
|
||||
r"(?P<header>CM Version:[\s\S]*?ABI:.*\n)"
|
||||
r"(?P<thread>pid:.*\n)"
|
||||
r"(?P<signal>signal.*\n)?"
|
||||
r"(?P<abort>Abort.*\n)?"
|
||||
r"(?P<registers>\s+x0[\s\S]*?\n)\n"
|
||||
r"(?:backtrace:\n"
|
||||
r"(?P<backtrace>[\s\S]*?\n)\n"
|
||||
r"stack:\n"
|
||||
r"(?P<stack>[\s\S]*?\n)\n"
|
||||
r")?", dat)
|
||||
|
||||
logtail = re.search(r"--------- tail end of.*\n([\s\S]*?\n)---", dat)
|
||||
logtail = logtail and logtail.group(1)
|
||||
|
||||
if parsed:
|
||||
parsedict = parsed.groupdict()
|
||||
else:
|
||||
parsedict = {}
|
||||
|
||||
thread_line = parsedict.get('thread', '')
|
||||
thread_parsed = re.match(r'pid: (?P<pid>\d+), tid: (?P<tid>\d+), name: (?P<name>.*) >>> (?P<cmd>.*) <<<', thread_line)
|
||||
if thread_parsed:
|
||||
thread_parseddict = thread_parsed.groupdict()
|
||||
else:
|
||||
thread_parseddict = {}
|
||||
pid = thread_parseddict.get('pid', '')
|
||||
tid = thread_parseddict.get('tid', '')
|
||||
name = thread_parseddict.get('name', 'unknown')
|
||||
cmd = thread_parseddict.get('cmd', 'unknown')
|
||||
|
||||
signal_line = parsedict.get('signal', '')
|
||||
signal_parsed = re.match(r'signal (?P<signal>.*?), code (?P<code>.*?), fault addr (?P<fault_addr>.*)\n', signal_line)
|
||||
if signal_parsed:
|
||||
signal_parseddict = signal_parsed.groupdict()
|
||||
else:
|
||||
signal_parseddict = {}
|
||||
signal = signal_parseddict.get('signal', 'unknown')
|
||||
code = signal_parseddict.get('code', 'unknown')
|
||||
fault_addr = signal_parseddict.get('fault_addr', '')
|
||||
|
||||
abort_line = parsedict.get('abort', '')
|
||||
|
||||
if parsed:
|
||||
message = 'Process {} ({}) got signal {} code {}'.format(name, cmd, signal, code)
|
||||
if abort_line:
|
||||
message += '\n'+abort_line
|
||||
else:
|
||||
message = fn+'\n'+dat[:1024]
|
||||
# Get summary for sentry title
|
||||
message = " ".join(contents.split('\n')[5:7])
|
||||
|
||||
# Cut off pid/tid, since that varies per run
|
||||
name_idx = message.find('name')
|
||||
if name_idx >= 0:
|
||||
message = message[name_idx:]
|
||||
|
||||
cloudlog.error({'tombstone': message})
|
||||
client.captureMessage(
|
||||
message=message,
|
||||
date=datetime.datetime.utcfromtimestamp(mtime),
|
||||
data={
|
||||
'logger':'tombstoned',
|
||||
'platform':'other',
|
||||
},
|
||||
sdk={'name': 'tombstoned', 'version': '0'},
|
||||
extra={
|
||||
'fault_addr': fault_addr,
|
||||
'abort_msg': abort_line,
|
||||
'pid': pid,
|
||||
'tid': tid,
|
||||
'name':'{} ({})'.format(name, cmd),
|
||||
'tombstone_fn': fn,
|
||||
'header': parsedict.get('header'),
|
||||
'registers': parsedict.get('registers'),
|
||||
'backtrace': parsedict.get('backtrace'),
|
||||
'logtail': logtail,
|
||||
},
|
||||
tags={
|
||||
'name':'{} ({})'.format(name, cmd),
|
||||
'signal':signal,
|
||||
'code':code,
|
||||
'fault_addr':fault_addr,
|
||||
'tombstone': contents
|
||||
},
|
||||
)
|
||||
cloudlog.error({'tombstone': message})
|
||||
|
||||
|
||||
def main():
|
||||
|
@ -112,11 +56,12 @@ def main():
|
|||
now_tombstones = set(get_tombstones())
|
||||
|
||||
for fn, ctime in (now_tombstones - initial_tombstones):
|
||||
cloudlog.info("reporting new tombstone %s", fn)
|
||||
cloudlog.info(f"reporting new tombstone {fn}")
|
||||
report_tombstone(fn, client)
|
||||
|
||||
initial_tombstones = now_tombstones
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
Loading…
Reference in New Issue