Simplify tombstoned, should fix #1258
This commit is contained in:
parent
106be23c4e
commit
d8d2d568dc
99
selfdrive/tombstoned.py
Normal file → Executable file
99
selfdrive/tombstoned.py
Normal file → Executable file
|
@ -1,7 +1,6 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import time
|
import time
|
||||||
import datetime
|
|
||||||
|
|
||||||
from raven import Client
|
from raven import Client
|
||||||
from raven.transport.http import HTTPTransport
|
from raven.transport.http import HTTPTransport
|
||||||
|
@ -9,96 +8,41 @@ from raven.transport.http import HTTPTransport
|
||||||
from selfdrive.version import version, dirty
|
from selfdrive.version import version, dirty
|
||||||
from selfdrive.swaglog import cloudlog
|
from selfdrive.swaglog import cloudlog
|
||||||
|
|
||||||
|
MAX_SIZE = 100000 * 10 # Normal size is 40-100k, allow up to 1M
|
||||||
|
|
||||||
|
|
||||||
def get_tombstones():
|
def get_tombstones():
|
||||||
|
"""Returns list of (c_time, filename) for all tombstones in /data/tombstones"""
|
||||||
DIR_DATA = "/data/tombstones/"
|
DIR_DATA = "/data/tombstones/"
|
||||||
return [(DIR_DATA + fn, int(os.stat(DIR_DATA + fn).st_ctime) )
|
return [(DIR_DATA + fn, int(os.stat(DIR_DATA + fn).st_ctime))
|
||||||
for fn in os.listdir(DIR_DATA) if fn.startswith("tombstone")]
|
for fn in os.listdir(DIR_DATA) if fn.startswith("tombstone")]
|
||||||
|
|
||||||
|
|
||||||
def report_tombstone(fn, client):
|
def report_tombstone(fn, client):
|
||||||
mtime = os.path.getmtime(fn)
|
f_size = os.path.getsize(fn)
|
||||||
|
if f_size > MAX_SIZE:
|
||||||
|
cloudlog.error(f"Tombstone {fn} too big, {f_size}. Skipping...")
|
||||||
|
|
||||||
with open(fn, encoding='ISO-8859-1') as f:
|
with open(fn, encoding='ISO-8859-1') as f:
|
||||||
dat = f.read()
|
contents = f.read()
|
||||||
|
|
||||||
# see system/core/debuggerd/tombstone.cpp
|
# Get summary for sentry title
|
||||||
parsed = re.match(r"[* ]*\n"
|
message = " ".join(contents.split('\n')[5:7])
|
||||||
r"(?P<header>CM Version:[\s\S]*?ABI:.*\n)"
|
|
||||||
r"(?P<thread>pid:.*\n)"
|
|
||||||
r"(?P<signal>signal.*\n)?"
|
|
||||||
r"(?P<abort>Abort.*\n)?"
|
|
||||||
r"(?P<registers>\s+x0[\s\S]*?\n)\n"
|
|
||||||
r"(?:backtrace:\n"
|
|
||||||
r"(?P<backtrace>[\s\S]*?\n)\n"
|
|
||||||
r"stack:\n"
|
|
||||||
r"(?P<stack>[\s\S]*?\n)\n"
|
|
||||||
r")?", dat)
|
|
||||||
|
|
||||||
logtail = re.search(r"--------- tail end of.*\n([\s\S]*?\n)---", dat)
|
|
||||||
logtail = logtail and logtail.group(1)
|
|
||||||
|
|
||||||
if parsed:
|
|
||||||
parsedict = parsed.groupdict()
|
|
||||||
else:
|
|
||||||
parsedict = {}
|
|
||||||
|
|
||||||
thread_line = parsedict.get('thread', '')
|
|
||||||
thread_parsed = re.match(r'pid: (?P<pid>\d+), tid: (?P<tid>\d+), name: (?P<name>.*) >>> (?P<cmd>.*) <<<', thread_line)
|
|
||||||
if thread_parsed:
|
|
||||||
thread_parseddict = thread_parsed.groupdict()
|
|
||||||
else:
|
|
||||||
thread_parseddict = {}
|
|
||||||
pid = thread_parseddict.get('pid', '')
|
|
||||||
tid = thread_parseddict.get('tid', '')
|
|
||||||
name = thread_parseddict.get('name', 'unknown')
|
|
||||||
cmd = thread_parseddict.get('cmd', 'unknown')
|
|
||||||
|
|
||||||
signal_line = parsedict.get('signal', '')
|
|
||||||
signal_parsed = re.match(r'signal (?P<signal>.*?), code (?P<code>.*?), fault addr (?P<fault_addr>.*)\n', signal_line)
|
|
||||||
if signal_parsed:
|
|
||||||
signal_parseddict = signal_parsed.groupdict()
|
|
||||||
else:
|
|
||||||
signal_parseddict = {}
|
|
||||||
signal = signal_parseddict.get('signal', 'unknown')
|
|
||||||
code = signal_parseddict.get('code', 'unknown')
|
|
||||||
fault_addr = signal_parseddict.get('fault_addr', '')
|
|
||||||
|
|
||||||
abort_line = parsedict.get('abort', '')
|
|
||||||
|
|
||||||
if parsed:
|
|
||||||
message = 'Process {} ({}) got signal {} code {}'.format(name, cmd, signal, code)
|
|
||||||
if abort_line:
|
|
||||||
message += '\n'+abort_line
|
|
||||||
else:
|
|
||||||
message = fn+'\n'+dat[:1024]
|
|
||||||
|
|
||||||
|
# Cut off pid/tid, since that varies per run
|
||||||
|
name_idx = message.find('name')
|
||||||
|
if name_idx >= 0:
|
||||||
|
message = message[name_idx:]
|
||||||
|
|
||||||
|
cloudlog.error({'tombstone': message})
|
||||||
client.captureMessage(
|
client.captureMessage(
|
||||||
message=message,
|
message=message,
|
||||||
date=datetime.datetime.utcfromtimestamp(mtime),
|
|
||||||
data={
|
|
||||||
'logger':'tombstoned',
|
|
||||||
'platform':'other',
|
|
||||||
},
|
|
||||||
sdk={'name': 'tombstoned', 'version': '0'},
|
sdk={'name': 'tombstoned', 'version': '0'},
|
||||||
extra={
|
extra={
|
||||||
'fault_addr': fault_addr,
|
|
||||||
'abort_msg': abort_line,
|
|
||||||
'pid': pid,
|
|
||||||
'tid': tid,
|
|
||||||
'name':'{} ({})'.format(name, cmd),
|
|
||||||
'tombstone_fn': fn,
|
'tombstone_fn': fn,
|
||||||
'header': parsedict.get('header'),
|
'tombstone': contents
|
||||||
'registers': parsedict.get('registers'),
|
|
||||||
'backtrace': parsedict.get('backtrace'),
|
|
||||||
'logtail': logtail,
|
|
||||||
},
|
|
||||||
tags={
|
|
||||||
'name':'{} ({})'.format(name, cmd),
|
|
||||||
'signal':signal,
|
|
||||||
'code':code,
|
|
||||||
'fault_addr':fault_addr,
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
cloudlog.error({'tombstone': message})
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
@ -112,11 +56,12 @@ def main():
|
||||||
now_tombstones = set(get_tombstones())
|
now_tombstones = set(get_tombstones())
|
||||||
|
|
||||||
for fn, ctime in (now_tombstones - initial_tombstones):
|
for fn, ctime in (now_tombstones - initial_tombstones):
|
||||||
cloudlog.info("reporting new tombstone %s", fn)
|
cloudlog.info(f"reporting new tombstone {fn}")
|
||||||
report_tombstone(fn, client)
|
report_tombstone(fn, client)
|
||||||
|
|
||||||
initial_tombstones = now_tombstones
|
initial_tombstones = now_tombstones
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
Loading…
Reference in a new issue