Direct logentries logging from manager (#1306)
* add function to add direct logentries handler * Whitespace * Add manager crash handler * Log scons build errors * Add line to format exception * Small cleanup * That is now a list * Read all data from stderralbatross
parent
83d8dd4717
commit
c95134defe
|
@ -3,12 +3,16 @@ import zmq
|
|||
from logentries import LogentriesHandler
|
||||
import cereal.messaging as messaging
|
||||
|
||||
def main():
|
||||
|
||||
def get_le_handler():
|
||||
# setup logentries. we forward log messages to it
|
||||
le_token = "e8549616-0798-4d7e-a2ca-2513ae81fa17"
|
||||
le_handler = LogentriesHandler(le_token, use_tls=False, verbose=False)
|
||||
return LogentriesHandler(le_token, use_tls=False, verbose=False)
|
||||
|
||||
le_level = 20 #logging.INFO
|
||||
|
||||
def main():
|
||||
le_handler = get_le_handler()
|
||||
le_level = 20 # logging.INFO
|
||||
|
||||
ctx = zmq.Context().instance()
|
||||
sock = ctx.socket(zmq.PULL)
|
||||
|
@ -36,5 +40,6 @@ def main():
|
|||
msg.logMessage = dat
|
||||
pub_sock.send(msg.to_bytes())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
@ -8,6 +8,7 @@ import signal
|
|||
import shutil
|
||||
import subprocess
|
||||
import datetime
|
||||
from selfdrive.swaglog import cloudlog, add_logentries_handler
|
||||
|
||||
from common.basedir import BASEDIR, PARAMS
|
||||
from common.android import ANDROID
|
||||
|
@ -88,25 +89,32 @@ if not prebuilt:
|
|||
j_flag = "" if nproc is None else "-j%d" % (nproc - 1)
|
||||
scons = subprocess.Popen(["scons", j_flag], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
|
||||
|
||||
compile_output = []
|
||||
|
||||
# Read progress from stderr and update spinner
|
||||
while scons.poll() is None:
|
||||
try:
|
||||
line = scons.stderr.readline()
|
||||
if line is None:
|
||||
continue
|
||||
|
||||
line = line.rstrip()
|
||||
|
||||
prefix = b'progress: '
|
||||
if line.startswith(prefix):
|
||||
i = int(line[len(prefix):])
|
||||
if spinner is not None:
|
||||
spinner.update("%d" % (50.0 * (i / TOTAL_SCONS_NODES)))
|
||||
elif len(line):
|
||||
print(line.decode('utf8'))
|
||||
compile_output.append(line)
|
||||
print(line.decode('utf8', 'replace'))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if scons.returncode != 0:
|
||||
# Read remaining output
|
||||
r = scons.stderr.read().split(b'\n')
|
||||
compile_output += r
|
||||
|
||||
if retry:
|
||||
print("scons build failed, cleaning in")
|
||||
for i in range(3,-1,-1):
|
||||
|
@ -115,7 +123,15 @@ if not prebuilt:
|
|||
subprocess.check_call(["scons", "-c"], cwd=BASEDIR, env=env)
|
||||
shutil.rmtree("/tmp/scons_cache")
|
||||
else:
|
||||
raise RuntimeError("scons build failed")
|
||||
# Build failed log errors
|
||||
errors = [line.decode('utf8', 'replace') for line in compile_output
|
||||
if any([err in line for err in [b'error: ', b'not found, needed by target']])]
|
||||
errors = "\n".join(errors)
|
||||
add_logentries_handler(cloudlog)
|
||||
cloudlog.error("scons build failed\n" + errors)
|
||||
|
||||
# TODO: Show errors in TextWindow
|
||||
exit(1)
|
||||
else:
|
||||
break
|
||||
|
||||
|
@ -124,7 +140,6 @@ import cereal.messaging as messaging
|
|||
|
||||
from common.params import Params
|
||||
import selfdrive.crash as crash
|
||||
from selfdrive.swaglog import cloudlog
|
||||
from selfdrive.registration import register
|
||||
from selfdrive.version import version, dirty
|
||||
from selfdrive.loggerd.config import ROOT
|
||||
|
@ -567,7 +582,17 @@ def main():
|
|||
if params.get("DoUninstall", encoding='utf8') == "1":
|
||||
uninstall()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
try:
|
||||
main()
|
||||
except Exception:
|
||||
add_logentries_handler(cloudlog)
|
||||
cloudlog.exception("Manager failed to start")
|
||||
# TODO: Show exception using TextWindow
|
||||
# error = traceback.format_exc()
|
||||
|
||||
raise
|
||||
|
||||
# manual exit because we are forked
|
||||
sys.exit(0)
|
||||
|
|
|
@ -5,6 +5,7 @@ import zmq
|
|||
|
||||
from common.logging_extra import SwagLogger, SwagFormatter
|
||||
|
||||
|
||||
class LogMessageHandler(logging.Handler):
|
||||
def __init__(self, formatter):
|
||||
logging.Handler.__init__(self)
|
||||
|
@ -31,10 +32,19 @@ class LogMessageHandler(logging.Handler):
|
|||
# drop :/
|
||||
pass
|
||||
|
||||
|
||||
def add_logentries_handler(log):
|
||||
"""Function to add the logentries handler to swaglog.
|
||||
This can be used to send logs when logmessaged is not running."""
|
||||
from selfdrive.logmessaged import get_le_handler
|
||||
handler = get_le_handler()
|
||||
handler.setFormatter(SwagFormatter(log))
|
||||
log.addHandler(handler)
|
||||
|
||||
|
||||
cloudlog = log = SwagLogger()
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
outhandler = logging.StreamHandler()
|
||||
log.addHandler(outhandler)
|
||||
|
||||
log.addHandler(LogMessageHandler(SwagFormatter(log)))
|
||||
|
|
Loading…
Reference in New Issue