retropilot
/
openpilot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

loggerd: split bootlog to a separate program (#19831) 

* bootlog

* blank lines

* move file_exists to util.h

* src = ['loggerd.cc']

* log_init_data

* Revert "src = ['loggerd.cc']"

This reverts commit 18a587023d75b3a3a54f1ceaf9cb31a51640a067.

* add bootlog to .gitignore

* use s->has_qlog instead of true

* add blank lines & remove extern C
albatross
Dean Lee 2021-01-20 04:16:12 +08:00 committed by GitHub
parent 978d0c8964
commit f973c56a36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 154 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -44,6 +44,7 @@ selfdrive/ui/_ui
selfdrive/test/longitudinal_maneuvers/out
selfdrive/visiond/visiond
selfdrive/loggerd/loggerd
selfdrive/loggerd/bootlog
selfdrive/sensord/_gpsd
selfdrive/sensord/_sensord
selfdrive/camerad/camerad

1
release/files_common
View File

@ -307,6 +307,7 @@ selfdrive/loggerd/omx_encoder.h
selfdrive/loggerd/logger.cc
selfdrive/loggerd/logger.h
selfdrive/loggerd/loggerd.cc
selfdrive/loggerd/bootlog.cc
selfdrive/loggerd/raw_logger.cc
selfdrive/loggerd/raw_logger.h
selfdrive/loggerd/include/msm_media_info.h

6
selfdrive/common/util.h
View File

@ -96,6 +96,12 @@ inline std::string getenv_default(const char* env_var, const char * suffix, cons
inline void sleep_for(const int milliseconds) {
std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));
}
inline bool file_exists(const std::string& fn) {
std::ifstream f(fn);
return f.good();
}
}
class ExitHandler {

7
selfdrive/loggerd/SConscript
View File

@ -1,7 +1,9 @@
Import('env', 'arch', 'cereal', 'messaging', 'common', 'visionipc', 'gpucommon')
src = ['loggerd.cc', 'logger.cc']
libs = ['zmq', 'capnp', 'kj', 'z',
logger_lib = env.Library('logger', ["logger.cc"])
src = ['loggerd.cc']
libs = [logger_lib, 'zmq', 'capnp', 'kj', 'z',
'avformat', 'avcodec', 'swscale', 'avutil',
'yuv', 'bz2', 'OpenCL', common, cereal, messaging, visionipc]
@ -22,3 +24,4 @@ if arch == "Darwin":
env['FRAMEWORKS'] = ['OpenCL']
env.Program(src, LIBS=libs)
env.Program('bootlog.cc', LIBS=libs)

37
selfdrive/loggerd/bootlog.cc 100644
View File

@ -0,0 +1,37 @@
#include <assert.h>
#include <string>
#include "common/swaglog.h"
#include "common/util.h"
#include "logger.h"
#include "messaging.hpp"
int main(int argc, char** argv) {
LoggerState logger;
logger_init(&logger, "bootlog", false);
char segment_path[4096];
int err = logger_next(&logger, LOG_ROOT.c_str(), segment_path, sizeof(segment_path), nullptr);
assert(err == 0);
LOGW("bootlog to %s", segment_path);
MessageBuilder msg;
auto boot = msg.initEvent().initBoot();
boot.setWallTimeNanos(nanos_since_epoch());
std::string lastKmsg = util::read_file("/sys/fs/pstore/console-ramoops");
boot.setLastKmsg(capnp::Data::Reader((const kj::byte*)lastKmsg.data(), lastKmsg.size()));
std::string lastPmsg = util::read_file("/sys/fs/pstore/pmsg-ramoops-0");
boot.setLastPmsg(capnp::Data::Reader((const kj::byte*)lastPmsg.data(), lastPmsg.size()));
std::string launchLog = util::read_file("/tmp/launch_log");
boot.setLaunchLog(capnp::Text::Reader(launchLog.data(), launchLog.size()));
auto bytes = msg.toBytes();
logger_log(&logger, bytes.begin(), bytes.size(), false);
logger_close(&logger);
return 0;
}

114
selfdrive/loggerd/logger.cc
View File

@ -12,12 +12,99 @@
#include <pthread.h>
#include <bzlib.h>
#include <iostream>
#include <fstream>
#include <streambuf>
#ifdef QCOM
#include <cutils/properties.h>
#endif
#include "messaging.hpp"
#include "common/swaglog.h"
#include "common/params.h"
#include "common/util.h"
#include "common/version.h"
#include "logger.h"
void append_property(const char* key, const char* value, void *cookie) {
std::vector<std::pair<std::string, std::string> > *properties =
(std::vector<std::pair<std::string, std::string> > *)cookie;
properties->push_back(std::make_pair(std::string(key), std::string(value)));
}
void log_init_data(LoggerState *s) {
MessageBuilder msg;
auto init = msg.initEvent().initInitData();
if (util::file_exists("/EON")) {
init.setDeviceType(cereal::InitData::DeviceType::NEO);
} else if (util::file_exists("/TICI")) {
init.setDeviceType(cereal::InitData::DeviceType::TICI);
} else {
init.setDeviceType(cereal::InitData::DeviceType::PC);
}
init.setVersion(capnp::Text::Reader(COMMA_VERSION));
std::ifstream cmdline_stream("/proc/cmdline");
std::vector<std::string> kernel_args;
std::string buf;
while (cmdline_stream >> buf) {
kernel_args.push_back(buf);
}
auto lkernel_args = init.initKernelArgs(kernel_args.size());
for (int i=0; i<kernel_args.size(); i++) {
lkernel_args.set(i, kernel_args[i]);
}
init.setKernelVersion(util::read_file("/proc/version"));
#ifdef QCOM
{
std::vector<std::pair<std::string, std::string> > properties;
property_list(append_property, (void*)&properties);
auto lentries = init.initAndroidProperties().initEntries(properties.size());
for (int i=0; i<properties.size(); i++) {
auto lentry = lentries[i];
lentry.setKey(properties[i].first);
lentry.setValue(properties[i].second);
}
}
#endif
const char* dongle_id = getenv("DONGLE_ID");
if (dongle_id) {
init.setDongleId(std::string(dongle_id));
}
init.setDirty(!getenv("CLEAN"));
// log params
Params params = Params();
init.setGitCommit(params.get("GitCommit"));
init.setGitBranch(params.get("GitBranch"));
init.setGitRemote(params.get("GitRemote"));
init.setPassive(params.read_db_bool("Passive"));
{
std::map<std::string, std::string> params_map;
params.read_db_all(&params_map);
auto lparams = init.initParams().initEntries(params_map.size());
int i = 0;
for (auto& kv : params_map) {
auto lentry = lparams[i];
lentry.setKey(kv.first);
lentry.setValue(kv.second);
i++;
}
}
auto bytes = msg.toBytes();
logger_log(s, bytes.begin(), bytes.size(), s->has_qlog);
}
static void log_sentinel(LoggerState *s, cereal::Sentinel::SentinelType type) {
MessageBuilder msg;
auto sen = msg.initEvent().initSentinel();
@ -43,15 +130,9 @@ static int mkpath(char* file_path) {
return 0;
}
void logger_init(LoggerState *s, const char* log_name, const uint8_t* init_data, size_t init_data_len, bool has_qlog) {
void logger_init(LoggerState *s, const char* log_name, bool has_qlog) {
memset(s, 0, sizeof(*s));
if (init_data) {
s->init_data = (uint8_t*)malloc(init_data_len);
assert(s->init_data);
memcpy(s->init_data, init_data, init_data_len);
s->init_data_len = init_data_len;
}
umask(0);
pthread_mutex_init(&s->lock, NULL);
@ -110,20 +191,10 @@ static LoggerHandle* logger_open(LoggerState *s, const char* root_path) {
h->bz_qlog = BZ2_bzWriteOpen(&bzerror, h->qlog_file, 9, 0, 30);
if (bzerror != BZ_OK) goto fail;
}
if (s->init_data) {
BZ2_bzWrite(&bzerror, h->bz_file, s->init_data, s->init_data_len);
if (bzerror != BZ_OK) goto fail;
if (s->has_qlog) {
// init data goes in the qlog too
BZ2_bzWrite(&bzerror, h->bz_qlog, s->init_data, s->init_data_len);
if (bzerror != BZ_OK) goto fail;
}
}
pthread_mutex_init(&h->lock, NULL);
h->refcnt++;
log_init_data(s);
return h;
fail:
LOGE("logger failed to open files");
@ -203,7 +274,6 @@ void logger_close(LoggerState *s) {
log_sentinel(s, cereal::Sentinel::SentinelType::END_OF_ROUTE);
pthread_mutex_lock(&s->lock);
free(s->init_data);
if (s->cur_handle) {
lh_close(s->cur_handle);
}

23
selfdrive/loggerd/logger.h
View File

@ -1,13 +1,16 @@
#ifndef LOGGER_H
#define LOGGER_H
#pragma once
#include <stdio.h>
#include <stdint.h>
#include <pthread.h>
#include <bzlib.h>
#include <kj/array.h>
#include <capnp/serialize.h>
#ifdef __cplusplus
extern "C" {
#if defined(QCOM) || defined(QCOM2)
const std::string LOG_ROOT = "/data/media/0/realdata";
#else
const std::string LOG_ROOT = util::getenv_default("HOME", "/.comma/media/0/realdata", "/data/media/0/realdata");
#endif
#define LOGGER_MAX_HANDLES 16
@ -28,10 +31,6 @@ typedef struct LoggerHandle {
typedef struct LoggerState {
pthread_mutex_t lock;
uint8_t* init_data;
size_t init_data_len;
int part;
char route_name[64];
char log_name[64];
@ -41,7 +40,7 @@ typedef struct LoggerState {
LoggerHandle* cur_handle;
} LoggerState;
void logger_init(LoggerState *s, const char* log_name, const uint8_t* init_data, size_t init_data_len, bool has_qlog);
void logger_init(LoggerState *s, const char* log_name, bool has_qlog);
int logger_next(LoggerState *s, const char* root_path,
char* out_segment_path, size_t out_segment_path_len,
int* out_part);
@ -51,9 +50,3 @@ void logger_log(LoggerState *s, uint8_t* data, size_t data_size, bool in_qlog);
void lh_log(LoggerHandle* h, uint8_t* data, size_t data_size, bool in_qlog);
void lh_close(LoggerHandle* h);
#ifdef __cplusplus
}
#endif
#endif

141
selfdrive/loggerd/loggerd.cc
View File

@ -10,9 +10,7 @@
#include <sys/resource.h>
#include <string>
#include <iostream>
#include <fstream>
#include <streambuf>
#include <thread>
#include <mutex>
#include <condition_variable>
@ -20,11 +18,6 @@
#include <random>
#include <ftw.h>
#ifdef QCOM
#include <cutils/properties.h>
#endif
#include "common/version.h"
#include "common/timing.h"
#include "common/params.h"
#include "common/swaglog.h"
@ -107,11 +100,6 @@ LogCameraInfo cameras_logged[LOG_CAMERA_ID_MAX] = {
namespace {
constexpr int SEGMENT_LENGTH = 60;
#if defined(QCOM) || defined(QCOM2)
std::string LOG_ROOT = "/data/media/0/realdata";
#else
std::string LOG_ROOT = util::getenv_default("HOME", "/.comma/media/0/realdata", "/data/media/0/realdata");
#endif
double randrange(double a, double b) __attribute__((unused));
double randrange(double a, double b) {
@ -123,11 +111,6 @@ double randrange(double a, double b) {
ExitHandler do_exit;
static bool file_exists(const std::string& fn) {
std::ifstream f(fn);
return f.good();
}
class RotateState {
public:
SubSocket* fpkt_sock;
@ -328,82 +311,6 @@ void encoder_thread(int cam_idx) {
}
void append_property(const char* key, const char* value, void *cookie) {
std::vector<std::pair<std::string, std::string> > *properties =
(std::vector<std::pair<std::string, std::string> > *)cookie;
properties->push_back(std::make_pair(std::string(key), std::string(value)));
}
kj::Array<capnp::word> gen_init_data() {
MessageBuilder msg;
auto init = msg.initEvent().initInitData();
if (file_exists("/EON")) {
init.setDeviceType(cereal::InitData::DeviceType::NEO);
} else if (file_exists("/TICI")) {
init.setDeviceType(cereal::InitData::DeviceType::TICI);
} else {
init.setDeviceType(cereal::InitData::DeviceType::PC);
}
init.setVersion(capnp::Text::Reader(COMMA_VERSION));
std::ifstream cmdline_stream("/proc/cmdline");
std::vector<std::string> kernel_args;
std::string buf;
while (cmdline_stream >> buf) {
kernel_args.push_back(buf);
}
auto lkernel_args = init.initKernelArgs(kernel_args.size());
for (int i=0; i<kernel_args.size(); i++) {
lkernel_args.set(i, kernel_args[i]);
}
init.setKernelVersion(util::read_file("/proc/version"));
#ifdef QCOM
{
std::vector<std::pair<std::string, std::string> > properties;
property_list(append_property, (void*)&properties);
auto lentries = init.initAndroidProperties().initEntries(properties.size());
for (int i=0; i<properties.size(); i++) {
auto lentry = lentries[i];
lentry.setKey(properties[i].first);
lentry.setValue(properties[i].second);
}
}
#endif
const char* dongle_id = getenv("DONGLE_ID");
if (dongle_id) {
init.setDongleId(std::string(dongle_id));
}
init.setDirty(!getenv("CLEAN"));
// log params
Params params = Params();
init.setGitCommit(params.get("GitCommit"));
init.setGitBranch(params.get("GitBranch"));
init.setGitRemote(params.get("GitRemote"));
init.setPassive(params.read_db_bool("Passive"));
{
std::map<std::string, std::string> params_map;
params.read_db_all(&params_map);
auto lparams = init.initParams().initEntries(params_map.size());
int i = 0;
for (auto& kv : params_map) {
auto lentry = lparams[i];
lentry.setKey(kv.first);
lentry.setValue(kv.second);
i++;
}
}
return capnp::messageToFlatArray(msg);
}
static int clear_locks_fn(const char* fpath, const struct stat *sb, int tyupeflag) {
const char* dot = strrchr(fpath, '.');
if (dot && strcmp(dot, ".lock") == 0) {
@ -416,50 +323,10 @@ static void clear_locks() {
ftw(LOG_ROOT.c_str(), clear_locks_fn, 16);
}
static void bootlog() {
int err;
{
auto words = gen_init_data();
auto bytes = words.asBytes();
logger_init(&s.logger, "bootlog", bytes.begin(), bytes.size(), false);
}
err = logger_next(&s.logger, LOG_ROOT.c_str(), s.segment_path, sizeof(s.segment_path), &s.rotate_segment);
assert(err == 0);
LOGW("bootlog to %s", s.segment_path);
{
MessageBuilder msg;
auto boot = msg.initEvent().initBoot();
boot.setWallTimeNanos(nanos_since_epoch());
std::string lastKmsg = util::read_file("/sys/fs/pstore/console-ramoops");
boot.setLastKmsg(capnp::Data::Reader((const kj::byte*)lastKmsg.data(), lastKmsg.size()));
std::string lastPmsg = util::read_file("/sys/fs/pstore/pmsg-ramoops-0");
boot.setLastPmsg(capnp::Data::Reader((const kj::byte*)lastPmsg.data(), lastPmsg.size()));
std::string launchLog = util::read_file("/tmp/launch_log");
boot.setLaunchLog(capnp::Text::Reader(launchLog.data(), launchLog.size()));
auto bytes = msg.toBytes();
logger_log(&s.logger, bytes.begin(), bytes.size(), false);
}
logger_close(&s.logger);
}
int main(int argc, char** argv) {
setpriority(PRIO_PROCESS, 0, -12);
if (argc > 1 && strcmp(argv[1], "--bootlog") == 0) {
bootlog();
return 0;
}
int segment_length = SEGMENT_LENGTH;
if (getenv("LOGGERD_TEST")) {
segment_length = atoi(getenv("LOGGERD_SEGMENT_LENGTH"));
@ -495,11 +362,7 @@ int main(int argc, char** argv) {
}
// init logger
{
auto words = gen_init_data();
auto bytes = words.asBytes();
logger_init(&s.logger, "rlog", bytes.begin(), bytes.size(), true);
}
logger_init(&s.logger, "rlog", true);
// init encoders
pthread_mutex_init(&s.rotate_lock, NULL);

2
selfdrive/loggerd/tests/test_loggerd.py
View File

@ -47,7 +47,7 @@ class TestLoggerd(unittest.TestCase):
def _gen_bootlog(self):
with Timeout(5):
out = subprocess.check_output(["./loggerd", "--bootlog"], cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8')
out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8')
# check existence
d = self._get_log_dir(out)

2
selfdrive/manager.py
View File

@ -440,7 +440,7 @@ def manager_thread():
cloudlog.info({"environ": os.environ})
# save boot log
subprocess.call(["./loggerd", "--bootlog"], cwd=os.path.join(BASEDIR, "selfdrive/loggerd"))
subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"))
# start daemon processes
for p in daemon_processes: