retropilot
/
openpilot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

log to file and send through athena (#20250) 

* log to file and send through athena

* rename logging level

* pass thru log formatter

* logMessage is TEXT

* send queue always strings

* switch to xattr and lower priority queue

* enable cloud logging for devices

* time or size based log rotation

* basename -> dirname

* remove HARDWARE.get_cloudlog_enabled

* fix errors

* fix another exception

* xattrs need to be bytes

* sending works

* cleanup files at start

* add id and adjust formatting

* do not send active log file

* better names

* separate log formatters

* fix formatter super init

* fix log file order

* ensure file always has file formatter

* i see why there was no formatter

* apply same formatting to cpp log msgs

* apply same formatting to cpp log msgs

* update queue names in tests

* strip deprecated keys in STATUS_PACKET

* strip DEPRECATED from dict recursively

* athena log queue test

* instanceof instead of type

* isinstance instead of type

* use super

* remove logentries

* last_scan param unused

* comment about special log msg attr names

* add dict_helpers.py to release files

* use monotonic time and counter for log rotation

* update for adjusted log file naming

* use monotonic clock for tracking last log file scan
albatross
Greg Hogan 2021-03-25 13:30:09 -07:00 committed by GitHub
parent dcc77451c4
commit 3d48bd934d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 453 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Pipfile
View File

@ -94,7 +94,6 @@ tqdm = "*"
Cython = "*" Cython = "*"
PyYAML = "*" PyYAML = "*"
websocket_client = "*" websocket_client = "*"
Logentries = {git = "https://github.com/commaai/le_python.git",ref = "feaeacb48f7f4bdb02c0a8fc092326d4e101b7f2"}
urllib3 = "*" urllib3 = "*"
gunicorn = "*" gunicorn = "*"
utm = "*" utm = "*"

371
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "2183375c59225a990d3bc013efd68e0702765d1b340d98f11112875efdc9a03f" "sha256": "fcb1581613337cb41eb3fb0382ca993caf9b71e469026c4acf35081d9b09cebe"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -128,10 +128,10 @@
}, },
"cysignals": { "cysignals": {
"hashes": [ "hashes": [
"sha256:8107b67a0c5991f74b0e000c6fa9fe8efcb2a22c7ede5b017aac4c3e20fb7db2" "sha256:5c4606c435775028316f725fdb7cb894e3cae0b6fd2f862a0d2971748469b43a"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.10.2" "version": "==1.10.3"
}, },
"cython": { "cython": {
"hashes": [ "hashes": [
@ -172,11 +172,11 @@
}, },
"flake8": { "flake8": {
"hashes": [ "hashes": [
"sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839", "sha256:12d05ab02614b6aee8df7c36b97d1a3b2372761222b19b58621355e82acddcff",
"sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b" "sha256:78873e372b12b093da7b5e5ed302e8ad9e988b38b063b61ad937f26ca58fc5f0"
], ],
"index": "pypi", "index": "pypi",
"version": "==3.8.4" "version": "==3.9.0"
}, },
"flask": { "flask": {
"hashes": [ "hashes": [
@ -211,11 +211,11 @@
}, },
"isort": { "isort": {
"hashes": [ "hashes": [
"sha256:c729845434366216d320e936b8ad6f9d681aab72dc7cbc2d51bedc3582f3ad1e", "sha256:0a943902919f65c5684ac4e0154b1ad4fac6dcaa5d9f3426b732f1c8b5419be6",
"sha256:fff4f0c04e1825522ce6949973e83110a6e907750cd92d128b0d14aaaadbffdc" "sha256:2bb1680aad211e3c9944dbce1d4ba09a989f04e238296c87fe2139faa26d655d"
], ],
"markers": "python_version >= '3.6' and python_version < '4'", "markers": "python_version >= '3.6' and python_version < '4'",
"version": "==5.7.0" "version": "==5.8.0"
}, },
"itsdangerous": { "itsdangerous": {
"hashes": [ "hashes": [
@ -243,33 +243,31 @@
}, },
"lazy-object-proxy": { "lazy-object-proxy": {
"hashes": [ "hashes": [
"sha256:1d33d6f789697f401b75ce08e73b1de567b947740f768376631079290118ad39", "sha256:17e0967ba374fc24141738c69736da90e94419338fd4c7c7bef01ee26b339653",
"sha256:2f2de8f8ac0be3e40d17730e0600619d35c78c13a099ea91ef7fb4ad944ce694", "sha256:1fee665d2638491f4d6e55bd483e15ef21f6c8c2095f235fef72601021e64f61",
"sha256:3782931963dc89e0e9a0ae4348b44762e868ea280e4f8c233b537852a8996ab9", "sha256:22ddd618cefe54305df49e4c069fa65715be4ad0e78e8d252a33debf00f6ede2",
"sha256:37d9c34b96cca6787fe014aeb651217944a967a5b165e2cacb6b858d2997ab84", "sha256:24a5045889cc2729033b3e604d496c2b6f588c754f7a62027ad4437a7ecc4837",
"sha256:38c3865bd220bd983fcaa9aa11462619e84a71233bafd9c880f7b1cb753ca7fa", "sha256:410283732af311b51b837894fa2f24f2c0039aa7f220135192b38fcc42bd43d3",
"sha256:429c4d1862f3fc37cd56304d880f2eae5bd0da83bdef889f3bd66458aac49128", "sha256:4732c765372bd78a2d6b2150a6e99d00a78ec963375f236979c0626b97ed8e43",
"sha256:522b7c94b524389f4a4094c4bf04c2b02228454ddd17c1a9b2801fac1d754871", "sha256:489000d368377571c6f982fba6497f2aa13c6d1facc40660963da62f5c379726",
"sha256:57fb5c5504ddd45ed420b5b6461a78f58cbb0c1b0cbd9cd5a43ad30a4a3ee4d0", "sha256:4f60460e9f1eb632584c9685bccea152f4ac2130e299784dbaf9fae9f49891b3",
"sha256:5944a9b95e97de1980c65f03b79b356f30a43de48682b8bdd90aa5089f0ec1f4", "sha256:5743a5ab42ae40caa8421b320ebf3a998f89c85cdc8376d6b2e00bd12bd1b587",
"sha256:6f4e5e68b7af950ed7fdb594b3f19a0014a3ace0fedb86acb896e140ffb24302", "sha256:85fb7608121fd5621cc4377a8961d0b32ccf84a7285b4f1d21988b2eae2868e8",
"sha256:71a1ef23f22fa8437974b2d60fedb947c99a957ad625f83f43fd3de70f77f458", "sha256:9698110e36e2df951c7c36b6729e96429c9c32b3331989ef19976592c5f3c77a",
"sha256:8a44e9901c0555f95ac401377032f6e6af66d8fc1fbfad77a7a8b1a826e0b93c", "sha256:9d397bf41caad3f489e10774667310d73cb9c4258e9aed94b9ec734b34b495fd",
"sha256:b6577f15d5516d7d209c1a8cde23062c0f10625f19e8dc9fb59268859778d7d7", "sha256:b579f8acbf2bdd9ea200b1d5dea36abd93cabf56cf626ab9c744a432e15c815f",
"sha256:c8fe2d6ff0ff583784039d0255ea7da076efd08507f2be6f68583b0da32e3afb", "sha256:b865b01a2e7f96db0c5d12cfea590f98d8c5ba64ad222300d93ce6ff9138bcad",
"sha256:cadfa2c2cf54d35d13dc8d231253b7985b97d629ab9ca6e7d672c35539d38163", "sha256:bf34e368e8dd976423396555078def5cfc3039ebc6fc06d1ae2c5a65eebbcde4",
"sha256:cd1bdace1a8762534e9a36c073cd54e97d517a17d69a17985961265be6d22847", "sha256:c6938967f8528b3668622a9ed3b31d145fab161a32f5891ea7b84f6b790be05b",
"sha256:ddbdcd10eb999d7ab292677f588b658372aadb9a52790f82484a37127a390108", "sha256:d1c2676e3d840852a2de7c7d5d76407c772927addff8d742b9808fe0afccebdf",
"sha256:e7273c64bccfd9310e9601b8f4511d84730239516bada26a0c9846c9697617ef", "sha256:d7124f52f3bd259f510651450e18e0fd081ed82f3c08541dffc7b94b883aa981",
"sha256:e7428977763150b4cf83255625a80a23dfdc94d43be7791ce90799d446b4e26f", "sha256:d900d949b707778696fdf01036f58c9876a0d8bfe116e8d220cfd4b15f14e741",
"sha256:e960e8be509e8d6d618300a6c189555c24efde63e85acaf0b14b2cd1ac743315", "sha256:ebfd274dcd5133e0afae738e6d9da4323c3eb021b3e13052d8cbd0e457b1256e",
"sha256:ecb5dd5990cec6e7f5c9c1124a37cb2c710c6d69b0c1a5c4aa4b35eba0ada068", "sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93",
"sha256:ef3f5e288aa57b73b034ce9c1f1ac753d968f9069cd0742d1d69c698a0167166", "sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b"
"sha256:fa5b2dee0e231fa4ad117be114251bdfe6afe39213bd629d43deb117b6a6c40a",
"sha256:fa7fb7973c622b9e725bee1db569d2c2ee64d2f9a089201c5e8185d482c7352d"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==1.5.2" "version": "==1.6.0"
}, },
"libusb1": { "libusb1": {
"hashes": [ "hashes": [
@ -284,10 +282,6 @@
"index": "pypi", "index": "pypi",
"version": "==1.9.2" "version": "==1.9.2"
}, },
"logentries": {
"git": "https://github.com/commaai/le_python.git",
"ref": "feaeacb48f7f4bdb02c0a8fc092326d4e101b7f2"
},
"markupsafe": { "markupsafe": {
"hashes": [ "hashes": [
"sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473",
@ -567,11 +561,11 @@
}, },
"pycodestyle": { "pycodestyle": {
"hashes": [ "hashes": [
"sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068",
"sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2.6.0" "version": "==2.7.0"
}, },
"pycparser": { "pycparser": {
"hashes": [ "hashes": [
@ -619,11 +613,11 @@
}, },
"pyflakes": { "pyflakes": {
"hashes": [ "hashes": [
"sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3",
"sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2.2.0" "version": "==2.3.1"
}, },
"pyjwt": { "pyjwt": {
"hashes": [ "hashes": [
@ -847,11 +841,11 @@
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
"sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.26.3" "version": "==1.26.4"
}, },
"utm": { "utm": {
"hashes": [ "hashes": [
@ -1021,11 +1015,11 @@
}, },
"azure-cli-core": { "azure-cli-core": {
"hashes": [ "hashes": [
"sha256:a23bc53d222a3c76e3de5299c1d3c27d8d005926b9d44b0c98722cbdc30505ab", "sha256:6776d7e63644bb0ef5aa05bd70b1b67a9dffe92c66fd1d773747399c9feee70c",
"sha256:e11f9edba62905623d54a75388afd8d21eeeb31a8aec22197abd24337ff5fa75" "sha256:78e5f0066ca7a89aeed2e68999ce2ebdd9c1b64a69578e5631e4bec2fd688ae5"
], ],
"index": "pypi", "index": "pypi",
"version": "==2.20.0" "version": "==2.21.0"
}, },
"azure-cli-telemetry": { "azure-cli-telemetry": {
"hashes": [ "hashes": [
@ -1143,19 +1137,19 @@
}, },
"boto3": { "boto3": {
"hashes": [ "hashes": [
"sha256:64a8900b3a110e2d6ff4d87f4d8cd56f0c8527361d9fc9385fcb50efe7a4975a", "sha256:1e6e06b2f1eee5a76acdde1e7b4f57c93c1bf2905341207d74f2a140ce060cd8",
"sha256:8e9ff8006c41889ed8a11831dee62adf922e071f14d54c52946d1f7855ae7a8e" "sha256:40e84a5f7888924db74a2710dbe48d066b51fe1f5549efaffe90e6efe813f37b"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.17.26" "version": "==1.17.35"
}, },
"botocore": { "botocore": {
"hashes": [ "hashes": [
"sha256:4a785847a351e59f2329627fc9a19cf50f07644ea68996a1595d5a20487a423f", "sha256:9119ffb231145ffadd55391c9356dcdb18e3de65c3a7c82844634e949f0ca5a0",
"sha256:d27cbe115a25bfa82b851861b62d71fc771c2883bf5645bf37a7c0114789407c" "sha256:e34bbb7d7de154c2ff2a73ae0691c601a69c5bda887374c8a6a23072380b07a4"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==1.20.26" "version": "==1.20.35"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
@ -1241,10 +1235,10 @@
}, },
"control": { "control": {
"hashes": [ "hashes": [
"sha256:fed8ca6c773175bea14b121c001eee20c48977629a498d9a2d19a85490112ba5" "sha256:34eeca077cf002a2f22a9334c8998ec5b3bcc0fdae2aac790a923cf8bc80245a"
], ],
"index": "pypi", "index": "pypi",
"version": "==0.8.4" "version": "==0.9.0"
}, },
"coverage": { "coverage": {
"hashes": [ "hashes": [
@ -1369,11 +1363,11 @@
}, },
"elasticsearch": { "elasticsearch": {
"hashes": [ "hashes": [
"sha256:1e24b33a82bf381b42d3b0d390f76fdb9d6a9d47b310dea8eaeb0a5933c394c0", "sha256:9a77172be02bc4855210d83f0f1346a1e7d421e3cb2ca47ba81ac0c5a717b3a0",
"sha256:a113cfcee9ba8565cd48a67b60e9903b67a81b3b80ddc6d3fb2c16789a58b763" "sha256:c67b0f6541eda6de9f92eaea319c070aa2710c5d4d4ee5e3dfa3c21bd95aa378"
], ],
"index": "pypi", "index": "pypi",
"version": "==7.11.0" "version": "==7.12.0"
}, },
"entrypoints": { "entrypoints": {
"hashes": [ "hashes": [
@ -1465,6 +1459,55 @@
"index": "pypi", "index": "pypi",
"version": "==2.5.1" "version": "==2.5.1"
}, },
"greenlet": {
"hashes": [
"sha256:0a77691f0080c9da8dfc81e23f4e3cffa5accf0f5b56478951016d7cfead9196",
"sha256:0ddd77586553e3daf439aa88b6642c5f252f7ef79a39271c25b1d4bf1b7cbb85",
"sha256:111cfd92d78f2af0bc7317452bd93a477128af6327332ebf3c2be7df99566683",
"sha256:122c63ba795fdba4fc19c744df6277d9cfd913ed53d1a286f12189a0265316dd",
"sha256:181300f826625b7fd1182205b830642926f52bd8cdb08b34574c9d5b2b1813f7",
"sha256:1a1ada42a1fd2607d232ae11a7b3195735edaa49ea787a6d9e6a53afaf6f3476",
"sha256:1bb80c71de788b36cefb0c3bb6bfab306ba75073dbde2829c858dc3ad70f867c",
"sha256:1d1d4473ecb1c1d31ce8fd8d91e4da1b1f64d425c1dc965edc4ed2a63cfa67b2",
"sha256:292e801fcb3a0b3a12d8c603c7cf340659ea27fd73c98683e75800d9fd8f704c",
"sha256:2c65320774a8cd5fdb6e117c13afa91c4707548282464a18cf80243cf976b3e6",
"sha256:4365eccd68e72564c776418c53ce3c5af402bc526fe0653722bc89efd85bf12d",
"sha256:5352c15c1d91d22902582e891f27728d8dac3bd5e0ee565b6a9f575355e6d92f",
"sha256:58ca0f078d1c135ecf1879d50711f925ee238fe773dfe44e206d7d126f5bc664",
"sha256:5d4030b04061fdf4cbc446008e238e44936d77a04b2b32f804688ad64197953c",
"sha256:5d69bbd9547d3bc49f8a545db7a0bd69f407badd2ff0f6e1a163680b5841d2b0",
"sha256:5f297cb343114b33a13755032ecf7109b07b9a0020e841d1c3cedff6602cc139",
"sha256:62afad6e5fd70f34d773ffcbb7c22657e1d46d7fd7c95a43361de979f0a45aef",
"sha256:647ba1df86d025f5a34043451d7c4a9f05f240bee06277a524daad11f997d1e7",
"sha256:719e169c79255816cdcf6dccd9ed2d089a72a9f6c42273aae12d55e8d35bdcf8",
"sha256:7cd5a237f241f2764324396e06298b5dee0df580cf06ef4ada0ff9bff851286c",
"sha256:875d4c60a6299f55df1c3bb870ebe6dcb7db28c165ab9ea6cdc5d5af36bb33ce",
"sha256:90b6a25841488cf2cb1c8623a53e6879573010a669455046df5f029d93db51b7",
"sha256:94620ed996a7632723a424bccb84b07e7b861ab7bb06a5aeb041c111dd723d36",
"sha256:b5f1b333015d53d4b381745f5de842f19fe59728b65f0fbb662dafbe2018c3a5",
"sha256:c5b22b31c947ad8b6964d4ed66776bcae986f73669ba50620162ba7c832a6b6a",
"sha256:c93d1a71c3fe222308939b2e516c07f35a849c5047f0197442a4d6fbcb4128ee",
"sha256:cdb90267650c1edb54459cdb51dab865f6c6594c3a47ebd441bc493360c7af70",
"sha256:cfd06e0f0cc8db2a854137bd79154b61ecd940dce96fad0cba23fe31de0b793c",
"sha256:d3789c1c394944084b5e57c192889985a9f23bd985f6d15728c745d380318128",
"sha256:da7d09ad0f24270b20f77d56934e196e982af0d0a2446120cb772be4e060e1a2",
"sha256:df3e83323268594fa9755480a442cabfe8d82b21aba815a71acf1bb6c1776218",
"sha256:df8053867c831b2643b2c489fe1d62049a98566b1646b194cc815f13e27b90df",
"sha256:e1128e022d8dce375362e063754e129750323b67454cac5600008aad9f54139e",
"sha256:e6e9fdaf6c90d02b95e6b0709aeb1aba5affbbb9ccaea5502f8638e4323206be",
"sha256:eac8803c9ad1817ce3d8d15d1bb82c2da3feda6bee1153eec5c58fa6e5d3f770",
"sha256:eb333b90036358a0e2c57373f72e7648d7207b76ef0bd00a4f7daad1f79f5203",
"sha256:ed1d1351f05e795a527abc04a0d82e9aecd3bdf9f46662c36ff47b0b00ecaf06",
"sha256:f3dc68272990849132d6698f7dc6df2ab62a88b0d36e54702a8fd16c0490e44f",
"sha256:f59eded163d9752fd49978e0bab7a1ff21b1b8d25c05f0995d140cc08ac83379",
"sha256:f5e2d36c86c7b03c94b8459c3bd2c9fe2c7dab4b258b8885617d44a22e453fb7",
"sha256:f6f65bf54215e4ebf6b01e4bb94c49180a589573df643735107056f7a910275b",
"sha256:f8450d5ef759dbe59f84f2c9f77491bb3d3c44bc1a573746daf086e70b14c243",
"sha256:f97d83049715fd9dec7911860ecf0e17b48d8725de01e45de07d8ac0bd5bc378"
],
"markers": "python_version >= '3'",
"version": "==1.0.0"
},
"gunicorn": { "gunicorn": {
"hashes": [ "hashes": [
"sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626", "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
@ -1506,11 +1549,11 @@
}, },
"identify": { "identify": {
"hashes": [ "hashes": [
"sha256:220169a38a0c977c8fef377dc808d6a3330641b5211ec7356c7bbe73cda487c7", "sha256:39c0b110c9d0cd2391b6c38cd0ff679ee4b4e98f8db8b06c5d9d9e502711a1e1",
"sha256:da3d757c94596c50865aae63db6ba4e2e5e3f666c3ea6a6da0cd09a8b2d34abc" "sha256:efbf090a619255bc31c4fbba709e2805f7d30913fd4854ad84ace52bd276e2f6"
], ],
"markers": "python_full_version >= '3.6.1'", "markers": "python_full_version >= '3.6.1'",
"version": "==2.1.1" "version": "==2.2.0"
}, },
"idna": { "idna": {
"hashes": [ "hashes": [
@ -1567,11 +1610,11 @@
}, },
"isort": { "isort": {
"hashes": [ "hashes": [
"sha256:c729845434366216d320e936b8ad6f9d681aab72dc7cbc2d51bedc3582f3ad1e", "sha256:0a943902919f65c5684ac4e0154b1ad4fac6dcaa5d9f3426b732f1c8b5419be6",
"sha256:fff4f0c04e1825522ce6949973e83110a6e907750cd92d128b0d14aaaadbffdc" "sha256:2bb1680aad211e3c9944dbce1d4ba09a989f04e238296c87fe2139faa26d655d"
], ],
"markers": "python_version >= '3.6' and python_version < '4'", "markers": "python_version >= '3.6' and python_version < '4'",
"version": "==5.7.0" "version": "==5.8.0"
}, },
"itsdangerous": { "itsdangerous": {
"hashes": [ "hashes": [
@ -1638,19 +1681,19 @@
}, },
"jupyter-client": { "jupyter-client": {
"hashes": [ "hashes": [
"sha256:5eaaa41df449167ebba5e1cf6ca9b31f7fd4f71625069836e2e4fee07fe3cb13", "sha256:c4bca1d0846186ca8be97f4d2fa6d2bae889cce4892a167ffa1ba6bd1f73e782",
"sha256:649ca3aca1e28f27d73ef15868a7c7f10d6e70f761514582accec3ca6bb13085" "sha256:e053a2c44b6fa597feebe2b3ecb5eea3e03d1d91cc94351a52931ee1426aecfc"
], ],
"markers": "python_version >= '3.5'", "markers": "python_version >= '3.5'",
"version": "==6.1.11" "version": "==6.1.12"
}, },
"jupyter-console": { "jupyter-console": {
"hashes": [ "hashes": [
"sha256:1d80c06b2d85bfb10bd5cc731b3db18e9023bc81ab00491d3ac31f206490aee3", "sha256:242248e1685039cd8bff2c2ecb7ce6c1546eb50ee3b08519729e6e881aec19c7",
"sha256:7f6194f4f4692d292da3f501c7f343ccd5e36c6a1becf7b7515e23e66d6bf1e9" "sha256:7799c4ea951e0e96ba8260575423cb323ea5a03fcf5503560fa3e15748869e27"
], ],
"markers": "python_version >= '3.6'", "markers": "python_version >= '3.6'",
"version": "==6.2.0" "version": "==6.4.0"
}, },
"jupyter-core": { "jupyter-core": {
"hashes": [ "hashes": [
@ -1730,33 +1773,31 @@
}, },
"lazy-object-proxy": { "lazy-object-proxy": {
"hashes": [ "hashes": [
"sha256:1d33d6f789697f401b75ce08e73b1de567b947740f768376631079290118ad39", "sha256:17e0967ba374fc24141738c69736da90e94419338fd4c7c7bef01ee26b339653",
"sha256:2f2de8f8ac0be3e40d17730e0600619d35c78c13a099ea91ef7fb4ad944ce694", "sha256:1fee665d2638491f4d6e55bd483e15ef21f6c8c2095f235fef72601021e64f61",
"sha256:3782931963dc89e0e9a0ae4348b44762e868ea280e4f8c233b537852a8996ab9", "sha256:22ddd618cefe54305df49e4c069fa65715be4ad0e78e8d252a33debf00f6ede2",
"sha256:37d9c34b96cca6787fe014aeb651217944a967a5b165e2cacb6b858d2997ab84", "sha256:24a5045889cc2729033b3e604d496c2b6f588c754f7a62027ad4437a7ecc4837",
"sha256:38c3865bd220bd983fcaa9aa11462619e84a71233bafd9c880f7b1cb753ca7fa", "sha256:410283732af311b51b837894fa2f24f2c0039aa7f220135192b38fcc42bd43d3",
"sha256:429c4d1862f3fc37cd56304d880f2eae5bd0da83bdef889f3bd66458aac49128", "sha256:4732c765372bd78a2d6b2150a6e99d00a78ec963375f236979c0626b97ed8e43",
"sha256:522b7c94b524389f4a4094c4bf04c2b02228454ddd17c1a9b2801fac1d754871", "sha256:489000d368377571c6f982fba6497f2aa13c6d1facc40660963da62f5c379726",
"sha256:57fb5c5504ddd45ed420b5b6461a78f58cbb0c1b0cbd9cd5a43ad30a4a3ee4d0", "sha256:4f60460e9f1eb632584c9685bccea152f4ac2130e299784dbaf9fae9f49891b3",
"sha256:5944a9b95e97de1980c65f03b79b356f30a43de48682b8bdd90aa5089f0ec1f4", "sha256:5743a5ab42ae40caa8421b320ebf3a998f89c85cdc8376d6b2e00bd12bd1b587",
"sha256:6f4e5e68b7af950ed7fdb594b3f19a0014a3ace0fedb86acb896e140ffb24302", "sha256:85fb7608121fd5621cc4377a8961d0b32ccf84a7285b4f1d21988b2eae2868e8",
"sha256:71a1ef23f22fa8437974b2d60fedb947c99a957ad625f83f43fd3de70f77f458", "sha256:9698110e36e2df951c7c36b6729e96429c9c32b3331989ef19976592c5f3c77a",
"sha256:8a44e9901c0555f95ac401377032f6e6af66d8fc1fbfad77a7a8b1a826e0b93c", "sha256:9d397bf41caad3f489e10774667310d73cb9c4258e9aed94b9ec734b34b495fd",
"sha256:b6577f15d5516d7d209c1a8cde23062c0f10625f19e8dc9fb59268859778d7d7", "sha256:b579f8acbf2bdd9ea200b1d5dea36abd93cabf56cf626ab9c744a432e15c815f",
"sha256:c8fe2d6ff0ff583784039d0255ea7da076efd08507f2be6f68583b0da32e3afb", "sha256:b865b01a2e7f96db0c5d12cfea590f98d8c5ba64ad222300d93ce6ff9138bcad",
"sha256:cadfa2c2cf54d35d13dc8d231253b7985b97d629ab9ca6e7d672c35539d38163", "sha256:bf34e368e8dd976423396555078def5cfc3039ebc6fc06d1ae2c5a65eebbcde4",
"sha256:cd1bdace1a8762534e9a36c073cd54e97d517a17d69a17985961265be6d22847", "sha256:c6938967f8528b3668622a9ed3b31d145fab161a32f5891ea7b84f6b790be05b",
"sha256:ddbdcd10eb999d7ab292677f588b658372aadb9a52790f82484a37127a390108", "sha256:d1c2676e3d840852a2de7c7d5d76407c772927addff8d742b9808fe0afccebdf",
"sha256:e7273c64bccfd9310e9601b8f4511d84730239516bada26a0c9846c9697617ef", "sha256:d7124f52f3bd259f510651450e18e0fd081ed82f3c08541dffc7b94b883aa981",
"sha256:e7428977763150b4cf83255625a80a23dfdc94d43be7791ce90799d446b4e26f", "sha256:d900d949b707778696fdf01036f58c9876a0d8bfe116e8d220cfd4b15f14e741",
"sha256:e960e8be509e8d6d618300a6c189555c24efde63e85acaf0b14b2cd1ac743315", "sha256:ebfd274dcd5133e0afae738e6d9da4323c3eb021b3e13052d8cbd0e457b1256e",
"sha256:ecb5dd5990cec6e7f5c9c1124a37cb2c710c6d69b0c1a5c4aa4b35eba0ada068", "sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93",
"sha256:ef3f5e288aa57b73b034ce9c1f1ac753d968f9069cd0742d1d69c698a0167166", "sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b"
"sha256:fa5b2dee0e231fa4ad117be114251bdfe6afe39213bd629d43deb117b6a6c40a",
"sha256:fa7fb7973c622b9e725bee1db569d2c2ee64d2f9a089201c5e8185d482c7352d"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==1.5.2" "version": "==1.6.0"
}, },
"lru-dict": { "lru-dict": {
"hashes": [ "hashes": [
@ -2038,11 +2079,11 @@
}, },
"notebook": { "notebook": {
"hashes": [ "hashes": [
"sha256:0464b28e18e7a06cec37e6177546c2322739be07962dd13bf712bcb88361f013", "sha256:cb271af1e8134e3d6fc6d458bdc79c40cbfc84c1eb036a493f216d58f0880e92",
"sha256:25ad93c982b623441b491e693ef400598d1a46cdf11b8c9c0b3be6c61ebbb6cd" "sha256:cbc9398d6c81473e9cdb891d2cae9c0d3718fca289dda6d26df5cb660fcadc7d"
], ],
"markers": "python_version >= '3.5'", "markers": "python_version >= '3.6'",
"version": "==6.2.0" "version": "==6.3.0"
}, },
"numpy": { "numpy": {
"hashes": [ "hashes": [
@ -2286,11 +2327,11 @@
}, },
"prompt-toolkit": { "prompt-toolkit": {
"hashes": [ "hashes": [
"sha256:4cea7d09e46723885cb8bc54678175453e5071e9449821dce6f017b1d1fbfc1a", "sha256:bf00f22079f5fadc949f42ae8ff7f05702826a97059ffcc6281036ad40ac6f04",
"sha256:9397a7162cf45449147ad6042fa37983a081b8a73363a5253dd4072666333137" "sha256:e1b4f11b9336a28fa11810bc623c357420f69dfdb6d2dac41ca2c21a55c033bc"
], ],
"markers": "python_full_version >= '3.6.1'", "markers": "python_full_version >= '3.6.1'",
"version": "==3.0.17" "version": "==3.0.18"
}, },
"psutil": { "psutil": {
"hashes": [ "hashes": [
@ -2517,11 +2558,11 @@
}, },
"pynmea2": { "pynmea2": {
"hashes": [ "hashes": [
"sha256:4fbb2a293883b24932e8403730f654b479828c218b00470031478c53202e2f71", "sha256:2d415c586bf2f40aebf452d62105528428806a5333321bfcdcfadf16caccbd74",
"sha256:93079d80a850d5d334635a82b85f68a67bcbdaaf4989c7fc7cffee3e68aac376" "sha256:7b84ec29b727946b8812082824c8a5298c144cc8b77a875f53d92b6832b6cdfc"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.16.0" "version": "==1.17.0"
}, },
"pyopenssl": { "pyopenssl": {
"hashes": [ "hashes": [
@ -2731,11 +2772,11 @@
}, },
"qtconsole": { "qtconsole": {
"hashes": [ "hashes": [
"sha256:0173486b9cd69e17df537fb4f1e0d62a88019f6661700a11fd7236fa89ed900b", "sha256:4a38053993ca2da058f76f8d75b3d8906efbf9183de516f92f222ac8e37d9614",
"sha256:404994edfe33c201d6bd0c4bd501b00c16125071573c938533224992bea0b30f" "sha256:c091a35607d2a2432e004c4a112d241ce908086570cf68594176dd52ccaa212d"
], ],
"markers": "python_version >= '3.6'", "markers": "python_version >= '3.6'",
"version": "==5.0.2" "version": "==5.0.3"
}, },
"qtpy": { "qtpy": {
"hashes": [ "hashes": [
@ -2785,10 +2826,10 @@
}, },
"s3transfer": { "s3transfer": {
"hashes": [ "hashes": [
"sha256:1e28620e5b444652ed752cf87c7e0cb15b0e578972568c6609f0f18212f259ed", "sha256:5d48b1fd2232141a9d5fb279709117aaba506cacea7f86f11bc392f06bfa8fc2",
"sha256:7fdddb4f22275cf1d32129e21f056337fd2a80b6ccef1664528145b72c49e6d2" "sha256:c5dadf598762899d8cfaecf68eba649cd25b0ce93b6c954b156aaa3eed160547"
], ],
"version": "==0.3.4" "version": "==0.3.6"
}, },
"scikit-image": { "scikit-image": {
"hashes": [ "hashes": [
@ -2942,47 +2983,43 @@
}, },
"sqlalchemy": { "sqlalchemy": {
"hashes": [ "hashes": [
"sha256:040bdfc1d76a9074717a3f43455685f781c581f94472b010cd6c4754754e1862", "sha256:06125670280111e39014af87f14d74599fd4b39a512c74f1a10e21e5626eb158",
"sha256:1fe5d8d39118c2b018c215c37b73fd6893c3e1d4895be745ca8ff6eb83333ed3", "sha256:09b08eb1bea621e47c2b0fcb0334fcbb00e1da2a3c2d45a98e56cd072b840719",
"sha256:23927c3981d1ec6b4ea71eb99d28424b874d9c696a21e5fbd9fa322718be3708", "sha256:0abab6d1044198993256f073340b14c459736777c550a7e914cd00444dcf9c30",
"sha256:24f9569e82a009a09ce2d263559acb3466eba2617203170e4a0af91e75b4f075", "sha256:0bb04fd7414718fb1f4dfa17efcb0be787363451cf99a5e992728925d298d9ae",
"sha256:2578dbdbe4dbb0e5126fb37ffcd9793a25dcad769a95f171a2161030bea850ff", "sha256:1b9f3c7b281aa1c3d0c74ef12c4633e5f8358bb94f01be7b964887183fd53e5e",
"sha256:269990b3ab53cb035d662dcde51df0943c1417bdab707dc4a7e4114a710504b4", "sha256:1ba6922331b3f38e116c9266206b044baf64576e5cebd87917b5ad872d7a025f",
"sha256:29cccc9606750fe10c5d0e8bd847f17a97f3850b8682aef1f56f5d5e1a5a64b1", "sha256:3b290ff34de625143a05d2d172a88a064bb04a7938265b09d4e4bf45f21948f6",
"sha256:37b83bf81b4b85dda273aaaed5f35ea20ad80606f672d94d2218afc565fb0173", "sha256:3fa75c854dba3f9b9c28bc5d88d246f6bc6f20b7480367c65339bcb2864d4707",
"sha256:63677d0c08524af4c5893c18dbe42141de7178001360b3de0b86217502ed3601", "sha256:4d1447183356c9679853926e81c7ebce3fbca9b1c607ea439975298c72137a36",
"sha256:639940bbe1108ac667dcffc79925db2966826c270112e9159439ab6bb14f8d80", "sha256:4e88549a5e58ba8c80c5ea071ac3b4e590236672a882bb80f56da4afcee45d96",
"sha256:6a939a868fdaa4b504e8b9d4a61f21aac11e3fecc8a8214455e144939e3d2aea", "sha256:5289cafee71037f15feeeaf736f01910b9e3572525b73b201bdd21816db010ed",
"sha256:6b8b8c80c7f384f06825612dd078e4a31f0185e8f1f6b8c19e188ff246334205", "sha256:59ec279f1bd55e1d703e3d4b651600cc463cc3eafa8d8e5a70ab844f736348d4",
"sha256:6c9e6cc9237de5660bcddea63f332428bb83c8e2015c26777281f7ffbd2efb84", "sha256:5fb8f6a391992dd6aafe4fdf1dffbf7934fba1f5938593f20b152aa7f9619f82",
"sha256:6ec1044908414013ebfe363450c22f14698803ce97fbb47e53284d55c5165848", "sha256:65c4df9517da9cce2c1255282d3e39f2afbc3a02deba60d99b0a3283ae80ec0b",
"sha256:6fca33672578666f657c131552c4ef8979c1606e494f78cd5199742dfb26918b", "sha256:6a8e4c2e65028933a6dc8643c8f5a4f295a367131195b3c708634925cb3e8ec1",
"sha256:751934967f5336a3e26fc5993ccad1e4fee982029f9317eb6153bc0bc3d2d2da", "sha256:6d6115edf1297bfa58994986ffe0dff21af18f0cba51dfa6d1769aa8a277be32",
"sha256:8be835aac18ec85351385e17b8665bd4d63083a7160a017bef3d640e8e65cadb", "sha256:6e517126d3bc13d455826befdc35a89f82f01d163848f68db02caa80d25433fc",
"sha256:927ce09e49bff3104459e1451ce82983b0a3062437a07d883a4c66f0b344c9b5", "sha256:7e1b0ed6d720750f02333d2f52502dfc2a23185aacc2cc6ce6ec29d28c21397c",
"sha256:94208867f34e60f54a33a37f1c117251be91a47e3bfdb9ab8a7847f20886ad06", "sha256:7eba42098a13a3bcd509080b5e44d73783d9129ba0383793979bf518d01e8bb3",
"sha256:94f667d86be82dd4cb17d08de0c3622e77ca865320e0b95eae6153faa7b4ecaf", "sha256:8383292298bb85d7ad79a13c6571aff213b96c49737f3c3af129de63bbfb42c9",
"sha256:9e9c25522933e569e8b53ccc644dc993cab87e922fb7e142894653880fdd419d", "sha256:8cfcfcf2582b19c874fa20d0b75100abe17be80a4c637c0683b4eb919946dfee",
"sha256:a0e306e9bb76fd93b29ae3a5155298e4c1b504c7cbc620c09c20858d32d16234", "sha256:920db115eb06fc507fe2c774fb5c82a898b05dffbdadc7fafad51ce2cfd8c549",
"sha256:a8bfc1e1afe523e94974132d7230b82ca7fa2511aedde1f537ec54db0399541a", "sha256:9406b96a979ab8d6de5d89f58b1f103c9aeef6fb5367448537a8228619f11258",
"sha256:ac2244e64485c3778f012951fdc869969a736cd61375fde6096d08850d8be729", "sha256:97e333260a99d989f2a131aa8aa74140636dfbd030987150cb3748da607ea7db",
"sha256:b4b0e44d586cd64b65b507fa116a3814a1a53d55dce4836d7c1a6eb2823ff8d1", "sha256:a6b4b7688fe7d251bbae3f9da4a487568bd584d13201bc7591c8639ad01fecdc",
"sha256:baeb451ee23e264de3f577fee5283c73d9bbaa8cb921d0305c0bbf700094b65b", "sha256:aed22be55a608787bb6875dbcf3561349a0e88fe33fd88c318c1e5b4eeb2306a",
"sha256:c7dc052432cd5d060d7437e217dd33c97025287f99a69a50e2dc1478dd610d64", "sha256:c6197c88ad53c31f58de5a8180936b8ef027356e788cd5f6514b3439d3d897ac",
"sha256:d1a85dfc5dee741bf49cb9b6b6b8d2725a268e4992507cf151cba26b17d97c37", "sha256:d3b2819f4d7ae56191efc6fc456eb1805ada2bd5ba93d918893bc24fa7a1e30c",
"sha256:d90010304abb4102123d10cbad2cdf2c25a9f2e66a50974199b24b468509bad5", "sha256:da72e3499bde4548e8b7d7f2ab23ceed09a5bac307bf51057e066c406a0ba2e1",
"sha256:ddfb511e76d016c3a160910642d57f4587dc542ce5ee823b0d415134790eeeb9", "sha256:dcde5067a7dab1ff2eaea2f3622b2055c5225ce2aaf589c5a4c703d43519c4ba",
"sha256:e273367f4076bd7b9a8dc2e771978ef2bfd6b82526e80775a7db52bff8ca01dd", "sha256:e1692bdf1b95c97caab1201773a4576f59627997f598d30bdadc50dd9f897fec",
"sha256:e5bb3463df697279e5459a7316ad5a60b04b0107f9392e88674d0ece70e9cf70", "sha256:edec945ed57d11a1123657e4066f0bf747aaa93c8a65ec1c2c98172d1f2a9b7d",
"sha256:e8a1750b44ad6422ace82bf3466638f1aa0862dbb9689690d5f2f48cce3476c8", "sha256:facacaea95e0822f7bbeaa6909b30b2836b14cff8790209d52a0c866e240b673",
"sha256:eab063a70cca4a587c28824e18be41d8ecc4457f8f15b2933584c6c6cccd30f0", "sha256:ff76d7dbf33f62e30e5a1d1b095d46afcdc49e42cbe33ce12014110147466700"
"sha256:ecce8c021894a77d89808222b1ff9687ad84db54d18e4bd0500ca766737faaf6",
"sha256:f4d972139d5000105fcda9539a76452039434013570d6059993120dc2a65e447",
"sha256:fd3b96f8c705af8e938eaa99cbd8fd1450f632d38cad55e7367c33b263bf98ec",
"sha256:fdd2ed7395df8ac2dbb10cefc44737b66c6a5cd7755c92524733d7a443e5b7e2"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.3.23" "version": "==1.4.2"
}, },
"subprocess32": { "subprocess32": {
"hashes": [ "hashes": [
@ -3010,11 +3047,11 @@
}, },
"terminado": { "terminado": {
"hashes": [ "hashes": [
"sha256:23a053e06b22711269563c8bb96b36a036a86be8b5353e85e804f89b84aaa23f", "sha256:261c0b7825fecf629666e1820b484a5380f7e54d6b8bd889fa482e99dcf9bde4",
"sha256:89e6d94b19e4bc9dce0ffd908dfaf55cc78a9bf735934e915a4a96f65ac9704c" "sha256:430e876ec9d4d93a4fd8a49e82dcfae0c25f846540d0c5ca774b397533e237e8"
], ],
"markers": "python_version >= '3.6'", "markers": "python_version >= '3.6'",
"version": "==0.9.2" "version": "==0.9.3"
}, },
"testpath": { "testpath": {
"hashes": [ "hashes": [
@ -3025,11 +3062,11 @@
}, },
"tifffile": { "tifffile": {
"hashes": [ "hashes": [
"sha256:8ae99b2d7a80512628dc7f7de5a178525b28cb50cc69e82578a67106c28beee6", "sha256:1b72c92ecd2273e52686c0f8792d1d1c4da4109b241dd1723dfe56ef4d1ad612",
"sha256:f55a4c8bcbfdd28157529c2e2a7c066da1636a202144f1696092b9290f52685d" "sha256:f6092aba910ed52b6087877c9a2f604ba67623a8703fa8079929b62992dcc69c"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==2021.3.5" "version": "==2021.3.17"
}, },
"toml": { "toml": {
"hashes": [ "hashes": [
@ -3139,19 +3176,19 @@
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
"sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.26.3" "version": "==1.26.4"
}, },
"virtualenv": { "virtualenv": {
"hashes": [ "hashes": [
"sha256:147b43894e51dd6bba882cf9c282447f780e2251cd35172403745fc381a0a80d", "sha256:49ec4eb4c224c6f7dd81bb6d0a28a09ecae5894f4e593c89b0db0885f565a107",
"sha256:2be72df684b74df0ea47679a7df93fd0e04e72520022c57b479d8f881485dbe3" "sha256:83f95875d382c7abafe06bd2a4cdd1b363e1bb77e02f155ebe8ac082a916b37c"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==20.4.2" "version": "==20.4.3"
}, },
"wcwidth": { "wcwidth": {
"hashes": [ "hashes": [

9
common/dict_helpers.py 100644
View File

@ -0,0 +1,9 @@
# remove all keys that end in DEPRECATED
def strip_deprecated_keys(d):
for k in list(d.keys()):
if isinstance(k, str):
if k.endswith('DEPRECATED'):
d.pop(k)
elif isinstance(d[k], dict):
strip_deprecated_keys(d[k])
return d

41
common/logging_extra.py
View File

@ -3,6 +3,7 @@ import os
import sys import sys
import copy import copy
import json import json
import uuid
import socket import socket
import logging import logging
import traceback import traceback
@ -62,8 +63,48 @@ class SwagFormatter(logging.Formatter):
return record_dict return record_dict
def format(self, record): def format(self, record):
if self.swaglogger is None:
raise Exception("must set swaglogger before calling format()")
return json_robust_dumps(self.format_dict(record)) return json_robust_dumps(self.format_dict(record))
class SwagLogFileFormatter(SwagFormatter):
def fix_kv(self, k, v):
# append type to names to preserve legacy naming in logs
# avoids overlapping key namespaces with different types
# e.g. log.info() creates 'msg' -> 'msg$s'
# log.event() creates 'msg.health.logMonoTime' -> 'msg.health.logMonoTime$i'
# because overlapping namespace 'msg' caused problems
if isinstance(v, (str, bytes)):
k += "$s"
elif isinstance(v, float):
k += "$f"
elif isinstance(v, bool):
k += "$b"
elif isinstance(v, int):
k += "$i"
elif isinstance(v, dict):
nv = {}
for ik, iv in v.items():
ik, iv = self.fix_kv(ik, iv)
nv[ik] = iv
v = nv
elif isinstance(v, list):
k += "$a"
return k, v
def format(self, record):
if isinstance(record, str):
v = json.loads(record)
else:
v = self.format_dict(record)
mk, mv = self.fix_kv('msg', v['msg'])
del v['msg']
v[mk] = mv
v['id'] = uuid.uuid4().hex
return json_robust_dumps(v)
class SwagErrorFilter(logging.Filter): class SwagErrorFilter(logging.Filter):
def filter(self, record): def filter(self, record):
return record.levelno < logging.ERROR return record.levelno < logging.ERROR

1
release/files_common
View File

@ -29,6 +29,7 @@ common/params_pyx.pyx
common/xattr.py common/xattr.py
common/profiler.py common/profiler.py
common/basedir.py common/basedir.py
common/dict_helpers.py
common/filter_simple.py common/filter_simple.py
common/stat_live.py common/stat_live.py
common/spinner.py common/spinner.py

115
selfdrive/athena/athenad.py
View File

@ -4,6 +4,7 @@ import hashlib
import io import io
import json import json
import os import os
import sys
import queue import queue
import random import random
import select import select
@ -24,18 +25,24 @@ from common.api import Api
from common.basedir import PERSIST from common.basedir import PERSIST
from common.params import Params from common.params import Params
from common.realtime import sec_since_boot from common.realtime import sec_since_boot
from selfdrive.hardware import HARDWARE from selfdrive.hardware import HARDWARE, PC
from selfdrive.loggerd.config import ROOT from selfdrive.loggerd.config import ROOT
from selfdrive.swaglog import cloudlog from selfdrive.loggerd.xattr_cache import getxattr, setxattr
from selfdrive.swaglog import cloudlog, SWAGLOG_DIR
ATHENA_HOST = os.getenv('ATHENA_HOST', 'wss://athena.comma.ai') ATHENA_HOST = os.getenv('ATHENA_HOST', 'wss://athena.comma.ai')
HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4")) HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4"))
LOCAL_PORT_WHITELIST = set([8022]) LOCAL_PORT_WHITELIST = set([8022])
LOG_ATTR_NAME = 'user.upload'
LOG_ATTR_VALUE_MAX_UNIX_TIME = int.to_bytes(2147483647, 4, sys.byteorder)
dispatcher["echo"] = lambda s: s dispatcher["echo"] = lambda s: s
payload_queue: Any = queue.Queue() recv_queue: Any = queue.Queue()
response_queue: Any = queue.Queue() send_queue: Any = queue.Queue()
upload_queue: Any = queue.Queue() upload_queue: Any = queue.Queue()
log_send_queue: Any = queue.Queue()
log_recv_queue: Any = queue.Queue()
cancelled_uploads: Any = set() cancelled_uploads: Any = set()
UploadItem = namedtuple('UploadItem', ['path', 'url', 'headers', 'created_at', 'id']) UploadItem = namedtuple('UploadItem', ['path', 'url', 'headers', 'created_at', 'id'])
@ -46,7 +53,8 @@ def handle_long_poll(ws):
threads = [ threads = [
threading.Thread(target=ws_recv, args=(ws, end_event)), threading.Thread(target=ws_recv, args=(ws, end_event)),
threading.Thread(target=ws_send, args=(ws, end_event)), threading.Thread(target=ws_send, args=(ws, end_event)),
threading.Thread(target=upload_handler, args=(end_event,)) threading.Thread(target=upload_handler, args=(end_event,)),
threading.Thread(target=log_handler, args=(end_event,)),
] + [ ] + [
threading.Thread(target=jsonrpc_handler, args=(end_event,)) threading.Thread(target=jsonrpc_handler, args=(end_event,))
for x in range(HANDLER_THREADS) for x in range(HANDLER_THREADS)
@ -64,19 +72,21 @@ def handle_long_poll(ws):
for thread in threads: for thread in threads:
thread.join() thread.join()
def jsonrpc_handler(end_event): def jsonrpc_handler(end_event):
dispatcher["startLocalProxy"] = partial(startLocalProxy, end_event) dispatcher["startLocalProxy"] = partial(startLocalProxy, end_event)
while not end_event.is_set(): while not end_event.is_set():
try: try:
data = payload_queue.get(timeout=1) data = recv_queue.get(timeout=1)
response = JSONRPCResponseManager.handle(data, dispatcher) if "method" in data and "params" in data:
response_queue.put_nowait(response) response = JSONRPCResponseManager.handle(data, dispatcher)
send_queue.put_nowait(response.json)
elif "result" in data and "id" in data:
log_recv_queue.put_nowait(data)
except queue.Empty: except queue.Empty:
pass pass
except Exception as e: except Exception as e:
cloudlog.exception("athena jsonrpc handler failed") cloudlog.exception("athena jsonrpc handler failed")
response_queue.put_nowait(json.dumps({"error": str(e)})) send_queue.put_nowait(json.dumps({"error": str(e)}))
def upload_handler(end_event): def upload_handler(end_event):
@ -244,6 +254,82 @@ def takeSnapshot():
raise Exception("not available while camerad is started") raise Exception("not available while camerad is started")
def get_logs_to_send_sorted():
# TODO: scan once then use inotify to detect file creation/deletion
curr_time = int(time.time())
logs = []
for log_entry in os.listdir(SWAGLOG_DIR):
log_path = os.path.join(SWAGLOG_DIR, log_entry)
try:
time_sent = int.from_bytes(getxattr(log_path, LOG_ATTR_NAME), sys.byteorder)
except (ValueError, TypeError):
time_sent = 0
# assume send failed and we lost the response if sent more than one hour ago
if not time_sent or curr_time - time_sent > 3600:
logs.append(log_entry)
# return logs in order they should be sent
# excluding most recent (active) log file
return sorted(logs[:-1])
def log_handler(end_event):
if PC:
return
log_files = []
last_scan = 0
log_retries = 0
while not end_event.is_set():
try:
try:
result = json.loads(log_recv_queue.get(timeout=1))
log_success = result.get("success")
log_entry = result.get("id")
log_path = os.path.join(SWAGLOG_DIR, log_entry)
if log_entry and log_success:
try:
setxattr(log_path, LOG_ATTR_NAME, LOG_ATTR_VALUE_MAX_UNIX_TIME)
except OSError:
pass # file could be deleted by log rotation
except queue.Empty:
pass
curr_scan = sec_since_boot()
if curr_scan - last_scan > 10:
log_files = get_logs_to_send_sorted()
last_scan = curr_scan
# never send last log file because it is the active log
# and only send one log file at a time (most recent first)
if not len(log_files) or not log_send_queue.empty():
continue
log_entry = log_files.pop()
try:
curr_time = int(time.time())
log_path = os.path.join(SWAGLOG_DIR, log_entry)
setxattr(log_path, LOG_ATTR_NAME, int.to_bytes(curr_time, 4, sys.byteorder))
with open(log_path, "r") as f:
jsonrpc = {
"method": "forwardLogs",
"params": {
"logs": f.read()
},
"jsonrpc": "2.0",
"id": log_entry
}
log_send_queue.put_nowait(json.dumps(jsonrpc))
except OSError:
pass # file could be deleted by log rotation
log_retries = 0
except Exception:
cloudlog.exception("athena.log_handler.exception")
log_retries += 1
if log_retries != 0:
time.sleep(backoff(log_retries))
def ws_proxy_recv(ws, local_sock, ssock, end_event, global_end_event): def ws_proxy_recv(ws, local_sock, ssock, end_event, global_end_event):
while not (end_event.is_set() or global_end_event.is_set()): while not (end_event.is_set() or global_end_event.is_set()):
try: try:
@ -290,7 +376,7 @@ def ws_recv(ws, end_event):
if opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY): if opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY):
if opcode == ABNF.OPCODE_TEXT: if opcode == ABNF.OPCODE_TEXT:
data = data.decode("utf-8") data = data.decode("utf-8")
payload_queue.put_nowait(data) recv_queue.put_nowait(data)
elif opcode == ABNF.OPCODE_PING: elif opcode == ABNF.OPCODE_PING:
Params().put("LastAthenaPingTime", str(int(sec_since_boot() * 1e9))) Params().put("LastAthenaPingTime", str(int(sec_since_boot() * 1e9)))
except WebSocketTimeoutException: except WebSocketTimeoutException:
@ -303,8 +389,11 @@ def ws_recv(ws, end_event):
def ws_send(ws, end_event): def ws_send(ws, end_event):
while not end_event.is_set(): while not end_event.is_set():
try: try:
response = response_queue.get(timeout=1) try:
ws.send(response.json) data = send_queue.get_nowait()
except queue.Empty:
data = log_send_queue.get(timeout=1)
ws.send(data)
except queue.Empty: except queue.Empty:
pass pass
except Exception: except Exception:

2
selfdrive/athena/tests/helpers.py
View File

@ -91,7 +91,7 @@ def with_http_server(func):
p.start() p.start()
time.sleep(0.1) time.sleep(0.1)
with Timeout(2): with Timeout(2, 'HTTP Server seeding failed'):
while True: while True:
try: try:
requests.put(f'http://{host}:{port}/qlog.bz2', data='') requests.put(f'http://{host}:{port}/qlog.bz2', data='')

11
selfdrive/athena/tests/test_athenad.py
View File

@ -103,6 +103,7 @@ class TestAthenadMethods(unittest.TestCase):
athenad.upload_queue.put_nowait(item) athenad.upload_queue.put_nowait(item)
try: try:
time.sleep(1) # give it time to process to prevent shutdown before upload completes
now = time.time() now = time.time()
while time.time() - now < 5: while time.time() - now < 5:
if athenad.upload_queue.qsize() == 0: if athenad.upload_queue.qsize() == 0:
@ -178,10 +179,14 @@ class TestAthenadMethods(unittest.TestCase):
thread = threading.Thread(target=athenad.jsonrpc_handler, args=(end_event,)) thread = threading.Thread(target=athenad.jsonrpc_handler, args=(end_event,))
thread.daemon = True thread.daemon = True
thread.start() thread.start()
athenad.payload_queue.put_nowait(json.dumps({"method": "echo", "params": ["hello"], "jsonrpc": "2.0", "id": 0}))
try: try:
resp = athenad.response_queue.get(timeout=3) athenad.recv_queue.put_nowait(json.dumps({"method": "echo", "params": ["hello"], "jsonrpc": "2.0", "id": 0}))
self.assertDictEqual(resp.data, {'result': 'hello', 'id': 0, 'jsonrpc': '2.0'}) resp = athenad.send_queue.get(timeout=3)
self.assertDictEqual(json.loads(resp), {'result': 'hello', 'id': 0, 'jsonrpc': '2.0'})
athenad.recv_queue.put_nowait(json.dumps({'result': {'success': 1}, 'id': 0, 'jsonrpc': '2.0'}))
resp = athenad.log_recv_queue.get(timeout=3)
self.assertDictEqual(json.loads(resp), {'result': {'success': 1}, 'id': 0, 'jsonrpc': '2.0'})
finally: finally:
end_event.set() end_event.set()
thread.join() thread.join()

23
selfdrive/logmessaged.py
View File

@ -1,12 +1,14 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import zmq import zmq
import cereal.messaging as messaging import cereal.messaging as messaging
from selfdrive.swaglog import get_le_handler from common.logging_extra import SwagLogFileFormatter
from selfdrive.swaglog import get_file_handler
def main(): def main():
le_handler = get_le_handler() log_handler = get_file_handler()
le_level = 20 # logging.INFO log_handler.setFormatter(SwagLogFileFormatter(None))
log_level = 20 # logging.INFO
ctx = zmq.Context().instance() ctx = zmq.Context().instance()
sock = ctx.socket(zmq.PULL) sock = ctx.socket(zmq.PULL)
@ -17,19 +19,14 @@ def main():
while True: while True:
dat = b''.join(sock.recv_multipart()) dat = b''.join(sock.recv_multipart())
dat = dat.decode('utf8') level = dat[0]
record = dat[1:].decode("utf-8")
levelnum = ord(dat[0]) if level >= log_level:
dat = dat[1:] log_handler.emit(record)
if levelnum >= le_level:
# push to logentries
# TODO: push to athena instead
le_handler.emit_raw(dat)
# then we publish them # then we publish them
msg = messaging.new_message() msg = messaging.new_message()
msg.logMessage = dat msg.logMessage = record
pub_sock.send(msg.to_bytes()) pub_sock.send(msg.to_bytes())

4
selfdrive/manager/build.py
View File

@ -10,7 +10,7 @@ import textwrap
from common.basedir import BASEDIR from common.basedir import BASEDIR
from common.spinner import Spinner from common.spinner import Spinner
from common.text_window import TextWindow from common.text_window import TextWindow
from selfdrive.swaglog import add_logentries_handler, cloudlog from selfdrive.swaglog import cloudlog, add_file_handler
from selfdrive.version import dirty from selfdrive.version import dirty
TOTAL_SCONS_NODES = 1225 TOTAL_SCONS_NODES = 1225
@ -70,7 +70,7 @@ def build(spinner, dirty=False):
errors = [line.decode('utf8', 'replace') for line in compile_output errors = [line.decode('utf8', 'replace') for line in compile_output
if any([err in line for err in [b'error: ', b'not found, needed by target']])] if any([err in line for err in [b'error: ', b'not found, needed by target']])]
error_s = "\n".join(errors) error_s = "\n".join(errors)
add_logentries_handler(cloudlog) add_file_handler(cloudlog)
cloudlog.error("scons build failed\n" + error_s) cloudlog.error("scons build failed\n" + error_s)
# Show TextWindow # Show TextWindow

4
selfdrive/manager/manager.py
View File

@ -16,7 +16,7 @@ from selfdrive.manager.helpers import unblock_stdout
from selfdrive.manager.process import ensure_running from selfdrive.manager.process import ensure_running
from selfdrive.manager.process_config import managed_processes from selfdrive.manager.process_config import managed_processes
from selfdrive.registration import register from selfdrive.registration import register
from selfdrive.swaglog import add_logentries_handler, cloudlog from selfdrive.swaglog import cloudlog, add_file_handler
from selfdrive.version import dirty, version from selfdrive.version import dirty, version
@ -183,7 +183,7 @@ if __name__ == "__main__":
try: try:
main() main()
except Exception: except Exception:
add_logentries_handler(cloudlog) add_file_handler(cloudlog)
cloudlog.exception("Manager failed to start") cloudlog.exception("Manager failed to start")
# Show last 3 lines of traceback # Show last 3 lines of traceback

82
selfdrive/swaglog.py
View File

@ -1,19 +1,72 @@
import os import os
from pathlib import Path
import logging import logging
from logging.handlers import BaseRotatingHandler
from logentries import LogentriesHandler
import zmq import zmq
from common.logging_extra import SwagLogger, SwagFormatter from common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
from common.realtime import sec_since_boot
from selfdrive.hardware import PC
if PC:
SWAGLOG_DIR = os.path.join(str(Path.home()), ".comma", "log")
else:
SWAGLOG_DIR = "/data/log/"
def get_le_handler(): def get_file_handler():
# setup logentries. we forward log messages to it Path(SWAGLOG_DIR).mkdir(parents=True, exist_ok=True)
le_token = "e8549616-0798-4d7e-a2ca-2513ae81fa17" base_filename = os.path.join(SWAGLOG_DIR, "swaglog")
return LogentriesHandler(le_token, use_tls=False, verbose=False) handler = SwaglogRotatingFileHandler(base_filename)
return handler
class SwaglogRotatingFileHandler(BaseRotatingHandler):
def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None):
super().__init__(base_filename, mode="a", encoding=encoding, delay=True)
self.base_filename = base_filename
self.interval = interval # seconds
self.max_bytes = max_bytes
self.backup_count = backup_count
self.log_files = self.get_existing_logfiles()
log_indexes = [f.split(".")[-1] for f in self.log_files]
self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
self.last_rollover = None
self.doRollover()
class LogMessageHandler(logging.Handler): def _open(self):
self.last_rollover = sec_since_boot()
self.last_file_idx += 1
next_filename = f"{self.base_filename}.{self.last_file_idx:010}"
stream = open(next_filename, self.mode, encoding=self.encoding)
self.log_files.insert(0, next_filename)
return stream
def get_existing_logfiles(self):
log_files = list()
base_dir = os.path.dirname(self.base_filename)
for fn in os.listdir(base_dir):
fp = os.path.join(base_dir, fn)
if fp.startswith(self.base_filename) and os.path.isfile(fp):
log_files.append(fp)
return sorted(log_files)
def shouldRollover(self, record):
size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= sec_since_boot()
return size_exceeded or time_exceeded
def doRollover(self):
if self.stream:
self.stream.close()
self.stream = self._open()
if self.backup_count > 0:
while len(self.log_files) > self.backup_count:
to_delete = self.log_files.pop()
if os.path.exists(to_delete): # just being safe, should always exist
os.remove(to_delete)
class UnixDomainSocketHandler(logging.Handler):
def __init__(self, formatter): def __init__(self, formatter):
logging.Handler.__init__(self) logging.Handler.__init__(self)
self.setFormatter(formatter) self.setFormatter(formatter)
@ -40,11 +93,13 @@ class LogMessageHandler(logging.Handler):
pass pass
def add_logentries_handler(log): def add_file_handler(log):
"""Function to add the logentries handler to swaglog. """
This can be used to send logs when logmessaged is not running.""" Function to add the file log handler to swaglog.
handler = get_le_handler() This can be used to store logs when logmessaged is not running.
handler.setFormatter(SwagFormatter(log)) """
handler = get_file_handler()
handler.setFormatter(SwagLogFileFormatter(log))
log.addHandler(handler) log.addHandler(handler)
@ -53,4 +108,5 @@ log.setLevel(logging.DEBUG)
outhandler = logging.StreamHandler() outhandler = logging.StreamHandler()
log.addHandler(outhandler) log.addHandler(outhandler)
log.addHandler(LogMessageHandler(SwagFormatter(log))) # logs are sent through IPC before writing to disk to prevent disk I/O blocking
log.addHandler(UnixDomainSocketHandler(SwagFormatter(log)))

7
selfdrive/thermald/thermald.py
View File

@ -13,6 +13,7 @@ from common.filter_simple import FirstOrderFilter
from common.numpy_fast import clip, interp from common.numpy_fast import clip, interp
from common.params import Params from common.params import Params
from common.realtime import DT_TRML, sec_since_boot from common.realtime import DT_TRML, sec_since_boot
from common.dict_helpers import strip_deprecated_keys
from selfdrive.controls.lib.alertmanager import set_offroad_alert from selfdrive.controls.lib.alertmanager import set_offroad_alert
from selfdrive.hardware import EON, TICI, HARDWARE from selfdrive.hardware import EON, TICI, HARDWARE
from selfdrive.loggerd.config import get_available_percent from selfdrive.loggerd.config import get_available_percent
@ -417,9 +418,9 @@ def thermald_thread():
location = messaging.recv_sock(location_sock) location = messaging.recv_sock(location_sock)
cloudlog.event("STATUS_PACKET", cloudlog.event("STATUS_PACKET",
count=count, count=count,
pandaState=(pandaState.to_dict() if pandaState else None), pandaState=(strip_deprecated_keys(pandaState.to_dict()) if pandaState else None),
location=(location.gpsLocationExternal.to_dict() if location else None), location=(strip_deprecated_keys(location.gpsLocationExternal.to_dict()) if location else None),
deviceState=msg.to_dict()) deviceState=strip_deprecated_keys(msg.to_dict()))
count += 1 count += 1