athena: uploads, show which items fail on error (#23412)
* athena: uploads, show which items fail on error * fix upload-id * no more 404 * Update selfdrive/athena/athenad.py Co-authored-by: Willem Melching <willem.melching@gmail.com>pull/23419/head
parent
8d80c0107f
commit
3ffebf4df5
|
@ -269,23 +269,29 @@ def uploadFileToUrl(fn, url, headers):
|
|||
@dispatcher.add_method
|
||||
def uploadFilesToUrls(files_data):
|
||||
items = []
|
||||
failed = []
|
||||
for fn, url, headers in files_data:
|
||||
if len(fn) == 0 or fn[0] == '/' or '..' in fn:
|
||||
return 500
|
||||
failed.append(fn)
|
||||
continue
|
||||
path = os.path.join(ROOT, fn)
|
||||
if not os.path.exists(path):
|
||||
return 404
|
||||
failed.append(fn)
|
||||
continue
|
||||
|
||||
item = UploadItem(path=path, url=url, headers=headers, created_at=int(time.time() * 1000), id=None)
|
||||
upload_id = hashlib.sha1(str(item).encode()).hexdigest()
|
||||
items.append(item._replace(id=upload_id))
|
||||
|
||||
for item in items:
|
||||
item = item._replace(id=upload_id)
|
||||
upload_queue.put_nowait(item)
|
||||
items.append(item._asdict())
|
||||
|
||||
UploadQueueCache.cache(upload_queue)
|
||||
|
||||
return {"enqueued": len(items), "items": [i._asdict() for i in items]}
|
||||
resp = {"enqueued": len(items), "items": items}
|
||||
if failed:
|
||||
resp["failed"] = failed
|
||||
|
||||
return resp
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
|
|
|
@ -134,13 +134,14 @@ class TestAthenadMethods(unittest.TestCase):
|
|||
@with_http_server
|
||||
def test_uploadFileToUrl(self, host):
|
||||
not_exists_resp = dispatcher["uploadFileToUrl"]("does_not_exist.bz2", "http://localhost:1238", {})
|
||||
self.assertEqual(not_exists_resp, 404)
|
||||
self.assertEqual(not_exists_resp, {'enqueued': 0, 'items': [], 'failed': ['does_not_exist.bz2']})
|
||||
|
||||
fn = os.path.join(athenad.ROOT, 'qlog.bz2')
|
||||
Path(fn).touch()
|
||||
|
||||
resp = dispatcher["uploadFileToUrl"]("qlog.bz2", f"{host}/qlog.bz2", {})
|
||||
self.assertEqual(resp['enqueued'], 1)
|
||||
self.assertNotIn('failed', resp)
|
||||
self.assertDictContainsSubset({"path": fn, "url": f"{host}/qlog.bz2", "headers": {}}, resp['items'][0])
|
||||
self.assertIsNotNone(resp['items'][0].get('id'))
|
||||
self.assertEqual(athenad.upload_queue.qsize(), 1)
|
||||
|
|
Loading…
Reference in New Issue