retropilot
/
retropilot-server
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Partial refactoring of v1.3/:dongleId/upload_url/, less code, started putting things into their own models. 

Won't work until I finish
pull/4/head
AdamSBlack 2021-05-19 22:55:19 +01:00
parent 446ab43887
commit 436aa3fda5
8 changed files with 840 additions and 798 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
controllers/authentication.js 100644
View File

@ -0,0 +1,12 @@
async function validateJWT(JWT, Key) {
}
module.exports = {
validateJWT
}

6
controllers/index.js 100644
View File

@ -0,0 +1,6 @@
module.exports = {
authenticationController: require('./authentication')
}

4
controllers/upload.js 100644
View File

@ -0,0 +1,4 @@
function getUploadURL(driveType) {}

0
database.empty.sqlite 100755 → 100644
View File

24
models/drives.js 100644
View File

@ -0,0 +1,24 @@
function getDrives(dongleId) {
}
async function getDevice(dongleId) {
return await db.get('SELECT * FROM devices WHERE dongle_id = ?', dongleId);
}
async function deviceCheckIn(dongleId) {
await db.run(
'UPDATE devices SET last_ping = ? WHERE dongle_id = ?',
Date.now(), dongle_id
);
}
module.exports = {
getDevice,
deviceCheckIn
}

6
models/index.js 100644
View File

@ -0,0 +1,6 @@
module.exports = {
drivesModel: require('./drives')
}

210
server.js
View File

@ -4,8 +4,8 @@ const path = require('path');
const crypto = require('crypto');
const log4js = require('log4js');
import sqlite3 from 'sqlite3'
import { open } from 'sqlite'
const sqlite3 = require('sqlite3')
const {open} = require('sqlite')
const lockfile = require('proper-lockfile');
@ -25,6 +25,9 @@ const htmlspecialchars = require('htmlspecialchars');
const dirTree = require("directory-tree");
const execSync = require('child_process').execSync;
const {drivesModel} = require('./models/index');
const {authenticationController} = require('./controllers/authentication');
var db = null;
var totalStorageUsed = null; // global variable that is regularly updated in the background to track the total used storage
@ -121,7 +124,7 @@ function simpleStringify (object){
simpleObject[prop] = object[prop];
}
return JSON.stringify(simpleObject); // returns cleaned up JSON
};
}
function writeFileSync(path, buffer, permission) {
var fileDescriptor;
@ -166,7 +169,7 @@ function moveUploadedFile(buffer, directory, filename) {
}
logger.error("moveUploadedFile invalid final path, check permissions to create / write '" + (config.storagePath + directory) + "'");
return false;
};
}
async function getAuthenticatedAccount(req) {
var sessionCookie = (req.signedCookies !== undefined ? req.signedCookies.session : null);
@ -188,7 +191,9 @@ function updateTotalStorageUsed() {
if (verifiedPath !== null) {
totalStorageUsed = execSync("du -hs " + verifiedPath + " | awk -F'\t' '{print $1;}'").toString();
}
setTimeout(function() {updateTotalStorageUsed();}, 120000); // update the used storage each 120 seconds
setTimeout(function () {
updateTotalStorageUsed();
}, 120000); // update the used storage each 120 seconds
}
function runAsyncWrapper(callback) {
@ -210,7 +215,11 @@ app.use(config.baseDriveDownloadPathMapping, express.static(config.storagePath))
app.use('/.well-known', express.static('.well-known'));
// DRIVE & BOOT/CRASH LOG FILE UPLOAD HANDLING
app.put('/backend/post_upload', bodyParser.raw({ inflate: true, limit: '100000kb', type: '*/*' }), runAsyncWrapper(async (req, res) => {
app.put('/backend/post_upload', bodyParser.raw({
inflate: true,
limit: '100000kb',
type: '*/*'
}), runAsyncWrapper(async (req, res) => {
var buf = new Buffer(req.body.toString('binary'), 'binary');
logger.info("HTTP.PUT /backend/post_upload for dongle " + req.query.dongleId + " with body length: " + buf.length);
@ -228,24 +237,21 @@ app.put('/backend/post_upload', bodyParser.raw({ inflate: true, limit: '100000kb
logger.error("HTTP.PUT /backend/post_upload token mismatch (" + token + " vs " + req.query.token + ")");
res.status(400);
res.send('Malformed request');
return;
}
else {
} else {
logger.info("HTTP.PUT /backend/post_upload permissions checked, calling moveUploadedFile");
var moveResult = moveUploadedFile(buf, directory, filename);
if (moveResult === false) {
logger.error("HTTP.PUT /backend/post_upload moveUploadedFile failed");
res.status(500);
res.send('Internal Server Error');
}
else {
} else {
logger.info("HTTP.PUT /backend/post_upload succesfully uploaded to " + moveResult);
res.status(200);
res.json(['OK']);
}
}
}
else { // boot or crash upload
} else { // boot or crash upload
var filename = req.query.file;
var token = crypto.createHmac('sha256', config.applicationSalt).update(dongleId + filename + ts).digest('hex');
var directory = req.query.dir;
@ -255,17 +261,15 @@ app.put('/backend/post_upload', bodyParser.raw({ inflate: true, limit: '100000kb
logger.error("HTTP.PUT /backend/post_upload token mismatch (" + token + " vs " + req.query.token + ")");
res.status(400);
res.send('Malformed request');
return;
}
else {
} else {
logger.info("HTTP.PUT /backend/post_upload permissions checked, calling moveUploadedFile");
var moveResult = moveUploadedFile(buf, directory, filename);
if (moveResult === false) {
logger.error("HTTP.PUT /backend/post_upload moveUploadedFile failed");
res.status(500);
res.send('Internal Server Error');
}
else {
} else {
logger.info("HTTP.PUT /backend/post_upload succesfully uploaded to " + moveResult);
res.status(200);
res.json(['OK']);
@ -280,122 +284,108 @@ app.get('/v1.3/:dongleId/upload_url/', runAsyncWrapper(async (req, res) => {
var path = req.query.path;
logger.info("HTTP.UPLOAD_URL called for " + req.params.dongleId + " and file " + path + ": " + JSON.stringify(req.headers));
const device = await db.get('SELECT * FROM devices WHERE dongle_id = ?', req.params.dongleId);
const dongleId = req.params.dongleId;
const auth = req.params.authorization;
if (device==null || device.account_id==0) {
logger.info("HTTP.UPLOAD_URL device "+req.params.dongleId+" not found or not linked to an account / refusing uploads");
res.status(400);
res.send('Unauthorized.');
return;
const device = await drivesModel.getDevice(dongleId);
if (!device) {
logger.info(`HTTP.UPLOAD_URL device ${dongleId} not found or not linked to an account / refusing uploads`);
return res.send('Unauthorized.').status(400)
}
var decoded=null;
if (device.public_key.length>0) {
decoded = validateJWTToken(req.headers.authorization, device.public_key);
let decoded = device.public_key ? await authenticationController.validateJWT(req.headers.authorization, device.public_key) : null;
if (decoded === null || decoded.identity !== req.params.dongleId) {
logger.info(`HTTP.UPLOAD_URL JWT authorization failed, token: ${auth} device: ${JSON.stringify(device)}, decoded: ${JSON.stringify(decoded)}`);
return res.send('Unauthorized.').status(400)
}
if (decoded==null || decoded.identity!==req.params.dongleId) {
logger.info("HTTP.UPLOAD_URL JWT authorization failed, token: '"+req.headers.authorization+"', device: "+JSON.stringify(device)+", decoded: "+JSON.stringify(decoded)+"");
res.status(400);
res.send('Unauthorized.');
return;
}
await drivesModel.deviceCheckIn(dongleId)
const result = await db.run(
'UPDATE devices SET last_ping = ? WHERE dongle_id = ?',
Date.now(), device.dongle_id
);
let responseUrl = null;
const ts = Date.now(); // we use this to make sure old URLs cannot be reused (timeout after 60min)
var responseUrl = null;
var ts = Date.now(); // we use this to make sure old URLs cannot be reused (timeout after 60min)
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(dongleId).digest('hex');
// boot log upload
if (path.indexOf("boot/")===0) {
var filename = path.replace("/", "-"); // "boot-2021-04-12--01-45-30.bz" for example
var token = crypto.createHmac('sha256', config.applicationSalt).update(req.params.dongleId+filename+ts).digest('hex');
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(req.params.dongleId).digest('hex');
var directory=req.params.dongleId+"/"+dongleIdHash+"/boot";
responseUrl = config.baseUploadUrl +'?file='+filename+'&dir='+directory+'&dongleId='+req.params.dongleId+'&ts='+ts+'&token='+token;
logger.info("HTTP.UPLOAD_URL matched 'boot' file upload, constructed responseUrl: "+responseUrl);
}
// crash log upload
if (path.indexOf("crash/")===0) {
var filename = path.replace("/", "-"); // "crash-2021-04-12--01-45-30.bz" for example
var token = crypto.createHmac('sha256', config.applicationSalt).update(req.params.dongleId+filename+ts).digest('hex');
var directory=req.params.dongleId+"/"+dongleIdHash+"/crash";
responseUrl = config.baseUploadUrl +'?file='+filename+'&dir='+directory+'&dongleId='+req.params.dongleId+'&ts='+ts+'&token='+token;
logger.info("HTTP.UPLOAD_URL matched 'crash' file upload, constructed responseUrl: "+responseUrl);
}
// drive upload
else {
if (path.indexOf("boot/") === 0 || path.indexOf("crash/") === 0) {
let filename = path.replace("/", "-");
const token = crypto.createHmac('sha256', config.applicationSalt).update(dongleId + filename + ts).digest('hex');
// TODO, allow multiple types
let uplaodType = path.indexOf("boot/") === 0 ? 'boot' : 'crash';
// "boot-2021-04-12--01-45-30.bz" for example
const directory = `${dongleId}/${dongleIdHash}/${uplaodType}`;
responseUrl = `${config.baseUploadUrl}?file=${filename}&dir=${directory}&dongleId=${dongleId}&ts=${ts}&token=${token}`;
logger.info(`HTTP.UPLOAD_URL matched '${uplaodType}' file upload, constructed responseUrl: ${responseUrl}`);
} else {
// "2021-04-12--01-44-25--0/qlog.bz2" for example
var subdirPosition = path.split("--", 2).join("--").length;
var filenamePosition = path.indexOf("/");
const subdirPosition = path.split("--", 2).join("--").length;
const filenamePosition = path.indexOf("/");
if (subdirPosition > 0 && filenamePosition > subdirPosition) {
var driveName = path.split("--")[0]+"--"+path.split("--")[1];
var segment = parseInt(path.split("--")[2].substr(0, path.split("--")[2].indexOf("/")));
var directory = path.split("--")[0]+"--"+path.split("--")[1]+"/"+segment;
var filename=path.split("/")[1];
const driveName = path.split("--")[0] + "--" + path.split("--")[1];
const segment = parseInt(path.split("--")[2].substr(0, path.split("--")[2].indexOf("/")));
let directory = path.split("--")[0] + "--" + path.split("--")[1] + "/" + segment;
const filename = path.split("/")[1];
var validRequest=false;
let validRequest = false;
if (filename=='fcamera.hevc' || filename=='qcamera.ts' || filename=='dcamera.hevc' || filename=='rlog.bz2' || filename=='qlog.bz2')
if (filename === 'fcamera.hevc' || filename === 'qcamera.ts' || filename === 'dcamera.hevc' || filename === 'rlog.bz2' || filename === 'qlog.bz2') {
validRequest = true;
if (segment==NaN || segment<0 || segment>10000)
} else if (isNaN(segment) || segment < 0 || segment > 10000) {
validRequest = false;
}
if (!validRequest) {
logger.error("HTTP.UPLOAD_URL invalid filename ("+filename+") or invalid segment ("+segment+"), responding with HTTP 400");
res.status(400);
res.send('Malformed Request.');
return;
logger.error(`HTTP.UPLOAD_URL invalid filename (${filename}) or invalid segment (${segment}), responding with HTTP 400`);
return res.send('Malformed Request.').status(400)
}
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(req.params.dongleId).digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(driveName).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(driveName).digest('hex');
directory=req.params.dongleId+"/"+dongleIdHash+"/"+driveIdentifierHash+"/"+directory;
directory = `${dongleId}/${dongleIdHash}/${driveIdentifierHash}/${directory}`;
var token = crypto.createHmac('sha256', config.applicationSalt).update(req.params.dongleId+filename+directory+ts).digest('hex');
responseUrl = config.baseUploadUrl +'?file='+filename+'&dir='+directory+'&dongleId='+req.params.dongleId+'&ts='+ts+'&token='+token;
logger.info("HTTP.UPLOAD_URL matched 'drive' file upload, constructed responseUrl: "+responseUrl);
const token = crypto.createHmac('sha256', config.applicationSalt).update(dongleId + filename + directory + ts).digest('hex');
responseUrl = `${config.baseUploadUrl}?file=${filename}&dir=${directory}&dongleId=${dongleId}&ts=${ts}&token=${token}`;
logger.info(`HTTP.UPLOAD_URL matched 'drive' file upload, constructed responseUrl: ${responseUrl}`);
const drive = await db.get('SELECT * FROM drives WHERE identifier = ? AND dongle_id = ?', driveName, req.params.dongleId);
const drive = await db.get('SELECT * FROM drives WHERE identifier = ? AND dongle_id = ?', driveName, dongleId);
if (drive == null) {
// create a new drive
var timeSplit = driveName.split('--');
var timeString = timeSplit[0]+' '+timeSplit[1].replace(/-/g, ':');
const timeSplit = driveName.split('--');
const timeString = timeSplit[0] + ' ' + timeSplit[1].replace(/-/g, ':');
const driveResult = await db.run(
'INSERT INTO drives (identifier, dongle_id, max_segment, duration, distance_meters, filesize, upload_complete, is_processed, drive_date, created, last_upload, is_preserved, is_deleted, is_physically_removed) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
driveName, req.params.dongleId, segment, 0, 0, 0, false, false, Date.parse(timeString), Date.now(), Date.now(), false, false, false);
driveName, dongleId, segment, 0, 0, 0, false, false, Date.parse(timeString), Date.now(), Date.now(), false, false, false);
const driveSegmentResult = await db.run(
'INSERT INTO drive_segments (segment_id, drive_identifier, dongle_id, duration, distance_meters, upload_complete, is_processed, is_stalled, created) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
segment, driveName, req.params.dongleId, 0, 0, false, false, false, Date.now());
segment, driveName, dongleId, 0, 0, false, false, false, Date.now());
logger.info("HTTP.UPLOAD_URL created new drive #" + JSON.stringify(driveResult.lastID));
}
else {
} else {
const driveResult = await db.run(
'UPDATE drives SET last_upload = ?, max_segment = ?, upload_complete = ?, is_processed = ? WHERE identifier = ? AND dongle_id = ?',
Date.now(), Math.max(drive.max_segment, segment), false, false, driveName, req.params.dongleId
Date.now(), Math.max(drive.max_segment, segment), false, false, driveName, dongleId
);
const drive_segment = await db.get('SELECT * FROM drive_segments WHERE drive_identifier = ? AND dongle_id = ? AND segment_id = ?', driveName, req.params.dongleId, segment);
const drive_segment = await db.get('SELECT * FROM drive_segments WHERE drive_identifier = ? AND dongle_id = ? AND segment_id = ?', driveName, dongleId, segment);
if (drive_segment == null) {
const driveSegmentResult = await db.run(
'INSERT INTO drive_segments (segment_id, drive_identifier, dongle_id, duration, distance_meters, upload_complete, is_processed, is_stalled, created) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
segment, driveName, req.params.dongleId, 0, 0, false, false, false, Date.now()
segment, driveName, dongleId, 0, 0, false, false, false, Date.now()
);
} else {
const driveSegmentResult = await db.run(
'UPDATE drive_segments SET upload_complete = ?, is_stalled = ? WHERE drive_identifier = ? AND dongle_id = ? AND segment_id = ?',
false, false, driveName, req.params.dongleId, segment
false, false, driveName, dongleId, segment
);
}
@ -408,8 +398,7 @@ app.get('/v1.3/:dongleId/upload_url/', runAsyncWrapper(async (req, res) => {
if (responseUrl != null) {
res.status(200);
res.json({'url': responseUrl, 'headers': {'Content-Type': 'application/octet-stream'}});
}
else {
} else {
logger.error("HTTP.UPLOAD_URL unable to match request, responding with HTTP 400");
res.status(400);
res.send('Malformed Request.');
@ -459,8 +448,7 @@ app.post('/v2/pilotauth/', bodyParser.urlencoded({ extended: true }), runAsyncWr
return;
}
}
}
else {
} else {
const result = await db.run(
'UPDATE devices SET last_ping = ?, public_key = ? WHERE dongle_id = ?',
Date.now(), public_key, device.dongle_id
@ -469,7 +457,7 @@ app.post('/v2/pilotauth/', bodyParser.urlencoded({ extended: true }), runAsyncWr
logger.info("HTTP.V2.PILOTAUTH REACTIVATING KNOWN DEVICE (" + imei1 + ", " + serial + ") with dongle_id " + device.dongle_id + "");
res.status(200);
res.json({dongle_id: device.dongle_id});
return;
}
})),
@ -534,7 +522,6 @@ app.post('/useradmin/auth', bodyParser.urlencoded({ extended: true }), runAsyncW
const account = await db.get('SELECT * FROM accounts WHERE email = ? AND password = ?', req.body.email, crypto.createHash('sha256').update(req.body.password + config.applicationSalt).digest('hex'));
if (!account || account.banned) {
res.status(200);
res.redirect('/useradmin?status=' + encodeURIComponent('Invalid credentials or banned account'));
@ -615,15 +602,12 @@ app.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true }),
if (err)
logger.error("USERADMIN REGISTRATION - failed to send registration token email " + (err && err.stack) + " " + reply);
});
}
else { // final registration form filled
} else { // final registration form filled
if (req.body.token != token) {
infoText = 'The registration token you entered was incorrect, please try again.<br><br>';
}
else if (req.body.password!=req.body.password2 || req.body.password.length<3) {
} else if (req.body.password != req.body.password2 || req.body.password.length < 3) {
infoText = 'The passwords you entered did not or were shorter than 3 characters, please try again.<br><br>';
}
else {
} else {
const result = await db.run(
'INSERT INTO accounts (email, password, created, banned) VALUES (?, ?, ?, ?)',
@ -633,11 +617,13 @@ app.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true }),
if (result.lastID != undefined) {
logger.info("USERADMIN REGISTRATION - created new account #" + result.lastID + " with email " + req.body.email + "");
res.cookie('session', {account: req.body.email, expires: Date.now()+1000*3600*24*365}, {signed: true});
res.cookie('session', {
account: req.body.email,
expires: Date.now() + 1000 * 3600 * 24 * 365
}, {signed: true});
res.redirect('/useradmin/overview');
return;
}
else {
} else {
logger.error("USERADMIN REGISTRATION - account creation failed, resulting account data for email " + req.body.email + " is: " + result);
infoText = 'Unable to complete account registration (database error).<br><br>';
}
@ -813,7 +799,11 @@ app.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
var timeSplit = bootlogDirectoryTree.children[i].name.replace('boot-', '').replace('crash-', '').replace('\.bz2', '').split('--');
var timeString = timeSplit[0] + ' ' + timeSplit[1].replace(/-/g, ':');
bootlogFiles.push({'name': bootlogDirectoryTree.children[i].name, 'size': bootlogDirectoryTree.children[i].size, 'date': Date.parse(timeString)});
bootlogFiles.push({
'name': bootlogDirectoryTree.children[i].name,
'size': bootlogDirectoryTree.children[i].size,
'date': Date.parse(timeString)
});
}
bootlogFiles.sort((a, b) => (a.date < b.date) ? 1 : -1);
}
@ -825,7 +815,11 @@ app.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
var timeSplit = crashlogDirectoryTree.children[i].name.replace('boot-', '').replace('crash-', '').replace('\.bz2', '').split('--');
var timeString = timeSplit[0] + ' ' + timeSplit[1].replace(/-/g, ':');
crashlogFiles.push({'name': crashlogDirectoryTree.children[i].name, 'size': crashlogDirectoryTree.children[i].size, 'date': Date.parse(timeString)});
crashlogFiles.push({
'name': crashlogDirectoryTree.children[i].name,
'size': crashlogDirectoryTree.children[i].size,
'date': Date.parse(timeString)
});
}
crashlogFiles.sort((a, b) => (a.date < b.date) ? 1 : -1);
}
@ -917,8 +911,7 @@ app.get('/useradmin/drive/:dongleId/:driveIdentifier/:action', runAsyncWrapper(a
'UPDATE drives SET is_deleted = ? WHERE id = ?',
true, drive.id
);
}
else if (req.params.action=='preserve') {
} else if (req.params.action == 'preserve') {
const result = await db.run(
'UPDATE drives SET is_preserved = ? WHERE id = ?',
true, drive.id
@ -987,7 +980,6 @@ app.get('/useradmin/drive/:dongleId/:driveIdentifier', runAsyncWrapper(async (re
`;
var directorySegments = {};
for (var i in directoryTree.children) {
// skip any non-directory entries (for example m3u8 file in the drive directory)
@ -1033,8 +1025,7 @@ app.get('/useradmin/drive/:dongleId/:driveIdentifier', runAsyncWrapper(async (re
for (var i = 0; i <= drive.max_segment; i++) {
if (directorySegments["seg-" + i] == undefined) {
response += '<tr><td>' + i + '</td><td>' + qcamera + '</td><td>' + qlog + '</td><td>' + fcamera + '</td><td>' + rlog + '</td><td>' + dcamera + '</td><td>' + isProcessed + '</td><td>' + isStalled + '</td></tr>';
}
else
} else
response += directorySegments["seg-" + i];
}
@ -1071,7 +1062,6 @@ app.post('*', runAsyncWrapper(async (req, res) => {
}));
lockfile.lock('retropilot_server.lock', {realpath: false, stale: 30000, update: 2000})
.then((release) => {
console.log("STARTING SERVER...");

6
worker.js
View File

@ -130,7 +130,7 @@ function simpleStringify (object){
simpleObject[prop] = object[prop];
}
return JSON.stringify(simpleObject); // returns cleaned up JSON
};
}
function writeFileSync(path, buffer, permission) {
var fileDescriptor;
@ -175,7 +175,7 @@ function moveUploadedFile(buffer, directory, filename) {
}
logger.error("moveUploadedFile invalid final path, check permissions to create / write '"+(config.storagePath+directory)+"'");
return false;
};
}
function deleteFolderRecursive(directoryPath) {
@ -190,7 +190,7 @@ function deleteFolderRecursive(directoryPath) {
});
fs.rmdirSync(directoryPath);
}
};
}
var segmentProcessQueue=[];
var segmentProcessPosition=0;