TODO: updated all quota / expire / delete drive functionality including recalculating device used storage in worker.js
also fixed some bugs, some performance improvements, removed drive count limit and added a "MOTD" field rendered on the login page.pull/4/head
parent
a2f7c65cea
commit
e000d7a7a6
|
@ -19,12 +19,13 @@ var config = {
|
||||||
baseDriveDownloadPathMapping: '/realdata', // path mapping of above download url for expressjs, prefix with "/"
|
baseDriveDownloadPathMapping: '/realdata', // path mapping of above download url for expressjs, prefix with "/"
|
||||||
|
|
||||||
storagePath: 'realdata/', // relative or absolute ( "/..." for absolute path )
|
storagePath: 'realdata/', // relative or absolute ( "/..." for absolute path )
|
||||||
|
|
||||||
deviceStorageQuotaMb: 200000,
|
deviceStorageQuotaMb: 200000,
|
||||||
deviceDriveQuota: 1000,
|
|
||||||
deviceDriveExpirationDays: 30,
|
deviceDriveExpirationDays: 30,
|
||||||
|
|
||||||
cabanaUrl: 'http://192.168.1.165:3001/'
|
cabanaUrl: 'http://192.168.1.165:3001/',
|
||||||
|
|
||||||
|
welcomeMessage: `<><><><><><><><><><><><><><><><><><><><><><><br>2021 RetroPilot`
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = config;
|
module.exports = config;
|
||||||
|
|
61
server.js
61
server.js
|
@ -23,11 +23,16 @@ const sendmail = require('sendmail')();
|
||||||
const htmlspecialchars = require('htmlspecialchars');
|
const htmlspecialchars = require('htmlspecialchars');
|
||||||
|
|
||||||
const dirTree = require("directory-tree");
|
const dirTree = require("directory-tree");
|
||||||
|
const execSync = require('child_process').execSync;
|
||||||
|
|
||||||
|
|
||||||
const adapter = new FileSync(config.databaseFile);
|
const adapter = new FileSync(config.databaseFile);
|
||||||
const db = low(adapter);
|
const db = low(adapter);
|
||||||
|
|
||||||
|
const ALL = 1E8;
|
||||||
|
|
||||||
|
var totalStorageUsed=null; // global variable that is regularly updated in the background to track the total used storage
|
||||||
|
|
||||||
|
|
||||||
log4js.configure({
|
log4js.configure({
|
||||||
appenders: { logfile: { type: "file", filename: "server.log" }, out: { type: "console"} },
|
appenders: { logfile: { type: "file", filename: "server.log" }, out: { type: "console"} },
|
||||||
|
@ -192,6 +197,13 @@ function getAuthenticatedAccount(req) {
|
||||||
return account.value();
|
return account.value();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function updateTotalStorageUsed() {
|
||||||
|
var verifiedPath = mkDirByPathSync(config.storagePath, {isRelativeToScript: (config.storagePath.indexOf("/")===0 ? false : true)});
|
||||||
|
if (verifiedPath!==null) {
|
||||||
|
totalStorageUsed = execSync("du -hs "+verifiedPath+" | awk -F'\t' '{print $1;}'").toString();
|
||||||
|
}
|
||||||
|
setTimeout(function() {updateTotalStorageUsed();}, 120000); // update the used storage each 120 seconds
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// CREATE OUR SERVER EXPRESS APP
|
// CREATE OUR SERVER EXPRESS APP
|
||||||
|
@ -271,7 +283,7 @@ app.put('/backend/post_upload', bodyParser.raw({ inflate: true, limit: '100000kb
|
||||||
|
|
||||||
// DRIVE & BOOT/CRASH LOG FILE UPLOAD URL REQUEST
|
// DRIVE & BOOT/CRASH LOG FILE UPLOAD URL REQUEST
|
||||||
app.get('/v1.3/:dongleId/upload_url/', (req, res) => {
|
app.get('/v1.3/:dongleId/upload_url/', (req, res) => {
|
||||||
var path = req.query.path; // todo: validate filename
|
var path = req.query.path;
|
||||||
logger.info("HTTP.UPLOAD_URL called for "+req.params.dongleId+" and file "+path+": "+JSON.stringify(req.headers));
|
logger.info("HTTP.UPLOAD_URL called for "+req.params.dongleId+" and file "+path+": "+JSON.stringify(req.headers));
|
||||||
|
|
||||||
var device = db.get('devices').find({ dongle_id: req.params.dongleId});
|
var device = db.get('devices').find({ dongle_id: req.params.dongleId});
|
||||||
|
@ -425,7 +437,7 @@ app.post('/v2/pilotauth/', bodyParser.urlencoded({ extended: true }), (req, res)
|
||||||
var device = db.get('devices').find({dongle_id: dongleId}).value();
|
var device = db.get('devices').find({dongle_id: dongleId}).value();
|
||||||
if (!device) {
|
if (!device) {
|
||||||
var resultingDevice = db.get('devices')
|
var resultingDevice = db.get('devices')
|
||||||
.push({ dongle_id: dongleId, account_id: 0, imei: imei1, serial: serial, device_type: 'freon', public_key: public_key, created: Date.now(), last_ping: Date.now()})
|
.push({ dongle_id: dongleId, account_id: 0, imei: imei1, serial: serial, device_type: 'freon', public_key: public_key, created: Date.now(), last_ping: Date.now(), storage_used: 0})
|
||||||
.write();
|
.write();
|
||||||
|
|
||||||
var device = db.get('devices').find({dongle_id: dongleId}).value();
|
var device = db.get('devices').find({dongle_id: dongleId}).value();
|
||||||
|
@ -529,12 +541,6 @@ app.get('/useradmin', (req, res) => {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var verifiedPath = mkDirByPathSync(config.storagePath, {isRelativeToScript: (config.storagePath.indexOf("/")===0 ? false : true)});
|
|
||||||
if (verifiedPath!==null) {
|
|
||||||
const execSync = require('child_process').execSync;
|
|
||||||
bytes = execSync("du -hs "+verifiedPath+" | awk -F'\t' '{print $1;}'").toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
res.status(200);
|
res.status(200);
|
||||||
res.send('<html style="font-family: monospace"><h2>Welcome To The RetroPilot Server Dashboard!</h2>'+
|
res.send('<html style="font-family: monospace"><h2>Welcome To The RetroPilot Server Dashboard!</h2>'+
|
||||||
`<br><br>
|
`<br><br>
|
||||||
|
@ -548,8 +554,7 @@ app.get('/useradmin', (req, res) => {
|
||||||
'Accounts: '+db.get('accounts').size().value()+' | '+
|
'Accounts: '+db.get('accounts').size().value()+' | '+
|
||||||
'Devices: '+db.get('devices').size().value()+' | '+
|
'Devices: '+db.get('devices').size().value()+' | '+
|
||||||
'Drives: '+db.get('drives').size().value()+' | '+
|
'Drives: '+db.get('drives').size().value()+' | '+
|
||||||
'Storage Used: '+(verifiedPath!==null ? bytes : '--')+'</html>');
|
'Storage Used: '+(totalStorageUsed!==null ? totalStorageUsed : '--')+'<br><br>'+config.welcomeMessage+'</html>');
|
||||||
|
|
||||||
}),
|
}),
|
||||||
|
|
||||||
|
|
||||||
|
@ -673,7 +678,7 @@ app.get('/useradmin/overview', (req, res) => {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const devices = db.get('devices').filter({account_id: account.id}).sortBy('dongle_id').take(1000).value();
|
const devices = db.get('devices').filter({account_id: account.id}).sortBy('dongle_id').take(ALL).value();
|
||||||
|
|
||||||
var response = '<html style="font-family: monospace"><h2>Welcome To The RetroPilot Server Dashboard!</h2>'+
|
var response = '<html style="font-family: monospace"><h2>Welcome To The RetroPilot Server Dashboard!</h2>'+
|
||||||
|
|
||||||
|
@ -683,11 +688,11 @@ app.get('/useradmin/overview', (req, res) => {
|
||||||
<b>Created:</b> `+formatDate(account.created)+`<br><br>
|
<b>Created:</b> `+formatDate(account.created)+`<br><br>
|
||||||
<b>Devices:</b><br>
|
<b>Devices:</b><br>
|
||||||
<table border=1 cellpadding=2 cellspacing=2>
|
<table border=1 cellpadding=2 cellspacing=2>
|
||||||
<tr><th>dongle_id</th><th>device_type</th><th>created</th><th>last_ping</th></tr>
|
<tr><th>dongle_id</th><th>device_type</th><th>created</th><th>last_ping</th><th>storage_used</th></tr>
|
||||||
`;
|
`;
|
||||||
|
|
||||||
for (var i in devices) {
|
for (var i in devices) {
|
||||||
response+='<tr><td><a href="/useradmin/device/'+devices[i].dongle_id+'">'+devices[i].dongle_id+'</a></td><td>'+devices[i].device_type+'</td><td>'+formatDate(devices[i].created)+'</td><td>'+formatDate(devices[i].last_ping)+'</td></tr>';
|
response+='<tr><td><a href="/useradmin/device/'+devices[i].dongle_id+'">'+devices[i].dongle_id+'</a></td><td>'+devices[i].device_type+'</td><td>'+formatDate(devices[i].created)+'</td><td>'+formatDate(devices[i].last_ping)+'</td><td>'+devices[i].storage_used+' MB</td></tr>';
|
||||||
}
|
}
|
||||||
response+=`</table>
|
response+=`</table>
|
||||||
<br>
|
<br>
|
||||||
|
@ -771,7 +776,7 @@ app.get('/useradmin/device/:dongleId', (req, res) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
device = device.value();
|
device = device.value();
|
||||||
const drives = db.get('drives').filter({dongle_id: device.dongle_id, is_deleted: false}).sortBy('created').take(1000).value();
|
const drives = db.get('drives').filter({dongle_id: device.dongle_id, is_deleted: false}).sortBy('created').take(ALL).value();
|
||||||
|
|
||||||
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
|
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
|
||||||
|
|
||||||
|
@ -812,8 +817,8 @@ app.get('/useradmin/device/:dongleId', (req, res) => {
|
||||||
<b>Last Ping:</b> `+formatDate(device.last_ping)+`<br>
|
<b>Last Ping:</b> `+formatDate(device.last_ping)+`<br>
|
||||||
<b>Public Key:</b><br><span style="font-size: 0.8em">`+device.public_key.replace(/\r?\n|\r/g, "<br>")+`</span>
|
<b>Public Key:</b><br><span style="font-size: 0.8em">`+device.public_key.replace(/\r?\n|\r/g, "<br>")+`</span>
|
||||||
<br>
|
<br>
|
||||||
<b>QuotaDrives:</b> `+drives.length+` / `+config.deviceDriveQuota+`<br>
|
<b>Stored Drives:</b> `+drives.length+`<br>
|
||||||
<b>Quota Storage:</b> `+(0)+` MB / `+config.deviceStorageQuotaMb+` MB<br>
|
<b>Quota Storage:</b> `+device.storage_used+` MB / `+config.deviceStorageQuotaMb+` MB<br>
|
||||||
<br>
|
<br>
|
||||||
`;
|
`;
|
||||||
|
|
||||||
|
@ -953,6 +958,7 @@ app.get('/useradmin/drive/:dongleId/:driveIdentifier', (req, res) => {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
var directorySegments={};
|
||||||
for (var i in directoryTree.children) {
|
for (var i in directoryTree.children) {
|
||||||
// skip any non-directory entries (for example m3u8 file in the drive directory)
|
// skip any non-directory entries (for example m3u8 file in the drive directory)
|
||||||
if (directoryTree.children[i].type!='directory') continue;
|
if (directoryTree.children[i].type!='directory') continue;
|
||||||
|
@ -982,9 +988,25 @@ app.get('/useradmin/drive/:dongleId/:driveIdentifier', (req, res) => {
|
||||||
isStalled=drive_segment.is_stalled;
|
isStalled=drive_segment.is_stalled;
|
||||||
}
|
}
|
||||||
|
|
||||||
response+='<tr><td>'+segment+'</td><td>'+qcamera+'</td><td>'+qlog+'</td><td>'+fcamera+'</td><td>'+rlog+'</td><td>'+dcamera+'</td><td>'+isProcessed+'</td><td>'+isStalled+'</td></tr>';
|
directorySegments["seg-"+segment] = '<tr><td>'+segment+'</td><td>'+qcamera+'</td><td>'+qlog+'</td><td>'+fcamera+'</td><td>'+rlog+'</td><td>'+dcamera+'</td><td>'+isProcessed+'</td><td>'+isStalled+'</td></tr>';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var qcamera = '--';
|
||||||
|
var fcamera = '--';
|
||||||
|
var dcamera = '--';
|
||||||
|
var qlog = '--';
|
||||||
|
var rlog = '--';
|
||||||
|
var isProcessed='?';
|
||||||
|
var isStalled='?';
|
||||||
|
|
||||||
|
for (var i=0; i<=drive.max_segment; i++) {
|
||||||
|
if (directorySegments["seg-"+i]==undefined) {
|
||||||
|
response+='<tr><td>'+i+'</td><td>'+qcamera+'</td><td>'+qlog+'</td><td>'+fcamera+'</td><td>'+rlog+'</td><td>'+dcamera+'</td><td>'+isProcessed+'</td><td>'+isStalled+'</td></tr>';
|
||||||
|
}
|
||||||
|
else
|
||||||
|
response+=directorySegments["seg-"+i];
|
||||||
|
}
|
||||||
|
|
||||||
response+=`</table>
|
response+=`</table>
|
||||||
<br><br>
|
<br><br>
|
||||||
<hr/>
|
<hr/>
|
||||||
|
@ -1019,11 +1041,12 @@ app.post('*', (req, res) => {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
lockfile.lock('retropilot_server.lock', { realpath: false, stale: 30000, update: 2000 })
|
lockfile.lock('retropilot_server.lock'+Math.random(), { realpath: false, stale: 30000, update: 2000 })
|
||||||
.then((release) => {
|
.then((release) => {
|
||||||
console.log("STARTING SERVER...");
|
console.log("STARTING SERVER...");
|
||||||
initializeDatabase();
|
initializeDatabase();
|
||||||
initializeStorage();
|
initializeStorage();
|
||||||
|
updateTotalStorageUsed();
|
||||||
|
|
||||||
var privateKey = fs.readFileSync(config.sslKey, 'utf8');
|
var privateKey = fs.readFileSync(config.sslKey, 'utf8');
|
||||||
var certificate = fs.readFileSync(config.sslCrt, 'utf8');
|
var certificate = fs.readFileSync(config.sslCrt, 'utf8');
|
||||||
|
|
176
worker.js
176
worker.js
|
@ -24,6 +24,7 @@ const htmlspecialchars = require('htmlspecialchars');
|
||||||
|
|
||||||
const dirTree = require("directory-tree");
|
const dirTree = require("directory-tree");
|
||||||
const { resolve } = require('path');
|
const { resolve } = require('path');
|
||||||
|
const execSync = require('child_process').execSync;
|
||||||
|
|
||||||
|
|
||||||
const Reader = require('@commaai/log_reader');
|
const Reader = require('@commaai/log_reader');
|
||||||
|
@ -34,6 +35,10 @@ const { exception } = require('console');
|
||||||
const adapter = new FileSync(config.databaseFile);
|
const adapter = new FileSync(config.databaseFile);
|
||||||
const db = low(adapter);
|
const db = low(adapter);
|
||||||
|
|
||||||
|
const ALL = 1E8;
|
||||||
|
var lastCleaningTime=0;
|
||||||
|
var startTime=Date.now();
|
||||||
|
|
||||||
|
|
||||||
log4js.configure({
|
log4js.configure({
|
||||||
appenders: { logfile: { type: "file", filename: "worker.log" }, out: { type: "console"} },
|
appenders: { logfile: { type: "file", filename: "worker.log" }, out: { type: "console"} },
|
||||||
|
@ -183,12 +188,25 @@ function moveUploadedFile(buffer, directory, filename) {
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
function deleteFolderRecursive(directoryPath) {
|
||||||
|
if (fs.existsSync(directoryPath)) {
|
||||||
|
fs.readdirSync(directoryPath).forEach((file, index) => {
|
||||||
|
const curPath = path.join(directoryPath, file);
|
||||||
|
if (fs.lstatSync(curPath).isDirectory()) {
|
||||||
|
deleteFolderRecursive(curPath);
|
||||||
|
} else {
|
||||||
|
fs.unlinkSync(curPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
fs.rmdirSync(directoryPath);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
var segmentProcessQueue=[];
|
var segmentProcessQueue=[];
|
||||||
var segmentProcessPosition=0;
|
var segmentProcessPosition=0;
|
||||||
|
|
||||||
var affectedDrives={};
|
var affectedDrives={};
|
||||||
|
var affectedDevices={};
|
||||||
|
|
||||||
var rlog_lastTs=0;
|
var rlog_lastTs=0;
|
||||||
var rlog_prevLat=-1000;
|
var rlog_prevLat=-1000;
|
||||||
|
@ -293,7 +311,7 @@ function updateSegments() {
|
||||||
segmentProcessPosition=0;
|
segmentProcessPosition=0;
|
||||||
affectedDrives={};
|
affectedDrives={};
|
||||||
|
|
||||||
drive_segments = db.get('drive_segments').filter({upload_complete: false, is_stalled: false}).sortBy('created').take(10000).value();
|
drive_segments = db.get('drive_segments').filter({upload_complete: false, is_stalled: false}).sortBy('created').take(ALL).value();
|
||||||
for (var t=0; t<drive_segments.length; t++) {
|
for (var t=0; t<drive_segments.length; t++) {
|
||||||
var segment = drive_segments[t];
|
var segment = drive_segments[t];
|
||||||
|
|
||||||
|
@ -345,6 +363,23 @@ function updateSegments() {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function updateDevices() {
|
||||||
|
// go through all affected devices (with deleted or updated drives) and update them (storage_used)
|
||||||
|
logger.info("updateDevices - affected drives: "+JSON.stringify(affectedDevices));
|
||||||
|
for (const [key, value] of Object.entries(affectedDevices)) {
|
||||||
|
var dongleId = key;
|
||||||
|
|
||||||
|
var device = db.get('devices').find({dongle_id: dongleId});
|
||||||
|
if (!device.value()) continue;
|
||||||
|
|
||||||
|
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.value().dongle_id).digest('hex');
|
||||||
|
var devicePath=config.storagePath+device.value().dongle_id+"/"+dongleIdHash;
|
||||||
|
var deviceQuotaMb = Math.round(parseInt(execSync("du -s "+devicePath+" | awk -F'\t' '{print $1;}'").toString())/1024);
|
||||||
|
logger.info("updateDevices device "+dongleId+" has an updated storage_used of: "+deviceQuotaMb+" MB");
|
||||||
|
device.assign({storage_used: deviceQuotaMb}).write();
|
||||||
|
}
|
||||||
|
affectedDevices=[];
|
||||||
|
}
|
||||||
|
|
||||||
function updateDrives() {
|
function updateDrives() {
|
||||||
// go through all affected drives and update them / complete and/or build m3u8
|
// go through all affected drives and update them / complete and/or build m3u8
|
||||||
|
@ -366,7 +401,7 @@ function updateDrives() {
|
||||||
var totalDurationSeconds=0;
|
var totalDurationSeconds=0;
|
||||||
var playlistSegmentStrings='';
|
var playlistSegmentStrings='';
|
||||||
|
|
||||||
drive_segments = db.get('drive_segments').filter({drive_identifier: driveIdentifier, dongle_id: dongleId}).sortBy('created').take(10000).value();
|
drive_segments = db.get('drive_segments').filter({drive_identifier: driveIdentifier, dongle_id: dongleId}).sortBy('created').take(ALL).value();
|
||||||
for (var t=0; t<drive_segments.length; t++) {
|
for (var t=0; t<drive_segments.length; t++) {
|
||||||
if (!drive_segments[t].upload_complete) uploadComplete=false;
|
if (!drive_segments[t].upload_complete) uploadComplete=false;
|
||||||
if (!drive_segments[t].is_processed) isProcessed=false;
|
if (!drive_segments[t].is_processed) isProcessed=false;
|
||||||
|
@ -382,7 +417,6 @@ function updateDrives() {
|
||||||
var updates = {distance_meters: Math.round(totalDistanceMeters), duration: totalDurationSeconds, upload_complete : uploadComplete, is_processed : isProcessed};
|
var updates = {distance_meters: Math.round(totalDistanceMeters), duration: totalDurationSeconds, upload_complete : uploadComplete, is_processed : isProcessed};
|
||||||
if (uploadComplete) {
|
if (uploadComplete) {
|
||||||
updates['filesize'] = 0;
|
updates['filesize'] = 0;
|
||||||
const execSync = require('child_process').execSync;
|
|
||||||
try {
|
try {
|
||||||
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(dongleId).digest('hex');
|
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(dongleId).digest('hex');
|
||||||
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(driveIdentifier).digest('hex');
|
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(driveIdentifier).digest('hex');
|
||||||
|
@ -393,6 +427,8 @@ function updateDrives() {
|
||||||
logger.info("updateDrives drive "+dongleId+" "+driveIdentifier+" uploadComplete: "+JSON.stringify(updates));
|
logger.info("updateDrives drive "+dongleId+" "+driveIdentifier+" uploadComplete: "+JSON.stringify(updates));
|
||||||
drive.assign(updates).write();
|
drive.assign(updates).write();
|
||||||
|
|
||||||
|
affectedDevices[dongleId]=true;
|
||||||
|
|
||||||
if (isProcessed) {
|
if (isProcessed) {
|
||||||
// create the playlist file m3u8 for cabana
|
// create the playlist file m3u8 for cabana
|
||||||
var playlist = `#EXTM3U\n`+
|
var playlist = `#EXTM3U\n`+
|
||||||
|
@ -407,16 +443,130 @@ function updateDrives() {
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
updateDevices();
|
||||||
|
|
||||||
setTimeout(function() {mainWorkerLoop();}, 0);
|
setTimeout(function() {mainWorkerLoop();}, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
function deleteExpiredDrives() {
|
function deleteExpiredDrives() {
|
||||||
// TODO: implement
|
var expirationTs = Date.now()-config.deviceDriveExpirationDays*24*3600*1000;
|
||||||
|
|
||||||
|
var expiredDrives = db.get('drives').filter({is_preserved: false, is_deleted: false}).orderBy('created', 'asc').take(ALL).value();
|
||||||
|
for (var t=0; t<expiredDrives.length; t++) {
|
||||||
|
if (expiredDrives[t].created>expirationTs) {
|
||||||
|
break; // the drives are queried ordered by date, so break at the first newer one
|
||||||
|
}
|
||||||
|
|
||||||
|
var drive = db.get('drives').find({ identifier: expiredDrives[t].identifier, dongle_id: expiredDrives[t].dongle_id});
|
||||||
|
if (!drive.value()) continue;
|
||||||
|
logger.info("deleteExpiredDrives drive "+expiredDrives[t].dongle_id+" "+expiredDrives[t].identifier+" is older than "+config.deviceDriveExpirationDays+" days, set is_deleted=true");
|
||||||
|
drive.assign({is_deleted: true}).write();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function removeDeletedDrivesPhysically() {
|
function removeDeletedDrivesPhysically() {
|
||||||
// TODO: implement
|
var expiredDrives = db.get('drives').filter({is_deleted: true}).orderBy('created', 'asc').take(ALL).value();
|
||||||
|
for (var t=0; t<expiredDrives.length; t++) {
|
||||||
|
logger.info("removeDeletedDrivesPhysically drive "+expiredDrives[t].dongle_id+" "+expiredDrives[t].identifier+" is deleted, remove physical files and clean database");
|
||||||
|
var drive = db.get('drives').find({ identifier: expiredDrives[t].identifier, dongle_id: expiredDrives[t].dongle_id});
|
||||||
|
if (!drive.value()) continue;
|
||||||
|
|
||||||
|
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(expiredDrives[t].dongle_id).digest('hex');
|
||||||
|
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(expiredDrives[t].identifier).digest('hex');
|
||||||
|
|
||||||
|
const drivePath = config.storagePath+expiredDrives[t].dongle_id+"/"+dongleIdHash+"/"+driveIdentifierHash+"";
|
||||||
|
logger.info("removeDeletedDrivesPhysically drive "+expiredDrives[t].dongle_id+" "+expiredDrives[t].identifier+" storage path is "+drivePath);
|
||||||
|
try {
|
||||||
|
deleteFolderRecursive(drivePath, { recursive: true });
|
||||||
|
db.get('drives').remove({ identifier: expiredDrives[t].identifier, dongle_id: expiredDrives[t].dongle_id}).write();
|
||||||
|
affectedDevices[expiredDrives[t].dongle_id]=true;
|
||||||
|
} catch (exception) {
|
||||||
|
logger.error(exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function deleteOverQuotaDrives() {
|
||||||
|
var devices = db.get('devices').filter({}).orderBy('storage_used', 'desc').take(ALL).value();
|
||||||
|
for (var t=0; t<devices.length; t++) {
|
||||||
|
if (devices[t].storage_used>config.deviceStorageQuotaMb) {
|
||||||
|
var foundDriveToDelete=false;
|
||||||
|
|
||||||
|
var allDrives = db.get('drives').filter({dongle_id: devices[t].dongle_id, is_preserved: false, is_deleted: false}).orderBy('created', 'asc').take(1).value();
|
||||||
|
for (var i=0; i<allDrives.length; i++) {
|
||||||
|
logger.info("deleteExpiredDrives drive "+allDrives[i].dongle_id+" "+allDrives[i].identifier+" (normal) is deleted for over-quota");
|
||||||
|
var drive = db.get('drives').find({ identifier: allDrives[i].identifier, dongle_id: allDrives[i].dongle_id});
|
||||||
|
if (!drive.value()) continue;
|
||||||
|
drive.assign({is_deleted: true}).write();
|
||||||
|
foundDriveToDelete=true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!foundDriveToDelete) {
|
||||||
|
var allDrives = db.get('drives').filter({dongle_id: devices[t].dongle_id, is_preserved: true, is_deleted: false}).orderBy('created', 'asc').take(1).value();
|
||||||
|
for (var i=0; i<allDrives.length; i++) {
|
||||||
|
logger.info("deleteOverQuotaDrives drive "+allDrives[i].dongle_id+" "+allDrives[i].identifier+" (preserved!) is deleted for over-quota");
|
||||||
|
var drive = db.get('drives').find({ identifier: allDrives[i].identifier, dongle_id: allDrives[i].dongle_id});
|
||||||
|
if (!drive.value()) continue;
|
||||||
|
drive.assign({is_deleted: true}).write();
|
||||||
|
foundDriveToDelete=true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function deleteBootAndCrashLogs() {
|
||||||
|
var devices = db.get('devices').filter({}).take(ALL).value();
|
||||||
|
for (var t=0; t<devices.length; t++) {
|
||||||
|
var device = devices[t];
|
||||||
|
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
|
||||||
|
|
||||||
|
const bootlogDirectoryTree = dirTree(config.storagePath+device.dongle_id+"/"+dongleIdHash+"/boot/", {attributes:['size']});
|
||||||
|
var bootlogFiles = [];
|
||||||
|
if (bootlogDirectoryTree!=undefined) {
|
||||||
|
for (var i=0; i<bootlogDirectoryTree.children.length; i++) {
|
||||||
|
|
||||||
|
var timeSplit = bootlogDirectoryTree.children[i].name.replace('boot-', '').replace('crash-', '').replace('\.bz2', '').split('--');
|
||||||
|
var timeString = timeSplit[0]+' '+timeSplit[1].replace('-',':');
|
||||||
|
bootlogFiles.push({'name': bootlogDirectoryTree.children[i].name, 'size': bootlogDirectoryTree.children[i].size, 'date': Date.parse(timeString), 'path' : bootlogDirectoryTree.children[i].path});
|
||||||
|
}
|
||||||
|
bootlogFiles.sort((a,b) => (a.date < b.date) ? 1 : -1);
|
||||||
|
for (var c=5; c<bootlogFiles.length; c++) {
|
||||||
|
logger.info("deleteBootAndCrashLogs deleting boot log "+bootlogFiles[c]['path']+"");
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(bootlogFiles[c]['path']);
|
||||||
|
affectedDevices[device.dongle_id]=true;
|
||||||
|
} catch (exception) {
|
||||||
|
logger.error(exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const crashlogDirectoryTree = dirTree(config.storagePath+device.dongle_id+"/"+dongleIdHash+"/crash/", {attributes:['size']});
|
||||||
|
var crashlogFiles = [];
|
||||||
|
if (crashlogDirectoryTree!=undefined) {
|
||||||
|
for (var i=0; i<crashlogDirectoryTree.children.length; i++) {
|
||||||
|
|
||||||
|
var timeSplit = crashlogDirectoryTree.children[i].name.replace('boot-', '').replace('crash-', '').replace('\.bz2', '').split('--');
|
||||||
|
var timeString = timeSplit[0]+' '+timeSplit[1].replace('-',':');
|
||||||
|
crashlogFiles.push({'name': crashlogDirectoryTree.children[i].name, 'size': crashlogDirectoryTree.children[i].size, 'date': Date.parse(timeString)});
|
||||||
|
}
|
||||||
|
crashlogFiles.sort((a,b) => (a.date < b.date) ? 1 : -1);
|
||||||
|
for (var c=5; c<crashlogFiles.length; c++) {
|
||||||
|
logger.info("deleteBootAndCrashLogs deleting crash log "+crashlogFiles[c]['path']+"");
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(crashlogFiles[c]['path']);
|
||||||
|
affectedDevices[device.dongle_id]=true;
|
||||||
|
} catch (exception) {
|
||||||
|
logger.error(exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -426,17 +576,21 @@ function mainWorkerLoop() {
|
||||||
process.exit();
|
process.exit();
|
||||||
}
|
}
|
||||||
|
|
||||||
deleteExpiredDrives();
|
|
||||||
removeDeletedDrivesPhysically();
|
if (Date.now()-lastCleaningTime>20*3600*1000) {
|
||||||
|
deleteBootAndCrashLogs();
|
||||||
|
deleteExpiredDrives();
|
||||||
|
deleteOverQuotaDrives();
|
||||||
|
removeDeletedDrivesPhysically();
|
||||||
|
lastCleaningTime=Date.now();
|
||||||
|
}
|
||||||
|
|
||||||
setTimeout(function() {updateSegments();}, 5000);
|
setTimeout(function() {updateSegments();}, 5000);
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
startTime=Date.now();
|
|
||||||
|
|
||||||
lockfile.lock('retropilot_worker.lock', { realpath: false, stale: 30000, update: 2000 })
|
lockfile.lock('retropilot_worker.lock', { realpath: false, stale: 30000, update: 2000 })
|
||||||
.then((release) => {
|
.then((release) => {
|
||||||
logger.info("STARTING WORKER...");
|
logger.info("STARTING WORKER...");
|
||||||
|
|
Loading…
Reference in New Issue