diff --git a/config.sample.js b/config.sample.js index afb313f..b7351c0 100644 --- a/config.sample.js +++ b/config.sample.js @@ -19,12 +19,13 @@ var config = { baseDriveDownloadPathMapping: '/realdata', // path mapping of above download url for expressjs, prefix with "/" storagePath: 'realdata/', // relative or absolute ( "/..." for absolute path ) - + deviceStorageQuotaMb: 200000, - deviceDriveQuota: 1000, deviceDriveExpirationDays: 30, - cabanaUrl: 'http://192.168.1.165:3001/' + cabanaUrl: 'http://192.168.1.165:3001/', + + welcomeMessage: `<><><><><><><><><><><><><><><><><><><><><><>
2021 RetroPilot` }; module.exports = config; diff --git a/server.js b/server.js index 896ed1e..296f130 100644 --- a/server.js +++ b/server.js @@ -23,11 +23,16 @@ const sendmail = require('sendmail')(); const htmlspecialchars = require('htmlspecialchars'); const dirTree = require("directory-tree"); +const execSync = require('child_process').execSync; const adapter = new FileSync(config.databaseFile); const db = low(adapter); +const ALL = 1E8; + +var totalStorageUsed=null; // global variable that is regularly updated in the background to track the total used storage + log4js.configure({ appenders: { logfile: { type: "file", filename: "server.log" }, out: { type: "console"} }, @@ -192,6 +197,13 @@ function getAuthenticatedAccount(req) { return account.value(); } +function updateTotalStorageUsed() { + var verifiedPath = mkDirByPathSync(config.storagePath, {isRelativeToScript: (config.storagePath.indexOf("/")===0 ? false : true)}); + if (verifiedPath!==null) { + totalStorageUsed = execSync("du -hs "+verifiedPath+" | awk -F'\t' '{print $1;}'").toString(); + } + setTimeout(function() {updateTotalStorageUsed();}, 120000); // update the used storage each 120 seconds +} // CREATE OUR SERVER EXPRESS APP @@ -271,7 +283,7 @@ app.put('/backend/post_upload', bodyParser.raw({ inflate: true, limit: '100000kb // DRIVE & BOOT/CRASH LOG FILE UPLOAD URL REQUEST app.get('/v1.3/:dongleId/upload_url/', (req, res) => { - var path = req.query.path; // todo: validate filename + var path = req.query.path; logger.info("HTTP.UPLOAD_URL called for "+req.params.dongleId+" and file "+path+": "+JSON.stringify(req.headers)); var device = db.get('devices').find({ dongle_id: req.params.dongleId}); @@ -425,7 +437,7 @@ app.post('/v2/pilotauth/', bodyParser.urlencoded({ extended: true }), (req, res) var device = db.get('devices').find({dongle_id: dongleId}).value(); if (!device) { var resultingDevice = db.get('devices') - .push({ dongle_id: dongleId, account_id: 0, imei: imei1, serial: serial, device_type: 'freon', public_key: public_key, created: Date.now(), last_ping: Date.now()}) + .push({ dongle_id: dongleId, account_id: 0, imei: imei1, serial: serial, device_type: 'freon', public_key: public_key, created: Date.now(), last_ping: Date.now(), storage_used: 0}) .write(); var device = db.get('devices').find({dongle_id: dongleId}).value(); @@ -529,12 +541,6 @@ app.get('/useradmin', (req, res) => { return; } - var verifiedPath = mkDirByPathSync(config.storagePath, {isRelativeToScript: (config.storagePath.indexOf("/")===0 ? false : true)}); - if (verifiedPath!==null) { - const execSync = require('child_process').execSync; - bytes = execSync("du -hs "+verifiedPath+" | awk -F'\t' '{print $1;}'").toString(); - } - res.status(200); res.send('

Welcome To The RetroPilot Server Dashboard!

'+ `

@@ -548,8 +554,7 @@ app.get('/useradmin', (req, res) => { 'Accounts: '+db.get('accounts').size().value()+' | '+ 'Devices: '+db.get('devices').size().value()+' | '+ 'Drives: '+db.get('drives').size().value()+' | '+ - 'Storage Used: '+(verifiedPath!==null ? bytes : '--')+''); - + 'Storage Used: '+(totalStorageUsed!==null ? totalStorageUsed : '--')+'

'+config.welcomeMessage+''); }), @@ -673,7 +678,7 @@ app.get('/useradmin/overview', (req, res) => { return; } - const devices = db.get('devices').filter({account_id: account.id}).sortBy('dongle_id').take(1000).value(); + const devices = db.get('devices').filter({account_id: account.id}).sortBy('dongle_id').take(ALL).value(); var response = '

Welcome To The RetroPilot Server Dashboard!

'+ @@ -683,11 +688,11 @@ app.get('/useradmin/overview', (req, res) => { Created: `+formatDate(account.created)+`

Devices:
- + `; for (var i in devices) { - response+=''; + response+=''; } response+=`
dongle_iddevice_typecreatedlast_ping
dongle_iddevice_typecreatedlast_pingstorage_used
'+devices[i].dongle_id+''+devices[i].device_type+''+formatDate(devices[i].created)+''+formatDate(devices[i].last_ping)+'
'+devices[i].dongle_id+''+devices[i].device_type+''+formatDate(devices[i].created)+''+formatDate(devices[i].last_ping)+''+devices[i].storage_used+' MB

@@ -771,7 +776,7 @@ app.get('/useradmin/device/:dongleId', (req, res) => { } device = device.value(); - const drives = db.get('drives').filter({dongle_id: device.dongle_id, is_deleted: false}).sortBy('created').take(1000).value(); + const drives = db.get('drives').filter({dongle_id: device.dongle_id, is_deleted: false}).sortBy('created').take(ALL).value(); var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex'); @@ -812,8 +817,8 @@ app.get('/useradmin/device/:dongleId', (req, res) => { Last Ping: `+formatDate(device.last_ping)+`
Public Key:
`+device.public_key.replace(/\r?\n|\r/g, "
")+`
- QuotaDrives: `+drives.length+` / `+config.deviceDriveQuota+`
- Quota Storage: `+(0)+` MB / `+config.deviceStorageQuotaMb+` MB
+ Stored Drives: `+drives.length+`
+ Quota Storage: `+device.storage_used+` MB / `+config.deviceStorageQuotaMb+` MB

`; @@ -953,6 +958,7 @@ app.get('/useradmin/drive/:dongleId/:driveIdentifier', (req, res) => { + var directorySegments={}; for (var i in directoryTree.children) { // skip any non-directory entries (for example m3u8 file in the drive directory) if (directoryTree.children[i].type!='directory') continue; @@ -982,9 +988,25 @@ app.get('/useradmin/drive/:dongleId/:driveIdentifier', (req, res) => { isStalled=drive_segment.is_stalled; } - response+=''+segment+''+qcamera+''+qlog+''+fcamera+''+rlog+''+dcamera+''+isProcessed+''+isStalled+''; + directorySegments["seg-"+segment] = ''+segment+''+qcamera+''+qlog+''+fcamera+''+rlog+''+dcamera+''+isProcessed+''+isStalled+''; } - + + var qcamera = '--'; + var fcamera = '--'; + var dcamera = '--'; + var qlog = '--'; + var rlog = '--'; + var isProcessed='?'; + var isStalled='?'; + + for (var i=0; i<=drive.max_segment; i++) { + if (directorySegments["seg-"+i]==undefined) { + response+=''+i+''+qcamera+''+qlog+''+fcamera+''+rlog+''+dcamera+''+isProcessed+''+isStalled+''; + } + else + response+=directorySegments["seg-"+i]; + } + response+=`

@@ -1019,11 +1041,12 @@ app.post('*', (req, res) => { -lockfile.lock('retropilot_server.lock', { realpath: false, stale: 30000, update: 2000 }) +lockfile.lock('retropilot_server.lock'+Math.random(), { realpath: false, stale: 30000, update: 2000 }) .then((release) => { console.log("STARTING SERVER..."); initializeDatabase(); initializeStorage(); + updateTotalStorageUsed(); var privateKey = fs.readFileSync(config.sslKey, 'utf8'); var certificate = fs.readFileSync(config.sslCrt, 'utf8'); diff --git a/worker.js b/worker.js index 8fc1209..985bf18 100644 --- a/worker.js +++ b/worker.js @@ -24,6 +24,7 @@ const htmlspecialchars = require('htmlspecialchars'); const dirTree = require("directory-tree"); const { resolve } = require('path'); +const execSync = require('child_process').execSync; const Reader = require('@commaai/log_reader'); @@ -34,6 +35,10 @@ const { exception } = require('console'); const adapter = new FileSync(config.databaseFile); const db = low(adapter); +const ALL = 1E8; +var lastCleaningTime=0; +var startTime=Date.now(); + log4js.configure({ appenders: { logfile: { type: "file", filename: "worker.log" }, out: { type: "console"} }, @@ -183,12 +188,25 @@ function moveUploadedFile(buffer, directory, filename) { }; - +function deleteFolderRecursive(directoryPath) { + if (fs.existsSync(directoryPath)) { + fs.readdirSync(directoryPath).forEach((file, index) => { + const curPath = path.join(directoryPath, file); + if (fs.lstatSync(curPath).isDirectory()) { + deleteFolderRecursive(curPath); + } else { + fs.unlinkSync(curPath); + } + }); + fs.rmdirSync(directoryPath); + } +}; var segmentProcessQueue=[]; var segmentProcessPosition=0; var affectedDrives={}; +var affectedDevices={}; var rlog_lastTs=0; var rlog_prevLat=-1000; @@ -293,7 +311,7 @@ function updateSegments() { segmentProcessPosition=0; affectedDrives={}; - drive_segments = db.get('drive_segments').filter({upload_complete: false, is_stalled: false}).sortBy('created').take(10000).value(); + drive_segments = db.get('drive_segments').filter({upload_complete: false, is_stalled: false}).sortBy('created').take(ALL).value(); for (var t=0; texpirationTs) { + break; // the drives are queried ordered by date, so break at the first newer one + } + + var drive = db.get('drives').find({ identifier: expiredDrives[t].identifier, dongle_id: expiredDrives[t].dongle_id}); + if (!drive.value()) continue; + logger.info("deleteExpiredDrives drive "+expiredDrives[t].dongle_id+" "+expiredDrives[t].identifier+" is older than "+config.deviceDriveExpirationDays+" days, set is_deleted=true"); + drive.assign({is_deleted: true}).write(); + } } function removeDeletedDrivesPhysically() { - // TODO: implement + var expiredDrives = db.get('drives').filter({is_deleted: true}).orderBy('created', 'asc').take(ALL).value(); + for (var t=0; tconfig.deviceStorageQuotaMb) { + var foundDriveToDelete=false; + + var allDrives = db.get('drives').filter({dongle_id: devices[t].dongle_id, is_preserved: false, is_deleted: false}).orderBy('created', 'asc').take(1).value(); + for (var i=0; i (a.date < b.date) ? 1 : -1); + for (var c=5; c (a.date < b.date) ? 1 : -1); + for (var c=5; c20*3600*1000) { + deleteBootAndCrashLogs(); + deleteExpiredDrives(); + deleteOverQuotaDrives(); + removeDeletedDrivesPhysically(); + lastCleaningTime=Date.now(); + } setTimeout(function() {updateSegments();}, 5000); + } - -startTime=Date.now(); - lockfile.lock('retropilot_worker.lock', { realpath: false, stale: 30000, update: 2000 }) .then((release) => { logger.info("STARTING WORKER...");