retropilot
/
retropilot-server
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

worker main method

pull/4/head
Cameron Clough 2022-03-21 11:48:58 +00:00
parent a8cb1e620e
commit 7680c2fb98
No known key found for this signature in database
GPG Key ID: BFB3B74B026ED43F
4 changed files with 21 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -7,10 +7,9 @@ If you don't want to host your own instance, check out https://api.retropilot.or
The server consists of 2 node scripts.
`src/server` is using expressjs and runs the backend (file upload / communication with openpilot) and the useradmin dashboard to manage / view / download drives & logs.
`src/worker` is a background worker that is processing drives (analyzing video files & logs) to prepare drives for playback in cabana and to gather statistics. It automatically terminates itself after 60 minutes to make sure the video/log libraries do not cause memory leaks.
- `src/server` is using expressjs and runs the backend (file upload / communication with openpilot) and the useradmin dashboard to manage / view / download drives & logs.
Both scripts can be started with a cronjob each minute, they use locking to make sure they run exclusively.
- `src/worker` is a background worker that is processing drives (analyzing video files & logs) to prepare drives for playback in cabana and to gather statistics. It automatically terminates itself after 60 minutes to make sure the video/log libraries do not cause memory leaks.
**Attention:** Minimum required node version is **node 10**.

35
package-lock.json generated
View File

@ -7,7 +7,6 @@
"": {
"name": "retropilot-server",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"@commaai/log_reader": "^0.9.0",
"@sendgrid/client": "^7.4.3",
@ -39,7 +38,6 @@
"multer": "^1.4.2",
"nodemailer": "^6.6.0",
"pg": "^8.7.1",
"proper-lockfile": "^4.1.2",
"sanitize": "^2.1.0",
"sequelize": "^6.6.5",
"simple-oauth2": "^4.3.0",
@ -4670,16 +4668,6 @@
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/proper-lockfile": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz",
"integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==",
"dependencies": {
"graceful-fs": "^4.2.4",
"retry": "^0.12.0",
"signal-exit": "^3.0.2"
}
},
"node_modules/property-expr": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/property-expr/-/property-expr-2.0.5.tgz",
@ -5094,14 +5082,6 @@
"node": ">=4"
}
},
"node_modules/retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
"integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=",
"engines": {
"node": ">= 4"
}
},
"node_modules/retry-as-promised": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-5.0.0.tgz",
@ -10106,16 +10086,6 @@
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"proper-lockfile": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz",
"integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==",
"requires": {
"graceful-fs": "^4.2.4",
"retry": "^0.12.0",
"signal-exit": "^3.0.2"
}
},
"property-expr": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/property-expr/-/property-expr-2.0.5.tgz",
@ -10426,11 +10396,6 @@
"integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
"dev": true
},
"retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
"integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs="
},
"retry-as-promised": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-5.0.0.tgz",

1
package.json
View File

@ -41,7 +41,6 @@
"multer": "^1.4.2",
"nodemailer": "^6.6.0",
"pg": "^8.7.1",
"proper-lockfile": "^4.1.2",
"sanitize": "^2.1.0",
"sequelize": "^6.6.5",
"simple-oauth2": "^4.3.0",

45
src/worker/index.js
View File

@ -5,7 +5,6 @@ import path, { dirname } from 'path';
import { fileURLToPath } from 'url';
import log4js from 'log4js';
import lockfile from 'proper-lockfile';
import dirTree from 'directory-tree';
import { execSync } from 'child_process';
import Reader from '@commaai/log_reader';
@ -739,29 +738,23 @@ async function mainWorkerLoop() {
}
}
// make sure bunzip2 is available
try {
//execSync('bunzip2 --help');
} catch (exception) {
logger.error('bunzip2 is not installed or not available in environment path');
process.exit();
}
lockfile.lock('retropilot_worker', {
realpath: false,
stale: 90000,
update: 2000
})
.then((release) => {
logger.info('STARTING WORKER...');
(async () => {
initializeStorage();
setTimeout(() => {
mainWorkerLoop();
}, 0);
})();
})
.catch((e) => {
console.error(e);
const main = async () => {
// make sure bunzip2 is available
try {
//execSync('bunzip2 --help');
} catch (exception) {
logger.error('bunzip2 is not installed or not available in environment path');
process.exit();
});
}
initializeStorage();
setTimeout(() => {
mainWorkerLoop();
}, 0);
};
try {
main();
} catch (e) {
console.error(e);
}