Working app locally using docker, removes config file for .env, fixes worker to only use Sequelize and fixes sequelize model type definitions

pull/4/head
Jose Vera 2022-03-01 00:04:36 -05:00
parent 6d79bd20fc
commit 66a5441eaf
36 changed files with 1033 additions and 387 deletions

34
.env.sample 100644
View File

@ -0,0 +1,34 @@
APP_SALT=RANDOM_SEED
DB_FILE=database.sqlite
DB_NAME=retro-pilot
DB_USER=root
DB_PASS=root
DB_HOST=db
DB_PORT=5432
ALLOW_REGISTRATION=true
HTTP_INTERFACE=0.0.0.0
HTTP_PORT=3000
CAN_SEND_MAIL=1 # Skips sending mail, all attempted mail is logged under DEBUG
SMTP_HOST="localhost" # credentials for smtp server to send account registration mails. if not filled in, get the generated tokens from the server.log manually
SMTP_PORT=25
SMTP_USER=root
SMTP_PASS=
SMTP_FROM="no-reply@retropilot.org"
BASE_URL="http://192.168.1.165:3000/" # base url of the retropilot server
BASE_UPLOAD_URL="http://192.168.1.165:3000/backend/post_upload" # base url sent to devices for POSTing drives & logs
BASE_DRIVE_DOWNLOAD_URL="http://192.168.1.165:3000/realdata/" # base download url for drive & log data
BASE_DRIVE_DOWNLOAD_PATH_MAPPING="/realdata" # path mapping of above download url for expressjs, prefix with "/"
STORAGE_PATH="realdata/"# relative or absolute ( "/..." for absolute path )
CABANA_URL="http://192.168.1.165:3000/cabana/index.html"
DEVICE_STORAGE_QUOTA_MB=200000
DEVICE_EXPIRATION_DAYS=30
WELCOME_MESSAGE="<><><><><><><><><><><><><><><><><><><><><><><br>2022 RetroPilot"
USE_USER_ADMIN_API=0
CLIENT_SOCKET_PORT=81
CLIENT_SOCKET_HOST="0.0.0.0"
ATHENA_ENABLED=1 # Enables Athena service
ATHENA_SECURE=1 # Disables crypto on Websocket server - use for testing on local network, change ATHENA_HOST in openpilot to ws:// instead of wss://
ATHENA_API_RATE_LIMIT=100 # Maxmium hits to /realtime/* per 30s
ATHENA_SOCKET_HOST="0.0.0.0"
ATHENA_SOCKET_PORT=4040
ATHENA_SOCKET_HEARTBEAT_FREQ=5000 # Higher the number = lower traffic, varies on how many devices are connected

1
.gitignore vendored
View File

@ -18,3 +18,4 @@ test/.devKeys
# Miscellaneous
.DS_Store
*.log
.env

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -1,54 +0,0 @@
const config = {
applicationSalt: 'RANDOM_SEED',
databaseFile: 'database.sqlite',
allowAccountRegistration: true,
httpInterface: '0.0.0.0',
httpPort: 3000,
canSendMail: true, // Skips sending mail, all attempted mail is logged under DEBUG
smtpHost: 'localhost', // credentials for smtp server to send account registration mails. if not filled in, get the generated tokens from the server.log manually
smtpPort: 25,
smtpUser: 'root',
smtpPassword: '',
smtpFrom: 'no-reply@retropilot.org',
baseUrl: 'http://192.168.1.165:3000/', // base url of the retropilot server
baseUploadUrl: 'http://192.168.1.165:3000/backend/post_upload', // base url sent to devices for POSTing drives & logs
baseDriveDownloadUrl: 'http://192.168.1.165:3000/realdata/', // base download url for drive & log data
baseDriveDownloadPathMapping: '/realdata', // path mapping of above download url for expressjs, prefix with "/"
storagePath: 'realdata/', // relative or absolute ( "/..." for absolute path )
cabanaUrl: 'http://192.168.1.165:3000/cabana/index.html',
deviceStorageQuotaMb: 200000,
deviceDriveExpirationDays: 30,
welcomeMessage: '<><><><><><><><><><><><><><><><><><><><><><><br>2021 RetroPilot',
flags: {
useUserAdminApi: false,
},
clientSocket: { // Used in development, remove before prod
port: 81,
host: '0.0.0.0',
},
athena: {
enabled: true, // Enables Athena service
secure: true, // Disables crypto on Websocket server - use for testing on local network, change ATHENA_HOST in openpilot to ws:// instead of wss://
api: {
ratelimit: 100, // Maxmium hits to /realtime/* per 30s
},
socket: {
port: 4040,
heartbeatFrequency: 5000, // Higher the number = lower traffic, varies on how many devices are connected
},
},
};
export default config;

View File

@ -1,7 +1,6 @@
import crypto from 'crypto';
import jsonwebtoken from 'jsonwebtoken';
import log4js from 'log4js';
import config from '../config';
import orm from '../models/index.model';
const logger = log4js.getLogger('default');
@ -29,9 +28,9 @@ async function signIn(email, password) {
if (account.dataValues) {
account = account.dataValues;
const inputPassword = crypto.createHash('sha256').update(password + config.applicationSalt).digest('hex');
const inputPassword = crypto.createHash('sha256').update(password + process.env.APP_SALT).digest('hex');
if (account.password === inputPassword) {
const token = jsonwebtoken.sign({ accountId: account.id }, config.applicationSalt);
const token = jsonwebtoken.sign({ accountId: account.id }, process.env.APP_SALT);
return { success: true, jwt: token };
}
@ -44,10 +43,10 @@ async function changePassword(account, newPassword, oldPassword) {
if (!account || !newPassword || !oldPassword) {
return { success: false, error: 'MISSING_DATA' };
}
const oldPasswordHash = crypto.createHash('sha256').update(oldPassword + config.applicationSalt).digest('hex');
const oldPasswordHash = crypto.createHash('sha256').update(oldPassword + process.env.APP_SALT).digest('hex');
if (account.password === oldPasswordHash) {
const newPasswordHash = crypto.createHash('sha256').update(newPassword + config.applicationSalt).digest('hex');
const newPasswordHash = crypto.createHash('sha256').update(newPassword + process.env.APP_SALT).digest('hex');
await orm.models.accounts.update(
{ password: newPasswordHash },
@ -76,7 +75,7 @@ async function getAccountFromJWT(jwt, limitData) {
let token;
try {
token = jsonwebtoken.verify(jwt, config.applicationSalt);
token = jsonwebtoken.verify(jwt, process.env.APP_SALT);
} catch (err) {
return null;// {success: false, msg: 'BAD_JWT'}
}

View File

@ -1,6 +1 @@
import crypto from 'crypto';
import { Logger } from 'log4js';
import { AUTH_REGISTER_OAUTH_NO_EMAIL, AUTH_REGISTER_ALREADY_REGISTERED } from '../../consistency/terms';
import { getAccountFromEmail } from '../users';
import orm from '../../models/index.model';
import config from '../../config';
//empty

View File

@ -1,9 +1,5 @@
import crypto from 'crypto';
import { Logger } from 'log4js';
import { AUTH_REGISTER_OAUTH_NO_EMAIL, AUTH_REGISTER_ALREADY_REGISTERED } from '../../consistency/terms';
import { getAccountFromEmail } from '../users';
import orm from '../../models/index.model';
import config from '../../config';
export function oauthRegister(email) {
if (!email) return { error: true, ...AUTH_REGISTER_OAUTH_NO_EMAIL };

View File

@ -1,5 +1,3 @@
import { log4js } from 'log4js';
import { crypto } from 'crypto';
import { generateSecret, verify } from '2fa-util';
import {
AUTH_2FA_BAD_ACCOUNT,
@ -8,9 +6,7 @@ import {
AUTH_2FA_ENROLLED,
AUTH_2FA_BAD_TOKEN,
} from '../../consistency/terms';
import { getAccountFromEmail } from '../users';
import orm from '../../models/index.model';
import config from '../../config';
export async function twoFactorOnboard(account) {
if (!account || !account.dataValues) { return { success: false, ...AUTH_2FA_BAD_ACCOUNT }; }

View File

@ -2,7 +2,6 @@ import sanitizeFactory from 'sanitize';
import crypto from 'crypto';
import dirTree from 'directory-tree';
import log4js from 'log4js';
import config from '../config';
import authenticationController from './authentication';
import orm from '../models/index.model';
import usersController from './users';
@ -211,9 +210,9 @@ async function getDriveFromidentifier(dongleId, identifier) {
*/
async function getCrashlogs(dongleId) {
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(dongleId).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId).digest('hex');
const directoryTree = dirTree(`${config.storagePath}${dongleId}/${dongleIdHash}/crash/`, { attributes: ['size'] });
const directoryTree = dirTree(`${process.env.STORAGE_PATH}${dongleId}/${dongleIdHash}/crash/`, { attributes: ['size'] });
const crashlogFiles = (directoryTree ? directoryTree.children : []).map((file) => {
const timeSplit = file.name.replace('boot-', '').replace('crash-', '').replace('.bz2', '').split('--');
let timeString = `${timeSplit[0]} ${timeSplit[1].replace(/-/g, ':')}`;
@ -236,7 +235,7 @@ async function getCrashlogs(dongleId) {
name: file.name,
size: file.size,
date: dateObj,
permalink: `${config.baseDriveDownloadUrl}${dongleId}/${dongleIdHash}/crash/${file.name}`,
permalink: `${process.env.BASE_DRIVE_DOWNLOAD_URL}${dongleId}/${dongleIdHash}/crash/${file.name}`,
};
});
crashlogFiles.sort((a, b) => ((a.date < b.date) ? 1 : -1));
@ -244,9 +243,9 @@ async function getCrashlogs(dongleId) {
}
async function getBootlogs(dongleId) {
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(dongleId).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId).digest('hex');
const directoryTree = dirTree(`${config.storagePath}${dongleId}/${dongleIdHash}/boot/`, { attributes: ['size'] });
const directoryTree = dirTree(`${process.env.STORAGE_PATH}${dongleId}/${dongleIdHash}/boot/`, { attributes: ['size'] });
const bootlogFiles = (directoryTree ? directoryTree.children : []).map((file) => {
const timeSplit = file.name.replace('boot-', '').replace('crash-', '').replace('.bz2', '').split('--');
const timeString = `${timeSplit[0]} ${timeSplit[1].replace(/-/g, ':')}`;
@ -263,7 +262,7 @@ async function getBootlogs(dongleId) {
name: file.name,
size: file.size,
date: dateObj,
permalink: `${config.baseDriveDownloadUrl}${dongleId}/${dongleIdHash}/boot/${file.name}`,
permalink: `${process.env.BASE_DRIVE_DOWNLOAD_URL}${dongleId}/${dongleIdHash}/boot/${file.name}`,
};
});
bootlogFiles.sort((a, b) => ((a.date < b.date) ? 1 : -1));

View File

@ -1,24 +1,23 @@
import nodemailer from 'nodemailer';
import log4js from 'log4js';
import config from '../config';
const logger = log4js.getLogger('default');
const transporter = nodemailer.createTransport(
{
host: config.smtpHost,
port: config.smtpPort,
host: process.env.SMTP_HOST,
port: process.env.SMTP_PORT,
auth: {
user: config.smtpUser,
pass: config.smtpPassword,
user: process.env.SMTP_USER,
pass: process.env.SMTP_PASS,
},
logger: true,
debug: false,
},
{ from: config.smtpFrom },
{ from: process.env.SMTP_FROM },
);
async function sendEmailVerification(token, email) {
if (!config.canSendMail) {
if (!process.env.CAN_SEND_MAIL) {
return logger.warn(`Mailing disabled. ${email} - ${token}`);
}
@ -28,7 +27,7 @@ async function sendEmailVerification(token, email) {
try {
message = {
from: config.smtpFrom,
from: process.env.SMTP_FROM,
to: email.trim(),
subject: 'RetroPilot Registration Token',
text: `Your Email Registration Token Is: "${token}"`,

View File

@ -2,18 +2,17 @@ import path from 'path';
import fs from 'fs';
import { execSync } from 'child_process';
import log4js from 'log4js';
import config from '../config.js';
const logger = log4js.getLogger('default');
let totalStorageUsed;
function initializeStorage() {
const verifiedPath = mkDirByPathSync(config.storagePath, { isRelativeToScript: (config.storagePath.indexOf('/') !== 0) });
const verifiedPath = mkDirByPathSync(process.env.STORAGE_PATH, { isRelativeToScript: (process.env.STORAGE_PATH.indexOf('/') !== 0) });
if (verifiedPath != null) {
logger.info(`Verified storage path ${verifiedPath}`);
} else {
logger.error(`Unable to verify storage path '${config.storagePath}', check filesystem / permissions`);
logger.error(`Unable to verify storage path '${process.env.STORAGE_PATH}', check filesystem / permissions`);
process.exit();
}
}
@ -79,16 +78,16 @@ function moveUploadedFile(buffer, directory, filename) {
return false;
}
if (config.storagePath.lastIndexOf('/') !== config.storagePath.length - 1) {
if (process.env.STORAGE_PATH.lastIndexOf('/') !== process.env.STORAGE_PATH.length - 1) {
directory = `/${directory}`;
}
if (directory.lastIndexOf('/') !== directory.length - 1) {
directory += '/';
}
const finalPath = mkDirByPathSync(config.storagePath + directory, { isRelativeToScript: (config.storagePath.indexOf('/') !== 0) });
const finalPath = mkDirByPathSync(process.env.STORAGE_PATH + directory, { isRelativeToScript: (process.env.STORAGE_PATH.indexOf('/') !== 0) });
if (!finalPath || finalPath.length === 0) {
logger.error(`moveUploadedFile invalid final path, check permissions to create / write '${config.storagePath + directory}'`);
logger.error(`moveUploadedFile invalid final path, check permissions to create / write '${process.env.STORAGE_PATH + directory}'`);
return false;
}
@ -102,7 +101,7 @@ function moveUploadedFile(buffer, directory, filename) {
}
async function updateTotalStorageUsed() {
const verifiedPath = mkDirByPathSync(config.storagePath, { isRelativeToScript: (config.storagePath.indexOf('/') !== 0) });
const verifiedPath = mkDirByPathSync(process.env.STORAGE_PATH, { isRelativeToScript: (process.env.STORAGE_PATH.indexOf('/') !== 0) });
if (!verifiedPath) {
return;
}

View File

@ -1,6 +1,5 @@
import crypto from 'crypto';
import log4js from 'log4js';
import config from '../config';
import orm from '../models/index.model';
const logger = log4js.getLogger('default');
@ -27,12 +26,12 @@ export async function createAccount(email, password) {
if (!email || !password) {
return { success: false, status: 400, data: { missingData: true } };
}
if (!config.allowAccountRegistration) {
if (!process.env.ALLOW_REGISTRATION) {
return { success: false, status: 403, data: { registerEnabled: false } };
}
const emailToken = crypto.createHmac('sha256', config.applicationSalt).update(email.trim()).digest('hex');
password = crypto.createHash('sha256').update(password + config.applicationSalt).digest('hex');
const emailToken = crypto.createHmac('sha256', process.env.APP_SALT).update(email.trim()).digest('hex');
password = crypto.createHash('sha256').update(password + process.env.APP_SALT).digest('hex');
const account = await orm.models.accounts.findOne({ where: { email } });
if (account != null && account.dataValues != null) {

Binary file not shown.

View File

@ -1,29 +1,31 @@
version: "3.9"
services:
server:
# Build image from Dockerfile
build: .
restart: unless-stopped
depends_on:
- db
volumes:
# Mount the config to the container
- ./config.js:/app/config.js
# Mount the database to the container
- ./database.sqlite:/app/database.sqlite
# Mount the data directory to the container
- ./realdata:/app/realdata
- ./:/app
ports:
# HTTP server
- "3000:3000"
# Athena WebSocket
- "4040:4040"
worker:
# Use the same image as the server
build: .
# But run the worker.js script instead
command: node --es-module-specifier-resolution=node worker.js
restart: unless-stopped
depends_on:
- db
volumes:
- ./config.js:/app/config.js
- ./database.sqlite:/app/database.sqlite
- ./realdata:/app/realdata
- ./:/app
db:
image: postgres
restart: always
ports:
- '5438:5432'
volumes:
- ./sql/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql
environment:
POSTGRES_USER: root
POSTGRES_PASSWORD: root
POSTGRES_DB: retro-pilot

View File

@ -18,15 +18,15 @@ export default (sequelize) => {
},
athena: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
unpair: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
view_drives: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
created_at: {
allowNull: true,

View File

@ -53,7 +53,7 @@ export default (sequelize) => {
},
ignore_uploads: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
nickname: {
allowNull: true,

View File

@ -20,15 +20,15 @@ export default (sequelize) => {
},
dongle_id: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.TEXT,
},
duration: {
allowedNull: true,
type: DataTypes.NUMBER,
type: DataTypes.INTEGER,
},
distance_meters: {
allowNull: true,
type: DataTypes.NUMBER,
type: DataTypes.INTEGER,
},
upload_complete: {
allowNull: true,
@ -48,7 +48,7 @@ export default (sequelize) => {
},
process_attempts: {
allowNull: true,
type: DataTypes.BOOLEAN,
type: DataTypes.INTEGER,
},
},
{

View File

@ -24,15 +24,15 @@ export default (sequelize) => {
},
upload_complete: {
allowedNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
duration: {
allowNull: true,
type: DataTypes.NUMBER,
type: DataTypes.INTEGER,
},
distance_meters: {
allowNull: true,
type: DataTypes.NUMBER,
type: DataTypes.INTEGER,
},
filesize: {
allowNull: true,

View File

@ -10,17 +10,14 @@ import athena_returned_data from './athena_returned_data.model';
import device_authorised_users from './device_authorised_users.model';
import drive_segments from './drive_segments.model';
import oauth_accounts from './oauth_accounts';
import config from '../config';
const sequelize = new Sequelize({
username: 'postgres',
password: config.sqltemp,
database: 'retro-dev',
host: '127.0.0.1',
port: 5432,
username: process.env.DB_USER,
password: process.env.DB_PASS,
database: process.env.DB_NAME || 'retro-pilot',
host: process.env.DB_HOST || '127.0.0.1',
port: process.env.DB_PORT || 5432,
dialect: 'postgres',
});
sequelize.options.logging = () => {};

View File

@ -18,11 +18,11 @@ export default (sequelize) => {
},
created: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.TIME,
},
last_used: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.CHAR,
},
refresh: {
allowNull: true,

View File

@ -14,15 +14,15 @@ export default (sequelize) => {
},
password: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.TEXT,
},
created: {
allowNull: true,
type: DataTypes.NUMBER,
type: DataTypes.INTEGER,
},
last_ping: {
allowNull: true,
type: DataTypes.NUMBER,
type: DataTypes.INTEGER,
},
'2fa_token': {
allowNull: true,
@ -30,7 +30,7 @@ export default (sequelize) => {
},
admin: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
email_verify_token: {
allowNull: true,

14
package-lock.json generated
View File

@ -21,6 +21,7 @@
"cors": "^2.8.5",
"crypto": "^1.0.1",
"directory-tree": "^2.2.9",
"dotenv": "^16.0.0",
"esm": "^3.2.25",
"express": "^4.17.1",
"express-fileupload": "^1.2.1",
@ -1549,6 +1550,14 @@
"node": ">=6.0.0"
}
},
"node_modules/dotenv": {
"version": "16.0.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.0.tgz",
"integrity": "sha512-qD9WU0MPM4SWLPJy/r2Be+2WgQj8plChsyrCNQzW/0WjvcJQiKQJ9mH3ZgB3fxbUUxgc/11ZJ0Fi5KiimWGz2Q==",
"engines": {
"node": ">=12"
}
},
"node_modules/dottie": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz",
@ -7824,6 +7833,11 @@
"esutils": "^2.0.2"
}
},
"dotenv": {
"version": "16.0.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.0.tgz",
"integrity": "sha512-qD9WU0MPM4SWLPJy/r2Be+2WgQj8plChsyrCNQzW/0WjvcJQiKQJ9mH3ZgB3fxbUUxgc/11ZJ0Fi5KiimWGz2Q=="
},
"dottie": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz",

View File

@ -26,6 +26,7 @@
"cors": "^2.8.5",
"crypto": "^1.0.1",
"directory-tree": "^2.2.9",
"dotenv": "^16.0.0",
"esm": "^3.2.25",
"express": "^4.17.1",
"express-fileupload": "^1.2.1",

View File

@ -4,10 +4,7 @@ import bodyParser from 'body-parser';
import crypto from 'crypto';
import log4js from 'log4js';
import storageController from '../controllers/storage';
import helperController from '../controllers/helpers';
import mailingController from '../controllers/mailing';
import deviceController from '../controllers/devices';
import config from '../config';
import authenticationController from './../controllers/authentication';
const logger = log4js.getLogger('default');
@ -45,7 +42,7 @@ router.put('/backend/post_upload', bodyParser.raw({
logger.info(`HTTP.PUT /backend/post_upload BOOT or CRASH upload with filename: ${filename}, token: ${req.query.token}`);
}
const token = crypto.createHmac('sha256', config.applicationSalt).update(dongleId + filename + directory + ts).digest('hex');
const token = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId + filename + directory + ts).digest('hex');
if (token !== req.query.token) {
logger.error(`HTTP.PUT /backend/post_upload token mismatch (${token} vs ${req.query.token})`);
return res.status(400).send('Malformed request');
@ -216,7 +213,7 @@ async function upload(req, res) {
let responseUrl = null;
const ts = Date.now(); // we use this to make sure old URLs cannot be reused (timeout after 60min)
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(dongleId).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId).digest('hex');
// boot log upload
@ -234,9 +231,9 @@ async function upload(req, res) {
// "boot-2021-04-12--01-45-30.bz" for example
const directory = `${dongleId}/${dongleIdHash}/${uploadType}`;
const token = crypto.createHmac('sha256', config.applicationSalt).update(dongleId + filename + directory + ts).digest('hex');
const token = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId + filename + directory + ts).digest('hex');
responseUrl = `${config.baseUploadUrl}?file=${filename}&dir=${directory}&dongleId=${dongleId}&ts=${ts}&token=${token}`;
responseUrl = `${process.env.BASE_UPLOAD_URL}?file=${filename}&dir=${directory}&dongleId=${dongleId}&ts=${ts}&token=${token}`;
logger.info(`HTTP.UPLOAD_URL matched '${uploadType}' file upload, constructed responseUrl: ${responseUrl}`);
} else {
// "2021-04-12--01-44-25--0/qlog.bz2" for example
@ -260,12 +257,12 @@ async function upload(req, res) {
return res.send('Malformed Request.').status(400);
}
const driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(driveName).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT).update(driveName).digest('hex');
directory = `${dongleId}/${dongleIdHash}/${driveIdentifierHash}/${directory}`;
const token = crypto.createHmac('sha256', config.applicationSalt).update(dongleId + filename + directory + ts).digest('hex');
responseUrl = `${config.baseUploadUrl}?file=${filename}&dir=${directory}&dongleId=${dongleId}&ts=${ts}&token=${token}`;
const token = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId + filename + directory + ts).digest('hex');
responseUrl = `${process.env.BASE_UPLOAD_URL}?file=${filename}&dir=${directory}&dongleId=${dongleId}&ts=${ts}&token=${token}`;
logger.info(`HTTP.UPLOAD_URL matched 'drive' file upload, constructed responseUrl: ${responseUrl}`);
const drive = await deviceController.getDriveFromidentifier(dongleId, driveName).catch((err)=>{
@ -405,9 +402,9 @@ router.get('/useradmin/cabana_drive/:extendedRouteIdentifier', runAsyncWrapper(a
return res.status(200).json({ status: 'drive not found' });
}
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(drive.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(drive.identifier).digest('hex');
const driveUrl = `${config.baseDriveDownloadUrl + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(drive.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT).update(drive.identifier).digest('hex');
const driveUrl = `${process.env.BASE_DRIVE_DOWNLOAD_URL + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
if (dongleIdHash !== dongleIdHashReq || driveIdentifierHash !== driveIdentifierHashReq) {
return res.status(200).json({ status: 'hashes not matching' });

View File

@ -2,11 +2,6 @@ import bodyParser from 'body-parser';
import express from 'express';
import authenticationController from '../../controllers/authentication';
/* eslint-disable no-unused-vars */
import userController from '../../controllers/users';
import config from '../../config';
/* eslint-enable no-unused-vars */
const router = express.Router();
async function isAuthenticated(req, res, next) {
const account = await authenticationController.getAuthenticatedAccount(req);

View File

@ -3,14 +3,9 @@ import crypto from 'crypto';
import dirTree from 'directory-tree';
import bodyParser from 'body-parser';
import deviceSchema from '../../schema/routes/devices.mjs';
import config from '../../config';
/* eslint-disable no-unused-vars */
import userController from '../../controllers/users';
import deviceController from '../../controllers/devices';
import authenticationController from '../../controllers/authentication';
/* eslint-enable no-unused-vars */
const router = express.Router();
async function isAuthenticated(req, res, next) {
const account = await authenticationController.getAuthenticatedAccount(req);
@ -70,10 +65,10 @@ router.get('/retropilot/0/device/:dongle_id/drives/:drive_identifier/segment', i
if (isUserAuthorised.success === false || isUserAuthorised.data.authorised === false) {
return res.json({ success: false, msg: isUserAuthorised.msg });
}
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(req.params.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(req.params.drive_identifier).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(req.params.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT).update(req.params.drive_identifier).digest('hex');
const directoryTree = dirTree(`${config.storagePath + req.params.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${req.params.drive_identifier}`);
const directoryTree = dirTree(`${process.env.STORAGE_PATH + req.params.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${req.params.drive_identifier}`);
return res.json({ success: true, msg: 'ok', data: directoryTree });
});

View File

@ -2,7 +2,6 @@
import express from 'express';
import bodyParser from 'body-parser';
import cookieParser from 'cookie-parser';
import config from '../config';
import controllers from '../controllers';
import deviceController from '../controllers/devices';
@ -47,8 +46,8 @@ router.get('/retropilot/0/useradmin', runAsyncWrapper(async (req, res) => {
data: {
serverStats: {
config: {
registerAllowed: config.allowAccountRegistration,
welcomeMessage: config.welcomeMessage,
registerAllowed: process.env.ALLOW_REGISTRATION,
welcomeMessage: process.env.WELCOME_MESSAGE,
},
accounts: accounts.num,
devices: devices.num,
@ -67,7 +66,7 @@ router.get('/retropilot/0/useradmin', runAsyncWrapper(async (req, res) => {
router.post('/useradmin/register/token', bodyParser.urlencoded({extended: true}), runAsyncWrapper(async (req, res) => {
const email = req.body.email;
if (!config.allowAccountRegistration) {
if (!process.env.ALLOW_REGISTRATION) {
res.send('Unauthorized.').status(401);
return;
}
@ -84,7 +83,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({extended: true})
return;
}
const token = crypto.createHmac('sha256', config.applicationSalt).update(email.trim()).digest('hex');
const token = crypto.createHmac('sha256', process.env.APP_SALT).update(email.trim()).digest('hex');
let infoText = '';
@ -103,7 +102,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({extended: true})
const result = await models.__db.run(
'INSERT INTO accounts (email, password, created, banned) VALUES (?, ?, ?, ?)',
email,
crypto.createHash('sha256').update(req.body.password + config.applicationSalt).digest('hex'),
crypto.createHash('sha256').update(req.body.password + process.env.APP_SALT).digest('hex'),
Date.now(), false);
if (result.lastID != undefined) {
@ -138,7 +137,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({extended: true})
}))
router.get('/useradmin/register', runAsyncWrapper(async (req, res) => {
if (!config.allowAccountRegistration) {
if (!process.env.ALLOW_REGISTRATION) {
res.status(400);
res.send('Unauthorized.');
return;
@ -262,9 +261,9 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
const drives = await models.__db.all('SELECT * FROM drives WHERE dongle_id = ? AND is_deleted = ? ORDER BY created DESC', device.dongle_id, false);
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
var dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(device.dongle_id).digest('hex');
const bootlogDirectoryTree = dirTree(config.storagePath + device.dongle_id + "/" + dongleIdHash + "/boot/", {attributes: ['size']});
const bootlogDirectoryTree = dirTree(process.env.STORAGE_PATH + device.dongle_id + "/" + dongleIdHash + "/boot/", {attributes: ['size']});
var bootlogFiles = [];
if (bootlogDirectoryTree != undefined) {
for (var i = 0; i < bootlogDirectoryTree.children.length; i++) {
@ -280,7 +279,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
bootlogFiles.sort((a, b) => (a.date < b.date) ? 1 : -1);
}
const crashlogDirectoryTree = dirTree(config.storagePath + device.dongle_id + "/" + dongleIdHash + "/crash/", {attributes: ['size']});
const crashlogDirectoryTree = dirTree(process.env.STORAGE_PATH + device.dongle_id + "/" + dongleIdHash + "/crash/", {attributes: ['size']});
var crashlogFiles = [];
if (crashlogDirectoryTree != undefined) {
for (var i = 0; i < crashlogDirectoryTree.children.length; i++) {
@ -309,7 +308,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
<b>Public Key:</b><br><span style="font-size: 0.8em">` + device.public_key.replace(/\r?\n|\r/g, "<br>") + `</span>
<br>
<b>Stored Drives:</b> ` + drives.length + `<br>
<b>Quota Storage:</b> ` + device.storage_used + ` MB / ` + config.deviceStorageQuotaMb + ` MB<br>
<b>Quota Storage:</b> ` + device.storage_used + ` MB / ` + process.env.DEVICE_STORAGE_QUOTA_MB + ` MB<br>
<br>
`;
@ -318,7 +317,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
<tr><th>date</th><th>file</th><th>size</th></tr>
`;
for (var i = 0; i < Math.min(5, bootlogFiles.length); i++) {
response += `<tr><td>` + controllers.helpers.formatDate(bootlogFiles[i].date) + `</td><td><a href="` + config.baseDriveDownloadUrl + device.dongle_id + "/" + dongleIdHash + "/boot/" + bootlogFiles[i].name + `" target=_blank>` + bootlogFiles[i].name + `</a></td><td>` + bootlogFiles[i].size + `</td></tr>`;
response += `<tr><td>` + controllers.helpers.formatDate(bootlogFiles[i].date) + `</td><td><a href="` + process.env.BASE_DRIVE_DOWNLOAD_URL + device.dongle_id + "/" + dongleIdHash + "/boot/" + bootlogFiles[i].name + `" target=_blank>` + bootlogFiles[i].name + `</a></td><td>` + bootlogFiles[i].size + `</td></tr>`;
}
response += `</table><br><br>`;
@ -327,11 +326,11 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
<tr><th>date</th><th>file</th><th>size</th></tr>
`;
for (var i = 0; i < Math.min(5, crashlogFiles.length); i++) {
response += `<tr><td>` + controllers.helpers.formatDate(crashlogFiles[i].date) + `</td><td><a href="` + config.baseDriveDownloadUrl + device.dongle_id + "/" + dongleIdHash + "/crash/" + crashlogFiles[i].name + `" target=_blank>` + crashlogFiles[i].name + `</a></td><td>` + crashlogFiles[i].size + `</td></tr>`;
response += `<tr><td>` + controllers.helpers.formatDate(crashlogFiles[i].date) + `</td><td><a href="` + process.env.BASE_DRIVE_DOWNLOAD_URL + device.dongle_id + "/" + dongleIdHash + "/crash/" + crashlogFiles[i].name + `" target=_blank>` + crashlogFiles[i].name + `</a></td><td>` + crashlogFiles[i].size + `</td></tr>`;
}
response += `</table><br><br>`;
response += `<b>Drives (non-preserved drives expire ` + config.deviceDriveExpirationDays + ` days after upload):</b><br>
response += `<b>Drives (non-preserved drives expire ` + process.env.DEVICE_EXPIRATION_DAYS + ` days after upload):</b><br>
<table border=1 cellpadding=2 cellspacing=2>
<tr><th>identifier</th><th>filesize</th><th>duration</th><th>distance_meters</th><th>upload_complete</th><th>is_processed</th><th>upload_date</th><th>actions</th></tr>
`;
@ -415,17 +414,17 @@ router.get('/useradmin/drive/:dongleId/:driveIdentifier', runAsyncWrapper(async
return;
}
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(drive.identifier).digest('hex');
var dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(device.dongle_id).digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT).update(drive.identifier).digest('hex');
var driveUrl = config.baseDriveDownloadUrl + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier + "/";
var driveUrl = process.env.BASE_DRIVE_DOWNLOAD_URL + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier + "/";
var cabanaUrl = null;
if (drive.is_processed) {
cabanaUrl = config.cabanaUrl + '?retropilotIdentifier=' + device.dongle_id + '|' + dongleIdHash + '|' + drive.identifier + '|' + driveIdentifierHash + '&retropilotHost=' + encodeURIComponent(config.baseUrl) + '&demo=1"';
cabanaUrl = process.env.CABANA_URL + '?retropilotIdentifier=' + device.dongle_id + '|' + dongleIdHash + '|' + drive.identifier + '|' + driveIdentifierHash + '&retropilotHost=' + encodeURIComponent(process.env.BASE_URL) + '&demo=1"';
}
const directoryTree = dirTree(config.storagePath + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier);
const directoryTree = dirTree(process.env.STORAGE_PATH + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier);
var response = '<html style="font-family: monospace"><h2>Welcome To The RetroPilot Server Dashboard!</h2>' +
`

View File

@ -5,9 +5,7 @@ import htmlspecialchars from 'htmlspecialchars';
import dirTree from 'directory-tree';
import cookieParser from 'cookie-parser';
import log4js from 'log4js';
import config from '../config';
import authenticationController from '../controllers/authentication';
import storageController from '../controllers/storage';
import helperController from '../controllers/helpers';
import mailingController from '../controllers/mailing';
import deviceController from '../controllers/devices';
@ -71,9 +69,9 @@ router.get('/useradmin', runAsyncWrapper(async (req, res) => {
<input type="submit">
</form>
<br><br>
${!config.allowAccountRegistration ? '<i>User Account Registration is disabled on this Server</i>' : '<a href="/useradmin/register">Register new Account</a>'}
${!process.env.ALLOW_REGISTRATION ? '<i>User Account Registration is disabled on this Server</i>' : '<a href="/useradmin/register">Register new Account</a>'}
<br><br>
<br><br>${config.welcomeMessage}
<br><br>${process.env.WELCOME_MESSAGE}
</html>
`, /*
Accounts: ${accounts.num} |
@ -82,7 +80,7 @@ router.get('/useradmin', runAsyncWrapper(async (req, res) => {
Distance Traveled: ${Math.round(drives.distance / 1000)} km |
Time Traveled: ${helperController.formatDuration(drives.duration)} |
Storage Used: ${await storageController.getTotalStorageUsed() !== null ? await storageController.getTotalStorageUsed() : '--'}
<br><br>${config.welcomeMessage}` */);
<br><br>${process.env.WELCOME_MESSAGE}` */);
}));
router.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true }), runAsyncWrapper(async (req, res) => {
@ -92,7 +90,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true
return res.status(400).send('Malformed Request');
}
if (!config.allowAccountRegistration) {
if (!process.env.ALLOW_REGISTRATION) {
return res.status(401).send('Unauthorized.');
}
@ -106,7 +104,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true
return res.redirect(`/useradmin/register?status=${encodeURIComponent('Email is already registered')}`);
}
const token = crypto.createHmac('sha256', config.applicationSalt).update(email.trim()).digest('hex');
const token = crypto.createHmac('sha256', process.env.APP_SALT).update(email.trim()).digest('hex');
let infoText = '';
@ -121,7 +119,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true
} else {
const result = await userController._dirtyCreateAccount(
email,
crypto.createHash('sha256').update(req.body.password + config.applicationSalt).digest('hex'),
crypto.createHash('sha256').update(req.body.password + process.env.APP_SALT).digest('hex'),
Date.now(),
false,
);
@ -153,7 +151,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true
}));
router.get('/useradmin/register', runAsyncWrapper(async (req, res) => {
if (!config.allowAccountRegistration) {
if (!process.env.ALLOW_REGISTRATION) {
return res.status(400).send('Unauthorized.');
}
@ -222,7 +220,7 @@ ${req.query.linkstatus !== undefined ? `<br><u>${htmlspecialchars(req.query.link
<hr/>
<a href="/useradmin/signout">Sign Out</a>`;
response += `<br>${config.welcomeMessage}</html>`;
response += `<br>${process.env.WELCOME_MESSAGE}</html>`;
return res.status(200).send(response);
}));
@ -274,7 +272,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
const drives = await deviceController.getDrives(device.dongle_id, false, true);
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(device.dongle_id).digest('hex');
const bootlogFiles = await deviceController.getBootlogs(device.dongle_id);
const crashlogFiles = await deviceController.getCrashlogs(device.dongle_id);
@ -292,7 +290,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
<b>Public Key:</b><br>
<span style="font-size: 0.8em">${device.public_key.replace(/\r?\n|\r/g, '<br>')}</span><br>
<b>Stored Drives:</b> ${drives.length}<br>
<b>Quota Storage:</b> ${device.storage_used} MB / ${config.deviceStorageQuotaMb} MB<br>
<b>Quota Storage:</b> ${device.storage_used} MB / ${process.env.DEVICE_STORAGE_QUOTA_MB} MB<br>
<br>`;
response += `<b>Boot Logs (last 5):</b>
@ -301,7 +299,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
<tr><th>date</th><th>file</th><th>size</th></tr>
`;
for (let i = 0; i < Math.min(5, bootlogFiles.length); i++) {
response += `<tr><td>${helperController.formatDate(bootlogFiles[i].date)}</td><td><a href="${config.baseDriveDownloadUrl}${device.dongle_id}/${dongleIdHash}/boot/${bootlogFiles[i].name}" target=_blank>${bootlogFiles[i].name}</a></td><td>${bootlogFiles[i].size}</td></tr>`;
response += `<tr><td>${helperController.formatDate(bootlogFiles[i].date)}</td><td><a href="${process.env.BASE_DRIVE_DOWNLOAD_URL}${device.dongle_id}/${dongleIdHash}/boot/${bootlogFiles[i].name}" target=_blank>${bootlogFiles[i].name}</a></td><td>${bootlogFiles[i].size}</td></tr>`;
}
response += '</table><br><br>';
@ -311,13 +309,13 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
for (let i = 0; i < Math.min(5, crashlogFiles.length); i++) {
response += `<tr>
<td>${helperController.formatDate(crashlogFiles[i].date)}</td>.
<td><a href="${config.baseDriveDownloadUrl}${device.dongle_id}/${dongleIdHash}/crash/${crashlogFiles[i].name}" target=_blank>${crashlogFiles[i].name}</a></td>
<td><a href="${process.env.BASE_DRIVE_DOWNLOAD_URL}${device.dongle_id}/${dongleIdHash}/crash/${crashlogFiles[i].name}" target=_blank>${crashlogFiles[i].name}</a></td>
<td>${crashlogFiles[i].size}</td>
</tr>`;
}
response += '</table><br><br>';
response += `<b>Drives (non-preserved drives expire ${config.deviceDriveExpirationDays} days after upload):</b><br>
response += `<b>Drives (non-preserved drives expire ${process.env.DEVICE_EXPIRATION_DAYS} days after upload):</b><br>
<table border=1 cellpadding=2 cellspacing=2>
<tr>
<th>identifier</th>
@ -423,14 +421,14 @@ router.get('/useradmin/drive/:dongleId/:driveIdentifier', runAsyncWrapper(async
return res.status(400).send('Unauthorized.');
}
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(drive.identifier).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(device.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT).update(drive.identifier).digest('hex');
const driveUrl = `${config.baseDriveDownloadUrl + device.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}/`;
const driveUrl = `${process.env.BASE_DRIVE_DOWNLOAD_URL + device.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}/`;
let cabanaUrl = null;
if (drive.is_processed) {
cabanaUrl = `${config.cabanaUrl}?retropilotIdentifier=${device.dongle_id}|${dongleIdHash}|${drive.identifier}|${driveIdentifierHash}&retropilotHost=${encodeURIComponent(config.baseUrl)}&demo=1"`;
cabanaUrl = `${process.env.CABANA_URL}?retropilotIdentifier=${device.dongle_id}|${dongleIdHash}|${drive.identifier}|${driveIdentifierHash}&retropilotHost=${encodeURIComponent(process.env.BASE_URL)}&demo=1"`;
}
let vehicle = '';
@ -573,7 +571,7 @@ router.get('/useradmin/drive/:dongleId/:driveIdentifier', runAsyncWrapper(async
<table border=1 cellpadding=2 cellspacing=2>
<tr><th>segment</th><th>qcamera</th><th>qlog</th><th>fcamera</th><th>rlog</th><th>dcamera</th><th>processed</th><th>stalled</th></tr>`;
const directoryTree = dirTree(config.storagePath + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier);
const directoryTree = dirTree(process.env.STORAGE_PATH + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier);
const directorySegments = {};
for (var i in directoryTree.children) {
// skip any non-directory entries (for example m3u8 file in the drive directory)

View File

@ -1,10 +1,9 @@
/* eslint-disable global-require */
import fs from 'fs';
import 'dotenv/config'
import log4js from 'log4js';
import lockfile from 'proper-lockfile';
import http from 'http';
import https from 'https';
import express from 'express';
import cors from 'cors';
import rateLimit from 'express-rate-limit';
@ -12,17 +11,13 @@ import cookieParser from 'cookie-parser';
import storageController from './controllers/storage.js';
/* eslint-disable no-unused-vars */
import webWebsocket from './websocket/web/index.js';
import athena from './websocket/athena/index.js';
import routers from './routes/index.js';
import orm from './models/index.model.js';
import controllers from './controllers/index.js';
import router from './routes/api/realtime.js';
/* eslint-enable no-unused-vars */
import config from './config.js';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
@ -55,7 +50,7 @@ const web = async () => {
const app = express();
app.use((req, res, next) => {
// TODO: can we use config.baseUrl here?
// TODO: can we use process.env.BASE_URL here?
res.header('Access-Control-Allow-Origin', 'http://localhost:3000');
res.header('Access-Control-Allow-Credentials', true);
res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept');
@ -69,10 +64,10 @@ const web = async () => {
app.use(routers.useradmin);
app.use(routers.authenticationApi);
if (config.athena.enabled) {
if (process.env.ATHENA_ENABLED) {
const athenaRateLimit = rateLimit({
windowMs: 30000,
max: config.athena.api.ratelimit,
max: process.env.ATHENA_API_RATE_LIMIT,
});
app.use((req, res, next) => {
@ -91,7 +86,7 @@ const web = async () => {
app.use(cors({ origin: 'http://localhost:3000' }));
app.use(cookieParser());
app.use('/favicon.ico', express.static('static/favicon.ico'));
app.use(config.baseDriveDownloadPathMapping, express.static(config.storagePath));
app.use(process.env.BASE_DRIVE_DOWNLOAD_PATH_MAPPING, express.static(process.env.STORAGE_PATH));
app.use(routers.deviceApi);
@ -127,8 +122,8 @@ lockfile.lock('retropilot_server', { realpath: false, stale: 30000, update: 2000
const httpServer = http.createServer(app);
httpServer.listen(config.httpPort, config.httpInterface, () => {
logger.info(`Retropilot Server listening at http://${config.httpInterface}:${config.httpPort}`);
httpServer.listen(process.env.HTTP_PORT, process.env.HTTP_INTERFACE, () => {
logger.info(`Retropilot Server listening at http://${process.env.HTTP_INTERFACE}:${process.env.HTTP_PORT}`);
});
}).catch((e) => {

View File

@ -0,0 +1,800 @@
--
-- PostgreSQL database dump
--
-- Dumped from database version 14.1
-- Dumped by pg_dump version 14.1
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET xmloption = content;
SET client_min_messages = warning;
SET row_security = off;
--
-- Name: accounts; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.accounts (
id integer NOT NULL,
email character varying(64) NOT NULL,
password character varying(246),
created bigint,
last_ping bigint,
banned boolean DEFAULT false,
"2fa_token" character varying(200),
admin boolean DEFAULT false,
session_seed character varying(256),
email_verify_token character varying,
g_oauth_sub character varying,
two_factor_enabled boolean DEFAULT false NOT NULL
);
--
-- Name: accounts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.accounts_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: accounts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.accounts_id_seq OWNED BY public.accounts.id;
--
-- Name: athena_action_logs; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.athena_action_logs (
id integer NOT NULL,
account_id integer,
device_id integer,
action character varying(38),
user_ip character varying(1),
device_ip character varying(19),
meta json,
created_at bigint,
dongle_id character varying(8)
);
--
-- Name: athena_action_logs_account_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.athena_action_logs_account_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: athena_action_logs_account_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.athena_action_logs_account_id_seq OWNED BY public.athena_action_logs.account_id;
--
-- Name: athena_action_logs_device_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.athena_action_logs_device_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: athena_action_logs_device_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.athena_action_logs_device_id_seq OWNED BY public.athena_action_logs.device_id;
--
-- Name: athena_action_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.athena_action_logs_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: athena_action_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.athena_action_logs_id_seq OWNED BY public.athena_action_logs.id;
--
-- Name: athena_returned_data; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.athena_returned_data (
id integer NOT NULL,
device_id integer NOT NULL,
type character varying(12),
data json,
created_at bigint,
uuid character varying(12),
resolved_at bigint
);
--
-- Name: athena_returned_data_device_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.athena_returned_data_device_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: athena_returned_data_device_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.athena_returned_data_device_id_seq OWNED BY public.athena_returned_data.device_id;
--
-- Name: athena_returned_data_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.athena_returned_data_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: athena_returned_data_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.athena_returned_data_id_seq OWNED BY public.athena_returned_data.id;
--
-- Name: device_authorised_users; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.device_authorised_users (
id integer NOT NULL,
account_id integer NOT NULL,
device_id integer NOT NULL,
athena boolean DEFAULT false NOT NULL,
unpair boolean DEFAULT false NOT NULL,
view_drives boolean DEFAULT false NOT NULL,
created_at bigint NOT NULL
);
--
-- Name: device_authorised_users_account_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.device_authorised_users_account_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: device_authorised_users_account_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.device_authorised_users_account_id_seq OWNED BY public.device_authorised_users.account_id;
--
-- Name: device_authorised_users_device_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.device_authorised_users_device_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: device_authorised_users_device_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.device_authorised_users_device_id_seq OWNED BY public.device_authorised_users.device_id;
--
-- Name: device_authorised_users_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.device_authorised_users_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: device_authorised_users_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.device_authorised_users_id_seq OWNED BY public.device_authorised_users.id;
--
-- Name: devices; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.devices (
id integer NOT NULL,
dongle_id text NOT NULL,
account_id integer,
imei text,
serial text,
device_type text,
public_key text,
created bigint,
last_ping bigint,
storage_used bigint,
max_storage bigint,
ignore_uploads boolean,
nickname character varying(20)
);
--
-- Name: devices_account_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.devices_account_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: devices_account_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.devices_account_id_seq OWNED BY public.devices.account_id;
--
-- Name: devices_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.devices_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: devices_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.devices_id_seq OWNED BY public.devices.id;
--
-- Name: drive_segments; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.drive_segments (
id integer NOT NULL,
segment_id bigint NOT NULL,
drive_identifier character varying(20) NOT NULL,
dongle_id character varying(12) NOT NULL,
duration bigint,
distance_meters bigint,
upload_complete boolean DEFAULT false NOT NULL,
is_processed boolean DEFAULT false NOT NULL,
is_stalled boolean DEFAULT false NOT NULL,
created bigint DEFAULT 0 NOT NULL,
process_attempts smallint DEFAULT 0 NOT NULL
);
--
-- Name: drive_segments_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.drive_segments_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: drive_segments_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.drive_segments_id_seq OWNED BY public.drive_segments.id;
--
-- Name: drives; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.drives (
id integer NOT NULL,
identifier character varying(20),
dongle_id character varying(8),
max_segment bigint,
duration double precision,
distance_meters double precision,
filesize bigint,
upload_complete boolean DEFAULT false NOT NULL,
is_processed boolean DEFAULT false NOT NULL,
created bigint,
last_upload bigint,
is_preserved boolean DEFAULT false NOT NULL,
is_deleted boolean DEFAULT false NOT NULL,
drive_date bigint DEFAULT 0,
is_physically_removed boolean DEFAULT false NOT NULL,
metadata text
);
--
-- Name: drives_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.drives_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: drives_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.drives_id_seq OWNED BY public.drives.id;
--
-- Name: oauth_accounts; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.oauth_accounts (
id integer NOT NULL,
account_id integer NOT NULL,
email character varying NOT NULL,
created time without time zone,
last_used character varying,
refresh character varying,
provider character varying,
external_id character varying,
enabled boolean DEFAULT false NOT NULL
);
--
-- Name: oauth_accounts_account_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.oauth_accounts_account_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: oauth_accounts_account_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.oauth_accounts_account_id_seq OWNED BY public.oauth_accounts.account_id;
--
-- Name: oauth_accounts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.oauth_accounts_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: oauth_accounts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.oauth_accounts_id_seq OWNED BY public.oauth_accounts.id;
--
-- Name: sessions; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.sessions (
id integer NOT NULL,
sessionkey character varying,
account_id bigint,
ip_address character varying,
expires bigint
);
--
-- Name: sessions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.sessions_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: sessions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.sessions_id_seq OWNED BY public.sessions.id;
--
-- Name: accounts id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.accounts ALTER COLUMN id SET DEFAULT nextval('public.accounts_id_seq'::regclass);
--
-- Name: athena_action_logs id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_action_logs ALTER COLUMN id SET DEFAULT nextval('public.athena_action_logs_id_seq'::regclass);
--
-- Name: athena_action_logs account_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_action_logs ALTER COLUMN account_id SET DEFAULT nextval('public.athena_action_logs_account_id_seq'::regclass);
--
-- Name: athena_action_logs device_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_action_logs ALTER COLUMN device_id SET DEFAULT nextval('public.athena_action_logs_device_id_seq'::regclass);
--
-- Name: athena_returned_data id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_returned_data ALTER COLUMN id SET DEFAULT nextval('public.athena_returned_data_id_seq'::regclass);
--
-- Name: athena_returned_data device_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_returned_data ALTER COLUMN device_id SET DEFAULT nextval('public.athena_returned_data_device_id_seq'::regclass);
--
-- Name: device_authorised_users id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users ALTER COLUMN id SET DEFAULT nextval('public.device_authorised_users_id_seq'::regclass);
--
-- Name: device_authorised_users account_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users ALTER COLUMN account_id SET DEFAULT nextval('public.device_authorised_users_account_id_seq'::regclass);
--
-- Name: device_authorised_users device_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users ALTER COLUMN device_id SET DEFAULT nextval('public.device_authorised_users_device_id_seq'::regclass);
--
-- Name: devices id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.devices ALTER COLUMN id SET DEFAULT nextval('public.devices_id_seq'::regclass);
--
-- Name: drive_segments id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drive_segments ALTER COLUMN id SET DEFAULT nextval('public.drive_segments_id_seq'::regclass);
--
-- Name: drives id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drives ALTER COLUMN id SET DEFAULT nextval('public.drives_id_seq'::regclass);
--
-- Name: oauth_accounts id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.oauth_accounts ALTER COLUMN id SET DEFAULT nextval('public.oauth_accounts_id_seq'::regclass);
--
-- Name: oauth_accounts account_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.oauth_accounts ALTER COLUMN account_id SET DEFAULT nextval('public.oauth_accounts_account_id_seq'::regclass);
--
-- Name: sessions id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.sessions ALTER COLUMN id SET DEFAULT nextval('public.sessions_id_seq'::regclass);
--
-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.accounts
ADD CONSTRAINT accounts_pkey PRIMARY KEY (id);
--
-- Name: accounts accounts_un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.accounts
ADD CONSTRAINT accounts_un UNIQUE (email);
--
-- Name: athena_action_logs athena_action_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_action_logs
ADD CONSTRAINT athena_action_logs_pkey PRIMARY KEY (id);
--
-- Name: athena_action_logs athena_action_logs_un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_action_logs
ADD CONSTRAINT athena_action_logs_un UNIQUE (id);
--
-- Name: athena_returned_data athena_returned_data_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_returned_data
ADD CONSTRAINT athena_returned_data_pkey PRIMARY KEY (id);
--
-- Name: device_authorised_users device_authorised_users_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users
ADD CONSTRAINT device_authorised_users_pkey PRIMARY KEY (id);
--
-- Name: device_authorised_users device_authorised_users_un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users
ADD CONSTRAINT device_authorised_users_un UNIQUE (id);
--
-- Name: devices devices_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.devices
ADD CONSTRAINT devices_pkey PRIMARY KEY (id);
--
-- Name: devices devices_un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.devices
ADD CONSTRAINT devices_un UNIQUE (id);
--
-- Name: devices devices_unique_dongle; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.devices
ADD CONSTRAINT devices_unique_dongle UNIQUE (dongle_id);
--
-- Name: drive_segments drive_segment_uk; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drive_segments
ADD CONSTRAINT drive_segment_uk UNIQUE (id);
--
-- Name: drive_segments drive_segments_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drive_segments
ADD CONSTRAINT drive_segments_pkey PRIMARY KEY (id);
--
-- Name: drives drives_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drives
ADD CONSTRAINT drives_pkey PRIMARY KEY (id);
--
-- Name: drives drives_un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drives
ADD CONSTRAINT drives_un UNIQUE (id);
--
-- Name: oauth_accounts oauth_accounts_pk; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.oauth_accounts
ADD CONSTRAINT oauth_accounts_pk PRIMARY KEY (id);
--
-- Name: accounts primary_uni; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.accounts
ADD CONSTRAINT primary_uni UNIQUE (id);
--
-- Name: sessions sessions_pk; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.sessions
ADD CONSTRAINT sessions_pk PRIMARY KEY (id);
--
-- Name: athena_returned_data un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_returned_data
ADD CONSTRAINT un UNIQUE (id);
--
-- Name: device_authorised_users device_authorised_users_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users
ADD CONSTRAINT device_authorised_users_fk FOREIGN KEY (account_id) REFERENCES public.accounts(id);
--
-- Name: device_authorised_users device_authorised_users_fk_1; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users
ADD CONSTRAINT device_authorised_users_fk_1 FOREIGN KEY (device_id) REFERENCES public.devices(id);
--
-- Name: athena_returned_data device_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_returned_data
ADD CONSTRAINT device_id_fk FOREIGN KEY (device_id) REFERENCES public.devices(id);
--
-- Name: devices devices_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.devices
ADD CONSTRAINT devices_fk FOREIGN KEY (account_id) REFERENCES public.accounts(id);
--
-- Name: drive_segments drive_segments_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drive_segments
ADD CONSTRAINT drive_segments_fk FOREIGN KEY (dongle_id) REFERENCES public.devices(dongle_id);
--
-- Name: drives drives_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drives
ADD CONSTRAINT drives_fk FOREIGN KEY (dongle_id) REFERENCES public.devices(dongle_id);
--
-- PostgreSQL database dump complete
--

View File

@ -23,7 +23,7 @@ export default (server) => {
body.data.hasOwnProperty('serverStats') &&
body.data.serverStats.hasOwnProperty('config') &&
typeof body.data.serverStats.config.registerAllowed === "boolean" &&
typeof body.data.serverStats.config.welcomeMessage === "string" &&
typeof body.data.serverStats.process.env.WELCOME_MESSAGE === "string" &&
typeof body.data.serverStats['accounts'] === "number" &&
typeof body.data.serverStats['devices'] === "number" &&
typeof body.data.serverStats['drives'] === "number" &&

View File

@ -1,6 +1,5 @@
import request from 'supertest';
import server from '../server';
import config from '../config';
// TODO better way to only run tests once server is up
describe('loading express', () => {
@ -18,4 +17,4 @@ describe('loading express', () => {
require('./routes/api.test')(server);
require('./routes/useradmin.test')(server);
if (config.flags.useUserAdminApi) require('./routes/userAdminApi.test')(server);
if (process.env.USE_USER_ADMIN_API) require('./routes/userAdminApi.test')(server);

View File

@ -1,16 +1,12 @@
import WebSocket, { WebSocketServer } from 'ws'; import cookie from 'cookie';
import { WebSocketServer } from 'ws'; import cookie from 'cookie';
import jsonwebtoken from 'jsonwebtoken';
import httpsServer from 'https';
import httpServer from 'http';
import { readFileSync } from 'fs';
import log4js from 'log4js';
import models from '../../models/index.model';
import config from '../../config';
import helperFunctions from './helpers';
// eslint-disable-next-line no-unused-vars
import authenticationController from '../../controllers/authentication';
import deviceController from '../../controllers/devices';
const logger = log4js.getLogger('default');
@ -21,7 +17,7 @@ let wss;
function __server() {
let server;
if (config.athena.secure) {
if (process.env.ATHENA_SECURE) {
server = httpsServer.createServer({
cert: readFileSync(config.sslCrt),
key: readFileSync(config.sslKey),
@ -48,10 +44,10 @@ function __server() {
ws.isAlive = false;
ws.ping();
});
}, config.athena.socket.heartbeatFrequency ? config.athena.socket.heartbeatFrequency : 5000);
}, process.env.ATHENA_SOCKET_HEARTBEAT_FREQ ? process.env.ATHENA_SOCKET_HEARTBEAT_FREQ : 5000);
server.listen(config.athena.socket.port, () => {
logger.info(`Athena(Server) - UP @ ${config.athena.host}:${config.athena.port}`);
server.listen(process.env.ATHENA_SOCKET_PORT, () => {
logger.info(`Athena(Server) - UP @ ${process.env.ATHENA_SOCKET_HOST}:${process.env.ATHENA_SOCKET_PORT}`);
});
wss.on('connection', manageConnection);

View File

@ -1,14 +1,10 @@
import WebSocket, { WebSocketServer } from 'ws';
import { WebSocketServer } from 'ws';
import cookie from 'cookie';
import httpServer from 'http';
import log4js from 'log4js';
import config from '../../config.js';
import controlsFunction from './controls.js';
import authenticationController from '../../controllers/authentication.js';
// eslint-disable-next-line no-unused-vars
import deviceController from '../../controllers/devices';
import athenaRealtime from '../athena/index.js';
import realtimeCommands from './commands.js';
@ -24,8 +20,8 @@ function __server() {
wss = new WebSocketServer({ server }, { path: '/realtime/v1', handshakeTimeout: 500 });
server.listen(config.clientSocket.port, config.clientSocket.host, () => {
logger.info(`Web(Server) - UP @ ${config.clientSocket.host}:${config.clientSocket.port}`);
server.listen(process.env.CLIENT_SOCKET_PORT, process.env.CLIENT_SOCKET_HOST, () => {
logger.info(`Web(Server) - UP @ ${process.env.CLIENT_SOCKET_HOST}:${process.env.CLIENT_SOCKET_PORT}`);
});
wss.on('connection', manageConnection);

197
worker.js
View File

@ -1,13 +1,11 @@
/* eslint-disable */
import 'dotenv/config'
import crypto from 'crypto';
import fs from 'fs';
import path, { dirname } from 'path';
import { fileURLToPath } from 'url';
import log4js from 'log4js';
import sqlite3 from 'sqlite3';
import { open } from 'sqlite';
import lockfile from 'proper-lockfile';
import dirTree from 'directory-tree';
import { execSync } from 'child_process';
@ -16,10 +14,6 @@ import ffprobe from 'ffprobe';
import ffprobeStatic from 'ffprobe-static';
import orm from './models/index.model';
import config from './config';
var db = null;
var lastCleaningTime = 0;
var startTime = Date.now();
@ -36,12 +30,12 @@ const __dirname = dirname(__filename);
global.__basedir = __dirname;
function initializeStorage() {
var verifiedPath = mkDirByPathSync(config.storagePath, { isRelativeToScript: (config.storagePath.indexOf('/') !== 0) });
var verifiedPath = mkDirByPathSync(process.env.STORAGE_PATH, { isRelativeToScript: (process.env.STORAGE_PATH.indexOf('/') !== 0) });
if (verifiedPath != null) {
logger.info(`Verified storage path ${verifiedPath}`);
}
else {
logger.error(`Unable to verify storage path '${config.storagePath}', check filesystem / permissions`);
logger.error(`Unable to verify storage path '${process.env.STORAGE_PATH}', check filesystem / permissions`);
process.exit();
}
}
@ -106,12 +100,12 @@ function moveUploadedFile(buffer, directory, filename) {
return false;
}
if (config.storagePath.lastIndexOf('/') !== config.storagePath.length - 1) {
if (process.env.STORAGE_PATH.lastIndexOf('/') !== process.env.STORAGE_PATH.length - 1) {
directory = `/${directory}`;
}
if (directory.lastIndexOf('/') !== directory.length - 1) directory += '/';
var finalPath = mkDirByPathSync(config.storagePath + directory, { isRelativeToScript: (config.storagePath.indexOf('/') !== 0) });
var finalPath = mkDirByPathSync(process.env.STORAGE_PATH + directory, { isRelativeToScript: (process.env.STORAGE_PATH.indexOf('/') !== 0) });
if (finalPath && finalPath.length > 0) {
if (writeFileSync(`${finalPath}/${filename}`, buffer, 0o660)) {
logger.info(`moveUploadedFile successfully written '${finalPath}/${filename}'`);
@ -120,7 +114,7 @@ function moveUploadedFile(buffer, directory, filename) {
logger.error('moveUploadedFile failed to writeFileSync');
return false;
}
logger.error(`moveUploadedFile invalid final path, check permissions to create / write '${config.storagePath + directory}'`);
logger.error(`moveUploadedFile invalid final path, check permissions to create / write '${process.env.STORAGE_PATH + directory}'`);
return false;
}
@ -141,60 +135,6 @@ function deleteFolderRecursive(directoryPath) {
}
}
async function dbProtectedRun() {
let retries = 0;
while (true) {
try {
return await db.run(...arguments);
} catch (error) {
logger.error(error);
retries++;
if (retries >= 10) {
break;
}
await new Promise((r) => setTimeout(r, 1000));
}
}
logger.error(`unable to complete dbProtectedRun for ${arguments}`);
return null;
}
async function dbProtectedGet() {
let retries = 0;
while (true) {
try {
return await db.get(...arguments);
} catch (error) {
logger.error(error);
retries++;
if (retries >= 10) {
break;
}
await new Promise((r) => setTimeout(r, 1000));
}
}
logger.error(`unable to complete dbProtectedGet for ${arguments}`);
return null;
}
async function dbProtectedAll() {
let retries = 0;
while (true) {
try {
return await db.all(...arguments);
} catch (error) {
logger.error(error);
retries++;
if (retries >= 10) {
break;
}
await new Promise((r) => setTimeout(r, 1000));
}
}
logger.error(`unable to complete dbProtectedGet for ${arguments}`);
return null;
}
var segmentProcessQueue = [];
var segmentProcessPosition = 0;
@ -356,19 +296,16 @@ function processSegmentsRecursive() {
logger.info(`processSegmentsRecursive ${segment.dongle_id} ${segment.drive_identifier} ${segment.segment_id} ${JSON.stringify(segment)}`);
const driveSegmentResult = dbProtectedRun(
'UPDATE drive_segments SET process_attempts = ? WHERE id = ?',
segment.process_attempts = segment.process_attempts + 1
segment.process_attempts = segment.process_attempts + 1,
segment.id
const [driveSegmentResult] = orm.query(
`UPDATE drive_segments SET process_attempts = ${segment.process_attempts} WHERE id = ${segment.id}`,
);
if (segment.process_attempts > 5) {
logger.error(`FAILING TO PROCESS SEGMENT,${segment.dongle_id} ${segment.drive_identifier} ${segment.segment_id} JSON: ${JSON.stringify(segment)} SKIPPING `);
segmentProcessPosition++;
}
else {
} else {
var p1 = processSegmentRLog(fileStatus['rlog.bz2']);
var p2 = processSegmentVideo(fileStatus['qcamera.ts']);
Promise.all([p1, p2])
@ -382,7 +319,6 @@ function processSegmentsRecursive() {
is_processed: true,
upload_complete: uploadComplete,
is_stalled: false
}, {where: {id: segment.id}})
@ -419,19 +355,19 @@ async function updateSegments() {
affectedDriveCarParams = {};
affectedDriveInitData = {};
const drive_segments = await dbProtectedAll('SELECT * FROM drive_segments WHERE upload_complete = ? AND is_stalled = ? AND process_attempts < ? ORDER BY created ASC', false, false, 5);
const [drive_segments] = await orm.query('SELECT * FROM drive_segments WHERE upload_complete = false AND is_stalled = false AND process_attempts < 5 ORDER BY created ASC');
if (drive_segments != null) {
for (var t = 0; t < drive_segments.length; t++) {
var segment = drive_segments[t];
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
var dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(segment.dongle_id)
.digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt)
var driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(segment.drive_identifier)
.digest('hex');
const directoryTree = dirTree(`${config.storagePath + segment.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${segment.drive_identifier}/${segment.segment_id}`);
const directoryTree = dirTree(`${process.env.STORAGE_PATH + segment.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${segment.drive_identifier}/${segment.segment_id}`);
if (directoryTree == null || directoryTree.children == undefined) continue; // happens if upload in progress (db entity written but directory not yet created)
var qcamera = false;
@ -468,13 +404,8 @@ async function updateSegments() {
else if (uploadComplete) {
logger.info(`updateSegments uploadComplete for ${segment.dongle_id} ${segment.drive_identifier} ${segment.segment_id}`);
const driveSegmentResult = await dbProtectedRun(
'UPDATE drive_segments SET upload_complete = ?, is_stalled = ? WHERE id = ?',
true,
false,
segment.id
const [driveSegmentResult] = await orm.query(
`UPDATE drive_segments SET upload_complete = true, is_stalled = false WHERE id = ${segment.id}`
);
affectedDrives[`${segment.dongle_id}|${segment.drive_identifier}`] = true;
@ -482,11 +413,8 @@ async function updateSegments() {
else if (Date.now() - segment.created > 10 * 24 * 3600 * 1000) { // ignore non-uploaded segments after 10 days until a new upload_url is requested (which resets is_stalled)
logger.info(`updateSegments isStalled for ${segment.dongle_id} ${segment.drive_identifier} ${segment.segment_id}`);
const driveSegmentResult = await dbProtectedRun(
'UPDATE drive_segments SET is_stalled = ? WHERE id = ?',
true,
segment.id
const driveSegmentResult = await orm.query(
`UPDATE drive_segments SET is_stalled = true WHERE id = ${segment.id}`
);
}
@ -512,13 +440,13 @@ async function updateDevices() {
for (const [key, value] of Object.entries(affectedDevices)) {
var dongleId = key;
const device = await dbProtectedGet('SELECT * FROM devices WHERE dongle_id = ?', dongleId);
const [device] = await orm.query(`SELECT * FROM devices WHERE dongle_id = ${dongleId}`);
if (device == null) continue;
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
var dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(device.dongle_id)
.digest('hex');
var devicePath = `${config.storagePath + device.dongle_id}/${dongleIdHash}`;
var devicePath = `${process.env.STORAGE_PATH + device.dongle_id}/${dongleIdHash}`;
var deviceQuotaMb = Math.round(parseInt(execSync(`du -s ${devicePath} | awk -F'\t' '{print $1;}'`)
.toString()) / 1024);
logger.info(`updateDevices device ${dongleId} has an updated storage_used of: ${deviceQuotaMb} MB`);
@ -543,14 +471,14 @@ async function updateDrives() {
let drive = await orm.models.drives({where: {driveIdentifier: driveIdentifier, dongleId: dongleId}})
if (drive == null) continue;
drive = drive.dataValues;
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
var dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(drive.dongle_id)
.digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt)
var driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(drive.identifier)
.digest('hex');
var driveUrl = `${config.baseDriveDownloadUrl + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
var drivePath = `${config.storagePath + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
var driveUrl = `${process.env.BASE_DRIVE_DOWNLOAD_URL + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
var drivePath = `${process.env.STORAGE_PATH + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
var uploadComplete = true;
var isProcessed = true;
@ -587,10 +515,10 @@ async function updateDrives() {
var { filesize } = drive;
if (uploadComplete) {
try {
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
var dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(dongleId)
.digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt)
var driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(driveIdentifier)
.digest('hex');
filesize = parseInt(execSync(`du -s ${drivePath} | awk -F'\t' '{print $1;}'`)
@ -650,12 +578,12 @@ async function updateDrives() {
}
async function deleteExpiredDrives() {
var expirationTs = Date.now() - config.deviceDriveExpirationDays * 24 * 3600 * 1000;
var expirationTs = Date.now() - process.env.DEVICE_EXPIRATION_DAYS * 24 * 3600 * 1000;
const expiredDrives = await dbProtectedAll('SELECT * FROM drives WHERE is_preserved = ? AND is_deleted = ? AND created < ?', false, false, expirationTs);
const [expiredDrives] = await orm.query(`SELECT * FROM drives WHERE is_preserved = false AND is_deleted = false AND created < ${expirationTs}`);
if (expiredDrives != null) {
for (var t = 0; t < expiredDrives.length; t++) {
logger.info(`deleteExpiredDrives drive ${expiredDrives[t].dongle_id} ${expiredDrives[t].identifier} is older than ${config.deviceDriveExpirationDays} days, set is_deleted=true`);
logger.info(`deleteExpiredDrives drive ${expiredDrives[t].dongle_id} ${expiredDrives[t].identifier} is older than ${process.env.DEVICE_EXPIRATION_DAYS} days, set is_deleted=true`);
const driveResult = await orm.models.drives.update({
is_deleted: true
},
@ -666,35 +594,27 @@ async function deleteExpiredDrives() {
}
async function removeDeletedDrivesPhysically() {
const deletedDrives = await dbProtectedAll('SELECT * FROM drives WHERE is_deleted = ? AND is_physically_removed = ?', true, false);
const [deletedDrives] = await orm.query('SELECT * FROM drives WHERE is_deleted = true AND is_physically_removed = false');
if (deletedDrives == null) {
return;
}
for (var t = 0; t < deletedDrives.length; t++) {
logger.info(`removeDeletedDrivesPhysically drive ${deletedDrives[t].dongle_id} ${deletedDrives[t].identifier} is deleted, remove physical files and clean database`);
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
var dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(deletedDrives[t].dongle_id)
.digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt)
var driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(deletedDrives[t].identifier)
.digest('hex');
const drivePath = `${config.storagePath + deletedDrives[t].dongle_id}/${dongleIdHash}/${driveIdentifierHash}`;
const drivePath = `${process.env.STORAGE_PATH + deletedDrives[t].dongle_id}/${dongleIdHash}/${driveIdentifierHash}`;
logger.info(`removeDeletedDrivesPhysically drive ${deletedDrives[t].dongle_id} ${deletedDrives[t].identifier} storage path is ${drivePath}`);
try {
const driveResult = await dbProtectedRun(
'UPDATE drives SET is_physically_removed = ? WHERE id = ?',
true,
const driveResult = await orm.query(`UPDATE drives SET is_physically_removed = true WHERE id = ${deletedDrives[t].id}`);
deletedDrives[t].id
);
const driveSegmentResult = await dbProtectedRun(
'DELETE FROM drive_segments WHERE drive_identifier = ? AND dongle_id = ?',
deletedDrives[t].identifier,
deletedDrives[t].dongle_id
const driveSegmentResult = await orm.query(
`DELETE FROM drive_segments WHERE drive_identifier = ${deletedDrives[t].identifier} AND dongle_id = ${deletedDrives[t].dongle_id}`
);
if (driveResult != null && driveSegmentResult != null) deleteFolderRecursive(drivePath, { recursive: true });
@ -706,7 +626,7 @@ async function removeDeletedDrivesPhysically() {
}
async function deleteOverQuotaDrives() {
const devices = await dbProtectedAll('SELECT * FROM devices WHERE storage_used > ?', config.deviceStorageQuotaMb);
const [devices] = await orm.query(`SELECT * FROM devices WHERE storage_used > ${process.env.DEVICE_STORAGE_QUOTA_MB}`);
if (devices == null) {
return;
}
@ -714,27 +634,21 @@ async function deleteOverQuotaDrives() {
for (var t = 0; t < devices.length; t++) {
var foundDriveToDelete = false;
const driveNormal = await dbProtectedGet('SELECT * FROM drives WHERE dongle_id = ? AND is_preserved = ? AND is_deleted = ? ORDER BY created ASC LIMIT 1', devices[t].dongle_id, false, false);
const [driveNormal] = await orm.query(`SELECT * FROM drives WHERE dongle_id = ${devices[t].dongle_id} AND is_preserved = false AND is_deleted = false ORDER BY created ASC LIMIT 1`);
if (driveNormal != null) {
logger.info(`deleteOverQuotaDrives drive ${driveNormal.dongle_id} ${driveNormal.identifier} (normal) is deleted for over-quota`);
const driveResult = await dbProtectedRun(
'UPDATE drives SET is_deleted = ? WHERE id = ?',
true,
driveNormal.id
const [driveResult] = await orm.query(
`UPDATE drives SET is_deleted = true WHERE id = ${driveNormal.id}`,
);
foundDriveToDelete = true;
}
if (!foundDriveToDelete) {
const drivePreserved = await dbProtectedGet('SELECT * FROM drives WHERE dongle_id = ? AND is_preserved = ? AND is_deleted = ? ORDER BY created ASC LIMIT 1', devices[t].dongle_id, true, false);
const [drivePreserved] = await orm.query(`SELECT * FROM drives WHERE dongle_id = devices[t].dongle_id AND is_preserved = true AND is_deleted = false ORDER BY created ASC LIMIT 1`);
if (drivePreserved != null) {
logger.info(`deleteOverQuotaDrives drive ${drivePreserved.dongle_id} ${drivePreserved.identifier} (preserved!) is deleted for over-quota`);
const driveResult = await dbProtectedRun(
'UPDATE drives SET is_deleted = ? WHERE id = ?',
true,
drivePreserved.id
const [driveResult] = await orm.query(
`UPDATE drives SET is_deleted = ? WHERE id = ${drivePreserved.id}`
);
foundDriveToDelete = true;
}
@ -743,18 +657,18 @@ async function deleteOverQuotaDrives() {
}
async function deleteBootAndCrashLogs() {
const devices = await dbProtectedAll('SELECT * FROM devices');
const [devices] = await orm.query('SELECT * FROM devices');
if (devices == null) {
return;
}
for (var t = 0; t < devices.length; t++) {
var device = devices[t];
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
var dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(device.dongle_id)
.digest('hex');
const bootlogDirectoryTree = dirTree(`${config.storagePath + device.dongle_id}/${dongleIdHash}/boot/`, { attributes: ['size'] });
const bootlogDirectoryTree = dirTree(`${process.env.STORAGE_PATH + device.dongle_id}/${dongleIdHash}/boot/`, { attributes: ['size'] });
var bootlogFiles = [];
if (bootlogDirectoryTree != undefined) {
for (var i = 0; i < bootlogDirectoryTree.children.length; i++) {
@ -782,7 +696,7 @@ async function deleteBootAndCrashLogs() {
}
}
const crashlogDirectoryTree = dirTree(`${config.storagePath + device.dongle_id}/${dongleIdHash}/crash/`, { attributes: ['size'] });
const crashlogDirectoryTree = dirTree(`${process.env.STORAGE_PATH + device.dongle_id}/${dongleIdHash}/crash/`, { attributes: ['size'] });
var crashlogFiles = [];
if (crashlogDirectoryTree != undefined) {
for (var i = 0; i < crashlogDirectoryTree.children.length; i++) {
@ -851,21 +765,6 @@ lockfile.lock('retropilot_worker', {
.then((release) => {
logger.info('STARTING WORKER...');
(async () => {
try {
db = await open({
filename: config.databaseFile,
driver: sqlite3.Database,
mode: sqlite3.OPEN_READWRITE
});
await db.get('SELECT * FROM accounts LIMIT 1');
await db.get('SELECT * FROM devices LIMIT 1');
await db.get('SELECT * FROM drives LIMIT 1');
await db.get('SELECT * FROM drive_segments LIMIT 1');
} catch (exception) {
logger.error(exception);
process.exit();
}
initializeStorage();
setTimeout(() => {
mainWorkerLoop();