Merge pull request #7 from jor3l/fix/welcome-2022

WIP - Fixes to get the service back up
pull/4/head
Adam Black 2022-03-03 03:22:10 +00:00 committed by GitHub
commit 1e250dcd98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 1355 additions and 567 deletions

35
.env.sample 100644
View File

@ -0,0 +1,35 @@
NODE_ENV=development
APP_SALT=RANDOM_SEED
DB_FILE=database.sqlite
DB_NAME=retro-pilot
DB_USER=root
DB_PASS=root
DB_HOST=db
DB_PORT=5432
ALLOW_REGISTRATION=true
HTTP_INTERFACE=0.0.0.0
HTTP_PORT=3000
CAN_SEND_MAIL=1 # Skips sending mail, all attempted mail is logged under DEBUG
SMTP_HOST="localhost" # credentials for smtp server to send account registration mails. if not filled in, get the generated tokens from the server.log manually
SMTP_PORT=25
SMTP_USER=root
SMTP_PASS=
SMTP_FROM="no-reply@retropilot.org"
BASE_URL="http://192.168.1.165:3000/" # base url of the retropilot server
BASE_UPLOAD_URL="http://192.168.1.165:3000/backend/post_upload" # base url sent to devices for POSTing drives & logs
BASE_DRIVE_DOWNLOAD_URL="http://192.168.1.165:3000/realdata/" # base download url for drive & log data
BASE_DRIVE_DOWNLOAD_PATH_MAPPING="/realdata" # path mapping of above download url for expressjs, prefix with "/"
STORAGE_PATH="realdata/"# relative or absolute ( "/..." for absolute path )
CABANA_URL="http://192.168.1.165:3000/cabana/index.html"
DEVICE_STORAGE_QUOTA_MB=200000
DEVICE_EXPIRATION_DAYS=30
WELCOME_MESSAGE="<><><><><><><><><><><><><><><><><><><><><><><br>2022 RetroPilot"
USE_USER_ADMIN_API=0
CLIENT_SOCKET_PORT=81
CLIENT_SOCKET_HOST="0.0.0.0"
ATHENA_ENABLED=1 # Enables Athena service
ATHENA_SECURE=1 # Disables crypto on Websocket server - use for testing on local network, change ATHENA_HOST in openpilot to ws:// instead of wss://
ATHENA_API_RATE_LIMIT=100 # Maxmium hits to /realtime/* per 30s
ATHENA_SOCKET_HOST="0.0.0.0"
ATHENA_SOCKET_PORT=4040
ATHENA_SOCKET_HEARTBEAT_FREQ=5000 # Higher the number = lower traffic, varies on how many devices are connected

3
.gitignore vendored
View File

@ -18,3 +18,6 @@ test/.devKeys
# Miscellaneous
.DS_Store
*.log
.env
data/certbot

View File

@ -6,9 +6,10 @@ WORKDIR /app
# Install app dependencies
COPY package*.json ./
RUN npm ci
RUN npm install pm2 -g
# Bundle app source
COPY . .
EXPOSE 3000
CMD [ "node", "--es-module-specifier-resolution=node", "server.js" ]
CMD ["pm2-runtime", "ecosystem.config.js"]

View File

@ -89,3 +89,11 @@ The athena websockets interface is not implemented yet, so the comma app and ath
![image](https://user-images.githubusercontent.com/48515354/118385075-2a459c80-b60c-11eb-976c-bc331a609391.png)
![image](https://user-images.githubusercontent.com/48515354/118385084-37fb2200-b60c-11eb-8d3e-6db458827808.png)
## UAT
Launch with:
```
docker-compose -f docker-compose.yml -f docker-compose.uat.yml up -d
```

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -1,54 +0,0 @@
const config = {
applicationSalt: 'RANDOM_SEED',
databaseFile: 'database.sqlite',
allowAccountRegistration: true,
httpInterface: '0.0.0.0',
httpPort: 3000,
canSendMail: true, // Skips sending mail, all attempted mail is logged under DEBUG
smtpHost: 'localhost', // credentials for smtp server to send account registration mails. if not filled in, get the generated tokens from the server.log manually
smtpPort: 25,
smtpUser: 'root',
smtpPassword: '',
smtpFrom: 'no-reply@retropilot.org',
baseUrl: 'http://192.168.1.165:3000/', // base url of the retropilot server
baseUploadUrl: 'http://192.168.1.165:3000/backend/post_upload', // base url sent to devices for POSTing drives & logs
baseDriveDownloadUrl: 'http://192.168.1.165:3000/realdata/', // base download url for drive & log data
baseDriveDownloadPathMapping: '/realdata', // path mapping of above download url for expressjs, prefix with "/"
storagePath: 'realdata/', // relative or absolute ( "/..." for absolute path )
cabanaUrl: 'http://192.168.1.165:3000/cabana/index.html',
deviceStorageQuotaMb: 200000,
deviceDriveExpirationDays: 30,
welcomeMessage: '<><><><><><><><><><><><><><><><><><><><><><><br>2021 RetroPilot',
flags: {
useUserAdminApi: false,
},
clientSocket: { // Used in development, remove before prod
port: 81,
host: '0.0.0.0',
},
athena: {
enabled: true, // Enables Athena service
secure: true, // Disables crypto on Websocket server - use for testing on local network, change ATHENA_HOST in openpilot to ws:// instead of wss://
api: {
ratelimit: 100, // Maxmium hits to /realtime/* per 30s
},
socket: {
port: 4040,
heartbeatFrequency: 5000, // Higher the number = lower traffic, varies on how many devices are connected
},
},
};
export default config;

View File

@ -1,7 +1,6 @@
import crypto from 'crypto';
import jsonwebtoken from 'jsonwebtoken';
import log4js from 'log4js';
import config from '../config';
import orm from '../models/index.model';
const logger = log4js.getLogger('default');
@ -29,9 +28,9 @@ async function signIn(email, password) {
if (account.dataValues) {
account = account.dataValues;
const inputPassword = crypto.createHash('sha256').update(password + config.applicationSalt).digest('hex');
const inputPassword = crypto.createHash('sha256').update(password + process.env.APP_SALT).digest('hex');
if (account.password === inputPassword) {
const token = jsonwebtoken.sign({ accountId: account.id }, config.applicationSalt);
const token = jsonwebtoken.sign({ accountId: account.id }, process.env.APP_SALT);
return { success: true, jwt: token };
}
@ -44,10 +43,10 @@ async function changePassword(account, newPassword, oldPassword) {
if (!account || !newPassword || !oldPassword) {
return { success: false, error: 'MISSING_DATA' };
}
const oldPasswordHash = crypto.createHash('sha256').update(oldPassword + config.applicationSalt).digest('hex');
const oldPasswordHash = crypto.createHash('sha256').update(oldPassword + process.env.APP_SALT).digest('hex');
if (account.password === oldPasswordHash) {
const newPasswordHash = crypto.createHash('sha256').update(newPassword + config.applicationSalt).digest('hex');
const newPasswordHash = crypto.createHash('sha256').update(newPassword + process.env.APP_SALT).digest('hex');
await orm.models.accounts.update(
{ password: newPasswordHash },
@ -76,7 +75,7 @@ async function getAccountFromJWT(jwt, limitData) {
let token;
try {
token = jsonwebtoken.verify(jwt, config.applicationSalt);
token = jsonwebtoken.verify(jwt, process.env.APP_SALT);
} catch (err) {
return null;// {success: false, msg: 'BAD_JWT'}
}
@ -95,10 +94,14 @@ async function getAccountFromJWT(jwt, limitData) {
return null; // {success: false, isInvalid: true}
}
await orm.models.accounts.update(
{ last_ping: Date.now() },
{ where: { id: account.id } },
);
try {
await orm.models.accounts.update(
{ last_ping: Date.now() },
{ where: { id: account.id } },
);
} catch(error) {
console.log(error);
}
if (!account || account.banned) {
return null; // {success: false, isBanned: true}

View File

@ -45,11 +45,11 @@ export async function getToken(code, scope) {
return { error: true, ...AUTH_OAUTH_ERR_GOOGLE_FAILED_TOKEN_FETCH };
}
console.log(accessToken);
logger.log(`accessToken: ${accessToken}`);
const id = jsonwebtoken.decode(accessToken.token.id_token);
console.log(id);
logger.log(`jsonwebtoken.${id}`);
return id;
}

View File

@ -1,6 +1 @@
import crypto from 'crypto';
import { Logger } from 'log4js';
import { AUTH_REGISTER_OAUTH_NO_EMAIL, AUTH_REGISTER_ALREADY_REGISTERED } from '../../consistency/terms';
import { getAccountFromEmail } from '../users';
import orm from '../../models/index.model';
import config from '../../config';
//empty

View File

@ -1,9 +1,5 @@
import crypto from 'crypto';
import { Logger } from 'log4js';
import { AUTH_REGISTER_OAUTH_NO_EMAIL, AUTH_REGISTER_ALREADY_REGISTERED } from '../../consistency/terms';
import { getAccountFromEmail } from '../users';
import orm from '../../models/index.model';
import config from '../../config';
export function oauthRegister(email) {
if (!email) return { error: true, ...AUTH_REGISTER_OAUTH_NO_EMAIL };

View File

@ -1,5 +1,3 @@
import { log4js } from 'log4js';
import { crypto } from 'crypto';
import { generateSecret, verify } from '2fa-util';
import {
AUTH_2FA_BAD_ACCOUNT,
@ -8,9 +6,7 @@ import {
AUTH_2FA_ENROLLED,
AUTH_2FA_BAD_TOKEN,
} from '../../consistency/terms';
import { getAccountFromEmail } from '../users';
import orm from '../../models/index.model';
import config from '../../config';
export async function twoFactorOnboard(account) {
if (!account || !account.dataValues) { return { success: false, ...AUTH_2FA_BAD_ACCOUNT }; }

View File

@ -2,7 +2,6 @@ import sanitizeFactory from 'sanitize';
import crypto from 'crypto';
import dirTree from 'directory-tree';
import log4js from 'log4js';
import config from '../config';
import authenticationController from './authentication';
import orm from '../models/index.model';
import usersController from './users';
@ -18,26 +17,26 @@ async function pairDevice(account, qrString) {
// Versions >= 0.8.3 uses only a pairtoken
const qrCodeParts = qrString.split('--');
let deviceQuery;
let device;
let pairJWT;
if (qrString.indexOf('--') >= 0) {
const [, serial, pairToken] = qrCodeParts;
deviceQuery = await orm.models.device.findOne({ where: { serial } });
device = await orm.models.device.findOne({ where: { serial } });
pairJWT = pairToken;
} else {
const data = await authenticationController.readJWT(qrString);
if (!data.pair) {
if (!data || !data.pair) {
return { success: false, noPair: true };
}
deviceQuery = await orm.models.device.findOne({ where: { dongle_id: data.identity } });
device = await orm.models.device.findOne({ where: { dongle_id: data.identity } });
pairJWT = qrString;
}
if (deviceQuery == null || !deviceQuery.dataValues) {
return { success: false, registered: false };
return { success: false, registered: false, noPair: true };
}
const device = deviceQuery.dataValues;
const decoded = await authenticationController.validateJWT(pairJWT, device.public_key);
if (decoded == null || !decoded.pair) {
return { success: false, badToken: true };
@ -196,7 +195,7 @@ async function getDrives(dongleId, includeDeleted, includeMeta) {
async function getDrive(identifier) {
const drive = await orm.models.drives.findOne({ where: { identifier } });
console.log(drive);
logger.log(drive);
if (drive.dataValues) return drive.dataValues;
return null;
@ -211,9 +210,9 @@ async function getDriveFromidentifier(dongleId, identifier) {
*/
async function getCrashlogs(dongleId) {
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(dongleId).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId).digest('hex');
const directoryTree = dirTree(`${config.storagePath}${dongleId}/${dongleIdHash}/crash/`, { attributes: ['size'] });
const directoryTree = dirTree(`${process.env.STORAGE_PATH}${dongleId}/${dongleIdHash}/crash/`, { attributes: ['size'] });
const crashlogFiles = (directoryTree ? directoryTree.children : []).map((file) => {
const timeSplit = file.name.replace('boot-', '').replace('crash-', '').replace('.bz2', '').split('--');
let timeString = `${timeSplit[0]} ${timeSplit[1].replace(/-/g, ':')}`;
@ -236,7 +235,7 @@ async function getCrashlogs(dongleId) {
name: file.name,
size: file.size,
date: dateObj,
permalink: `${config.baseDriveDownloadUrl}${dongleId}/${dongleIdHash}/crash/${file.name}`,
permalink: `${process.env.BASE_DRIVE_DOWNLOAD_URL}${dongleId}/${dongleIdHash}/crash/${file.name}`,
};
});
crashlogFiles.sort((a, b) => ((a.date < b.date) ? 1 : -1));
@ -244,9 +243,9 @@ async function getCrashlogs(dongleId) {
}
async function getBootlogs(dongleId) {
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(dongleId).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId).digest('hex');
const directoryTree = dirTree(`${config.storagePath}${dongleId}/${dongleIdHash}/boot/`, { attributes: ['size'] });
const directoryTree = dirTree(`${process.env.STORAGE_PATH}${dongleId}/${dongleIdHash}/boot/`, { attributes: ['size'] });
const bootlogFiles = (directoryTree ? directoryTree.children : []).map((file) => {
const timeSplit = file.name.replace('boot-', '').replace('crash-', '').replace('.bz2', '').split('--');
const timeString = `${timeSplit[0]} ${timeSplit[1].replace(/-/g, ':')}`;
@ -263,7 +262,7 @@ async function getBootlogs(dongleId) {
name: file.name,
size: file.size,
date: dateObj,
permalink: `${config.baseDriveDownloadUrl}${dongleId}/${dongleIdHash}/boot/${file.name}`,
permalink: `${process.env.BASE_DRIVE_DOWNLOAD_URL}${dongleId}/${dongleIdHash}/boot/${file.name}`,
};
});
bootlogFiles.sort((a, b) => ((a.date < b.date) ? 1 : -1));
@ -274,7 +273,7 @@ async function updateOrCreateDrive(dongleId, identifier, data) {
logger.info('updateOrCreate Drive', dongleId, identifier, data);
const check = await orm.models.drives.findOne({ where: { dongle_id: dongleId, identifier } });
console.log('checking for existing drive....', check);
logger.log('checking for existing drive....', check);
if (check) {
return orm.models.drives.update(

View File

@ -1,24 +1,23 @@
import nodemailer from 'nodemailer';
import log4js from 'log4js';
import config from '../config';
const logger = log4js.getLogger('default');
const transporter = nodemailer.createTransport(
{
host: config.smtpHost,
port: config.smtpPort,
host: process.env.SMTP_HOST,
port: process.env.SMTP_PORT,
auth: {
user: config.smtpUser,
pass: config.smtpPassword,
user: process.env.SMTP_USER,
pass: process.env.SMTP_PASS,
},
logger: true,
debug: false,
},
{ from: config.smtpFrom },
{ from: process.env.SMTP_FROM },
);
async function sendEmailVerification(token, email) {
if (!config.canSendMail) {
if (!process.env.CAN_SEND_MAIL) {
return logger.warn(`Mailing disabled. ${email} - ${token}`);
}
@ -28,7 +27,7 @@ async function sendEmailVerification(token, email) {
try {
message = {
from: config.smtpFrom,
from: process.env.SMTP_FROM,
to: email.trim(),
subject: 'RetroPilot Registration Token',
text: `Your Email Registration Token Is: "${token}"`,

View File

@ -2,18 +2,17 @@ import path from 'path';
import fs from 'fs';
import { execSync } from 'child_process';
import log4js from 'log4js';
import config from '../config.js';
const logger = log4js.getLogger('default');
let totalStorageUsed;
function initializeStorage() {
const verifiedPath = mkDirByPathSync(config.storagePath, { isRelativeToScript: (config.storagePath.indexOf('/') !== 0) });
const verifiedPath = mkDirByPathSync(process.env.STORAGE_PATH, { isRelativeToScript: (process.env.STORAGE_PATH.indexOf('/') !== 0) });
if (verifiedPath != null) {
logger.info(`Verified storage path ${verifiedPath}`);
} else {
logger.error(`Unable to verify storage path '${config.storagePath}', check filesystem / permissions`);
logger.error(`Unable to verify storage path '${process.env.STORAGE_PATH}', check filesystem / permissions`);
process.exit();
}
}
@ -79,16 +78,16 @@ function moveUploadedFile(buffer, directory, filename) {
return false;
}
if (config.storagePath.lastIndexOf('/') !== config.storagePath.length - 1) {
if (process.env.STORAGE_PATH.lastIndexOf('/') !== process.env.STORAGE_PATH.length - 1) {
directory = `/${directory}`;
}
if (directory.lastIndexOf('/') !== directory.length - 1) {
directory += '/';
}
const finalPath = mkDirByPathSync(config.storagePath + directory, { isRelativeToScript: (config.storagePath.indexOf('/') !== 0) });
const finalPath = mkDirByPathSync(process.env.STORAGE_PATH + directory, { isRelativeToScript: (process.env.STORAGE_PATH.indexOf('/') !== 0) });
if (!finalPath || finalPath.length === 0) {
logger.error(`moveUploadedFile invalid final path, check permissions to create / write '${config.storagePath + directory}'`);
logger.error(`moveUploadedFile invalid final path, check permissions to create / write '${process.env.STORAGE_PATH + directory}'`);
return false;
}
@ -102,7 +101,7 @@ function moveUploadedFile(buffer, directory, filename) {
}
async function updateTotalStorageUsed() {
const verifiedPath = mkDirByPathSync(config.storagePath, { isRelativeToScript: (config.storagePath.indexOf('/') !== 0) });
const verifiedPath = mkDirByPathSync(process.env.STORAGE_PATH, { isRelativeToScript: (process.env.STORAGE_PATH.indexOf('/') !== 0) });
if (!verifiedPath) {
return;
}

View File

@ -1,6 +1,5 @@
import crypto from 'crypto';
import log4js from 'log4js';
import config from '../config';
import orm from '../models/index.model';
const logger = log4js.getLogger('default');
@ -17,9 +16,10 @@ export async function getAccountFromEmail(email) {
return null;
}
export async function _dirtyCreateAccount(email, password, created, banned) {
export async function _dirtyCreateAccount(email, password, created, admin) {
logger.log('creating acount: ', email, password, created, admin);
return orm.models.accounts.create({
email, password, created, banned,
email, password, created, admin,
});
}
@ -27,12 +27,12 @@ export async function createAccount(email, password) {
if (!email || !password) {
return { success: false, status: 400, data: { missingData: true } };
}
if (!config.allowAccountRegistration) {
if (!process.env.ALLOW_REGISTRATION) {
return { success: false, status: 403, data: { registerEnabled: false } };
}
const emailToken = crypto.createHmac('sha256', config.applicationSalt).update(email.trim()).digest('hex');
password = crypto.createHash('sha256').update(password + config.applicationSalt).digest('hex');
const emailToken = crypto.createHmac('sha256', process.env.APP_SALT).update(email.trim()).digest('hex');
password = crypto.createHash('sha256').update(password + process.env.APP_SALT).digest('hex');
const account = await orm.models.accounts.findOne({ where: { email } });
if (account != null && account.dataValues != null) {

View File

@ -0,0 +1,29 @@
server {
listen 80;
server_name uat.api.retropilot.org;
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
location / {
return 301 https://$host$request_uri;
}
}
server {
listen 443 ssl;
server_name uat.api.retropilot.org;
location / {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://server:3000;
}
ssl_certificate /etc/letsencrypt/live/uat.api.retropilot.org/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/uat.api.retropilot.org/privkey.pem;
include /etc/letsencrypt/options-ssl-nginx.conf;
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
}

Binary file not shown.

View File

@ -0,0 +1,18 @@
version: "3.0"
services:
nginx:
image: nginx:1.15-alpine
command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'"
ports:
- "80:80"
- "443:443"
volumes:
- ./data/nginx:/etc/nginx/conf.d
- ./data/certbot/conf:/etc/letsencrypt
- ./data/certbot/www:/var/www/certbot
certbot:
image: certbot/certbot
entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
volumes:
- ./data/certbot/conf:/etc/letsencrypt
- ./data/certbot/www:/var/www/certbot

View File

@ -1,29 +1,23 @@
version: "3.9"
version: "3.0"
services:
server:
# Build image from Dockerfile
build: .
restart: unless-stopped
depends_on:
- db
volumes:
# Mount the config to the container
- ./config.js:/app/config.js
# Mount the database to the container
- ./database.sqlite:/app/database.sqlite
# Mount the data directory to the container
- ./realdata:/app/realdata
- ./:/app
ports:
# HTTP server
- "3000:3000"
# Athena WebSocket
- "4040:4040"
worker:
# Use the same image as the server
build: .
# But run the worker.js script instead
command: node --es-module-specifier-resolution=node worker.js
restart: unless-stopped
db:
image: postgres
restart: always
ports:
- '5438:5432'
volumes:
- ./config.js:/app/config.js
- ./database.sqlite:/app/database.sqlite
- ./realdata:/app/realdata
- ./sql/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql
environment:
POSTGRES_USER: root
POSTGRES_PASSWORD: root
POSTGRES_DB: retro-pilot

View File

@ -1,10 +1,9 @@
export default {
apps: [{
name: 'Retropilot Service',
script: './server.js',
env_development: {
NODE_ENV: 'development',
},
}],
};
module.exports = [{
script: 'server.js',
name: 'server',
node_args: '-r esm',
}, {
script: 'worker.js',
node_args: '-r esm',
name: 'worker'
}]

View File

@ -0,0 +1,80 @@
#!/bin/bash
if ! [ -x "$(command -v docker-compose)" ]; then
echo 'Error: docker-compose is not installed.' >&2
exit 1
fi
domains=(uat.api.retropilot.org)
rsa_key_size=4096
data_path="./data/certbot"
email="" # Adding a valid address is strongly recommended
staging=0 # Set to 1 if you're testing your setup to avoid hitting request limits
if [ -d "$data_path" ]; then
read -p "Existing data found for $domains. Continue and replace existing certificate? (y/N) " decision
if [ "$decision" != "Y" ] && [ "$decision" != "y" ]; then
exit
fi
fi
if [ ! -e "$data_path/conf/options-ssl-nginx.conf" ] || [ ! -e "$data_path/conf/ssl-dhparams.pem" ]; then
echo "### Downloading recommended TLS parameters ..."
mkdir -p "$data_path/conf"
curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > "$data_path/conf/options-ssl-nginx.conf"
curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot/certbot/ssl-dhparams.pem > "$data_path/conf/ssl-dhparams.pem"
echo
fi
echo "### Creating dummy certificate for $domains ..."
path="/etc/letsencrypt/live/$domains"
mkdir -p "$data_path/conf/live/$domains"
docker-compose -f docker-compose.yml -f docker-compose.uat.yml run --rm --entrypoint "\
openssl req -x509 -nodes -newkey rsa:$rsa_key_size -days 1\
-keyout '$path/privkey.pem' \
-out '$path/fullchain.pem' \
-subj '/CN=localhost'" certbot
echo
echo "### Starting nginx ..."
docker-compose -f docker-compose.yml -f docker-compose.uat.yml up --force-recreate -d nginx
echo
echo "### Deleting dummy certificate for $domains ..."
docker-compose -f docker-compose.yml -f docker-compose.uat.yml run --rm --entrypoint "\
rm -Rf /etc/letsencrypt/live/$domains && \
rm -Rf /etc/letsencrypt/archive/$domains && \
rm -Rf /etc/letsencrypt/renewal/$domains.conf" certbot
echo
echo "### Requesting Let's Encrypt certificate for $domains ..."
#Join $domains to -d args
domain_args=""
for domain in "${domains[@]}"; do
domain_args="$domain_args -d $domain"
done
# Select appropriate email arg
case "$email" in
"") email_arg="--register-unsafely-without-email" ;;
*) email_arg="--email $email" ;;
esac
# Enable staging mode if needed
if [ $staging != "0" ]; then staging_arg="--staging"; fi
docker-compose -f docker-compose.yml -f docker-compose.uat.yml run --rm --entrypoint "\
certbot certonly --webroot -w /var/www/certbot \
$staging_arg \
$email_arg \
$domain_args \
--rsa-key-size $rsa_key_size \
--agree-tos \
--force-renewal" certbot
echo
echo "### Reloading nginx ..."
docker-compose -f docker-compose.yml -f docker-compose.uat.yml exec nginx nginx -s reload

View File

@ -14,15 +14,15 @@ export default (sequelize) => {
},
password: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.TEXT,
},
created: {
allowNull: true,
type: DataTypes.NUMBER,
type: DataTypes.BIGINT,
},
last_ping: {
allowNull: true,
type: DataTypes.NUMBER,
type: DataTypes.BIGINT,
},
'2fa_token': {
allowNull: true,
@ -30,7 +30,7 @@ export default (sequelize) => {
},
admin: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
email_verify_token: {
allowNull: true,

View File

@ -34,7 +34,7 @@ export default (sequelize) => {
},
created_at: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BIGINT,
},
dongle_id: {
allowNull: true,

View File

@ -18,19 +18,19 @@ export default (sequelize) => {
},
athena: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
unpair: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
view_drives: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
created_at: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BIGINT,
},
}, {

View File

@ -36,11 +36,11 @@ export default (sequelize) => {
},
created: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BIGINT,
},
last_ping: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BIGINT,
},
storage_used: {
allowNull: true,
@ -53,7 +53,7 @@ export default (sequelize) => {
},
ignore_uploads: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
nickname: {
allowNull: true,

View File

@ -20,15 +20,15 @@ export default (sequelize) => {
},
dongle_id: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.TEXT,
},
duration: {
allowedNull: true,
type: DataTypes.NUMBER,
type: DataTypes.INTEGER,
},
distance_meters: {
allowNull: true,
type: DataTypes.NUMBER,
type: DataTypes.INTEGER,
},
upload_complete: {
allowNull: true,
@ -44,11 +44,11 @@ export default (sequelize) => {
},
created: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BIGINT,
},
process_attempts: {
allowNull: true,
type: DataTypes.BOOLEAN,
type: DataTypes.INTEGER,
},
},
{

View File

@ -24,15 +24,15 @@ export default (sequelize) => {
},
upload_complete: {
allowedNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BOOLEAN,
},
duration: {
allowNull: true,
type: DataTypes.NUMBER,
type: DataTypes.INTEGER,
},
distance_meters: {
allowNull: true,
type: DataTypes.NUMBER,
type: DataTypes.INTEGER,
},
filesize: {
allowNull: true,
@ -44,11 +44,11 @@ export default (sequelize) => {
},
created: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BIGINT,
},
last_upload: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BIGINT,
},
is_preserved: {
allowNull: true,
@ -60,7 +60,7 @@ export default (sequelize) => {
},
drive_date: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.BIGINT,
},
is_physically_removed: {
allowNull: true,

View File

@ -4,23 +4,20 @@
import { Sequelize } from 'sequelize';
import devices from './devices.model';
import drives from './drives.model';
import users from './users.model';
import accounts from './accounts.model';
import athena_action_log from './athena_action_log.model';
import athena_returned_data from './athena_returned_data.model';
import device_authorised_users from './device_authorised_users.model';
import drive_segments from './drive_segments.model';
import oauth_accounts from './oauth_accounts';
import config from '../config';
const sequelize = new Sequelize({
username: 'postgres',
password: config.sqltemp,
database: 'retro-dev',
host: '127.0.0.1',
port: 5432,
username: process.env.DB_USER,
password: process.env.DB_PASS,
database: process.env.DB_NAME || 'retro-pilot',
host: process.env.DB_HOST || '127.0.0.1',
port: process.env.DB_PORT || 5432,
dialect: 'postgres',
});
sequelize.options.logging = () => {};
@ -28,7 +25,7 @@ sequelize.options.logging = () => {};
const modelDefiners = [
devices,
drives,
users,
accounts,
athena_action_log,
athena_returned_data,
device_authorised_users,

View File

@ -18,11 +18,11 @@ export default (sequelize) => {
},
created: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.TIME,
},
last_used: {
allowNull: true,
type: DataTypes.INTEGER,
type: DataTypes.CHAR,
},
refresh: {
allowNull: true,

14
package-lock.json generated
View File

@ -21,6 +21,7 @@
"cors": "^2.8.5",
"crypto": "^1.0.1",
"directory-tree": "^2.2.9",
"dotenv": "^16.0.0",
"esm": "^3.2.25",
"express": "^4.17.1",
"express-fileupload": "^1.2.1",
@ -1549,6 +1550,14 @@
"node": ">=6.0.0"
}
},
"node_modules/dotenv": {
"version": "16.0.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.0.tgz",
"integrity": "sha512-qD9WU0MPM4SWLPJy/r2Be+2WgQj8plChsyrCNQzW/0WjvcJQiKQJ9mH3ZgB3fxbUUxgc/11ZJ0Fi5KiimWGz2Q==",
"engines": {
"node": ">=12"
}
},
"node_modules/dottie": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz",
@ -7824,6 +7833,11 @@
"esutils": "^2.0.2"
}
},
"dotenv": {
"version": "16.0.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.0.tgz",
"integrity": "sha512-qD9WU0MPM4SWLPJy/r2Be+2WgQj8plChsyrCNQzW/0WjvcJQiKQJ9mH3ZgB3fxbUUxgc/11ZJ0Fi5KiimWGz2Q=="
},
"dottie": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz",

View File

@ -3,7 +3,6 @@
"version": "1.0.0",
"description": "replacement for comma.ai backend and useradmin dashboard. can be combined with a modified cabana instance.",
"main": "server.js",
"type": "module",
"scripts": {
"test": "mocha",
"start": "node --es-module-specifier-resolution=node server.js",
@ -26,6 +25,7 @@
"cors": "^2.8.5",
"crypto": "^1.0.1",
"directory-tree": "^2.2.9",
"dotenv": "^16.0.0",
"esm": "^3.2.25",
"express": "^4.17.1",
"express-fileupload": "^1.2.1",
@ -61,4 +61,4 @@
"eslint-plugin-import": "^2.25.4",
"eslint-plugin-no-floating-promise": "^1.0.2"
}
}
}

View File

@ -44,8 +44,6 @@ router.get('/user/:userId/get/devices', runAsyncWrapper(async (req, res) => {
}));
router.get('/user/', runAsyncWrapper(async (req, res) => {
console.warn('PROCESSED');
return res.status(200).json({ success: true, data: await controllers.users.getAllUsers() });
}));
@ -71,7 +69,6 @@ router.get('/device/:dongle_id/pair/:user_id', runAsyncWrapper(async (req, res)
router.get('/device', runAsyncWrapper(async (req, res) => {
const filteredDevices = await controllers.devices.getAllDevicesFiltered();
console.log('fil', filteredDevices);
return res.status(200).json({ success: true, data: filteredDevices });
}));

View File

@ -4,10 +4,7 @@ import bodyParser from 'body-parser';
import crypto from 'crypto';
import log4js from 'log4js';
import storageController from '../controllers/storage';
import helperController from '../controllers/helpers';
import mailingController from '../controllers/mailing';
import deviceController from '../controllers/devices';
import config from '../config';
import authenticationController from './../controllers/authentication';
const logger = log4js.getLogger('default');
@ -45,7 +42,7 @@ router.put('/backend/post_upload', bodyParser.raw({
logger.info(`HTTP.PUT /backend/post_upload BOOT or CRASH upload with filename: ${filename}, token: ${req.query.token}`);
}
const token = crypto.createHmac('sha256', config.applicationSalt).update(dongleId + filename + directory + ts).digest('hex');
const token = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId + filename + directory + ts).digest('hex');
if (token !== req.query.token) {
logger.error(`HTTP.PUT /backend/post_upload token mismatch (${token} vs ${req.query.token})`);
return res.status(400).send('Malformed request');
@ -216,7 +213,7 @@ async function upload(req, res) {
let responseUrl = null;
const ts = Date.now(); // we use this to make sure old URLs cannot be reused (timeout after 60min)
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(dongleId).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId).digest('hex');
// boot log upload
@ -234,9 +231,9 @@ async function upload(req, res) {
// "boot-2021-04-12--01-45-30.bz" for example
const directory = `${dongleId}/${dongleIdHash}/${uploadType}`;
const token = crypto.createHmac('sha256', config.applicationSalt).update(dongleId + filename + directory + ts).digest('hex');
const token = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId + filename + directory + ts).digest('hex');
responseUrl = `${config.baseUploadUrl}?file=${filename}&dir=${directory}&dongleId=${dongleId}&ts=${ts}&token=${token}`;
responseUrl = `${process.env.BASE_UPLOAD_URL}?file=${filename}&dir=${directory}&dongleId=${dongleId}&ts=${ts}&token=${token}`;
logger.info(`HTTP.UPLOAD_URL matched '${uploadType}' file upload, constructed responseUrl: ${responseUrl}`);
} else {
// "2021-04-12--01-44-25--0/qlog.bz2" for example
@ -260,21 +257,20 @@ async function upload(req, res) {
return res.send('Malformed Request.').status(400);
}
const driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(driveName).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT).update(driveName).digest('hex');
directory = `${dongleId}/${dongleIdHash}/${driveIdentifierHash}/${directory}`;
const token = crypto.createHmac('sha256', config.applicationSalt).update(dongleId + filename + directory + ts).digest('hex');
responseUrl = `${config.baseUploadUrl}?file=${filename}&dir=${directory}&dongleId=${dongleId}&ts=${ts}&token=${token}`;
const token = crypto.createHmac('sha256', process.env.APP_SALT).update(dongleId + filename + directory + ts).digest('hex');
responseUrl = `${process.env.BASE_UPLOAD_URL}?file=${filename}&dir=${directory}&dongleId=${dongleId}&ts=${ts}&token=${token}`;
logger.info(`HTTP.UPLOAD_URL matched 'drive' file upload, constructed responseUrl: ${responseUrl}`);
const drive = await deviceController.getDriveFromidentifier(dongleId, driveName).catch((err)=>{
logger.warn("drive failed to make", err)
})
console.log("drive value", drive)
console.log("drive name:", driveName)
logger.log("drive value", drive)
logger.log("drive name:", driveName)
if (drive === undefined || drive === null) {
logger.info("CREATING NEW DRIVE")
@ -345,9 +341,11 @@ router.get('/v1.4/:dongleId/upload_url', upload);
// DEVICE REGISTRATION OR RE-ACTIVATION
router.post('/v2/pilotauth/', bodyParser.urlencoded({ extended: true }), async (req, res) => {
const imei1 = req.query.imei;
const { serial } = req.query;
const { public_key: publicKey } = req.query;
const { register_token: registerToken } = req.query;
const {
serial,
public_key: publicKey,
register_token: registerToken
} = req.query;
if (
serial == null || serial.length < 5
@ -405,9 +403,9 @@ router.get('/useradmin/cabana_drive/:extendedRouteIdentifier', runAsyncWrapper(a
return res.status(200).json({ status: 'drive not found' });
}
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(drive.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(drive.identifier).digest('hex');
const driveUrl = `${config.baseDriveDownloadUrl + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(drive.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT).update(drive.identifier).digest('hex');
const driveUrl = `${process.env.BASE_DRIVE_DOWNLOAD_URL + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
if (dongleIdHash !== dongleIdHashReq || driveIdentifierHash !== driveIdentifierHashReq) {
return res.status(200).json({ status: 'hashes not matching' });

View File

@ -2,11 +2,6 @@ import bodyParser from 'body-parser';
import express from 'express';
import authenticationController from '../../controllers/authentication';
/* eslint-disable no-unused-vars */
import userController from '../../controllers/users';
import config from '../../config';
/* eslint-enable no-unused-vars */
const router = express.Router();
async function isAuthenticated(req, res, next) {
const account = await authenticationController.getAuthenticatedAccount(req);

View File

@ -1,9 +1,10 @@
import express from 'express';
import jsonwebtoken from 'jsonwebtoken';
import { getURL, getToken } from '../../../controllers/authentication/oauth/google';
import authenticationController from '../../../controllers/authentication';
import log4js from 'log4js';
const router = express.Router();
const logger = log4js.getLogger('default');
async function isAuthenticated(req, res, next) {
const account = await authenticationController.getAuthenticatedAccount(req);
@ -22,14 +23,13 @@ async function isAuthenticated(req, res, next) {
}
router.get('/authentication/oauth/callback', async (req, res) => {
console.log(req.query);
logger.log(req.query);
res.json(await getToken(req.query.code, req.query.scope));
});
router.get('/authentication/oauth/:provider', async (req, res) => {
const { provider } = req.params;
console.log('provider', provider);
logger.log('provider', provider);
let url;
switch (provider) {
case 'google':
@ -48,9 +48,7 @@ router.get('/authentication/oauth/:provider', async (req, res) => {
});
router.get('/authentication/oauth/pair/:provider', isAuthenticated, async (req, res) => {
res.status(200);
});
export default router;

View File

@ -2,15 +2,14 @@ import express from 'express';
import crypto from 'crypto';
import dirTree from 'directory-tree';
import bodyParser from 'body-parser';
import deviceSchema from '../../schema/routes/devices.mjs';
import config from '../../config';
/* eslint-disable no-unused-vars */
import userController from '../../controllers/users';
import deviceSchema from '../../schema/routes/devices';
import log4js from 'log4js';
import deviceController from '../../controllers/devices';
import authenticationController from '../../controllers/authentication';
/* eslint-enable no-unused-vars */
const logger = log4js.getLogger('default');
const router = express.Router();
async function isAuthenticated(req, res, next) {
const account = await authenticationController.getAuthenticatedAccount(req);
@ -55,7 +54,7 @@ router.put('/retropilot/0/device/:dongle_id/', [isAuthenticated, bodyParser.json
}
const { body } = req;
console.log(deviceSchema.MutateDevice.isValid(body));
logger.log(deviceSchema.MutateDevice.isValid(body));
});
router.get('/retropilot/0/device/:dongle_id/drives/:drive_identifier/segment', isAuthenticated, async (req, res) => {
@ -70,10 +69,10 @@ router.get('/retropilot/0/device/:dongle_id/drives/:drive_identifier/segment', i
if (isUserAuthorised.success === false || isUserAuthorised.data.authorised === false) {
return res.json({ success: false, msg: isUserAuthorised.msg });
}
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(req.params.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(req.params.drive_identifier).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(req.params.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT).update(req.params.drive_identifier).digest('hex');
const directoryTree = dirTree(`${config.storagePath + req.params.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${req.params.drive_identifier}`);
const directoryTree = dirTree(`${process.env.STORAGE_PATH + req.params.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${req.params.drive_identifier}`);
return res.json({ success: true, msg: 'ok', data: directoryTree });
});

View File

@ -2,7 +2,6 @@
import express from 'express';
import bodyParser from 'body-parser';
import cookieParser from 'cookie-parser';
import config from '../config';
import controllers from '../controllers';
import deviceController from '../controllers/devices';
@ -47,8 +46,8 @@ router.get('/retropilot/0/useradmin', runAsyncWrapper(async (req, res) => {
data: {
serverStats: {
config: {
registerAllowed: config.allowAccountRegistration,
welcomeMessage: config.welcomeMessage,
registerAllowed: process.env.ALLOW_REGISTRATION,
welcomeMessage: process.env.WELCOME_MESSAGE,
},
accounts: accounts.num,
devices: devices.num,
@ -67,7 +66,7 @@ router.get('/retropilot/0/useradmin', runAsyncWrapper(async (req, res) => {
router.post('/useradmin/register/token', bodyParser.urlencoded({extended: true}), runAsyncWrapper(async (req, res) => {
const email = req.body.email;
if (!config.allowAccountRegistration) {
if (!process.env.ALLOW_REGISTRATION) {
res.send('Unauthorized.').status(401);
return;
}
@ -84,7 +83,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({extended: true})
return;
}
const token = crypto.createHmac('sha256', config.applicationSalt).update(email.trim()).digest('hex');
const token = crypto.createHmac('sha256', process.env.APP_SALT).update(email.trim()).digest('hex');
let infoText = '';
@ -103,7 +102,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({extended: true})
const result = await models.__db.run(
'INSERT INTO accounts (email, password, created, banned) VALUES (?, ?, ?, ?)',
email,
crypto.createHash('sha256').update(req.body.password + config.applicationSalt).digest('hex'),
crypto.createHash('sha256').update(req.body.password + process.env.APP_SALT).digest('hex'),
Date.now(), false);
if (result.lastID != undefined) {
@ -138,7 +137,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({extended: true})
}))
router.get('/useradmin/register', runAsyncWrapper(async (req, res) => {
if (!config.allowAccountRegistration) {
if (!process.env.ALLOW_REGISTRATION) {
res.status(400);
res.send('Unauthorized.');
return;
@ -262,9 +261,9 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
const drives = await models.__db.all('SELECT * FROM drives WHERE dongle_id = ? AND is_deleted = ? ORDER BY created DESC', device.dongle_id, false);
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
var dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(device.dongle_id).digest('hex');
const bootlogDirectoryTree = dirTree(config.storagePath + device.dongle_id + "/" + dongleIdHash + "/boot/", {attributes: ['size']});
const bootlogDirectoryTree = dirTree(process.env.STORAGE_PATH + device.dongle_id + "/" + dongleIdHash + "/boot/", {attributes: ['size']});
var bootlogFiles = [];
if (bootlogDirectoryTree != undefined) {
for (var i = 0; i < bootlogDirectoryTree.children.length; i++) {
@ -280,7 +279,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
bootlogFiles.sort((a, b) => (a.date < b.date) ? 1 : -1);
}
const crashlogDirectoryTree = dirTree(config.storagePath + device.dongle_id + "/" + dongleIdHash + "/crash/", {attributes: ['size']});
const crashlogDirectoryTree = dirTree(process.env.STORAGE_PATH + device.dongle_id + "/" + dongleIdHash + "/crash/", {attributes: ['size']});
var crashlogFiles = [];
if (crashlogDirectoryTree != undefined) {
for (var i = 0; i < crashlogDirectoryTree.children.length; i++) {
@ -309,7 +308,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
<b>Public Key:</b><br><span style="font-size: 0.8em">` + device.public_key.replace(/\r?\n|\r/g, "<br>") + `</span>
<br>
<b>Stored Drives:</b> ` + drives.length + `<br>
<b>Quota Storage:</b> ` + device.storage_used + ` MB / ` + config.deviceStorageQuotaMb + ` MB<br>
<b>Quota Storage:</b> ` + device.storage_used + ` MB / ` + process.env.DEVICE_STORAGE_QUOTA_MB + ` MB<br>
<br>
`;
@ -318,7 +317,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
<tr><th>date</th><th>file</th><th>size</th></tr>
`;
for (var i = 0; i < Math.min(5, bootlogFiles.length); i++) {
response += `<tr><td>` + controllers.helpers.formatDate(bootlogFiles[i].date) + `</td><td><a href="` + config.baseDriveDownloadUrl + device.dongle_id + "/" + dongleIdHash + "/boot/" + bootlogFiles[i].name + `" target=_blank>` + bootlogFiles[i].name + `</a></td><td>` + bootlogFiles[i].size + `</td></tr>`;
response += `<tr><td>` + controllers.helpers.formatDate(bootlogFiles[i].date) + `</td><td><a href="` + process.env.BASE_DRIVE_DOWNLOAD_URL + device.dongle_id + "/" + dongleIdHash + "/boot/" + bootlogFiles[i].name + `" target=_blank>` + bootlogFiles[i].name + `</a></td><td>` + bootlogFiles[i].size + `</td></tr>`;
}
response += `</table><br><br>`;
@ -327,11 +326,11 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
<tr><th>date</th><th>file</th><th>size</th></tr>
`;
for (var i = 0; i < Math.min(5, crashlogFiles.length); i++) {
response += `<tr><td>` + controllers.helpers.formatDate(crashlogFiles[i].date) + `</td><td><a href="` + config.baseDriveDownloadUrl + device.dongle_id + "/" + dongleIdHash + "/crash/" + crashlogFiles[i].name + `" target=_blank>` + crashlogFiles[i].name + `</a></td><td>` + crashlogFiles[i].size + `</td></tr>`;
response += `<tr><td>` + controllers.helpers.formatDate(crashlogFiles[i].date) + `</td><td><a href="` + process.env.BASE_DRIVE_DOWNLOAD_URL + device.dongle_id + "/" + dongleIdHash + "/crash/" + crashlogFiles[i].name + `" target=_blank>` + crashlogFiles[i].name + `</a></td><td>` + crashlogFiles[i].size + `</td></tr>`;
}
response += `</table><br><br>`;
response += `<b>Drives (non-preserved drives expire ` + config.deviceDriveExpirationDays + ` days after upload):</b><br>
response += `<b>Drives (non-preserved drives expire ` + process.env.DEVICE_EXPIRATION_DAYS + ` days after upload):</b><br>
<table border=1 cellpadding=2 cellspacing=2>
<tr><th>identifier</th><th>filesize</th><th>duration</th><th>distance_meters</th><th>upload_complete</th><th>is_processed</th><th>upload_date</th><th>actions</th></tr>
`;
@ -415,17 +414,17 @@ router.get('/useradmin/drive/:dongleId/:driveIdentifier', runAsyncWrapper(async
return;
}
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(drive.identifier).digest('hex');
var dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(device.dongle_id).digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT).update(drive.identifier).digest('hex');
var driveUrl = config.baseDriveDownloadUrl + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier + "/";
var driveUrl = process.env.BASE_DRIVE_DOWNLOAD_URL + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier + "/";
var cabanaUrl = null;
if (drive.is_processed) {
cabanaUrl = config.cabanaUrl + '?retropilotIdentifier=' + device.dongle_id + '|' + dongleIdHash + '|' + drive.identifier + '|' + driveIdentifierHash + '&retropilotHost=' + encodeURIComponent(config.baseUrl) + '&demo=1"';
cabanaUrl = process.env.CABANA_URL + '?retropilotIdentifier=' + device.dongle_id + '|' + dongleIdHash + '|' + drive.identifier + '|' + driveIdentifierHash + '&retropilotHost=' + encodeURIComponent(process.env.BASE_URL) + '&demo=1"';
}
const directoryTree = dirTree(config.storagePath + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier);
const directoryTree = dirTree(process.env.STORAGE_PATH + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier);
var response = '<html style="font-family: monospace"><h2>Welcome To The RetroPilot Server Dashboard!</h2>' +
`

View File

@ -5,9 +5,7 @@ import htmlspecialchars from 'htmlspecialchars';
import dirTree from 'directory-tree';
import cookieParser from 'cookie-parser';
import log4js from 'log4js';
import config from '../config';
import authenticationController from '../controllers/authentication';
import storageController from '../controllers/storage';
import helperController from '../controllers/helpers';
import mailingController from '../controllers/mailing';
import deviceController from '../controllers/devices';
@ -29,7 +27,7 @@ function runAsyncWrapper(callback) {
router.post('/useradmin/auth', bodyParser.urlencoded({ extended: true }), runAsyncWrapper(async (req, res) => {
const signIn = await authenticationController.signIn(req.body.email, req.body.password);
console.log(signIn);
logger.log(signIn);
if (signIn.success) {
res.cookie('jwt', signIn.jwt);
@ -71,9 +69,9 @@ router.get('/useradmin', runAsyncWrapper(async (req, res) => {
<input type="submit">
</form>
<br><br>
${!config.allowAccountRegistration ? '<i>User Account Registration is disabled on this Server</i>' : '<a href="/useradmin/register">Register new Account</a>'}
${!process.env.ALLOW_REGISTRATION ? '<i>User Account Registration is disabled on this Server</i>' : '<a href="/useradmin/register">Register new Account</a>'}
<br><br>
<br><br>${config.welcomeMessage}
<br><br>${process.env.WELCOME_MESSAGE}
</html>
`, /*
Accounts: ${accounts.num} |
@ -82,7 +80,7 @@ router.get('/useradmin', runAsyncWrapper(async (req, res) => {
Distance Traveled: ${Math.round(drives.distance / 1000)} km |
Time Traveled: ${helperController.formatDuration(drives.duration)} |
Storage Used: ${await storageController.getTotalStorageUsed() !== null ? await storageController.getTotalStorageUsed() : '--'}
<br><br>${config.welcomeMessage}` */);
<br><br>${process.env.WELCOME_MESSAGE}` */);
}));
router.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true }), runAsyncWrapper(async (req, res) => {
@ -92,7 +90,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true
return res.status(400).send('Malformed Request');
}
if (!config.allowAccountRegistration) {
if (!process.env.ALLOW_REGISTRATION) {
return res.status(401).send('Unauthorized.');
}
@ -106,32 +104,39 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true
return res.redirect(`/useradmin/register?status=${encodeURIComponent('Email is already registered')}`);
}
const token = crypto.createHmac('sha256', config.applicationSalt).update(email.trim()).digest('hex');
const token = (process.env.NODE_ENV === 'development') ? 'verysecrettoken' : crypto.createHmac('sha256', process.env.APP_SALT).update(email.trim()).digest('hex');
let infoText = '';
if (req.body.token === undefined) { // email entered, token request
infoText = 'Please check your inbox (<b>SPAM</b>) for an email with the registration token.<br>If the token was not delivered, please ask the administrator to check the <i>server.log</i> for the token generated for your email.<br><br>';
const emailStatus = await mailingController.sendEmailVerification(token, email);
await mailingController.sendEmailVerification(token, email);
} else if (req.body.token !== token) {
infoText = 'The registration token you entered was incorrect, please try again.<br><br>';
} else if (req.body.password !== req.body.password2 || req.body.password.length < 3) {
infoText = 'The passwords you entered did not match or were shorter than 3 characters, please try again.<br><br>';
} else {
const result = await userController._dirtyCreateAccount(
email,
crypto.createHash('sha256').update(req.body.password + config.applicationSalt).digest('hex'),
Date.now(),
false,
);
let result = false;
console.log(result);
try {
result = await userController._dirtyCreateAccount(
email,
crypto.createHash('sha256').update(req.body.password + process.env.APP_SALT).digest('hex'),
Date.now(),
false,
);
} catch(error) {
console.error(error);
}
logger.log(result);
if (result.dataValues) {
logger.info(`USERADMIN REGISTRATION - created new account #${result.lastID} with email ${email}`);
return res.redirect(`/useradmin?status=${encodeURIComponent('Successfully registered')}`);
}
logger.error(`USERADMIN REGISTRATION - account creation failed, resulting account data for email ${email} is: ${result}`);
infoText = 'Unable to complete account registration (database error).<br><br>';
}
@ -153,7 +158,7 @@ router.post('/useradmin/register/token', bodyParser.urlencoded({ extended: true
}));
router.get('/useradmin/register', runAsyncWrapper(async (req, res) => {
if (!config.allowAccountRegistration) {
if (!process.env.ALLOW_REGISTRATION) {
return res.status(400).send('Unauthorized.');
}
@ -222,7 +227,7 @@ ${req.query.linkstatus !== undefined ? `<br><u>${htmlspecialchars(req.query.link
<hr/>
<a href="/useradmin/signout">Sign Out</a>`;
response += `<br>${config.welcomeMessage}</html>`;
response += `<br>${process.env.WELCOME_MESSAGE}</html>`;
return res.status(200).send(response);
}));
@ -274,7 +279,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
const drives = await deviceController.getDrives(device.dongle_id, false, true);
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(device.dongle_id).digest('hex');
const bootlogFiles = await deviceController.getBootlogs(device.dongle_id);
const crashlogFiles = await deviceController.getCrashlogs(device.dongle_id);
@ -292,7 +297,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
<b>Public Key:</b><br>
<span style="font-size: 0.8em">${device.public_key.replace(/\r?\n|\r/g, '<br>')}</span><br>
<b>Stored Drives:</b> ${drives.length}<br>
<b>Quota Storage:</b> ${device.storage_used} MB / ${config.deviceStorageQuotaMb} MB<br>
<b>Quota Storage:</b> ${device.storage_used} MB / ${process.env.DEVICE_STORAGE_QUOTA_MB} MB<br>
<br>`;
response += `<b>Boot Logs (last 5):</b>
@ -301,7 +306,7 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
<tr><th>date</th><th>file</th><th>size</th></tr>
`;
for (let i = 0; i < Math.min(5, bootlogFiles.length); i++) {
response += `<tr><td>${helperController.formatDate(bootlogFiles[i].date)}</td><td><a href="${config.baseDriveDownloadUrl}${device.dongle_id}/${dongleIdHash}/boot/${bootlogFiles[i].name}" target=_blank>${bootlogFiles[i].name}</a></td><td>${bootlogFiles[i].size}</td></tr>`;
response += `<tr><td>${helperController.formatDate(bootlogFiles[i].date)}</td><td><a href="${process.env.BASE_DRIVE_DOWNLOAD_URL}${device.dongle_id}/${dongleIdHash}/boot/${bootlogFiles[i].name}" target=_blank>${bootlogFiles[i].name}</a></td><td>${bootlogFiles[i].size}</td></tr>`;
}
response += '</table><br><br>';
@ -311,13 +316,13 @@ router.get('/useradmin/device/:dongleId', runAsyncWrapper(async (req, res) => {
for (let i = 0; i < Math.min(5, crashlogFiles.length); i++) {
response += `<tr>
<td>${helperController.formatDate(crashlogFiles[i].date)}</td>.
<td><a href="${config.baseDriveDownloadUrl}${device.dongle_id}/${dongleIdHash}/crash/${crashlogFiles[i].name}" target=_blank>${crashlogFiles[i].name}</a></td>
<td><a href="${process.env.BASE_DRIVE_DOWNLOAD_URL}${device.dongle_id}/${dongleIdHash}/crash/${crashlogFiles[i].name}" target=_blank>${crashlogFiles[i].name}</a></td>
<td>${crashlogFiles[i].size}</td>
</tr>`;
}
response += '</table><br><br>';
response += `<b>Drives (non-preserved drives expire ${config.deviceDriveExpirationDays} days after upload):</b><br>
response += `<b>Drives (non-preserved drives expire ${process.env.DEVICE_EXPIRATION_DAYS} days after upload):</b><br>
<table border=1 cellpadding=2 cellspacing=2>
<tr>
<th>identifier</th>
@ -423,14 +428,14 @@ router.get('/useradmin/drive/:dongleId/:driveIdentifier', runAsyncWrapper(async
return res.status(400).send('Unauthorized.');
}
const dongleIdHash = crypto.createHmac('sha256', config.applicationSalt).update(device.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt).update(drive.identifier).digest('hex');
const dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT).update(device.dongle_id).digest('hex');
const driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT).update(drive.identifier).digest('hex');
const driveUrl = `${config.baseDriveDownloadUrl + device.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}/`;
const driveUrl = `${process.env.BASE_DRIVE_DOWNLOAD_URL + device.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}/`;
let cabanaUrl = null;
if (drive.is_processed) {
cabanaUrl = `${config.cabanaUrl}?retropilotIdentifier=${device.dongle_id}|${dongleIdHash}|${drive.identifier}|${driveIdentifierHash}&retropilotHost=${encodeURIComponent(config.baseUrl)}&demo=1"`;
cabanaUrl = `${process.env.CABANA_URL}?retropilotIdentifier=${device.dongle_id}|${dongleIdHash}|${drive.identifier}|${driveIdentifierHash}&retropilotHost=${encodeURIComponent(process.env.BASE_URL)}&demo=1"`;
}
let vehicle = '';
@ -573,7 +578,7 @@ router.get('/useradmin/drive/:dongleId/:driveIdentifier', runAsyncWrapper(async
<table border=1 cellpadding=2 cellspacing=2>
<tr><th>segment</th><th>qcamera</th><th>qlog</th><th>fcamera</th><th>rlog</th><th>dcamera</th><th>processed</th><th>stalled</th></tr>`;
const directoryTree = dirTree(config.storagePath + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier);
const directoryTree = dirTree(process.env.STORAGE_PATH + device.dongle_id + "/" + dongleIdHash + "/" + driveIdentifierHash + "/" + drive.identifier);
const directorySegments = {};
for (var i in directoryTree.children) {
// skip any non-directory entries (for example m3u8 file in the drive directory)

View File

@ -1,10 +1,9 @@
/* eslint-disable global-require */
import fs from 'fs';
import 'dotenv/config'
import log4js from 'log4js';
import lockfile from 'proper-lockfile';
import http from 'http';
import https from 'https';
import express from 'express';
import cors from 'cors';
import rateLimit from 'express-rate-limit';
@ -12,17 +11,13 @@ import cookieParser from 'cookie-parser';
import storageController from './controllers/storage.js';
/* eslint-disable no-unused-vars */
import webWebsocket from './websocket/web/index.js';
import athena from './websocket/athena/index.js';
import routers from './routes/index.js';
import orm from './models/index.model.js';
import controllers from './controllers/index.js';
import router from './routes/api/realtime.js';
/* eslint-enable no-unused-vars */
import config from './config.js';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
@ -31,6 +26,7 @@ process.on('unhandledRejection', (error, p) => {
console.log(error.promise, p);
console.dir(error.stack);
});
log4js.configure({
appenders: { logfile: { type: 'file', filename: 'server.log' }, out: { type: 'console' } /* {type: "file", filename: "server1.log"} */ },
categories: { default: { appenders: ['out', 'logfile'], level: 'info' } },
@ -55,8 +51,7 @@ const web = async () => {
const app = express();
app.use((req, res, next) => {
// TODO: can we use config.baseUrl here?
res.header('Access-Control-Allow-Origin', 'http://localhost:3000');
res.header('Access-Control-Allow-Origin', `${process.env.BASE_URL}`);
res.header('Access-Control-Allow-Credentials', true);
res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept');
next();
@ -69,10 +64,10 @@ const web = async () => {
app.use(routers.useradmin);
app.use(routers.authenticationApi);
if (config.athena.enabled) {
if (process.env.ATHENA_ENABLED) {
const athenaRateLimit = rateLimit({
windowMs: 30000,
max: config.athena.api.ratelimit,
max: process.env.ATHENA_API_RATE_LIMIT,
});
app.use((req, res, next) => {
@ -91,7 +86,7 @@ const web = async () => {
app.use(cors({ origin: 'http://localhost:3000' }));
app.use(cookieParser());
app.use('/favicon.ico', express.static('static/favicon.ico'));
app.use(config.baseDriveDownloadPathMapping, express.static(config.storagePath));
app.use(process.env.BASE_DRIVE_DOWNLOAD_PATH_MAPPING, express.static(process.env.STORAGE_PATH));
app.use(routers.deviceApi);
@ -122,13 +117,10 @@ lockfile.lock('retropilot_server', { realpath: false, stale: 30000, update: 2000
.then(async () => {
console.log('STARTING SERVER...');
const app = await web();
const httpServer = http.createServer(app);
httpServer.listen(config.httpPort, config.httpInterface, () => {
logger.info(`Retropilot Server listening at http://${config.httpInterface}:${config.httpPort}`);
httpServer.listen(process.env.HTTP_PORT, () => {
logger.info(`Retropilot Server listening at http://${process.env.BASE_URL}`);
});
}).catch((e) => {

View File

@ -0,0 +1,800 @@
--
-- PostgreSQL database dump
--
-- Dumped from database version 14.1
-- Dumped by pg_dump version 14.1
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET xmloption = content;
SET client_min_messages = warning;
SET row_security = off;
--
-- Name: accounts; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.accounts (
id integer NOT NULL,
email character varying(64) NOT NULL,
password character varying(246),
created bigint,
last_ping bigint,
banned boolean DEFAULT false,
"2fa_token" character varying(200),
admin boolean DEFAULT false,
session_seed character varying(256),
email_verify_token character varying,
g_oauth_sub character varying,
two_factor_enabled boolean DEFAULT false NOT NULL
);
--
-- Name: accounts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.accounts_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: accounts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.accounts_id_seq OWNED BY public.accounts.id;
--
-- Name: athena_action_logs; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.athena_action_logs (
id integer NOT NULL,
account_id integer,
device_id integer,
action character varying(38),
user_ip character varying(1),
device_ip character varying(19),
meta json,
created_at bigint,
dongle_id character varying(8)
);
--
-- Name: athena_action_logs_account_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.athena_action_logs_account_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: athena_action_logs_account_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.athena_action_logs_account_id_seq OWNED BY public.athena_action_logs.account_id;
--
-- Name: athena_action_logs_device_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.athena_action_logs_device_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: athena_action_logs_device_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.athena_action_logs_device_id_seq OWNED BY public.athena_action_logs.device_id;
--
-- Name: athena_action_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.athena_action_logs_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: athena_action_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.athena_action_logs_id_seq OWNED BY public.athena_action_logs.id;
--
-- Name: athena_returned_data; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.athena_returned_data (
id integer NOT NULL,
device_id integer NOT NULL,
type character varying(12),
data json,
created_at bigint,
uuid character varying(12),
resolved_at bigint
);
--
-- Name: athena_returned_data_device_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.athena_returned_data_device_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: athena_returned_data_device_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.athena_returned_data_device_id_seq OWNED BY public.athena_returned_data.device_id;
--
-- Name: athena_returned_data_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.athena_returned_data_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: athena_returned_data_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.athena_returned_data_id_seq OWNED BY public.athena_returned_data.id;
--
-- Name: device_authorised_users; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.device_authorised_users (
id integer NOT NULL,
account_id integer NOT NULL,
device_id integer NOT NULL,
athena boolean DEFAULT false NOT NULL,
unpair boolean DEFAULT false NOT NULL,
view_drives boolean DEFAULT false NOT NULL,
created_at bigint NOT NULL
);
--
-- Name: device_authorised_users_account_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.device_authorised_users_account_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: device_authorised_users_account_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.device_authorised_users_account_id_seq OWNED BY public.device_authorised_users.account_id;
--
-- Name: device_authorised_users_device_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.device_authorised_users_device_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: device_authorised_users_device_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.device_authorised_users_device_id_seq OWNED BY public.device_authorised_users.device_id;
--
-- Name: device_authorised_users_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.device_authorised_users_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: device_authorised_users_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.device_authorised_users_id_seq OWNED BY public.device_authorised_users.id;
--
-- Name: devices; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.devices (
id integer NOT NULL,
dongle_id text NOT NULL,
account_id integer,
imei text,
serial text,
device_type text,
public_key text,
created bigint,
last_ping bigint,
storage_used bigint,
max_storage bigint,
ignore_uploads boolean,
nickname character varying(20)
);
--
-- Name: devices_account_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.devices_account_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: devices_account_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.devices_account_id_seq OWNED BY public.devices.account_id;
--
-- Name: devices_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.devices_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: devices_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.devices_id_seq OWNED BY public.devices.id;
--
-- Name: drive_segments; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.drive_segments (
id integer NOT NULL,
segment_id bigint NOT NULL,
drive_identifier character varying(20) NOT NULL,
dongle_id character varying(12) NOT NULL,
duration bigint,
distance_meters bigint,
upload_complete boolean DEFAULT false NOT NULL,
is_processed boolean DEFAULT false NOT NULL,
is_stalled boolean DEFAULT false NOT NULL,
created bigint DEFAULT 0 NOT NULL,
process_attempts smallint DEFAULT 0 NOT NULL
);
--
-- Name: drive_segments_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.drive_segments_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: drive_segments_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.drive_segments_id_seq OWNED BY public.drive_segments.id;
--
-- Name: drives; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.drives (
id integer NOT NULL,
identifier character varying(20),
dongle_id character varying(8),
max_segment bigint,
duration double precision,
distance_meters double precision,
filesize bigint,
upload_complete boolean DEFAULT false NOT NULL,
is_processed boolean DEFAULT false NOT NULL,
created bigint,
last_upload bigint,
is_preserved boolean DEFAULT false NOT NULL,
is_deleted boolean DEFAULT false NOT NULL,
drive_date bigint DEFAULT 0,
is_physically_removed boolean DEFAULT false NOT NULL,
metadata text
);
--
-- Name: drives_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.drives_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: drives_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.drives_id_seq OWNED BY public.drives.id;
--
-- Name: oauth_accounts; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.oauth_accounts (
id integer NOT NULL,
account_id integer NOT NULL,
email character varying NOT NULL,
created time without time zone,
last_used character varying,
refresh character varying,
provider character varying,
external_id character varying,
enabled boolean DEFAULT false NOT NULL
);
--
-- Name: oauth_accounts_account_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.oauth_accounts_account_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: oauth_accounts_account_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.oauth_accounts_account_id_seq OWNED BY public.oauth_accounts.account_id;
--
-- Name: oauth_accounts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.oauth_accounts_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: oauth_accounts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.oauth_accounts_id_seq OWNED BY public.oauth_accounts.id;
--
-- Name: sessions; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.sessions (
id integer NOT NULL,
sessionkey character varying,
account_id bigint,
ip_address character varying,
expires bigint
);
--
-- Name: sessions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.sessions_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: sessions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.sessions_id_seq OWNED BY public.sessions.id;
--
-- Name: accounts id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.accounts ALTER COLUMN id SET DEFAULT nextval('public.accounts_id_seq'::regclass);
--
-- Name: athena_action_logs id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_action_logs ALTER COLUMN id SET DEFAULT nextval('public.athena_action_logs_id_seq'::regclass);
--
-- Name: athena_action_logs account_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_action_logs ALTER COLUMN account_id SET DEFAULT nextval('public.athena_action_logs_account_id_seq'::regclass);
--
-- Name: athena_action_logs device_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_action_logs ALTER COLUMN device_id SET DEFAULT nextval('public.athena_action_logs_device_id_seq'::regclass);
--
-- Name: athena_returned_data id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_returned_data ALTER COLUMN id SET DEFAULT nextval('public.athena_returned_data_id_seq'::regclass);
--
-- Name: athena_returned_data device_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_returned_data ALTER COLUMN device_id SET DEFAULT nextval('public.athena_returned_data_device_id_seq'::regclass);
--
-- Name: device_authorised_users id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users ALTER COLUMN id SET DEFAULT nextval('public.device_authorised_users_id_seq'::regclass);
--
-- Name: device_authorised_users account_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users ALTER COLUMN account_id SET DEFAULT nextval('public.device_authorised_users_account_id_seq'::regclass);
--
-- Name: device_authorised_users device_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users ALTER COLUMN device_id SET DEFAULT nextval('public.device_authorised_users_device_id_seq'::regclass);
--
-- Name: devices id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.devices ALTER COLUMN id SET DEFAULT nextval('public.devices_id_seq'::regclass);
--
-- Name: drive_segments id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drive_segments ALTER COLUMN id SET DEFAULT nextval('public.drive_segments_id_seq'::regclass);
--
-- Name: drives id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drives ALTER COLUMN id SET DEFAULT nextval('public.drives_id_seq'::regclass);
--
-- Name: oauth_accounts id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.oauth_accounts ALTER COLUMN id SET DEFAULT nextval('public.oauth_accounts_id_seq'::regclass);
--
-- Name: oauth_accounts account_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.oauth_accounts ALTER COLUMN account_id SET DEFAULT nextval('public.oauth_accounts_account_id_seq'::regclass);
--
-- Name: sessions id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.sessions ALTER COLUMN id SET DEFAULT nextval('public.sessions_id_seq'::regclass);
--
-- Name: accounts accounts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.accounts
ADD CONSTRAINT accounts_pkey PRIMARY KEY (id);
--
-- Name: accounts accounts_un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.accounts
ADD CONSTRAINT accounts_un UNIQUE (email);
--
-- Name: athena_action_logs athena_action_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_action_logs
ADD CONSTRAINT athena_action_logs_pkey PRIMARY KEY (id);
--
-- Name: athena_action_logs athena_action_logs_un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_action_logs
ADD CONSTRAINT athena_action_logs_un UNIQUE (id);
--
-- Name: athena_returned_data athena_returned_data_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_returned_data
ADD CONSTRAINT athena_returned_data_pkey PRIMARY KEY (id);
--
-- Name: device_authorised_users device_authorised_users_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users
ADD CONSTRAINT device_authorised_users_pkey PRIMARY KEY (id);
--
-- Name: device_authorised_users device_authorised_users_un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users
ADD CONSTRAINT device_authorised_users_un UNIQUE (id);
--
-- Name: devices devices_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.devices
ADD CONSTRAINT devices_pkey PRIMARY KEY (id);
--
-- Name: devices devices_un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.devices
ADD CONSTRAINT devices_un UNIQUE (id);
--
-- Name: devices devices_unique_dongle; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.devices
ADD CONSTRAINT devices_unique_dongle UNIQUE (dongle_id);
--
-- Name: drive_segments drive_segment_uk; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drive_segments
ADD CONSTRAINT drive_segment_uk UNIQUE (id);
--
-- Name: drive_segments drive_segments_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drive_segments
ADD CONSTRAINT drive_segments_pkey PRIMARY KEY (id);
--
-- Name: drives drives_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drives
ADD CONSTRAINT drives_pkey PRIMARY KEY (id);
--
-- Name: drives drives_un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drives
ADD CONSTRAINT drives_un UNIQUE (id);
--
-- Name: oauth_accounts oauth_accounts_pk; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.oauth_accounts
ADD CONSTRAINT oauth_accounts_pk PRIMARY KEY (id);
--
-- Name: accounts primary_uni; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.accounts
ADD CONSTRAINT primary_uni UNIQUE (id);
--
-- Name: sessions sessions_pk; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.sessions
ADD CONSTRAINT sessions_pk PRIMARY KEY (id);
--
-- Name: athena_returned_data un; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_returned_data
ADD CONSTRAINT un UNIQUE (id);
--
-- Name: device_authorised_users device_authorised_users_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users
ADD CONSTRAINT device_authorised_users_fk FOREIGN KEY (account_id) REFERENCES public.accounts(id);
--
-- Name: device_authorised_users device_authorised_users_fk_1; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.device_authorised_users
ADD CONSTRAINT device_authorised_users_fk_1 FOREIGN KEY (device_id) REFERENCES public.devices(id);
--
-- Name: athena_returned_data device_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.athena_returned_data
ADD CONSTRAINT device_id_fk FOREIGN KEY (device_id) REFERENCES public.devices(id);
--
-- Name: devices devices_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.devices
ADD CONSTRAINT devices_fk FOREIGN KEY (account_id) REFERENCES public.accounts(id);
--
-- Name: drive_segments drive_segments_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drive_segments
ADD CONSTRAINT drive_segments_fk FOREIGN KEY (dongle_id) REFERENCES public.devices(dongle_id);
--
-- Name: drives drives_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.drives
ADD CONSTRAINT drives_fk FOREIGN KEY (dongle_id) REFERENCES public.devices(dongle_id);
--
-- PostgreSQL database dump complete
--

View File

@ -23,7 +23,7 @@ export default (server) => {
body.data.hasOwnProperty('serverStats') &&
body.data.serverStats.hasOwnProperty('config') &&
typeof body.data.serverStats.config.registerAllowed === "boolean" &&
typeof body.data.serverStats.config.welcomeMessage === "string" &&
typeof body.data.serverStats.process.env.WELCOME_MESSAGE === "string" &&
typeof body.data.serverStats['accounts'] === "number" &&
typeof body.data.serverStats['devices'] === "number" &&
typeof body.data.serverStats['drives'] === "number" &&

View File

@ -1,6 +1,5 @@
import request from 'supertest';
import server from '../server';
import config from '../config';
// TODO better way to only run tests once server is up
describe('loading express', () => {
@ -18,4 +17,4 @@ describe('loading express', () => {
require('./routes/api.test')(server);
require('./routes/useradmin.test')(server);
if (config.flags.useUserAdminApi) require('./routes/userAdminApi.test')(server);
if (process.env.USE_USER_ADMIN_API) require('./routes/userAdminApi.test')(server);

View File

@ -1,16 +1,12 @@
import WebSocket, { WebSocketServer } from 'ws'; import cookie from 'cookie';
import { WebSocketServer } from 'ws'; import cookie from 'cookie';
import jsonwebtoken from 'jsonwebtoken';
import httpsServer from 'https';
import httpServer from 'http';
import { readFileSync } from 'fs';
import log4js from 'log4js';
import models from '../../models/index.model';
import config from '../../config';
import helperFunctions from './helpers';
// eslint-disable-next-line no-unused-vars
import authenticationController from '../../controllers/authentication';
import deviceController from '../../controllers/devices';
const logger = log4js.getLogger('default');
@ -21,10 +17,10 @@ let wss;
function __server() {
let server;
if (config.athena.secure) {
if (process.env.ATHENA_SECURE && process.env.SSL_CRT) {
server = httpsServer.createServer({
cert: readFileSync(config.sslCrt),
key: readFileSync(config.sslKey),
cert: readFileSync(process.env.SSL_CRT),
key: readFileSync(process.env.SSL_KEY),
});
} else {
server = httpServer.createServer();
@ -48,10 +44,10 @@ function __server() {
ws.isAlive = false;
ws.ping();
});
}, config.athena.socket.heartbeatFrequency ? config.athena.socket.heartbeatFrequency : 5000);
}, process.env.ATHENA_SOCKET_HEARTBEAT_FREQ ? process.env.ATHENA_SOCKET_HEARTBEAT_FREQ : 5000);
server.listen(config.athena.socket.port, () => {
logger.info(`Athena(Server) - UP @ ${config.athena.host}:${config.athena.port}`);
server.listen(process.env.ATHENA_SOCKET_PORT, () => {
logger.info(`Athena(Server) - UP @ ${process.env.ATHENA_SOCKET_HOST}:${process.env.ATHENA_SOCKET_PORT}`);
});
wss.on('connection', manageConnection);

View File

@ -1,14 +1,10 @@
import WebSocket, { WebSocketServer } from 'ws';
import { WebSocketServer } from 'ws';
import cookie from 'cookie';
import httpServer from 'http';
import log4js from 'log4js';
import config from '../../config.js';
import controlsFunction from './controls.js';
import authenticationController from '../../controllers/authentication.js';
// eslint-disable-next-line no-unused-vars
import deviceController from '../../controllers/devices';
import athenaRealtime from '../athena/index.js';
import realtimeCommands from './commands.js';
@ -24,8 +20,8 @@ function __server() {
wss = new WebSocketServer({ server }, { path: '/realtime/v1', handshakeTimeout: 500 });
server.listen(config.clientSocket.port, config.clientSocket.host, () => {
logger.info(`Web(Server) - UP @ ${config.clientSocket.host}:${config.clientSocket.port}`);
server.listen(process.env.CLIENT_SOCKET_PORT, process.env.CLIENT_SOCKET_HOST, () => {
logger.info(`Web(Server) - UP @ ${process.env.CLIENT_SOCKET_HOST}:${process.env.CLIENT_SOCKET_PORT}`);
});
wss.on('connection', manageConnection);

409
worker.js
View File

@ -1,13 +1,11 @@
/* eslint-disable */
import 'dotenv/config'
import crypto from 'crypto';
import fs from 'fs';
import path, { dirname } from 'path';
import { fileURLToPath } from 'url';
import log4js from 'log4js';
import sqlite3 from 'sqlite3';
import { open } from 'sqlite';
import lockfile from 'proper-lockfile';
import dirTree from 'directory-tree';
import { execSync } from 'child_process';
@ -16,32 +14,28 @@ import ffprobe from 'ffprobe';
import ffprobeStatic from 'ffprobe-static';
import orm from './models/index.model';
import config from './config';
var db = null;
var lastCleaningTime = 0;
var startTime = Date.now();
let lastCleaningTime = 0;
let startTime = Date.now();
log4js.configure({
appenders: { logfile: { type: 'file', filename: 'worker.log' }, out: { type: 'console' } },
categories: { default: { appenders: ['out', 'logfile'], level: 'info' } }
});
var logger = log4js.getLogger('default');
let logger = log4js.getLogger('default');
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
global.__basedir = __dirname;
function initializeStorage() {
var verifiedPath = mkDirByPathSync(config.storagePath, { isRelativeToScript: (config.storagePath.indexOf('/') !== 0) });
let verifiedPath = mkDirByPathSync(process.env.STORAGE_PATH, { isRelativeToScript: (process.env.STORAGE_PATH.indexOf('/') !== 0) });
if (verifiedPath != null) {
logger.info(`Verified storage path ${verifiedPath}`);
}
else {
logger.error(`Unable to verify storage path '${config.storagePath}', check filesystem / permissions`);
logger.error(`Unable to verify storage path '${process.env.STORAGE_PATH}', check filesystem / permissions`);
process.exit();
}
}
@ -79,7 +73,7 @@ function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
}
function writeFileSync(path, buffer, permission) {
var fileDescriptor;
let fileDescriptor;
try {
fileDescriptor = fs.openSync(path, 'w', permission);
} catch (e) {
@ -106,12 +100,12 @@ function moveUploadedFile(buffer, directory, filename) {
return false;
}
if (config.storagePath.lastIndexOf('/') !== config.storagePath.length - 1) {
if (process.env.STORAGE_PATH.lastIndexOf('/') !== process.env.STORAGE_PATH.length - 1) {
directory = `/${directory}`;
}
if (directory.lastIndexOf('/') !== directory.length - 1) directory += '/';
var finalPath = mkDirByPathSync(config.storagePath + directory, { isRelativeToScript: (config.storagePath.indexOf('/') !== 0) });
let finalPath = mkDirByPathSync(process.env.STORAGE_PATH + directory, { isRelativeToScript: (process.env.STORAGE_PATH.indexOf('/') !== 0) });
if (finalPath && finalPath.length > 0) {
if (writeFileSync(`${finalPath}/${filename}`, buffer, 0o660)) {
logger.info(`moveUploadedFile successfully written '${finalPath}/${filename}'`);
@ -120,7 +114,7 @@ function moveUploadedFile(buffer, directory, filename) {
logger.error('moveUploadedFile failed to writeFileSync');
return false;
}
logger.error(`moveUploadedFile invalid final path, check permissions to create / write '${config.storagePath + directory}'`);
logger.error(`moveUploadedFile invalid final path, check permissions to create / write '${process.env.STORAGE_PATH + directory}'`);
return false;
}
@ -141,80 +135,26 @@ function deleteFolderRecursive(directoryPath) {
}
}
async function dbProtectedRun() {
let retries = 0;
while (true) {
try {
return await db.run(...arguments);
} catch (error) {
logger.error(error);
retries++;
if (retries >= 10) {
break;
}
await new Promise((r) => setTimeout(r, 1000));
}
}
logger.error(`unable to complete dbProtectedRun for ${arguments}`);
return null;
}
let segmentProcessQueue = [];
let segmentProcessPosition = 0;
async function dbProtectedGet() {
let retries = 0;
while (true) {
try {
return await db.get(...arguments);
} catch (error) {
logger.error(error);
retries++;
if (retries >= 10) {
break;
}
await new Promise((r) => setTimeout(r, 1000));
}
}
logger.error(`unable to complete dbProtectedGet for ${arguments}`);
return null;
}
let affectedDrives = {};
let affectedDriveInitData = {};
let affectedDriveCarParams = {};
async function dbProtectedAll() {
let retries = 0;
while (true) {
try {
return await db.all(...arguments);
} catch (error) {
logger.error(error);
retries++;
if (retries >= 10) {
break;
}
await new Promise((r) => setTimeout(r, 1000));
}
}
logger.error(`unable to complete dbProtectedGet for ${arguments}`);
return null;
}
let affectedDevices = {};
var segmentProcessQueue = [];
var segmentProcessPosition = 0;
var affectedDrives = {};
var affectedDriveInitData = {};
var affectedDriveCarParams = {};
var affectedDevices = {};
var rlog_lastTsInternal = 0;
var rlog_prevLatInternal = -1000;
var rlog_prevLngInternal = -1000;
var rlog_totalDistInternal = 0;
var rlog_lastTsExternal = 0;
var rlog_prevLatExternal = -1000;
var rlog_prevLngExternal = -1000;
var rlog_totalDistExternal = 0;
var rlog_CarParams = null;
var rlog_InitData = null;
var qcamera_duration = 0;
let rlog_lastTsInternal = 0;
let rlog_prevLatInternal = -1000;
let rlog_prevLngInternal = -1000;
let rlog_totalDistInternal = 0;
let rlog_lastTsExternal = 0;
let rlog_prevLatExternal = -1000;
let rlog_prevLngExternal = -1000;
let rlog_totalDistExternal = 0;
let rlog_CarParams = null;
let rlog_InitData = null;
let qcamera_duration = 0;
function processSegmentRLog(rLogPath) {
rlog_lastTsInternal = 0;
@ -230,7 +170,7 @@ function processSegmentRLog(rLogPath) {
return new Promise(
(resolve) => {
var temporaryFile = rLogPath.replace('.bz2', '');
let temporaryFile = rLogPath.replace('.bz2', '');
try {
execSync(`bunzip2 -k -f "${rLogPath}"`);
@ -245,6 +185,7 @@ function processSegmentRLog(rLogPath) {
let readStream;
let reader;
try {
readStream = fs.createReadStream(temporaryFile);
reader = Reader(readStream);
@ -259,24 +200,27 @@ function processSegmentRLog(rLogPath) {
} catch (exception) { }
resolve();
});
//const jsonLog = fs.createWriteStream(rLogPath.replace('.bz2', '.json'));
try {
reader((obj) => {
//jsonLog.write(JSON.stringify(obj));
try {
if (obj.LogMonoTime !== undefined && obj.LogMonoTime - rlog_lastTsInternal >= 1000000 * 1000 * 0.99 && obj.GpsLocation !== undefined) {
logger.info(`processSegmentRLog GpsLocation @ ${obj.LogMonoTime}: ${obj.GpsLocation.Latitude} ${obj.GpsLocation.Longitude}`);
if (rlog_prevLatInternal != -1000) {
var lat1 = rlog_prevLatInternal;
var lat2 = obj.GpsLocation.Latitude;
var lon1 = rlog_prevLngInternal;
var lon2 = obj.GpsLocation.Longitude;
var p = 0.017453292519943295; // Math.PI / 180
var c = Math.cos;
var a = 0.5 - c((lat2 - lat1) * p) / 2
let lat1 = rlog_prevLatInternal;
let lat2 = obj.GpsLocation.Latitude;
let lon1 = rlog_prevLngInternal;
let lon2 = obj.GpsLocation.Longitude;
let p = 0.017453292519943295; // Math.PI / 180
let c = Math.cos;
let a = 0.5 - c((lat2 - lat1) * p) / 2
+ c(lat1 * p) * c(lat2 * p)
* (1 - c((lon2 - lon1) * p)) / 2;
var dist_m = 1000 * 12742 * Math.asin(Math.sqrt(a)); // 2 * R; R = 6371 km
let dist_m = 1000 * 12742 * Math.asin(Math.sqrt(a)); // 2 * R; R = 6371 km
if (dist_m > 70) dist_m = 0; // each segment is max. 60s. if the calculated speed would exceed ~250km/h for this segment, we assume the coordinates off / defective and skip it
rlog_totalDistInternal += dist_m;
}
@ -288,17 +232,17 @@ function processSegmentRLog(rLogPath) {
logger.info(`processSegmentRLog GpsLocationExternal @ ${obj.LogMonoTime}: ${obj.GpsLocationExternal.Latitude} ${obj.GpsLocationExternal.Longitude}`);
if (rlog_prevLatExternal != -1000) {
var lat1 = rlog_prevLatExternal;
var lat2 = obj.GpsLocationExternal.Latitude;
var lon1 = rlog_prevLngExternal;
var lon2 = obj.GpsLocationExternal.Longitude;
var p = 0.017453292519943295; // Math.PI / 180
var c = Math.cos;
var a = 0.5 - c((lat2 - lat1) * p) / 2
let lat1 = rlog_prevLatExternal;
let lat2 = obj.GpsLocationExternal.Latitude;
let lon1 = rlog_prevLngExternal;
let lon2 = obj.GpsLocationExternal.Longitude;
let p = 0.017453292519943295; // Math.PI / 180
let c = Math.cos;
let a = 0.5 - c((lat2 - lat1) * p) / 2
+ c(lat1 * p) * c(lat2 * p)
* (1 - c((lon2 - lon1) * p)) / 2;
var dist_m = 1000 * 12742 * Math.asin(Math.sqrt(a)); // 2 * R; R = 6371 km
let dist_m = 1000 * 12742 * Math.asin(Math.sqrt(a)); // 2 * R; R = 6371 km
if (dist_m > 70) dist_m = 0; // each segment is max. 60s. if the calculated speed would exceed ~250km/h for this segment, we assume the coordinates off / defective and skip it
rlog_totalDistExternal += dist_m;
}
@ -342,12 +286,12 @@ function processSegmentVideo(qcameraPath) {
});
}
function processSegmentsRecursive() {
async function processSegmentsRecursive() {
if (segmentProcessQueue.length <= segmentProcessPosition) {
return updateDrives();
}
var segmentWrapper = segmentProcessQueue[segmentProcessPosition];
let segmentWrapper = segmentProcessQueue[segmentProcessPosition];
const { segment } = segmentWrapper;
const { uploadComplete } = segmentWrapper;
@ -356,33 +300,29 @@ function processSegmentsRecursive() {
logger.info(`processSegmentsRecursive ${segment.dongle_id} ${segment.drive_identifier} ${segment.segment_id} ${JSON.stringify(segment)}`);
const driveSegmentResult = dbProtectedRun(
'UPDATE drive_segments SET process_attempts = ? WHERE id = ?',
segment.process_attempts = segment.process_attempts + 1
segment.process_attempts = segment.process_attempts + 1,
segment.id
await orm.query(
`UPDATE drive_segments SET process_attempts = ${segment.process_attempts} WHERE id = ${segment.id}`,
);
if (segment.process_attempts > 5) {
logger.error(`FAILING TO PROCESS SEGMENT,${segment.dongle_id} ${segment.drive_identifier} ${segment.segment_id} JSON: ${JSON.stringify(segment)} SKIPPING `);
segmentProcessPosition++;
}
else {
var p1 = processSegmentRLog(fileStatus['rlog.bz2']);
var p2 = processSegmentVideo(fileStatus['qcamera.ts']);
} else {
let p1 = processSegmentRLog(fileStatus['rlog.bz2']);
let p2 = processSegmentVideo(fileStatus['qcamera.ts']);
Promise.all([p1, p2])
.then((values) => {
(async () => {
logger.info(`processSegmentsRecursive ${segment.dongle_id} ${segment.drive_identifier} ${segment.segment_id} internal gps: ${Math.round(rlog_totalDistInternal * 100) / 100}m, external gps: ${Math.round(rlog_totalDistExternal * 100) / 100}m, duration: ${qcamera_duration}s`);
const driveSegmentResult = await orm.models.drive_segments.update({
duration: qcamera_duration,
duration: Math.round(qcamera_duration),
distance_meters: Math.round(Math.max(rlog_totalDistInternal, rlog_totalDistExternal) * 10) / 10,
is_processed: true,
upload_complete: uploadComplete,
is_stalled: false
}, {where: {id: segment.id}})
@ -419,27 +359,29 @@ async function updateSegments() {
affectedDriveCarParams = {};
affectedDriveInitData = {};
const drive_segments = await dbProtectedAll('SELECT * FROM drive_segments WHERE upload_complete = ? AND is_stalled = ? AND process_attempts < ? ORDER BY created ASC', false, false, 5);
const [drive_segments] = await orm.query('SELECT * FROM drive_segments WHERE upload_complete = false AND is_stalled = false AND process_attempts < 5 ORDER BY created ASC');
logger.info('updateSegments - total drive_segments', drive_segments.length);
if (drive_segments != null) {
for (var t = 0; t < drive_segments.length; t++) {
var segment = drive_segments[t];
for (let t = 0; t < drive_segments.length; t++) {
let segment = drive_segments[t];
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
let dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(segment.dongle_id)
.digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt)
let driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(segment.drive_identifier)
.digest('hex');
const directoryTree = dirTree(`${config.storagePath + segment.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${segment.drive_identifier}/${segment.segment_id}`);
if (directoryTree == null || directoryTree.children == undefined) continue; // happens if upload in progress (db entity written but directory not yet created)
const directoryTreePath = `${process.env.STORAGE_PATH + segment.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${segment.drive_identifier}/${segment.segment_id}`;
const directoryTree = dirTree(directoryTreePath);
var qcamera = false;
var fcamera = false;
var dcamera = false;
var qlog = false;
var rlog = false;
var fileStatus = {
if (directoryTree == null || directoryTree.children == undefined) {
console.log('missing directory', directoryTreePath);
continue; // happens if upload in progress (db entity written but directory not yet created)
}
let fileStatus = {
'fcamera.hevc': false,
'dcamera.hevc': false,
'qcamera.ts': false,
@ -447,13 +389,12 @@ async function updateSegments() {
'rlog.bz2': false
};
for (var i in directoryTree.children) {
for (let i in directoryTree.children) {
fileStatus[directoryTree.children[i].name] = directoryTree.children[i].path;
}
var uploadComplete = false;
if (fileStatus['qcamera.ts'] !== false && fileStatus['fcamera.hevc'] !== false && fileStatus['rlog.bz2'] !== false && fileStatus['qlog.bz2'] !== false) // upload complete
{
let uploadComplete = false;
if (Object.keys(fileStatus).filter(key => fileStatus[key] === false).length === 0) {
uploadComplete = true;
}
@ -468,13 +409,8 @@ async function updateSegments() {
else if (uploadComplete) {
logger.info(`updateSegments uploadComplete for ${segment.dongle_id} ${segment.drive_identifier} ${segment.segment_id}`);
const driveSegmentResult = await dbProtectedRun(
'UPDATE drive_segments SET upload_complete = ?, is_stalled = ? WHERE id = ?',
true,
false,
segment.id
await orm.query(
`UPDATE drive_segments SET upload_complete = true, is_stalled = false WHERE id = ${segment.id}`
);
affectedDrives[`${segment.dongle_id}|${segment.drive_identifier}`] = true;
@ -482,11 +418,8 @@ async function updateSegments() {
else if (Date.now() - segment.created > 10 * 24 * 3600 * 1000) { // ignore non-uploaded segments after 10 days until a new upload_url is requested (which resets is_stalled)
logger.info(`updateSegments isStalled for ${segment.dongle_id} ${segment.drive_identifier} ${segment.segment_id}`);
const driveSegmentResult = await dbProtectedRun(
'UPDATE drive_segments SET is_stalled = ? WHERE id = ?',
true,
segment.id
await orm.query(
`UPDATE drive_segments SET is_stalled = true WHERE id = ${segment.id}`
);
}
@ -510,26 +443,31 @@ async function updateDevices() {
// go through all affected devices (with deleted or updated drives) and update them (storage_used)
logger.info(`updateDevices - affected drives: ${JSON.stringify(affectedDevices)}`);
for (const [key, value] of Object.entries(affectedDevices)) {
var dongleId = key;
let dongleId = key;
const device = await dbProtectedGet('SELECT * FROM devices WHERE dongle_id = ?', dongleId);
const [device] = await orm.query(`SELECT * FROM devices WHERE dongle_id = ${dongleId}`);
if (device == null) continue;
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
let dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(device.dongle_id)
.digest('hex');
var devicePath = `${config.storagePath + device.dongle_id}/${dongleIdHash}`;
var deviceQuotaMb = Math.round(parseInt(execSync(`du -s ${devicePath} | awk -F'\t' '{print $1;}'`)
let devicePath = `${process.env.STORAGE_PATH + device.dongle_id}/${dongleIdHash}`;
let deviceQuotaMb = Math.round(parseInt(execSync(`du -s ${devicePath} | awk -F'\t' '{print $1;}'`)
.toString()) / 1024);
logger.info(`updateDevices device ${dongleId} has an updated storage_used of: ${deviceQuotaMb} MB`);
const deviceResult = await orm.models.drives.update(
{storage_used: deviceQuotaMb},
{where: {dongle_id: device.dongle_id}}
)
await orm.models.drives.update(
{
storage_used: deviceQuotaMb
},
{
where: {
dongle_id: device.dongle_id
}
}
);
}
affectedDevices = [];
}
@ -537,29 +475,29 @@ async function updateDrives() {
// go through all affected drives and update them / complete and/or build m3u8
logger.info(`updateDrives - affected drives: ${JSON.stringify(affectedDrives)}`);
for (const [key, value] of Object.entries(affectedDrives)) {
var dongleId,
let dongleId,
driveIdentifier;
[dongleId, driveIdentifier] = key.split('|');
let drive = await orm.models.drives({where: {driveIdentifier: driveIdentifier, dongleId: dongleId}})
let drive = await orm.models.drives.findOne({where: {identifier: driveIdentifier, dongle_id: dongleId}});
if (drive == null) continue;
drive = drive.dataValues;
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
let dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(drive.dongle_id)
.digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt)
let driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(drive.identifier)
.digest('hex');
var driveUrl = `${config.baseDriveDownloadUrl + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
var drivePath = `${config.storagePath + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
let driveUrl = `${process.env.BASE_DRIVE_DOWNLOAD_URL + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
let drivePath = `${process.env.STORAGE_PATH + drive.dongle_id}/${dongleIdHash}/${driveIdentifierHash}/${drive.identifier}`;
var uploadComplete = true;
var isProcessed = true;
let uploadComplete = true;
let isProcessed = true;
var totalDistanceMeters = 0;
var totalDurationSeconds = 0;
var playlistSegmentStrings = '';
let totalDistanceMeters = 0;
let totalDurationSeconds = 0;
let playlistSegmentStrings = '';
const drive_segments= await orm.models.drive_segments.findAll({
const drive_segments = await orm.models.drive_segments.findAll({
where: {
drive_identifier: driveIdentifier,
dongle_id: dongleId
@ -570,7 +508,7 @@ async function updateDrives() {
})
if (drive_segments != null) {
for (var t = 0; t < drive_segments.length; t++) {
for (let t = 0; t < drive_segments.length; t++) {
if (!drive_segments[t].upload_complete) uploadComplete = false;
if (!drive_segments[t].is_processed) {
isProcessed = false;
@ -584,15 +522,9 @@ async function updateDrives() {
}
}
var { filesize } = drive;
let { filesize } = drive;
if (uploadComplete) {
try {
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
.update(dongleId)
.digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt)
.update(driveIdentifier)
.digest('hex');
filesize = parseInt(execSync(`du -s ${drivePath} | awk -F'\t' '{print $1;}'`)
.toString()); // in kilobytes
} catch (exception) { }
@ -615,23 +547,22 @@ async function updateDrives() {
logger.info(`updateDrives drive ${dongleId} ${driveIdentifier} uploadComplete: ${uploadComplete}`);
const driveResult = await orm.models.drives.update(
await orm.models.drives.update(
{distance_meters: Math.round(totalDistanceMeters),
duration: totalDurationSeconds,
duration: Math.round(totalDurationSeconds),
upload_complete: uploadComplete,
is_processed: isProcessed,
filesize,
metadata:JSON.stringify(metadata)
},
{where: {id: drive.id}}
)
);
affectedDevices[dongleId] = true;
if (isProcessed) {
// create the playlist file m3u8 for cabana
var playlist = '#EXTM3U\n'
let playlist = '#EXTM3U\n'
+ '#EXT-X-VERSION:3\n'
+ '#EXT-X-TARGETDURATION:61\n'
+ '#EXT-X-MEDIA-SEQUENCE:0\n'
@ -650,51 +581,44 @@ async function updateDrives() {
}
async function deleteExpiredDrives() {
var expirationTs = Date.now() - config.deviceDriveExpirationDays * 24 * 3600 * 1000;
let expirationTs = Date.now() - process.env.DEVICE_EXPIRATION_DAYS * 24 * 3600 * 1000;
const expiredDrives = await dbProtectedAll('SELECT * FROM drives WHERE is_preserved = ? AND is_deleted = ? AND created < ?', false, false, expirationTs);
const [expiredDrives] = await orm.query(`SELECT * FROM drives WHERE is_preserved = false AND is_deleted = false AND created < ${expirationTs}`);
if (expiredDrives != null) {
for (var t = 0; t < expiredDrives.length; t++) {
logger.info(`deleteExpiredDrives drive ${expiredDrives[t].dongle_id} ${expiredDrives[t].identifier} is older than ${config.deviceDriveExpirationDays} days, set is_deleted=true`);
const driveResult = await orm.models.drives.update({
is_deleted: true
},
{where: {id: expiredDrives[t].id}})
for (let t = 0; t < expiredDrives.length; t++) {
logger.info(`deleteExpiredDrives drive ${expiredDrives[t].dongle_id} ${expiredDrives[t].identifier} is older than ${process.env.DEVICE_EXPIRATION_DAYS} days, set is_deleted=true`);
await orm.models.drives.update(
{
is_deleted: true
},
{where: {id: expiredDrives[t].id}}
);
}
}
}
async function removeDeletedDrivesPhysically() {
const deletedDrives = await dbProtectedAll('SELECT * FROM drives WHERE is_deleted = ? AND is_physically_removed = ?', true, false);
const [deletedDrives] = await orm.query('SELECT * FROM drives WHERE is_deleted = true AND is_physically_removed = false');
if (deletedDrives == null) {
return;
}
for (var t = 0; t < deletedDrives.length; t++) {
for (let t = 0; t < deletedDrives.length; t++) {
logger.info(`removeDeletedDrivesPhysically drive ${deletedDrives[t].dongle_id} ${deletedDrives[t].identifier} is deleted, remove physical files and clean database`);
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
let dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(deletedDrives[t].dongle_id)
.digest('hex');
var driveIdentifierHash = crypto.createHmac('sha256', config.applicationSalt)
let driveIdentifierHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(deletedDrives[t].identifier)
.digest('hex');
const drivePath = `${config.storagePath + deletedDrives[t].dongle_id}/${dongleIdHash}/${driveIdentifierHash}`;
const drivePath = `${process.env.STORAGE_PATH + deletedDrives[t].dongle_id}/${dongleIdHash}/${driveIdentifierHash}`;
logger.info(`removeDeletedDrivesPhysically drive ${deletedDrives[t].dongle_id} ${deletedDrives[t].identifier} storage path is ${drivePath}`);
try {
const driveResult = await dbProtectedRun(
'UPDATE drives SET is_physically_removed = ? WHERE id = ?',
true,
const driveResult = await orm.query(`UPDATE drives SET is_physically_removed = true WHERE id = ${deletedDrives[t].id}`);
deletedDrives[t].id
);
const driveSegmentResult = await dbProtectedRun(
'DELETE FROM drive_segments WHERE drive_identifier = ? AND dongle_id = ?',
deletedDrives[t].identifier,
deletedDrives[t].dongle_id
const driveSegmentResult = await orm.query(
`DELETE FROM drive_segments WHERE drive_identifier = ${deletedDrives[t].identifier} AND dongle_id = ${deletedDrives[t].dongle_id}`
);
if (driveResult != null && driveSegmentResult != null) deleteFolderRecursive(drivePath, { recursive: true });
@ -706,35 +630,29 @@ async function removeDeletedDrivesPhysically() {
}
async function deleteOverQuotaDrives() {
const devices = await dbProtectedAll('SELECT * FROM devices WHERE storage_used > ?', config.deviceStorageQuotaMb);
const [devices] = await orm.query(`SELECT * FROM devices WHERE storage_used > ${process.env.DEVICE_STORAGE_QUOTA_MB}`);
if (devices == null) {
return;
}
for (var t = 0; t < devices.length; t++) {
var foundDriveToDelete = false;
for (let t = 0; t < devices.length; t++) {
let foundDriveToDelete = false;
const driveNormal = await dbProtectedGet('SELECT * FROM drives WHERE dongle_id = ? AND is_preserved = ? AND is_deleted = ? ORDER BY created ASC LIMIT 1', devices[t].dongle_id, false, false);
const [driveNormal] = await orm.query(`SELECT * FROM drives WHERE dongle_id = ${devices[t].dongle_id} AND is_preserved = false AND is_deleted = false ORDER BY created ASC LIMIT 1`);
if (driveNormal != null) {
logger.info(`deleteOverQuotaDrives drive ${driveNormal.dongle_id} ${driveNormal.identifier} (normal) is deleted for over-quota`);
const driveResult = await dbProtectedRun(
'UPDATE drives SET is_deleted = ? WHERE id = ?',
true,
driveNormal.id
const [driveResult] = await orm.query(
`UPDATE drives SET is_deleted = true WHERE id = ${driveNormal.id}`,
);
foundDriveToDelete = true;
}
if (!foundDriveToDelete) {
const drivePreserved = await dbProtectedGet('SELECT * FROM drives WHERE dongle_id = ? AND is_preserved = ? AND is_deleted = ? ORDER BY created ASC LIMIT 1', devices[t].dongle_id, true, false);
const [drivePreserved] = await orm.query(`SELECT * FROM drives WHERE dongle_id = devices[t].dongle_id AND is_preserved = true AND is_deleted = false ORDER BY created ASC LIMIT 1`);
if (drivePreserved != null) {
logger.info(`deleteOverQuotaDrives drive ${drivePreserved.dongle_id} ${drivePreserved.identifier} (preserved!) is deleted for over-quota`);
const driveResult = await dbProtectedRun(
'UPDATE drives SET is_deleted = ? WHERE id = ?',
true,
drivePreserved.id
const [driveResult] = await orm.query(
`UPDATE drives SET is_deleted = ? WHERE id = ${drivePreserved.id}`
);
foundDriveToDelete = true;
}
@ -743,26 +661,26 @@ async function deleteOverQuotaDrives() {
}
async function deleteBootAndCrashLogs() {
const devices = await dbProtectedAll('SELECT * FROM devices');
const [devices] = await orm.query('SELECT * FROM devices');
if (devices == null) {
return;
}
for (var t = 0; t < devices.length; t++) {
var device = devices[t];
var dongleIdHash = crypto.createHmac('sha256', config.applicationSalt)
for (let t = 0; t < devices.length; t++) {
let device = devices[t];
let dongleIdHash = crypto.createHmac('sha256', process.env.APP_SALT)
.update(device.dongle_id)
.digest('hex');
const bootlogDirectoryTree = dirTree(`${config.storagePath + device.dongle_id}/${dongleIdHash}/boot/`, { attributes: ['size'] });
var bootlogFiles = [];
const bootlogDirectoryTree = dirTree(`${process.env.STORAGE_PATH + device.dongle_id}/${dongleIdHash}/boot/`, { attributes: ['size'] });
let bootlogFiles = [];
if (bootlogDirectoryTree != undefined) {
for (var i = 0; i < bootlogDirectoryTree.children.length; i++) {
var timeSplit = bootlogDirectoryTree.children[i].name.replace('boot-', '')
for (let i = 0; i < bootlogDirectoryTree.children.length; i++) {
let timeSplit = bootlogDirectoryTree.children[i].name.replace('boot-', '')
.replace('crash-', '')
.replace('\.bz2', '')
.split('--');
var timeString = `${timeSplit[0]} ${timeSplit[1].replace(/-/g, ':')}`;
let timeString = `${timeSplit[0]} ${timeSplit[1].replace(/-/g, ':')}`;
bootlogFiles.push({
name: bootlogDirectoryTree.children[i].name,
size: bootlogDirectoryTree.children[i].size,
@ -771,7 +689,7 @@ async function deleteBootAndCrashLogs() {
});
}
bootlogFiles.sort((a, b) => ((a.date < b.date) ? 1 : -1));
for (var c = 5; c < bootlogFiles.length; c++) {
for (let c = 5; c < bootlogFiles.length; c++) {
logger.info(`deleteBootAndCrashLogs deleting boot log ${bootlogFiles[c].path}`);
try {
fs.unlinkSync(bootlogFiles[c].path);
@ -782,15 +700,15 @@ async function deleteBootAndCrashLogs() {
}
}
const crashlogDirectoryTree = dirTree(`${config.storagePath + device.dongle_id}/${dongleIdHash}/crash/`, { attributes: ['size'] });
var crashlogFiles = [];
const crashlogDirectoryTree = dirTree(`${process.env.STORAGE_PATH + device.dongle_id}/${dongleIdHash}/crash/`, { attributes: ['size'] });
let crashlogFiles = [];
if (crashlogDirectoryTree != undefined) {
for (var i = 0; i < crashlogDirectoryTree.children.length; i++) {
var timeSplit = crashlogDirectoryTree.children[i].name.replace('boot-', '')
for (let i = 0; i < crashlogDirectoryTree.children.length; i++) {
let timeSplit = crashlogDirectoryTree.children[i].name.replace('boot-', '')
.replace('crash-', '')
.replace('\.bz2', '')
.split('--');
var timeString = `${timeSplit[0]} ${timeSplit[1].replace(/-/g, ':')}`;
let timeString = `${timeSplit[0]} ${timeSplit[1].replace(/-/g, ':')}`;
crashlogFiles.push({
name: crashlogDirectoryTree.children[i].name,
size: crashlogDirectoryTree.children[i].size,
@ -799,7 +717,7 @@ async function deleteBootAndCrashLogs() {
});
}
crashlogFiles.sort((a, b) => ((a.date < b.date) ? 1 : -1));
for (var c = 5; c < crashlogFiles.length; c++) {
for (let c = 5; c < crashlogFiles.length; c++) {
logger.info(`deleteBootAndCrashLogs deleting crash log ${crashlogFiles[c].path}`);
try {
fs.unlinkSync(crashlogFiles[c].path);
@ -851,21 +769,6 @@ lockfile.lock('retropilot_worker', {
.then((release) => {
logger.info('STARTING WORKER...');
(async () => {
try {
db = await open({
filename: config.databaseFile,
driver: sqlite3.Database,
mode: sqlite3.OPEN_READWRITE
});
await db.get('SELECT * FROM accounts LIMIT 1');
await db.get('SELECT * FROM devices LIMIT 1');
await db.get('SELECT * FROM drives LIMIT 1');
await db.get('SELECT * FROM drive_segments LIMIT 1');
} catch (exception) {
logger.error(exception);
process.exit();
}
initializeStorage();
setTimeout(() => {
mainWorkerLoop();