From 81524d760c3d18af295cd610c0d62bcfcf44c206 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Thu, 14 Apr 2022 15:51:13 +0100 Subject: [PATCH] add 'development' environment and improve documentation --- .env.docker.sample | 2 - .env.sample | 7 +- .gitignore | 5 +- README.md | 36 ++++------- environment/README.md | 5 ++ environment/development/.env.sample | 61 ++++++++++++++++++ environment/development/README.md | 37 +++++++++++ .../development/docker-compose.yml | 64 ++++++++++--------- environment/uat/.env.sample | 61 ++++++++++++++++++ environment/uat/.gitignore | 2 - environment/uat/README.md | 34 ++++++++++ environment/uat/docker-compose.yml | 26 +++++--- environment/uat/traefik/traefik.toml | 3 +- init-letsencrypt.sh | 2 + 14 files changed, 270 insertions(+), 75 deletions(-) delete mode 100644 .env.docker.sample create mode 100644 environment/README.md create mode 100644 environment/development/.env.sample create mode 100644 environment/development/README.md rename docker-compose.yml => environment/development/docker-compose.yml (55%) create mode 100644 environment/uat/.env.sample delete mode 100644 environment/uat/.gitignore diff --git a/.env.docker.sample b/.env.docker.sample deleted file mode 100644 index 7d4f0a2..0000000 --- a/.env.docker.sample +++ /dev/null @@ -1,2 +0,0 @@ -DB_HOST=db -STORAGE_PATH=/realdata/ diff --git a/.env.sample b/.env.sample index 6a166c1..0e75b4b 100644 --- a/.env.sample +++ b/.env.sample @@ -1,12 +1,12 @@ NODE_ENV=development APP_SALT=RANDOM_SEED -LOG_LEVEL=info +LOG_LEVEL=debug DB_NAME=retro-pilot DB_USER=root DB_PASS=root # If using docker compose, this should match the container service name -DB_HOST=db +DB_HOST=localhost DB_PORT=5432 # Whether or not to DROP all tables and recreate to match the current models DB_FORCE_SYNC=false @@ -18,7 +18,7 @@ HTTP_INTERFACE=0.0.0.0 HTTP_PORT=8080 # Set to false to skip sending mail, all attempted mail is logged under DEBUG -CAN_SEND_MAIL=true +CAN_SEND_MAIL=false # credentials for smtp server to send account registration mails. if not filled in, get the generated tokens from the server.log manually SMTP_HOST="localhost" SMTP_PORT=25 @@ -36,6 +36,7 @@ BASE_DRIVE_DOWNLOAD_URL="http://192.168.1.165:8080/realdata/" BASE_DRIVE_DOWNLOAD_PATH_MAPPING="/realdata" # relative or absolute ( "/..." for absolute path ) STORAGE_PATH="realdata/" + CABANA_URL="http://192.168.1.165:8080/cabana/index.html" DEVICE_STORAGE_QUOTA_MB=200000 diff --git a/.gitignore b/.gitignore index c56bc16..95f021d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,14 +8,13 @@ node_modules/ # Data -realdata database +realdata # Miscellaneous .DS_Store *.log -.env* -!.env*.sample +.env # Build dist diff --git a/README.md b/README.md index 34945cd..9cfcd4a 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,14 @@ If you don't want to host your own instance, check out https://api.retropilot.or The server consists of 2 node scripts. - `src/server` is using expressjs and runs the backend (file upload / communication with openpilot) and the useradmin dashboard to manage / view / download drives & logs. - - `src/worker` is a background worker that is processing drives (analyzing video files & logs) to prepare drives for playback in cabana and to gather statistics. It automatically terminates itself after 60 minutes to make sure the video/log libraries do not cause memory leaks. **Attention:** Minimum required node version is **node 10**. ### [Server] Installation +TODO: write instructions for PostgreSQL database + ``` npm install cp config.sample.js config.js @@ -39,7 +40,8 @@ node -r esm src/worker ### [Server] CABANA Support -A compiled version of a custom cabana fork (https://github.com/florianbrede-ayet/retropilot-cabana) is directly bundled in the `cabana/` subdirectory and will be served by the express app. After starting `index.js`, cabana is ready to use. + +A compiled version of a custom cabana fork (https://github.com/RetroPilot/cabana) is directly bundled in the `cabana/` subdirectory and will be served by the express app. After starting `index.js`, cabana is ready to use. ----- @@ -48,12 +50,16 @@ A compiled version of a custom cabana fork (https://github.com/florianbrede-ayet On the device or in your fork's code, replace all API endpoints with your own server endpoint. This could be executed directly on the device in the shell to use `https://api.retropilot.org` as backend: + ``` find /data/openpilot -type f -exec sed -i 's/https:\/\/api.commadotai.com/https:\/\/api.retropilot.org/g' {} + ``` +TODO: add git patch instructions + ### [Device] Swapping Servers (Back) To switch a device between different servers, you have to remove the old `DongleId` and reboot: + ``` rm /data/params/d/DongleID reboot @@ -62,10 +68,10 @@ reboot There is no need to backup the `DongleId`, as the new server will identify your device based on its imei, serial and public key. ### [Device] Raw Drives Not Uploading (fcamera & rlog) + 1. Raw data is only uploaded if the device is sufficiently charged, not connected to an active panda (offroad) and there are no immediate files (boot, crash, qcamera, qlog) remaining.
2. Your branch might have raw uploads disabled, check *Device Settings > Upload Raw Logs*. - If that doesn't help or the option is not available, try: ``` @@ -95,26 +101,6 @@ The athena websockets interface is not implemented yet, so the comma app and ath ![image](https://user-images.githubusercontent.com/48515354/118385084-37fb2200-b60c-11eb-8d3e-6db458827808.png) -## UAT +## Deployment -Note that the first time you run the PostgreSQL container it will have to initialise. -The server and worker cannot interact with it before this happens. - -Before first run: -``` -cd environment/uat - -# Create the database -docker-compose up db -# CTRL-C when "database system is ready to accept connections" message appears - -# Allow the API program to initialise the database schema -docker-compose up db api -# CTRL-C when "RetroPilot Server listening at" message appears -``` - -Launch with: -``` -cd environment/uat -docker-compose up -d -``` +For deployment instructions and configuration, take a look at the `environment` directory. diff --git a/environment/README.md b/environment/README.md new file mode 100644 index 0000000..259e755 --- /dev/null +++ b/environment/README.md @@ -0,0 +1,5 @@ +# Environments + +The subdirectories contain configuration for different environments. The services are managed using Docker Compose. + +Read the Docker Compose documentation [here](https://docs.docker.com/compose/). diff --git a/environment/development/.env.sample b/environment/development/.env.sample new file mode 100644 index 0000000..c895dd5 --- /dev/null +++ b/environment/development/.env.sample @@ -0,0 +1,61 @@ +NODE_ENV=development +APP_SALT=RANDOM_SEED +LOG_LEVEL=debug + +DB_NAME=retro-pilot +DB_USER=root +DB_PASS=root +# If using docker compose, this should match the container service name +DB_HOST=localhost +DB_PORT=5432 +# Whether or not to DROP all tables and recreate to match the current models +DB_FORCE_SYNC=false + +ALLOW_REGISTRATION=true +AUTH_2FA_ISSUER=RetroPilot + +HTTP_INTERFACE=0.0.0.0 +HTTP_PORT=8080 + +# Set to false to skip sending mail, all attempted mail is logged under DEBUG +CAN_SEND_MAIL=false +# credentials for smtp server to send account registration mails. if not filled in, get the generated tokens from the server.log manually +SMTP_HOST="localhost" +SMTP_PORT=25 +SMTP_USER=root +SMTP_PASS= +SMTP_FROM="no-reply@retropilot.org" + +# base url of the retropilot server +BASE_URL="http://192.168.1.165:8080/" +# base url sent to devices for POSTing drives & logs +BASE_UPLOAD_URL="http://192.168.1.165:8080/backend/post_upload" +# base download url for drive & log data +BASE_DRIVE_DOWNLOAD_URL="http://192.168.1.165:8080/realdata/" +# path mapping of above download url for expressjs, prefix with "/" +BASE_DRIVE_DOWNLOAD_PATH_MAPPING="/realdata" +# relative or absolute ( "/..." for absolute path ) +STORAGE_PATH="/realdata/" + +CABANA_URL="http://192.168.1.165:8080/cabana/index.html" + +DEVICE_STORAGE_QUOTA_MB=200000 +DEVICE_EXPIRATION_DAYS=30 + +WELCOME_MESSAGE="<><><><><><><><><><><><><><><><><><><><><><>
2022 RetroPilot" + +USE_USER_ADMIN_API=0 + +CLIENT_SOCKET_PORT=81 +CLIENT_SOCKET_HOST="0.0.0.0" + +# Enables Athena service +ATHENA_ENABLED=1 +# Disables crypto on Websocket server - use for testing on local network, change ATHENA_HOST in openpilot to ws:// instead of wss:// +ATHENA_SECURE=1 +# Maxmium hits to /realtime/* per 30s +ATHENA_API_RATE_LIMIT=100 +ATHENA_SOCKET_HOST="0.0.0.0" +ATHENA_SOCKET_PORT=4040 +# Higher the number = lower traffic, varies on how many devices are connected +ATHENA_SOCKET_HEARTBEAT_FREQ=5000 diff --git a/environment/development/README.md b/environment/development/README.md new file mode 100644 index 0000000..da90a08 --- /dev/null +++ b/environment/development/README.md @@ -0,0 +1,37 @@ +# Development environment + +Test in docker containers on your development machine + +## Usage + +### Environment + +Copy `.env.sample` to `.env` and edit as needed. + +``` +cp .env.sample .env +``` + +### Deployment + +Note that the first time you run the PostgreSQL container it will have to initialise. +The server and worker cannot interact with it before this happens. + +Before first run: +``` +cd environment/development + +# Create the database +docker-compose up db +# CTRL-C when "database system is ready to accept connections" message appears + +# Allow the API program to initialise the database schema +docker-compose up db api +# CTRL-C when "RetroPilot Server listening at" message appears +``` + +To start all the services: +``` +cd environment/development +docker-compose up -d +``` diff --git a/docker-compose.yml b/environment/development/docker-compose.yml similarity index 55% rename from docker-compose.yml rename to environment/development/docker-compose.yml index a3a4c4f..aadf25f 100644 --- a/docker-compose.yml +++ b/environment/development/docker-compose.yml @@ -1,42 +1,48 @@ version: "3.0" services: - server: - build: . - restart: unless-stopped - depends_on: - - db - volumes: - - retropilot-realdata:/realdata - ports: - - "8080:8080" - - "4040:4040" - env_file: - - .env - - .env.docker - worker: - build: . - command: npm run worker - restart: unless-stopped - depends_on: - - db - volumes: - - retropilot-realdata:/realdata - env_file: - - .env - - .env.docker + # PostgreSQL database db: image: postgres:14-bullseye restart: always ports: - "5432:5432" volumes: - - ./sql/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql - - retropilot-db:/var/lib/postgresql/data + - ./database:/var/lib/postgresql/data environment: POSTGRES_USER: ${DB_USER} POSTGRES_PASSWORD: ${DB_PASS} POSTGRES_DB: ${DB_NAME} -volumes: - retropilot-realdata: - retropilot-db: + # API server + api: + build: ../.. + restart: unless-stopped + depends_on: + - db + volumes: + - ./realdata:/realdata + ports: + - "8080:8080" + env_file: + - .env + + # API worker + worker: + build: ../.. + command: npm run worker + restart: unless-stopped + depends_on: + - db + volumes: + - ./realdata:/realdata + env_file: + - .env + + pgadmin: + image: dpage/pgadmin4 + restart: unless-stopped + ports: + - "5050:80" + environment: + PGADMIN_DEFAULT_EMAIL: admin@retropilot.org + PGADMIN_DEFAULT_PASSWORD: ${DB_PASS} diff --git a/environment/uat/.env.sample b/environment/uat/.env.sample new file mode 100644 index 0000000..e2e8126 --- /dev/null +++ b/environment/uat/.env.sample @@ -0,0 +1,61 @@ +NODE_ENV=development +APP_SALT=RANDOM_SEED +LOG_LEVEL=debug + +DB_NAME=retro-pilot +DB_USER=root +DB_PASS=root +# If using docker compose, this should match the container service name +DB_HOST=db +DB_PORT=5432 +# Whether or not to DROP all tables and recreate to match the current models +DB_FORCE_SYNC=false + +ALLOW_REGISTRATION=true +AUTH_2FA_ISSUER=RetroPilot + +HTTP_INTERFACE=0.0.0.0 +HTTP_PORT=8080 + +# Set to false to skip sending mail, all attempted mail is logged under DEBUG +CAN_SEND_MAIL=true +# credentials for smtp server to send account registration mails. if not filled in, get the generated tokens from the server.log manually +SMTP_HOST="localhost" +SMTP_PORT=25 +SMTP_USER=root +SMTP_PASS= +SMTP_FROM="no-reply@retropilot.org" + +# base url of the retropilot server +BASE_URL="https://uat.api.retropilot.org/" +# base url sent to devices for POSTing drives & logs +BASE_UPLOAD_URL="https://uat.api.retropilot.org/backend/post_upload" +# base download url for drive & log data +BASE_DRIVE_DOWNLOAD_URL="https://uat.api.retropilot.org/realdata/" +# path mapping of above download url for expressjs, prefix with "/" +BASE_DRIVE_DOWNLOAD_PATH_MAPPING="/realdata" +# relative or absolute ( "/..." for absolute path ) +STORAGE_PATH="/realdata/" + +CABANA_URL="https://uat.api.retropilot.org/cabana/index.html" + +DEVICE_STORAGE_QUOTA_MB=200000 +DEVICE_EXPIRATION_DAYS=30 + +WELCOME_MESSAGE="<><><><><><><><><><><><><><><><><><><><><><>
2022 RetroPilot" + +USE_USER_ADMIN_API=0 + +CLIENT_SOCKET_PORT=81 +CLIENT_SOCKET_HOST="0.0.0.0" + +# Enables Athena service +ATHENA_ENABLED=1 +# Disables crypto on Websocket server - use for testing on local network, change ATHENA_HOST in openpilot to ws:// instead of wss:// +ATHENA_SECURE=1 +# Maxmium hits to /realtime/* per 30s +ATHENA_API_RATE_LIMIT=100 +ATHENA_SOCKET_HOST="0.0.0.0" +ATHENA_SOCKET_PORT=4040 +# Higher the number = lower traffic, varies on how many devices are connected +ATHENA_SOCKET_HEARTBEAT_FREQ=5000 diff --git a/environment/uat/.gitignore b/environment/uat/.gitignore deleted file mode 100644 index 76075ce..0000000 --- a/environment/uat/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -database -realdata diff --git a/environment/uat/README.md b/environment/uat/README.md index 84ef801..5c60828 100644 --- a/environment/uat/README.md +++ b/environment/uat/README.md @@ -1,3 +1,37 @@ # User Acceptance Testing (UAT) environnment Hostname: `uat.api.retropilot.org` + +## Usage + +### Environment + +Copy `.env.sample` to `.env` and edit as needed. + +``` +cp .env.sample .env +``` + +### Deployment + +Note that the first time you run the PostgreSQL container it will have to initialise. +The server and worker cannot interact with it before this happens. + +Before first run: +``` +cd environment/uat + +# Create the database +docker-compose up db +# CTRL-C when "database system is ready to accept connections" message appears + +# Allow the API program to initialise the database schema +docker-compose up db api +# CTRL-C when "RetroPilot Server listening at" message appears +``` + +To start all the services: +``` +cd environment/uat +docker-compose up -d +``` diff --git a/environment/uat/docker-compose.yml b/environment/uat/docker-compose.yml index 3e82026..97140f2 100644 --- a/environment/uat/docker-compose.yml +++ b/environment/uat/docker-compose.yml @@ -9,13 +9,19 @@ services: ports: - "80:80" - "443:443" - - "127.0.0.1:8080:8080" volumes: - /var/run/docker.sock:/var/run/docker.sock # See traefik/traefik.toml for static config - ./traefik:/etc/traefik labels: - "traefik.enable=true" + # Expose traefik dashboard at http://localhost:8080 + - "traefik.http.routers.dashboard.rule=Host(`localhost`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))" + - "traefik.http.routers.dashboard.entrypoints=dashboard" + - "traefik.http.routers.dashboard.service=api@internal" + # Secure the dashboard with IPWhiteList middleware + - "traefik.http.routers.dashboard.middlewares=dashboard-allowlist@docker" + - "traefik.http.middlewares.dashboard-allowlist.ipwhitelist.sourcerange=127.0.0.1/32" # PostgreSQL database db: @@ -51,18 +57,18 @@ services: - "traefik.http.routers.api.tls=true" - "traefik.http.routers.api.tls.certresolver=retropilot" + # Athena service forwards requests to container port 4040 + - "traefik.http.services.athena.loadbalancer.server.port=4040" + # Expose athena at https://uat.athena.retropilot.org + - "traefik.http.routers.athena.rule=Host(`athena.uat.retropilot.org`)" + - "traefik.http.routers.athena.entrypoints=websecure" + - "traefik.http.routers.athena.service=athena@docker" + - "traefik.http.routers.athena.tls=true" + - "traefik.http.routers.athena.tls.certresolver=retropilot" + # Monitor for image updates and restart automatically - "com.centurylinklabs.watchtower.enable=true" -# # Athena service forwards requests to container port 4040 -# - "traefik.http.services.athena.loadbalancer.server.port=4040" -# # Expose athena at https://uat.athena.retropilot.org -# - "traefik.http.routers.athena.rule=Host(`athena.uat.retropilot.org`)" -# - "traefik.http.routers.athena.entrypoints=websecure" -# - "traefik.http.routers.athena.service=athena@docker" -# - "traefik.http.routers.athena.tls=true" -# - "traefik.http.routers.athena.tls.certresolver=retropilot" - # API worker worker: # Use the same image as the server diff --git a/environment/uat/traefik/traefik.toml b/environment/uat/traefik/traefik.toml index 8eb75b5..c953af2 100644 --- a/environment/uat/traefik/traefik.toml +++ b/environment/uat/traefik/traefik.toml @@ -3,7 +3,6 @@ [api] dashboard = true - insecure = true [log] level = "INFO" @@ -22,6 +21,8 @@ to = "websecure" [entryPoints.websecure] address = ":443" + [entryPoints.dashboard] + address = ":8080" [certificatesResolvers] [certificatesResolvers.retropilot] diff --git a/init-letsencrypt.sh b/init-letsencrypt.sh index 51b3d32..6bac7a0 100755 --- a/init-letsencrypt.sh +++ b/init-letsencrypt.sh @@ -1,5 +1,7 @@ #!/bin/bash +# NOTE: you don't need to run this script unless you are deploying without docker and traefik + if ! [ -x "$(command -v docker-compose)" ]; then echo 'Error: docker-compose is not installed.' >&2 exit 1