added new link to GPS status

ubxtool.cc: Use monotonic clock for uptime calculation.

.github/workflows/ccpp.yml

@@ -4,15 +4,15 @@ on: [push]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
-    
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
+      with:
+        submodules: "recursive"
     - name: deps
-      run: sudo apt-get install protobuf-compiler libh2o-dev libcurl4-openssl-dev libssl-dev libprotobuf-dev libh2o-evloop-dev libwslay-dev libncurses5-dev libeigen3-dev libzstd-dev
-    - name: submodules
-      run: git submodule update --init --recursive
+      run: |
+        sudo apt-get update
+        sudo apt-get install protobuf-compiler libh2o-dev libcurl4-openssl-dev libssl-dev libprotobuf-dev libh2o-evloop-dev libwslay-dev libncurses5-dev libeigen3-dev libzstd-dev
     - name: config
       run: echo WSLAY=-lwslay > Makefile.local
     - name: make

.github/workflows/docker.yml

@@ -0,0 +1,32 @@
+name: Build and publish Docker image
+
+on:
+  push:
+    branches: master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Checkout submodules
+        run: git submodule update --init --recursive
+      - name: Set up docker buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v3
+        with:
+          buildx-version: latest
+          qemu-version: latest
+      - name: Login to docker registry
+        run: |
+          docker login --username ${{ secrets.DOCKER_USERNAME }} --password ${{ secrets.DOCKER_TOKEN }}
+      - name: Run buildx
+        run: |
+          docker buildx build \
+            --tag berthubert/galmon \
+            --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 \
+            --output "type=registry" \
+            --build-arg MAKE_FLAGS=-j1 \
+            --file Dockerfile \
+            .

Dockerfile

@@ -1,25 +1,38 @@
-FROM ubuntu:disco
+#
+# First stage - builder
+#
+FROM debian:10-slim AS builder
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV LC_ALL C.UTF-8
 
-# This allows you to use a local Ubuntu mirror
-ARG APT_URL=
-ENV APT_URL ${APT_URL:-http://archive.ubuntu.com/ubuntu/}
-RUN sed -i "s%http://archive.ubuntu.com/ubuntu/%${APT_URL}%" /etc/apt/sources.list
+# This allows you to use a local Debian mirror
+ARG APT_URL=http://deb.debian.org/debian/
+ARG MAKE_FLAGS=-j2
 
-
-# Update packages and install dependencies
-RUN apt-get update && apt-get -y upgrade && apt-get -y clean
-RUN apt-get install -y protobuf-compiler libh2o-dev libcurl4-openssl-dev \
-        libssl-dev libprotobuf-dev libh2o-evloop-dev libwslay-dev libeigen3-dev libzstd-dev \
-	make gcc g++ git build-essential curl autoconf automake libfmt-dev libncurses5-dev \
-    && apt-get -y clean
+RUN sed -i "s%http://deb.debian.org/debian/%${APT_URL}%" /etc/apt/sources.list \
+    && apt-get update && apt-get -y upgrade \
+    && apt-get install -y protobuf-compiler libh2o-dev libcurl4-openssl-dev \
+           libssl-dev libprotobuf-dev libh2o-evloop-dev libwslay-dev \
+           libeigen3-dev libzstd-dev libfmt-dev libncurses-dev \
+           make gcc g++ git build-essential curl autoconf automake help2man
 
 # Build
-ARG MAKE_FLAGS=-j2
-ADD . /galmon/
-WORKDIR /galmon/
-RUN make $MAKE_FLAGS
-ENV PATH=/galmon:${PATH}
+ADD . /galmon-src/
+RUN cd /galmon-src/ \
+    && make $MAKE_FLAGS \
+    && prefix=/galmon make install
 
+#
+# Second stage - contains just the binaries
+#
+FROM debian:10-slim
+RUN apt-get update && apt-get -y upgrade \
+    && apt-get install -y libcurl4 libssl1.1 libprotobuf17 libh2o-evloop0.13 \
+           libncurses6 \
+    && apt-get -y clean \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /galmon/ /galmon/
+ENV PATH=/galmon/bin:${PATH}
+ENV LC_ALL C.UTF-8
+WORKDIR /galmon/bin

Dockerfile-pi

@@ -1,18 +0,0 @@
-#FROM ubuntu:disco
-FROM arm32v7/debian:unstable
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV LC_ALL C.UTF-8
-
-# Update packages and install dependencies
-RUN apt-get update && apt-get -y upgrade && apt-get -y clean
-RUN apt-get install -y protobuf-compiler libh2o-dev libcurl4-openssl-dev \
-        libssl-dev libprotobuf-dev libh2o-evloop-dev libwslay-dev libeigen3-dev libzstd-dev \
-	make clang-9 git build-essential curl autoconf automake libfmt-dev libncurses5-dev \
-    && apt-get -y clean
-
-# Build
-ADD . /galmon/
-WORKDIR /galmon/
-RUN make 
-ENV PATH=/galmon:${PATH}

Makefile

@@ -1,6 +1,6 @@
 CFLAGS = -O3 -Wall -ggdb 
 
-CXXFLAGS:= -std=gnu++17 -Wall -O2 -ggdb -MMD -MP -fno-omit-frame-pointer -Iext/CLI11 \
+CXXFLAGS:= -std=gnu++17 -Wall -O3 -ggdb -MMD -MP -fno-omit-frame-pointer -Iext/CLI11 \
 	 -Iext/fmt-6.1.2/include/ -Iext/powerblog/ext/simplesocket -Iext/powerblog/ext/ \
 	 -I/usr/local/opt/openssl/include/  \
 	 -Iext/sgp4/libsgp4/ \
@@ -25,7 +25,7 @@ endif
 CHEAT_ARG := $(shell ./update-git-hash-if-necessary)
 
 PROGRAMS = navparse ubxtool navnexus navcat navrecv navdump testrunner navdisplay tlecatch reporter sp3feed \
-	galmonmon rinreport rtcmtool
+	galmonmon rinreport rinjoin rtcmtool gndate septool navmerge
 
 all: navmon.pb.cc $(PROGRAMS)
 
@@ -76,10 +76,10 @@ download-raspbian-package:
 decrypt: decrypt.o bits.o ext/fmt-6.1.2/src/format.o
 	$(CXX) -std=gnu++17 $^ -o $@ 
 
-navparse: navparse.o ext/fmt-6.1.2/src/format.o $(H2OPP) $(SIMPLESOCKETS) minicurl.o ubx.o bits.o navmon.pb.o gps.o ephemeris.o beidou.o glonass.o $(patsubst %.cc,%.o,$(wildcard ext/sgp4/libsgp4/*.cc)) tle.o navmon.o coverage.o osen.o trkmeas.o influxpush.o ${EXTRADEP} githash.o sbas.o rtcm.o
+navparse: navparse.o ext/fmt-6.1.2/src/format.o $(H2OPP) $(SIMPLESOCKETS) minicurl.o ubx.o bits.o navmon.pb.o gps.o ephemeris.o beidou.o glonass.o $(patsubst %.cc,%.o,$(wildcard ext/sgp4/libsgp4/*.cc)) tle.o navmon.o coverage.o osen.o trkmeas.o influxpush.o ${EXTRADEP} githash.o sbas.o rtcm.o galileo.o
 	$(CXX) -std=gnu++17 $^ -o $@ -pthread -L/usr/local/lib -L/usr/local/opt/openssl/lib/  -lh2o-evloop -lssl -lcrypto -lz  -lcurl -lprotobuf  $(WSLAY)
 
-reporter: reporter.o ext/fmt-6.1.2/src/format.o $(SIMPLESOCKETS) minicurl.o ubx.o bits.o navmon.pb.o gps.o ephemeris.o beidou.o glonass.o $(patsubst %.cc,%.o,$(wildcard ext/sgp4/libsgp4/*.cc)) tle.o navmon.o coverage.o osen.o githash.o
+reporter: reporter.o ext/fmt-6.1.2/src/format.o $(SIMPLESOCKETS) minicurl.o ubx.o bits.o navmon.pb.o gps.o ephemeris.o beidou.o glonass.o $(patsubst %.cc,%.o,$(wildcard ext/sgp4/libsgp4/*.cc)) tle.o navmon.o coverage.o osen.o githash.o influxpush.o
 	$(CXX) -std=gnu++17 $^ -o $@ -pthread -L/usr/local/lib -lprotobuf -lcurl
 
 sp3feed: sp3feed.o ext/fmt-6.1.2/src/format.o $(SIMPLESOCKETS) minicurl.o ubx.o bits.o navmon.pb.o gps.o ephemeris.o beidou.o glonass.o $(patsubst %.cc,%.o,$(wildcard ext/sgp4/libsgp4/*.cc)) tle.o navmon.o coverage.o osen.o influxpush.o githash.o sp3.o
@@ -94,14 +94,14 @@ galmonmon: galmonmon.o ext/fmt-6.1.2/src/format.o $(SIMPLESOCKETS) minicurl.o ub
 	$(CXX) -std=gnu++17 $^ -o $@ -pthread -L/usr/local/lib -lprotobuf -lcurl
 
 
-navdump: navdump.o ext/fmt-6.1.2/src/format.o bits.o navmon.pb.o gps.o ephemeris.o beidou.o glonass.o navmon.o $(patsubst %.cc,%.o,$(wildcard ext/sgp4/libsgp4/*.cc)) tle.o sp3.o osen.o trkmeas.o githash.o rinex.o sbas.o rtcm.o ${EXTRADEP}
+navdump: navdump.o ext/fmt-6.1.2/src/format.o bits.o navmon.pb.o gps.o ephemeris.o beidou.o glonass.o navmon.o $(patsubst %.cc,%.o,$(wildcard ext/sgp4/libsgp4/*.cc)) tle.o sp3.o osen.o trkmeas.o githash.o rinex.o sbas.o rtcm.o galileo.o ${EXTRADEP}
 	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -pthread  -lprotobuf -lz
 
 navdisplay: navdisplay.o ext/fmt-6.1.2/src/format.o bits.o navmon.pb.o gps.o ephemeris.o beidou.o glonass.o ephemeris.o navmon.o osen.o githash.o
 	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -pthread  -lprotobuf -lncurses
 
 
-navnexus: navnexus.o ext/fmt-6.1.2/src/format.o  $(SIMPLESOCKETS) ubx.o bits.o navmon.pb.o storage.o githash.o
+navnexus: navnexus.o ext/fmt-6.1.2/src/format.o  $(SIMPLESOCKETS) bits.o navmon.pb.o storage.o githash.o
 	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -pthread -lprotobuf
 
 navcat: navcat.o ext/fmt-6.1.2/src/format.o  $(SIMPLESOCKETS) ubx.o bits.o navmon.pb.o storage.o navmon.o githash.o
@@ -111,12 +111,20 @@ navcat: navcat.o ext/fmt-6.1.2/src/format.o  $(SIMPLESOCKETS) ubx.o bits.o navmo
 navrecv: navrecv.o ext/fmt-6.1.2/src/format.o $(SIMPLESOCKETS) navmon.pb.o storage.o githash.o zstdwrap.o navmon.o
 	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -pthread -lprotobuf -lzstd 
 
+navmerge: navmerge.o ext/fmt-6.1.2/src/format.o $(SIMPLESOCKETS) navmon.pb.o storage.o githash.o zstdwrap.o navmon.o nmmsender.o
+	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -pthread -lprotobuf -lzstd 
+
+
 tlecatch: tlecatch.o $(patsubst %.cc,%.o,$(wildcard ext/sgp4/libsgp4/*.cc)) githash.o
 	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -pthread -lprotobuf  
 
 rinreport: rinreport.o rinex.o githash.o navmon.o ext/fmt-6.1.2/src/format.o  ephemeris.o osen.o
 	$(CXX) -std=gnu++17 $^ -o $@ -lz -pthread
 
+rinjoin: rinjoin.o rinex.o githash.o navmon.o ext/fmt-6.1.2/src/format.o  ephemeris.o osen.o
+	$(CXX) -std=gnu++17 $^ -o $@ -lz -pthread
+
+
 rtcmtool: rtcmtool.o navmon.pb.o githash.o ext/fmt-6.1.2/src/format.o  bits.o nmmsender.o $(SIMPLESOCKETS)  navmon.o rtcm.o zstdwrap.o
 	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -lz -pthread -lprotobuf -lzstd
 
@@ -124,8 +132,15 @@ rtcmtool: rtcmtool.o navmon.pb.o githash.o ext/fmt-6.1.2/src/format.o  bits.o nm
 ubxtool: navmon.pb.o ubxtool.o ubx.o bits.o ext/fmt-6.1.2/src/format.o galileo.o  gps.o beidou.o navmon.o ephemeris.o $(SIMPLESOCKETS) osen.o githash.o nmmsender.o zstdwrap.o 
 	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -lprotobuf -pthread -lzstd
 
+septool: navmon.pb.o septool.o bits.o ext/fmt-6.1.2/src/format.o galileo.o  gps.o beidou.o navmon.o ephemeris.o $(SIMPLESOCKETS) osen.o githash.o nmmsender.o zstdwrap.o 
+	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -lprotobuf -pthread -lzstd
+
+
 testrunner: navmon.pb.o testrunner.o ubx.o bits.o ext/fmt-6.1.2/src/format.o galileo.o  gps.o beidou.o ephemeris.o sp3.o osen.o navmon.o rinex.o githash.o
-	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -lprotobuf -lz 
+	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib -lprotobuf -lz  -pthread
+
+gndate: gndate.o githash.o ext/fmt-6.1.2/src/format.o navmon.o
+	$(CXX) -std=gnu++17 $^ -o $@ -L/usr/local/lib
 
 check: testrunner
 	./testrunner

README.md

@@ -4,14 +4,17 @@ galileo/GPS/GLONASS/BeiDou open source monitoring. GPL3 licensed.
 
 Live website: https://galmon.eu/
 
-Theoretically multi-vendor, although currently only the U-blox 8 and 9
-chipsets are supported.  Navilock NL-8012U receiver works really well, as
-does the U-blox evaluation kit for the 8MT.  In addition, many stations have
-reported success with this very cheap [AliExpress sourced
-device](https://www.aliexpress.com/item/32816656706.html).  The best and
-most high-end receiver, which does all bands, all the time, is the Ublox
-F9P, several of us use the
-[ArdusimpleRTK2B](https://www.ardusimple.com/simplertk2b/) board.
+Multi-vendor, with support for U-blox 8 and 9 chipsets and many Septentrio
+devices.  Navilock NL-8012U receiver works really well, as does the U-blox
+evaluation kit for the 8MT.  In addition, many stations have reported
+success with this very cheap [AliExpress sourced
+device](https://www.aliexpress.com/item/32816656706.html).  
+
+For ublox, there is good support for the F9P, several of us use the
+[ArdusimpleRTK2B](https://www.ardusimple.com/simplertk2b/) board. It adds
+the Galileo E5b band.
+
+Septentrio devices support even more bands.
 
 An annotated presentation about our project aimed at GNSS professionals can
 be found [here](https://berthub.eu/galileo/The%20galmon.eu%20project.pdf). 
@@ -29,9 +32,10 @@ guidelines](https://github.com/ahupowerdns/galmon/blob/master/Operator.md).
 Highlights
 ----------
 
+ * Support for Septentrio and U-blox.
  * Processes raw frames/strings/words from GPS, GLONASS, BeiDou and Galileo
- * All-band support (E1, E5b, B1I, B2I, Glonass L1, Glonass L2, GPS L1C/A)
-   so far, GPS L2C and Galileo E5a pending).
+ * All-band support (E1, E5a, E5b, B1I, B2I, Glonass L1, Glonass L2, GPS L1C/A)
+   so far.
  * Calculate ephemeris positions
  * Comparison of ephemerides to independent SP3 data to determine SISE
    * Globally, locally, worst user location
@@ -68,8 +72,8 @@ Goals:
 
 Works on Linux (including Raspbian Buster on Pi Zero W), OSX and OpenBSD.
 
-Build locally
--------------
+Build locally (Linux, Debian, Ubuntu)
+-------------------------------------
 
 To get started, make sure you have a C++17 compiler (like g++ 8 or higher),
 git, protobuf-compiler.  Then run 'make ubxtool navdump' to build the
@@ -79,7 +83,7 @@ To build everything, including the webserver, try:
 
 ```
 apt-get install protobuf-compiler libh2o-dev libcurl4-openssl-dev libssl-dev libprotobuf-dev \
-libh2o-evloop-dev libwslay-dev libncurses5-dev libeigen3-dev libzstd-dev
+libh2o-evloop-dev libwslay-dev libncurses5-dev libeigen3-dev libzstd-dev g++
 git clone https://github.com/ahupowerdns/galmon.git --recursive
 cd galmon
 make
@@ -92,29 +96,57 @@ library installed. If you get an error about 'wslay', do the following, and run
 echo WSLAY=-lwslay > Makefile.local
 ```
 
-Build in Docker
+Building on OSX
 ---------------
+With thanks to a contributor from Prague.  First make sure you've installed
+brew, which you can get [here](https://brew.sh/). Then do:
 
-To build it in Docker:
+```
+brew install protobuf lzlib zstd h2o eigen
+```
 
+And then:
 ```
 git clone https://github.com/ahupowerdns/galmon.git --recursive
-docker build -t galmon --build-arg MAKE_FLAGS=-j2 .
+cd galmon
+make
 ```
 
-To run a container with a shell in there (this will also expose a port so you can view the UI too and assumes a ublox GPS device too - you may need to tweak as necessary):
+Running in Docker
+-----------------
+
+We publish official Docker images for galmon on
+[docker hub](https://hub.docker.com/r/berthubert/galmon) for multiple architectures.
+
+To run a container with a shell in there (this will also expose a port so you
+can view the UI too and assumes a ublox GPS device too -
+you may need to tweak as necessary):
 
 ```
-docker run -it --rm --device=/dev/ttyACM0 -p 10000:10000 galmon
+docker run -it --rm --device=/dev/ttyACM0 -p 10000:10000 berthubert/galmon
 ```
 
+Running a daemonized docker container reporting data to a remote server
+might look like:
+
+```
+docker run -d --restart=always --device=/dev/ttyACM0 --name=galmon berthubert/galmon ubxtool --wait --port /dev/ttyACM0 --gps --galileo --glonass --destination [server] --station [station-id] --owner [owner]
+```
+
+To make your docker container update automatically you could use a tool such as
+[watchtower](https://containrrr.github.io/watchtower/).
 
 Running
 -------
-
+On u-blox: 
 Once compiled, run for example `./ubxtool --wait --port /dev/ttyACM0
 --station 1 --stdout --galileo | ./navparse --bind [::1]:10000`
 
+For Septentrio, try: `nc 192.168.1.1 29000 | ./septool --station x --stdout |
+./navparse --bind [::1]:10000`, assuming your Septentrio can be reached on
+192.168.1.1.1 and you have defined an SBF stream on port 29000. For more
+details, please see below.
+
 Next up, browse to http://[::1]:10000 (or try http://localhost:10000/ and
 you should be in business. ubxtool changes (non-permanently) the
 configuration of your u-blox receiver so it emits the required frames for
@@ -152,6 +184,10 @@ Tooling:
  * ubxtool: can configure a u-blox 8 chipset, parses its output & will
    convert it into a protbuf stream of GNSS NAV frames + metadata
    Adds 64-bit timestamps plus origin information to each message
+ * septool: ingests the Septentrio binary format (SBF) and converts it to our
+   protobuf format. Supports same protocol as ubxtool.
+ * rtcmtool: ingest ntripclient output, decodes RTCM messages and converts
+   them to our protobuf format
  * xtool: if you have another chipset, build something that extracts NAV
    frames & metadata. Not done yet.
  * navrecv: receives GNSS NAV frames and stores them on disk, split out per
@@ -222,6 +258,47 @@ This also works for `navparse` for the pretty website and influx storage, `nc 12
 if you have an influxdb running on localhost with a galileo database in there.
 The default URL is http://127.0.0.1:29599/ 
 
+Septentrio specifics
+--------------------
+Unlike `ubxtool`, our `septool` does not (re)configure your Septentrio
+device. Instead, the tool expects Septentrio Binary Format (SBF) on input,
+and that this stream includes at least the following messages:
+
+ * MeasEpoch
+ * PVTCartesian
+
+We currently parse and understand:
+ * GALRawFNAV
+ * GALRawINAV
+
+Support will be added soon for:
+ * GPSRawCA
+ * GPSRawL2C
+ * GPSRawL5
+ * GLORawCA
+ * BDSRaw
+ * BDSRawB1C
+ * BDSRawB2a
+
+A typical invocation of `septool` looks like this:
+
+```
+nc 192.168.1.1 29000 | ./septool --station x --destination galmon-eu-server.example.com
+```
+
+Or to test, try:
+
+```
+nc 192.168.1.1 29000 | ./septool --station x --stdout | ./navdump
+```
+
+This is assuming that you can reach your Septentrio on 192.168.1.1 and that
+you have defined a TCP server stream on port 29000. 
+
+Septool will also accept input from a serial port or basically anything that
+can provide SBF. Please let us know if our tooling can make your life
+easier.
+
 Internals
 ---------
 The transport format consists of repeats of:
@@ -268,17 +345,39 @@ The software can interpret SP3 files, good sources:
      to have less of a delay than the ESA ESM series.
    * GBU = ultra rapid, still a few days delay, but much more recent.
 
-Uncompress and concatenate all downloaded files into 'all.sp3' and run
-'navdump ' on collected protobuf, and it will output 'sp3.csv' with fit data.
+To get SP3 GBM from GFZ Potsdam for GPS week number 2111:
+
+```
+WN=2111
+lftp -c "mget ftp://ftp.gfz-potsdam.de/GNSS/products/mgnss/${WN}/gbm*sp3.Z"
+gunzip gbm*sp3.Z
+```
+
+To feed data, use: 
+
+```
+./sp3feed --sp3src=gbm --influxdb=galileo gbm*sp3
+```
+
+This will populate the sp3 tables in the database. A subsequent run of
+`reporter`, while setting the `--sp3src` parameter, will provide SP3
+deviation statistics, and fill out the `sp3delta` table in there, which
+stores deviations from the SP3 provided position, per sp3src.
+
+Further interesting (ephemeris) data is on http://mgex.igs.org/IGS_MGEX_Products.php
 
 RTCM
 ----
 RTCM is the Radio Technical Commission for Maritime Services, and
-confusingly, also the name of a protocol. This project can parse RTCM 10403.1 
-messages, and currently processes State Space Representation (SSR) messages,
-specifically types 1057/1240 (GPS/Galileo Orbit corrections to broadcast
-ephemeris) and 1058/1241 (GPS/Galileo Clock corrections to broadcast
-ephemeris).
+confusingly, also the name of a protocol. 
+
+This protocol is proprietary, but search for a file called `RTCM3.2.pdf` or
+`104-2013-SC104-STD - Vers. 3.2.docx` and you might find a copy. 
+
+This project can parse RTCM 10403.1 messages, and currently processes State
+Space Representation (SSR) messages, specifically types 1057/1240
+(GPS/Galileo Orbit corrections to broadcast ephemeris) and 1058/1241
+(GPS/Galileo Clock corrections to broadcast ephemeris).
 
 RTCM messages need to be converted to protobuf format, and the `rtcmtool` is
 provided for this purpose.
@@ -292,6 +391,17 @@ $ ntripclient ntrip:CLKA0_DEU1/user:password@navcast.spaceopal.com:2101 | ./rtcm
 User and password can be obtained from https://spaceopal.com/navcast/ - the
 Galileo operating company. 
 
+The IGS also offers excellent streams, but without Galileo. Information is
+[here](http://www.igs.org/rts/products). A typical commandline is:
+
+```
+$ ntripclient ntrip:IGS01/user:password@products.igs-ip.net:2101 | ./rtcmtool --station x --destination y
+```
+
+User and password can be requested through http://www.igs.org/rts/access
+
+An interesting list is here: http://products.igs-ip.net/
+
 There are many other sources of RTCM but currently not many offer the SSR
 messages we can use.
 

beidou.hh

@@ -228,7 +228,8 @@ struct BeidouMessage : GPSLikeEphemeris
 
   //                                            2^-30  2^-50
   int a0gps, a1gps, a0gal, a1gal, a0glo, a1glo, a0utc, a1utc;
-  int8_t deltaTLS;
+  int8_t deltaTLS, deltaTLSF;
+  uint8_t wnLSF, dn;
 
   // in Beidou the offset is a0utc + SOW * a1utc
   std::pair<double, double> getUTCOffset(int tow) const
@@ -268,6 +269,9 @@ struct BeidouMessage : GPSLikeEphemeris
       a0utc = bbits(91, 32);
       a1utc = bbits(131, 24);
       deltaTLS = bbits(31+12+1+7, 8);
+      deltaTLSF = bbits(61+6, 8);
+      wnLSF = bbits(61+6+8, 8);
+      dn = bbits(151+12, 8);
     }
     else {
       alma.sqrtA = bbitu(51, 24);

coverage.cc

@@ -87,6 +87,7 @@ covmap_t emitCoverage(const vector<Point>& sats)
     double phi = M_PI* latitude / 180;
     double longsteps = 1 + 360.0 * cos(phi);
     double step = 4*180.0 / longsteps;
+    // this does sorta equi-distanced measurements
     vector<tuple<double, int, int, int, double, double, double, double, double, double,double, double, double>> latvect;
     for(double longitude = -180; longitude < 180; longitude += step) { // east - west
       Point p;

ext/CLI11/CLI/CLI.hpp

@@ -1,16 +1,36 @@
-#pragma once
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
 
-// Distributed under the 3-Clause BSD License.  See accompanying
-// file LICENSE or https://github.com/CLIUtils/CLI11 for details.
+#pragma once
 
 // CLI Library includes
 // Order is important for combiner script
 
-#include "CLI/Error.hpp"
-#include "CLI/TypeTools.hpp"
-#include "CLI/StringTools.hpp"
-#include "CLI/Split.hpp"
-#include "CLI/Ini.hpp"
-#include "CLI/Validators.hpp"
-#include "CLI/Option.hpp"
-#include "CLI/App.hpp"
+#include "Version.hpp"
+
+#include "Macros.hpp"
+
+#include "StringTools.hpp"
+
+#include "Error.hpp"
+
+#include "TypeTools.hpp"
+
+#include "Split.hpp"
+
+#include "ConfigFwd.hpp"
+
+#include "Validators.hpp"
+
+#include "FormatterFwd.hpp"
+
+#include "Option.hpp"
+
+#include "App.hpp"
+
+#include "Config.hpp"
+
+#include "Formatter.hpp"

ext/CLI11/CLI/Config.hpp

@@ -0,0 +1,346 @@
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "App.hpp"
+#include "ConfigFwd.hpp"
+#include "StringTools.hpp"
+
+namespace CLI {
+
+namespace detail {
+
+inline std::string convert_arg_for_ini(const std::string &arg) {
+    if(arg.empty()) {
+        return std::string(2, '"');
+    }
+    // some specifically supported strings
+    if(arg == "true" || arg == "false" || arg == "nan" || arg == "inf") {
+        return arg;
+    }
+    // floating point conversion can convert some hex codes, but don't try that here
+    if(arg.compare(0, 2, "0x") != 0 && arg.compare(0, 2, "0X") != 0) {
+        double val;
+        if(detail::lexical_cast(arg, val)) {
+            return arg;
+        }
+    }
+    // just quote a single non numeric character
+    if(arg.size() == 1) {
+        return std::string("'") + arg + '\'';
+    }
+    // handle hex, binary or octal arguments
+    if(arg.front() == '0') {
+        if(arg[1] == 'x') {
+            if(std::all_of(arg.begin() + 2, arg.end(), [](char x) {
+                   return (x >= '0' && x <= '9') || (x >= 'A' && x <= 'F') || (x >= 'a' && x <= 'f');
+               })) {
+                return arg;
+            }
+        } else if(arg[1] == 'o') {
+            if(std::all_of(arg.begin() + 2, arg.end(), [](char x) { return (x >= '0' && x <= '7'); })) {
+                return arg;
+            }
+        } else if(arg[1] == 'b') {
+            if(std::all_of(arg.begin() + 2, arg.end(), [](char x) { return (x == '0' || x == '1'); })) {
+                return arg;
+            }
+        }
+    }
+    if(arg.find_first_of('"') == std::string::npos) {
+        return std::string("\"") + arg + '"';
+    } else {
+        return std::string("'") + arg + '\'';
+    }
+}
+
+/// Comma separated join, adds quotes if needed
+inline std::string
+ini_join(const std::vector<std::string> &args, char sepChar = ',', char arrayStart = '[', char arrayEnd = ']') {
+    std::string joined;
+    if(args.size() > 1 && arrayStart != '\0') {
+        joined.push_back(arrayStart);
+    }
+    std::size_t start = 0;
+    for(const auto &arg : args) {
+        if(start++ > 0) {
+            joined.push_back(sepChar);
+            if(isspace(sepChar) == 0) {
+                joined.push_back(' ');
+            }
+        }
+        joined.append(convert_arg_for_ini(arg));
+    }
+    if(args.size() > 1 && arrayEnd != '\0') {
+        joined.push_back(arrayEnd);
+    }
+    return joined;
+}
+
+inline std::vector<std::string> generate_parents(const std::string &section, std::string &name) {
+    std::vector<std::string> parents;
+    if(detail::to_lower(section) != "default") {
+        if(section.find('.') != std::string::npos) {
+            parents = detail::split(section, '.');
+        } else {
+            parents = {section};
+        }
+    }
+    if(name.find('.') != std::string::npos) {
+        std::vector<std::string> plist = detail::split(name, '.');
+        name = plist.back();
+        detail::remove_quotes(name);
+        plist.pop_back();
+        parents.insert(parents.end(), plist.begin(), plist.end());
+    }
+
+    // clean up quotes on the parents
+    for(auto &parent : parents) {
+        detail::remove_quotes(parent);
+    }
+    return parents;
+}
+
+/// assuming non default segments do a check on the close and open of the segments in a configItem structure
+inline void checkParentSegments(std::vector<ConfigItem> &output, const std::string &currentSection) {
+
+    std::string estring;
+    auto parents = detail::generate_parents(currentSection, estring);
+    if(!output.empty() && output.back().name == "--") {
+        std::size_t msize = (parents.size() > 1U) ? parents.size() : 2;
+        while(output.back().parents.size() >= msize) {
+            output.push_back(output.back());
+            output.back().parents.pop_back();
+        }
+
+        if(parents.size() > 1) {
+            std::size_t common = 0;
+            std::size_t mpair = (std::min)(output.back().parents.size(), parents.size() - 1);
+            for(std::size_t ii = 0; ii < mpair; ++ii) {
+                if(output.back().parents[ii] != parents[ii]) {
+                    break;
+                }
+                ++common;
+            }
+            if(common == mpair) {
+                output.pop_back();
+            } else {
+                while(output.back().parents.size() > common + 1) {
+                    output.push_back(output.back());
+                    output.back().parents.pop_back();
+                }
+            }
+            for(std::size_t ii = common; ii < parents.size() - 1; ++ii) {
+                output.emplace_back();
+                output.back().parents.assign(parents.begin(), parents.begin() + static_cast<std::ptrdiff_t>(ii) + 1);
+                output.back().name = "++";
+            }
+        }
+    } else if(parents.size() > 1) {
+        for(std::size_t ii = 0; ii < parents.size() - 1; ++ii) {
+            output.emplace_back();
+            output.back().parents.assign(parents.begin(), parents.begin() + static_cast<std::ptrdiff_t>(ii) + 1);
+            output.back().name = "++";
+        }
+    }
+
+    // insert a section end which is just an empty items_buffer
+    output.emplace_back();
+    output.back().parents = std::move(parents);
+    output.back().name = "++";
+}
+}  // namespace detail
+
+inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) const {
+    std::string line;
+    std::string section = "default";
+
+    std::vector<ConfigItem> output;
+    bool defaultArray = (arrayStart == '\0' || arrayStart == ' ') && arrayStart == arrayEnd;
+    char aStart = (defaultArray) ? '[' : arrayStart;
+    char aEnd = (defaultArray) ? ']' : arrayEnd;
+    char aSep = (defaultArray && arraySeparator == ' ') ? ',' : arraySeparator;
+
+    while(getline(input, line)) {
+        std::vector<std::string> items_buffer;
+        std::string name;
+
+        detail::trim(line);
+        std::size_t len = line.length();
+        if(len > 1 && line.front() == '[' && line.back() == ']') {
+            if(section != "default") {
+                // insert a section end which is just an empty items_buffer
+                output.emplace_back();
+                output.back().parents = detail::generate_parents(section, name);
+                output.back().name = "--";
+            }
+            section = line.substr(1, len - 2);
+            // deal with double brackets for TOML
+            if(section.size() > 1 && section.front() == '[' && section.back() == ']') {
+                section = section.substr(1, section.size() - 2);
+            }
+            if(detail::to_lower(section) == "default") {
+                section = "default";
+            } else {
+                detail::checkParentSegments(output, section);
+            }
+            continue;
+        }
+        if(len == 0) {
+            continue;
+        }
+        // comment lines
+        if(line.front() == ';' || line.front() == '#' || line.front() == commentChar) {
+            continue;
+        }
+
+        // Find = in string, split and recombine
+        auto pos = line.find(valueDelimiter);
+        if(pos != std::string::npos) {
+            name = detail::trim_copy(line.substr(0, pos));
+            std::string item = detail::trim_copy(line.substr(pos + 1));
+            if(item.size() > 1 && item.front() == aStart && item.back() == aEnd) {
+                items_buffer = detail::split_up(item.substr(1, item.length() - 2), aSep);
+            } else if(defaultArray && item.find_first_of(aSep) != std::string::npos) {
+                items_buffer = detail::split_up(item, aSep);
+            } else if(defaultArray && item.find_first_of(' ') != std::string::npos) {
+                items_buffer = detail::split_up(item);
+            } else {
+                items_buffer = {item};
+            }
+        } else {
+            name = detail::trim_copy(line);
+            items_buffer = {"true"};
+        }
+        if(name.find('.') == std::string::npos) {
+            detail::remove_quotes(name);
+        }
+        // clean up quotes on the items
+        for(auto &it : items_buffer) {
+            detail::remove_quotes(it);
+        }
+
+        std::vector<std::string> parents = detail::generate_parents(section, name);
+
+        if(!output.empty() && name == output.back().name && parents == output.back().parents) {
+            output.back().inputs.insert(output.back().inputs.end(), items_buffer.begin(), items_buffer.end());
+        } else {
+            output.emplace_back();
+            output.back().parents = std::move(parents);
+            output.back().name = std::move(name);
+            output.back().inputs = std::move(items_buffer);
+        }
+    }
+    if(section != "default") {
+        // insert a section end which is just an empty items_buffer
+        std::string ename;
+        output.emplace_back();
+        output.back().parents = detail::generate_parents(section, ename);
+        output.back().name = "--";
+        while(output.back().parents.size() > 1) {
+            output.push_back(output.back());
+            output.back().parents.pop_back();
+        }
+    }
+    return output;
+}
+
+inline std::string
+ConfigBase::to_config(const App *app, bool default_also, bool write_description, std::string prefix) const {
+    std::stringstream out;
+    std::string commentLead;
+    commentLead.push_back(commentChar);
+    commentLead.push_back(' ');
+
+    std::vector<std::string> groups = app->get_groups();
+    bool defaultUsed = false;
+    groups.insert(groups.begin(), std::string("Options"));
+    if(write_description) {
+        out << commentLead << app->get_description() << '\n';
+    }
+    for(auto &group : groups) {
+        if(group == "Options" || group.empty()) {
+            if(defaultUsed) {
+                continue;
+            }
+            defaultUsed = true;
+        }
+        if(write_description && group != "Options" && !group.empty()) {
+            out << '\n' << commentLead << group << " Options\n";
+        }
+        for(const Option *opt : app->get_options({})) {
+
+            // Only process option with a long-name and configurable
+            if(!opt->get_lnames().empty() && opt->get_configurable()) {
+                if(opt->get_group() != group) {
+                    if(!(group == "Options" && opt->get_group().empty())) {
+                        continue;
+                    }
+                }
+                std::string name = prefix + opt->get_lnames()[0];
+                std::string value = detail::ini_join(opt->reduced_results(), arraySeparator, arrayStart, arrayEnd);
+
+                if(value.empty() && default_also) {
+                    if(!opt->get_default_str().empty()) {
+                        value = detail::convert_arg_for_ini(opt->get_default_str());
+                    } else if(opt->get_expected_min() == 0) {
+                        value = "false";
+                    }
+                }
+
+                if(!value.empty()) {
+                    if(write_description && opt->has_description()) {
+                        out << '\n';
+                        out << commentLead << detail::fix_newlines(commentLead, opt->get_description()) << '\n';
+                    }
+                    out << name << valueDelimiter << value << '\n';
+                }
+            }
+        }
+    }
+    auto subcommands = app->get_subcommands({});
+    for(const App *subcom : subcommands) {
+        if(subcom->get_name().empty()) {
+            if(write_description && !subcom->get_group().empty()) {
+                out << '\n' << commentLead << subcom->get_group() << " Options\n";
+            }
+            out << to_config(subcom, default_also, write_description, prefix);
+        }
+    }
+
+    for(const App *subcom : subcommands) {
+        if(!subcom->get_name().empty()) {
+            if(subcom->get_configurable() && app->got_subcommand(subcom)) {
+                if(!prefix.empty() || app->get_parent() == nullptr) {
+                    out << '[' << prefix << subcom->get_name() << "]\n";
+                } else {
+                    std::string subname = app->get_name() + "." + subcom->get_name();
+                    auto p = app->get_parent();
+                    while(p->get_parent() != nullptr) {
+                        subname = p->get_name() + "." + subname;
+                        p = p->get_parent();
+                    }
+                    out << '[' << subname << "]\n";
+                }
+                out << to_config(subcom, default_also, write_description, "");
+            } else {
+                out << to_config(subcom, default_also, write_description, prefix + subcom->get_name() + ".");
+            }
+        }
+    }
+
+    return out.str();
+}
+
+}  // namespace CLI

ext/CLI11/CLI/ConfigFwd.hpp

@@ -0,0 +1,131 @@
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "Error.hpp"
+#include "StringTools.hpp"
+
+namespace CLI {
+
+class App;
+
+/// Holds values to load into Options
+struct ConfigItem {
+    /// This is the list of parents
+    std::vector<std::string> parents{};
+
+    /// This is the name
+    std::string name{};
+
+    /// Listing of inputs
+    std::vector<std::string> inputs{};
+
+    /// The list of parents and name joined by "."
+    std::string fullname() const {
+        std::vector<std::string> tmp = parents;
+        tmp.emplace_back(name);
+        return detail::join(tmp, ".");
+    }
+};
+
+/// This class provides a converter for configuration files.
+class Config {
+  protected:
+    std::vector<ConfigItem> items{};
+
+  public:
+    /// Convert an app into a configuration
+    virtual std::string to_config(const App *, bool, bool, std::string) const = 0;
+
+    /// Convert a configuration into an app
+    virtual std::vector<ConfigItem> from_config(std::istream &) const = 0;
+
+    /// Get a flag value
+    virtual std::string to_flag(const ConfigItem &item) const {
+        if(item.inputs.size() == 1) {
+            return item.inputs.at(0);
+        }
+        throw ConversionError::TooManyInputsFlag(item.fullname());
+    }
+
+    /// Parse a config file, throw an error (ParseError:ConfigParseError or FileError) on failure
+    std::vector<ConfigItem> from_file(const std::string &name) {
+        std::ifstream input{name};
+        if(!input.good())
+            throw FileError::Missing(name);
+
+        return from_config(input);
+    }
+
+    /// Virtual destructor
+    virtual ~Config() = default;
+};
+
+/// This converter works with INI/TOML files; to write proper TOML files use ConfigTOML
+class ConfigBase : public Config {
+  protected:
+    /// the character used for comments
+    char commentChar = ';';
+    /// the character used to start an array '\0' is a default to not use
+    char arrayStart = '\0';
+    /// the character used to end an array '\0' is a default to not use
+    char arrayEnd = '\0';
+    /// the character used to separate elements in an array
+    char arraySeparator = ' ';
+    /// the character used separate the name from the value
+    char valueDelimiter = '=';
+
+  public:
+    std::string
+    to_config(const App * /*app*/, bool default_also, bool write_description, std::string prefix) const override;
+
+    std::vector<ConfigItem> from_config(std::istream &input) const override;
+    /// Specify the configuration for comment characters
+    ConfigBase *comment(char cchar) {
+        commentChar = cchar;
+        return this;
+    }
+    /// Specify the start and end characters for an array
+    ConfigBase *arrayBounds(char aStart, char aEnd) {
+        arrayStart = aStart;
+        arrayEnd = aEnd;
+        return this;
+    }
+    /// Specify the delimiter character for an array
+    ConfigBase *arrayDelimiter(char aSep) {
+        arraySeparator = aSep;
+        return this;
+    }
+    /// Specify the delimiter between a name and value
+    ConfigBase *valueSeparator(char vSep) {
+        valueDelimiter = vSep;
+        return this;
+    }
+};
+
+/// the default Config is the INI file format
+using ConfigINI = ConfigBase;
+
+/// ConfigTOML generates a TOML compliant output
+class ConfigTOML : public ConfigINI {
+
+  public:
+    ConfigTOML() {
+        commentChar = '#';
+        arrayStart = '[';
+        arrayEnd = ']';
+        arraySeparator = ',';
+        valueDelimiter = '=';
+    }
+};
+}  // namespace CLI

ext/CLI11/CLI/Error.hpp

@@ -1,14 +1,38 @@
-#pragma once
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
 
-// Distributed under the 3-Clause BSD License.  See accompanying
-// file LICENSE or https://github.com/CLIUtils/CLI11 for details.
+#pragma once
 
 #include <exception>
 #include <stdexcept>
 #include <string>
+#include <utility>
+#include <vector>
+
+// CLI library includes
+#include "StringTools.hpp"
 
 namespace CLI {
 
+// Use one of these on all error classes.
+// These are temporary and are undef'd at the end of this file.
+#define CLI11_ERROR_DEF(parent, name)                                                                                  \
+  protected:                                                                                                           \
+    name(std::string ename, std::string msg, int exit_code) : parent(std::move(ename), std::move(msg), exit_code) {}   \
+    name(std::string ename, std::string msg, ExitCodes exit_code)                                                      \
+        : parent(std::move(ename), std::move(msg), exit_code) {}                                                       \
+                                                                                                                       \
+  public:                                                                                                              \
+    name(std::string msg, ExitCodes exit_code) : parent(#name, std::move(msg), exit_code) {}                           \
+    name(std::string msg, int exit_code) : parent(#name, std::move(msg), exit_code) {}
+
+// This is added after the one above if a class is used directly and builds its own message
+#define CLI11_ERROR_SIMPLE(name)                                                                                       \
+    explicit name(std::string msg) : name(#name, msg, ExitCodes::name) {}
+
 /// These codes are part of every error in CLI. They can be obtained from e using e.exit_code or as a quick shortcut,
 /// int values from e.get_error_code().
 enum class ExitCodes {
@@ -16,18 +40,19 @@ enum class ExitCodes {
     IncorrectConstruction = 100,
     BadNameString,
     OptionAlreadyAdded,
-    File,
-    Conversion,
-    Validation,
-    Required,
-    Requires,
-    Excludes,
-    Extras,
-    ExtrasINI,
-    Invalid,
-    Horrible,
+    FileError,
+    ConversionError,
+    ValidationError,
+    RequiredError,
+    RequiresError,
+    ExcludesError,
+    ExtrasError,
+    ConfigError,
+    InvalidError,
+    HorribleError,
     OptionNotFound,
-    BaseClass = 255
+    ArgumentMismatch,
+    BaseClass = 127
 };
 
 // Error definitions
@@ -39,127 +64,277 @@ enum class ExitCodes {
 /// @{
 
 /// All errors derive from this one
-struct Error : public std::runtime_error {
-    int exit_code;
-    bool print_help;
-    int get_exit_code() const { return exit_code; }
-    Error(std::string parent, std::string name, ExitCodes exit_code = ExitCodes::BaseClass, bool print_help = true)
-        : runtime_error(parent + ": " + name), exit_code(static_cast<int>(exit_code)), print_help(print_help) {}
-    Error(std::string parent,
-          std::string name,
-          int exit_code = static_cast<int>(ExitCodes::BaseClass),
-          bool print_help = true)
-        : runtime_error(parent + ": " + name), exit_code(exit_code), print_help(print_help) {}
+class Error : public std::runtime_error {
+    int actual_exit_code;
+    std::string error_name{"Error"};
+
+  public:
+    int get_exit_code() const { return actual_exit_code; }
+
+    std::string get_name() const { return error_name; }
+
+    Error(std::string name, std::string msg, int exit_code = static_cast<int>(ExitCodes::BaseClass))
+        : runtime_error(msg), actual_exit_code(exit_code), error_name(std::move(name)) {}
+
+    Error(std::string name, std::string msg, ExitCodes exit_code) : Error(name, msg, static_cast<int>(exit_code)) {}
 };
 
+// Note: Using Error::Error constructors does not work on GCC 4.7
+
 /// Construction errors (not in parsing)
-struct ConstructionError : public Error {
-    // Using Error::Error constructors seem to not work on GCC 4.7
-    ConstructionError(std::string parent,
-                      std::string name,
-                      ExitCodes exit_code = ExitCodes::BaseClass,
-                      bool print_help = true)
-        : Error(parent, name, exit_code, print_help) {}
+class ConstructionError : public Error {
+    CLI11_ERROR_DEF(Error, ConstructionError)
 };
 
 /// Thrown when an option is set to conflicting values (non-vector and multi args, for example)
-struct IncorrectConstruction : public ConstructionError {
-    IncorrectConstruction(std::string name)
-        : ConstructionError("IncorrectConstruction", name, ExitCodes::IncorrectConstruction) {}
+class IncorrectConstruction : public ConstructionError {
+    CLI11_ERROR_DEF(ConstructionError, IncorrectConstruction)
+    CLI11_ERROR_SIMPLE(IncorrectConstruction)
+    static IncorrectConstruction PositionalFlag(std::string name) {
+        return IncorrectConstruction(name + ": Flags cannot be positional");
+    }
+    static IncorrectConstruction Set0Opt(std::string name) {
+        return IncorrectConstruction(name + ": Cannot set 0 expected, use a flag instead");
+    }
+    static IncorrectConstruction SetFlag(std::string name) {
+        return IncorrectConstruction(name + ": Cannot set an expected number for flags");
+    }
+    static IncorrectConstruction ChangeNotVector(std::string name) {
+        return IncorrectConstruction(name + ": You can only change the expected arguments for vectors");
+    }
+    static IncorrectConstruction AfterMultiOpt(std::string name) {
+        return IncorrectConstruction(
+            name + ": You can't change expected arguments after you've changed the multi option policy!");
+    }
+    static IncorrectConstruction MissingOption(std::string name) {
+        return IncorrectConstruction("Option " + name + " is not defined");
+    }
+    static IncorrectConstruction MultiOptionPolicy(std::string name) {
+        return IncorrectConstruction(name + ": multi_option_policy only works for flags and exact value options");
+    }
 };
 
 /// Thrown on construction of a bad name
-struct BadNameString : public ConstructionError {
-    BadNameString(std::string name) : ConstructionError("BadNameString", name, ExitCodes::BadNameString) {}
+class BadNameString : public ConstructionError {
+    CLI11_ERROR_DEF(ConstructionError, BadNameString)
+    CLI11_ERROR_SIMPLE(BadNameString)
+    static BadNameString OneCharName(std::string name) { return BadNameString("Invalid one char name: " + name); }
+    static BadNameString BadLongName(std::string name) { return BadNameString("Bad long name: " + name); }
+    static BadNameString DashesOnly(std::string name) {
+        return BadNameString("Must have a name, not just dashes: " + name);
+    }
+    static BadNameString MultiPositionalNames(std::string name) {
+        return BadNameString("Only one positional name allowed, remove: " + name);
+    }
 };
 
 /// Thrown when an option already exists
-struct OptionAlreadyAdded : public ConstructionError {
-    OptionAlreadyAdded(std::string name)
-        : ConstructionError("OptionAlreadyAdded", name, ExitCodes::OptionAlreadyAdded) {}
+class OptionAlreadyAdded : public ConstructionError {
+    CLI11_ERROR_DEF(ConstructionError, OptionAlreadyAdded)
+    explicit OptionAlreadyAdded(std::string name)
+        : OptionAlreadyAdded(name + " is already added", ExitCodes::OptionAlreadyAdded) {}
+    static OptionAlreadyAdded Requires(std::string name, std::string other) {
+        return OptionAlreadyAdded(name + " requires " + other, ExitCodes::OptionAlreadyAdded);
+    }
+    static OptionAlreadyAdded Excludes(std::string name, std::string other) {
+        return OptionAlreadyAdded(name + " excludes " + other, ExitCodes::OptionAlreadyAdded);
+    }
 };
 
 // Parsing errors
 
 /// Anything that can error in Parse
-struct ParseError : public Error {
-    ParseError(std::string parent, std::string name, ExitCodes exit_code = ExitCodes::BaseClass, bool print_help = true)
-        : Error(parent, name, exit_code, print_help) {}
+class ParseError : public Error {
+    CLI11_ERROR_DEF(Error, ParseError)
 };
 
 // Not really "errors"
 
 /// This is a successful completion on parsing, supposed to exit
-struct Success : public ParseError {
-    Success() : ParseError("Success", "Successfully completed, should be caught and quit", ExitCodes::Success, false) {}
+class Success : public ParseError {
+    CLI11_ERROR_DEF(ParseError, Success)
+    Success() : Success("Successfully completed, should be caught and quit", ExitCodes::Success) {}
 };
 
 /// -h or --help on command line
-struct CallForHelp : public ParseError {
-    CallForHelp()
-        : ParseError("CallForHelp", "This should be caught in your main function, see examples", ExitCodes::Success) {}
+class CallForHelp : public ParseError {
+    CLI11_ERROR_DEF(ParseError, CallForHelp)
+    CallForHelp() : CallForHelp("This should be caught in your main function, see examples", ExitCodes::Success) {}
+};
+
+/// Usually something like --help-all on command line
+class CallForAllHelp : public ParseError {
+    CLI11_ERROR_DEF(ParseError, CallForAllHelp)
+    CallForAllHelp()
+        : CallForAllHelp("This should be caught in your main function, see examples", ExitCodes::Success) {}
+};
+
+/// Does not output a diagnostic in CLI11_PARSE, but allows to return from main() with a specific error code.
+class RuntimeError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, RuntimeError)
+    explicit RuntimeError(int exit_code = 1) : RuntimeError("Runtime error", exit_code) {}
 };
 
 /// Thrown when parsing an INI file and it is missing
-struct FileError : public ParseError {
-    FileError(std::string name) : ParseError("FileError", name, ExitCodes::File) {}
+class FileError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, FileError)
+    CLI11_ERROR_SIMPLE(FileError)
+    static FileError Missing(std::string name) { return FileError(name + " was not readable (missing?)"); }
 };
 
-/// Thrown when conversion call back fails, such as when an int fails to coerse to a string
-struct ConversionError : public ParseError {
-    ConversionError(std::string name) : ParseError("ConversionError", name, ExitCodes::Conversion) {}
+/// Thrown when conversion call back fails, such as when an int fails to coerce to a string
+class ConversionError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, ConversionError)
+    CLI11_ERROR_SIMPLE(ConversionError)
+    ConversionError(std::string member, std::string name)
+        : ConversionError("The value " + member + " is not an allowed value for " + name) {}
+    ConversionError(std::string name, std::vector<std::string> results)
+        : ConversionError("Could not convert: " + name + " = " + detail::join(results)) {}
+    static ConversionError TooManyInputsFlag(std::string name) {
+        return ConversionError(name + ": too many inputs for a flag");
+    }
+    static ConversionError TrueFalse(std::string name) {
+        return ConversionError(name + ": Should be true/false or a number");
+    }
 };
 
 /// Thrown when validation of results fails
-struct ValidationError : public ParseError {
-    ValidationError(std::string name) : ParseError("ValidationError", name, ExitCodes::Validation) {}
+class ValidationError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, ValidationError)
+    CLI11_ERROR_SIMPLE(ValidationError)
+    explicit ValidationError(std::string name, std::string msg) : ValidationError(name + ": " + msg) {}
 };
 
 /// Thrown when a required option is missing
-struct RequiredError : public ParseError {
-    RequiredError(std::string name) : ParseError("RequiredError", name, ExitCodes::Required) {}
+class RequiredError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, RequiredError)
+    explicit RequiredError(std::string name) : RequiredError(name + " is required", ExitCodes::RequiredError) {}
+    static RequiredError Subcommand(std::size_t min_subcom) {
+        if(min_subcom == 1) {
+            return RequiredError("A subcommand");
+        }
+        return RequiredError("Requires at least " + std::to_string(min_subcom) + " subcommands",
+                             ExitCodes::RequiredError);
+    }
+    static RequiredError
+    Option(std::size_t min_option, std::size_t max_option, std::size_t used, const std::string &option_list) {
+        if((min_option == 1) && (max_option == 1) && (used == 0))
+            return RequiredError("Exactly 1 option from [" + option_list + "]");
+        if((min_option == 1) && (max_option == 1) && (used > 1)) {
+            return RequiredError("Exactly 1 option from [" + option_list + "] is required and " + std::to_string(used) +
+                                     " were given",
+                                 ExitCodes::RequiredError);
+        }
+        if((min_option == 1) && (used == 0))
+            return RequiredError("At least 1 option from [" + option_list + "]");
+        if(used < min_option) {
+            return RequiredError("Requires at least " + std::to_string(min_option) + " options used and only " +
+                                     std::to_string(used) + "were given from [" + option_list + "]",
+                                 ExitCodes::RequiredError);
+        }
+        if(max_option == 1)
+            return RequiredError("Requires at most 1 options be given from [" + option_list + "]",
+                                 ExitCodes::RequiredError);
+
+        return RequiredError("Requires at most " + std::to_string(max_option) + " options be used and " +
+                                 std::to_string(used) + "were given from [" + option_list + "]",
+                             ExitCodes::RequiredError);
+    }
+};
+
+/// Thrown when the wrong number of arguments has been received
+class ArgumentMismatch : public ParseError {
+    CLI11_ERROR_DEF(ParseError, ArgumentMismatch)
+    CLI11_ERROR_SIMPLE(ArgumentMismatch)
+    ArgumentMismatch(std::string name, int expected, std::size_t received)
+        : ArgumentMismatch(expected > 0 ? ("Expected exactly " + std::to_string(expected) + " arguments to " + name +
+                                           ", got " + std::to_string(received))
+                                        : ("Expected at least " + std::to_string(-expected) + " arguments to " + name +
+                                           ", got " + std::to_string(received)),
+                           ExitCodes::ArgumentMismatch) {}
+
+    static ArgumentMismatch AtLeast(std::string name, int num, std::size_t received) {
+        return ArgumentMismatch(name + ": At least " + std::to_string(num) + " required but received " +
+                                std::to_string(received));
+    }
+    static ArgumentMismatch AtMost(std::string name, int num, std::size_t received) {
+        return ArgumentMismatch(name + ": At Most " + std::to_string(num) + " required but received " +
+                                std::to_string(received));
+    }
+    static ArgumentMismatch TypedAtLeast(std::string name, int num, std::string type) {
+        return ArgumentMismatch(name + ": " + std::to_string(num) + " required " + type + " missing");
+    }
+    static ArgumentMismatch FlagOverride(std::string name) {
+        return ArgumentMismatch(name + " was given a disallowed flag override");
+    }
 };
 
 /// Thrown when a requires option is missing
-struct RequiresError : public ParseError {
-    RequiresError(std::string name, std::string subname)
-        : ParseError("RequiresError", name + " requires " + subname, ExitCodes::Requires) {}
+class RequiresError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, RequiresError)
+    RequiresError(std::string curname, std::string subname)
+        : RequiresError(curname + " requires " + subname, ExitCodes::RequiresError) {}
 };
 
-/// Thrown when a exludes option is present
-struct ExcludesError : public ParseError {
-    ExcludesError(std::string name, std::string subname)
-        : ParseError("ExcludesError", name + " excludes " + subname, ExitCodes::Excludes) {}
+/// Thrown when an excludes option is present
+class ExcludesError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, ExcludesError)
+    ExcludesError(std::string curname, std::string subname)
+        : ExcludesError(curname + " excludes " + subname, ExitCodes::ExcludesError) {}
 };
 
 /// Thrown when too many positionals or options are found
-struct ExtrasError : public ParseError {
-    ExtrasError(std::string name) : ParseError("ExtrasError", name, ExitCodes::Extras) {}
+class ExtrasError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, ExtrasError)
+    explicit ExtrasError(std::vector<std::string> args)
+        : ExtrasError((args.size() > 1 ? "The following arguments were not expected: "
+                                       : "The following argument was not expected: ") +
+                          detail::rjoin(args, " "),
+                      ExitCodes::ExtrasError) {}
+    ExtrasError(const std::string &name, std::vector<std::string> args)
+        : ExtrasError(name,
+                      (args.size() > 1 ? "The following arguments were not expected: "
+                                       : "The following argument was not expected: ") +
+                          detail::rjoin(args, " "),
+                      ExitCodes::ExtrasError) {}
 };
 
 /// Thrown when extra values are found in an INI file
-struct ExtrasINIError : public ParseError {
-    ExtrasINIError(std::string name) : ParseError("ExtrasINIError", name, ExitCodes::ExtrasINI) {}
+class ConfigError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, ConfigError)
+    CLI11_ERROR_SIMPLE(ConfigError)
+    static ConfigError Extras(std::string item) { return ConfigError("INI was not able to parse " + item); }
+    static ConfigError NotConfigurable(std::string item) {
+        return ConfigError(item + ": This option is not allowed in a configuration file");
+    }
 };
 
 /// Thrown when validation fails before parsing
-struct InvalidError : public ParseError {
-    InvalidError(std::string name) : ParseError("InvalidError", name, ExitCodes::Invalid) {}
+class InvalidError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, InvalidError)
+    explicit InvalidError(std::string name)
+        : InvalidError(name + ": Too many positional arguments with unlimited expected args", ExitCodes::InvalidError) {
+    }
 };
 
-/// This is just a safety check to verify selection and parsing match
-struct HorribleError : public ParseError {
-    HorribleError(std::string name)
-        : ParseError("HorribleError", "(You should never see this error) " + name, ExitCodes::Horrible) {}
+/// This is just a safety check to verify selection and parsing match - you should not ever see it
+/// Strings are directly added to this error, but again, it should never be seen.
+class HorribleError : public ParseError {
+    CLI11_ERROR_DEF(ParseError, HorribleError)
+    CLI11_ERROR_SIMPLE(HorribleError)
 };
 
 // After parsing
 
 /// Thrown when counting a non-existent option
-struct OptionNotFound : public Error {
-    OptionNotFound(std::string name) : Error("OptionNotFound", name, ExitCodes::OptionNotFound) {}
+class OptionNotFound : public Error {
+    CLI11_ERROR_DEF(Error, OptionNotFound)
+    explicit OptionNotFound(std::string name) : OptionNotFound(name + " not found", ExitCodes::OptionNotFound) {}
 };
 
+#undef CLI11_ERROR_DEF
+#undef CLI11_ERROR_SIMPLE
+
 /// @}
 
-} // namespace CLI
+}  // namespace CLI

ext/CLI11/CLI/Formatter.hpp

@@ -0,0 +1,281 @@
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include "App.hpp"
+#include "FormatterFwd.hpp"
+
+namespace CLI {
+
+inline std::string
+Formatter::make_group(std::string group, bool is_positional, std::vector<const Option *> opts) const {
+    std::stringstream out;
+
+    out << "\n" << group << ":\n";
+    for(const Option *opt : opts) {
+        out << make_option(opt, is_positional);
+    }
+
+    return out.str();
+}
+
+inline std::string Formatter::make_positionals(const App *app) const {
+    std::vector<const Option *> opts =
+        app->get_options([](const Option *opt) { return !opt->get_group().empty() && opt->get_positional(); });
+
+    if(opts.empty())
+        return std::string();
+
+    return make_group(get_label("Positionals"), true, opts);
+}
+
+inline std::string Formatter::make_groups(const App *app, AppFormatMode mode) const {
+    std::stringstream out;
+    std::vector<std::string> groups = app->get_groups();
+
+    // Options
+    for(const std::string &group : groups) {
+        std::vector<const Option *> opts = app->get_options([app, mode, &group](const Option *opt) {
+            return opt->get_group() == group                     // Must be in the right group
+                   && opt->nonpositional()                       // Must not be a positional
+                   && (mode != AppFormatMode::Sub                // If mode is Sub, then
+                       || (app->get_help_ptr() != opt            // Ignore help pointer
+                           && app->get_help_all_ptr() != opt));  // Ignore help all pointer
+        });
+        if(!group.empty() && !opts.empty()) {
+            out << make_group(group, false, opts);
+
+            if(group != groups.back())
+                out << "\n";
+        }
+    }
+
+    return out.str();
+}
+
+inline std::string Formatter::make_description(const App *app) const {
+    std::string desc = app->get_description();
+    auto min_options = app->get_require_option_min();
+    auto max_options = app->get_require_option_max();
+    if(app->get_required()) {
+        desc += " REQUIRED ";
+    }
+    if((max_options == min_options) && (min_options > 0)) {
+        if(min_options == 1) {
+            desc += " \n[Exactly 1 of the following options is required]";
+        } else {
+            desc += " \n[Exactly " + std::to_string(min_options) + "options from the following list are required]";
+        }
+    } else if(max_options > 0) {
+        if(min_options > 0) {
+            desc += " \n[Between " + std::to_string(min_options) + " and " + std::to_string(max_options) +
+                    " of the follow options are required]";
+        } else {
+            desc += " \n[At most " + std::to_string(max_options) + " of the following options are allowed]";
+        }
+    } else if(min_options > 0) {
+        desc += " \n[At least " + std::to_string(min_options) + " of the following options are required]";
+    }
+    return (!desc.empty()) ? desc + "\n" : std::string{};
+}
+
+inline std::string Formatter::make_usage(const App *app, std::string name) const {
+    std::stringstream out;
+
+    out << get_label("Usage") << ":" << (name.empty() ? "" : " ") << name;
+
+    std::vector<std::string> groups = app->get_groups();
+
+    // Print an Options badge if any options exist
+    std::vector<const Option *> non_pos_options =
+        app->get_options([](const Option *opt) { return opt->nonpositional(); });
+    if(!non_pos_options.empty())
+        out << " [" << get_label("OPTIONS") << "]";
+
+    // Positionals need to be listed here
+    std::vector<const Option *> positionals = app->get_options([](const Option *opt) { return opt->get_positional(); });
+
+    // Print out positionals if any are left
+    if(!positionals.empty()) {
+        // Convert to help names
+        std::vector<std::string> positional_names(positionals.size());
+        std::transform(positionals.begin(), positionals.end(), positional_names.begin(), [this](const Option *opt) {
+            return make_option_usage(opt);
+        });
+
+        out << " " << detail::join(positional_names, " ");
+    }
+
+    // Add a marker if subcommands are expected or optional
+    if(!app->get_subcommands(
+               [](const CLI::App *subc) { return ((!subc->get_disabled()) && (!subc->get_name().empty())); })
+            .empty()) {
+        out << " " << (app->get_require_subcommand_min() == 0 ? "[" : "")
+            << get_label(app->get_require_subcommand_max() < 2 || app->get_require_subcommand_min() > 1 ? "SUBCOMMAND"
+                                                                                                        : "SUBCOMMANDS")
+            << (app->get_require_subcommand_min() == 0 ? "]" : "");
+    }
+
+    out << std::endl;
+
+    return out.str();
+}
+
+inline std::string Formatter::make_footer(const App *app) const {
+    std::string footer = app->get_footer();
+    if(footer.empty()) {
+        return std::string{};
+    }
+    return footer + "\n";
+}
+
+inline std::string Formatter::make_help(const App *app, std::string name, AppFormatMode mode) const {
+
+    // This immediately forwards to the make_expanded method. This is done this way so that subcommands can
+    // have overridden formatters
+    if(mode == AppFormatMode::Sub)
+        return make_expanded(app);
+
+    std::stringstream out;
+    if((app->get_name().empty()) && (app->get_parent() != nullptr)) {
+        if(app->get_group() != "Subcommands") {
+            out << app->get_group() << ':';
+        }
+    }
+
+    out << make_description(app);
+    out << make_usage(app, name);
+    out << make_positionals(app);
+    out << make_groups(app, mode);
+    out << make_subcommands(app, mode);
+    out << '\n' << make_footer(app);
+
+    return out.str();
+}
+
+inline std::string Formatter::make_subcommands(const App *app, AppFormatMode mode) const {
+    std::stringstream out;
+
+    std::vector<const App *> subcommands = app->get_subcommands({});
+
+    // Make a list in definition order of the groups seen
+    std::vector<std::string> subcmd_groups_seen;
+    for(const App *com : subcommands) {
+        if(com->get_name().empty()) {
+            if(!com->get_group().empty()) {
+                out << make_expanded(com);
+            }
+            continue;
+        }
+        std::string group_key = com->get_group();
+        if(!group_key.empty() &&
+           std::find_if(subcmd_groups_seen.begin(), subcmd_groups_seen.end(), [&group_key](std::string a) {
+               return detail::to_lower(a) == detail::to_lower(group_key);
+           }) == subcmd_groups_seen.end())
+            subcmd_groups_seen.push_back(group_key);
+    }
+
+    // For each group, filter out and print subcommands
+    for(const std::string &group : subcmd_groups_seen) {
+        out << "\n" << group << ":\n";
+        std::vector<const App *> subcommands_group = app->get_subcommands(
+            [&group](const App *sub_app) { return detail::to_lower(sub_app->get_group()) == detail::to_lower(group); });
+        for(const App *new_com : subcommands_group) {
+            if(new_com->get_name().empty())
+                continue;
+            if(mode != AppFormatMode::All) {
+                out << make_subcommand(new_com);
+            } else {
+                out << new_com->help(new_com->get_name(), AppFormatMode::Sub);
+                out << "\n";
+            }
+        }
+    }
+
+    return out.str();
+}
+
+inline std::string Formatter::make_subcommand(const App *sub) const {
+    std::stringstream out;
+    detail::format_help(out, sub->get_name(), sub->get_description(), column_width_);
+    return out.str();
+}
+
+inline std::string Formatter::make_expanded(const App *sub) const {
+    std::stringstream out;
+    out << sub->get_display_name() << "\n";
+
+    out << make_description(sub);
+    out << make_positionals(sub);
+    out << make_groups(sub, AppFormatMode::Sub);
+    out << make_subcommands(sub, AppFormatMode::Sub);
+
+    // Drop blank spaces
+    std::string tmp = detail::find_and_replace(out.str(), "\n\n", "\n");
+    tmp = tmp.substr(0, tmp.size() - 1);  // Remove the final '\n'
+
+    // Indent all but the first line (the name)
+    return detail::find_and_replace(tmp, "\n", "\n  ") + "\n";
+}
+
+inline std::string Formatter::make_option_name(const Option *opt, bool is_positional) const {
+    if(is_positional)
+        return opt->get_name(true, false);
+
+    return opt->get_name(false, true);
+}
+
+inline std::string Formatter::make_option_opts(const Option *opt) const {
+    std::stringstream out;
+
+    if(opt->get_type_size() != 0) {
+        if(!opt->get_type_name().empty())
+            out << " " << get_label(opt->get_type_name());
+        if(!opt->get_default_str().empty())
+            out << "=" << opt->get_default_str();
+        if(opt->get_expected_max() == detail::expected_max_vector_size)
+            out << " ...";
+        else if(opt->get_expected_min() > 1)
+            out << " x " << opt->get_expected();
+
+        if(opt->get_required())
+            out << " " << get_label("REQUIRED");
+    }
+    if(!opt->get_envname().empty())
+        out << " (" << get_label("Env") << ":" << opt->get_envname() << ")";
+    if(!opt->get_needs().empty()) {
+        out << " " << get_label("Needs") << ":";
+        for(const Option *op : opt->get_needs())
+            out << " " << op->get_name();
+    }
+    if(!opt->get_excludes().empty()) {
+        out << " " << get_label("Excludes") << ":";
+        for(const Option *op : opt->get_excludes())
+            out << " " << op->get_name();
+    }
+    return out.str();
+}
+
+inline std::string Formatter::make_option_desc(const Option *opt) const { return opt->get_description(); }
+
+inline std::string Formatter::make_option_usage(const Option *opt) const {
+    // Note that these are positionals usages
+    std::stringstream out;
+    out << make_option_name(opt, true);
+    if(opt->get_expected_max() >= detail::expected_max_vector_size)
+        out << "...";
+    else if(opt->get_expected_max() > 1)
+        out << "(" << opt->get_expected() << "x)";
+
+    return opt->get_required() ? out.str() : "[" + out.str() + "]";
+}
+
+}  // namespace CLI

ext/CLI11/CLI/FormatterFwd.hpp

@@ -0,0 +1,180 @@
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "StringTools.hpp"
+
+namespace CLI {
+
+class Option;
+class App;
+
+/// This enum signifies the type of help requested
+///
+/// This is passed in by App; all user classes must accept this as
+/// the second argument.
+
+enum class AppFormatMode {
+    Normal,  ///< The normal, detailed help
+    All,     ///< A fully expanded help
+    Sub,     ///< Used when printed as part of expanded subcommand
+};
+
+/// This is the minimum requirements to run a formatter.
+///
+/// A user can subclass this is if they do not care at all
+/// about the structure in CLI::Formatter.
+class FormatterBase {
+  protected:
+    /// @name Options
+    ///@{
+
+    /// The width of the first column
+    std::size_t column_width_{30};
+
+    /// @brief The required help printout labels (user changeable)
+    /// Values are Needs, Excludes, etc.
+    std::map<std::string, std::string> labels_{};
+
+    ///@}
+    /// @name Basic
+    ///@{
+
+  public:
+    FormatterBase() = default;
+    FormatterBase(const FormatterBase &) = default;
+    FormatterBase(FormatterBase &&) = default;
+
+    /// Adding a destructor in this form to work around bug in GCC 4.7
+    virtual ~FormatterBase() noexcept {}  // NOLINT(modernize-use-equals-default)
+
+    /// This is the key method that puts together help
+    virtual std::string make_help(const App *, std::string, AppFormatMode) const = 0;
+
+    ///@}
+    /// @name Setters
+    ///@{
+
+    /// Set the "REQUIRED" label
+    void label(std::string key, std::string val) { labels_[key] = val; }
+
+    /// Set the column width
+    void column_width(std::size_t val) { column_width_ = val; }
+
+    ///@}
+    /// @name Getters
+    ///@{
+
+    /// Get the current value of a name (REQUIRED, etc.)
+    std::string get_label(std::string key) const {
+        if(labels_.find(key) == labels_.end())
+            return key;
+        else
+            return labels_.at(key);
+    }
+
+    /// Get the current column width
+    std::size_t get_column_width() const { return column_width_; }
+
+    ///@}
+};
+
+/// This is a specialty override for lambda functions
+class FormatterLambda final : public FormatterBase {
+    using funct_t = std::function<std::string(const App *, std::string, AppFormatMode)>;
+
+    /// The lambda to hold and run
+    funct_t lambda_;
+
+  public:
+    /// Create a FormatterLambda with a lambda function
+    explicit FormatterLambda(funct_t funct) : lambda_(std::move(funct)) {}
+
+    /// Adding a destructor (mostly to make GCC 4.7 happy)
+    ~FormatterLambda() noexcept override {}  // NOLINT(modernize-use-equals-default)
+
+    /// This will simply call the lambda function
+    std::string make_help(const App *app, std::string name, AppFormatMode mode) const override {
+        return lambda_(app, name, mode);
+    }
+};
+
+/// This is the default Formatter for CLI11. It pretty prints help output, and is broken into quite a few
+/// overridable methods, to be highly customizable with minimal effort.
+class Formatter : public FormatterBase {
+  public:
+    Formatter() = default;
+    Formatter(const Formatter &) = default;
+    Formatter(Formatter &&) = default;
+
+    /// @name Overridables
+    ///@{
+
+    /// This prints out a group of options with title
+    ///
+    virtual std::string make_group(std::string group, bool is_positional, std::vector<const Option *> opts) const;
+
+    /// This prints out just the positionals "group"
+    virtual std::string make_positionals(const App *app) const;
+
+    /// This prints out all the groups of options
+    std::string make_groups(const App *app, AppFormatMode mode) const;
+
+    /// This prints out all the subcommands
+    virtual std::string make_subcommands(const App *app, AppFormatMode mode) const;
+
+    /// This prints out a subcommand
+    virtual std::string make_subcommand(const App *sub) const;
+
+    /// This prints out a subcommand in help-all
+    virtual std::string make_expanded(const App *sub) const;
+
+    /// This prints out all the groups of options
+    virtual std::string make_footer(const App *app) const;
+
+    /// This displays the description line
+    virtual std::string make_description(const App *app) const;
+
+    /// This displays the usage line
+    virtual std::string make_usage(const App *app, std::string name) const;
+
+    /// This puts everything together
+    std::string make_help(const App * /*app*/, std::string, AppFormatMode) const override;
+
+    ///@}
+    /// @name Options
+    ///@{
+
+    /// This prints out an option help line, either positional or optional form
+    virtual std::string make_option(const Option *opt, bool is_positional) const {
+        std::stringstream out;
+        detail::format_help(
+            out, make_option_name(opt, is_positional) + make_option_opts(opt), make_option_desc(opt), column_width_);
+        return out.str();
+    }
+
+    /// @brief This is the name part of an option, Default: left column
+    virtual std::string make_option_name(const Option *, bool) const;
+
+    /// @brief This is the options part of the name, Default: combined into left column
+    virtual std::string make_option_opts(const Option *) const;
+
+    /// @brief This is the description. Default: Right column, on new line if left column too large
+    virtual std::string make_option_desc(const Option *) const;
+
+    /// @brief This is used to print the name on the USAGE line
+    virtual std::string make_option_usage(const Option *opt) const;
+
+    ///@}
+};
+
+}  // namespace CLI

ext/CLI11/CLI/Ini.hpp

@@ -1,115 +0,0 @@
-#pragma once
-
-// Distributed under the 3-Clause BSD License.  See accompanying
-// file LICENSE or https://github.com/CLIUtils/CLI11 for details.
-
-#include <algorithm>
-#include <fstream>
-#include <iostream>
-#include <string>
-
-#include "CLI/StringTools.hpp"
-
-namespace CLI {
-namespace detail {
-
-inline std::string inijoin(std::vector<std::string> args) {
-    std::ostringstream s;
-    size_t start = 0;
-    for(const auto &arg : args) {
-        if(start++ > 0)
-            s << " ";
-
-        auto it = std::find_if(arg.begin(), arg.end(), [](char ch) { return std::isspace<char>(ch, std::locale()); });
-        if(it == arg.end())
-            s << arg;
-        else if(arg.find(R"(")") == std::string::npos)
-            s << R"(")" << arg << R"(")";
-        else
-            s << R"(')" << arg << R"(')";
-    }
-
-    return s.str();
-}
-
-struct ini_ret_t {
-    /// This is the full name with dots
-    std::string fullname;
-
-    /// Listing of inputs
-    std::vector<std::string> inputs;
-
-    /// Current parent level
-    size_t level = 0;
-
-    /// Return parent or empty string, based on level
-    ///
-    /// Level 0, a.b.c would return a
-    /// Level 1, a.b.c could return b
-    std::string parent() const {
-        std::vector<std::string> plist = detail::split(fullname, '.');
-        if(plist.size() > (level + 1))
-            return plist[level];
-        else
-            return "";
-    }
-
-    /// Return name
-    std::string name() const {
-        std::vector<std::string> plist = detail::split(fullname, '.');
-        return plist.at(plist.size() - 1);
-    }
-};
-
-/// Internal parsing function
-inline std::vector<ini_ret_t> parse_ini(std::istream &input) {
-    std::string name, line;
-    std::string section = "default";
-
-    std::vector<ini_ret_t> output;
-
-    while(getline(input, line)) {
-        std::vector<std::string> items;
-
-        detail::trim(line);
-        size_t len = line.length();
-        if(len > 1 && line[0] == '[' && line[len - 1] == ']') {
-            section = line.substr(1, len - 2);
-        } else if(len > 0 && line[0] != ';') {
-            output.emplace_back();
-            ini_ret_t &out = output.back();
-
-            // Find = in string, split and recombine
-            auto pos = line.find("=");
-            if(pos != std::string::npos) {
-                name = detail::trim_copy(line.substr(0, pos));
-                std::string item = detail::trim_copy(line.substr(pos + 1));
-                items = detail::split_up(item);
-            } else {
-                name = detail::trim_copy(line);
-                items = {"ON"};
-            }
-
-            if(detail::to_lower(section) == "default")
-                out.fullname = name;
-            else
-                out.fullname = section + "." + name;
-
-            out.inputs.insert(std::end(out.inputs), std::begin(items), std::end(items));
-        }
-    }
-    return output;
-}
-
-/// Parse an INI file, throw an error (ParseError:INIParseError or FileError) on failure
-inline std::vector<ini_ret_t> parse_ini(const std::string &name) {
-
-    std::ifstream input{name};
-    if(!input.good())
-        throw FileError(name);
-
-    return parse_ini(input);
-}
-
-} // namespace detail
-} // namespace CLI

ext/CLI11/CLI/Macros.hpp

@@ -0,0 +1,44 @@
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+// [CLI11:verbatim]
+
+// The following version macro is very similar to the one in PyBind11
+#if !(defined(_MSC_VER) && __cplusplus == 199711L) && !defined(__INTEL_COMPILER)
+#if __cplusplus >= 201402L
+#define CLI11_CPP14
+#if __cplusplus >= 201703L
+#define CLI11_CPP17
+#if __cplusplus > 201703L
+#define CLI11_CPP20
+#endif
+#endif
+#endif
+#elif defined(_MSC_VER) && __cplusplus == 199711L
+// MSVC sets _MSVC_LANG rather than __cplusplus (supposedly until the standard is fully implemented)
+// Unless you use the /Zc:__cplusplus flag on Visual Studio 2017 15.7 Preview 3 or newer
+#if _MSVC_LANG >= 201402L
+#define CLI11_CPP14
+#if _MSVC_LANG > 201402L && _MSC_VER >= 1910
+#define CLI11_CPP17
+#if __MSVC_LANG > 201703L && _MSC_VER >= 1910
+#define CLI11_CPP20
+#endif
+#endif
+#endif
+#endif
+
+#if defined(CLI11_CPP14)
+#define CLI11_DEPRECATED(reason) [[deprecated(reason)]]
+#elif defined(_MSC_VER)
+#define CLI11_DEPRECATED(reason) __declspec(deprecated(reason))
+#else
+#define CLI11_DEPRECATED(reason) __attribute__((deprecated(reason)))
+#endif
+
+// [CLI11:verbatim]

ext/CLI11/CLI/Split.hpp

@@ -1,14 +1,18 @@
-#pragma once
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
 
-// Distributed under the 3-Clause BSD License.  See accompanying
-// file LICENSE or https://github.com/CLIUtils/CLI11 for details.
+#pragma once
 
 #include <string>
 #include <tuple>
+#include <utility>
 #include <vector>
 
-#include "CLI/Error.hpp"
-#include "CLI/StringTools.hpp"
+#include "Error.hpp"
+#include "StringTools.hpp"
 
 namespace CLI {
 namespace detail {
@@ -19,14 +23,14 @@ inline bool split_short(const std::string &current, std::string &name, std::stri
         name = current.substr(1, 1);
         rest = current.substr(2);
         return true;
-    } else
-        return false;
+    }
+    return false;
 }
 
 // Returns false if not a long option. Otherwise, sets opt name and other side of = and returns true
 inline bool split_long(const std::string &current, std::string &name, std::string &value) {
     if(current.size() > 2 && current.substr(0, 2) == "--" && valid_first_char(current[2])) {
-        auto loc = current.find("=");
+        auto loc = current.find_first_of('=');
         if(loc != std::string::npos) {
             name = current.substr(2, loc - 2);
             value = current.substr(loc + 1);
@@ -35,19 +39,62 @@ inline bool split_long(const std::string &current, std::string &name, std::strin
             value = "";
         }
         return true;
-    } else
-        return false;
+    }
+    return false;
+}
+
+// Returns false if not a windows style option. Otherwise, sets opt name and value and returns true
+inline bool split_windows_style(const std::string &current, std::string &name, std::string &value) {
+    if(current.size() > 1 && current[0] == '/' && valid_first_char(current[1])) {
+        auto loc = current.find_first_of(':');
+        if(loc != std::string::npos) {
+            name = current.substr(1, loc - 1);
+            value = current.substr(loc + 1);
+        } else {
+            name = current.substr(1);
+            value = "";
+        }
+        return true;
+    }
+    return false;
 }
 
 // Splits a string into multiple long and short names
 inline std::vector<std::string> split_names(std::string current) {
     std::vector<std::string> output;
-    size_t val;
+    std::size_t val;
     while((val = current.find(",")) != std::string::npos) {
-        output.push_back(current.substr(0, val));
+        output.push_back(trim_copy(current.substr(0, val)));
         current = current.substr(val + 1);
     }
-    output.push_back(current);
+    output.push_back(trim_copy(current));
+    return output;
+}
+
+/// extract default flag values either {def} or starting with a !
+inline std::vector<std::pair<std::string, std::string>> get_default_flag_values(const std::string &str) {
+    std::vector<std::string> flags = split_names(str);
+    flags.erase(std::remove_if(flags.begin(),
+                               flags.end(),
+                               [](const std::string &name) {
+                                   return ((name.empty()) || (!(((name.find_first_of('{') != std::string::npos) &&
+                                                                 (name.back() == '}')) ||
+                                                                (name[0] == '!'))));
+                               }),
+                flags.end());
+    std::vector<std::pair<std::string, std::string>> output;
+    output.reserve(flags.size());
+    for(auto &flag : flags) {
+        auto def_start = flag.find_first_of('{');
+        std::string defval = "false";
+        if((def_start != std::string::npos) && (flag.back() == '}')) {
+            defval = flag.substr(def_start + 1);
+            defval.pop_back();
+            flag.erase(def_start, std::string::npos);
+        }
+        flag.erase(0, flag.find_first_not_of("-!"));
+        output.emplace_back(flag, defval);
+    }
     return output;
 }
 
@@ -60,24 +107,25 @@ get_names(const std::vector<std::string> &input) {
     std::string pos_name;
 
     for(std::string name : input) {
-        if(name.length() == 0)
+        if(name.length() == 0) {
             continue;
-        else if(name.length() > 1 && name[0] == '-' && name[1] != '-') {
+        }
+        if(name.length() > 1 && name[0] == '-' && name[1] != '-') {
             if(name.length() == 2 && valid_first_char(name[1]))
                 short_names.emplace_back(1, name[1]);
             else
-                throw BadNameString("Invalid one char name: " + name);
+                throw BadNameString::OneCharName(name);
         } else if(name.length() > 2 && name.substr(0, 2) == "--") {
             name = name.substr(2);
             if(valid_name_string(name))
                 long_names.push_back(name);
             else
-                throw BadNameString("Bad long name: " + name);
+                throw BadNameString::BadLongName(name);
         } else if(name == "-" || name == "--") {
-            throw BadNameString("Must have a name, not just dashes");
+            throw BadNameString::DashesOnly(name);
         } else {
             if(pos_name.length() > 0)
-                throw BadNameString("Only one positional name allowed, remove: " + name);
+                throw BadNameString::MultiPositionalNames(name);
             pos_name = name;
         }
     }
@@ -86,5 +134,5 @@ get_names(const std::vector<std::string> &input) {
         short_names, long_names, pos_name);
 }
 
-} // namespace detail
-} // namespace CLI
+}  // namespace detail
+}  // namespace CLI

ext/CLI11/CLI/StringTools.hpp

@@ -1,26 +1,50 @@
-#pragma once
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
 
-// Distributed under the 3-Clause BSD License.  See accompanying
-// file LICENSE or https://github.com/CLIUtils/CLI11 for details.
+#pragma once
 
 #include <algorithm>
 #include <iomanip>
 #include <locale>
 #include <sstream>
+#include <stdexcept>
 #include <string>
 #include <type_traits>
+#include <vector>
 
 namespace CLI {
-namespace detail {
 
+/// Include the items in this namespace to get free conversion of enums to/from streams.
+/// (This is available inside CLI as well, so CLI11 will use this without a using statement).
+namespace enums {
+
+/// output streaming for enumerations
+template <typename T, typename = typename std::enable_if<std::is_enum<T>::value>::type>
+std::ostream &operator<<(std::ostream &in, const T &item) {
+    // make sure this is out of the detail namespace otherwise it won't be found when needed
+    return in << static_cast<typename std::underlying_type<T>::type>(item);
+}
+
+}  // namespace enums
+
+/// Export to CLI namespace
+using enums::operator<<;
+
+namespace detail {
+/// a constant defining an expected max vector size defined to be a big number that could be multiplied by 4 and not
+/// produce overflow for some expected uses
+constexpr int expected_max_vector_size{1 << 29};
 // Based on http://stackoverflow.com/questions/236129/split-a-string-in-c
 /// Split a string by a delim
 inline std::vector<std::string> split(const std::string &s, char delim) {
     std::vector<std::string> elems;
-    // Check to see if emtpy string, give consistent result
-    if(s == "")
-        elems.emplace_back("");
-    else {
+    // Check to see if empty string, give consistent result
+    if(s.empty()) {
+        elems.emplace_back();
+    } else {
         std::stringstream ss;
         ss.str(s);
         std::string item;
@@ -34,11 +58,28 @@ inline std::vector<std::string> split(const std::string &s, char delim) {
 /// Simple function to join a string
 template <typename T> std::string join(const T &v, std::string delim = ",") {
     std::ostringstream s;
-    size_t start = 0;
-    for(const auto &i : v) {
-        if(start++ > 0)
-            s << delim;
-        s << i;
+    auto beg = std::begin(v);
+    auto end = std::end(v);
+    if(beg != end)
+        s << *beg++;
+    while(beg != end) {
+        s << delim << *beg++;
+    }
+    return s.str();
+}
+
+/// Simple function to join a string from processed elements
+template <typename T,
+          typename Callable,
+          typename = typename std::enable_if<!std::is_constructible<std::string, Callable>::value>::type>
+std::string join(const T &v, Callable func, std::string delim = ",") {
+    std::ostringstream s;
+    auto beg = std::begin(v);
+    auto end = std::end(v);
+    if(beg != end)
+        s << func(*beg++);
+    while(beg != end) {
+        s << delim << func(*beg++);
     }
     return s.str();
 }
@@ -46,7 +87,7 @@ template <typename T> std::string join(const T &v, std::string delim = ",") {
 /// Join a string in reverse order
 template <typename T> std::string rjoin(const T &v, std::string delim = ",") {
     std::ostringstream s;
-    for(size_t start = 0; start < v.size(); start++) {
+    for(std::size_t start = 0; start < v.size(); start++) {
         if(start > 0)
             s << delim;
         s << v[v.size() - start - 1];
@@ -97,30 +138,47 @@ inline std::string trim_copy(const std::string &str) {
     return trim(s);
 }
 
+/// remove quotes at the front and back of a string either '"' or '\''
+inline std::string &remove_quotes(std::string &str) {
+    if(str.length() > 1 && (str.front() == '"' || str.front() == '\'')) {
+        if(str.front() == str.back()) {
+            str.pop_back();
+            str.erase(str.begin(), str.begin() + 1);
+        }
+    }
+    return str;
+}
+
 /// Make a copy of the string and then trim it, any filter string can be used (any char in string is filtered)
 inline std::string trim_copy(const std::string &str, const std::string &filter) {
     std::string s = str;
     return trim(s, filter);
 }
 /// Print a two part "help" string
-inline void format_help(std::stringstream &out, std::string name, std::string description, size_t wid) {
+inline std::ostream &format_help(std::ostream &out, std::string name, std::string description, std::size_t wid) {
     name = "  " + name;
     out << std::setw(static_cast<int>(wid)) << std::left << name;
-    if(description != "") {
+    if(!description.empty()) {
         if(name.length() >= wid)
-            out << std::endl << std::setw(static_cast<int>(wid)) << "";
-        out << description;
+            out << "\n" << std::setw(static_cast<int>(wid)) << "";
+        for(const char c : description) {
+            out.put(c);
+            if(c == '\n') {
+                out << std::setw(static_cast<int>(wid)) << "";
+            }
+        }
     }
-    out << std::endl;
+    out << "\n";
+    return out;
 }
 
 /// Verify the first character of an option
-template <typename T> bool valid_first_char(T c) { return std::isalpha(c, std::locale()) || c == '_'; }
+template <typename T> bool valid_first_char(T c) {
+    return std::isalnum(c, std::locale()) || c == '_' || c == '?' || c == '@';
+}
 
 /// Verify following characters of an option
-template <typename T> bool valid_later_char(T c) {
-    return std::isalnum(c, std::locale()) || c == '_' || c == '.' || c == '-';
-}
+template <typename T> bool valid_later_char(T c) { return valid_first_char(c) || c == '.' || c == '-'; }
 
 /// Verify an option name
 inline bool valid_name_string(const std::string &str) {
@@ -132,6 +190,11 @@ inline bool valid_name_string(const std::string &str) {
     return true;
 }
 
+/// Verify that str consists of letters only
+inline bool isalpha(const std::string &str) {
+    return std::all_of(str.begin(), str.end(), [](char c) { return std::isalpha(c, std::locale()); });
+}
+
 /// Return a lower case version of a string
 inline std::string to_lower(std::string str) {
     std::transform(std::begin(str), std::end(str), std::begin(str), [](const std::string::value_type &x) {
@@ -140,18 +203,105 @@ inline std::string to_lower(std::string str) {
     return str;
 }
 
-/// Split a string '"one two" "three"' into 'one two', 'three'
-inline std::vector<std::string> split_up(std::string str) {
+/// remove underscores from a string
+inline std::string remove_underscore(std::string str) {
+    str.erase(std::remove(std::begin(str), std::end(str), '_'), std::end(str));
+    return str;
+}
 
-    std::vector<char> delims = {'\'', '\"'};
-    auto find_ws = [](char ch) { return std::isspace<char>(ch, std::locale()); };
+/// Find and replace a substring with another substring
+inline std::string find_and_replace(std::string str, std::string from, std::string to) {
+
+    std::size_t start_pos = 0;
+
+    while((start_pos = str.find(from, start_pos)) != std::string::npos) {
+        str.replace(start_pos, from.length(), to);
+        start_pos += to.length();
+    }
+
+    return str;
+}
+
+/// check if the flag definitions has possible false flags
+inline bool has_default_flag_values(const std::string &flags) {
+    return (flags.find_first_of("{!") != std::string::npos);
+}
+
+inline void remove_default_flag_values(std::string &flags) {
+    auto loc = flags.find_first_of('{');
+    while(loc != std::string::npos) {
+        auto finish = flags.find_first_of("},", loc + 1);
+        if((finish != std::string::npos) && (flags[finish] == '}')) {
+            flags.erase(flags.begin() + static_cast<std::ptrdiff_t>(loc),
+                        flags.begin() + static_cast<std::ptrdiff_t>(finish) + 1);
+        }
+        loc = flags.find_first_of('{', loc + 1);
+    }
+    flags.erase(std::remove(flags.begin(), flags.end(), '!'), flags.end());
+}
+
+/// Check if a string is a member of a list of strings and optionally ignore case or ignore underscores
+inline std::ptrdiff_t find_member(std::string name,
+                                  const std::vector<std::string> names,
+                                  bool ignore_case = false,
+                                  bool ignore_underscore = false) {
+    auto it = std::end(names);
+    if(ignore_case) {
+        if(ignore_underscore) {
+            name = detail::to_lower(detail::remove_underscore(name));
+            it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) {
+                return detail::to_lower(detail::remove_underscore(local_name)) == name;
+            });
+        } else {
+            name = detail::to_lower(name);
+            it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) {
+                return detail::to_lower(local_name) == name;
+            });
+        }
+
+    } else if(ignore_underscore) {
+        name = detail::remove_underscore(name);
+        it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) {
+            return detail::remove_underscore(local_name) == name;
+        });
+    } else {
+        it = std::find(std::begin(names), std::end(names), name);
+    }
+
+    return (it != std::end(names)) ? (it - std::begin(names)) : (-1);
+}
+
+/// Find a trigger string and call a modify callable function that takes the current string and starting position of the
+/// trigger and returns the position in the string to search for the next trigger string
+template <typename Callable> inline std::string find_and_modify(std::string str, std::string trigger, Callable modify) {
+    std::size_t start_pos = 0;
+    while((start_pos = str.find(trigger, start_pos)) != std::string::npos) {
+        start_pos = modify(str, start_pos);
+    }
+    return str;
+}
+
+/// Split a string '"one two" "three"' into 'one two', 'three'
+/// Quote characters can be ` ' or "
+inline std::vector<std::string> split_up(std::string str, char delimiter = '\0') {
+
+    const std::string delims("\'\"`");
+    auto find_ws = [delimiter](char ch) {
+        return (delimiter == '\0') ? (std::isspace<char>(ch, std::locale()) != 0) : (ch == delimiter);
+    };
     trim(str);
 
     std::vector<std::string> output;
-
+    bool embeddedQuote = false;
+    char keyChar = ' ';
     while(!str.empty()) {
-        if(str[0] == '\'') {
-            auto end = str.find('\'', 1);
+        if(delims.find_first_of(str[0]) != std::string::npos) {
+            keyChar = str[0];
+            auto end = str.find_first_of(keyChar, 1);
+            while((end != std::string::npos) && (str[end - 1] == '\\')) {  // deal with escaped quotes
+                end = str.find_first_of(keyChar, end + 1);
+                embeddedQuote = true;
+            }
             if(end != std::string::npos) {
                 output.push_back(str.substr(1, end - 1));
                 str = str.substr(end + 1);
@@ -159,32 +309,71 @@ inline std::vector<std::string> split_up(std::string str) {
                 output.push_back(str.substr(1));
                 str = "";
             }
-        } else if(str[0] == '\"') {
-            auto end = str.find('\"', 1);
-            if(end != std::string::npos) {
-                output.push_back(str.substr(1, end - 1));
-                str = str.substr(end + 1);
-            } else {
-                output.push_back(str.substr(1));
-                str = "";
-            }
-
         } else {
             auto it = std::find_if(std::begin(str), std::end(str), find_ws);
             if(it != std::end(str)) {
                 std::string value = std::string(str.begin(), it);
                 output.push_back(value);
-                str = std::string(it, str.end());
+                str = std::string(it + 1, str.end());
             } else {
                 output.push_back(str);
                 str = "";
             }
         }
+        // transform any embedded quotes into the regular character
+        if(embeddedQuote) {
+            output.back() = find_and_replace(output.back(), std::string("\\") + keyChar, std::string(1, keyChar));
+            embeddedQuote = false;
+        }
         trim(str);
     }
-
     return output;
 }
 
-} // namespace detail
-} // namespace CLI
+/// Add a leader to the beginning of all new lines (nothing is added
+/// at the start of the first line). `"; "` would be for ini files
+///
+/// Can't use Regex, or this would be a subs.
+inline std::string fix_newlines(const std::string &leader, std::string input) {
+    std::string::size_type n = 0;
+    while(n != std::string::npos && n < input.size()) {
+        n = input.find('\n', n);
+        if(n != std::string::npos) {
+            input = input.substr(0, n + 1) + leader + input.substr(n + 1);
+            n += leader.size();
+        }
+    }
+    return input;
+}
+
+/// This function detects an equal or colon followed by an escaped quote after an argument
+/// then modifies the string to replace the equality with a space.  This is needed
+/// to allow the split up function to work properly and is intended to be used with the find_and_modify function
+/// the return value is the offset+1 which is required by the find_and_modify function.
+inline std::size_t escape_detect(std::string &str, std::size_t offset) {
+    auto next = str[offset + 1];
+    if((next == '\"') || (next == '\'') || (next == '`')) {
+        auto astart = str.find_last_of("-/ \"\'`", offset - 1);
+        if(astart != std::string::npos) {
+            if(str[astart] == ((str[offset] == '=') ? '-' : '/'))
+                str[offset] = ' ';  // interpret this as a space so the split_up works properly
+        }
+    }
+    return offset + 1;
+}
+
+/// Add quotes if the string contains spaces
+inline std::string &add_quotes_if_needed(std::string &str) {
+    if((str.front() != '"' && str.front() != '\'') || str.front() != str.back()) {
+        char quote = str.find('"') < str.find('\'') ? '\'' : '"';
+        if(str.find(' ') != std::string::npos) {
+            str.insert(0, 1, quote);
+            str.append(1, quote);
+        }
+    }
+    return str;
+}
+
+}  // namespace detail
+
+}  // namespace CLI

ext/CLI11/CLI/Timer.hpp

@@ -1,9 +1,19 @@
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
 #pragma once
 
-// Distributed under the 3-Clause BSD License.  See accompanying
-// file LICENSE or https://github.com/CLIUtils/CLI11 for details.
+// On GCC < 4.8, the following define is often missing. Due to the
+// fact that this library only uses sleep_for, this should be safe
+#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 5 && __GNUC_MINOR__ < 8
+#define _GLIBCXX_USE_NANOSLEEP
+#endif
 
-#include <chrono>
+#include <array>
+#include <chrono>  // NOLINT(build/c++11)
 #include <functional>
 #include <iostream>
 #include <string>
@@ -11,6 +21,7 @@
 
 namespace CLI {
 
+/// This is a simple timer with pretty printing. Creating the timer starts counting.
 class Timer {
   protected:
     /// This is a typedef to make clocks easier to use
@@ -32,7 +43,7 @@ class Timer {
     time_point start_;
 
     /// This is the number of times cycles (print divides by this number)
-    size_t cycles{1};
+    std::size_t cycles{1};
 
   public:
     /// Standard print function, this one is set by default
@@ -46,7 +57,7 @@ class Timer {
 
   public:
     /// Standard constructor, can set title and print function
-    Timer(std::string title = "Timer", time_print_t time_print = Simple)
+    explicit Timer(std::string title = "Timer", time_print_t time_print = Simple)
         : title_(std::move(title)), time_print_(std::move(time_print)), start_(clock::now()) {}
 
     /// Time a function by running it multiple times. Target time is the len to target.
@@ -55,14 +66,14 @@ class Timer {
         double total_time;
 
         start_ = clock::now();
-        size_t n = 0;
+        std::size_t n = 0;
         do {
             f();
             std::chrono::duration<double> elapsed = clock::now() - start_;
             total_time = elapsed.count();
-        } while(n++ < 100 && total_time < target_time);
+        } while(n++ < 100u && total_time < target_time);
 
-        std::string out = make_time_str(total_time / n) + " for " + std::to_string(n) + " tries";
+        std::string out = make_time_str(total_time / static_cast<double>(n)) + " for " + std::to_string(n) + " tries";
         start_ = start;
         return out;
     }
@@ -71,16 +82,18 @@ class Timer {
     std::string make_time_str() const {
         time_point stop = clock::now();
         std::chrono::duration<double> elapsed = stop - start_;
-        double time = elapsed.count() / cycles;
+        double time = elapsed.count() / static_cast<double>(cycles);
         return make_time_str(time);
     }
 
     // LCOV_EXCL_START
+    /// This prints out a time string from a time
     std::string make_time_str(double time) const {
         auto print_it = [](double x, std::string unit) {
-            char buffer[50];
-            std::snprintf(buffer, 50, "%.5g", x);
-            return buffer + std::string(" ") + unit;
+            const unsigned int buffer_length = 50;
+            std::array<char, buffer_length> buffer;
+            std::snprintf(buffer.data(), buffer_length, "%.5g", x);
+            return buffer.data() + std::string(" ") + unit;
         };
 
         if(time < .000001)
@@ -92,13 +105,13 @@ class Timer {
         else
             return print_it(time, "s");
     }
-    // LCOV_EXCL_END
+    // LCOV_EXCL_STOP
 
     /// This is the main function, it creates a string
     std::string to_string() const { return time_print_(title_, make_time_str()); }
 
     /// Division sets the number of cycles to divide by (no graphical change)
-    Timer &operator/(size_t val) {
+    Timer &operator/(std::size_t val) {
         cycles = val;
         return *this;
     }
@@ -108,14 +121,14 @@ class Timer {
 class AutoTimer : public Timer {
   public:
     /// Reimplementing the constructor is required in GCC 4.7
-    AutoTimer(std::string title = "Timer", time_print_t time_print = Simple) : Timer(title, time_print) {}
+    explicit AutoTimer(std::string title = "Timer", time_print_t time_print = Simple) : Timer(title, time_print) {}
     // GCC 4.7 does not support using inheriting constructors.
 
-    /// This desctructor prints the string
+    /// This destructor prints the string
     ~AutoTimer() { std::cout << to_string() << std::endl; }
 };
 
-} // namespace CLI
+}  // namespace CLI
 
 /// This prints out the time if shifted into a std::cout like stream.
 inline std::ostream &operator<<(std::ostream &in, const CLI::Timer &timer) { return in << timer.to_string(); }

ext/CLI11/CLI/Version.hpp

@@ -0,0 +1,16 @@
+// Copyright (c) 2017-2020, University of Cincinnati, developed by Henry Schreiner
+// under NSF AWARD 1414736 and by the respective contributors.
+// All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+// [CLI11:verbatim]
+
+#define CLI11_VERSION_MAJOR 1
+#define CLI11_VERSION_MINOR 9
+#define CLI11_VERSION_PATCH 1
+#define CLI11_VERSION "1.9.1"
+
+// [CLI11:verbatim]

galileo.cc

@@ -24,3 +24,91 @@ bool getTOWFromInav(std::basic_string_view<uint8_t> inav, uint32_t *satTOW, uint
 
   return false;
 }
+
+int GalileoMessage::parseFnav(std::basic_string_view<uint8_t> page)
+{
+  const uint8_t* ptr = &page[0];
+  int offset=0;
+  auto gbum=[&ptr, &offset](int bits) {
+    unsigned int ret = getbitu(ptr, offset, bits);
+    offset += bits;
+    return ret;
+  };
+  
+  auto gbsm=[&ptr, &offset](int bits) {
+    int ret = getbits(ptr, offset, bits);
+    offset += bits;
+    return ret;
+  };
+  
+  wtype = gbum(6);
+  if(wtype == 1) {
+    /*int sv = */    (void)gbum(6);
+    iodnav = gbum(10);
+    t0c =    gbum(14);
+    af0 =    gbsm(31);
+    af1 =    gbsm(21);
+    af2 =    gbsm(6);
+    sisa =   gbum(8);
+    ai0 =    gbum(11);
+    ai1 =    gbsm(11);
+    ai2 =    gbsm(14);
+    sf1 =    gbum(1);
+    sf2 =    gbum(1);
+    sf3 =    gbum(1);
+    sf4 =    gbum(1);
+    sf5 =    gbum(1);
+    BGDE1E5a = gbsm(10);
+    e5ahs = gbum(2);
+    wn = gbum(12);
+    tow = gbum(20);
+    e5advs=gbum(1);
+  }
+  else if(wtype==2)  {
+    iodnav = gbum(10);
+    m0 = gbsm(32);
+    omegadot = gbsm(24);
+    e = gbum(32);
+    sqrtA = gbum(32);
+    omega0 = gbsm(32);
+    idot = gbsm(14);
+    wn = gbum(12);
+    tow = gbum(20);
+  }
+  else if(wtype == 3) {
+    iodnav = gbum(10);
+    i0 = gbsm(32);
+    omega = gbsm(32);
+    deltan = gbsm(16);
+    cuc = gbsm(16);
+    cus = gbsm(16);
+    crc =gbsm(16);
+    crs = gbsm(16);
+    t0e = gbum(14);
+    wn = gbum(12);
+    tow = gbum(20);
+  }
+  else if(wtype == 4) {
+    iodnav = gbum(10);
+    cic = gbsm(16);
+    cis = gbsm(16);
+
+    a0 = gbsm(32);
+    a1 = gbsm(24);
+
+    dtLS= gbsm(8);
+    t0t = gbum(8);
+    wn0t = gbum(8);
+    wnLSF = gbum(8);
+    
+    dn = gbum(3);
+    dtLSF = gbsm(8);
+    
+    t0g = gbum(8);
+    a0g = gbsm(16);
+    a1g = gbsm(12);
+    wn0g = gbum(6);
+    tow = gbum(20);
+  }
+  return wtype;
+}

galileo.hh

@@ -40,6 +40,8 @@ struct GalileoMessage : GPSLikeEphemeris
     return wtype;
   }
 
+  int parseFnav(std::basic_string_view<uint8_t> page);
+  
   uint8_t sparetime{0};
   uint16_t wn{0};  
   uint32_t tow{0}; 
@@ -63,13 +65,13 @@ struct GalileoMessage : GPSLikeEphemeris
   {
   }
 
-  uint8_t e5bhs{0}, e1bhs{0};
+  uint8_t e5ahs{0}, e5bhs{0}, e1bhs{0};
   uint8_t gpshealth{0};
   uint16_t ai0{0};
   int16_t ai1{0}, ai2{0};
   bool sf1{0}, sf2{0}, sf3{0}, sf4{0}, sf5{0};
   int BGDE1E5a{0}, BGDE1E5b{0};
-  bool e5bdvs{false}, e1bdvs{false};
+  bool e5advs{false}, e5bdvs{false}, e1bdvs{false};
   bool disturb1{false}, disturb2{false}, disturb3{false}, disturb4{false}, disturb5{false};
   
   //                     
@@ -222,7 +224,9 @@ struct GalileoMessage : GPSLikeEphemeris
   // pair of nanosecond, nanosecond/s 
   std::pair<double, double> getGPSOffset(int tow, int wn) const
   {
-    int dw = (int)(uint8_t)wn - (int)(uint8_t) wn0g;
+    int dw = (int)(wn%64) - (int)(wn0g%64);
+    if(dw > 31)
+      dw = dw - 64;
     int delta = dw*7*86400  + tow - getT0g(); // NOT ephemeris age tricks
 
     // 2^-35  2^-51   3600
@@ -266,8 +270,8 @@ struct GalileoMessage : GPSLikeEphemeris
     sf3 = getbitu(&page[0], 44, 1);
     sf4 = getbitu(&page[0], 45, 1);
     sf5 = getbitu(&page[0], 46, 1);
-    BGDE1E5a = getbits(&page[0], 47, 10);
-    BGDE1E5b = getbits(&page[0], 57, 10);
+    BGDE1E5a = getbits(&page[0], 47, 10); // 2^-32 s
+    BGDE1E5b = getbits(&page[0], 57, 10); // 2^-32 s
     
     e5bhs = getbitu(&page[0], 67, 2);
     e1bhs = getbitu(&page[0], 69, 2);

galmonmon.cc

@@ -1,4 +1,4 @@
-
+#include <optional>
 #include "minicurl.hh"
 #include <iostream>
 #include "navmon.hh"
@@ -45,7 +45,6 @@ public:
   std::optional<string> reportState(string_view thing, string_view name, var_t state, const std::string& state_text="");
   std::optional<string> getState(string_view thing, string_view name);
 
-
   std::optional<string> getPrevState(string_view thing, string_view name);  
 
   struct State
@@ -166,23 +165,7 @@ std::optional<string> StateKeeper::reportState(string_view thing, string_view na
 
 
 StateKeeper g_sk;
-#if 0
-static std::string string_replace(const std::string& str, const std::string& match, 
-        const std::string& replacement, unsigned int max_replacements = UINT_MAX)
-{
-    size_t pos = 0;
-    std::string newstr = str;
-    unsigned int replacements = 0;
-    while ((pos = newstr.find(match, pos)) != std::string::npos
-            && replacements < max_replacements)
-    {
-         newstr.replace(pos, match.length(), replacement);
-         pos += replacement.length();
-         replacements++;
-    }
-    return newstr;
-}
-#endif 
+
 void sendTweet(const string& tweet)
 {
   string etweet = tweet;
@@ -197,8 +180,8 @@ int main(int argc, char **argv)
 {
   MiniCurl mc;
   MiniCurl::MiniCurlHeaders mch;
-  //  string url="https://galmon.eu/svs.json";
-  string url="http://[::1]:29599/";
+  string url="https://galmon.eu/";
+  //  string url="http://[::1]:29599/";
   bool doVERSION{false};
 
   CLI::App app(program);
@@ -222,6 +205,7 @@ int main(int argc, char **argv)
   g_sk.setBoolNames("health", "healthy", "unhealthy");
   g_sk.setBoolNames("eph-too-old", "ephemeris fresh", "ephemeris aged");
   g_sk.setBoolNames("silent", "observed", "not observed");
+  g_sk.setBoolNames("osnma", "OFF", "ON");
 
   std::variant<bool, string> tst;
 
@@ -273,7 +257,8 @@ int main(int argc, char **argv)
       
       res = mc.getURL(url+"svs.json");
       j = nlohmann::json::parse(res);
-      bool first=true;       
+      bool first=true;
+      bool globalOsnma=false;
       for(const auto& sv : j) {
         if(!sv.count("gnssid") || !sv.count("fullName") || !sv.count("sigid")) {
           cout<<"Skipping "<< sv.count("gnssid") <<", "<< sv.count("fullName") <<", " <<sv.count("sigid") <<endl;
@@ -295,7 +280,8 @@ int main(int argc, char **argv)
           //          cout<<"Skipping "<<fullName<<" in loop: "<<sv.count("healthissue")<<", "<<sv.count("eph-age-m") << ", "<<sv.count("perrecv")<<endl;
           continue;
         }
-        
+
+
         for(const auto& recv : sv["perrecv"]) {
           if(!recv.count("last-seen-s")) {
             cout<<"Missing last-seen-s"<<endl;
@@ -305,9 +291,11 @@ int main(int argc, char **argv)
             numfresh++;
           if((int)recv["last-seen-s"] < 3600)
             notseen=false;
-          
         }
+	if(sv.count("osnma") && sv["osnma"]==true)
+	  globalOsnma |= 1;
 
+	
         auto healthchange = g_sk.reportState(fullName, "health", sv["healthissue"]!=0);
         std::optional<string> tooOldChange;
         if(gnssid == 2)
@@ -410,6 +398,17 @@ int main(int argc, char **argv)
           cout<<humanTimeNow() <<" " << tweet << endl;
         }
       }
+
+      auto osnmachange = g_sk.reportState("global", "osnma", globalOsnma);
+      if(osnmachange) {
+	string tweet= "Galileo OSNMA new state: "+*osnmachange;
+	cout<<humanTimeNow()<< " " <<tweet<<endl;
+	if(doTweet) {
+	  sendTweet(tweet);
+	}
+      }
+
+      
       cout<<".";
       cout.flush();
     }

glonass.cc

@@ -110,16 +110,16 @@ static double passedMsec(const Clock::time_point& then, const Clock::time_point&
   return std::chrono::duration_cast<std::chrono::microseconds>(now - then).count()/1000.0;
 }
 
-
+#if 0
 static double passedMsec(const Clock::time_point& then)
 {
   return passedMsec(then, Clock::now());
 }
-
+#endif
 
 double getCoordinates(double tow, const GlonassMessage& eph, Point* p)
 {
-  auto start = Clock::now();
+  //  auto start = Clock::now();
   
   double y0[6] = {ldexp(eph.x, -11), ldexp(eph.y, -11), ldexp(eph.z, -11),
     ldexp(eph.dx, -20), ldexp(eph.dy, -20), ldexp(eph.dz, -20)};
@@ -137,7 +137,7 @@ double getCoordinates(double tow, const GlonassMessage& eph, Point* p)
     rk4step (A, y0, h);
 
   *p = Point (1E3*y0[0], 1E3*y0[1], 1E3*y0 [2]);
-  static double total=0;
+  //  static double total=0;
   //  cout<<"Took: "<<(total+=passedMsec(start))<<" ms" <<endl;
   return 0;
 }

gndate.cc

@@ -0,0 +1,81 @@
+#include "navmon.hh"
+#include <iostream>
+#include "CLI/CLI.hpp"
+#include "version.hh"
+
+extern const char* g_gitHash;
+using namespace std;
+
+int main(int argc, char** argv)
+try
+{
+  string program("gndate");
+  CLI::App app(program);
+  string date;
+  int galwn{-1};
+  bool doProgOutput{false};
+  bool doGPSWN{false}, doGALWN{false}, doBEIDOUWN{false}, doVERSION{false}, doUTC{false};
+  app.add_flag("--version", doVERSION, "show program version and copyright");
+  app.add_option("--date,-d", date, "yyyy-mm-dd hh:mm[:ss] hh:mm yyyymmdd hhmm");
+  app.add_option("--date-gal-wn", galwn, "Give data for this Galileo week number");
+  app.add_flag("--utc,-u", doUTC, "Interpret --date,-d as UTC");
+  app.add_flag("--gps-wn", doGPSWN, "Print GPS week number");
+  app.add_flag("--gal-wn", doGALWN, "Print GPS week number");
+  app.add_flag("--prog-output", doProgOutput, "Modulate some date formats for use as parameters to programs");
+  try {
+    app.parse(argc, argv);
+  } catch(const CLI::Error &e) {
+    return app.exit(e);
+  }
+
+  if(doVERSION) {
+    showVersion(program.c_str(), g_gitHash);
+    exit(0);
+  }
+
+  
+  time_t now;
+  if(date.empty())
+    now = time(0);
+  else {
+    if(doUTC)
+      setenv("TZ", "UTC", 1);
+    now = parseTime(date);
+  }
+  
+  int wn, tow;
+
+  if(galwn >= 0) {
+    time_t week=utcFromGST(galwn, 0);
+    if(doProgOutput)
+      cout<<influxTime(week) << endl;
+    else
+      cout<<humanTime(week)<< " - " << humanTime(week+7*86400) << endl;
+    return 0;
+  }
+  
+  if(doGPSWN) {
+    getGPSDateFromUTC(now, wn, tow);
+    cout<<wn<<endl;
+  }
+  else if(doGALWN) {
+    getGalDateFromUTC(now, wn, tow);
+    cout<<wn<<endl;
+  }
+  else if(doBEIDOUWN) {
+    getBeiDouDateFromUTC(now, wn, tow);
+    cout<<wn<<endl;
+  }
+  else {
+    getGPSDateFromUTC(now, wn, tow);
+    cout<<"GPS Week Number (non-wrapped): "<< wn << ", tow " << tow << endl;
+    getGalDateFromUTC(now, wn, tow);
+    cout<<"Galileo Week Number: "<< wn << ", tow " << tow << endl;
+    getBeiDouDateFromUTC(now, wn, tow);
+    cout<<"BeiDou Week Number: "<< wn << ", sow " << tow << endl;
+    
+  }
+}
+catch(exception& e) {
+  cerr<<"Error: "<<e.what()<<endl;
+}

gps.cc

@@ -96,8 +96,12 @@ int GPSState::parseGPSMessage(std::basic_string_view<uint8_t> cond, uint8_t* pag
 
       t0t = getbitu(&cond[0], 7*24 + 8, 8) * 4096; // WE SCALE THIS FOR THE USER!
       wn0t = getbitu(&cond[0], 7*24  + 16, 8);
+      
       dtLS = getbits(&cond[0], 8*24, 8);
+      wnLSF= getbitu(&cond[0], 8*24 + 8, 8);
+      dn = getbitu(&cond[0], 8*24 + 16, 8);
       dtLSF = getbits(&cond[0], 9*24, 8);
+
       
       //      cerr<<": a0: "<<a0<<", a1: "<<a1<<", t0t: "<< t0t * (1<<12) <<", wn0t: "<< wn0t<<", rough offset: "<<ldexp(a0, -30)<<endl;
       //      cerr<<"deltaTLS: "<< (int)dtLS<<", post "<< (int)dtLSF<<endl;

html/doalles.js

@@ -14,7 +14,7 @@ function maketable(str, arr)
         enter().
         append("tr");
     
-    var columns = ["sv", "best-tle", "iod", "eph-age-m", "latest-disco", "time-disco", "sisa", "health", "alma-dist", "delta-utc", "sources", "hqsources", "db", "rtcm-eph-delta-cm","prres", "elev", "last-seen-s"];    
+    var columns = ["sv", "best-tle", "iod", "eph-age-m", "latest-disco", "time-disco", "sisa", "health", "alma-dist", "osnma", "sources", "db", "rtcm-eph-delta-cm","rtcm-clock-dclock0", "prres", "elev", "last-seen-s"];    
     
     // append the header row
     thead.append("tr")
@@ -27,6 +27,9 @@ function maketable(str, arr)
                 return "ΔHz";
             if(column == "rtcm-eph-delta-cm")
                 return "Δrtcm";
+            if(column == "rtcm-clock-dclock0")
+                return "Δclk";
+
             if(column == "delta-gps")
                 return "ΔGPS ns";
             if(column == "delta-utc")
@@ -58,7 +61,7 @@ function maketable(str, arr)
                     else if(row["gnssid"] == 6)
                         img='ext/glo.png';
                     
-                    ret.value = '<img width="16" height="16" src="https://ds9a.nl/tmp/'+ img +'"/>';
+                    ret.value = '<img width="16" height="16" src="https://berthub.eu/tmp/'+ img +'"/>';
 //                    ret.value="";
                     ret.value += "&nbsp;<a href='sv.html?gnssid="+row.gnssid+"&sv="+row.svid+"&sigid="+row.sigid+"'>"+row.sv+"</a>";
                 }
@@ -68,6 +71,18 @@ function maketable(str, arr)
                     else
                         ret.value="";
                 }
+                else if(column == "rtcm-clock-dclock0") {
+                    if(row[column] != null) {
+                        if(Math.abs(row[column]) > 150)
+                            ret.color="#ff2222";
+                        else if(Math.abs(row[column]) > 100) 
+                            ret.color="#ff4444";
+                        ret.value = row[column].toFixed(1)+" cm";
+                    }
+                    else
+                        ret.value="";
+                }
+
                 else if(column == "aodc/e") {
                     if(row["aodc"] != null && row["aode"] != null)
                         ret.value = row["aodc"]+"/"+row["aode"];
@@ -115,6 +130,13 @@ function maketable(str, arr)
                 else if(column == "norad") {
                     ret.value = row["best-tle-norad"];
                 }
+                else if(column == "osnma" && row["osnma"] != null) {
+                    if(row["osnma"]==true)
+                        ret.value="ON!";
+                    else
+                        ret.value="off";
+                }
+
                 else if(column == "delta-utc" && row["delta-utc"] != null) {
                     ret.value = row["delta-utc"]+'<span class="CellComment">a0: '+row["a0"]+'<br/>a1: '+row["a1"]+'<br/>wn0t: ' + row["wn0t"]+'<br/>t0t: '+row["t0t"]+'</span>';
                     ret.Class = 'CellWithComment';
@@ -273,6 +295,8 @@ function updateSats()
             let wantIt = false;
             if(d3.select("#GalE1").property("checked") && arr[n].gnssid==2 && arr[n].sigid == 1)
                 wantIt = true;
+            if(d3.select("#GalE5a").property("checked") && arr[n].gnssid==2 && arr[n].sigid == 6)
+                wantIt = true;
             if(d3.select("#GalE5b").property("checked") && arr[n].gnssid==2 && arr[n].sigid == 5)
                 wantIt = true;
             if(d3.select("#GPSL1CA").property("checked") && arr[n].gnssid==0 && arr[n].sigid == 0)

html/index.html

@@ -10,6 +10,7 @@
   <div class="centered">
     <hr/>
     <input type="checkbox"  id="GalE1" onclick="updateSats();">  <label for="GalE1">Galileo E1</label> &nbsp;&nbsp;
+    <input type="checkbox"  id="GalE5a" onclick="updateSats();">  <label for="GalE5a">Galileo E5a</label> &nbsp;&nbsp;
     <input type="checkbox"  id="GalE5b" onclick="updateSats();">  <label for="GalE5b">Galileo E5b</label> &nbsp;&nbsp;
     <input type="checkbox"  id="BeiDouB1I" onclick="updateSats();">  <label for="BeiDouB1I">BeiDou B1I</label> &nbsp;&nbsp;
     <input type="checkbox"  id="BeiDouB2I" onclick="updateSats();">  <label for="BeiDouB2I">BeiDou B2I</label> &nbsp;&nbsp;
@@ -27,12 +28,8 @@
   Stale:<br/>
   <table id="svsstale"></table>
   <p>
-    This table shows live output from four Galileo/GPS/BeiDou/GLONASS receivers hosted in Nootdorp, The Netherlands and California, United States, Tonga, Brazil, Singapore, Austria, India and Uruguay.
-    It is very much a work in progress, and will not be available at all times. Extremely rough code is on
-    <a href="https://github.com/ahuPowerDNS/galmon">GitHub</a>.
-  </p>
-  <p>
-    Some technical detail behind this setup can be found in <a href="https://berthub.eu/articles/posts/galileo-notes/">this post</a>.
+  Source code of this website &amp; the whole system is available on
+    <a href="https://github.com/berthubert/galmon">GitHub</a>.
   </p>
   <p>
     For updates, follow <a href="https://twitter.com/GalileoSats">@GalileoSats</a> on Twitter, or join us on our IRC channel (chat) via the
@@ -64,10 +61,11 @@
     The official Galileo constellation status can be found on the <a href="https://www.gsc-europa.eu/system-status/Constellation-Information">European GNSS Service Centre page</a>, which also lists "NAGUs", <a href="https://www.gsc-europa.eu/system-status/user-notifications">notifications about outages or changes</a>.
   </p>
   <p>
-    Official GLONASS status can be found on <a href="https://www.glonass-iac.ru/en/GLONASS/">this page</a> from the Russian Information and Analysis Center for Positioning, Navigation and Timing.
+    Official GLONASS status can be found on <a href="https://www.glonass-iac.ru/en/sostavOG/">this page</a> from the Russian Information and Analysis Center for Positioning, Navigation and Timing.
   </p>
   <p>
-    Sadly reduced status updates on GPS can be found on <a href="https://www.navcen.uscg.gov/?Do=constellationStatus">this page</a> from the US Department of Homeland Security.
+    Status updates on GPS can be found on <a
+    href="https://www.navcen.uscg.gov/gps-constellation">this page</a>.
   </p>
   <p>
     Information on the status of BeiDou can be found on <a href="http://www.csno-tarc.cn/system/constellation&amp;ce=english">this page</a> from the Chinese Test and Assessment Research Center of China Satellite Navigation Office.

html/observer.html

@@ -6,7 +6,7 @@
   <link rel="stylesheet" type="text/css" href="style.css">
 </head>
 <body>
-  Last update: <span id="freshness"></span>. More information about this Galileo/GPS/BeiDou/Glonass open source monitor can be found <a href="https://github.com/ahupowerdns/galmon/blob/master/README.md#galmon">here</a>. Live map <a href="geo">here!</a>. Contact <a href="https://ds9a.nl/">me</a> if you want access to the Grafana dashboard.<br/>
+  Last update: <span id="freshness"></span>. More information about this Galileo/GPS/BeiDou/Glonass open source monitor can be found <a href="https://github.com/ahupowerdns/galmon/blob/master/README.md#galmon">here</a>. Live map <a href="geo">here!</a>. Contact <a href="https://berthub.eu/">me</a> if you want access to the Grafana dashboard.<br/>
   <table>
     <tr style="background: #FFF">
       <td style="vertical-align:top">
@@ -27,7 +27,7 @@
     It is very much a work in progress, and will not be available at all times. Extremely rough code is on
     <a href="https://github.com/ahuPowerDNS/galmon">GitHub</a>.
 
-    Some technical detail behind this setup can be found in <a href="https://ds9a.nl/articles/posts/galileo-notes/">this post</a>.
+    Some technical detail behind this setup can be found in <a href="https://berthub.eu/articles/posts/galileo-notes/">this post</a>.
 
     For updates, follow <a href="https://twitter.com/GalileoSats">@GalileoSats</a> on Twitter, or join us on our IRC channel (chat) via the
     <a href="https://webchat.oftc.net/?channels=galileo">web gateway</a>.

html/observer.js

@@ -85,7 +85,6 @@ function componentDidMount() {
         .text(function(d) { return d + "°"; });
 
         sats.select('g.satellites').remove();
-    console.log(gnss_position);
     
         let points = sats
             .insert("g")
@@ -259,7 +258,6 @@ function update()
       
 }
 
-console.log(window.location.href);
 var url = new URL(window.location.href);
 observer = url.searchParams.get("observer");
 

html/observers.html

@@ -6,14 +6,14 @@
   <link rel="stylesheet" type="text/css" href="style.css">
 </head>
 <body>
-  Last update: <span id="freshness"></span>. More information about this Galileo/GPS/BeiDou/Glonass open source monitor can be found <a href="https://github.com/ahupowerdns/galmon/blob/master/README.md#galmon">here</a>. Live map <a href="geo">here!</a>. Contact <a href="https://ds9a.nl/">me</a> if you want access to the Grafana dashboard.<br/>
+  Last update: <span id="freshness"></span>. More information about this Galileo/GPS/BeiDou/Glonass open source monitor can be found <a href="https://github.com/ahupowerdns/galmon/blob/master/README.md#galmon">here</a>. Live map <a href="geo">here!</a>. Contact <a href="https://berthub.eu/">me</a> if you want access to the Grafana dashboard.<br/>
   <table id="galileo"></table>
   <p>
     This table shows live output from four Galileo/GPS/BeiDou/GLONASS receivers hosted in Nootdorp, The Netherlands and California, United States.
     It is very much a work in progress, and will not be available at all times. Extremely rough code is on
     <a href="https://github.com/ahuPowerDNS/galmon">GitHub</a>.
 
-    Some technical detail behind this setup can be found in <a href="https://ds9a.nl/articles/posts/galileo-notes/">this post</a>.
+    Some technical detail behind this setup can be found in <a href="https://berthub.eu/articles/posts/galileo-notes/">this post</a>.
 
     For updates, follow <a href="https://twitter.com/GalileoSats">@GalileoSats</a> on Twitter, or join us on our IRC channel (chat) via the
     <a href="https://webchat.oftc.net/?channels=galileo">web gateway</a>.

html/overview.html

@@ -6,14 +6,14 @@
   <link rel="stylesheet" type="text/css" href="style.css">
 </head>
 <body>
-  Last update: <span id="freshness"></span>. More information about this Galileo/GPS/BeiDou/Glonass open source monitor can be found <a href="https://github.com/ahupowerdns/galmon/blob/master/README.md#galmon">here</a>. Live map <a href="geo">here!</a>. Contact <a href="https://ds9a.nl/">me</a> if you want access to the Grafana dashboard.<br/>
+  Last update: <span id="freshness"></span>. More information about this Galileo/GPS/BeiDou/Glonass open source monitor can be found <a href="https://github.com/ahupowerdns/galmon/blob/master/README.md#galmon">here</a>. Live map <a href="geo">here!</a>. Contact <a href="https://berthub.eu/">me</a> if you want access to the Grafana dashboard.<br/>
   <table id="galileo"></table>
   <p>
     This table shows live output from four Galileo/GPS/BeiDou/GLONASS receivers hosted in Nootdorp, The Netherlands and California, United States.
     It is very much a work in progress, and will not be available at all times. Extremely rough code is on
     <a href="https://github.com/ahuPowerDNS/galmon">GitHub</a>.
 
-    Some technical detail behind this setup can be found in <a href="https://ds9a.nl/articles/posts/galileo-notes/">this post</a>.
+    Some technical detail behind this setup can be found in <a href="https://berthub.eu/articles/posts/galileo-notes/">this post</a>.
 
     For updates, follow <a href="https://twitter.com/GalileoSats">@GalileoSats</a> on Twitter, or join us on our IRC channel (chat) via the
     <a href="https://webchat.oftc.net/?channels=galileo">web gateway</a>.

html/sbas.html

@@ -18,7 +18,7 @@
   <table id="sbasstale"></table>
   <p>
 
-    Some technical detail behind this setup can be found in <a href="https://ds9a.nl/articles/posts/galileo-notes/">this post</a>.
+    Some technical detail behind this setup can be found in <a href="https://berthub.eu/articles/posts/galileo-notes/">this post</a>.
 
     For updates, follow <a href="https://twitter.com/GalileoSats">@GalileoSats</a> on Twitter, or join us on our IRC channel (chat) via the
     <a href="https://webchat.oftc.net/?channels=galileo">web gateway</a>.

html/sbas.js

@@ -0,0 +1,276 @@
+var repeat;
+
+moment.relativeTimeThreshold('m', 120);
+
+function maketable(str, arr)
+{
+    var table=d3.select(str);
+    table.html("");
+    var thead=table.append("thead");
+    var tbody=table.append("tbody");
+    
+    var rows=tbody.selectAll("tr").
+        data(arr).
+        enter().
+        append("tr");
+    
+    var columns = ["sv",  "health", "sources", "db", "last-do-not-use", "last-seen-s"];    
+    
+    // append the header row
+    thead.append("tr")
+        .selectAll("th")
+        .data(columns)
+        .enter()
+        .append("th")
+        .html(function(column) {
+            if(column == "delta_hz_corr")
+                return "ΔHz";
+            if(column == "delta-gps")
+                return "ΔGPS ns";
+            if(column == "delta-utc")
+                return "ΔUTC ns";
+            if(column == "sources")
+                return '<a href="observers.html">sources</a>';
+            if(column == "alma-dist")
+                return '<a href="almanac.html">alma-dist</a>';
+            
+            else
+                return column;
+        });
+    
+    var cells = rows.selectAll("td").
+        data(function(row) {
+            return columns.map(function(column) {
+                var ret={};
+                ret.column = column;
+                ret.color=null;
+                ret.Class = null;
+                if(column == "last-do-not-use") {
+                    if(row["last-type-0-s"] < 30*86400) {
+                        var b = moment.duration(-row["last-type-0-s"], 's');
+                        ret.value = b.humanize(true);
+                    }
+                    else ret.value="";
+                }
+                else if(column == "sv") {
+                    var img="";
+                    var sv = row["sv"];
+                    var sbas="";
+                    if(sv == 138 || sv == 131 || sv == 133) {
+                        img = 'ext/gps.png';
+                        sbas = "WAAS";
+                    }
+                    else if(sv== 126 || sv == 136 || sv == 123 ) {
+                        img='ext/gal.png';
+                        sbas = "EGNOS";
+                    }
+                    else if(sv == 140 || sv == 125 || sv == 141) {
+                        img='ext/glo.png';
+                        sbas = "SDCM";
+                    }
+                    else if(sv == 127 || sv == 128 || sv == 138) {
+                        img='ext/gagan.png';
+                        sbas ="GAGAN";
+                    }
+
+                    ret.value = sbas + "&nbsp;";
+                    if(img != "")
+                        ret.value += '<img width="16" height="16" src="https://berthub.eu/tmp/'+ img +'"/>';
+                    else
+                        ret.value += "";
+                    //                    ret.value="";
+
+                    ret.value += "&nbsp;<a href='sv.html?gnssid="+row.gnssid+"&sv="+row.svid+"&sigid="+row.sigid+"'>"+row.sv+"</a>";
+                }
+                else if(column == "aodc/e") {
+                    if(row["aodc"] != null && row["aode"] != null)
+                        ret.value = row["aodc"]+"/"+row["aode"];
+                    else if(row["aode"] != null)
+                        ret.value = row["aode"];
+                    else
+                        ret.value="";
+                }
+                else if(column == "eph-age-m") {
+                    if(row["gnssid"]==0 && Math.abs(row[column]) > 120 && row["last-seen-s"] < 120)
+                        ret.color="orange";
+                    if(row["gnissid"]==2 && Math.abs(row[column]) > 60 && row["last-seen-s"] < 120)
+                        ret.color="orange";
+
+                    if(Math.abs(row[column]) >120 && row["last-seen-s"] < 120)
+                        ret.color="#ff4444";
+                    if(Math.abs(row[column]) > 4*60 && row["last-seen-s"] < 120)
+                        ret.color="#ff2222";
+
+                    
+                    if(row[column] != null) {
+                        var b = moment.duration(-row[column], 'm');
+                        ret.value = b.humanize(true);
+                    }
+                    else
+                        ret.value="";
+                }
+                else if(row[column] != null && (column == "delta_hz_corr" || column =="delta_hz")) {
+                    ret.value = row[column];
+                }
+                else if((column == "tle-dist")) {
+                    if(row["best-tle-dist"] != null) {
+                        ret.value = row["best-tle-dist"].toFixed(1) + " km";
+
+                        if (Math.abs(row["best-tle-dist"]) > 10000)
+                            ret.color="red";
+                        else if (Math.abs(row["best-tle-dist"]) > 1000)
+                            ret.color="#ffaaaa";
+                    } 
+                }
+
+                else if(column == "last-seen-s") {
+                    var b = moment.duration(row[column], 's');
+                    ret.value = b.humanize();
+                }
+                else if(column == "health")  {
+                    ret.value = "<small>"+row[column]+"</small>";
+                    if(row["healthissue"] == 1) {
+                        ret.color="orange";
+                    }
+                    if(row["healthissue"] == 2) {
+                        ret.color="red";
+                    }
+                }                
+                else {
+                    ret.value= row[column];
+                }
+
+                if(column == "sisa" && parseInt(row[column]) > 312)
+                    ret.color="#ffaaaa";
+                var myRe = RegExp('[0-9]* m');
+                if(column == "sisa" && myRe.test(row[column]))
+                    ret.color="#ff2222";
+
+                if(column == "sisa" && row[column]=="NO SISA AVAILABLE")
+                    ret.color="#ff2222";                    
+                return ret;
+            })
+        })
+        .enter().append("td").attr("class", function(d) { return d.Class; }).html(function(d) { return d.value; }).attr("align", "right").style("background-color", function(d) {
+            return d.color;
+        });
+
+}
+
+var sats={};
+var lastseen=null;
+
+function updateSats()
+{
+    var arr=[];
+    Object.keys(sats).forEach(function(e) {
+        var o = sats[e];
+        o.sv=e;
+            o.sources="";
+            o.db="";
+            o.elev="";
+            o.delta_hz_corr="";
+            o.sources="";
+            let prrestot = 0, prresnum=0, dbtot=0, hztot=0, hznum=0;
+            let mindb=1000, maxdb=0;
+            let minelev=90, maxelev=-1;
+
+            Object.keys(o.perrecv).forEach(function(k) {
+                if(o.perrecv[k]["last-seen-s"] < 30) {
+                    o.sources += k+" ";
+                    
+                    dbtot += o.perrecv[k].db;
+                    if(o.perrecv[k].db != null) {
+                        let db=o.perrecv[k].db;
+                        if(db > maxdb)
+                            maxdb = db;
+                        if(db <mindb)
+                            mindb =db;
+                    }
+                    if(o.perrecv[k].elev != null) {
+                        let elev=o.perrecv[k].elev;
+                        if(elev > maxelev)
+                            maxelev = elev;
+                        if(elev <minelev)
+                            minelev =elev;
+                    }
+
+                }
+            });
+
+            if(mindb != 1000)
+                o.db = mindb+" - " +maxdb;
+            else
+                o.db = "-";
+
+            if(maxelev != -1)
+                o.elev = minelev.toFixed(0)+" - " +maxelev.toFixed(0);
+            else
+                o.elev = "-";
+
+            if(o.hqsources > 2) {
+                if(prresnum)
+                    o.prres = (prrestot / prresnum).toFixed(2);
+                else
+                    o.prres ="-";
+                if(hznum)
+                    o.delta_hz_corr = (hztot / hznum).toFixed(2);
+                else
+                    o.delta_hz_corr ="-";
+            }
+            arr.push(o);
+        });
+
+        // sort it on SV
+        arr.sort(function(a, b) {
+            if(Number(a.sv) < Number(b.sv))
+                return -1;
+            if(Number(b.sv) < Number(a.sv))
+                return 1;
+            return 0;
+        });
+
+        var livearr=[], stalearr=[];
+
+
+        for(n = 0 ; n < arr.length; n++)
+        {
+            if(arr[n]["last-seen-s"] < 600)
+                livearr.push(arr[n]);
+            else
+                stalearr.push(arr[n]);
+        }
+    
+        maketable("#sbas", livearr);
+        maketable("#sbasstale", stalearr);
+
+}
+
+function update()
+{
+    var seconds = 2;
+    clearTimeout(repeat);
+    repeat=setTimeout(update, 1000.0*seconds);
+
+    if(lastseen != null)
+        d3.select("#freshness").html(lastseen.fromNow());
+
+    
+    d3.json("global.json", function(d) {
+
+        lastseen = moment(1000*d["last-seen"]);
+        d3.select("#freshness").html(lastseen.fromNow());
+    });
+    
+    d3.json("sbas.json", function(d) {
+        // put data in an array
+        sats=d;
+        updateSats();
+        
+    });
+}
+
+
+repeat=update();
+
+

html/sbstatus.html

@@ -18,7 +18,7 @@
   <table id="sbasstale"></table>
   <p>
 
-    Some technical detail behind this setup can be found in <a href="https://ds9a.nl/articles/posts/galileo-notes/">this post</a>.
+    Some technical detail behind this setup can be found in <a href="https://berthub.eu/articles/posts/galileo-notes/">this post</a>.
 
     For updates, follow <a href="https://twitter.com/GalileoSats">@GalileoSats</a> on Twitter, or join us on our IRC channel (chat) via the
     <a href="https://webchat.oftc.net/?channels=galileo">web gateway</a>.

html/sv.html

@@ -6,14 +6,14 @@
   <link rel="stylesheet" type="text/css" href="style.css">
 </head>
 <body>
-  Last update: <span id="freshness"></span>. More information about this Galileo/GPS/BeiDou/Glonass open source monitor can be found <a href="https://github.com/ahupowerdns/galmon/blob/master/README.md#galmon">here</a>. Live map <a href="geo">here!</a>. Contact <a href="https://ds9a.nl/">me</a> if you want access to the Grafana dashboard.<br/>
+  Last update: <span id="freshness"></span>. More information about this Galileo/GPS/BeiDou/Glonass open source monitor can be found <a href="https://github.com/ahupowerdns/galmon/blob/master/README.md#galmon">here</a>. Live map <a href="geo">here!</a>. Contact <a href="https://berthub.eu/">me</a> if you want access to the Grafana dashboard.<br/>
   <table id="galileo"></table>
   <p>
     This table shows live output from four Galileo/GPS/BeiDou/GLONASS receivers hosted in Nootdorp, The Netherlands and California, United States.
     It is very much a work in progress, and will not be available at all times. Extremely rough code is on
     <a href="https://github.com/ahuPowerDNS/galmon">GitHub</a>.
 
-    Some technical detail behind this setup can be found in <a href="https://ds9a.nl/articles/posts/galileo-notes/">this post</a>.
+    Some technical detail behind this setup can be found in <a href="https://berthub.eu/articles/posts/galileo-notes/">this post</a>.
 
     For updates, follow <a href="https://twitter.com/GalileoSats">@GalileoSats</a> on Twitter, or join us on our IRC channel (chat) via the
     <a href="https://webchat.oftc.net/?channels=galileo">web gateway</a>.

influxdb.md

@@ -84,8 +84,8 @@ These measurements are tagged by gnssid, sv
     * tow
     * udi
   * rtcm-clock-correction
-    * dclock0
-    * dclock1
+    * dclock0: in meters
+    * dclock1: meters/s
     * dclock2
     * ssr-iod
     * ssr-provider
@@ -131,9 +131,37 @@ Observer and SV measurements:
    * ele: calculated elevation for SV from this receiver
    * prres: pseudorange residual according to receiver
    * qi: 0-7, quality indicator according to receiver
+   * used: did the receiver use this SV?
  * ubx\_jamming
    * noise\_per\_ms: the Ublox noisePerMS field
    * agccnt: the Ublox automatic gain correction "count"
    * jamind: The Ublox jamming indicator
    * flag: The Ublox jamming flag field
 
+Fed by separate tool:
+
+SP3 design, tagged by GNSSID, SV:
+ * sp3\_data:
+   * x
+   * y
+   * z
+   * clk
+   * provider
+
+ephemeris, tagged by GNSSID, SV, SIGID:
+ * active-ephemeris
+   * all the raw parameters
+
+GDOP/PDOP stats?
+ * covdop
+   * lat
+   * lon
+   * cov5
+   * cov10
+   * cov20
+   * hdop5
+   * hdop10
+   * hdop20
+
+etc
+

influxpush.cc

@@ -13,9 +13,8 @@ InfluxPusher::InfluxPusher(std::string_view dbname) : d_dbname(dbname)
 
 void InfluxPusher::queueValue(const std::string& line)
 {
-  d_buffer.insert(line);
-  //  if(d_buffer.insert(line).second)
-  //  cout<<line;
+  if(!d_buffer.insert(line).second)
+    d_numdedupmsmts++;
   checkSend();
 }
 
@@ -41,6 +40,9 @@ void InfluxPusher::addValueObserver(int src, string_view name, const initializer
   buffer+= " ";
   bool lefirst=true;
   for(const auto& v : values) {
+    if(!v.first) // trick to null out certain fields
+      continue;
+    d_numvalues++;
     if(!lefirst) {
       buffer +=",";
     }
@@ -48,18 +50,33 @@ void InfluxPusher::addValueObserver(int src, string_view name, const initializer
     buffer += string(v.first) + "="+to_string(v.second);
   }
   buffer += " " + to_string((uint64_t)(t* 1000000000))+"\n";
-
+  d_nummsmts++;
+  d_msmtmap[(string)name]++;
   queueValue(buffer);
 }
 
-  
+
 void InfluxPusher::addValue(const SatID& id, string_view name, const initializer_list<pair<const char*, var_t>>& values, double t, std::optional<int> src, std::optional<string> tag)
+{
+
+  vector<pair<string,var_t>> tags{{"sv", id.sv}, {"gnssid", id.gnss}, {"sigid", id.sigid}};
+
+  if(src) {
+    tags.push_back({*tag, *src});
+  }
+  addValue(tags, name, values, t);
+}
+
+void InfluxPusher::addValue(const vector<pair<string,var_t>>& tags, string_view name, const initializer_list<pair<const char*, var_t>>& values, double t)
 {
   if(d_mute)
     return;
 
   if(t > 2200000000 || t < 0) {
-    cerr<<"Unable to store item "<<name<<" for sv "<<id.gnss<<","<<id.sv<<": time out of range "<<t<<endl;
+    cerr<<"Unable to store item "<<name<<" for ";
+    //    for(const auto& t: tags)
+    //      cerr<<t.first<<"="<<t.second<<" ";
+    cerr<<": time out of range "<<t<<endl;
     return;
   }
   for(const auto& p : values) {
@@ -68,13 +85,29 @@ void InfluxPusher::addValue(const SatID& id, string_view name, const initializer
         return;
   }
 
-  string buffer = string(name) +",gnssid="+to_string(id.gnss)+",sv=" +to_string(id.sv)+",sigid="+to_string(id.sigid);
-  if(src)
-    buffer += ","+*tag+"="+to_string(*src);
+  string buffer = string(name);
+  for(const auto& t : tags) {
+    buffer += ","+t.first + "=";
+    std::visit([&buffer](auto&& arg) {
+        using T = std::decay_t<decltype(arg)>;
+        if constexpr (std::is_same_v<T, string>) {
+                      // string tags really don't need a "
+                       buffer += arg;
+          }
+        else {
+          // resist the urge to do integer tags, it sucks
+          buffer += to_string(arg);
+        }
+      }, t.second);
+  }
   
   buffer+= " ";
   bool lefirst=true;
   for(const auto& v : values) {
+    if(!v.first) // trick to null out certain fields
+      continue;
+
+    d_numvalues++;
     if(!lefirst) {
       buffer +=",";
     }
@@ -83,14 +116,19 @@ void InfluxPusher::addValue(const SatID& id, string_view name, const initializer
 
     std::visit([&buffer](auto&& arg) {
         using T = std::decay_t<decltype(arg)>;
-        buffer += to_string(arg);
-        if constexpr (!std::is_same_v<T, double>)
-                       buffer+="i";
+        if constexpr (std::is_same_v<T, string>)
+                       buffer += "\""+arg+"\"";
+        else {
+          buffer += to_string(arg);
+          if constexpr (!std::is_same_v<T, double>)
+                         buffer+="i";
+        }
       }, v.second);
   }
   buffer += " " + to_string((uint64_t)(t*1000000000))+"\n";
+  d_nummsmts++;
+  d_msmtmap[(string)name]++;
   queueValue(buffer);
-  
 }
 
 

influxpush.hh

@@ -8,11 +8,13 @@
 
 struct InfluxPusher
 {
-  typedef std::variant<double, int32_t, uint32_t, int64_t> var_t;
+  typedef std::variant<double, int32_t, uint32_t, int64_t, string> var_t;
   explicit InfluxPusher(std::string_view dbname);
   void addValueObserver(int src, std::string_view name, const std::initializer_list<std::pair<const char*, double>>& values, double t, std::optional<SatID> satid=std::optional<SatID>());
   void addValue(const SatID& id, std::string_view name, const std::initializer_list<std::pair<const char*, var_t>>& values, double t, std::optional<int> src = std::optional<int>(), std::optional<string> tag = std::optional<string>("src"));
-  
+
+
+  void addValue(const vector<pair<string, var_t>>& tags, string_view name, const initializer_list<pair<const char*, var_t>>& values, double t);
   void checkSend();
   void doSend(const std::set<std::string>& buffer);
   ~InfluxPusher();
@@ -22,4 +24,8 @@ struct InfluxPusher
   time_t d_lastsent{0};
   string d_dbname;
   bool d_mute{false};
+  int64_t d_nummsmts{0};
+  int64_t d_numvalues{0};
+  int64_t d_numdedupmsmts{0};
+  map<std::string, int64_t> d_msmtmap;
 };

navcat.cc

@@ -26,29 +26,7 @@ using namespace std;
 
 extern const char* g_gitHash;
 
-time_t parseTime(std::string_view in)
-{
-  time_t now=time(0);
-
-  vector<string> formats({"%Y-%m-%d %H:%M", "%Y%m%d %H%M", "%H:%M", "%H%M"});
-  for(const auto& f : formats) {
-    struct tm tm;
-    memset(&tm, 0, sizeof(tm));
-
-    localtime_r(&now, &tm);
-    tm.tm_isdst = -1;
-    tm.tm_sec = 0;
-    char* res = strptime(&in[0], f.c_str(), &tm);
-    if(res && !*res) {
-      cerr<<"Matched on "<<f<<endl;
-      return mktime(&tm);
-    }
-  }
-  
-  throw runtime_error("Can only parse %Y-%m-%d %H:%M");
-}
-
-
+// get all stations (numerical) from a directory
 vector<uint64_t> getSources(string_view dirname)
 {
   DIR *dir = opendir(&dirname[0]);
@@ -76,8 +54,13 @@ vector<uint64_t> getSources(string_view dirname)
   return ret;
 }
 
+static bool operator==(const timespec& a, const timespec& b)
+{
+  return a.tv_sec == b.tv_sec && a.tv_nsec && b.tv_nsec;
+}
 
-void sendProtobuf(string_view dir, time_t startTime, time_t stopTime=0)
+// send protobuf data from the named directories and stations, between start and stoptime
+void sendProtobuf(const vector<string>& dirs, vector<uint64_t> stations, time_t startTime, time_t stopTime=0)
 {
   timespec start;
   start.tv_sec = startTime;
@@ -86,43 +69,56 @@ void sendProtobuf(string_view dir, time_t startTime, time_t stopTime=0)
   // so we have a ton of files, and internally these are not ordered
   map<string,uint32_t> fpos;
   vector<pair<timespec,string> > rnmms;
+
   for(;;) {
-    auto srcs = getSources(dir);
     rnmms.clear();
-    for(const auto& src: srcs) {
-      string fname = getPath(dir, start.tv_sec, src);
-      FILE* fp = fopen(fname.c_str(), "r");
-      if(!fp)
-        continue;
-      uint32_t offset= fpos[fname];
-      if(fseek(fp, offset, SEEK_SET) < 0) {
-        cerr<<"Error seeking: "<<strerror(errno) <<endl;
-        fclose(fp);
-        continue;
-      }
-      cerr <<"Seeked to position "<<fpos[fname]<<" of "<<fname<<endl;
-
-      uint32_t looked=0;
-      string msg;
-      struct timespec ts;
-      while(getRawNMM(fp, ts, msg, offset)) {
-        // don't drop data that is only 5 seconds too old
-        if(make_pair(ts.tv_sec + 5, ts.tv_nsec) >= make_pair(start.tv_sec, start.tv_nsec)) {
-          rnmms.push_back({ts, msg});
+    for(const auto& dir : dirs) {
+      cerr<<"Gathering data from "<<humanTime(start.tv_sec)<<" from "<<dir<<".. ";
+          
+      vector<uint64_t> srcs = stations.empty() ? getSources(dir) : stations;
+      int count=0;
+      for(const auto& src: srcs) {
+        string fname = getPath(dir, start.tv_sec, src);
+        FILE* fp = fopen(fname.c_str(), "r");
+        if(!fp)
+          continue;
+        uint32_t offset= fpos[fname];
+        if(fseek(fp, offset, SEEK_SET) < 0) {
+          cerr<<"Error seeking: "<<strerror(errno) <<endl;
+          fclose(fp);
+          continue;
         }
-        ++looked;
-      }
-      cerr<<"Harvested "<<rnmms.size()<<" events out of "<<looked<<endl;
-      fpos[fname]=offset;
-      fclose(fp);
-    }
+        //      cerr <<"Seeked to position "<<fpos[fname]<<" of "<<fname<<endl;
+        
+        string msg;
+        struct timespec ts;
+        while(getRawNMM(fp, ts, msg, offset)) {
+          // don't drop data that is only 5 seconds too old
+          if(make_pair(ts.tv_sec + 5, ts.tv_nsec) >= make_pair(start.tv_sec, start.tv_nsec)) {
+            rnmms.push_back({ts, msg});
+            ++count;
+          }
+        }
+        //      cerr<<"Harvested "<<rnmms.size()<<" events out of "<<looked<<endl;
 
+        fpos[fname]=offset;
+        fclose(fp);
+      }
+      cerr<<" added "<<count<<endl;
+    }
+    //    cerr<<"Sorting data"<<endl;
     sort(rnmms.begin(), rnmms.end(), [](const auto& a, const auto& b)
          {
            return std::tie(a.first.tv_sec, a.first.tv_nsec)
                 < std::tie(b.first.tv_sec, b.first.tv_nsec);
          });
 
+    auto newend=unique(rnmms.begin(), rnmms.end());
+    cerr<<"Removed "<<rnmms.end() - newend <<" duplicates, ";
+    
+    rnmms.erase(newend, rnmms.end());
+    cerr<<"sending data"<<endl;
+    unsigned int count=0;
     for(const auto& nmm: rnmms) {
       if(nmm.first.tv_sec > stopTime)
         break;
@@ -132,7 +128,9 @@ void sendProtobuf(string_view dir, time_t startTime, time_t stopTime=0)
       buf += nmm.second;
       //fwrite(buf.c_str(), 1, buf.size(), stdout);
       writen2(1, buf.c_str(), buf.size());
+      ++count;
     }
+    cerr<<"Done sending " << count<<" messages"<<endl;
     if(3600 + start.tv_sec - (start.tv_sec%3600) < stopTime)
       start.tv_sec = 3600 + start.tv_sec - (start.tv_sec%3600);
     else {
@@ -145,32 +143,46 @@ void sendProtobuf(string_view dir, time_t startTime, time_t stopTime=0)
 int main(int argc, char** argv)
 {
   bool doVERSION{false};
-  /*
+
   CLI::App app(program);
-
+  string beginarg, endarg;
+  vector<string> storages;
+  int galwn{-1};
+  app.add_option("--storage,-s", storages, "Locations of storage files");  
+  vector<uint64_t> stations;
   app.add_flag("--version", doVERSION, "show program version and copyright");
-  app.allow_extras(true); // allow bare positional parameters
-  try {
-    app.parse(argc, argv);
-  } catch(const CLI::Error &e) {
-    return app.exit(e);
-  }
+  app.add_option("--begin,-b", beginarg, "Begin time (2020-01-01 00:00, or 12:30)");
+  app.add_option("--end,-e", endarg, "End time. Now if omitted");
+  app.add_option("--stations", stations, "only send data from listed stations");
+  app.add_option("--gal-wn", galwn, "Galileo week number to report on");
+  CLI11_PARSE(app, argc, argv);
 
+  
   if(doVERSION) {
     showVersion(program, g_gitHash);
     exit(0);
   }
-  */
-  signal(SIGPIPE, SIG_IGN);
-  if(argc < 3) {
-    cout<<"Syntax: navcat storage start stop"<<endl;
-    cout<<"Example: ./navcat storage \"2020-01-01 00:00\" \"2020-01-02 00:00\" | ./navdump  "<<endl;
-    return(EXIT_FAILURE);
+
+  time_t startTime, stopTime;
+  if(galwn >=0) {
+    startTime=utcFromGST(galwn, 0);
+    stopTime=startTime + 7*86400;
+  }
+  else if(!beginarg.empty()) {
+    startTime = parseTime(beginarg);
+    stopTime = endarg.empty()  ? time(0) :  parseTime(endarg);
+  }
+  else {
+    cerr<<"No time range specified, use -b or --gal-wn"<<endl;
+    return 1;
   }
-  time_t startTime = parseTime(argv[2]);
-  time_t stopTime = parseTime(argv[3]);
 
   cerr<<"Emitting from "<<humanTime(startTime) << " to " << humanTime(stopTime) << endl;
-  sendProtobuf(argv[1], startTime, stopTime);
-  
+  if(!stations.empty()) {
+    cerr<<"Restricting to stations:";
+    for(const auto& s : stations)
+      cerr<<" "<<s;
+    cerr<<endl;
+  }
+  sendProtobuf(storages, stations, startTime, stopTime);
 }

navdump.cc

@@ -25,6 +25,7 @@
 #include "tle.hh"
 #include "sp3.hh"
 #include "ubx.hh"
+#include <optional>
 #include <unistd.h>
 #include "sbas.hh"
 #include "version.hh"
@@ -257,12 +258,20 @@ try
   bool doReceptionData{false};
   bool doRFData{true};
   bool doObserverPosition{false};
+  bool doObserverDetails{false};
+  bool doTimeOffsets{false};
   bool doVERSION{false};
+  string rinexfname;
+  string osnmafname;
   app.add_option("--svs", svpairs, "Listen to specified svs. '0' = gps, '2' = Galileo, '2,1' is E01");
   app.add_option("--stations", stations, "Listen to specified stations.");
   app.add_option("--positions,-p", doObserverPosition, "Print out observer positions (or not)");
   app.add_option("--rfdata,-r", doRFData, "Print out RF data (or not)");
+  app.add_option("--observerdetails,-o", doObserverDetails, "Print out observer detail data (or not)");
+  app.add_option("--timeoffsets,-t", doTimeOffsets, "Print out timeoffset data (or not)");
   app.add_option("--recdata", doReceptionData, "Print out reception data (or not)");
+  app.add_option("--rinex", rinexfname, "If set, emit ephemerides to this filename");
+  app.add_option("--osnma", osnmafname, "If set, emit OSNMA CSV to this filename");
   app.add_flag("--version", doVERSION, "show program version and copyright");
     
   try {
@@ -299,7 +308,18 @@ try
 
   sp3csv<<"timestamp gnssid sv ephAge sp3X sp3Y sp3Z ephX ephY ephZ  sp3Clock ephClock distance along clockDelta E speed"<<endl;
 
-  //  RINEXNavWriter rnw("test.rnx");
+  std::optional<RINEXNavWriter> rnw;
+
+  if(!rinexfname.empty())
+    rnw = RINEXNavWriter(rinexfname);
+
+  std::optional<ofstream> osnmacsv;
+  if(!osnmafname.empty()) {
+    osnmacsv = ofstream(osnmafname);
+    (*osnmacsv)<<"wn,tow,wtype,sv,osnma\n";
+
+  }
+  
   for(;;) {
     char bert[4];
     int res = readn2(0, bert, 4);
@@ -310,7 +330,23 @@ try
     
     // I am so sorry
     if(bert[0]!='b' || bert[1]!='e' || bert[2] !='r' || bert[3]!='t') {
-      cerr<<"Bad magic"<<endl;
+      cerr<<"Bad magic: "<<makeHexDump(string(bert, 4))<<endl;
+      int res;
+      for(int s=0;;++s) {
+        cerr<<"Skipping character hunting for good magic.. "<<s<<endl;
+        bert[0] = bert[1];
+        bert[1] = bert[2];
+        bert[2] = bert[3];
+        res = readn2(0, bert + 3, 1);
+        if(res != 1)
+          break;
+        if(bert[0]=='b' && bert[1]=='e' && bert[2] =='r' && bert[3]=='t')
+          break;
+      }
+      if(res != 1) {
+        cerr<<"EOF2, res = "<<res<<endl;
+        break;
+      }
     }
     
     uint16_t len;
@@ -372,13 +408,26 @@ try
       int wtype = gm.parse(inav);
       
       gm.tow = nmm.gi().gnsstow();
+      bool isnew = gmwtypes[{nmm.gi().gnsssv(), wtype}].tow != gm.tow;
       gmwtypes[{nmm.gi().gnsssv(), wtype}] = gm;
       static map<int,GalileoMessage> oldEph;
       cout << "gal inav wtype "<<wtype<<" for "<<nmm.gi().gnssid()<<","<<nmm.gi().gnsssv()<<","<<nmm.gi().sigid()<<" pbwn "<<nmm.gi().gnsswn()<<" pbtow "<< nmm.gi().gnsstow();
       static uint32_t tow;
+
+      if(osnmacsv && isnew)
+	(*osnmacsv)<<nmm.gi().gnsswn()<<","<<gm.tow<<","<<wtype<<","<<nmm.gi().gnsssv()<<","<<makeHexDump(nmm.gi().reserved1())<<endl;
+
+      
+      if(wtype >=1 && wtype <= 5) {
+        if(nmm.gi().has_reserved1()) {
+          cout<<" res1 "<<makeHexDump(nmm.gi().reserved1());
+	}
+      }
       if(wtype == 4) {
         //              2^-34       2^-46
         cout <<" iodnav "<<gm.iodnav <<" af0 "<<gm.af0 <<" af1 "<<gm.af1 <<", scaled: "<<ldexp(1.0*gm.af0, 19-34)<<", "<<ldexp(1.0*gm.af1, 38-46);
+        cout << " t0g " << gm.t0g <<" a0g " << gm.a0g <<" a1g " << gm.a1g <<" WN0g " << gm.wn0g;
+
         if(tow && oldgm4s.count(nmm.gi().gnsssv()) && oldgm4s[nmm.gi().gnsssv()].iodnav != gm.iodnav) {
           
           auto& oldgm4 = oldgm4s[nmm.gi().gnsssv()];
@@ -437,7 +486,9 @@ try
           if(!oldEph[sv].sqrtA)
             oldEph[sv] = gm;
           else if(oldEph[sv].iodnav != gm.iodnav) {
-            //            rnw.emitEphemeris(sid, gm);
+	    if(rnw)
+	      rnw->emitEphemeris(sid, gm);
+	    //	    gm.toJSON();
 
             cout<<" disco! "<< oldEph[sv].iodnav << " - > "<<gm.iodnav <<", "<< (gm.getT0e() - oldEph[sv].getT0e())/3600.0 <<" hours-jump insta-age "<<ephAge(gm.tow, gm.getT0e())/3600.0<<" hours";
             Point oldPoint, newPoint;
@@ -482,7 +533,9 @@ try
       }
       if(wtype == 6) {
         cout<<" a0 " << gm.a0 <<" a1 " << gm.a1 <<" t0t "<<gm.t0t << " dtLS "<<(int)gm.dtLS;
+        cout <<" wnLSF "<< (unsigned int)gm.wnLSF<<" dn " << (unsigned int)gm.dn<< " dtLSF "<<(int)gm.dtLSF<<endl;
       }
+
       
       //      if(wtype < 7)
       //        gm = GalileoMessage{};
@@ -519,11 +572,66 @@ try
         if(gm.alma3.e1bhs != 0) {
           cout <<" gm.tow "<<gm.tow<<" gmwtypes.tow "<< gmwtypes[{sv,9}].tow <<" ";
         }
-        cout<<"  "<<gmwtypes[{sv,9}].alma3.svid <<" af0 "<<gm.alma3.af0<<" af1 "<< gm.alma3.af1 <<" e5bhs "<< gm.alma3.e5bhs<<" e1bhs "<< gm.alma3.e1bhs <<" a0g " << gm.a0g <<" a1g "<< gm.a1g <<" t0g "<<gm.t0g <<" wn0g "<<gm.wn0g;
+        cout<<"  alma3.sv "<<gmwtypes[{sv,9}].alma3.svid <<" af0 "<<gm.alma3.af0<<" af1 "<< gm.alma3.af1 <<" e5bhs "<< gm.alma3.e5bhs<<" e1bhs "<< gm.alma3.e1bhs <<" a0g " << gm.a0g <<" a1g "<< gm.a1g <<" t0g "<<gm.t0g <<" wn0g "<<gm.wn0g <<" delta-gps "<< gm.getGPSOffset(gm.tow, gm.wn).first<<"ns";
+
+        int dw = (int)(gm.wn%64) - (int)(gm.wn0g);
+        if(dw > 31)
+          dw = 31- dw;
+        int delta = dw*7*86400  + gm.tow - gm.getT0g(); // NOT ephemeris age tricks
+        cout<<" wn%64 "<< (gm.wn%64) << " dw " << dw <<" delta " << delta;
       }
       
       cout<<endl;      
     }
+    else if(nmm.type() == NavMonMessage::GalileoCnavType) {
+      basic_string<uint8_t> cnav((uint8_t*)nmm.gc().contents().c_str(), nmm.gc().contents().size());
+      int sv = nmm.gc().gnsssv();
+      if(!svfilter.check(2, sv, nmm.gc().sigid()))
+        continue;
+      etstamp();
+      cout << "C/NAV for " << nmm.gc().gnssid()<<","<<nmm.gc().gnsssv()<<","<<nmm.gc().sigid() <<": header ";
+      cout<<fmt::sprintf("%02x%02x%02x (status %d, MT %d, MID %d, MS %d, PID %d)  rest ",
+			 getbitu(cnav.c_str(), 14, 8),
+			 getbitu(cnav.c_str(), 22, 8),
+			 getbitu(cnav.c_str(), 30, 8),
+			 getbitu(cnav.c_str(), 14+0, 2),  // status
+			 getbitu(cnav.c_str(), 14+4, 2),  // MT
+			 getbitu(cnav.c_str(), 14+6, 5),  // MID
+			 getbitu(cnav.c_str(), 14+11, 5),  // MIS
+			 getbitu(cnav.c_str(), 14+16, 8)  // PID
+			 
+			 );
+      for(int n=0; n < 51; ++n)
+	cout << fmt::sprintf("%02x ", getbitu(cnav.c_str(), 38 +n *8, 8));
+      cout<<endl;
+
+    }
+    else if(nmm.type() == NavMonMessage::GalileoFnavType) {
+      basic_string<uint8_t> fnav((uint8_t*)nmm.gf().contents().c_str(), nmm.gf().contents().size());
+      int sv = nmm.gf().gnsssv();
+      if(!svfilter.check(2, sv, nmm.gf().sigid()))
+        continue;
+      etstamp();
+      GalileoMessage gm;
+      gm.parseFnav(fnav);
+      cout<<"gal F/NAV wtype "<< (int)gm.wtype << " for "<<nmm.gf().gnssid()<<","<<nmm.gf().gnsssv()<<","<<nmm.gf().sigid();
+      if(gm.wtype ==1 || gm.wtype == 2 || gm.wtype == 3 || gm.wtype == 4)
+        cout<<" iodnav " <<gm.iodnav <<" tow " << gm.tow;
+      if(gm.wtype == 1) {
+        cout <<" af0 "<<gm.af0 << " af1 "<<gm.af1 <<" af2 "<< (int)gm.af2;
+      }
+      if(gm.wtype == 2) {
+        cout <<" sqrtA "<<gm.sqrtA;
+      }
+      if(gm.wtype == 3) {
+        cout <<" t0e "<<gm.t0e;
+      }
+      if(gm.wtype == 4) {
+        cout <<" dtLS "<<(int)gm.dtLS <<" wnLSF "<< (unsigned int)gm.wnLSF<<" dn " << (unsigned int)gm.dn<< " dtLSF "<<(int)gm.dtLSF<<endl;
+      }
+
+      cout<<endl;
+    }
     else if(nmm.type() == NavMonMessage::GPSInavType) {
       int sv = nmm.gpsi().gnsssv();
 
@@ -545,7 +653,7 @@ try
       if(frame == 1) {
 
         gpswn = gs.wn;
-        cout << "gpshealth = "<<(int)gs.gpshealth<<", wn "<<gs.wn << " t0c "<<gs.t0c << " af0 " << gs.af0 << " af1 " << gs.af1 <<" af2 " << gs.af2;
+        cout << "gpshealth = "<<(int)gs.gpshealth<<", wn "<<gs.wn << " t0c "<<gs.t0c << " af0 " << gs.af0 << " af1 " << gs.af1 <<" af2 " << (int)gs.af2;
         if(auto iter = oldgs1s.find(sv); iter != oldgs1s.end() && iter->second.t0c != gs.t0c) {
           auto oldOffset = getGPSAtomicOffset(gs.tow, iter->second);
           auto newOffset = getGPSAtomicOffset(gs.tow, gs);
@@ -630,7 +738,7 @@ try
           cout<<" 2nd-match "<<second.name << " dist "<<second.distance/1000<<" km t0e "<<gs.gpsalma.getT0e() << " t " <<nmm.localutcseconds();
         }
         if(page == 18)
-          cout << " dtLS " << (int)gs.dtLS <<" dtLSF "<< (int)gs.dtLSF;
+          cout << " wnLSF "<< (int)gs.wnLSF <<" dn " << (int)gs.dn << " t0t " << (int)gs.t0t <<" wn0t "<<(int)gs.wn0t<<" dtLS " << (int)gs.dtLS <<" dtLSF "<< (int)gs.dtLSF;
       }
       else if(frame == 5) {
         if(gs.gpsalma.sv <= 24) {
@@ -653,17 +761,72 @@ try
       etstamp();
       RTCMMessage rm;
       rm.parse(nmm.rm().contents());
+      cout<<" rtcm-msg "<<rm.type<<" ";
       if(rm.type == 1057 || rm.type == 1240) {
+        cout<<"iod-ssr "<<rm.ssrIOD<<" ";
         for(const auto& ed : rm.d_ephs) {
-          cout<<makeSatPartialName(ed.id)<<":  iode "<< ed.iod<<" ("<< ed.radial<<", "<<ed.along<<", "<<ed.cross<<") mm -> (";
+          cout<<makeSatIDName(ed.id)<<":  iode "<< ed.iod<<" ("<< ed.radial<<", "<<ed.along<<", "<<ed.cross<<") mm -> (";
           cout<< ed.dradial<<", "<<ed.dalong<<", "<<ed.dcross<< ") mm/s\n";
         }
       }
       else if(rm.type == 1058 || rm.type == 1241) {
+        cout<<"iod-ssr "<<rm.ssrIOD<<" ";
         for(const auto& cd : rm.d_clocks) {
-          cout<<makeSatPartialName(cd.id)<<":  dclock0 "<< cd.dclock0 <<" dclock1 " << cd.dclock1 <<" dclock2 "<< cd.dclock2 << endl;
+          cout<<makeSatIDName(cd.id)<<":  dclock0 "<< cd.dclock0 <<" dclock1 " << cd.dclock1 <<" dclock2 "<< cd.dclock2 << endl;
         }
       }
+      else if (rm.type == 1060 || rm.type == 1243) {
+        for(const auto& ed : rm.d_ephs) {
+          cout<<makeSatIDName(ed.id)<<":  iode "<< ed.iod<<" ("<< ed.radial<<", "<<ed.along<<", "<<ed.cross<<") mm -> (";
+          cout<< ed.dradial<<", "<<ed.dalong<<", "<<ed.dcross<< ") mm/s\n";
+        }
+
+        for(const auto& cd : rm.d_clocks) {
+          cout<<makeSatIDName(cd.id)<<":  dclock0 "<< cd.dclock0 <<" dclock1 " << cd.dclock1 <<" dclock2 "<< cd.dclock2 << endl;
+        }
+      }
+      else if(rm.type == 1045 || rm.type == 1046) {
+        static ofstream af0inavstr("af0inav.csv"), af0fnavstr("af0fnav.csv"), bgdstr("bgdstr.csv");
+        static bool first{true};
+
+        if(first) {
+          af0inavstr<<"timestamp sv wn t0c af0 af1\n";
+          af0fnavstr<<"timestamp sv wn t0c af0 af1\n";
+          first=false;
+        }
+        SatID sid;
+        sid.gnss = 2;
+        sid.sv = rm.d_sv;
+        sid.sigid = (rm.type == 1045) ? 5 : 1;
+        
+        cout<< makeSatIDName(sid)<<" ";
+        if(rm.type == 1045) {
+          af0fnavstr << nmm.localutcseconds()<<" " << rm.d_sv <<" " << rm.d_gm.wn<<" "<< rm.d_gm.t0c << " " << rm.d_gm.af0 << " " << rm.d_gm.af1<<"\n";
+          cout<<"F/NAV";
+        }
+        else {
+          af0inavstr << nmm.localutcseconds() <<" " << rm.d_sv <<" " << rm.d_gm.wn<<" "<<rm.d_gm.t0c<<" "<< rm.d_gm.af0 << " " << rm.d_gm.af1<<"\n";
+          bgdstr << nmm.localutcseconds() <<" " << rm.d_sv<<" " <<rm.d_gm.BGDE1E5a <<" " << rm.d_gm.BGDE1E5b << "\n";
+          cout <<"I/NAV";
+        }
+
+        cout <<" iode " << rm.d_gm.iodnav << " sisa " << (unsigned int) rm.d_gm.sisa << " t0c " << rm.d_gm.t0c << " af0 "<<rm.d_gm.af0 <<" af1 " << rm.d_gm.af1 <<" af2 " << (int) rm.d_gm.af2 << " BGDE1E5a " << rm.d_gm.BGDE1E5a;
+        if(rm.type == 1046) // I/NAV
+          cout <<" BGDE1E5b "<< rm.d_gm.BGDE1E5b;
+        cout<<endl;
+      }
+      else if(rm.type == 1059 || rm.type==1242) {
+        cout<<"\n";
+        for(const auto& dcb : rm.d_dcbs) {
+          cout<<"   "<<makeSatIDName(dcb.first)<<":  "<<dcb.second<<" meters\n";
+        }
+
+        cout<<endl;
+      }
+      else {
+        cout<<" len " << nmm.rm().contents().size() << endl;
+        
+      }
 
     }
     else if(nmm.type() == NavMonMessage::GPSCnavType) {
@@ -715,7 +878,7 @@ try
           cout<<"  Timejump: "<<oldOffset.first - newOffset.first<<" after "<<(bm.getT0c() - msgs[sv].getT0c() )<<" seconds";
         }
         msgs[sv]=bm;
-        cout<<" wn "<<bm.wn<<" t0c "<<(int)bm.t0c<<" aodc "<< (int)bm.aodc <<" aode "<< (int)bm.aode <<" sath1 "<< (int)bm.sath1 << " urai "<<(int)bm.urai << " af0 "<<bm.a0 <<" af1 " <<bm.a1 <<" af2 "<<bm.a2;
+        cout<<" wn "<<bm.wn<<" t0c "<<(int)bm.t0c<<" aodc "<< (int)bm.aodc <<" aode "<< (int)bm.aode <<" sath1 "<< (int)bm.sath1 << " urai "<<(int)bm.urai << " af0 "<<bm.a0 <<" af1 " <<bm.a1 <<" af2 "<< (int)bm.a2;
         auto offset = bm.getAtomicOffset();
         cout<< ", "<<offset.first<<"ns " << (offset.second * 3600) <<" ns/hour "<< ephAge(bm.sow, bm.t0c*8);
       }
@@ -727,8 +890,6 @@ try
         cout<<" best-tle-match "<<match.name <<" dist "<<match.distance /1000<<" km";
         cout<<" norad " <<match.norad <<" int-desig " << match.internat;
         cout<<" 2nd-match "<<second.name << " dist "<<second.distance/1000<<" km";
-
-
       }
       else if((fraid == 4 && 1<= pageno && pageno <= 24) ||
               (fraid == 5 && 1<= pageno && pageno <= 6) ||
@@ -759,7 +920,7 @@ try
         cout<<" WNa "<<getbitu(&cond[0], beidouBitconv(190), 8)<<" t0a "<<getbitu(&cond[0], beidouBitconv(198), 8);
       }
       else if(bm.fraid == 5 && pageno==10) {
-        cout <<" dTLS "<< (int)bm.deltaTLS;
+        cout <<" dTLS "<< (int)bm.deltaTLS << " dTLSF " << (int) bm.deltaTLSF <<" wnLSF " << (unsigned int)bm.wnLSF <<" dn "<<(unsigned int)  bm.dn<<endl;
       }
       else if(bm.fraid == 5 && pageno==24) {
         int AmID= getbitu(&cond[0], beidouBitconv(216), 2);
@@ -960,17 +1121,19 @@ try
       
     }
     else if(nmm.type() == NavMonMessage::ObserverDetailsType) {
-      etstamp();
-      cout<<"vendor "<<nmm.od().vendor()<<" hwversion " <<nmm.od().hwversion()<<" modules "<<nmm.od().modules()<<" swversion "<<nmm.od().swversion();
-      cout<<" serial "<<nmm.od().serialno();
-      if(nmm.od().has_owner())
-        cout<<" owner "<<nmm.od().owner();
-      if(nmm.od().has_clockoffsetdriftns())
-        cout<<" drift "<<nmm.od().clockoffsetdriftns();
-      if(nmm.od().has_clockaccuracyns())
-        cout<<" clock-accuracy "<<nmm.od().clockaccuracyns();
-      
-      cout<<endl;
+      if(doObserverDetails) {
+	etstamp();
+	cout<<"vendor "<<nmm.od().vendor()<<" hwversion " <<nmm.od().hwversion()<<" modules "<<nmm.od().modules()<<" swversion "<<nmm.od().swversion();
+	cout<<" serial "<<nmm.od().serialno();
+	if(nmm.od().has_owner())
+	  cout<<" owner "<<nmm.od().owner();
+	if(nmm.od().has_clockoffsetdriftns())
+	  cout<<" drift "<<nmm.od().clockoffsetdriftns();
+	if(nmm.od().has_clockaccuracyns())
+	  cout<<" clock-accuracy "<<nmm.od().clockaccuracyns();
+	
+	cout<<endl;
+      }
     }
     else if(nmm.type() == NavMonMessage::UbloxJammingStatsType) {
       etstamp();
@@ -1128,6 +1291,29 @@ try
       }
       cout<<endl;
     }
+    else if(nmm.type() == NavMonMessage::SARResponseType) {    
+      etstamp();
+
+      string hexstring;
+      string id = nmm.sr().identifier();
+      for(int n = 0; n < 15; ++n)
+        hexstring+=fmt::sprintf("%x", (int)getbitu((unsigned char*)id.c_str(), 4 + 4*n, 4));
+
+      
+      cout<<" SAR RLM type "<< nmm.sr().type() <<" from gal sv ";
+      cout<< nmm.sr().gnsssv() << " beacon "<<hexstring <<" code "<<(int)nmm.sr().code()<<" params "<< makeHexDump(nmm.sr().params()) <<endl;
+    }
+    else if(nmm.type() == NavMonMessage::TimeOffsetType) {
+      if(doTimeOffsets) {
+	etstamp();
+	cout<<" got a time-offset message with "<< nmm.to().offsets().size()<<" offsets: ";
+	for(const auto& o : nmm.to().offsets()) {
+	  cout << "gnssid "<<o.gnssid()<<" offset " << o.offsetns() << " +- "<<o.tacc()<<" ("<<o.valid()<<") , ";
+	}
+	cout<<endl;
+      }
+            
+    }
     else {
       etstamp();
       cout<<"Unknown type "<< (int)nmm.type()<<endl;

navmerge.cc

@@ -0,0 +1,172 @@
+#include "sclasses.hh"
+#include <map>
+#include "navmon.hh"
+#include "navmon.pb.h"
+#include <thread>
+#include "nmmsender.hh"
+#include "CLI/CLI.hpp"
+#include "version.hh"
+
+using namespace std;
+
+static char program[]="navmerge";
+
+/* ubxtool/rtcmtool/septool deliver data to one of several `navrecv` instances
+   This means we need a 'merge' tool.
+
+   The merge tool should be able to stream data from multiple `navnexus` instances 
+      (that correspond to the `navrecv` instances)
+
+   Currently, `navnexus` is really simple - it will send you a feed from x hours back, where 
+   you don't get to pick x.
+
+   The simplest navmerge implementation does nothing but connect to a few navnexus instances
+   and it mixes them together.
+
+   Every message "should" only appear on one of the upstreams, but you never know. 
+
+   On initial connection, the different navnexuses may start up from a different time, currently.
+   Let us state that This Should Not Happen.
+
+   On initial connect, a navnexus might take dozens of seconds before it starts coughing up data. 
+
+   Initial goal for navmerge is: only make sure realtime works. 
+
+   Every upstream has a thread that loops trying to connect
+   If a new message comes in, it is stored in a shared data structure
+   If a new connect is made, set a "don't send" marker for a whole minute
+
+   There is a sender thread that periodically polls this data structure
+   Any data that is older than the previous high-water mark gets sent out & removed from structure
+   However, transmission stops 10 seconds before realtime
+   If a "don't send" marker is set, we don't do a thing
+
+*/
+
+multimap<pair<uint64_t, uint64_t>, string> g_buffer;
+std::mutex g_mut;
+
+void recvSession(ComboAddress upstream)
+{
+  for(;;) {
+    try {
+      Socket sock(upstream.sin4.sin_family, SOCK_STREAM);
+      cerr<<"Connecting to "<< upstream.toStringWithPort()<<" to source data..";
+      SConnectWithTimeout(sock, upstream, 5);
+      cerr<<" done"<<endl;
+
+      for(int count=0;;++count) {
+        string part=SRead(sock, 4);
+        if(part.empty()) {
+          cerr<<"EOF from "<<upstream.toStringWithPort()<<endl;
+          break;
+        }
+        if(part != "bert") {
+          cerr << "Message "<<count<<", wrong magic from "<<upstream.toStringWithPort()<<": "<<makeHexDump(part)<<endl;
+          break;
+        }
+        if(!count)
+          cerr<<"Receiving messages from "<<upstream.toStringWithPort()<<endl;
+        string out=part;
+        
+        part = SRead(sock, 2);
+        out += part;
+        
+        uint16_t len;
+        memcpy(&len, part.c_str(), 2);
+        len = htons(len);
+        
+        part = SRead(sock, len);  // XXXXX ???
+        if(part.size() != len) {
+          cerr<<"Mismatch, "<<part.size()<<", len "<<len<<endl;
+          // XX AND THEN WHAT??
+        }
+        out += part;
+      
+        NavMonMessage nmm;
+        nmm.ParseFromString(part);
+        // writeToDisk(nmm.localutcseconds(), nmm.sourceid(), out);
+        // do something with the message
+        
+        std::lock_guard<std::mutex> mut(g_mut);
+        g_buffer.insert({{nmm.localutcseconds(), nmm.localutcnanoseconds()}, part});
+      }
+    }
+    catch(std::exception& e) {
+      cerr<<"Error in receiving thread: "<<e.what()<<endl;
+      sleep(1);
+    }
+  }
+  cerr<<"Thread for "<<upstream.toStringWithPort()<< " exiting"<<endl;
+}
+
+int main(int argc, char** argv)
+{
+  GOOGLE_PROTOBUF_VERIFY_VERSION;
+  vector<string> destinations;
+  vector<string> sources;
+
+  bool doVERSION{false}, doSTDOUT{false};
+  CLI::App app(program);
+  app.add_option("--source", sources, "Connect to these IP address:port to source protobuf");
+  app.add_option("--destination,-d", destinations, "Send output to this IPv4/v6 address");
+  app.add_flag("--version", doVERSION, "show program version and copyright");
+  app.add_flag("--stdout", doSTDOUT, "Emit output to stdout");
+
+  CLI11_PARSE(app, argc, argv);
+  
+  if(doVERSION) {
+    showVersion(program, g_gitHash);
+    exit(0);
+  }
+
+
+  signal(SIGPIPE, SIG_IGN);
+  NMMSender ns;
+
+  ns.d_debug = true;
+  for(const auto& s : destinations) {
+    auto res=resolveName(s, true, true);
+    if(res.empty()) {
+      cerr<<"Unable to resolve '"<<s<<"' as destination for data, exiting"<<endl;
+      exit(EXIT_FAILURE);
+    }
+    ns.addDestination(s); // ComboAddress(s, 29603));
+  }
+  if(doSTDOUT)
+    ns.addDestination(1);
+
+  
+  for(const auto& s : sources) {
+    ComboAddress oneaddr(s, 29601);
+    std::thread one(recvSession, oneaddr);
+    one.detach();
+  }
+
+  time_t start=time(0);
+  for(;;) {
+    sleep(1);
+    vector<string> tosend;
+    {
+      std::lock_guard<std::mutex> mut(g_mut);
+
+      time_t now = time(0);
+      if(now - start < 30) {
+        cerr<<"Have "<<g_buffer.size()<<" messages"<<endl;
+        continue;
+      }
+
+      for(auto iter = g_buffer.begin(); iter != g_buffer.end(); ) {
+        if(iter->first.first > (uint64_t)now - 5)
+          break;
+        tosend.push_back(iter->second);
+        iter = g_buffer.erase(iter);
+      }
+    }
+    //    cerr<<"Have "<<tosend.size()<<" messages to send, "<<g_buffer.size()<<" left in queue"<<endl;
+    std::string buf;
+    for(const auto& m : tosend) {
+      ns.emitNMM(m);
+    }
+  }
+}

navmon.cc

@@ -9,7 +9,7 @@
 #include <signal.h>
 #include <sys/poll.h>
 #include <iostream>
-
+#include <vector>
 using namespace std;
 
 using Clock = std::chrono::steady_clock; 
@@ -159,6 +159,18 @@ std::string humanTimeShort(time_t t)
   return buffer;
 }
 
+// influx ascii time format, in UTC
+std::string influxTime(time_t t)
+{
+  struct tm tm={0};
+  gmtime_r(&t, &tm);
+
+  char buffer[80];
+  std::string fmt = "%Y-%m-%d %H:%M:%S";
+  
+  strftime(buffer, sizeof(buffer), fmt.c_str(), &tm);
+  return buffer;
+}
 
 std::string humanTime(time_t t, uint32_t nanoseconds)
 {
@@ -280,8 +292,33 @@ std::string makeSatPartialName(const SatID& satid)
   return fmt::sprintf("%c%02d", getGNSSChar(satid.gnss), satid.sv);
 }
 
-
 int g_dtLS{18}, g_dtLSBeidou{4};
+
+void getGPSDateFromUTC(time_t t, int& wn, int& tow)
+{
+  t -= 315964800;
+  t += 18; // XXXXXX LEAP SECOND
+  wn = t/(7*86400);
+  tow = t%(7*86400);
+}
+void getGalDateFromUTC(time_t t, int& wn, int& tow)
+{
+  t -= 935280000;
+  t += 18; // XXXXXXX LEAP SECOND
+  wn = t/(7*86400);
+  tow = t%(7*86400);
+}
+
+void getBeiDouDateFromUTC(time_t t, int&wn, int& sow)
+{
+  // Week number count started from zero at 00:00:00 on Jan. 1, 2006 of BDT
+  t -= 1136070000;
+  t+= g_dtLSBeidou;
+  wn = t/(7*86400);
+  sow = t%(7*86400);
+}
+
+
 uint64_t utcFromGST(int wn, int tow)
 {
   return (935280000 + wn * 7*86400 + tow - g_dtLS); 
@@ -310,6 +347,19 @@ string makeHexDump(const string& str)
   return ret;
 }
 
+string makeHexDump(const basic_string<uint8_t>& str)
+{
+  char tmp[5];
+  string ret;
+  ret.reserve((int)(str.size()*2.2));
+
+  for(string::size_type n=0;n<str.size();++n) {
+    snprintf(tmp, sizeof(tmp), "%02x ", (unsigned char)str[n]);
+    ret+=tmp;
+  }
+  return ret;
+}
+
 std::string sbasName(int prn)
 {
   string sbas;
@@ -356,3 +406,46 @@ void unixDie(const std::string& reason)
 {
   throw std::runtime_error(reason+": "+strerror(errno));
 }
+
+time_t parseTime(std::string_view in)
+{
+  time_t now=time(0);
+
+  vector<string> formats({"%Y-%m-%d %H:%M", "%Y-%m-%d %H:%M:%S", "%Y%m%d %H%M", "%H:%M", "%H%M"});
+  for(const auto& f : formats) {
+    struct tm tm;
+    memset(&tm, 0, sizeof(tm));
+
+    localtime_r(&now, &tm);
+    tm.tm_isdst = -1;
+    tm.tm_sec = 0;
+    char* res = strptime(&in[0], f.c_str(), &tm);
+    if(res && !*res) {
+      //      cerr<<"Matched on "<<f<<endl;
+      return mktime(&tm);
+    }
+  }
+
+  string err="Can only parse on";
+  for(const auto& f : formats)
+    err += " '"+ f+"'";
+  throw runtime_error(err);
+}
+
+
+std::string string_replace(const std::string& str, const std::string& match, 
+        const std::string& replacement, unsigned int max_replacements)
+{
+    size_t pos = 0;
+    std::string newstr = str;
+    unsigned int replacements = 0;
+    while ((pos = newstr.find(match, pos)) != std::string::npos
+            && replacements < max_replacements)
+    {
+         newstr.replace(pos, match.length(), replacement);
+         pos += replacement.length();
+         replacements++;
+    }
+    return newstr;
+}
+

navmon.hh

@@ -5,7 +5,7 @@
 #include <string>
 #include <tuple>
 #include <mutex>
-
+#include <limits.h>
 extern const char* g_gitHash;
 
 
@@ -18,6 +18,8 @@ std::string humanTimeNow();
 std::string humanTime(time_t t);
 std::string humanTimeShort(time_t t);
 std::string humanTime(time_t t, uint32_t nanoseconds);
+// influx ascii time format, in UTC
+std::string influxTime(time_t t);
 struct SatID
 {
   uint32_t gnss{255}; // these could all be 'int16_t' but leads to howling numbers of warnings with protobuf
@@ -76,6 +78,14 @@ uint64_t utcFromGST(int wn, int tow);
 double utcFromGST(int wn, double tow);
 double utcFromGPS(int wn, double tow);
 
+void getGPSDateFromUTC(time_t t, int& wn, int& tow);
+void getGalDateFromUTC(time_t t, int& wn, int& tow);
+void getBeiDouDateFromUTC(time_t t, int&wn, int& sow);
+
 std::string makeHexDump(const std::string& str);
+std::string makeHexDump(const std::basic_string<uint8_t>& str);
 size_t writen2(int fd, const void *buf, size_t count);
 void unixDie(const std::string& reason);
+time_t parseTime(std::string_view in);
+std::string string_replace(const std::string& str, const std::string& match, 
+                           const std::string& replacement, unsigned int max_replacements = UINT_MAX);

navmon.proto

@@ -17,6 +17,9 @@ message NavMonMessage {
     SBASMessageType = 13;
     GPSCnavType = 14;
     RTCMMessageType = 15;
+    TimeOffsetType = 16;
+    GalileoFnavType = 17;
+    GalileoCnavType = 18;
   }
 
   required uint64 sourceID = 1;
@@ -32,7 +35,31 @@ message NavMonMessage {
 	required uint32 gnssID =3;
 	required uint32 gnssSV =4;
 	required bytes contents =5;
-        optional uint32 sigid = 6;        
+        optional uint32 sigid = 6;
+        optional bytes reserved1 = 7;
+        optional bytes reserved2 = 8;
+        optional bytes sar = 9;
+        optional bytes spare = 10;
+        optional bytes crc = 11;      
+  }
+
+message GalileoFnav {
+	required uint32 gnssWN =1;
+  	required uint32 gnssTOW =2; // INTEGERS!
+
+	required uint32 gnssID =3;
+	required uint32 gnssSV =4;
+	required bytes contents =5;
+        required uint32 sigid = 6;        
+  }
+  message GalileoCnav {
+	required uint32 gnssWN =1;
+  	required uint32 gnssTOW =2; // INTEGERS!
+
+	required uint32 gnssID =3;
+	required uint32 gnssSV =4;
+	required bytes contents =5;
+        required uint32 sigid = 6;        
   }
 
   message GPSInav {
@@ -172,10 +199,32 @@ message NavMonMessage {
         required uint32 sigid = 6;
   }
 
+
   message RTCMMessage {
 	required bytes contents =5;
   }
 
+  message GNSSOffset
+  {
+        required uint32 gnssid = 1;
+
+        required int32 offsetNS = 2;
+        required uint32 tAcc = 3;
+        required bool valid = 4;
+        optional int32 leapS = 5;
+        required uint32 tow = 6;
+        optional uint32 wn = 7;
+        optional uint32 nT = 8;
+        optional uint32 n4 = 9;
+
+  }
+  
+
+  message TimeOffsetMessage
+  {
+       required uint32 itow = 1;
+       repeated GNSSOffset offsets = 2;
+  }
 
 
   optional GalileoInav gi=5;
@@ -192,5 +241,8 @@ message NavMonMessage {
   optional UbloxJammingStats ujs = 16;
   optional SBASMessage sbm = 17;
   optional GPSCnav gpsc = 18;
-  optional RTCMMessage rm = 19;   
+  optional RTCMMessage rm = 19;
+  optional TimeOffsetMessage to = 20;
+  optional GalileoFnav gf=21;
+  optional GalileoCnav gc=22;
 }

navnexus.cc

@@ -88,7 +88,7 @@ try
         close(fd);
         continue;
       }
-      cout <<"Seeked to position "<<fpos[fname]<<" of "<<fname<<endl;
+      //      cout <<"Seeked to position "<<fpos[fname]<<" of "<<fname<<endl;
       NavMonMessage nmm;
 
       uint32_t looked=0;
@@ -106,15 +106,15 @@ try
       fpos[fname]=offset;
       close(fd);
     }
-    cout<<"Sorting.. ";
-    cout.flush();
+    //    cout<<"Sorting.. ";
+    //    cout.flush();
     sort(rnmms.begin(), rnmms.end(), [](const auto& a, const auto& b)
          {
            return std::tie(a.first.tv_sec, a.first.tv_nsec)
              < std::tie(b.first.tv_sec, b.first.tv_nsec);
          });
-    cout<<"Sending.. ";
-    cout.flush();
+    //    cout<<"Sending.. ";
+    //    cout.flush();
     for(const auto& nmm: rnmms) {
       std::string buf="bert";
       uint16_t len = htons(nmm.second.size());
@@ -122,7 +122,7 @@ try
       buf += nmm.second;
       SWriten(clientfd, buf);
     }
-    cout<<"Done"<<endl;
+    //    cout<<"Done"<<endl;
     if(3600 + start.tv_sec - (start.tv_sec % 3600) < time(0))
       start.tv_sec = 3600 + start.tv_sec - (start.tv_sec % 3600);
     else {

navparse.hh

@@ -49,6 +49,8 @@ struct SVStat
   GalileoMessage ephgalmsg, galmsg, oldephgalmsg;
   // internal
   map<int, GalileoMessage> galmsgTyped;
+  bool osnma{false};
+  time_t osnmaTime{0};
   
   // Glonass
   GlonassMessage ephglomsg, glonassMessage, oldephglomsg;
@@ -61,6 +63,7 @@ struct SVStat
   map<int, SBASCombo> sbas;
 
   RTCMMessage::EphemerisDelta rtcmEphDelta;
+  RTCMMessage::ClockDelta rtcmClockDelta;
   
   const GPSLikeEphemeris& liveIOD() const;
   const GPSLikeEphemeris& prevIOD() const;

navrecv.cc

@@ -16,6 +16,7 @@
 #include "version.hh"
 #include <netinet/tcp.h>
 #include "navmon.hh"
+#include <mutex>
 
 static char program[]="navrecv";
 
@@ -140,8 +141,94 @@ void writeToDisk(time_t s, uint64_t sourceid, std::string_view message)
   }
 }
 
+// This is used to report clients, so we can log them
+// The idea is that cleanup runs from the Sentinel which, when destroyed, will remove the entry
+struct ClientKeeper
+{
+  struct ClientStatus
+  {
+    bool oldProtocol;
+    time_t lastMessage;
+    int station;
+    uint64_t messages{0};
+  };
+
+  struct Sentinel
+  {
+    Sentinel(ClientKeeper* parent, const ComboAddress& us) : d_parent(parent), d_us(us)
+    {
+    }
+
+    Sentinel(Sentinel&& s)
+    {
+      //      cerr<<"Moved!"<<endl;
+      d_parent = s.d_parent;
+      d_us = s.d_us;
+      s.d_parent=0;
+    }
+    
+    ~Sentinel()
+    {
+      //      cerr<<"Destructor"<<endl;
+      if(d_parent) {
+	d_parent->remove(d_us);
+      }
+      else
+	; //cerr<<"   but we were moved already!\n";
+
+    }
+    void update(int station, bool oldProtocol)
+    {
+      time_t now = time(0);
+      std::lock_guard<std::mutex> l(d_parent->d_mut);
+      ClientStatus& cs = d_parent->d_clients[d_us];
+      cs.station = station;
+      cs.lastMessage = now;
+      cs.messages++;
+      cs.oldProtocol = oldProtocol;
+    }
+    ClientKeeper* d_parent;
+    ComboAddress d_us;
+  };
+
+  Sentinel reportClient(const ComboAddress& client)
+  {
+    Sentinel s2(this, client);
+    
+    std::lock_guard<std::mutex> l(d_mut);
+    d_clients[client];
+    return s2;
+  }
+
+  void remove(const ComboAddress& client)
+  {
+    std::lock_guard<std::mutex> l(d_mut);
+    d_clients.erase(client);
+  }
+
+  void dump()
+  {
+    std::lock_guard<std::mutex> l(d_mut);
+    string format("{:<50}{:<5}{:<10}{:<10}{:<10}\n");
+    ofstream out("clients.bak");
+    time_t now=time(0);
+    out<< fmt::format(format, "IP Address", "ID", "Protocol", "Messages", "Age");
+    for(const auto& c : d_clients) {
+      out << fmt::format(format, c.first.toStringWithPort(), c.second.station, c.second.oldProtocol ? "Old" : "New", c.second.messages, now-c.second.lastMessage);
+    }
+    out.close();
+    unlink("clients.txt");
+    rename("clients.bak", "clients.txt");
+  }
+  
+  map<ComboAddress, ClientStatus> d_clients;
+  std::mutex d_mut;
+};
+
+ClientKeeper g_ckeeper;
+
 // note that this moves the socket
-void recvSession2(Socket&& uns, ComboAddress client)
+void recvSession2(Socket&& uns, ComboAddress client, ClientKeeper::Sentinel& sentinel)
 {
   string secret = SRead(uns, 8); // ignored for now
   cerr << "Entering compressed session for "<<client.toStringWithPort()<<endl;
@@ -177,6 +264,7 @@ void recvSession2(Socket&& uns, ComboAddress client)
     memcpy(&denum, num.c_str(), 4);
     denum = htonl(denum);
     //    cerr<<"Received message "<<denum<< " "<<nmm.localutcseconds()<<" " << nmm.localutcnanoseconds()/1000000000.0<<endl;
+    sentinel.update(nmm.sourceid(), false);
     writeToDisk(nmm.localutcseconds(), nmm.sourceid(), out);
 
     if(first) {
@@ -201,6 +289,9 @@ void recvSession(int s, ComboAddress client)
     SSetsockopt(s, SOL_SOCKET, SO_KEEPALIVE, 1); // saves file descriptors
     cerr<<"Receiving messages from "<<client.toStringWithPort()<<endl;
     bool first=true;
+
+    ClientKeeper::Sentinel sentinel=g_ckeeper.reportClient(client);
+    
     for(int count=0;;++count) {
       string part=SRead(sock, 4);
       if(part.empty()) {
@@ -209,7 +300,7 @@ void recvSession(int s, ComboAddress client)
       }
       if(part != "bert") {
         if(part == "RNIE")
-          return recvSession2(std::move(sock), client);  // protocol v2, socket is moved cuz cleanup is special
+          return recvSession2(std::move(sock), client, sentinel);  // protocol v2, socket is moved cuz cleanup is special
         cerr << "Message "<<count<<", wrong magic from "<<client.toStringWithPort()<<": "<<makeHexDump(part)<<endl;
         break;
       }
@@ -225,6 +316,7 @@ void recvSession(int s, ComboAddress client)
       part = SRead(s, len);
       if(part.size() != len) {
         cerr<<"Mismatch, "<<part.size()<<", len "<<len<<endl;
+        // XX AND THEN WHAT??
       }
       out += part;
       
@@ -234,6 +326,7 @@ void recvSession(int s, ComboAddress client)
         cerr<<"\tstation: "<<nmm.sourceid() << endl;
         first=false;
       }
+      sentinel.update(nmm.sourceid(), true);
       writeToDisk(nmm.localutcseconds(), nmm.sourceid(), out);
     }
   }
@@ -285,8 +378,10 @@ int main(int argc, char** argv)
 
   thread recvThread(recvListener, std::move(receiver), recvaddr);
   recvThread.detach();
- 
+
+  sleep(5);
   for(;;) {
-    sleep(1);
+    g_ckeeper.dump();
+    sleep(10);
   }
 }

nmmsender.cc

@@ -175,16 +175,22 @@ void NMMSender::sendTCPThread(Destination* d)
 
 
 void NMMSender::emitNMM(const NavMonMessage& nmm)
+{
+  string out;
+  nmm.SerializeToString(& out);
+  emitNMM(out);
+}
+
+void NMMSender::emitNMM(const std::string& out)
 {
   for(auto& d : d_dests) {
-    d->emitNMM(nmm, d_compress);
+    d->emitNMM(out, d_compress);
   }
 }
 
-void NMMSender::Destination::emitNMM(const NavMonMessage& nmm, bool compressed)
+
+void NMMSender::Destination::emitNMM(const std::string& out, bool compressed)
 {
-  string out;
-  nmm.SerializeToString(& out);
   string msg;
   if(dst.empty() || !compressed)
     msg="bert";

nmmsender.hh

@@ -16,7 +16,7 @@ class NMMSender
 
     std::deque<std::string> queue;
     std::mutex mut;
-    void emitNMM(const NavMonMessage& nmm, bool compress);
+    void emitNMM(const std::string& out, bool compress);
   };
   
 public:
@@ -45,6 +45,7 @@ public:
   void sendTCPThread(Destination* d);
 
   void emitNMM(const NavMonMessage& nmm);
+  void emitNMM(const std::string& out);
   bool d_debug{false};
   bool d_compress{false}; // set BEFORE launch
   bool d_pleaseQuit{false};

reporter.cc

@@ -4,9 +4,13 @@
 #include "navmon.hh"
 #include "fmt/format.h"
 #include "fmt/printf.h"
-
+#include "galileo.hh"
+#include "gps.hh"
 #include "CLI/CLI.hpp"
 #include "version.hh"
+#include "ephemeris.hh"
+#include "influxpush.hh"
+#include "sp3.hh"
 
 static char program[]="reporter";
 
@@ -25,7 +29,7 @@ public:
   struct Results
   {
     vector<double> d;
-    bool dirty{false};
+    mutable bool dirty{false};
     double median() const
     {
       return quantile(0.5);
@@ -41,7 +45,7 @@ public:
     }
 
   };
-  const Results& done()
+  const Results& done() const
   {
     if(results.dirty) {
       sort(results.d.begin(), results.d.end());
@@ -49,9 +53,12 @@ public:
     }
     return results;
   }
-
+  bool empty() const
+  {
+    return results.d.empty();
+  }
 private:
-  Results results;
+  mutable Results results;
 };
 
 /*
@@ -77,16 +84,20 @@ private:
 struct IntervalStat
 {
   std::optional<int> unhealthy;
+  std::optional<int> dataunhealthy;
+  std::optional<int> osnma;
   std::optional<int> sisa;
   bool ripe{false};
   bool expired{false};
   double rtcmDist{-1};
+  std::optional<double> rtcmDClock;
 };
 
 
 map<SatID, map<time_t,IntervalStat>> g_stats;
 
 int main(int argc, char **argv)
+try
 {
   MiniCurl mc;
   MiniCurl::MiniCurlHeaders mch;
@@ -100,12 +111,22 @@ int main(int argc, char **argv)
   CLI::App app(program);
   string periodarg("1d");
   string beginarg, endarg;
+  string sp3src("default");
+  int gnssid=2;
+  int rtcmsrc=300;
+  int galwn=-1;
+  string influxserver="http://127.0.0.1:8086";
   app.add_flag("--version", doVERSION, "show program version and copyright");
   app.add_option("--period,-p", periodarg, "period over which to report (1h, 1w)");
   app.add_option("--begin,-b", beginarg, "Beginning");
   app.add_option("--end,-e", endarg, "End");
+  app.add_option("--gal-wn", galwn, "Galileo week number to report on");
+  app.add_option("--sp3src", sp3src, "Identifier of SP3 source");
+  app.add_option("--rtcmsrc", rtcmsrc, "Identifier of RTCM source");
   app.add_option("--sigid,-s", sigid, "Signal identifier. 1 or 5 for Galileo.");
+  app.add_option("--gnssid,-g", gnssid, "gnssid, 0 GPS, 2 Galileo");
   app.add_option("--influxdb", influxDBName, "Name of influxdb database");
+  app.add_option("--influxserver", influxserver, "Address of influx server");
   try {
     app.parse(argc, argv);
   } catch(const CLI::Error &e) {
@@ -117,7 +138,11 @@ int main(int argc, char **argv)
     exit(0);
   }
 
-  if(beginarg.empty() && endarg.empty()) 
+  if(galwn>= 0) {
+    time_t w = utcFromGST(galwn, 0);
+    period = "time >= '"+influxTime(w)+"' and time < '"+influxTime(w+7*86400) +"'";
+  }
+  else if(beginarg.empty() && endarg.empty()) 
     period = "time > now() - "+periodarg;
   else {
     period = "time > '"+ beginarg +"' and time <= '" + endarg +"'";
@@ -128,12 +153,16 @@ int main(int argc, char **argv)
 
   // auto res = mc.getURL(url + mc.urlEncode("select distinct(value) from sisa where "+period+" and sigid='"+to_string(sigid)+"' group by gnssid,sv,sigid,time(10m)"));
 
-
-  
-  string url="http://127.0.0.1:8086/query?db="+influxDBName+"&epoch=s&q=";
-  string query="select distinct(value) from sisa where "+period+" and sigid='"+to_string(sigid)+"' group by gnssid,sv,sigid,time(10m)";
+  if(influxserver.find("http"))
+    influxserver="http://"+influxserver;
+  if(influxserver.empty() || influxserver[influxserver.size()-1]!='/')
+    influxserver+="/";
+  string url=influxserver+"query?db="+influxDBName+"&epoch=s&q=";
+  string sisaname = (gnssid==2) ? "sisa" : "gpsura";
+  string query="select distinct(value) from "+sisaname+" where "+period+" and sigid='"+to_string(sigid)+"' group by gnssid,sv,sigid,time(10m)";
 
   cout<<"query: "<<query<<endl;
+  cout<<"url: "<<(url + mc.urlEncode(query))<<endl;
   auto res = mc.getURL(url + mc.urlEncode(query));
 
   auto j = nlohmann::json::parse(res);
@@ -150,8 +179,36 @@ int main(int argc, char **argv)
     }
   }
 
+  string healthname = (gnssid == 2) ? "galhealth" : "gpshealth";
+  string healthfieldname = (gnssid==2) ? "e1bhs" : "value";
+  res = mc.getURL(url + mc.urlEncode("select distinct("+healthfieldname+") from "+healthname+" where "+period+" and sigid='"+to_string(sigid)+"' group by gnssid,sv,sigid,time(10m)"));
+  j = nlohmann::json::parse(res);
   
-  res = mc.getURL(url + mc.urlEncode("select distinct(e1bhs) from galhealth where "+period+" and sigid='"+to_string(sigid)+"' group by gnssid,sv,sigid,time(10m)"));
+  for(const auto& sv : j["results"][0]["series"]) {
+    const auto& tags=sv["tags"];
+    SatID id{(unsigned int)std::stoi((string)tags["gnssid"]), (unsigned int)std::stoi((string)tags["sv"]), (unsigned int)std::stoi((string)tags["sigid"])};
+
+    for(const auto& v : sv["values"]) {
+      auto healthy = (int)v[1];
+      g_stats[id][(int)v[0]].unhealthy = healthy; // hngg
+    }
+  }
+
+  if(gnssid == 2) {
+    res = mc.getURL(url + mc.urlEncode("select distinct(e1bdvs) from galhealth where "+period+" and sigid='"+to_string(sigid)+"' group by gnssid,sv,sigid,time(10m)"));
+    j = nlohmann::json::parse(res);
+    
+    for(const auto& sv : j["results"][0]["series"]) {
+      const auto& tags=sv["tags"];
+      SatID id{(unsigned int)std::stoi((string)tags["gnssid"]), (unsigned int)std::stoi((string)tags["sv"]), (unsigned int)std::stoi((string)tags["sigid"])};
+      
+      for(const auto& v : sv["values"]) {
+	auto dhealthy = (int)v[1]; // if true, "working without guarantee"
+	g_stats[id][(int)v[0]].dataunhealthy = dhealthy; 
+      }
+    }
+  }  
+  res = mc.getURL(url + mc.urlEncode("select count(\"field\") from osnma where "+period+" and sigid='"+to_string(sigid)+"' group by gnssid,sv,sigid,time(10m)"));
   j = nlohmann::json::parse(res);
   
   for(const auto& sv : j["results"][0]["series"]) {
@@ -160,17 +217,21 @@ int main(int argc, char **argv)
 
 
     for(const auto& v : sv["values"]) {
-      auto healthy = (int)v[1];
-      g_stats[id][(int)v[0]].unhealthy = healthy;
+      auto osnma = (int)v[1];
+      if(!g_stats[id][(int)v[0]].osnma)
+	g_stats[id][(int)v[0]].osnma = osnma;
+      else
+	(*g_stats[id][(int)v[0]].osnma) += osnma;
     }
   }
 
+  
   res = mc.getURL(url + mc.urlEncode("select max(\"eph-age\") from ephemeris where "+period+" and sigid='"+to_string(sigid)+"' group by gnssid,sv,sigid,time(10m)"));
   j = nlohmann::json::parse(res);
   for(const auto& sv : j["results"][0]["series"]) {
     const auto& tags=sv["tags"];
     SatID id{(unsigned int)std::stoi((string)tags["gnssid"]), (unsigned int)std::stoi((string)tags["sv"]), (unsigned int)std::stoi((string)tags["sigid"])};
-
+    
 
     for(const auto& v : sv["values"]) {
       if(v.size() > 1 && v[1] != nullptr) {
@@ -187,17 +248,51 @@ int main(int argc, char **argv)
     }
   }
 
-  /////////////////////
+  ///////////////////// rtcm-eph
 
-  res = mc.getURL(url + mc.urlEncode("select mean(\"total-dist\") from \"rtcm-eph-correction\" where "+period+" and sigid='"+to_string(sigid)+"' group by gnssid,sv,sigid,time(10m)"));
+  
+  string rtcmQuery = "select mean(\"radial\") from \"rtcm-eph-correction\" where "+period+" and sigid='"+to_string(sigid)+"' and gnssid='"+to_string(gnssid)+"' and src='"+to_string(rtcmsrc)+"' group by gnssid,sv,sigid,time(10m)";
+  cout<<"rtcmquery: "<<rtcmQuery<<endl;
+  res = mc.getURL(url + mc.urlEncode(rtcmQuery));
   j = nlohmann::json::parse(res);
   for(const auto& sv : j["results"][0]["series"]) {
     try {
       const auto& tags=sv["tags"];
       SatID id{(unsigned int)std::stoi((string)tags["gnssid"]), (unsigned int)std::stoi((string)tags["sv"]), (unsigned int)std::stoi((string)tags["sigid"])};
       for(const auto& v : sv["values"]) {
-        
-        g_stats[id][(int)v[0]].rtcmDist = v[1];
+        try {
+          auto val = (double)v[1]; // might trigger exception
+          g_stats[id][(int)v[0]].rtcmDist = val;
+        }
+        catch(...){ continue; }
+      }
+    }
+    catch(...) {
+      continue;
+    }
+  }
+
+  
+  /////////////////////
+
+  ///////////////////// rtcm-clock
+
+  
+  string rtcmClockQuery = "select mean(\"dclock0\") from \"rtcm-clock-correction\" where "+period+" and sigid='"+to_string(sigid)+"' and gnssid='"+to_string(gnssid)+"' and src='"+to_string(rtcmsrc)+"' group by gnssid,sv,sigid,time(10m)";
+  cout<<"rtcmquery: "<<rtcmClockQuery<<endl;
+  res = mc.getURL(url + mc.urlEncode(rtcmClockQuery));
+  j = nlohmann::json::parse(res);
+  for(const auto& sv : j["results"][0]["series"]) {
+    try {
+      const auto& tags=sv["tags"];
+      SatID id{(unsigned int)std::stoi((string)tags["gnssid"]), (unsigned int)std::stoi((string)tags["sv"]), (unsigned int)std::stoi((string)tags["sigid"])};
+      for(const auto& v : sv["values"]) {
+        try {
+          auto val = (double) v[1]; // might trigger an exception
+          if(g_stats.count(id)) // we have some bad data it appears
+            g_stats[id][(int)v[0]].rtcmDClock = val;
+        }
+        catch(...){ continue; }
       }
     }
     catch(...) {
@@ -209,10 +304,295 @@ int main(int argc, char **argv)
   /////////////////////
 
 
-  g_stats.erase({2,14,1});
-  g_stats.erase({2,18,1});
-  g_stats.erase({2,14,5});
-  g_stats.erase({2,18,5});
+                                  
+  map<SatID, map<time_t, GalileoMessage>> galephs;
+  map<SatID, map<time_t, GPSState>> gpsephs;
+
+  res = mc.getURL(url + mc.urlEncode("select * from \"ephemeris-actual\" where "+period+" and sigid='"+to_string(sigid)+"' and gnssid='"+to_string(gnssid)+"' group by gnssid,sv,sigid"));
+  j = nlohmann::json::parse(res);
+  for(const auto& sv : j["results"][0]["series"]) {
+    try {
+      const auto& tags=sv["tags"];
+      SatID id{(unsigned int)std::stoi((string)tags["gnssid"]), (unsigned int)std::stoi((string)tags["sv"]), (unsigned int)std::stoi((string)tags["sigid"])};
+      //      cout << makeSatIDName(id) <<": "<<sv["columns"] << endl;
+      map<string, int> cmap;
+      for(const auto& c : sv["columns"]) {
+        cmap[c]=cmap.size();
+      }
+
+      for(const auto& v : sv["values"]) {
+          //        cout << makeSatIDName(id)<<": crc "<<v[cmap["crc"]]<<" e " <<v[cmap["e"]] <<  endl;
+        if(id.gnss==2) {
+          GalileoMessage gm;
+          gm.e = v[cmap["e"]];
+          gm.crc = v[cmap["crc"]];
+          gm.crs = v[cmap["crs"]];
+          gm.cuc = v[cmap["cuc"]];
+          gm.cus = v[cmap["cus"]];
+          gm.cic = v[cmap["cic"]];
+          gm.cis = v[cmap["cis"]];
+          gm.sqrtA = v[cmap["sqrta"]];
+          gm.t0e = v[cmap["t0e"]];
+          gm.m0 = v[cmap["m0"]];
+          gm.deltan = v[cmap["deltan"]];
+          gm.omega0 = v[cmap["omega0"]];
+          gm.omegadot = v[cmap["omegadot"]];
+          gm.idot = v[cmap["idot"]];
+          gm.omega = v[cmap["omega"]];
+          gm.i0 = v[cmap["i0"]];
+          gm.t0e = v[cmap["t0e"]];
+          gm.iodnav = v[cmap["iod"]];
+          if(cmap.count("af0")) {
+            gm.af0 = v[cmap["af0"]];
+            gm.af1 = v[cmap["af1"]];
+            gm.af2 = v[cmap["af2"]];
+            gm.t0c = v[cmap["t0c"]];
+          }
+
+          Point pos;
+          galephs[id][v[cmap["time"]]]= gm;
+        }
+        else if(id.gnss==0) {
+          GPSState gm{};
+          gm.e = v[cmap["e"]];
+          gm.crc = v[cmap["crc"]];
+          gm.crs = v[cmap["crs"]];
+          gm.cuc = v[cmap["cuc"]];
+          gm.cus = v[cmap["cus"]];
+          gm.cic = v[cmap["cic"]];
+          gm.cis = v[cmap["cis"]];
+          gm.sqrtA = v[cmap["sqrta"]];
+          gm.t0e = v[cmap["t0e"]];
+          gm.m0 = v[cmap["m0"]];
+          gm.deltan = v[cmap["deltan"]];
+          gm.omega0 = v[cmap["omega0"]];
+          gm.omegadot = v[cmap["omegadot"]];
+          gm.idot = v[cmap["idot"]];
+          gm.omega = v[cmap["omega"]];
+          gm.i0 = v[cmap["i0"]];
+          gm.t0e = v[cmap["t0e"]];
+          gm.gpsiod = v[cmap["iod"]];
+          if(cmap.count("af0")) {
+            gm.af0 = v[cmap["af0"]];
+            gm.af1 = v[cmap["af1"]];
+            gm.af2 = v[cmap["af2"]];
+            gm.t0c = v[cmap["t0c"]];
+          }
+
+          Point pos;
+          gpsephs[id][v[cmap["time"]]]= gm;
+        }
+      }
+
+    }
+    catch(...) {
+      continue;
+    }
+  }
+
+  cout<<"Gathered ephemerides for "<<galephs.size()<<" galileo + "<<gpsephs.size()<<" GPS signals"<<endl;
+  /////////////////////
+
+  map<SatID, map<time_t, SP3Entry>> sp3s;
+  string spq3="select * from \"sp3\" where "+period+" and sp3src =~ /"+sp3src+"/ and gnssid='"+to_string(gnssid)+"' group by gnssid,sv,sigid";
+  cout<<"sp3 query: "<<spq3<<endl;
+  res = mc.getURL(url + mc.urlEncode(spq3));
+  j = nlohmann::json::parse(res);
+  cout<<"Gathered sp, got "<< j["results"][0]["series"].size()<< " tags"<<endl;
+  for(const auto& sv : j["results"][0]["series"]) {
+    try {
+      const auto& tags=sv["tags"];
+      SatID id{(unsigned int)std::stoi((string)tags["gnssid"]), (unsigned int)std::stoi((string)tags["sv"]), (unsigned int)sigid};
+
+      // SP3 data does not have a sigid, it refers to the center of mass, not the antenna phase center
+      //      cout << makeSatIDName(id) <<": "<<sv["columns"] << endl;
+      map<string, int> cmap;
+      for(const auto& c : sv["columns"]) {
+        cmap[c]=cmap.size();
+      }
+            
+      for(const auto& v : sv["values"]) {
+        //        cout << makeSatIDName(id)<<": time "<<v[cmap["time"]] <<" x "<<v[cmap["x"]]<<" y " <<v[cmap["y"]] << " z " << v[cmap["z"]] << endl;
+        SP3Entry e;
+        e.t = v[cmap["time"]]; // UTC!!
+        e.x = v[cmap["x"]];
+        e.y = v[cmap["y"]];
+        e.z = v[cmap["z"]];
+        e.clockBias = v[cmap["clock-bias"]];
+        sp3s[id][e.t]=e;
+      }
+    }
+    catch(std::exception& e)
+    {
+      cerr<<"Error: "<<e.what()<<endl;
+    }
+  }
+
+  ofstream csv("sp3.csv");
+  csv<<"timestamp gnss sv sigid zerror"<<endl;
+
+  InfluxPusher idb(influxDBName);
+
+  map<SatID, Stats> sp3zerrors, sp3clockerrors;
+  /////////////////////
+  for(const auto& svsp3 : sp3s) {
+    const auto& id = svsp3.first;
+    const auto& es = svsp3.second;
+    //    cout<<"Looking at SP3 for " << makeSatIDName(id)<<", have "<<es.size()<< " entries"<<endl;
+    for(const auto& e : es) {
+      //      cout << humanTimeShort(e.first)<<": ";
+
+      if(id.gnss == 2) {
+        const auto& svephs = galephs[id];
+        auto iter = svephs.lower_bound(e.first);
+
+        // this logic is actually sort of wrong & ignores the last ephemeris
+        
+        
+        if(iter != svephs.end() && iter != svephs.begin()) {
+          --iter;
+          //        cout << "found ephemeris from "<< humanTimeShort(iter->first)<<" iod "<<iter->second.iodnav;
+
+          // our UTC timestamp from SP3 need to be converted into a tow
+          
+          int offset = id.gnss ? 935280000 : 315964800;
+          int sp3tow = (e.first - offset) % (7*86400);
+          Point epos;
+          getCoordinates(sp3tow, iter->second, &epos);
+          
+          double clkoffset= iter->second.getAtomicOffset(sp3tow).first - e.second.clockBias;
+          
+          //        cout<<" "<<iter->second.getAtomicOffset(sp3tow).first <<" v " << e.second.clockBias<<" ns ";
+          //        cout << " ("<<epos.x<<", "<<epos.y<<", "<<epos.z<<")  - > ("<<e.second.x<<", "<<e.second.y<<", "<<e.second.z<<") " ;
+          //        cout <<" ("<<epos.x - e.second.x<<", "<<epos.y - e.second.y <<", "<<epos.z - e.second.z<<")";
+          
+          Point sp3pos(e.second.x, e.second.y, e.second.z);
+          Vector v(epos, sp3pos);
+          
+          //        cout<< " -> " << v.length();
+          
+          Vector dir(Point(0,0,0), sp3pos);
+          dir.norm();
+          
+          Point cv=sp3pos;
+          cv.x -= 0.519 * dir.x;
+          cv.y -= 0.519 * dir.y;
+          cv.z -= 0.519 * dir.z;
+          
+          Vector v2(epos, cv);
+          //        cout<< " -> " << v2.length();
+          
+          //        cout<<" z-error: "<<dir.inner(v);
+          
+          csv << e.first << " " << id.gnss <<" " << id.sv << " " << id.sigid <<" " << dir.inner(v) << endl;
+          idb.addValue({{"gnssid", id.gnss}, {"sv", id.sv}, {"sp3src", sp3src}},
+                       "sp3delta",
+                       {{"ecef-dx", v.x}, {"ecef-dy", v.y}, {"ecef-dz", v.z}, {"sv-dz", dir.inner(v)}, {"dclock", clkoffset},
+                                                                                                         {"iod-nav",iter->second.iodnav}},
+                       e.first);
+          
+          sp3clockerrors[id].add(clkoffset); // nanoseconds
+          sp3zerrors[id].add(100*dir.inner(v) - 80); // meters -> cm
+        }
+      }
+      else if(id.gnss==0) {
+        const auto& svephs = gpsephs[id];
+        // this is keyed on the moment of _issue_
+        auto iter = svephs.lower_bound(e.first);
+        if(iter != svephs.end() && iter != svephs.begin()) {
+          --iter;
+          //        cout << "found ephemeris from "<< humanTimeShort(iter->first)<<" iod "<<iter->second.iodnav;
+          
+          // our UTC timestamp from SP3 need to be converted into a tow
+          
+          int offset = 315964800;
+          int sp3tow = (e.first - offset) % (7*86400);
+          Point epos;
+          getCoordinates(sp3tow, iter->second, &epos);
+          
+          double clkoffset= getGPSAtomicOffset(sp3tow, iter->second).first - e.second.clockBias;
+          
+          //        cout<<" "<<iter->second.getAtomicOffset(sp3tow).first <<" v " << e.second.clockBias<<" ns ";
+          //        cout << " ("<<epos.x<<", "<<epos.y<<", "<<epos.z<<")  - > ("<<e.second.x<<", "<<e.second.y<<", "<<e.second.z<<") " ;
+          //        cout <<" ("<<epos.x - e.second.x<<", "<<epos.y - e.second.y <<", "<<epos.z - e.second.z<<")";
+          
+          Point sp3pos(e.second.x, e.second.y, e.second.z);
+          Vector v(epos, sp3pos);
+          
+          //        cout<< " -> " << v.length();
+          
+          Vector dir(Point(0,0,0), sp3pos);
+          dir.norm();
+          
+          Point cv=sp3pos;
+          cv.x -= 0.519 * dir.x;
+          cv.y -= 0.519 * dir.y;
+          cv.z -= 0.519 * dir.z;
+          
+          Vector v2(epos, cv);
+          //        cout<< " -> " << v2.length();
+          
+          //        cout<<" z-error: "<<dir.inner(v);
+          
+          csv << e.first << " " << id.gnss <<" " << id.sv << " " << id.sigid <<" " << dir.inner(v) << endl;
+          idb.addValue({{"gnssid", id.gnss}, {"sv", id.sv}, {"sp3src", sp3src}},
+                       "sp3delta",
+                       {{"ecef-dx", v.x}, {"ecef-dy", v.y}, {"ecef-dz", v.z}, {"sv-dz", dir.inner(v)}, {"dclock", clkoffset},
+                                                                                                         {"iod-nav",iter->second.gpsiod}},
+                       e.first);
+          
+          sp3clockerrors[id].add(clkoffset); // nanoseconds
+          sp3zerrors[id].add(100*dir.inner(v)); // meters -> cm
+        }
+      }
+    }
+  }
+
+  ///// 
+  string dishesQuery = "select iod,sv from \"ephemeris-actual\" where "+period+" and sigid='"+to_string(sigid)+"' and gnssid='"+to_string(gnssid)+"' and iod < 128";
+  cout<<"dishesquery: "<<dishesQuery<<endl;
+  res = mc.getURL(url + mc.urlEncode(dishesQuery));
+  cout<<res<<endl;
+  j = nlohmann::json::parse(res);
+  map<time_t, set<int>> dishcount;
+  set<int> totsvs;
+  for(const auto& sv : j["results"][0]["series"]) {
+    for(const auto& v : sv["values"]) {
+      try {
+	int sv = (unsigned int)std::stoi((string)v[2]);
+	int t = (int)v[0];
+	//	t &= (~31);
+	dishcount[t].insert(sv);
+	totsvs.insert(sv);
+      }
+      catch(exception& e) {
+	cerr<<"error: "<<e.what()<<endl;
+	continue;
+      }
+    }
+  }
+  map<time_t, unsigned int> maxcounts;
+  for(const auto& dc : dishcount) {
+    auto& bin = maxcounts[dc.first - (dc.first % 3600)];
+    if(bin < dc.second.size())
+      bin = dc.second.size();
+    cout << dc.first<<" "<<humanTimeShort(dc.first) <<", " << fmt::sprintf("%2d", dc.second.size())<<": ";
+    for(const auto& n : totsvs) {
+      if(dc.second.count(n))
+	cout<<fmt::sprintf("%2d ", n);
+      else
+	cout<<"   ";
+    }
+    cout<<"\n";
+  }
+
+  ofstream hrcounts("hrcounts.csv");
+  hrcounts<<"timestamp,dishcount\n";
+  for(const auto& mc: maxcounts)
+    hrcounts<<mc.first<<","<<mc.second<<"\n";
+  
+  /////////////////////
+  
   /*
   g_stats[{2,19,1}];
   */
@@ -225,9 +605,17 @@ int main(int argc, char **argv)
       if(sv.second.rbegin()->first > stop)
         stop = sv.second.rbegin()->first;
     }
-      
-    if(sv.second.size() > maxintervals) 
-      maxintervals = sv.second.size();
+    unsigned int liveInterval=0;
+    for(const auto i : sv.second) {
+      if(i.second.sisa.has_value())
+        liveInterval++;
+      else {
+        //        cout<<makeSatIDName(sv.first)<<": no Sisa, "<< i.second.rtcmDClock.has_value()<<" " << i.second.unhealthy.has_value()<<" " <<i.second.rtcmDist<<" ripe "<<i.second.ripe<<endl;
+      }
+    }
+    
+    if(liveInterval > maxintervals) 
+      maxintervals = liveInterval;
   }
 
   cout<<"Report on "<<g_stats.size()<<" SVs from "<<humanTime(start) <<" to " <<humanTime(stop) << endl;
@@ -235,17 +623,22 @@ int main(int argc, char **argv)
   int totunobserved=0;
   int totripe = 0,  totexpired = 0;
   Stats totRTCM;
+  ofstream texstream("stats.tex");
   for(const auto& sv : g_stats) {
 
     int napa=0, unhealthy=0, healthy=0, testing=0, ripe=0, expired=0;
 
-    Stats rtcm;
-    for(const auto& i : sv.second) {
+    Stats rtcm, clockRtcm;
 
+    for(const auto& i : sv.second) {
+      //      cout<<humanTimeShort(i.first)<<" "<<((i.second.sisa.has_value()) ? "S" : " ")<<" ";
       if(i.second.rtcmDist >= 0) {
         rtcm.add(i.second.rtcmDist);
         totRTCM.add(i.second.rtcmDist);
       }
+
+      if(i.second.rtcmDClock)
+        clockRtcm.add(*i.second.rtcmDClock * 100);
       
       if(i.second.ripe)
         ripe++;
@@ -258,7 +651,10 @@ int main(int argc, char **argv)
         else if(*i.second.unhealthy==3)
           testing++;
         else {
-          if(i.second.sisa) {
+	  if(i.second.dataunhealthy && *i.second.dataunhealthy) {  // this is 'working without guarantee'
+	    unhealthy++;
+	  }
+	  else if(i.second.sisa) {
             if(*i.second.sisa == 255)
               napa++;
             else
@@ -273,38 +669,93 @@ int main(int argc, char **argv)
           napa++;
       }
     }
+    //    cout<<endl;
     totnapa += napa;
     totunhealthy += unhealthy;
     tottesting += testing;
     tothealthy += healthy;
     totripe += ripe;
     totexpired += expired;
-    totunobserved += maxintervals-sv.second.size();
-     
-    cout<<fmt::sprintf("E%02d: %6.2f%% unobserved, %6.2f%% unhealthy, %6.2f%% healthy, %6.2f%% testing, %6.2f%% napa, %6.2f%% ripe, %6.2f%% expired, %.1f - %.1f - %.1f cm",
-                       sv.first.sv,
-                       100.0*(maxintervals-sv.second.size())/maxintervals,
+    int liveInterval=0;
+
+    for(const auto i : sv.second)
+      if(i.second.sisa.has_value())
+        liveInterval++;
+
+    
+    totunobserved += maxintervals - liveInterval;
+    
+    cout<<fmt::sprintf("%s: %6.2f%% unobserved, %6.2f%% unhealthy, %6.2f%% healthy, %6.2f%% testing, %6.2f%% napa, %6.2f%% ripe, %6.2f%% expired",
+                       makeSatPartialName(sv.first),
+                       100.0*(maxintervals-liveInterval)/maxintervals,
                        100.0*unhealthy/maxintervals,
                        100.0*healthy/maxintervals,
                        100.0*testing/maxintervals,
                        100.0*napa/maxintervals,
                        100.0*ripe/maxintervals,
-                       100.0*expired/maxintervals,
-                       rtcm.done().quantile(0.10)/10, rtcm.done().median()/10, rtcm.done().quantile(0.9)/10
-                       )<<endl;
+                       100.0*expired/maxintervals);
+
+    texstream << fmt::sprintf("%s & %6.2f\\%% & %6.2f\\%% & %6.2f\\%% & %6.2f\\%% & %6.2f\\%% & %6.2f\\%% & %6.2f\\%%\\\\",
+                       makeSatPartialName(sv.first),
+                       100.0*(maxintervals-liveInterval)/maxintervals,
+                       100.0*unhealthy/maxintervals,
+                       100.0*healthy/maxintervals,
+                       100.0*testing/maxintervals,
+                       100.0*napa/maxintervals,
+                       100.0*ripe/maxintervals,
+                             100.0*expired/maxintervals) << endl;
+    
+    if(!rtcm.empty()) 
+      cout<<fmt::sprintf(", %.1f - %.1f - %.1f cm",
+                         rtcm.done().quantile(0.10)/10, rtcm.done().median()/10, rtcm.done().quantile(0.9)/10);
+
+
+    if(!clockRtcm.empty()) 
+      cout<<fmt::sprintf(", c %.1f - %.1f - %.1f cm",
+                         clockRtcm.done().quantile(0.10), clockRtcm.done().median(), clockRtcm.done().quantile(0.9));
+
+    if(!sp3zerrors[sv.first].empty()) {
+      const auto& z = sp3zerrors[sv.first];
+      cout<<fmt::sprintf(", z %.1f - %.1f - %.1f cm",
+                         z.done().quantile(0.10), z.done().median(), z.done().quantile(0.9));
+    }
+    
+    cout<<endl;
+    
   }
+
+ 
+  
   cout<<"------------------------------------------------------------------------------------------"<<endl;
-  cout<<fmt::sprintf("Tot: %6.2f%% unobserved, %6.2f%% unhealthy, %6.2f%% healthy, %6.2f%% testing, %6.2f%% napa, %6.2f%% ripe, %6.2f%% expired, %.1f - %.1f - %.1f cm",
+  cout<<fmt::sprintf("Tot: %6.2f%% unobserved, %6.2f%% unhealthy, %6.2f%% healthy, %6.2f%% testing, %6.2f%% napa, %6.2f%% ripe, %6.2f%% expired",
                      100.0*(totunobserved)/maxintervals/g_stats.size(),
                      100.0*totunhealthy/maxintervals/g_stats.size(),
                      100.0*tothealthy/maxintervals/g_stats.size(),
                      100.0*tottesting/maxintervals/g_stats.size(),
                      100.0*totnapa/maxintervals/g_stats.size(),
                      100.0*totripe/maxintervals/g_stats.size(),
-                     100.0*totexpired/maxintervals/g_stats.size(),
-                     totRTCM.done().quantile(0.10)/10, totRTCM.done().median()/10, totRTCM.done().quantile(0.9)/10
-                     )<<endl;
+                     100.0*totexpired/maxintervals/g_stats.size());
+
+  texstream<<fmt::sprintf("\\hline\nTot & %6.2f\\%% & %6.2f\\%% & %6.2f\\%% & %6.2f\\%% & %6.2f\\%% & %6.2f\\%% & %6.2f\\%%\\\\",
+                     100.0*(totunobserved)/maxintervals/g_stats.size(),
+                     100.0*totunhealthy/maxintervals/g_stats.size(),
+                     100.0*tothealthy/maxintervals/g_stats.size(),
+                     100.0*tottesting/maxintervals/g_stats.size(),
+                     100.0*totnapa/maxintervals/g_stats.size(),
+                     100.0*totripe/maxintervals/g_stats.size(),
+                          100.0*totexpired/maxintervals/g_stats.size()) <<endl;
+
+
+  if(!totRTCM.empty())
+    cout<<fmt::sprintf(", %.1f - %.1f - %.1f cm",
+                       totRTCM.done().quantile(0.10)/10, totRTCM.done().median()/10, totRTCM.done().quantile(0.9)/10);
+  
+  cout<<endl;
 
 }
-
+catch(exception& e)
+{
+  cerr<<"Fatal error: "<<e.what()<<endl;
+  return EXIT_FAILURE;
+}
 

rinex.cc

@@ -61,6 +61,22 @@ RINEXReader::~RINEXReader()
     gzclose(d_fp);
 }
 
+/* RINEX format.. is very special. This extracts a value from a line
+   where it should be noted values can be and often are back to back
+*/
+
+static double getRINEXValue(char* line, int offset)
+{
+  char* ptr=line+offset+19;
+  char tmp = *ptr;
+  *ptr = 0;
+  double ret;
+  sscanf(line + offset, "%lf", &ret);
+  //  cout<<"'"<<string(line+offset)<<"'\n";
+  *ptr = tmp;
+  return ret;
+}
+
 bool RINEXReader::get(RINEXEntry& entry)
 {
   char line[300];
@@ -79,7 +95,7 @@ G02 2019 12 16 00 00 00-3.670863807201E-04-7.389644451905E-12 0.000000000000E+00
   */
   
 
-  //  SV   YR  MN DY HR MN SS_______________====================______________________
+  //  SV   YR  MN DY HR MN SS___________________===================___________________
   //  G02 2019 12 16 00 00 00-3.670863807201E-04-7.389644451905E-12 0.000000000000E+00
 
   for(;;) {
@@ -115,7 +131,8 @@ G02 2019 12 16 00 00 00-3.670863807201E-04-7.389644451905E-12 0.000000000000E+00
       continue;
 
     }
-    
+
+    char tmp=line[24];
     line[24]=0;
     char gnss;
     if(sscanf(line, "%c%02d %d %d %d %d %d %d",
@@ -123,52 +140,48 @@ G02 2019 12 16 00 00 00-3.670863807201E-04-7.389644451905E-12 0.000000000000E+00
               &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 8) {
       throw std::runtime_error("Failed to parse '"+string(line)+"'");
     }
-
+    line[24]=tmp;
     bool skip=false;
-    /*
-    if(tm.tm_year != 2019 || tm.tm_mon != 7)
-      skip=true;
-
-    if((entry.sv == 33 || entry.sv == 36 || entry.sv == 13 || entry.sv == 15)
-       && tm.tm_mon < 3)
-      skip = true;
-    */
     tm.tm_mon -= 1;
     tm.tm_year -= 1900;
 
     entry.t=timegm(&tm);
 
-    // skip 5 lines, store 6th
-    for(int n=0 ; n < 6; ++n) {
+    // af0, af1, af2
+    entry.af0 = getRINEXValue(line, 23);
+    entry.af1 = getRINEXValue(line, 42);
+    entry.af2 = getRINEXValue(line, 61);
+    
+    // 5 lines of which we store a bit, store 6th
+    for(int n=1 ; n < 7; ++n) {
       if(!gzgets(d_fp, line, sizeof(line))) 
         return false;
-      if(n==2) {
-        line[23]=0;
-        double toe;
-        sscanf(line+4, "%lf", &toe);
+      if(n==1) {
+        entry.iodnav = getRINEXValue(line, 4);
+      }
+      if(n==3) { 
+        double toe = getRINEXValue(line, 4);
         entry.toe = toe;
       }
+      if(n==5) {
+        entry.clkflags = getRINEXValue(line, 23);
+      }
+      if(n==6) {
+        entry.BGDE1E5a = getRINEXValue(line, 42);
+        entry.BGDE1E5b = getRINEXValue(line, 61);
+      }
       //      cerr<<"Line "<<n<<": "<<line;
     }
-    char tmp=line[23];
-    line[23]=0;
-    sscanf(line+4, "%lf", &entry.sisa);
-    line[23]=tmp;
-
-    tmp=line[42];
-    line[42]=0;
-
-    double health;
-    sscanf(line+23, "%lf", &health);
+    // line 6
+    entry.sisa = getRINEXValue(line, 4);
+    double health = getRINEXValue(line, 23);
     entry.health = health; // yeah..
     
-    //last line
+    //last line, number 7
     if(!gzgets(d_fp, line, sizeof(line)))
       return false;
 
-    line[23]=0;
-    double tow;
-    sscanf(line+4, "%lf", &tow);
+    double tow = getRINEXValue(line, 4);
     entry.tow = tow;
     
     if(skip)

rinex.hh

@@ -18,6 +18,10 @@ struct RINEXEntry
   int health;
   int toe;
   int tow;
+  int iodnav;
+  double af0, af1, af2;
+  double clkflags;
+  double BGDE1E5a, BGDE1E5b;
 };
 
 class RINEXReader
@@ -69,7 +73,8 @@ E01 2019 09 21 23 30 00-6.949011585675E-04-7.943867785798E-12 0.000000000000E+00
   time_t then = utcFromGST(e.wn, (int)e.tow);
   struct tm tm;
   gmtime_r(&then, &tm);
-  
+
+  // 0
   d_ofs << makeSatPartialName(sid)<<" " << fmt::sprintf("%04d %02d %02d %02d %02d %02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
 
   emit(ldexp(e.af0, -34));
@@ -77,39 +82,48 @@ E01 2019 09 21 23 30 00-6.949011585675E-04-7.943867785798E-12 0.000000000000E+00
   emit(ldexp(e.af2, -59));
   d_ofs<<"\n    ";
 
+  // 1
   emit(e.iodnav);
   emit(e.getCrs());
   emit(e.getDeltan());
   emit(e.getM0());
   d_ofs<<"\n    ";
 
-
+  // 2
   emit(e.getCuc());
   emit(e.getE());
   emit(e.getCus());
   emit(e.getSqrtA());
   d_ofs<<"\n    ";
 
+  // 3
   emit(e.getT0e());
   emit(e.getCic());
   emit(e.getOmega0());
   emit(e.getCis());
 
   d_ofs<<"\n    ";
+
+  // 4
   emit(e.getI0());
   emit(e.getCrc());
   emit(e.getOmega());
   emit(e.getOmegadot());
 
   d_ofs<<"\n    ";
+
+  // 5 
   emit(e.getIdot());
-  emit(257);
+               
+  emit(257);   // bit 0 = I/NAV E1, bit 1 = F/NAV E5a, bit 2 = I/NAV E5b
+               // bit 8 = E1,E5a clock aka F/NAV, bit 9 = E1,E5b aka I/NAV
   emit(e.wn + 1024); // so it aligns with GPS
 
 
+  // 6
   d_ofs<<"\n    ";
   emit(numSisa(e.sisa));
-  int health=0;
+  int health=0; 
   health |= e.e1bdvs;
   health |= (e.e1bhs << 2);
   // don't have e5advs
@@ -120,6 +134,7 @@ E01 2019 09 21 23 30 00-6.949011585675E-04-7.943867785798E-12 0.000000000000E+00
   emit(ldexp(e.BGDE1E5a, -32));
   emit(ldexp(e.BGDE1E5b, -32));
 
+  // 7
   d_ofs<<"\n    ";
   emit(e.tow); // XXX
 

rinjoin.cc

@@ -0,0 +1,52 @@
+#include <iostream>
+#include "rinex.hh"
+#include <map>
+#include <optional>
+using namespace std;
+
+struct Value
+{
+  optional<int> af0Inav;
+  optional<int> af0Fnav;
+  int af1;
+  int iod;
+  optional<int> BGDE1E5a;
+  optional<int> BGDE1E5b;
+};
+
+map<pair<time_t, int>, Value> satmap;
+
+int main(int argc, char** argv)
+{
+  for(int n = 1; n < argc; ++n) {
+    RINEXReader rr(argv[n]);
+    RINEXEntry e;
+    while(rr.get(e)) {
+      if(e.gnss != 2)
+        continue;
+      //      cout << e.t <<" " << e.sv <<" " << (int64_t)(rint(ldexp(e.af0,34))) <<" " << (int64_t)(rint(ldexp(e.BGDE1E5a,32)))<<" " << (int64_t)(rint(ldexp(e.BGDE1E5b,32))) <<" "<<e.clkflags <<endl;
+      auto& s=satmap[{e.t, e.sv}];
+      if(((unsigned int)e.clkflags) & 512) { // I/NAV
+        s.af0Inav = rint(ldexp(e.af0,34));
+        s.af1 = rint(ldexp(e.af1,46));
+        s.BGDE1E5a = rint(ldexp(e.BGDE1E5a,32));
+        s.BGDE1E5b = rint(ldexp(e.BGDE1E5b,32));
+        s.iod = e.iodnav;
+      }
+      else {
+        s.af0Fnav = rint(ldexp(e.af0,34));
+        s.af1 = rint(ldexp(e.af1,46));
+        s.BGDE1E5a = rint(ldexp(e.BGDE1E5a,32));
+        // E1E5b unreliable on F/NAV somehow
+      }
+                         
+    }
+    
+  }
+  cout<<"timestamp sv iod af0fnav af0inav af1 bgde1e5a bgde1e5b\n";
+  for(const auto& s : satmap) {
+    if(s.second.af0Fnav.has_value() && s.second.af0Inav.has_value() && s.second.BGDE1E5a.has_value() && s.second.BGDE1E5b.has_value())
+      cout << s.first.first<<" " <<s.first.second<<" " << s.second.iod<<" "<<
+      *s.second.af0Fnav << " " << *s.second.af0Inav <<" " << s.second.af1<<" " <<*s.second.BGDE1E5a <<" " << *s.second.BGDE1E5b << "\n";
+  }
+}

rinreport.cc

@@ -85,8 +85,6 @@ auto worker(HanderOuter<string>* ho)
 
 int main(int argc, char** argv)
 {
-
-
   ifstream filefile(argv[1]);
   string fname;
   deque<string> files;

rtcm.cc

@@ -1,18 +1,33 @@
 #include "rtcm.hh"
 #include "bits.hh"
 #include <iostream>
-
+#include <string.h>
 using namespace std;
 
 void RTCMMessage::parse(const std::string& str)
 {
+  d_gm={};
+  // memset(&d_gm, 0, sizeof(d_gm));
   auto gbu=[&str](int offset, int bits) {
     return getbitu((const unsigned char*)str.c_str(), offset, bits);
   };
+  auto gbum=[&str](int& offset, int bits) {
+    unsigned int ret = getbitu((const unsigned char*)str.c_str(), offset, bits);
+    offset += bits;
+    return ret;
+  };
+  
   auto gbs=[&str](int offset, int bits) {
     return getbits((const unsigned char*)str.c_str(), offset, bits);
   };
-    
+  
+  auto gbsm=[&str](int& offset, int bits) {
+    int ret = getbits((const unsigned char*)str.c_str(), offset, bits);
+    offset += bits;
+    return ret;
+  };
+  
+  
   type = gbu(0, 12);
   //  cout<<"Message number: "<<type << " of size "<<str.size()<<"\n";
   if(type == 1057 || type == 1240) {
@@ -29,7 +44,7 @@ void RTCMMessage::parse(const std::string& str)
     }
     
     int sats = gbu(62, 6);
-    sow = gbu(12, 20);
+    sow = gbu(12, 20);  // this is DF385
     udi = gbu(32, 4);
     mmi = gbu(36, 1);
     reference = gbu(37,1);
@@ -45,16 +60,16 @@ void RTCMMessage::parse(const std::string& str)
       EphemerisDelta ed;
       
       int off = 68+stride*n;
-      ed.radial = gbs(off+     iodlen + 6, 22) * 0.1;
+      ed.radial = gbs(off+     iodlen + 6, 22) * 0.1;   // we store this in millimeters
       ed.along = gbs(off+ iodlen+ 28, 20) * 0.4;
       ed.cross = gbs(off+ iodlen+48, 20) * 0.4;
       
-      
-      ed.dradial = gbs(off +   iodlen+  68, 21) * 0.001; 
+      ed.dradial = gbs(off +   iodlen+  68, 21) * 0.001;   // we store this in mm/s
       ed.dalong = gbs(off + iodlen + 89, 19) * 0.004;
       ed.dcross = gbs(off + iodlen +108, 19) * 0.004;
       ed.iod = gbu(off +6, iodlen);
       ed.sow = sow;
+      ed.udi = udi;
       if(type == 1057) {
         ed.id.gnss = 0;
         ed.id.sigid = 0;
@@ -83,9 +98,11 @@ void RTCMMessage::parse(const std::string& str)
     //    cout <<" sow "<< sow <<" sats "<<sats<<" update interval " << udi <<" mmi " << mmi;
     //    cout << " iod-ssr "<< ssrIOD << " ssr-provider " << ssrProvider << " ssr-solution ";
     //    cout<< ssrSolution <<":\n";
-    
+
     for(int n = 0; n < sats; ++n) {
       ClockDelta cd;
+      cd.sow = sow;
+      cd.udi = udi;
       if(type == 1058) {
         cd.id.gnss = 0;
         cd.id.sigid = 0;
@@ -97,9 +114,20 @@ void RTCMMessage::parse(const std::string& str)
 
       int off = 67+76*n;
       cd.id.sv = gbu(off +0, 6);
-      cd.dclock0 = gbs(off + 6, 22)*1e-4;
-      cd.dclock1 = gbs(off + 28, 21)*1e-6;
-      cd.dclock2 = gbs(off + 49, 27)*2e-8;
+
+      /*
+C0 polynomial coefficient for correction of broadcast satellite clock.
+The reference time t0 is Epoch Time (DF385, DF386) plus 1⁄2 SSR
+Update Interval. The reference time t0 for SSR Update Interval “0” is
+Epoch Time
+
+DF 385: Full seconds since the beginning of the GPS week
+
+      */
+      
+      cd.dclock0 = gbs(off + 6, 22)*1e-4;     // in 0.1 mm, this converts to meters
+      cd.dclock1 = gbs(off + 28, 21)*1e-6;    // meter/s
+      cd.dclock2 = gbs(off + 49, 27)*2e-8;    // meter/s^2
       d_clocks.push_back(cd);
       //      cout<<"  "<< makeSatIDName(cd.id)<<" ";
       //      cout<< cd.dclock0 <<" ";
@@ -107,6 +135,184 @@ void RTCMMessage::parse(const std::string& str)
       //      cout<< cd.dclock2 << endl;
     }
   }
-    
+  else if(type == 1060 || type == 1243) { // combined
+    int sow = gbu(12, 20);
+    int udi = gbu(32, 4);
+    //    int mmi = gbu(36, 1);
+    //    int srd = gbu(37, 1);
+    ssrIOD = gbu(38, 4);
+    ssrProvider = gbu(42, 16);
+    ssrSolution=gbu(58, 4);
+    unsigned int numsats=gbu(62, 6);
 
+    int offset=68;
+    d_ephs.clear();
+    d_clocks.clear();
+    int iodlen = type == 1060 ? 8 : 10; 
+    for(unsigned int n=0; n < numsats; ++n) {
+      ClockDelta cd;
+      EphemerisDelta ed;
+
+      int off = offset + n*(197 + iodlen);
+      cd.sow = ed.sow = sow;
+      cd.udi = ed.udi = udi;
+      cd.id.gnss = (type == 1060) ? 0 : 2; // GPS or Galileo
+      cd.id.sv = gbu(off + 0, 6);
+      cd.id.sigid = (type == 1060) ? 0 : 1;
+
+      ed.id = cd.id;
+      ed.iod =    gbu(off +  6,  iodlen);
+      int shift = iodlen - 8;
+      ed.radial = gbs(off + 14 + shift,  22 ) * 0.1;   // we store this in millimeters
+      ed.along =  gbs(off + 36 + shift,  20 ) * 0.4;
+      ed.cross =  gbs(off + 56 + shift,  20 ) * 0.4;
+      
+      ed.dradial= gbs(off + 76 + shift,  21) * 0.001;   // we store this in mm/s
+      ed.dalong = gbs(off + 97 + shift,  19) * 0.004;
+      ed.dcross = gbs(off +116 + shift,  19) * 0.004;
+
+      d_ephs.push_back(ed);
+      cd.iod = ed.iod;
+      cd.dclock0 = gbs(off + 135 + shift,  22)*1e-4;     // in 0.1 mm, this converts to meters
+      cd.dclock1 = gbs(off + 157 + shift,  21)*1e-6;    // meter/s
+      cd.dclock2 = gbs(off + 178 + shift,  27)*2e-8;    // meter/s^2
+      // 205 / 207
+      d_clocks.push_back(cd);
+    }
+  }
+  else if(type == 1045 || type == 1046) { // F/NAV or I/NAV respectively ephemeris
+    int off=12;
+    d_sv = gbum(off, 6);
+    d_gm.wn = gbum(off, 12);
+    d_gm.iodnav = gbum(off, 10);
+    d_gm.sisa = gbum(off, 8);
+    d_gm.idot = gbsm(off, 14);
+    d_gm.t0c = gbum(off, 14);
+    d_gm.af2 = gbsm(off, 6);
+    d_gm.af1 = gbsm(off, 21);
+    d_gm.af0 = gbsm(off, 31);
+    //
+    d_gm.crs = gbsm(off, 16);
+    d_gm.deltan = gbsm(off, 16);
+    d_gm.m0 = gbsm(off, 32);
+    d_gm.cuc = gbsm(off, 16);
+    d_gm.e = gbum(off, 32);
+    d_gm.cus = gbsm(off, 16);
+    d_gm.sqrtA = gbum(off, 32);
+    d_gm.t0e = gbum(off, 14); 
+    //
+
+    d_gm.cic = gbsm(off, 16);
+    d_gm.omega0 = gbsm(off, 32);
+    d_gm.cis = gbsm(off, 16);
+    d_gm.i0 = gbsm(off, 32);
+    d_gm.crc = gbsm(off, 16);
+    d_gm.omega = gbsm(off, 32);
+    d_gm.omegadot = gbsm(off, 24);
+
+    //     16 + 16 + 32 + 16 + 32 + 16 + 32 + 14 +
+    //     crs deln  M0  cuc   e   cus  sqrA toe  cic OM0 cis i0   crc  omeg omegdot  
+    //    off +=                                         16+ 32 +16 + 32 + 16 + 32 +24;
+    d_gm.BGDE1E5a = gbsm(off, 10);
+    if(type == 1046) { // I/NAV
+      d_gm.BGDE1E5b = gbsm(off, 10);
+    }
+    else {
+      d_gm.BGDE1E5b = 9999999;
+    }
+
+    // thank you RTKLIB:
+#if 0
+    setbitu(rtcm->buff,i,12,1045     ); i+=12;
+    setbitu(rtcm->buff,i, 6,prn      ); i+= 6;
+    setbitu(rtcm->buff,i,12,week     ); i+=12;
+    setbitu(rtcm->buff,i,10,eph->iode); i+=10;
+    setbitu(rtcm->buff,i, 8,eph->sva ); i+= 8;
+    setbits(rtcm->buff,i,14,idot     ); i+=14;
+    setbitu(rtcm->buff,i,14,toc      ); i+=14;
+    setbits(rtcm->buff,i, 6,af2      ); i+= 6;
+    setbits(rtcm->buff,i,21,af1      ); i+=21;
+    setbits(rtcm->buff,i,31,af0      ); i+=31;
+
+    setbits(rtcm->buff,i,16,crs      ); i+=16;
+    setbits(rtcm->buff,i,16,deln     ); i+=16;
+    setbits(rtcm->buff,i,32,M0       ); i+=32;
+    setbits(rtcm->buff,i,16,cuc      ); i+=16;
+    setbitu(rtcm->buff,i,32,e        ); i+=32;
+    setbits(rtcm->buff,i,16,cus      ); i+=16;
+    setbitu(rtcm->buff,i,32,sqrtA    ); i+=32;
+    setbitu(rtcm->buff,i,14,toe      ); i+=14;
+    setbits(rtcm->buff,i,16,cic      ); i+=16;
+    setbits(rtcm->buff,i,32,OMG0     ); i+=32;
+    setbits(rtcm->buff,i,16,cis      ); i+=16;
+    setbits(rtcm->buff,i,32,i0       ); i+=32;
+    setbits(rtcm->buff,i,16,crc      ); i+=16;
+    setbits(rtcm->buff,i,32,omg      ); i+=32;
+    setbits(rtcm->buff,i,24,OMGd     ); i+=24;
+    setbits(rtcm->buff,i,10,bgd1     ); i+=10;
+
+1045:  F/NAV
+    setbitu(rtcm->buff,i, 2,oshs     ); i+= 2; /* E5a SVH */
+    setbitu(rtcm->buff,i, 1,osdvs    ); i+= 1; /* E5a DVS */
+    setbitu(rtcm->buff,i, 7,0        ); i+= 7; /* reserved */
+1046:  I/NAV
+    setbits(rtcm->buff,i,10,bgd2     ); i+=10;
+    setbitu(rtcm->buff,i, 2,oshs1    ); i+= 2; /* E5b SVH */
+    setbitu(rtcm->buff,i, 1,osdvs1   ); i+= 1; /* E5b DVS */
+    setbitu(rtcm->buff,i, 2,oshs2    ); i+= 2; /* E1 SVH */
+    setbitu(rtcm->buff,i, 1,osdvs2   ); i+= 1; /* E1 DVS */
+
+#endif
+  }
+  else if(type == 1059 || type == 1242) { // GPS/Galileo bias
+    int off = 0;
+    int msgnum = gbum(off, 12);
+
+    int gpstime = gbum(off, 20);
+    int uinterval = gbum(off, 4);
+    int mmi = gbum(off, 1);
+    int iodssr = gbum(off, 4);
+    int ssrprov = gbum(off, 16);
+    int ssrsol = gbum(off, 4);
+    int numsats = gbum(off, 6);
+
+    //    cout <<"msgnum "<<msgnum<<" gpstime " << gpstime<<" numsats "<< numsats<<endl;
+    d_dcbs.clear();
+    for(int n=0; n < numsats; ++n) {
+      int gpsid = gbum(off, 6);
+      int numdcbs = gbum(off, 5);
+      //      cout<<"  "<< (type==1059 ? "G" : "E") <<gpsid<<" has "<<numdcbs <<" DCBs\n";
+      SatID id;
+      id.gnss = (type==1059 ? 0 : 2); // GPS or Galileo
+      id.sv = gpsid;
+      for(int m = 0 ; m < numdcbs; ++m) {
+        int sig = gbum(off, 5);
+        id.sigid = sig;
+        int dcb = gbsm(off, 14); // 0.01 meter
+        d_dcbs[id] = 0.01*dcb;
+        //        cout<<"     sig "<<sig <<" dcb " << dcb*0.01 << "\n";
+
+        /*
+Indicator to specify the GPS signal and tracking mode:
+0 - L1 C/A
+1- L1 P
+2- L1 Z-tracking and similar (AS on)
+3 - Reserved
+4 - Reserved
+5 - L2 C/A
+6 - L2 L1(C/A)+(P2-P1) (semi-codeless)
+7 - L2 L2C (M)
+8 - L2 L2C (L)
+9 - L2 L2C (M+L)
+10 - L2 P
+11 - L2 Z-tracking and similar (AS on)
+12 - Reserved
+13 - Reserved
+14 - L5 I
+15 - L5 Q
+>15 - Reserved.
+        */
+      }
+    }
+  }
 }

rtcm.hh

@@ -3,6 +3,9 @@
 #include <stdio.h>
 #include "navmon.hh"
 #include <vector>
+#include "galileo.hh"
+#include <map>
+
 struct RTCMFrame
 {
   std::string payload;
@@ -31,10 +34,11 @@ struct RTCMMessage
   {
     SatID id;
     // in millimeters
-    double radial, along, cross;
-    double dradial, dalong, dcross;
+    double radial, along, cross;   // mm
+    double dradial, dalong, dcross; // mm/s
     int iod;
     int sow;
+    int udi;
   };
   struct ClockDelta
   {
@@ -42,9 +46,14 @@ struct RTCMMessage
     double dclock0; // in meters
     double dclock1;
     double dclock2;
+    int sow;
+    int udi;
+    int iod{-1};
   };
 
   std::vector<EphemerisDelta> d_ephs;
   std::vector<ClockDelta> d_clocks;
-  
+  std::map<SatID, double> d_dcbs;
+  GalileoMessage d_gm;
+  int d_sv;
 };

rtcmtool.cc

@@ -13,10 +13,14 @@ using namespace std;
 bool RTCMReader::get(RTCMFrame& rf)
 {
   int c;
+  bool skipped=false;
   while( ((c=fgetc(d_fp)) != -1) && c != 211) {
-    cerr<<"Skipped.. "<<endl;
+    skipped=true;
+    cerr<<".";
     continue;
   }
+  if(skipped)
+    cerr<<endl;
 
   if(c != 211) {
     cerr<<"EOF"<<endl;
@@ -148,8 +152,9 @@ int main(int argc, char** argv)
   
   RTCMReader rr(0);
   RTCMFrame rf;
+  cerr<<"Station "<<g_srcid<<endl;
   while(rr.get(rf)) {
-    //    cout<<"Got a "<<rf.payload.size()<<" byte frame"<<endl;
+    //    cerr<<"Got a "<<rf.payload.size()<<" byte frame"<<endl;
     RTCMMessage rm;
     NavMonMessage nmm;
     struct timespec ts;

septool.cc

@@ -0,0 +1,603 @@
+#include <string>
+#include "navmon.hh"
+#include <iostream>
+#include <string.h>
+#include <time.h>
+#include "bits.hh"
+#include <sys/time.h>
+#include <arpa/inet.h>
+#include "galileo.hh"
+#include "nmmsender.hh"
+#include "CLI/CLI.hpp"
+#include "swrappers.hh"
+#include "sclasses.hh"
+#include "version.hh"
+
+using namespace std;
+
+
+static int sepsig2ubx(int sep)
+{
+  if(sep == 1) // GPS L1P
+    return 0;
+  else if(sep == 2) // GPS L2P
+    return 4;
+  else if(sep== 4) // GPS L5
+    return 7; // ??
+  else if(sep == 17)
+    return 1; // Galileo E1
+  else if(sep == 19)
+    return 8; // Galileo E6
+  else if(sep == 20)
+    return 6; // Galileo E5a
+  else if(sep==21)
+    return 5; // Galileo E5b
+  else if(sep==22)
+    return 6; // Galileo "AltBoc"
+  
+  else if(sep==0)
+    return 0; // GPS L1
+  else if(sep==3)
+    return 4; // GPS L2c
+  else throw runtime_error("Asked to convert unknown Septentrio signal id "+to_string(sep));
+  return -1;
+}
+
+struct SEPMessage
+{
+  SEPMessage(const std::basic_string<uint8_t>& str) : d_store(str) {}
+
+  uint16_t getID() // includes revision
+  {
+    return d_store[4] + 256*d_store[5];
+  }
+
+  uint16_t getIDBare() // includes revision
+  {
+    return getID() & 0xFFF;
+  }
+
+  
+  std::basic_string<uint8_t> getPayload()
+  {
+    return d_store.substr(8);
+  }
+  std::basic_string<uint8_t> d_store;
+};
+
+/* format:
+
+   01  23   45   67    
+   $@ CRC blk-id len [len-8 bytes] 
+                  |
+            multiple of four
+
+   bits 0-12 of blk-id are the type
+*/
+
+std::pair<SEPMessage, struct timeval> getSEPMessage(int fd, double* timeout)
+{
+  uint8_t marker[2]={0};
+  bool hadskip=false;
+  for(;;) {
+    marker[0] = marker[1];
+    int res = readn2Timeout(fd, marker+1, 1, timeout);
+
+    if(res < 0) {
+      cerr<<"Readn2Timeout failed: "<<strerror(errno)<<endl;
+      throw EofException();
+    }
+    if(marker[0]=='$' && marker[1]=='@') { // bingo
+      if(hadskip) {
+        cerr<<"\n";
+        hadskip=false;
+      }
+      struct timeval tv;
+      gettimeofday(&tv, 0);
+      basic_string<uint8_t> msg;
+      msg.append(marker, 2);  // 0,1
+      uint8_t b[6];
+      readn2Timeout(fd, b, 6, timeout);
+      msg.append(b, 6); // crc id len
+
+      // 0,1 = crc, 2-3 = marker, 4, 5
+      //      uint16_t blkid = htons(b[2] + 256*b[3]);
+      uint16_t len = b[4] + 256*b[5];
+      //      cerr<<"Got message of type "<<getbitu((uint8_t*)&blkid, 0, 12)<<", revision "<<
+      //        getbitu((uint8_t*)&blkid, 12, 4)<<" ("<<ntohs(blkid)<<"), len= "<<len<<endl;
+      
+      uint8_t buffer[len-8];
+      res=readn2Timeout(fd, buffer, len-8, timeout);
+
+      msg.append(buffer, len - 8); // checksum
+      return make_pair(SEPMessage(msg), tv);
+    }
+    else if(marker[1] != '$') {
+      hadskip=true;
+      cerr<<".";
+    }
+         
+  }                                                                       
+}
+
+static char program[]="septool";
+uint16_t g_srcid{0};
+
+int main(int argc, char** argv)
+try
+{
+  time_t starttime=time(0);
+  GOOGLE_PROTOBUF_VERIFY_VERSION;
+  vector<string> destinations;
+  g_dtLS = 18;
+  bool doVERSION{false}, doSTDOUT{false};
+  CLI::App app(program);
+  string sourceaddr;
+  bool quiet{false};
+  app.add_option("--source", sourceaddr, "Connect to this IP address:port to source SBF (otherwise stdin)");
+  app.add_option("--destination,-d", destinations, "Send output to this IPv4/v6 address");
+  app.add_option("--station", g_srcid, "Station id")->required();
+  app.add_option("--quiet", quiet, "Don't emit noise");
+  app.add_flag("--version", doVERSION, "show program version and copyright");
+  app.add_flag("--stdout", doSTDOUT, "Emit output to stdout");
+  try {
+    app.parse(argc, argv);
+  } catch(const CLI::Error &e) {
+    return app.exit(e);
+  }
+
+  if(doVERSION) {
+    showVersion(program, g_gitHash);
+    exit(0);
+  }
+
+
+  signal(SIGPIPE, SIG_IGN);
+  NMMSender ns;
+  ns.d_debug = true;
+  for(const auto& s : destinations) {
+    auto res=resolveName(s, true, true);
+    if(res.empty()) {
+      cerr<<"Unable to resolve '"<<s<<"' as destination for data, exiting"<<endl;
+      exit(EXIT_FAILURE);
+    }
+    ns.addDestination(s); // ComboAddress(s, 29603));
+  }
+  if(doSTDOUT)
+    ns.addDestination(1);
+
+  int srcfd=0;
+  if(!sourceaddr.empty()) {
+    ComboAddress src(sourceaddr);
+    
+    srcfd = socket(src.sin4.sin_family, SOCK_STREAM, 0);
+    if(srcfd < 0)
+      unixDie("making socket for SBF connection");
+    cerr<<"Connecting to "<< src.toStringWithPort()<<" to source data..";
+    SConnectWithTimeout(srcfd, src, 5);
+    cerr<<" done"<<endl;
+  }
+  ns.d_compress = true;
+  ns.launch();
+  cerr<<"Station "<<g_srcid<<endl;
+  for(;;) {
+    double to=1000;
+    auto res = getSEPMessage(srcfd, &to);
+    if(!quiet)
+      cerr<<res.first.getID()<<" - " <<res.first.getIDBare() << endl;
+    if(res.first.getID() == 4023) { // I/NAV
+      auto str = res.first.getPayload();
+      struct SEPInav
+      {
+        uint32_t towMsec;
+        uint16_t wn;
+        uint8_t sv;
+        uint8_t crcPassed;
+        uint8_t viterbiCount;
+        uint8_t src;
+        uint8_t ign1;
+        uint8_t rxChannel;
+        uint8_t navBits[32];
+      } __attribute__((packed));
+      SEPInav si;
+      memcpy(&si, str.c_str(), sizeof(si));
+      //      cerr<<"tow "<<si.towMsec /1000<<" wn "<<si.wn <<" sv " << (int) si.sv - 70<<" ";
+      
+      if(!si.crcPassed) {
+        cerr<<"I/NAV CRC error, skipping"<<endl;
+        continue;
+      }
+      int sigid = si.src & 31;
+
+      std::string inav((char*)si.navBits, 32);
+      //      cerr<<makeHexDump(inav)<<endl;
+
+
+      // byte order adjustment
+      std::basic_string<uint8_t> payload;
+      
+      for(unsigned int i = 0 ; i < 8; ++i) {
+        payload.append(1, si.navBits[4 * i + 3]);
+        payload.append(1, si.navBits[4 * i + 2]);
+        payload.append(1, si.navBits[4 * i + 1]);
+        payload.append(1, si.navBits[4 * i + 0]);
+      }
+
+      
+      basic_string<uint8_t> inav2;
+      // copy in the even page
+      for(int n = 0 ; n < 14; ++n)
+        inav2.append(1, getbitu(payload.c_str(), 2 + n*8, 8));
+      // odd page
+      for(int n = 0 ; n < 2; ++n)
+        inav2.append(1, getbitu(payload.c_str(), 116 + n*8, 8));
+      //      cerr<<makeHexDump(inav2) << endl;
+
+      
+      basic_string<uint8_t> reserved1;
+      for(int n=0; n < 5 ; ++n)
+	reserved1.append(1, getbitu(payload.c_str(), 116 + 16 + n*8, 8));
+
+      NavMonMessage nmm;
+      double t = utcFromGST(si.wn - 1024, si.towMsec / 1000.0);
+      //      cerr<<t<< " " <<si.wn - 1024 <<" " <<si.towMsec /1000.0 <<" " << g_dtLS<<endl;
+      nmm.set_sourceid(g_srcid);
+      nmm.set_type(NavMonMessage::GalileoInavType);
+
+      nmm.set_localutcseconds(t);
+      nmm.set_localutcnanoseconds(0); // yeah XXX
+            
+      nmm.mutable_gi()->set_gnsswn(si.wn - 1024);
+      
+      nmm.mutable_gi()->set_gnsstow(si.towMsec/1000.0);
+      nmm.mutable_gi()->set_gnssid(2);
+      nmm.mutable_gi()->set_gnsssv(si.sv - 70);
+      nmm.mutable_gi()->set_contents((const char*)&inav2[0], inav2.size());
+      nmm.mutable_gi()->set_sigid(sepsig2ubx(sigid));
+      nmm.mutable_gi()->set_reserved1((const char*)&reserved1[0], reserved1.size());
+      ns.emitNMM( nmm);
+      
+    }
+    else if(res.first.getID() == 5914) {
+      // current time
+    }
+
+    else if(res.first.getID() == 4026) {
+      // GLONASS
+    }
+    else if(res.first.getID() == 4017) { // GPS-CA
+
+    }
+    else if(res.first.getID() == 4018) { // GPS-L2C
+
+    }
+    else if(res.first.getID() == 4019) { // GPS raw L5
+
+    }
+    else if(res.first.getID() == 4093) { // navic raw
+
+    }
+    else if(res.first.getID() == 4022) {  // F/NAV
+      auto str = res.first.getPayload();
+      struct SEPFnav
+      {
+        uint32_t towMsec;
+        uint16_t wn;
+        uint8_t sv;
+        uint8_t crcPassed;
+        uint8_t viterbiCount;
+        uint8_t src;
+        uint8_t ign1;
+        uint8_t rxChannel;
+        uint8_t navBits[32];
+      } __attribute__((packed));
+      SEPFnav sf;
+      memcpy(&sf, str.c_str(), sizeof(sf));
+      int sigid = sf.src & 31;
+      //      cerr<<"tow "<<sf.towMsec /1000<<" wn "<<sf.wn <<" sv " << (int) sf.sv - 70<<" sigid " << sigid <<" ";
+      if(!sf.crcPassed) {
+        cerr<<"F/NAV CRC error, skipping"<<endl;
+        continue;
+      }
+
+      std::string fnav((char*)sf.navBits, 32);
+      // byte order adjustment
+      std::basic_string<uint8_t> payload;
+      
+      for(unsigned int i = 0 ; i < 8; ++i) {
+        payload.append(1, sf.navBits[4 * i + 3]);
+        payload.append(1, sf.navBits[4 * i + 2]);
+        payload.append(1, sf.navBits[4 * i + 1]);
+        payload.append(1, sf.navBits[4 * i + 0]);
+      }
+
+      NavMonMessage nmm;
+      double t = utcFromGST(sf.wn - 1024, sf.towMsec / 1000.0);
+      //      cerr<<t<< " " <<si.wn - 1024 <<" " <<si.towMsec /1000.0 <<" " << g_dtLS<<endl;
+      nmm.set_sourceid(g_srcid);
+      nmm.set_type(NavMonMessage::GalileoFnavType);
+
+      nmm.set_localutcseconds(t);
+      nmm.set_localutcnanoseconds(0); // yeah XXX
+            
+      nmm.mutable_gf()->set_gnsswn(sf.wn - 1024);
+      
+      nmm.mutable_gf()->set_gnsstow(sf.towMsec/1000.0);
+      nmm.mutable_gf()->set_gnssid(2);
+      nmm.mutable_gf()->set_gnsssv(sf.sv - 70);
+      nmm.mutable_gf()->set_contents((const char*)&payload[0], payload.size());
+      nmm.mutable_gf()->set_sigid(sepsig2ubx(sigid));
+      ns.emitNMM( nmm);
+    }
+    else if(res.first.getIDBare() == 4047) {
+      // BDSRaw
+    }
+    else if(res.first.getIDBare() == 4006) {
+      auto str = res.first.getPayload();
+      struct PVTCartesian
+      {
+        uint32_t towMsec;
+        uint16_t wn;
+        uint8_t mode, error;
+        double x, y, z;
+        float undulation;
+        float vx, vy, vz;
+      } __attribute__((packed));
+      PVTCartesian pc;
+      memcpy(&pc, str.c_str(), sizeof(pc));
+
+      NavMonMessage nmm;
+      nmm.set_type(NavMonMessage::ObserverPositionType);
+      nmm.set_localutcseconds(utcFromGST(pc.wn - 1024, pc.towMsec / 1000.0));
+      nmm.set_localutcnanoseconds(0);
+      nmm.set_sourceid(g_srcid);
+      
+      nmm.mutable_op()->set_x(pc.x);
+      nmm.mutable_op()->set_y(pc.y);
+      nmm.mutable_op()->set_z(pc.z);
+      nmm.mutable_op()->set_acc(3.14);
+      
+      ns.emitNMM( nmm);
+
+      {
+        NavMonMessage nmm;
+        nmm.set_sourceid(g_srcid);
+        nmm.set_localutcseconds(utcFromGST(pc.wn - 1024, pc.towMsec / 1000.0));
+        nmm.set_localutcnanoseconds(0);
+        
+        nmm.set_type(NavMonMessage::ObserverDetailsType);
+        nmm.mutable_od()->set_vendor("Septentrio");
+        nmm.mutable_od()->set_hwversion("Mosaic");
+        nmm.mutable_od()->set_swversion("");
+        nmm.mutable_od()->set_serialno("3060601");
+        nmm.mutable_od()->set_modules("");
+        nmm.mutable_od()->set_clockoffsetns(0);
+        nmm.mutable_od()->set_clockoffsetdriftns(0);
+        nmm.mutable_od()->set_clockaccuracyns(0);
+        nmm.mutable_od()->set_freqaccuracyps(0);
+
+        nmm.mutable_od()->set_owner("Septentrio");
+        nmm.mutable_od()->set_remark("");
+        nmm.mutable_od()->set_recvgithash(g_gitHash);
+        nmm.mutable_od()->set_uptime(time(0) - starttime);
+        ns.emitNMM( nmm);
+      }
+      
+    }
+    else if(res.first.getIDBare() == 4024) { // GALRawCNAV
+      struct SEPCnav
+      {
+        uint32_t towMsec;
+        uint16_t wn;
+        uint8_t sv;
+        uint8_t crcPassed;
+        uint8_t viterbiCount;
+        uint8_t src;
+        uint8_t ign1;
+        uint8_t rxChannel;
+        uint8_t navBits[64];
+      } __attribute__((packed));
+
+      SEPCnav sc;
+      
+      auto str = res.first.getPayload();
+      memcpy(&sc, str.c_str(), sizeof(sc));
+      int sigid = sc.src & 31;
+      //      cerr<<"C/NAV tow "<<sc.towMsec /1000<<" wn "<<sc.wn <<" sv " << (int) sc.sv - 70<<" sigid " << sigid <<" ";
+      if(!sc.crcPassed) {
+        cerr<<"C/NAV CRC error, skipping"<<endl;
+        continue;
+      }
+
+      std::string cnav((char*)sc.navBits, 64);
+      // byte order adjustment
+      std::basic_string<uint8_t> payload;    
+      
+      for(unsigned int i = 0 ; i < 16; ++i) {
+        payload.append(1, sc.navBits[4 * i + 3]);
+        payload.append(1, sc.navBits[4 * i + 2]);
+        payload.append(1, sc.navBits[4 * i + 1]);
+        payload.append(1, sc.navBits[4 * i + 0]);
+      }
+
+      unsigned char crc_buff[58]={0};
+      unsigned int i;
+      for (i=0; i< 462;i++)
+	setbitu(crc_buff, 2+i, 1,getbitu(payload.c_str(),i, 1));
+
+      int calccrc=rtk_crc24q(crc_buff,58);
+      int realcrc= getbitu(payload.c_str(), 14+448, 24);
+      if (calccrc !=  realcrc) {
+	cerr << "CRC mismatch, " << calccrc << " != " << realcrc <<endl;
+      }
+      //      else
+      //	cerr<<"Checksum Correct: "<<calccrc << " = " << realcrc<<endl;
+
+      
+      
+      NavMonMessage nmm;
+      double t = utcFromGST(sc.wn - 1024, sc.towMsec / 1000.0);
+      //      cerr<<t<< " " <<si.wn - 1024 <<" " <<si.towMsec /1000.0 <<" " << g_dtLS<<endl;
+      nmm.set_sourceid(g_srcid);
+      nmm.set_type(NavMonMessage::GalileoCnavType);
+
+      nmm.set_localutcseconds(t);
+      nmm.set_localutcnanoseconds(0); // yeah XXX
+            
+      nmm.mutable_gc()->set_gnsswn(sc.wn - 1024);
+      
+      nmm.mutable_gc()->set_gnsstow(sc.towMsec/1000.0);
+      nmm.mutable_gc()->set_gnssid(2);
+      nmm.mutable_gc()->set_gnsssv(sc.sv - 70);
+      nmm.mutable_gc()->set_contents((const char*)&payload[0], payload.size());
+      nmm.mutable_gc()->set_sigid(sepsig2ubx(sigid));
+      ns.emitNMM( nmm);
+    }
+    else if(res.first.getIDBare() == 4027) {
+      auto str = res.first.getPayload();
+      struct MeasEpoch
+      {
+        uint32_t towMsec;
+        uint16_t wn;
+        uint8_t n1;
+        uint8_t sb1len;
+        uint8_t sb2len;
+        uint8_t commonFlags;
+        uint8_t clkJumps;
+        uint8_t res1;
+      } __attribute__((packed));
+      MeasEpoch me;
+      memcpy(&me, str.c_str(), sizeof(me));
+      //      cerr<<"Got "<<(int)me.n1<<" signal statuses, block1 "<<(int)me.sb1len<<", block2 "<<(int)me.sb2len<<endl;
+
+      struct Block1
+      {
+        uint8_t rxchannel, type, sv, misc; // misc contains 4 bits of codeMSB
+        uint32_t codeLSB;
+        int32_t doppler;
+        uint16_t carrierLSB;
+        int8_t carrierMSB;
+        uint8_t cn0;
+        uint16_t lockTime;
+        uint8_t obsinfo;
+        uint8_t n2;
+      } __attribute__((packed));
+      struct Block2
+      {
+        uint8_t type, locktime, cn0, offsetMSB;
+        int8_t carrierMSB;
+        uint8_t obsinfo;
+        uint16_t codeoffsetLSB;
+        uint16_t carrierLSB;
+        uint16_t dopplerOffsetLSB;
+      } __attribute__((packed));
+
+      int pos = sizeof(me);
+      for(int n = 0 ; n < me.n1; ++n) {
+        Block1 b1;
+        memcpy(&b1, str.c_str() + pos, sizeof(b1));
+        uint8_t sigid = b1.type & 31;
+        //        cerr<<"sv "<<(int)b1.sv<<" sigid "<< (int)sigid <<" cn0 ";
+        double db;
+        if(sigid==1 || sigid ==2)
+          db = b1.cn0 *0.25;
+        else db = b1.cn0 * 0.25 + 10;
+        //        cerr<<" "<<db;
+        //        cerr<<" n2 "<< (int)b1.n2;
+        //        cerr<<endl;
+        pos += me.sb1len;
+
+        if(b1.sv <= 36 || (b1.sv > 70 && b1.sv <= 106)) {
+          NavMonMessage nmm;
+          nmm.set_sourceid(g_srcid);
+          nmm.set_localutcseconds(utcFromGST(me.wn - 1024, me.towMsec / 1000.0));
+          nmm.set_localutcnanoseconds(0);
+          
+          nmm.set_type(NavMonMessage::ReceptionDataType);
+          nmm.mutable_rd()->set_gnssid(b1.sv > 70 ? 2 : 0 );
+          nmm.mutable_rd()->set_gnsssv(b1.sv <= 37 ? b1.sv : b1.sv - 70);
+          nmm.mutable_rd()->set_db(db);
+          nmm.mutable_rd()->set_el(0);
+          nmm.mutable_rd()->set_azi(0);
+          nmm.mutable_rd()->set_prres(-1);
+          nmm.mutable_rd()->set_qi(7);
+                      
+          try {
+            nmm.mutable_rd()->set_sigid(sepsig2ubx(sigid));
+            ns.emitNMM(nmm);
+          }
+          catch(...){}
+
+          /*
+          LSB of the pseudorange. The pseudorange expressed in meters
+            is computed as follows:
+            PR type1 [m] = ( CodeMSB *4294967296+ CodeLSB )*0.001
+
+            codeMSB hides in bits 0-3 of misc. 
+          */
+          
+        }
+        
+        for(int m = 0 ; m < b1.n2; ++m) {
+          Block2 b2;
+          memcpy(&b2, str.c_str() + pos, sizeof(b2));
+          pos += me.sb2len;
+          sigid = b2.type & 31;
+          //          cerr<<"\t sigid  "<<(int)sigid<<" cn0 ";
+          if(sigid==1 || sigid ==2)
+            db= b2.cn0 *0.25;
+          else db = b2.cn0 * 0.25 + 10;
+          //          cerr<<db;
+          //          cerr<<endl;
+
+          if(b1.sv <= 36 || (b1.sv > 70 && b1.sv <= 106)) {
+            NavMonMessage nmm;
+            nmm.set_sourceid(g_srcid);
+            nmm.set_localutcseconds(utcFromGST(me.wn - 1024, me.towMsec / 1000.0));
+            nmm.set_localutcnanoseconds(0);
+            
+            nmm.set_type(NavMonMessage::ReceptionDataType);
+            nmm.mutable_rd()->set_gnssid(b1.sv > 70 ? 2 : 0 );
+            nmm.mutable_rd()->set_gnsssv(b1.sv <= 37 ? b1.sv : b1.sv - 70);
+            nmm.mutable_rd()->set_db(db);
+            nmm.mutable_rd()->set_el(0);
+            nmm.mutable_rd()->set_azi(0);
+            nmm.mutable_rd()->set_prres(0);
+            nmm.mutable_rd()->set_qi(7);
+            try {
+              nmm.mutable_rd()->set_sigid(sepsig2ubx(sigid));            
+              ns.emitNMM(nmm);
+            }catch(...){} // might be unknown signal type
+          }
+        }
+      }
+      
+    }
+    else {
+    if(!quiet)
+      cerr<<"Unknown message "<<res.first.getID() << " / " <<res.first.getIDBare()<<"  ("<<res.first.d_store.size()<<" bytes)"<<endl;
+    }
+  }
+}
+catch(std::exception& e)
+{
+  cerr<<"Fatal: "<<e.what()<<endl;
+}
+catch(EofException& e)
+{
+  
+}
+
+      /*
+        NAVBits contains the 234 bits of an I/NAV navigation page (in nominal
+or alert mode). Note that the I/NAV page is transmitted as two sub-pages
+(the so-called even and odd pages) of duration 1 second each (120 bits
+each). 
+
+In this block, the even and odd pages are concatenated, even page
+first and odd page last. The 6 tails bits at the end of the even page are
+removed (hence a total of 234 bits). If the even and odd pages have been
+received from two different carriers (E5b and L1), bit 5 of the Source
+field is set.
+      */
+         

sp3.hh

@@ -8,8 +8,8 @@ struct SP3Entry
   int gnss;
   int sv;
   time_t t;
-  double x, y, z;
-  double clockBias;
+  double x, y, z; // meters
+  double clockBias; // nanoseconds
 };
 
 class SP3Reader

sp3feed.cc

@@ -18,11 +18,12 @@ int main(int argc, char **argv)
 {
   string influxDBName("galileo2");
   bool doVERSION=false;
-  int sigid=1;
+
   CLI::App app(program);
   vector<string> fnames;
+  string sp3src("default");
   app.add_flag("--version", doVERSION, "show program version and copyright");
-  app.add_option("--sigid,-s", sigid, "Signal identifier. 1 or 5 for Galileo.");
+  app.add_option("--sp3src,-s", sp3src, "Identifier of SP3 source");
   app.add_option("--influxdb", influxDBName, "Name of influxdb database");
   app.add_option("files", fnames, "filenames to parse");
   try {
@@ -40,14 +41,10 @@ int main(int argc, char **argv)
   for(const auto& fn : fnames) {
     SP3Reader sp3(fn);
     SP3Entry e;
-    SatID sid;
     cout<<fn<<endl;
     while(sp3.get(e)) {
-      sid.gnss = e.gnss;
-      sid.sigid = sigid;
-      sid.sv = e.sv;
       // XXX LEAP SECOND ADJUSTMENT FIXED AT 18 SECONDS
-      idb.addValue(sid, "sp3", {{"x", e.x}, {"y", e.y}, {"z", e.z}, {"clock-bias", e.clockBias}}, e.t + 18);
+      idb.addValue({{"gnssid", e.gnss}, {"sv", e.sv}, {"sp3src", sp3src}}, "sp3", {{"x", e.x}, {"y", e.y}, {"z", e.z}, {"clock-bias", e.clockBias}}, e.t + 18);
     }
   }
 }

storage.cc

@@ -5,6 +5,7 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <arpa/inet.h>
+#include <iostream>
 using namespace std;
 
 
@@ -82,8 +83,22 @@ bool getNMM(FILE* fp, NavMonMessage& nmm, uint32_t& offset)
 bool getRawNMM(int fd, timespec& t, string& raw, uint32_t& offset)
 {
   char bert[4];
-  if(read(fd, bert, 4) != 4 || bert[0]!='b' || bert[1]!='e' || bert[2] !='r' || bert[3]!='t') {
-    return false;
+  int res;
+  if((res=read(fd, bert, 4)) != 4 || bert[0]!='b' || bert[1]!='e' || bert[2] !='r' || bert[3]!='t') {
+    if(res != 4)
+      return false;
+
+    for(int s=0;; ++s ) {
+      cerr<<"Skipping character hunting for good magic.. "<<s<<endl;
+      bert[0] = bert[1];
+      bert[1] = bert[2];
+      bert[2] = bert[3];
+      res = read(fd, bert + 3, 1);
+      if(res != 1)
+        return false;
+      if(bert[0]=='b' && bert[1]=='e' && bert[2] =='r' && bert[3]=='t')
+        break;
+    }
   }
     
     
@@ -109,8 +124,26 @@ bool getRawNMM(int fd, timespec& t, string& raw, uint32_t& offset)
 bool getRawNMM(FILE* fp, timespec& t, string& raw, uint32_t& offset)
 {
   char bert[4];
-  if(fread(bert, 1, 4, fp) != 4 || bert[0]!='b' || bert[1]!='e' || bert[2] !='r' || bert[3]!='t') {
-    return false;
+  int res;
+  if((res=fread(bert, 1, 4, fp)) != 4 || bert[0]!='b' || bert[1]!='e' || bert[2] !='r' || bert[3]!='t') {
+    if(res != 4) {
+      //      cerr<<"EOF"<<endl;
+      return false;
+    }
+
+    for(int s=0;; ++s ) {
+      cerr<<"Skipping character hunting for good magic.. "<<s<<endl;
+      bert[0] = bert[1];
+      bert[1] = bert[2];
+      bert[2] = bert[3];
+      res = fread(bert+3, 1,1, fp);
+      if(res != 1)
+        return false;
+      if(bert[0]=='b' && bert[1]=='e' && bert[2] =='r' && bert[3]=='t') {
+        cerr<<"Resync!"<<endl;
+        break;
+      }
+    }
   }
     
   uint16_t len;

tle.cc

@@ -101,6 +101,11 @@ TLERepo::Match TLERepo::getBestMatch(time_t now, double x, double y, double z, T
       //      cerr<<"TLE error: "<<se.what()<<endl;
       continue;
     }
+    catch(DecayedException& se) {
+      //      cerr<<"TLE error: "<<se.what()<<endl;
+      continue;
+    }
+
   }
   if(distances.empty())
     return TLERepo::Match();

ubx.cc

@@ -3,6 +3,7 @@
 #include "fmt/format.h"
 #include "fmt/printf.h"
 #include "bits.hh"
+#include "navmon.hh"
 
 using namespace std;
 uint16_t calcUbxChecksum(uint8_t ubxClass, uint8_t ubxType, std::basic_string_view<uint8_t> str)
@@ -60,7 +61,12 @@ std::basic_string<uint8_t> buildUbxMessage(uint8_t ubxClass, uint8_t ubxType, st
   return msg;
 }
 
-basic_string<uint8_t> getInavFromSFRBXMsg(std::basic_string_view<uint8_t> msg)
+basic_string<uint8_t> getInavFromSFRBXMsg(std::basic_string_view<uint8_t> msg,
+                                          basic_string<uint8_t>& reserved1,
+                                          basic_string<uint8_t>& reserved2,
+                                          basic_string<uint8_t>& sar,
+                                          basic_string<uint8_t>& spare,
+                                          basic_string<uint8_t>& crc)
 {
   // byte order adjustment
   std::basic_string<uint8_t> payload;
@@ -74,10 +80,14 @@ basic_string<uint8_t> getInavFromSFRBXMsg(std::basic_string_view<uint8_t> msg)
   for (i=0,j=  4;i<15;i++,j+=8) setbitu(crc_buff,j,8,getbitu(payload.c_str()   ,i*8,8));
   for (i=0,j=118;i<11;i++,j+=8) setbitu(crc_buff,j,8,getbitu(payload.c_str()+16,i*8,8));
   if (rtk_crc24q(crc_buff,25) != getbitu(payload.c_str()+16,82,24)) {
-    cout << "CRC mismatch, " << rtk_crc24q(crc_buff, 25) << " != " << getbitu(payload.c_str()+16,82,24) <<endl;
+    cerr << "CRC mismatch, " << rtk_crc24q(crc_buff, 25) << " != " << getbitu(payload.c_str()+16,82,24) <<endl;
     throw CRCMismatch();
   }
 
+  crc.clear();
+  for(i=0; i < 3; ++i)
+    crc.append(1, getbitu(payload.c_str()+16,82+i*8,8));
+  
   std::basic_string<uint8_t> inav;
   
   for (i=0,j=2; i<14; i++, j+=8)
@@ -85,6 +95,21 @@ basic_string<uint8_t> getInavFromSFRBXMsg(std::basic_string_view<uint8_t> msg)
   for (i=0,j=2; i< 2; i++, j+=8)
     inav.append(1, (unsigned char)getbitu(payload.c_str()+16,j,8));
 
+  reserved1.clear();
+  for(i=0, j=18; i < 5 ; i++, j+=8)
+    reserved1.append(1, (unsigned char)getbitu(payload.c_str()+16, j, 8));
+  //  cerr<<"reserved1: "<<makeHexDump(reserved1)<<endl;
+
+  sar.clear();
+  for(i=0, j=58; i < 3 ; i++, j+=8) // you get 24 bits
+    sar.append(1, (unsigned char)getbitu(payload.c_str()+16, j, 8));
+
+  spare.clear();
+  spare.append(1, (unsigned char)getbitu(payload.c_str()+16, 80, 2));
+
+  reserved2.clear();
+  reserved2.append(1, (unsigned char)getbitu(payload.c_str()+16, 106, 8));
+  
   return inav;
 }
 

ubx.hh

@@ -6,7 +6,13 @@ std::basic_string<uint8_t> buildUbxMessage(uint8_t ubxClass, uint8_t ubxType, st
 
 std::basic_string<uint8_t> buildUbxMessage(uint8_t ubxClass, uint8_t ubxType, const std::initializer_list<uint8_t>& str);
 
-std::basic_string<uint8_t> getInavFromSFRBXMsg(std::basic_string_view<uint8_t> msg);
+std::basic_string<uint8_t> getInavFromSFRBXMsg(std::basic_string_view<uint8_t> msg,
+                                               std::basic_string<uint8_t>& reserved1,
+                                               std::basic_string<uint8_t>& reserved2,
+                                               std::basic_string<uint8_t>& sar,
+                                               std::basic_string<uint8_t>& spare,
+                                               std::basic_string<uint8_t>& crc);
+
 std::basic_string<uint8_t> getGPSFromSFRBXMsg(std::basic_string_view<uint8_t> msg);
 std::basic_string<uint8_t> getGlonassFromSFRBXMsg(std::basic_string_view<uint8_t> msg);
 std::basic_string<uint8_t> getBeidouFromSFRBXMsg(std::basic_string_view<uint8_t> msg);

ubxtool.cc

@@ -446,16 +446,63 @@ static string format_serial(basic_string<uint8_t> payload)
                             payload[8]);
 }
 
+// these are four structs to capture Ublox F9P time offset stats
+namespace {
+struct TIMEGAL
+{
+  uint32_t itow;
+  uint32_t galTow;
+  int32_t fGalTow;
+  int16_t galWno;
+  int8_t leapS;
+  uint8_t valid;
+  uint32_t tAcc;          
+} __attribute__((packed));
+
+struct TIMEBDS
+{
+  uint32_t itow;
+  uint32_t sow;
+  int32_t fSow;
+  int16_t week;
+  int8_t leapS;
+  uint8_t valid;
+  uint32_t tAcc;          
+} __attribute__((packed));
+
+struct TIMEGLO
+{
+  uint32_t itow;
+  uint32_t tod;
+  int32_t fTod;
+  uint16_t nT;
+  uint8_t n4;
+  uint8_t valid;
+  uint32_t tAcc;          
+} __attribute__((packed));
+
+struct TIMEGPS
+{
+  uint32_t itow;
+  int32_t ftow;
+  int16_t week;
+  int8_t leapS;
+  uint8_t valid;
+  uint32_t tAcc;          
+} __attribute__((packed));
+
+}
+
 // ubxtool device srcid
 int main(int argc, char** argv)
 {
-  time_t starttime=time(0);
+  auto starttime = std::chrono::steady_clock::now();
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
   CLI::App app(program);
     
 
-  bool doGPS{true}, doGalileo{true}, doGlonass{false}, doBeidou{true}, doReset{false}, doWait{true}, doRTSCTS{true}, doSBAS{false};
+  bool doGPS{true}, doGalileo{true}, doGlonass{false}, doBeidou{false}, doReset{false}, doWait{true}, doRTSCTS{true}, doSBAS{false};
   bool doFakeFix{false};
   bool doKeepNMEA{false};
   bool doSTDOUT=false;
@@ -672,9 +719,9 @@ int main(int argc, char** argv)
         }
       }
       else { // UBX-CFG-VALSET
-
+        //                                 vers  ram   res   res    
         msg = buildUbxMessage(0x06, 0x8a, {0x00, 0x01, 0x00, 0x00,
-              0x1f,0x00,0x31,0x10, doGPS,
+              0x1f,0x00,0x31,0x10, doGPS,   // 
               0x01,0x00,0x31,0x10, doGPS,
               0x03,0x00,0x31,0x10, doGPS,
 
@@ -874,11 +921,13 @@ int main(int argc, char** argv)
         cerr<<"Got timeout waiting for ack of port protocol poll, no problem"<<endl;
       }
 
-      
-      if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling UBX-RXM-RLM"<<endl; } // SAR
-      enableUBXMessageOnPort(fd, 0x02, 0x59, ubxport); // UBX-RXM-RLM
 
-      if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling UBX-MON-HW"<<endl; } // SAR
+      if(mods.find("NEO-M8P") ==string::npos) {
+        if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling UBX-RXM-RLM"<<endl; } // SAR
+        enableUBXMessageOnPort(fd, 0x02, 0x59, ubxport); // UBX-RXM-RLM
+      }
+
+      if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling UBX-MON-HW"<<endl; } 
       enableUBXMessageOnPort(fd, 0x0A, 0x09, ubxport, 16); // UBX-MON-HW
 
       
@@ -891,13 +940,30 @@ int main(int argc, char** argv)
         enableUBXMessageOnPort(fd, 0x0d, 0x04, ubxport, 2);       
       }
 
-      if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling UBX-RXM-RAWX"<<endl; } // RF doppler
-      enableUBXMessageOnPort(fd, 0x02, 0x15, ubxport, 8); // RXM-RAWX
+      if(mods.find("NEO-M9N") == string::npos) {
+        if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling UBX-RXM-RAWX"<<endl; } // RF doppler
+        enableUBXMessageOnPort(fd, 0x02, 0x15, ubxport, 8); // RXM-RAWX
+      }
 
       if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling UBX-NAV-CLOCK"<<endl; } // clock details
       enableUBXMessageOnPort(fd, 0x01, 0x22, ubxport, 16); // UBX-NAV-CLOCK
 
-      
+      if(1) {
+        if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling/disabling UBX-NAV-TIMEGPS"<<endl; } // GPS time solution
+        enableUBXMessageOnPort(fd, 0x01, 0x20, ubxport, doGPS ? 16 : 0); // UBX-NAV-TIMEGPS
+
+        if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling/disabling "<< doGlonass<< " UBX-NAV-TIMEGLO"<<endl; } // GLONASS time solution
+        enableUBXMessageOnPort(fd, 0x01, 0x23, ubxport, doGlonass ? 16 : 0); // UBX-NAV-TIMEGLO
+	
+        if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling/disabling UBX-NAV-TIMEBDS"<<endl; } // Beidou time solution
+        enableUBXMessageOnPort(fd, 0x01, 0x24, ubxport, doBeidou ? 16 : 0); // UBX-NAV-TIMEBDS
+
+        if(mods.find("NEO-M8P") ==string::npos) { 
+          if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling/disabling UBX-NAV-TIMEGAL"<<endl; } // Galileo time solution
+          enableUBXMessageOnPort(fd, 0x01, 0x25, ubxport, doGalileo ? 16 : 0); // UBX-NAV-TIMEGAL
+        }
+      }
+            
       if(!version9 && !m8t && !fuzzPositionMeters) {
         if (doDEBUG) { cerr<<humanTimeNow()<<" Enabling debugging data"<<endl; } // RF doppler
         enableUBXMessageOnPort(fd, 0x03, 0x10, ubxport, 4);
@@ -946,12 +1012,104 @@ int main(int argc, char** argv)
   ns.launch();
   
   cerr<<humanTimeNow()<<" Entering main loop"<<endl;
+
+  struct TIMEXState
+  {
+    TIMEGAL gal;
+    TIMEGPS gps;
+    TIMEGLO glo;
+    TIMEBDS bds;
+    bool doGlonass, doGalileo, doBeidou, doGPS;
+    void transmitIfComplete(NMMSender& ns)
+    {
+      vector<int> itows;
+      if(doGlonass)
+        itows.push_back(glo.itow);
+      if(doGalileo)
+        itows.push_back(gal.itow);
+      if(doBeidou)
+        itows.push_back(bds.itow);
+      if(doGPS)
+        itows.push_back(gps.itow);
+
+      if(itows.empty())
+        return;
+      if(itows[0] == 0)
+        return;
+        
+      for(const auto& itow : itows)
+        if(itow != itows[0])
+           return;
+        
+      NavMonMessage nmm;
+      nmm.set_sourceid(g_srcid);
+      nmm.set_localutcseconds(g_gnssutc.tv_sec);
+      nmm.set_localutcnanoseconds(g_gnssutc.tv_nsec);
+      
+      nmm.set_type(NavMonMessage::TimeOffsetType);
+      nmm.mutable_to()->set_itow(gps.itow);
+
+      NavMonMessage::GNSSOffset* no;
+      if(doGPS) {
+        no = nmm.mutable_to()->add_offsets();
+        no->set_gnssid(0);
+        no->set_offsetns(gps.ftow);
+        no->set_tacc(gps.tAcc);
+        no->set_tow(gps.itow); // this is for consistency
+        no->set_leaps(gps.leapS);
+        no->set_wn(gps.week);
+        no->set_valid(gps.valid);
+      }
+      
+      if(doGalileo) {
+        no = nmm.mutable_to()->add_offsets();
+        no->set_gnssid(2);
+        no->set_offsetns(gal.fGalTow);
+        no->set_tacc(gal.tAcc);
+        no->set_leaps(gal.leapS);
+        no->set_wn(gal.galWno);
+        no->set_valid(gal.valid);
+        no->set_tow(gal.galTow);
+      }
+      
+      if(doBeidou) {
+        no = nmm.mutable_to()->add_offsets();
+        no->set_gnssid(3);
+        no->set_offsetns(bds.fSow);
+        no->set_tacc(bds.tAcc);
+        no->set_leaps(bds.leapS);
+        no->set_wn(bds.week);
+        no->set_valid(bds.valid);
+        no->set_tow(bds.sow);
+      }
+      
+      if(doGlonass) {
+        no = nmm.mutable_to()->add_offsets();
+        no->set_gnssid(6);
+        no->set_offsetns(glo.fTod);
+        no->set_tacc(glo.tAcc);
+        no->set_nt(glo.nT);
+        no->set_n4(glo.n4);
+        no->set_valid(glo.valid);
+        no->set_tow(glo.tod);
+      }
+      ns.emitNMM(nmm);
+      gal.itow = 0;
+      gps.itow = 0;
+      glo.itow = 0;
+      bds.itow = 0;
+    }
+  } tstate;
+  
+  tstate.doGPS = doGPS; tstate.doGalileo = doGalileo; tstate.doGlonass = doGlonass;
+  tstate.doBeidou = doBeidou;
+  
   for(;;) {
     try {
       auto [msg, timestamp] = getUBXMessage(fd, nullptr);
       (void)timestamp;
       auto payload = msg.getPayload();
-
+      
       if(msg.getClass() == 0x01 && msg.getType() == 0x07) {  // UBX-NAV-PVT
         struct PVT
         {
@@ -1040,10 +1198,7 @@ int main(int argc, char** argv)
         if(doDEBUG)
           cerr<<humanTimeNow()<<" Serial number from stream "<< serialno <<endl;
       }
-
-      
-      
-      if(msg.getClass() == 0x02 && msg.getType() == 0x15) {  // RAWX, the doppler stuff
+      else       if(msg.getClass() == 0x02 && msg.getType() == 0x15) {  // RAWX, the doppler stuff
         //        if (doDEBUG) { cerr<<humanTimeNow()<<" Got "<<(int)payload[11] <<" measurements "<<endl; }
         double rcvTow;
         memcpy(&rcvTow, &payload[0], 8);
@@ -1230,7 +1385,8 @@ int main(int argc, char** argv)
             ns.emitNMM( nmm);            
           }
           else if(id.first ==2) { // GALILEO
-            auto inav = getInavFromSFRBXMsg(payload);
+            basic_string<uint8_t> reserved1, reserved2, sar, spare, crc;
+            auto inav = getInavFromSFRBXMsg(payload, reserved1, reserved2, sar, spare, crc);  
             unsigned int wtype = getbitu(&inav[0], 0, 6);
 
             uint32_t satTOW;
@@ -1311,6 +1467,11 @@ int main(int argc, char** argv)
             nmm.mutable_gi()->set_gnsssv(id.second);
             nmm.mutable_gi()->set_sigid(sigid);
             nmm.mutable_gi()->set_contents((const char*)&inav[0], inav.size());
+            nmm.mutable_gi()->set_reserved1((const char*)&reserved1[0], reserved1.size());
+            nmm.mutable_gi()->set_reserved2((const char*)&reserved2[0], reserved2.size());
+            nmm.mutable_gi()->set_sar((const char*)    &sar[0],   sar.size());
+            nmm.mutable_gi()->set_crc((const char*)    &crc[0],   crc.size());
+            nmm.mutable_gi()->set_spare((const char*)&spare[0], spare.size());
             
             ns.emitNMM( nmm);
           }
@@ -1542,7 +1703,7 @@ int main(int argc, char** argv)
         nmm.mutable_od()->set_owner(owner);
         nmm.mutable_od()->set_remark(remark);
         nmm.mutable_od()->set_recvgithash(g_gitHash);
-        nmm.mutable_od()->set_uptime(time(0) - starttime);
+        nmm.mutable_od()->set_uptime(std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now()-starttime).count());
         
         
         ns.emitNMM( nmm);
@@ -1564,8 +1725,6 @@ int main(int argc, char** argv)
         }
       }
       else if(msg.getClass() == 0x02 && msg.getType() == 0x59) { // UBX-RXM-RLM
-        int type = (int)payload[1];      
-        int sv = (int)payload[2];
 
         NavMonMessage nmm;
         nmm.set_sourceid(g_srcid);
@@ -1573,34 +1732,29 @@ int main(int argc, char** argv)
         nmm.set_localutcnanoseconds(g_gnssutc.tv_nsec);
         
         nmm.set_type(NavMonMessage::SARResponseType);
+
+        // short version:
+        //   0         1   2    3      4   -  11    12     13,14 15
+        // version, type, sv, reserved beacon id  msg-code param res2
+
+        // long version:
+        //   0         1   2    3      4   -  11    12     13-24  25
+        // version, type, sv, reserved beacon id  msg-code params res2
+        
         nmm.mutable_sr()->set_gnssid(2); // Galileo only for now
-        nmm.mutable_sr()->set_gnsssv(sv);
-        nmm.mutable_sr()->set_sigid(0); // we should fill this in later
+        nmm.mutable_sr()->set_gnsssv(payload[2]);
+        nmm.mutable_sr()->set_sigid(1); // 
         nmm.mutable_sr()->set_type(payload[1]);
         nmm.mutable_sr()->set_identifier(string((char*)payload.c_str()+4, 8));
         nmm.mutable_sr()->set_code(payload[12]);
         nmm.mutable_sr()->set_params(string((char*)payload.c_str()+13, payload.size()-14));
+
         
         string hexstring;
-        for(int n = 0; n < 15; ++n)
+        for(int n = 0; n < 15; ++n)                                   
           hexstring+=fmt::sprintf("%x", (int)getbitu(payload.c_str(), 36 + 4*n, 4));
 
-        //        if (doDEBUG) { cerr<<humanTimeNow()<<" "<<humanTime(g_gnssutc.tv_sec)<<" SAR RLM type "<<type<<" from gal sv " << sv << " beacon "<<hexstring <<" code "<<(int)payload[12]<<" params "<<payload[12] + 256*payload[13]<<endl; }
-
-      //      wk.emitLine(sv, "SAR "+hexstring);
-      //      cout<<"SAR: sv = "<< (int)msg[2] <<" ";
-      //      for(int n=4; n < 12; ++n)
-      //        fmt::printf("%02x", (int)msg[n]);
-
-      //      for(int n = 0; n < 15; ++n)
-      //        fmt::printf("%x", (int)getbitu(msg.c_str(), 36 + 4*n, 4));
-      
-      //      cout << " Type: "<< (int) msg[12] <<"\n";
-      //      cout<<"Parameter: (len = "<<msg.length()<<") ";
-      //      for(unsigned int n = 13; n < msg.length(); ++n)
-      //        fmt::printf("%02x ", (int)msg[n]);
-      //      cout<<"\n";
-
+        ns.emitNMM(nmm);
       }
       else if(msg.getClass()==39 && msg.getType()==0) {
         NavMonMessage nmm;
@@ -1712,6 +1866,29 @@ int main(int argc, char** argv)
         nmm.mutable_ujs()->set_jamind(mhw.jamInd);
         ns.emitNMM(nmm);
       }
+      else if(msg.getClass() == 0x01 && msg.getType() == 0x25) { // UBX-NAV-TIMEGAL
+        memcpy(&tstate.gal, &payload[0], sizeof(TIMEGAL));
+        //        cerr << "TIMEGAL itow: "<<tstate.gal.itow<<", fGalTow: "<<tstate.gal.fGalTow<<", tAcc: "<<tstate.gal.tAcc<< ", valid: "<< !!tstate.gal.valid<< endl;
+        tstate.transmitIfComplete(ns);
+      }
+      else
+      if(msg.getClass() == 0x01 && msg.getType() == 0x24) { // UBX-NAV-TIMEBDS
+        memcpy(&tstate.bds, &payload[0], sizeof(TIMEBDS));
+//        cerr << "TIMEBDS itow: "<<tstate.bds.itow<<", fSow: "<<tstate.bds.fSow<<", tAcc: "<<tstate.bds.tAcc<< ", valid: "<< !!tstate.bds.valid  << endl;
+        tstate.transmitIfComplete(ns);
+      }
+      else
+      if(msg.getClass() == 0x01 && msg.getType() == 0x23) { // UBX-NAV-TIMEGLO
+        memcpy(&tstate.glo, &payload[0], sizeof(TIMEGLO));
+//        cerr << "TIMEGLO itow: "<<tstate.glo.itow<<", fTod: "<<tstate.glo.fTod<<", tAcc: "<<tstate.glo.tAcc<< ", valid: "<<!!tstate.glo.valid<<endl;
+        tstate.transmitIfComplete(ns);
+      }
+      else
+      if(msg.getClass() == 0x01 && msg.getType() == 0x20) { // UBX-NAV-TIMEGPS
+        memcpy(&tstate.gps, &payload[0], sizeof(TIMEGPS));
+        //        cerr << "TIMEGPS itow: "<<tstate.gps.itow<<", ftow: "<<tstate.gps.ftow<<", tAcc: "<<tstate.gps.tAcc<< ", valid: "<< !!tstate.gps.valid<<endl;
+        tstate.transmitIfComplete(ns);
+      }
       else 
         if (doDEBUG) { cerr<<humanTimeNow()<<" Unknown UBX message of class "<<(int) msg.getClass() <<" and type "<< (int) msg.getType()<< " of "<<payload.size()<<" bytes"<<endl; }
 

version.hh

@@ -1,6 +1,6 @@
 #pragma once
 
-void showVersion(char *pname, const char *hash) {
+void showVersion(const char *pname, const char *hash) {
   std::cout <<"galmon tools (" <<pname <<") " <<hash <<std::endl;
   std::cout <<"built date " <<__DATE__ <<std::endl;
   std::cout <<"(C) AHU Holding BV - bert@hubertnet.nl - https://berthub.eu/" <<std::endl;

zstdwrap.cc

@@ -149,17 +149,21 @@ void ZStdReader::worker()
 
   for(;;) {
     input.pos=0;
-    input.size=read(d_sourcefd, (char*)input.src, inputcapacity);
-    if(input.size <= 0) {
+    int ret = read(d_sourcefd, (char*)input.src, inputcapacity);
+    if(ret <= 0) {
       cerr<<"Got EOF on input fd "<<d_sourcefd<<", terminating thread"<<endl;
       break;
     }
+    input.size = ret; // this is unsigned, so we need 'ret' to see the error
     while(input.pos != input.size) {
       output.pos=0;
       output.size=outputcapacity;
-      ZSTD_decompressStream(z, &output, &input);
+      int res = ZSTD_decompressStream(z, &output, &input);
+      if(ZSTD_isError(res)) {
+        cerr<<"Error decompressing ZSTD data"<<endl;
+        break;
+      }
 
-      int res;
       res = writen(d_writepipe, output.dst, output.pos);
       if(!res) // we are history
         break;

zstdwrap.hh

@@ -1,6 +1,7 @@
 #pragma once
 #include <cstdint>
-#include <zstd.h> // can't easily be moved to zstdwrap.cc, trust me
+#include <zstd.h> // apt-get install libzstd-dev if you miss this. 
+// can't easily be moved to zstdwrap.cc, trust me
 #include <functional>
 #include <thread>
 #include <atomic>