%
% Upload.tex
%
% SatNOGS Optical Unofficial Guide
%
% Copyright (C) 2022, Jeff Moe
%
% This document is licensed under the Creative Commons Attribution 4.0
% International Public License (CC BY-SA 4.0) by Jeff Moe.
%

\section{Push Data Upstream}
\label{sec:upload}
\index{upload}
The \gls{SatNOGS} network isn't ready for optical data yet.
So there is nothing that can be uploaded. This is under
development.

\section{File Extensions}
\label{sec:upload-extensions}
\index{formats}
\index{.cal}\index{.cat}\index{.csv}\index{.dat}\index{.fits}\index{.png}
The following file extensions are in use:

\begin{mdframed}[backgroundcolor=blue!10,linecolor=blue!30]
\begin{description}
        \item [.cal] --- Cal % XXX
        \item [.cat] --- Cat % XXX
        \item [.csv] --- \gls{CSV}
        \item [.dat] --- Data.
        \item [.fits] --- \gls{FITS}.
        \item [.png] --- \gls{PNG}.
\end{description}
\end{mdframed}
\index{FITS}\index{PNG}\index{CSV}


\section{Data Files}
\label{sec:upload-data}
\index{data}\index{stvid}\index{process}\index{acquire}
After the \texttt{stvid} acquisition and processing steps have been
run, files such as below will be generated:

\begin{minted}{sh}
2022-08-23T04:16:26.633.fits
2022-08-23T04:16:26.633.fits.cal
2022-08-23T04:16:26.633.fits.cat
2022-08-23T04:16:26.633.fits.png
2022-08-23T04:16:26.633_48473_catalog.dat
2022-08-23T04:16:26.633_48473_catalog.png
2022-08-23T04:16:26.633_52718_catalog.dat
2022-08-23T04:16:26.633_52718_catalog.png
2022-08-23T04:16:26.633_90000_unid.dat
2022-08-23T04:16:26.633_90000_unid.png
2022-08-23T04:16:26.633_hough.csv
2022-08-23T04:16:26.633_predict.csv
2022-08-23T04:16:26.633_threshold.csv
\end{minted}
\index{.cal}\index{.cat}\index{.csv}\index{.dat}\index{.fits}\index{.png}


\subsection{\texttt{.fits} FITS Files}
\gls{FITS} files are the main files created during acquisition and
are the main files used in processing. It has the data of
the actual image taken with the camera. \gls{FITS} files will be significantly
larger than the other files, at around 8 to 35 megabytes each, depending
on sample rate, frame size, and \gls{binning}.
\index{FITS}\index{acquire}\index{process}
\index{frame size}\index{sample rate}

The \texttt{stvid} program creates it's customized \gls{FITS} format
file, which is a stack up of multiple images.
I'll leave explanation of the \texttt{stvid} \gls{FITS} file structure
to the program's author, Cees Bassa:

\begin{quote}
With stvid, I typically store 10 seconds worth of video frames, 100 frames at 10 \gls{Hz} frame rate or 250 frames at 25 \gls{Hz} frame rate into a so-called ``four frame''. Here, for each pixel of the initial 100 or 250 frames, the mean, standard deviation, maximum and argmax are stored, each as its own frame. Objects that move will stand out because they will have a high (max - mean) / standard deviation, which allows me to detect trails. Once a trail is detected, the argmax frame tells me the frame number and hence the motion of the trail, and using the timestamps of the individual 100 or 250 frames, I can reconstruct the pixel position as a function of time during the 10 second exposure, using only 4 frames instead of the 100 or 250 individual video frames.
\end{quote}
\index{Bassa, Cees}\index{stvid}\index{four frame}\index{trail}
\index{.fits}


\subsection{\texttt{.cal} CAL Files}
\index{.cal}\index{stvid}
Example \texttt{.cal} filename:\\
\texttt{2022-08-23T04:16:26.633.fits.cal}\\

Sample data from \texttt{.cal} file, just the top select lines of a 577 line file:

\begin{minted}{sh}
1172.876   11.776  7.1775 221.117561  45.185979  6.761
 786.524    1.817  6.0213 222.623526  37.272049  5.590
 838.954   21.772  8.0430 222.941344  38.403949  7.653
 872.352   25.919  8.3855 222.909215  39.103246  7.884
 692.542   37.123  8.2615 223.889916  35.455924  8.718
 738.829   47.126  8.2037 223.973142  36.430116  7.777
 691.964   52.656  7.7910 224.278547  35.490201  7.411
 951.962   53.159  8.0072 223.315748  40.816697  7.844
1209.704   60.030  6.0799 222.327791  46.116208  5.811
 893.057   66.086  7.1867 223.943996  39.648460  8.039
...
\end{minted}


\subsection{\texttt{.cat} CAT Files}
\index{.cat}\index{stvid}\index{FITS}
\texttt{stvid} generates \texttt{.cat} files, generated from \gls{FITS} files, such as:\\
\texttt{2022-08-23T04:16:26.633.fits.cat} \\

Sample data from \texttt{.cat} file, just the top select lines of a 1,033 line file:

\begin{minted}{sh}
#   1 X_IMAGE                Object position along x                                    [pixel]
#   2 Y_IMAGE                Object position along y                                    [pixel]
#   3 MAG_BEST               Best of MAG_AUTO and MAG_ISOCOR                            [mag]
#   4 FLUX_BEST              Best of FLUX_AUTO and FLUX_ISOCOR                          [count]
#   5 FLUX_MAX               Peak flux above background                                 [count]
#   6 BACKGROUND             Background at centroid position                            [count]
  1172.8755     11.7756   7.1775     33.80787     11.09625     16.06616
    28.8250     11.4696   6.7443     50.38253     5.008112     15.81194
  1347.4933     10.4887   8.3649     11.32553     2.619246     16.04644
   269.9134      9.8019   8.4220     10.74506      1.56768     15.97937
   468.9823      7.9780   8.8072     7.535751     5.987358     16.02673
  1475.9681      6.6672   8.6115     9.024131     2.281431     16.03049
   847.0105      5.9795   8.1992     13.19223     14.31796     16.08833
   181.5139      5.5141   7.9473     16.63711     2.278276     15.95168
   460.1482      5.1577   8.5318     9.711239     2.795708     16.02688
  1559.9583      2.9245   7.5915     23.08835     6.981803      15.9909
   786.5237      1.8174   6.0213     98.05751     27.88283     16.09418
...
\end{minted}


\subsection{\texttt{.fits.png} FITS PNG Files}
\index{FITS}\index{PNG}\index{stvid}
\index{.fits.png}
The \texttt{stvid} application can generate multiple \gls{PNG} files,
depending how many satellites it finds. In every case it creates at
least one \gls{PNG} file, with a name like:

\texttt{2022-08-23T04:16:26.633.fits.png}

In this image, as can be see in figure \ref{fig:stvid-png}, page \pageref{fig:stvid-png},
no detected satellites are indicated.
The blue lines are from the ``Catalog'', which in this case means \glspl{TLE}.
The blue lines are drawn whether no, a few, or many satellites are detected.
They are generated by taking data from the \gls{plate-solver} about where the picture
\gls{FOV} is, and overlaying satellite \glspl{orbit} that are calculated from
\glspl{TLE}.
\index{detect}\index{plate solver}\index{orbit}\index{TLE}

The main blue line is slightly offset from where the satellite is calculated to
appear.
The blue line forms a small \texttt{L} shape.
The smaller segment indicates the area where the satellite may pass.
Next to the small segment line is the \gls{NORAD-ID} for the satellite.
If it is unknown, it will be given the number \texttt{90000} or larger.

\begin{figure}[h!]
        \begin{center}
                \includegraphics[keepaspectratio=true,height=0.85\textheight,width=0.85\textwidth,angle=0]{stvid/data/2022-08-23T04:16:26.633.fits.png}
                \caption{\texttt{stvid} PNG.}
                \label{fig:stvid-png}
                \index{stvid}\index{PNG}\index{FITS}
        \end{center}
\end{figure}
\index{.fits.png}


\subsection{\texttt{catalog.png} catalog PNG Files}
\index{FITS}\index{PNG}\index{stvid}
\index{detect}\index{identify}
When \texttt{stvid} identifies a satellite, it creates a new \gls{PNG} file for each
satellite detected in the image. A file name example follows, with satellite
\gls{NORAD-ID} 48473 identified, also shown in figure
\ref{fig:stvid-png-48473-catalog}, page \pageref{fig:stvid-png-48473-catalog}.

\texttt{2022-08-23T04:16:26.633\_48473\_catalog.png}

\begin{figure}[h!]
        \begin{center}
                \includegraphics[keepaspectratio=true,height=0.85\textheight,width=0.85\textwidth,angle=0]{stvid/data/2022-08-23T04:16:26.633_48473_catalog.png}
                \caption{\texttt{stvid} PNG of the satellite trail of NORAD ID 48473 identified in red.}
                \label{fig:stvid-png-48473-catalog}
                \index{stvid}\index{PNG}\index{FITS}
        \end{center}
\end{figure}
\index{stvid}\index{identify}\index{PNG}

If multiple satellites are detected in the image, they will each get a \gls{PNG} file.
For the example image, a satellite with \gls{NORAD-ID} 52718 was also identified.
The file name is:

\texttt{2022-08-23T04:16:26.633\_52718\_catalog.png}

\begin{figure}[h!]
        \begin{center}
                \includegraphics[keepaspectratio=true,height=0.85\textheight,width=0.85\textwidth,angle=0]{stvid/data/2022-08-23T04:16:26.633_52718_catalog.png}
                \caption{\texttt{stvid} PNG of the satellite trail of NORAD ID 52718 identified in red.}
                \label{fig:stvid-png-52718-catalog}
                \index{stvid}\index{PNG}\index{FITS}\index{identify}
        \end{center}
\end{figure}


\subsection{\texttt{unid.png} Unidentified PNG Files}
\index{FITS}\index{PNG}\index{stvid}
\index{unidentified}
As shown previously, \texttt{stvid} will create a new \gls{PNG} with a
red line when it identifies a satellite.
When \texttt{stvid} finds a trail it can't identify in the
\gls{TLE} catalogs, it gives it a \gls{NORAD-ID} starting with \texttt{90000},
incrementing by one.
See figure \ref{fig:stvid-png-90000-unid}, page \pageref{fig:stvid-png-90000-unid},
for an example of \texttt{stvid} marking an unidentified trail red.
In this case, the trail was created by an airplane, but \texttt{stvid} would
do the same with an unidentified satellite trail.
\index{airplane}

\begin{figure}[h!]
        \begin{center}
                \includegraphics[keepaspectratio=true,height=0.85\textheight,width=0.85\textwidth,angle=0]{stvid/data/2022-08-23T04:16:26.633_90000_unid.png}
                \caption{\texttt{stvid} PNG of an unidentified trail in red.}
                \label{fig:stvid-png-90000-unid}
                \index{stvid}\index{PNG}\index{FITS}
        \end{center}
\end{figure}


\subsection{\texttt{.dat} Data Files}
\index{.dat}\index{PNG}
For each trail detected as shown in previous sections, such as for which
a \gls{PNG} file was created,
\texttt{stvid} will output a \texttt{.dat} file. The \texttt{.dat} filenames
corresponding to the figures at \ref{fig:stvid-png-48473-catalog}, \ref{fig:stvid-png-52718-catalog},
and \ref{fig:stvid-png-90000-unid} are:

\begin{minted}{sh}
2022-08-23T04:16:26.633_48473_catalog.dat
2022-08-23T04:16:26.633_52718_catalog.dat
2022-08-23T04:16:26.633_90000_unid.dat
\end{minted}

Note, there is no \texttt{.dat} file created corresponding to the
\texttt{2022-08-23T04:16:26.633.fits.png} file, since it doesn't show
detected trails.

The \texttt{.dat} file contents look like this, for \gls{NORAD-ID} 48473:

\begin{minted}{sh}
48473 21 040AX  9999 G 20220823041631625 17 25 1533406+483563 37 S
\end{minted} 

XXX This is the IOD ? XXX
The first field is the \gls{NORAD-ID}. The fourth field looks like COSPAR. XXX
Large number starting with 2022 is date and time stamp.

The \texttt{.dat} file contents look like this, for \gls{NORAD-ID} 52718:
\index{.dat}

\begin{minted}{sh}
52718 22 056C   9999 G 20220823041631625 17 25 1527277+381243 37 S
\end{minted} 

The \texttt{.dat} file contents look like this, for the unidentified trail
(the airplane):
\index{airplane}\index{unidentified}

\begin{minted}{sh}
90000 22 500A   9999 G 20220823041631625 17 25 1457613+535844 37 S
\end{minted} 


\subsection{\texttt{stvid} \texttt{.csv} Files}
\index{.csv}\index{stvid}\index{hough}\index{predict}\index{threshold}
During processing, \texttt{stvid} generates (at least!) three \gls{CSV} files.
For the observation discussed above, staring with figure \ref{fig:stvid-png},
the following \gls{CSV} files are generated:

\begin{minted}{sh}
2022-08-23T04:16:26.633_hough.csv
2022-08-23T04:16:26.633_predict.csv
2022-08-23T04:16:26.633_threshold.csv
\end{minted} 


\subsubsection{\texttt{stvid} \texttt{hough.csv} Files}
\index{.csv}\index{stvid}\index{hough}
Full contents of the \texttt{2022-08-23T04:16:26.633\_hough.csv} file:

\begin{minted}{sh}
ax,ay,az,bx,by,bz,n
1585.699766,186.468458,51.073014,0.879795,0.416052,0.229917,3424
794.849123,355.614035,48.596491,0.780147,-0.382158,0.495304,285
1297.660777,432.469965,52.989399,0.923059,0.193205,0.332617,283
\end{minted} 

\subsubsection{\texttt{stvid} \texttt{predict.csv} Files}
\index{.csv}\index{stvid}\index{predict}
The \texttt{2022-08-23T04:16:26.633\_predict.csv} is 3,764 lines long.
Below is an extract of the lines. I shortened the path, which
included the full path to my home directory, for clarity:

\begin{minted}{sh}
satno,cospar,mjd,ra,dec,state,tlefile,age
00134,61015U  ,59814.17808603,182.367480,+37.582914,sunlit,/data/tle/catalog.tle,0.441
00134,61015U  ,59814.17809887,182.231429,+37.736730,sunlit,/data/tle/catalog.tle,0.441
00134,61015U  ,59814.17811171,182.094454,+37.890457,sunlit,/data/tle/catalog.tle,0.441
00134,61015U  ,59814.17812455,181.956550,+38.044091,sunlit,/data/tle/catalog.tle,0.441
...
00735,64002C  ,59814.17808603,227.004270,+34.763570,sunlit,/data/tle/catalog.tle,0.027
00735,64002C  ,59814.17809887,227.192764,+34.379114,sunlit,/data/tle/catalog.tle,0.027
00735,64002C  ,59814.17811171,227.378969,+33.994697,sunlit,/data/tle/catalog.tle,0.027
...
46083,20055BJ ,59814.17808603,204.492490,+40.972573,sunlit,/data/tle/catalog.tle,0.022
46083,20055BJ ,59814.17809887,204.840813,+41.039396,sunlit,/data/tle/catalog.tle,0.022
46083,20055BJ ,59814.17811171,205.193715,+41.105756,sunlit,/data/tle/catalog.tle,0.022
...
32706,08010A  ,59814.17808603,169.555855,+50.073199,sunlit,/data/tle/inttles.tle,0.624
32706,08010A  ,59814.17809887,169.560127,+50.075268,sunlit,/data/tle/inttles.tle,0.624
32706,08010A  ,59814.17811171,169.564400,+50.077336,sunlit,/data/tle/inttles.tle,0.624
...
\end{minted} 
\index{.csv}\index{COSPAR}\index{TLE}


\subsubsection{\texttt{stvid} \texttt{threshold.csv} Files}
\index{.csv}\index{stvid}\index{threshold}
\texttt{stvid} creates a \texttt{threshold.csv} file with a filename like:

\texttt{2022-08-23T04:16:26.633\_threshold.csv}

The above file has 5,293 lines. Here are a few of them:

\begin{minted}{sh}
58.000000,0.000000,38.000000
69.000000,1.000000,59.000000
1832.000000,1.000000,52.000000
418.000000,2.000000,11.000000
1183.000000,2.000000,71.000000
...
1276.000000,1078.000000,1.000000
11.000000,1079.000000,38.000000
1500.000000,1079.000000,45.000000
\end{minted}

