json noise

This also includes a refresh of the requirements after the Python3
transition.

Signed-off-by: Fabian P. Schmidt <kerel@mailbox.org>

.gitignore

@@ -15,6 +15,10 @@ pip-log.txt
 nosetests.xml
 .pytest_cache
 
+# build artifacts
+build
+dist
+
 # Sqlite
 *.db
 *.sqlite
@@ -23,10 +27,19 @@ nosetests.xml
 # Media & Static
 media
 /staticfiles/*
+/network/static/lib/
 node_modules
-yarn-error.log
 *.css.map
 .sass-cache
 
 # Celery
 celerybeat-schedule
+
+# Documentation
+/docs/_build/
+
+# Text editor
+*.swp
+
+# Python generated files
+satnogs_network.egg-info/

.gitlab-ci.yml

@@ -1,3 +1,10 @@
+variables:
+  GITLAB_CI_IMAGE_ALPINE: 'alpine:3.9'
+  GITLAB_CI_IMAGE_DOCKER: 'docker:18.09'
+  GITLAB_CI_IMAGE_NODE: 'node:12.13.1'
+  GITLAB_CI_IMAGE_PYTHON: 'python:3'
+  GITLAB_CI_PYPI_DOCKER_COMPOSE: 'docker-compose~=1.23.0'
+  GITLAB_CI_PYPI_TOX: 'tox~=3.8.0'
 stages:
   - static
   - build
@@ -6,32 +13,38 @@ stages:
   - trigger
 static_js_css:
   stage: static
-  image: node:11.13
+  image: ${GITLAB_CI_IMAGE_NODE}
   script:
-    - npm install -g eslint stylelint
-    - eslint 'network/static/js/*.js'
-    - stylelint 'network/static/css/*.css' 'network/static/css/*.scss'
-static_python:
-  stage: static
-  image: python:2.7
-  before_script:
-    - pip install tox~=3.8.0
-  script:
-    - tox -e flake8
-build:
-  stage: build
-  image: python:2.7
-  script:
-    - rm -rf dist
-    - python setup.py sdist bdist_wheel
+    - npm ci
+    - node_modules/.bin/gulp
   artifacts:
     expire_in: 1 week
     when: always
     paths:
-      - dist
-build3:
+      - network/static/lib
+static_python:
+  stage: static
+  image: ${GITLAB_CI_IMAGE_PYTHON}
+  before_script:
+    - pip install "$GITLAB_CI_PYPI_TOX"
+  script:
+    - tox -e "flake8,isort,yapf,pylint"
+docs:
   stage: build
-  image: python:3.6
+  image: ${GITLAB_CI_IMAGE_PYTHON}
+  before_script:
+    - pip install "$GITLAB_CI_PYPI_TOX"
+  script:
+    - rm -rf docs/_build
+    - tox -e "docs"
+  artifacts:
+    expire_in: 1 week
+    when: always
+    paths:
+      - docs/_build/html
+build:
+  stage: build
+  image: ${GITLAB_CI_IMAGE_PYTHON}
   script:
     - rm -rf dist
     - python setup.py sdist bdist_wheel
@@ -42,21 +55,21 @@ build3:
       - dist
 test:
   stage: test
-  image: python:2.7
+  image: ${GITLAB_CI_IMAGE_PYTHON}
   before_script:
-    - pip install tox~=3.8.0
+    - pip install "$GITLAB_CI_PYPI_TOX"
     - apt-get update
     - apt-get install -y ruby-sass
   script:
     - tox -e deps,pytest
 docker:
   stage: deploy
-  image: docker:18.09
+  image: ${GITLAB_CI_IMAGE_DOCKER}
   services:
-    - docker:18.09-dind
+    - ${GITLAB_CI_IMAGE_DOCKER}-dind
   before_script:
     - apk --update add py-pip
-    - pip install docker-compose~=1.23.0
+    - pip install "$GITLAB_CI_PYPI_DOCKER_COMPOSE"
   script:
     - |
       [ -z "$CI_REGISTRY_IMAGE" ] || {
@@ -91,7 +104,7 @@ docker:
       - tags
 trigger_master:
   stage: trigger
-  image: alpine:3.9
+  image: ${GITLAB_CI_IMAGE_ALPINE}
   before_script:
     - apk add --no-cache curl
   script:
@@ -103,7 +116,7 @@ trigger_master:
       - $PIPELINE_TRIGGERS_MASTER
 trigger_latest:
   stage: trigger
-  image: alpine:3.9
+  image: ${GITLAB_CI_IMAGE_ALPINE}
   before_script:
     - apk add --no-cache curl
   script:
@@ -113,3 +126,13 @@ trigger_latest:
       - tags
     variables:
       - $PIPELINE_TRIGGERS_LATEST
+pages:
+  stage: deploy
+  image: ${GITLAB_CI_IMAGE_ALPINE}
+  script:
+    - mv docs/_build/html/ public/
+  artifacts:
+    paths:
+      - public
+  only:
+    - tags

.pylintrc

@@ -0,0 +1,12 @@
+[MASTER]
+ignore=_version.py,migrations
+load-plugins=pylint_django
+ignored-argument-names=args|kwargs
+
+[MESSAGES CONTROL]
+disable=
+    E1121, # too-many-function-args
+    R0801, # needs to remain disabled see https://github.com/PyCQA/pylint/issues/214
+    R0912, # too-many-branches
+    R0913, # too-many-arguments
+    R0914, # too-many-locals

.yapfignore

@@ -0,0 +1,7 @@
+node_modules
+network/_version.py
+network/*/migrations
+.tox
+build
+docs
+versioneer.py

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2
+FROM python:3
 MAINTAINER SatNOGS project <dev@satnogs.org>
 
 WORKDIR /workdir/

README.md

@@ -1,3 +1,81 @@
+# Fork
+This is a fork of the SatNOGS Network. It is available here:
+
+* http://network.satnogs.spacecruft.org
+
+
+# Install SatNOGS-DB
+Set up for Debian Buster.
+
+## Install docs
+
+* https://docs.satnogs.org/projects/satnogs-network/en/stable/installation-virtualenv.html
+
+## Install dependencies
+```
+apt update
+apt install -t buster-backports apache2
+apt install python3 virtualenvwrapper python3-pip git python3-certbot-apache npm default-libmysqlclient-dev libmariadbclient-dev libjpeg-dev sass redis-server
+# Set up SSL
+certbot
+```
+
+## Install Virtual Environment
+Set up `virtualenvs`  (like this? Not in docs).
+
+```
+mkdir $HOME/.virtualenvs
+echo "export WORKON_HOME=$HOME/.virtualenvs" >> $HOME/.bashrc
+echo "source /usr/share/virtualenvwrapper/virtualenvwrapper.sh" >> $HOME/.bashrc
+source ~/.bashrc
+# Make sure it looks ok:
+workon
+```
+
+## Install satnogs-network from git
+```
+git clone https://spacecruft.org/spacecruft/satnogs-network.git
+cd satnogs-network
+mkvirtualenv --python=/usr/bin/python3 satnogs-network -a .
+cp env-dist .env
+# Set values in .env thusly:
+ALLOWED_HOSTS=network.satnogs.spacecruft.org
+# and SECRET_KEY to a long random string
+vim .env
+npm install
+./node_modules/.bin/gulp
+```
+
+## Run satnogs-network
+Note, you may have to change the listen IP and port:
+
+```
+vim bin/djangoctl.sh
+# Change:
+exec "$MANAGE_CMD" runserver 0.0.0.0:8000
+# To:
+exec "$MANAGE_CMD" runserver 127.0.0.1:8100
+```
+
+Then start the thing:
+```
+cd satnogs-network # if you aren't there already
+workon satnogs-network
+./bin/djangoctl.sh develop .
+```
+
+## Populate satnogs-network
+While `./bin/djangoctl.sh develop .` is running above, in another terminal
+run:
+
+```
+workon satnogs-network
+./bin/djangoctl.sh initialize
+```
+
+# Upstream
+See below for upstream README.
+
 # SatNOGS Network
 
 SatNOGS Network is a web application, implementing a global scheduling and monitoring network for ground station operations.
@@ -6,7 +84,7 @@ It features multiple observers to multiple intrumentation functionality and mana
 
 ## Contribute
 
-Check out the [documentation](https://docs.satnogs.org/en/latest/satnogs-network/docs/) on how to setup a local development instance.
+Check out the [documentation](https://docs.satnogs.org/projects/satnogs-network/en/stable/) on how to setup a local development instance.
 
 The main repository lives on [Gitlab](https://gitlab.com/librespacefoundation/satnogs/satnogs-network) and all Merge Request should happen there.
 

auth0login/admin.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+"""SatNOGS Network Auth0 login module admin class"""
 from __future__ import unicode_literals
 
 # from django.contrib import admin

auth0login/apps.py

@@ -1,8 +1,10 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+"""SatNOGS Network Auth0 login app config"""
+from __future__ import absolute_import, unicode_literals
 
 from django.apps import AppConfig
 
 
 class Auth0LoginConfig(AppConfig):
+    """Set the name of the django app for auth0login"""
     name = 'auth0login'

auth0login/auth0backend.py

@@ -1,3 +1,6 @@
+"""SatNOGS Network Auth0 login module auth backend"""
+from __future__ import absolute_import
+
 import requests
 from social_core.backends.oauth import BaseOAuth2
 
@@ -7,9 +10,7 @@ class Auth0(BaseOAuth2):
     name = 'auth0'
     SCOPE_SEPARATOR = ' '
     ACCESS_TOKEN_METHOD = 'POST'
-    EXTRA_DATA = [
-        ('email', 'email')
-    ]
+    EXTRA_DATA = [('email', 'email')]
 
     def authorization_url(self):
         """Return the authorization endpoint."""
@@ -19,6 +20,10 @@ class Auth0(BaseOAuth2):
         """Return the token endpoint."""
         return "https://" + self.setting('DOMAIN') + "/oauth/token"
 
+    def auth_html(self):
+        """Return the login endpoint."""
+        return "https://" + self.setting('DOMAIN') + "/login/auth0"
+
     def get_user_id(self, details, response):
         """Return current user id."""
         return details['user_id']
@@ -29,7 +34,9 @@ class Auth0(BaseOAuth2):
         resp = requests.get(url, headers=headers)
         userinfo = resp.json()
 
-        return {'username': userinfo['nickname'],
-                'email': userinfo['email'],
-                # 'first_name': userinfo['name'],
-                'user_id': userinfo['sub']}
+        return {
+            'username': userinfo['nickname'],
+            'email': userinfo['email'],
+            # 'first_name': userinfo['name'],
+            'user_id': userinfo['sub']
+        }

auth0login/models.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+"""SatNOGS Network Auth0 login module models"""
 from __future__ import unicode_literals
 
 # from django.db import models

auth0login/tests.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+"""SatNOGS Network Auth0 login module test suites"""
 from __future__ import unicode_literals
 
 # from django.test import TestCase

auth0login/urls.py

@@ -1,4 +1,8 @@
-from django.conf.urls import url, include
+"""SatNOGS Network Auth0 login module URL routers"""
+from __future__ import absolute_import
+
+from django.conf.urls import include, url
+
 from . import views
 
 urlpatterns = [

auth0login/views.py

@@ -1,7 +1,9 @@
-from __future__ import unicode_literals
+"""SatNOGS Network Auth0 login module views"""
+from __future__ import absolute_import, unicode_literals
 
 from django.shortcuts import render
 
 
 def index(request):
+    """Returns the index view"""
     return render(request, 'index.html')

bin/djangoctl.sh

@@ -86,7 +86,7 @@ develop() {
 		install_editable "$1"
 	fi
 	prepare
-	exec "$MANAGE_CMD" runserver 0.0.0.0:8000
+	exec "$MANAGE_CMD" runserver 127.0.0.1:8100
 }
 
 develop_celery() {

docker-compose.yml

@@ -1,18 +1,26 @@
 version: '3.2'
 services:
   db:
-    image: 'mariadb:10.4.1'
+    image: 'mariadb:10.4.11'
     environment:
       MYSQL_DATABASE: 'satnogsnetwork'
       MYSQL_USER: 'satnogsnetwork'
       MYSQL_PASSWORD: 'satnogsnetwork'
       MYSQL_ROOT_PASSWORD: 'toor'
+    volumes:
+      - type: 'volume'
+        source: 'db'
+        target: '/var/lib/mysql'
   redis:
-    image: 'redis:3.2.12'
+    image: 'redis:5.0.7'
+    volumes:
+      - type: 'volume'
+        source: 'redis'
+        target: '/data'
   celery:
     build:
       context: '.'
-    links:
+    depends_on:
       - 'db'
       - 'redis'
     environment:
@@ -27,14 +35,16 @@ services:
     command: ["djangoctl.sh", "develop_celery", "/usr/local/src/satnogs-network"]
     volumes:
       - '.:/usr/local/src/satnogs-network:z'
-      - 'media:/var/lib/satnogs-network/media'
+      - type: 'volume'
+        source: 'media'
+        target: '/var/lib/satnogs-network/media'
   web:
     build:
       context: '.'
     image: 'satnogs-network'
     ports:
       - '8000:8000'
-    links:
+    depends_on:
       - 'db'
       - 'redis'
     environment:
@@ -50,8 +60,14 @@ services:
     command: ["djangoctl.sh", "develop", "/usr/local/src/satnogs-network"]
     volumes:
       - '.:/usr/local/src/satnogs-network:z'
-      - 'static:/var/lib/satnogs-network/static'
-      - 'media:/var/lib/satnogs-network/media'
+      - type: 'volume'
+        source: 'static'
+        target: '/var/lib/satnogs-network/static'
+      - type: 'volume'
+        source: 'media'
+        target: '/var/lib/satnogs-network/media'
 volumes:
+  db:
+  redis:
   static:
   media:

docs/developer-guide.rst

@@ -6,6 +6,7 @@ There are always bugs to file; bugs to fix in code; improvements to be made to t
 
 The below instructions are for software developers who want to work on `satnogs-network code <http://gitlab.com/librespacefoundation/satnogs/satnogs-network>`_.
 
+
 Workflow
 --------
 
@@ -31,6 +32,7 @@ If you're asked to change your commit message or code, you can amend or rebase a
 
 If you need more Git expertise, a good resource is the `Git book <http://git-scm.com/book>`_.
 
+
 Templates
 ---------
 
@@ -40,23 +42,34 @@ satnogs-network uses `Django's template engine <https://docs.djangoproject.com/e
 Frontend development
 --------------------
 
-To be able to manage the required javascript libraries, install the development dependencies with npm::
+Third-party static assets are not included in this repository.
+The frontend dependencies are managed with ``npm``.
+Development tasks like the copying of assets, code linting and tests are managed with ``gulp``.
 
-  $ npm install
+To download third-party static assets:
 
-Development tasks like the download of assets, code linting and tests are managed with gulp::
+#. Install dependencies with ``npm``::
 
-  $ gulp
+     $ npm install
 
-Frontend dependencies are stored in packages.json, handled by yarn. To add a new dependency, e.g. satellite.js, call::
+#. Test and copy the newly downlodaded static assets::
 
-  $ yarn add satellite.js
+     $ ./node_modules/.bin/gulp
 
-Manually add the new required files to the list of "assets" in packages.json, then start the download with::
+To add new or remove existing third-party static assets:
 
-  $ gulp assets
+#. Install a new dependency::
+
+     $ npm install <package>
+
+#. Uninstall an existing dependency::
+
+     $ npm uninstall <package>
+
+#. Copy the newly downlodaded static assets::
+
+     $ ./node_modules/.bin/gulp assets
 
-The assets are stored in the repository, thus don't forget to create a commit after you add/update/remove dependencies.
 
 Simulating station heartbeats
 -----------------------------
@@ -64,13 +77,36 @@ Simulating station heartbeats
 Only stations which have been seen by the server in the last hour (by default, can be customized by `STATION_HEARTBEAT_TIME`) are taken into consideration when scheduling observations.
 In order to simulate an heartbeat of the stations 7, 23 and 42, the following command can be used::
 
-  $ sudo docker-compose run web python manage.py update_station_last_seen 7 23 42
+  $ docker-compose exec web django-admin update_station_last_seen 7 23 42
+
+
+Manually run a celery tasks
+---------------------------
+
+The following procedure can be used to manually run celery tasks in the local (docker-based) development environment:
+
+- Setup local dev env (docker).
+- Start django shell
+  ```
+  docker-compose exec web django-admin shell
+  ```
+- Run an asnyc task and check if it succeeded.
+  ```
+  > from network.base.tasks import update_all_tle
+  > task = update_all_tle.delay()
+  > assert(task.ready())
+  ```
+- (optional) Check the celery log for the task output:
+  ```
+  docker-compose logs celery
+  ```
 
 Coding Style
 ------------
 
 Follow the `PEP8 <http://www.python.org/dev/peps/pep-0008/>`_ and `PEP257 <http://www.python.org/dev/peps/pep-0257/#multi-line-docstrings>`_ Style Guides.
 
+
 What to work on
 ---------------
 You can check `open issues <https://gitlab.com/librespacefoundation/satnogs/satnogs-network/issues>`_.

docs/installation-docker.rst

@@ -19,6 +19,16 @@ Docker Installation
 
      $ cp env-dist .env
 
+#. **Install frontend dependencies**
+
+   Install dependencies with ``npm``::
+
+     $ npm install
+
+   Test and copy the newly downlodaded static assets::
+
+     $ ./node_modules/.bin/gulp
+
 #. **Run it!**
 
    Run satnogs-network::
@@ -41,4 +51,4 @@ Docker Installation
 
 #. **Build the documentation locally**
 
-     $ docker run -it -v ${PWD}:/documents/ plaindocs/docker-sphinx make html
+     $ tox -e docs

docs/installation-virtualenv.rst

@@ -24,6 +24,16 @@ VirtualEnv Installation
 
      $ cp env-dist .env
 
+#. **Install frontend dependencies**
+
+   Install dependencies with ``npm``::
+
+     $ npm install
+
+   Test and copy the newly downlodaded static assets::
+
+     $ ./node_modules/.bin/gulp
+
 #. **Run it!**
 
    Activate your python virtual environment::

docs/maintenance.rst

@@ -4,28 +4,28 @@ Maintenance
 
 Updating Python dependencies
 ----------------------------
+
 To update the Python dependencies:
 
-#. Execute script to refresh `requirements{-dev}.txt` files:
+#. Execute script to refresh ``requirements{-dev}.txt`` files::
 
-    $ ./contrib/refresh-requirements.sh
+     $ ./contrib/refresh-requirements.sh
 
-#. Stage and commit `requirements{-dev}.txt` files
+#. Stage and commit ``requirements{-dev}.txt`` files.
 
 
 Updating frontend dependencies
 ------------------------------
-The frontend dependencies are managed with `npm` as defined in the `package.json`.
-The following are required to perform an update of the dependencies:
 
-#. Bump versions in `package.json`
+The frontend dependencies are managed with ``npm``.
+To update the frontend dependencies, while respecting semver:
 
-#. Download and install the latest version of the dependencies
+#. Update all the packages listed in ``package.json``::
 
-    $ npm install
+     $ npm update
 
-#. Move the installed version into to satnogs-network source tree
+#. Test and copy the newly downlodaded static assets::
 
-    $ ./node_modules/.bin/gulp
+     $ ./node_modules/.bin/gulp
 
-#. Stage & commit the updated files in `network/static/`.
+#. Stage and commit ``package-lock.json`` file.

gulpfile.js

@@ -1,46 +1,46 @@
 /* global require */
 
-var gulp = require('gulp');
-var eslint = require('gulp-eslint');
-var stylelint = require('gulp-stylelint');
+const gulp = require('gulp');
 
-var lintPathsJS = [
+const lintPathsJS = [
     'network/static/js/*.js',
     'gulpfile.js'
 ];
 
-var lintPathsCSS = [
+const lintPathsCSS = [
     'network/static/css/*.scss',
+    'network/static/css/common/*.scss',
+    'network/static/css/pages/*.scss',
+    'network/static/css/partials/*.scss',
     'network/static/css/*.css'
 ];
 
-gulp.task('js:lint', () => {
+gulp.task('js:lint', function() {
+    const eslint = require('gulp-eslint');
+
     return gulp.src(lintPathsJS)
         .pipe(eslint())
         .pipe(eslint.format())
         .pipe(eslint.failAfterError());
 });
 
-gulp.task('css:lint', () => {
+gulp.task('css:lint', function() {
+    const stylelint = require('gulp-stylelint');
+
     return gulp.src(lintPathsCSS)
         .pipe(stylelint({
             reporters: [{ formatter: 'string', console: true}]
         }));
 });
 
-gulp.task('assets', function(){
-    var p = require('./package.json');
-    var assets = p.assets;
+gulp.task('assets', function() {
+    const p = require('./package.json');
+    const assets = p.assets;
+
     return gulp.src(assets, {cwd : 'node_modules/**'})
         .pipe(gulp.dest('network/static/lib'));
 });
 
-gulp.task('test', () => {
-    gulp.start('js:lint');
-    gulp.start('css:lint');
-});
+gulp.task('test', gulp.parallel('js:lint', 'css:lint'));
 
-gulp.task('default', function() {
-    gulp.start('assets');
-    gulp.start('test');
-});
+gulp.task('default', gulp.series('assets', 'test'));

manage.py

@@ -2,7 +2,6 @@
 import os
 import sys
 
-
 if __name__ == "__main__":
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'network.settings')
     from django.core.management import execute_from_command_line

network/__init__.py

@@ -1,10 +1,10 @@
+"""The core django app for SatNOGS Network"""
 from __future__ import absolute_import
 
-from .celery import app as celery_app  # noqa
-
+from ._version import get_versions
+from .celery import APP as celery_app  # noqa
 
 __all__ = ['celery_app']
 
-from ._version import get_versions
 __version__ = get_versions()['version']
 del get_versions

network/api/filters.py

@@ -1,26 +1,34 @@
-from django_filters.rest_framework import FilterSet
+"""SatNOGS Network django rest framework Filters class"""
+from __future__ import absolute_import
+
 import django_filters
+from django_filters.rest_framework import FilterSet
 
 from network.base.models import Observation, Station, Transmitter
 
 
 class ObservationViewFilter(FilterSet):
+    """SatNOGS Network Observation API View Filter"""
     start = django_filters.IsoDateTimeFilter(name='start', lookup_expr='gte')
     end = django_filters.IsoDateTimeFilter(name='end', lookup_expr='lte')
 
     class Meta:
         model = Observation
-        fields = ['id', 'ground_station', 'satellite__norad_cat_id', 'transmitter_uuid',
-                  'transmitter_mode', 'transmitter_type', 'vetted_status', 'vetted_user']
+        fields = [
+            'id', 'ground_station', 'satellite__norad_cat_id', 'transmitter_uuid',
+            'transmitter_mode', 'transmitter_type', 'vetted_status', 'vetted_user'
+        ]
 
 
 class StationViewFilter(FilterSet):
+    """SatNOGS Network Station API View Filter"""
     class Meta:
         model = Station
         fields = ['id', 'name', 'status', 'client_version']
 
 
 class TransmitterViewFilter(FilterSet):
+    """SatNOGS Network Transmitter API View Filter"""
     class Meta:
         model = Transmitter
         fields = ['uuid', 'sync_to_db']

network/api/pagination.py

@@ -1,6 +1,7 @@
 """
 Custom pagination classes for REST framework
 """
+from __future__ import absolute_import
 
 from rest_framework.pagination import PageNumberPagination
 from rest_framework.response import Response

network/api/perms.py

@@ -1,5 +1,10 @@
+"""SatNOGS Network API permissions, django rest framework"""
+from __future__ import absolute_import
+
 from rest_framework import permissions
 
+from network.base.perms import schedule_perms
+
 
 class SafeMethodsOnlyPermission(permissions.BasePermission):
     """Anyone can access non-destructive methods (like GET and HEAD)"""
@@ -10,12 +15,11 @@ class SafeMethodsOnlyPermission(permissions.BasePermission):
         return request.method in permissions.SAFE_METHODS
 
 
-class StationOwnerCanEditPermission(permissions.BasePermission):
+class StationOwnerPermission(permissions.BasePermission):
     """Only the owner can edit station jobs"""
     def has_object_permission(self, request, view, obj):
         if request.method in permissions.SAFE_METHODS:
             return True
-        if request.user.is_authenticated() and request.user == obj.ground_station.owner:
+        if request.method == 'POST' and schedule_perms(request.user):
             return True
-        else:
-            return False
+        return request.user.is_authenticated() and request.user == obj.ground_station.owner

network/api/serializers.py

@@ -1,18 +1,35 @@
+"""SatNOGS Network API serializers, django rest framework"""
+#  pylint: disable=no-self-use
+from __future__ import absolute_import, division
+
+from collections import defaultdict
+
+from future.builtins import round
 from rest_framework import serializers
 
-from network.base.models import Observation, Station, DemodData, Antenna, Transmitter
+from network.base.db_api import DBConnectionError, get_transmitters_by_uuid_set
+from network.base.models import Antenna, DemodData, Observation, Station, \
+    Transmitter
+from network.base.perms import UserNoPermissionError, \
+    check_schedule_perms_per_station
+from network.base.scheduling import create_new_observation
 from network.base.stats import transmitter_stats_by_uuid
+from network.base.validators import ObservationOverlapError, OutOfRangeError, \
+    check_end_datetime, check_overlaps, check_start_datetime, \
+    check_start_end_datetimes, check_transmitter_station_pairs
 
 
 class DemodDataSerializer(serializers.ModelSerializer):
+    """SatNOGS Network DemodData API Serializer"""
     class Meta:
         model = DemodData
         fields = ('payload_demod', )
 
 
 class ObservationSerializer(serializers.ModelSerializer):
+    """SatNOGS Network Observation API Serializer"""
     transmitter = serializers.SerializerMethodField()
-    transmitter_updated = serializers.DateTimeField(source='transmitter_created')
+    transmitter_updated = serializers.SerializerMethodField()
     norad_cat_id = serializers.SerializerMethodField()
     station_name = serializers.SerializerMethodField()
     station_lat = serializers.SerializerMethodField()
@@ -22,75 +39,258 @@ class ObservationSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Observation
-        fields = ('id', 'start', 'end', 'ground_station', 'transmitter', 'norad_cat_id',
-                  'payload', 'waterfall', 'demoddata', 'station_name', 'station_lat',
-                  'station_lng', 'station_alt', 'vetted_status', 'archived', 'archive_url',
-                  'client_version', 'client_metadata', 'vetted_user', 'vetted_datetime',
-                  'rise_azimuth', 'set_azimuth', 'max_altitude', 'transmitter_uuid',
-                  'transmitter_description', 'transmitter_type', 'transmitter_uplink_low',
-                  'transmitter_uplink_high', 'transmitter_uplink_drift',
-                  'transmitter_downlink_low', 'transmitter_downlink_high',
-                  'transmitter_downlink_drift', 'transmitter_mode', 'transmitter_invert',
-                  'transmitter_baud', 'transmitter_updated', 'tle')
-        read_only_fields = ['id', 'start', 'end', 'observation', 'ground_station',
-                            'transmitter', 'norad_cat_id', 'archived', 'archive_url',
-                            'station_name', 'station_lat', 'station_lng', 'vetted_user',
-                            'station_alt', 'vetted_status', 'vetted_datetime', 'rise_azimuth',
-                            'set_azimuth', 'max_altitude', 'transmitter_uuid',
-                            'transmitter_description', 'transmitter_type',
-                            'transmitter_uplink_low', 'transmitter_uplink_high',
-                            'transmitter_uplink_drift', 'transmitter_downlink_low',
-                            'transmitter_downlink_high', 'transmitter_downlink_drift',
-                            'transmitter_mode', 'transmitter_invert', 'transmitter_baud',
-                            'transmitter_created', 'transmitter_updated', 'tle']
+        fields = (
+            'id', 'start', 'end', 'ground_station', 'transmitter', 'norad_cat_id', 'payload',
+            'waterfall', 'demoddata', 'station_name', 'station_lat', 'station_lng', 'station_alt',
+            'vetted_status', 'archived', 'archive_url', 'client_version', 'client_metadata',
+            'vetted_user', 'vetted_datetime', 'rise_azimuth', 'set_azimuth', 'max_altitude',
+            'transmitter_uuid', 'transmitter_description', 'transmitter_type',
+            'transmitter_uplink_low', 'transmitter_uplink_high', 'transmitter_uplink_drift',
+            'transmitter_downlink_low', 'transmitter_downlink_high', 'transmitter_downlink_drift',
+            'transmitter_mode', 'transmitter_invert', 'transmitter_baud', 'transmitter_updated',
+            'tle'
+        )
+        read_only_fields = [
+            'id', 'start', 'end', 'observation', 'ground_station', 'transmitter', 'norad_cat_id',
+            'archived', 'archive_url', 'station_name', 'station_lat', 'station_lng', 'vetted_user',
+            'station_alt', 'vetted_status', 'vetted_datetime', 'rise_azimuth', 'set_azimuth',
+            'max_altitude', 'transmitter_uuid', 'transmitter_description', 'transmitter_type',
+            'transmitter_uplink_low', 'transmitter_uplink_high', 'transmitter_uplink_drift',
+            'transmitter_downlink_low', 'transmitter_downlink_high', 'transmitter_downlink_drift',
+            'transmitter_mode', 'transmitter_invert', 'transmitter_baud', 'transmitter_created',
+            'transmitter_updated', 'tle'
+        ]
 
     def update(self, instance, validated_data):
+        """Updates observation object with validated data"""
         validated_data.pop('demoddata')
         super(ObservationSerializer, self).update(instance, validated_data)
         return instance
 
     def get_transmitter(self, obj):
+        """Returns Transmitter UUID"""
         try:
             return obj.transmitter_uuid
         except AttributeError:
             return ''
 
+    def get_transmitter_updated(self, obj):
+        """Returns Transmitter last update date"""
+        try:
+            return obj.transmitter_created
+        except AttributeError:
+            return ''
+
     def get_norad_cat_id(self, obj):
+        """Returns Satellite NORAD ID"""
         return obj.satellite.norad_cat_id
 
     def get_station_name(self, obj):
+        """Returns Station name"""
         try:
             return obj.ground_station.name
         except AttributeError:
             return None
 
     def get_station_lat(self, obj):
+        """Returns Station latitude"""
         try:
             return obj.ground_station.lat
         except AttributeError:
             return None
 
     def get_station_lng(self, obj):
+        """Returns Station longitude"""
         try:
             return obj.ground_station.lng
         except AttributeError:
             return None
 
     def get_station_alt(self, obj):
+        """Returns Station elevation"""
         try:
             return obj.ground_station.alt
         except AttributeError:
             return None
 
 
-class AntennaSerializer(serializers.ModelSerializer):
+class NewObservationListSerializer(serializers.ListSerializer):
+    """SatNOGS Network New Observation API List Serializer"""
+    transmitters = {}
 
+    def validate(self, attrs):
+        """Validates data from a list of new observations"""
+        station_set = set()
+        transmitter_uuid_set = set()
+        transmitter_uuid_station_set = set()
+        start_end_per_station = defaultdict(list)
+
+        for observation in attrs:
+            station = observation.get('ground_station')
+            transmitter_uuid = observation.get('transmitter_uuid')
+
+            station_set.add(station)
+            transmitter_uuid_set.add(transmitter_uuid)
+            transmitter_uuid_station_set.add((transmitter_uuid, station))
+            start_end_per_station[int(station.id)].append(
+                (observation.get('start'), observation.get('end'))
+            )
+
+        try:
+            check_overlaps(start_end_per_station)
+        except ObservationOverlapError as error:
+            raise serializers.ValidationError(error, code='invalid')
+
+        try:
+            check_schedule_perms_per_station(self.context['request'].user, station_set)
+        except UserNoPermissionError as error:
+            raise serializers.ValidationError(error, code='forbidden')
+
+        try:
+            transmitters = get_transmitters_by_uuid_set(transmitter_uuid_set)
+            self.transmitters = transmitters
+        except ValueError as error:
+            raise serializers.ValidationError(error, code='invalid')
+        except DBConnectionError as error:
+            raise serializers.ValidationError(error)
+
+        transmitter_station_list = [
+            (transmitters[pair[0]], pair[1]) for pair in transmitter_uuid_station_set
+        ]
+        try:
+            check_transmitter_station_pairs(transmitter_station_list)
+        except OutOfRangeError as error:
+            raise serializers.ValidationError(error, code='invalid')
+        return attrs
+
+    def create(self, validated_data):
+        """Creates new observations from a list of new observations validated data"""
+        new_observations = []
+        for observation_data in validated_data:
+
+            transmitter_uuid = observation_data['transmitter_uuid']
+            transmitter = self.transmitters[transmitter_uuid]
+
+            observation = create_new_observation(
+                station=observation_data['ground_station'],
+                transmitter=transmitter,
+                start=observation_data['start'],
+                end=observation_data['end'],
+                author=self.context['request'].user
+            )
+            new_observations.append(observation)
+
+        for observation in new_observations:
+            observation.save()
+
+        return new_observations
+
+    def update(self, instance, validated_data):
+        """Updates observations from a list of validated data
+
+        currently disabled and returns None
+        """
+        return None
+
+
+class NewObservationSerializer(serializers.Serializer):
+    """SatNOGS Network New Observation API Serializer"""
+    start = serializers.DateTimeField(
+        input_formats=['%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d %H:%M:%S'],
+        error_messages={
+            'invalid':
+            'Start datetime should have either \'%Y-%m-%d %H:%M:%S.%f\' or \'%Y-%m-%d %H:%M:%S\' '
+            'format.',
+            'required':
+            'Start(\'start\' key) datetime is required.'
+        }
+    )
+    end = serializers.DateTimeField(
+        input_formats=['%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d %H:%M:%S'],
+        error_messages={
+            'invalid':
+            'End datetime should have either \'%Y-%m-%d %H:%M:%S.%f\' or \'%Y-%m-%d %H:%M:%S\' '
+            'format.',
+            'required':
+            'End datetime(\'end\' key) is required.'
+        }
+    )
+    ground_station = serializers.PrimaryKeyRelatedField(
+        queryset=Station.objects.filter(status__gt=0),
+        allow_null=False,
+        error_messages={
+            'does_not_exist': 'Station should exist and be online.',
+            'required': 'Station(\'ground_station\' key) is required.'
+        }
+    )
+    transmitter_uuid = serializers.CharField(
+        max_length=22,
+        min_length=22,
+        error_messages={
+            'invalid': 'Transmitter UUID should be valid.',
+            'required': 'Transmitter UUID(\'transmitter_uuid\' key) is required.'
+        }
+    )
+
+    def validate_start(self, value):
+        """Validates start datetime of a new observation"""
+        try:
+            check_start_datetime(value)
+        except ValueError as error:
+            raise serializers.ValidationError(error, code='invalid')
+        return value
+
+    def validate_end(self, value):
+        """Validates end datetime of a new observation"""
+        try:
+            check_end_datetime(value)
+        except ValueError as error:
+            raise serializers.ValidationError(error, code='invalid')
+        return value
+
+    def validate(self, attrs):
+        """Validates combination of start and end datetimes of a new observation"""
+        start = attrs['start']
+        end = attrs['end']
+        try:
+            check_start_end_datetimes(start, end)
+        except ValueError as error:
+            raise serializers.ValidationError(error, code='invalid')
+        return attrs
+
+    def create(self, validated_data):
+        """Creates a new observation
+
+        Currently not implemented and raises exception. If in the future we want to implement this
+        serializer accepting and creating observation from single object instead from a list of
+        objects, we should remove raising the exception below and implement the validations that
+        exist now only on NewObservationListSerializer
+        """
+        raise serializers.ValidationError(
+            "Serializer is implemented for accepting and schedule\
+                                           only lists of observations"
+        )
+
+    def update(self, instance, validated_data):
+        """Updates an observation from validated data
+
+        currently disabled and returns None
+        """
+        return None
+
+    class Meta:
+        list_serializer_class = NewObservationListSerializer
+
+
+class AntennaSerializer(serializers.ModelSerializer):
+    """SatNOGS Network Antenna API Serializer"""
     class Meta:
         model = Antenna
         fields = ('frequency', 'frequency_max', 'band', 'antenna_type')
 
 
 class StationSerializer(serializers.ModelSerializer):
+    """SatNOGS Network Station API Serializer"""
     antenna = AntennaSerializer(many=True)
     altitude = serializers.SerializerMethodField()
     min_horizon = serializers.SerializerMethodField()
@@ -99,32 +299,40 @@ class StationSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Station
-        fields = ('id', 'name', 'altitude', 'min_horizon', 'lat', 'lng',
-                  'qthlocator', 'location', 'antenna', 'created', 'last_seen',
-                  'status', 'observations', 'description', 'client_version',
-                  'target_utilization')
+        fields = (
+            'id', 'name', 'altitude', 'min_horizon', 'lat', 'lng', 'qthlocator', 'location',
+            'antenna', 'created', 'last_seen', 'status', 'observations', 'description',
+            'client_version', 'target_utilization'
+        )
 
     def get_altitude(self, obj):
+        """Returns Station elevation"""
         return obj.alt
 
     def get_min_horizon(self, obj):
+        """Returns Station minimum horizon"""
         return obj.horizon
 
     def get_antenna(self, obj):
+        """Returns Station antenna list"""
         def antenna_name(antenna):
-            return (antenna.band + " " + antenna.get_antenna_type_display())
+            """Returns Station antenna"""
+            return antenna.band + " " + antenna.get_antenna_type_display()
+
         try:
             return [antenna_name(ant) for ant in obj.antenna.all()]
         except AttributeError:
             return None
 
     def get_observations(self, obj):
+        """Returns Station observations number"""
         try:
             return obj.observations_count
         except AttributeError:
             return None
 
     def get_status(self, obj):
+        """Returns Station status"""
         try:
             return obj.get_status_display()
         except AttributeError:
@@ -132,6 +340,7 @@ class StationSerializer(serializers.ModelSerializer):
 
 
 class JobSerializer(serializers.ModelSerializer):
+    """SatNOGS Network Job API Serializer"""
     frequency = serializers.SerializerMethodField()
     tle0 = serializers.SerializerMethodField()
     tle1 = serializers.SerializerMethodField()
@@ -142,40 +351,49 @@ class JobSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Observation
-        fields = ('id', 'start', 'end', 'ground_station', 'tle0', 'tle1', 'tle2',
-                  'frequency', 'mode', 'transmitter', 'baud')
+        fields = (
+            'id', 'start', 'end', 'ground_station', 'tle0', 'tle1', 'tle2', 'frequency', 'mode',
+            'transmitter', 'baud'
+        )
 
     def get_frequency(self, obj):
+        """Returns Transmitter downlink low frequency"""
         frequency = obj.transmitter_downlink_low
         frequency_drift = obj.transmitter_downlink_drift
         if frequency_drift is None:
             return frequency
-        else:
-            return int(round(frequency + ((frequency * frequency_drift) / float(pow(10, 9)))))
+        return int(round(frequency + ((frequency * frequency_drift) / 1e9)))
 
     def get_transmitter(self, obj):
+        """Returns Transmitter UUID"""
         return obj.transmitter_uuid
 
     def get_tle0(self, obj):
+        """Returns line 0 of TLE"""
         return obj.tle.tle0
 
     def get_tle1(self, obj):
+        """Returns line 1 of TLE"""
         return obj.tle.tle1
 
     def get_tle2(self, obj):
+        """Returns line 2 of TLE"""
         return obj.tle.tle2
 
     def get_mode(self, obj):
+        """Returns Transmitter mode"""
         try:
             return obj.transmitter_mode
         except AttributeError:
             return ''
 
     def get_baud(self, obj):
+        """Returns Transmitter baudrate"""
         return obj.transmitter_baud
 
 
 class TransmitterSerializer(serializers.ModelSerializer):
+    """SatNOGS Network Transmitter API Serializer"""
     stats = serializers.SerializerMethodField()
 
     class Meta:
@@ -183,4 +401,8 @@ class TransmitterSerializer(serializers.ModelSerializer):
         fields = ('uuid', 'sync_to_db', 'stats')
 
     def get_stats(self, obj):
-        return transmitter_stats_by_uuid(obj.uuid)
+        """Returns Transmitter statistics"""
+        stats = transmitter_stats_by_uuid(obj.uuid)
+        for statistic in stats:
+            stats[statistic] = int(stats[statistic])
+        return stats

network/api/tests.py

@@ -1,16 +1,14 @@
+"""SatNOGS Network API test suites"""
+from __future__ import absolute_import
+
 import json
-import pytest
 
 from django.test import TestCase
-
 from rest_framework.utils.encoders import JSONEncoder
 
-from network.base.tests import (
-    ObservationFactory,
-    SatelliteFactory,
-    StationFactory,
-    AntennaFactory
-)
+import pytest
+from network.base.tests import AntennaFactory, ObservationFactory, \
+    SatelliteFactory, StationFactory
 
 
 @pytest.mark.django_db(transaction=True)
@@ -23,13 +21,14 @@ class JobViewApiTest(TestCase):
     stations = []
 
     def setUp(self):
-        for x in xrange(1, 10):
+        for _ in range(1, 10):
             self.satellites.append(SatelliteFactory())
-        for x in xrange(1, 10):
+        for _ in range(1, 10):
             self.stations.append(StationFactory())
         self.observation = ObservationFactory()
 
     def test_job_view_api(self):
+        """Test the Job View API"""
         response = self.client.get('/api/jobs/')
         response_json = json.loads(response.content)
         self.assertEqual(response_json, [])
@@ -48,12 +47,17 @@ class StationViewApiTest(TestCase):
         self.station = StationFactory.create(antennas=[self.antenna])
 
     def test_station_view_api(self):
+        """Test the Station View API"""
 
         ants = self.station.antenna.all()
-        ser_ants = [{u'band': ant.band,
-                     u'frequency': ant.frequency,
-                     u'frequency_max': ant.frequency_max,
-                     u'antenna_type': ant.antenna_type} for ant in ants]
+        ser_ants = [
+            {
+                u'band': ant.band,
+                u'frequency': ant.frequency,
+                u'frequency_max': ant.frequency_max,
+                u'antenna_type': ant.antenna_type
+            } for ant in ants
+        ]
 
         station_serialized = {
             u'altitude': self.station.alt,
@@ -71,7 +75,8 @@ class StationViewApiTest(TestCase):
             u'observations': 0,
             u'qthlocator': self.station.qthlocator,
             u'target_utilization': self.station.target_utilization,
-            u'status': self.station.get_status_display()}
+            u'status': self.station.get_status_display()
+        }
 
         response = self.client.get('/api/stations/')
         response_json = json.loads(response.content)

network/api/urls.py

@@ -1,13 +1,16 @@
+"""SatNOGS Network django rest framework API url routings"""
+from __future__ import absolute_import
+
 from rest_framework import routers
+
 from network.api import views
 
+ROUTER = routers.DefaultRouter()
 
-router = routers.DefaultRouter()
+ROUTER.register(r'jobs', views.JobView, base_name='jobs')
+ROUTER.register(r'data', views.ObservationView, base_name='data')
+ROUTER.register(r'observations', views.ObservationView, base_name='observations')
+ROUTER.register(r'stations', views.StationView, base_name='stations')
+ROUTER.register(r'transmitters', views.TransmitterView, base_name='transmitters')
 
-router.register(r'jobs', views.JobView, base_name='jobs')
-router.register(r'data', views.ObservationView, base_name='data')
-router.register(r'observations', views.ObservationView, base_name='observations')
-router.register(r'stations', views.StationView, base_name='stations')
-router.register(r'transmitters', views.TransmitterView, base_name='transmitters')
-
-api_urlpatterns = router.urls
+API_URLPATTERNS = ROUTER.urls

network/api/views.py

@@ -1,26 +1,60 @@
+"""SatNOGS Network API django rest framework Views"""
+from __future__ import absolute_import
+
 from django.core.exceptions import ObjectDoesNotExist
 from django.shortcuts import get_object_or_404
 from django.utils.timezone import now
-
-from rest_framework import viewsets, mixins, status
+from requests.exceptions import RequestException
+from rest_framework import mixins, status, viewsets
 from rest_framework.response import Response
+from rest_framework.serializers import ValidationError
 
-from network.api.perms import StationOwnerCanEditPermission
-from network.api import serializers, filters, pagination
-from network.base.models import Observation, Station, Transmitter
+from network.api import filters, pagination, serializers
+from network.api.perms import StationOwnerPermission
+from network.base.models import LatestTle, Observation, Station, Transmitter
+from network.base.utils import sync_demoddata_to_db
+from network.base.validators import NegativeElevationError, \
+    ObservationOverlapError, SinglePassError
 
 
-class ObservationView(mixins.ListModelMixin, mixins.RetrieveModelMixin,
-                      mixins.UpdateModelMixin, viewsets.GenericViewSet):
-    queryset = Observation.objects.all()
-    serializer_class = serializers.ObservationSerializer
+class ObservationView(  # pylint: disable=R0901
+        mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.UpdateModelMixin,
+        mixins.CreateModelMixin, viewsets.GenericViewSet):
+    """SatNOGS Network Observation API view class"""
+    queryset = Observation.objects.prefetch_related('satellite', 'demoddata', 'ground_station')
     filter_class = filters.ObservationViewFilter
-    permission_classes = [
-        StationOwnerCanEditPermission
-    ]
+    permission_classes = [StationOwnerPermission]
     pagination_class = pagination.LinkedHeaderPageNumberPagination
 
+    def get_serializer_class(self):
+        """Returns the right serializer depending on http method that is used"""
+        if self.request.method == 'POST':
+            return serializers.NewObservationSerializer
+        return serializers.ObservationSerializer
+
+    def create(self, request, *args, **kwargs):
+        """Creates observations from a list of observation data"""
+        serializer = self.get_serializer(data=request.data, many=True, allow_empty=False)
+        try:
+            if serializer.is_valid():
+                observations = serializer.save()
+                serialized_obs = serializers.ObservationSerializer(observations, many=True)
+                data = serialized_obs.data
+                response = Response(data, status=status.HTTP_200_OK)
+            else:
+                data = serializer.errors
+                response = Response(data, status=status.HTTP_400_BAD_REQUEST)
+        except (NegativeElevationError, SinglePassError, ValidationError, ValueError) as error:
+            response = Response(str(error), status=status.HTTP_400_BAD_REQUEST)
+        except LatestTle.DoesNotExist:
+            data = 'Scheduling failed: Satellite without TLE'
+            response = Response(data, status=status.HTTP_501_NOT_IMPLEMENTED)
+        except ObservationOverlapError as error:
+            response = Response(str(error), status=status.HTTP_409_CONFLICT)
+        return response
+
     def update(self, request, *args, **kwargs):
+        """Updates observation with audio, waterfall or demoded data"""
         instance = self.get_object()
         if request.data.get('client_version'):
             instance.ground_station.client_version = request.data.get('client_version')
@@ -29,56 +63,74 @@ class ObservationView(mixins.ListModelMixin, mixins.RetrieveModelMixin,
             try:
                 file_path = 'data_obs/{0}/{1}'.format(instance.id, request.data.get('demoddata'))
                 instance.demoddata.get(payload_demod=file_path)
-                return Response(data='This data file has already been uploaded',
-                                status=status.HTTP_403_FORBIDDEN)
+                return Response(
+                    data='This data file has already been uploaded',
+                    status=status.HTTP_403_FORBIDDEN
+                )
             except ObjectDoesNotExist:
-                instance.demoddata.create(payload_demod=request.data.get('demoddata'))
+                demoddata = instance.demoddata.create(payload_demod=request.data.get('demoddata'))
+                if Transmitter.objects.get(uuid=instance.transmitter_uuid).sync_to_db:
+                    try:
+                        sync_demoddata_to_db(demoddata.id)
+                    except RequestException:
+                        # Sync to db failed, let the periodic task handle the sync to db later
+                        pass
         if request.data.get('waterfall'):
             if instance.has_waterfall:
-                return Response(data='Watefall has already been uploaded',
-                                status=status.HTTP_403_FORBIDDEN)
+                return Response(
+                    data='Watefall has already been uploaded', status=status.HTTP_403_FORBIDDEN
+                )
         if request.data.get('payload'):
             if instance.has_audio:
-                return Response(data='Audio has already been uploaded',
-                                status=status.HTTP_403_FORBIDDEN)
+                return Response(
+                    data='Audio has already been uploaded', status=status.HTTP_403_FORBIDDEN
+                )
 
-        super(ObservationView, self).update(request, *args, **kwargs)
+        # False-positive no-member (E1101) pylint error:
+        # Parent class rest_framework.mixins.UpdateModelMixin provides the 'update' method
+        super(ObservationView, self).update(request, *args, **kwargs)  # pylint: disable=E1101
         return Response(status=status.HTTP_200_OK)
 
 
-class StationView(mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
-    queryset = Station.objects.all()
+class StationView(  # pylint: disable=R0901
+        mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+    """SatNOGS Network Station API view class"""
+    queryset = Station.objects.prefetch_related('antenna', 'observations')
     serializer_class = serializers.StationSerializer
     filter_class = filters.StationViewFilter
     pagination_class = pagination.LinkedHeaderPageNumberPagination
 
 
-class TransmitterView(mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+class TransmitterView(  # pylint: disable=R0901
+        mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+    """SatNOGS Network Transmitter API view class"""
     queryset = Transmitter.objects.all().order_by('uuid')
     serializer_class = serializers.TransmitterSerializer
     filter_class = filters.TransmitterViewFilter
     pagination_class = pagination.LinkedHeaderPageNumberPagination
 
 
-class JobView(viewsets.ReadOnlyModelViewSet):
+class JobView(viewsets.ReadOnlyModelViewSet):  # pylint: disable=R0901
+    """SatNOGS Network Job API view class"""
     queryset = Observation.objects.filter(payload='')
     serializer_class = serializers.JobSerializer
     filter_class = filters.ObservationViewFilter
     filter_fields = ('ground_station')
 
     def get_queryset(self):
-        queryset = self.queryset.filter(start__gte=now())
-        gs_id = self.request.query_params.get('ground_station', None)
-        if gs_id and self.request.user.is_authenticated():
-            gs = get_object_or_404(Station, id=gs_id)
-            if gs.owner == self.request.user:
+        """Returns queryset for Job API view"""
+        queryset = self.queryset.filter(start__gte=now()).prefetch_related('tle')
+        ground_station_id = self.request.query_params.get('ground_station', None)
+        if ground_station_id and self.request.user.is_authenticated():
+            ground_station = get_object_or_404(Station, id=ground_station_id)
+            if ground_station.owner == self.request.user:
                 lat = self.request.query_params.get('lat', None)
                 lon = self.request.query_params.get('lon', None)
                 alt = self.request.query_params.get('alt', None)
                 if not (lat is None or lon is None or alt is None):
-                    gs.lat = float(lat)
-                    gs.lng = float(lon)
-                    gs.alt = int(alt)
-                gs.last_seen = now()
-                gs.save()
+                    ground_station.lat = float(lat)
+                    ground_station.lng = float(lon)
+                    ground_station.alt = int(alt)
+                ground_station.last_seen = now()
+                ground_station.save()
         return queryset

network/base/admin.py

@@ -1,26 +1,47 @@
-from django.contrib import admin
+"""Define functions and settings for the django admin base interface"""
+from __future__ import absolute_import
 
-from network.base.models import (Antenna, Satellite, Station, Transmitter,
-                                 Observation, Tle, DemodData)
+from django.conf.urls import url
+from django.contrib import admin, messages
+from django.shortcuts import redirect
+from django.urls import reverse
+
+from network.base.models import Antenna, DemodData, Observation, Satellite, \
+    Station, Tle, Transmitter
+from network.base.tasks import sync_to_db
 from network.base.utils import export_as_csv, export_station_status
 
 
 @admin.register(Antenna)
 class AntennaAdmin(admin.ModelAdmin):
-    list_display = ('id', '__unicode__', 'antenna_count', 'station_list', )
-    list_filter = ('band', 'antenna_type', )
+    """Define Antenna view in django admin UI"""
+    list_display = (
+        'id',
+        '__str__',
+        'antenna_count',
+        'station_list',
+    )
+    list_filter = (
+        'band',
+        'antenna_type',
+    )
 
-    def antenna_count(self, obj):
+    def antenna_count(self, obj):  # pylint: disable=no-self-use
+        """Return the number of antennas"""
         return obj.stations.all().count()
 
-    def station_list(self, obj):
+    def station_list(self, obj):  # pylint: disable=no-self-use
+        """Return stations that use the antenna"""
         return ",\n".join([str(s.id) for s in obj.stations.all()])
 
 
 @admin.register(Station)
 class StationAdmin(admin.ModelAdmin):
-    list_display = ('id', 'name', 'owner', 'get_email', 'lng', 'lat', 'qthlocator',
-                    'client_version', 'created_date', 'state', 'target_utilization')
+    """Define Station view in django admin UI"""
+    list_display = (
+        'id', 'name', 'owner', 'get_email', 'lng', 'lat', 'qthlocator', 'client_version',
+        'created_date', 'state', 'target_utilization'
+    )
     list_filter = ('status', 'created', 'client_version')
     search_fields = ('id', 'name', 'owner__username')
 
@@ -28,15 +49,19 @@ class StationAdmin(admin.ModelAdmin):
     export_as_csv.short_description = "Export selected as CSV"
     export_station_status.short_description = "Export selected status"
 
-    def created_date(self, obj):
+    def created_date(self, obj):  # pylint: disable=no-self-use
+        """Return when the station was created"""
         return obj.created.strftime('%d.%m.%Y, %H:%M')
 
-    def get_email(self, obj):
+    def get_email(self, obj):  # pylint: disable=no-self-use
+        """Return station owner email address"""
         return obj.owner.email
+
     get_email.admin_order_field = 'email'
     get_email.short_description = 'Owner Email'
 
     def get_actions(self, request):
+        """Return the list of actions for station admin view"""
         actions = super(StationAdmin, self).get_actions(request)
         if 'delete_selected' in actions:
             del actions['delete_selected']
@@ -45,39 +70,72 @@ class StationAdmin(admin.ModelAdmin):
 
 @admin.register(Satellite)
 class SatelliteAdmin(admin.ModelAdmin):
+    """Define Satellite view in django admin UI"""
     list_display = ('id', 'name', 'norad_cat_id', 'manual_tle', 'norad_follow_id', 'status')
-    list_filter = ('status', 'manual_tle',)
+    list_filter = (
+        'status',
+        'manual_tle',
+    )
     readonly_fields = ('name', 'names', 'image')
     search_fields = ('name', 'norad_cat_id', 'norad_follow_id')
 
 
 @admin.register(Tle)
 class TleAdmin(admin.ModelAdmin):
+    """Define TLE view in django admin UI"""
     list_display = ('satellite_name', 'tle0', 'tle1', 'updated')
-    list_filter = ('satellite__name',)
+    list_filter = ('satellite__name', )
 
-    def satellite_name(self, obj):
+    def satellite_name(self, obj):  # pylint: disable=no-self-use
+        """Return the satellite name"""
         return obj.satellite.name
 
 
 @admin.register(Transmitter)
 class TransmitterAdmin(admin.ModelAdmin):
+    """Define Transmitter view in django admin UI"""
     list_display = ('uuid', 'sync_to_db')
-    search_fields = ('uuid',)
-    list_filter = ('sync_to_db',)
-    readonly_fields = ('uuid', 'sync_to_db')
+    search_fields = ('uuid', )
+    list_filter = ('sync_to_db', )
+    readonly_fields = ('uuid', )
+
+    def get_urls(self):
+        """Returns django urls for the Transmitter view
+
+        sync_to_db -- url for the sync_to_db function
+
+        :returns: Django urls for the Transmitter admin view
+        """
+        urls = super(TransmitterAdmin, self).get_urls()
+        my_urls = [
+            url(r'^sync_to_db/$', self.sync_to_db, name='sync_to_db'),
+        ]
+        return my_urls + urls
+
+    def sync_to_db(self, request):  # pylint: disable=R0201
+        """Returns the admin home page, while triggering a Celery sync to DB task
+
+        Forces sync of unsynced demoddata for all transmitters that have set to be synced
+
+        :returns: admin home page redirect with popup message
+        """
+        sync_to_db.delay()
+        messages.success(request, 'Sync to DB task was triggered successfully!')
+        return redirect(reverse('admin:index'))
 
 
-class DataDemodInline(admin.TabularInline):
+class DemodDataInline(admin.TabularInline):
+    """Define DemodData inline template for use in Observation view in django admin UI"""
     model = DemodData
 
 
 @admin.register(Observation)
 class ObservationAdmin(admin.ModelAdmin):
+    """Define Observation view in django admin UI"""
     list_display = ('id', 'author', 'satellite', 'transmitter_uuid', 'start', 'end')
     list_filter = ('start', 'end')
     search_fields = ('satellite', 'author')
     inlines = [
-        DataDemodInline,
+        DemodDataInline,
     ]
-    readonly_fields = ('tle',)
+    readonly_fields = ('tle', )

network/base/context_processors.py

@@ -1,7 +1,11 @@
+"""SatNOGS Network django context processors"""
+from __future__ import absolute_import
+
 from django.conf import settings
 from django.template.loader import render_to_string
 from django.utils.timezone import now
 
+from network import __version__
 from network.base.models import Observation
 
 
@@ -9,39 +13,40 @@ def analytics(request):
     """Returns analytics code."""
     if settings.ENVIRONMENT == 'production':
         return {'analytics_code': render_to_string('includes/analytics.html')}
-    else:
-        return {'analytics_code': ''}
+    return {'analytics_code': ''}
 
 
 def stage_notice(request):
     """Displays stage notice."""
     if settings.ENVIRONMENT == 'stage':
         return {'stage_notice': render_to_string('includes/stage_notice.html')}
-    else:
-        return {'stage_notice': ''}
+    return {'stage_notice': ''}
 
 
 def user_processor(request):
+    """Returns number of user's unvetted observations."""
     if request.user.is_authenticated():
-        owner_vetting_count = Observation.objects.filter(author=request.user,
-                                                         vetted_status='unknown',
-                                                         end__lt=now()).count()
+        owner_vetting_count = Observation.objects.filter(
+            author=request.user, vetted_status='unknown', end__lt=now()
+        ).count()
         return {'owner_vetting_count': owner_vetting_count}
-    else:
-        return {'owner_vetting_count': ''}
+    return {'owner_vetting_count': ''}
 
 
 def auth_block(request):
     """Displays auth links local vs auth0."""
     if settings.AUTH0:
         return {'auth_block': render_to_string('includes/auth_auth0.html')}
-    else:
-        return {'auth_block': render_to_string('includes/auth_local.html')}
+    return {'auth_block': render_to_string('includes/auth_local.html')}
 
 
 def logout_block(request):
     """Displays logout links local vs auth0."""
     if settings.AUTH0:
         return {'logout_block': render_to_string('includes/logout_auth0.html')}
-    else:
-        return {'logout_block': render_to_string('includes/logout_local.html')}
+    return {'logout_block': render_to_string('includes/logout_local.html')}
+
+
+def version(request):
+    """Displays the current satnogs-network version."""
+    return {'version': 'Version: {}'.format(__version__)}

network/base/db_api.py

@@ -1,35 +1,71 @@
-import requests
+"""SatNOGS Network functions that consume DB API"""
+from __future__ import absolute_import
 
+import requests
 from django.conf import settings
 
-db_api_url = settings.DB_API_ENDPOINT
+DB_API_URL = settings.DB_API_ENDPOINT
+
+
+class DBConnectionError(Exception):
+    """Error when there are connection issues with DB API"""
 
 
 def transmitters_api_request(url):
-    if len(db_api_url) == 0:
-        return None
+    """Perform transmitter query on SatNOGS DB API and return the results"""
+    if not DB_API_URL:
+        raise DBConnectionError('Error in DB API connection. Blank DB API URL!')
     try:
         request = requests.get(url)
     except requests.exceptions.RequestException:
-        return None
+        raise DBConnectionError('Error in DB API connection. Please try again!')
     return request.json()
 
 
 def get_transmitter_by_uuid(uuid):
-    transmitters_url = "{}transmitters/?uuid={}".format(db_api_url, uuid)
+    """Returns transmitter filtered by Transmitter UUID"""
+    transmitters_url = "{}transmitters/?uuid={}".format(DB_API_URL, uuid)
     return transmitters_api_request(transmitters_url)
 
 
 def get_transmitters_by_norad_id(norad_id):
-    transmitters_url = "{}transmitters/?satellite__norad_cat_id={}".format(db_api_url, norad_id)
+    """Returns transmitters filtered by NORAD ID"""
+    transmitters_url = "{}transmitters/?satellite__norad_cat_id={}".format(DB_API_URL, norad_id)
     return transmitters_api_request(transmitters_url)
 
 
 def get_transmitters_by_status(status):
-    transmitters_url = "{}transmitters/?status={}".format(db_api_url, status)
+    """Returns transmitters filtered by status"""
+    transmitters_url = "{}transmitters/?status={}".format(DB_API_URL, status)
     return transmitters_api_request(transmitters_url)
 
 
 def get_transmitters():
-    transmitters_url = "{}transmitters".format(db_api_url)
+    """Returns all transmitters"""
+    transmitters_url = "{}transmitters".format(DB_API_URL)
     return transmitters_api_request(transmitters_url)
+
+
+def get_transmitters_by_uuid_set(uuid_set):
+    """Returns transmitters filtered by Transmitter UUID list"""
+    if not uuid_set:
+        raise ValueError('Expected a non empty list of UUIDs.')
+    if len(uuid_set) == 1:
+        transmitter_uuid = next(iter(uuid_set))
+        transmitter = get_transmitter_by_uuid(transmitter_uuid)
+        if not transmitter:
+            raise ValueError('Invalid Transmitter UUID: {0}'.format(str(transmitter_uuid)))
+        return {transmitter[0]['uuid']: transmitter[0]}
+
+    transmitters_list = get_transmitters()
+
+    transmitters = {t['uuid']: t for t in transmitters_list if t['uuid'] in uuid_set}
+    invalid_transmitters = [str(uuid) for uuid in uuid_set.difference(set(transmitters.keys()))]
+
+    if not invalid_transmitters:
+        return transmitters
+
+    if len(invalid_transmitters) == 1:
+        raise ValueError('Invalid Transmitter UUID: {0}'.format(invalid_transmitters[0]))
+
+    raise ValueError('Invalid Transmitter UUIDs: {0}'.format(invalid_transmitters))

network/base/decorators.py

@@ -1,22 +1,30 @@
+"""SatNOGS Network base decorators"""
+from __future__ import absolute_import
+
 from django.core.urlresolvers import reverse
 from django.http import HttpResponseBadRequest
 from django.shortcuts import redirect
 
 
 def admin_required(function):
+    """Decorator for requiring admin permission"""
     def wrap(request, *args, **kwargs):
+        """Wrap function of decorator"""
         if not request.user.is_authenticated():
             return redirect(reverse('account_login'))
         if request.user.is_superuser:
             return function(request, *args, **kwargs)
-        else:
-            return redirect(reverse('base:home'))
+        return redirect(reverse('base:home'))
+
     return wrap
 
 
 def ajax_required(function):
+    """Decorator for requiring request to be and ajax one"""
     def wrap(request, *args, **kwargs):
+        """Wrap function of decorator"""
         if not request.is_ajax():
             return HttpResponseBadRequest()
         return function(request, *args, **kwargs)
+
     return wrap

network/base/forms.py

@@ -1,20 +1,163 @@
+"""SatNOGS Network django base Forms class"""
+from __future__ import absolute_import
+
 from django import forms
 
-from network.base.models import Station
+from network.base.db_api import DBConnectionError, get_transmitters_by_uuid_set
+from network.base.models import Observation, Station
+from network.base.perms import UserNoPermissionError, \
+    check_schedule_perms_per_station
+from network.base.validators import ObservationOverlapError, OutOfRangeError, \
+    check_end_datetime, check_overlaps, check_start_datetime, \
+    check_start_end_datetimes, check_transmitter_station_pairs
+
+
+class ObservationForm(forms.ModelForm):
+    """Model Form class for Observation objects"""
+    start = forms.DateTimeField(
+        input_formats=['%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d %H:%M:%S'],
+        error_messages={
+            'invalid':
+            'Start datetime should have either "%Y-%m-%d %H:%M:%S.%f" or "%Y-%m-%d %H:%M:%S" '
+            'format.',
+            'required':
+            'Start datetime is required.'
+        }
+    )
+    end = forms.DateTimeField(
+        input_formats=['%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d %H:%M:%S'],
+        error_messages={
+            'invalid':
+            'End datetime should have either "%Y-%m-%d %H:%M:%S.%f" or "%Y-%m-%d %H:%M:%S" '
+            'format.',
+            'required':
+            'End datetime is required.'
+        }
+    )
+    ground_station = forms.ModelChoiceField(
+        queryset=Station.objects.filter(status__gt=0),
+        error_messages={
+            'invalid_choice': 'Station(s) should exist and be online.',
+            'required': 'Station is required.'
+        }
+    )
+
+    def clean_start(self):
+        """Validates start datetime of a new observation"""
+        start = self.cleaned_data['start']
+        try:
+            check_start_datetime(start)
+        except ValueError as error:
+            raise forms.ValidationError(error, code='invalid')
+        return start
+
+    def clean_end(self):
+        """Validates end datetime of a new observation"""
+        end = self.cleaned_data['end']
+        try:
+            check_end_datetime(end)
+        except ValueError as error:
+            raise forms.ValidationError(error, code='invalid')
+        return end
+
+    def clean(self):
+        """Validates combination of start and end datetimes of a new observation"""
+        if any(self.errors):
+            # If there are errors in fields validation no need for validating the form
+            return
+        cleaned_data = super(ObservationForm, self).clean()
+        start = cleaned_data['start']
+        end = cleaned_data['end']
+        try:
+            check_start_end_datetimes(start, end)
+        except ValueError as error:
+            raise forms.ValidationError(error, code='invalid')
+
+    class Meta:
+        model = Observation
+        fields = ['transmitter_uuid', 'start', 'end', 'ground_station']
+        error_messages = {'transmitter_uuid': {'required': "Transmitter is required"}}
+
+
+class BaseObservationFormSet(forms.BaseFormSet):
+    """Base FormSet class for Observation objects forms"""
+    transmitters = {}
+
+    def __init__(self, user, *args, **kwargs):
+        """Initializes Observation FormSet"""
+        self.user = user
+        super(BaseObservationFormSet, self).__init__(*args, **kwargs)
+
+    def clean(self):
+        """Validates Observation FormSet data"""
+        if any(self.errors):
+            # If there are errors in forms validation no need for validating the formset
+            return
+
+        station_list = []
+        transmitter_uuid_set = set()
+        transmitter_uuid_station_list = []
+        start_end_per_station = {}
+
+        for form in self.forms:
+            station = form.cleaned_data.get('ground_station')
+            transmitter_uuid = form.cleaned_data.get('transmitter_uuid')
+            start = form.cleaned_data.get('start')
+            end = form.cleaned_data.get('end')
+            station_id = int(station.id)
+            station_list.append(station)
+            transmitter_uuid_set.add(transmitter_uuid)
+            transmitter_uuid_station_list.append((transmitter_uuid, station))
+            if station_id in start_end_per_station:
+                start_end_per_station[station_id].append((start, end))
+            else:
+                start_end_per_station[station_id] = []
+                start_end_per_station[station_id].append((start, end))
+
+        try:
+            check_overlaps(start_end_per_station)
+        except ObservationOverlapError as error:
+            raise forms.ValidationError(error, code='invalid')
+
+        station_list = list(set(station_list))
+        try:
+            check_schedule_perms_per_station(self.user, station_list)
+        except UserNoPermissionError as error:
+            raise forms.ValidationError(error, code='forbidden')
+
+        try:
+            transmitters = get_transmitters_by_uuid_set(transmitter_uuid_set)
+            self.transmitters = transmitters
+        except ValueError as error:
+            raise forms.ValidationError(error, code='invalid')
+        except DBConnectionError as error:
+            raise forms.ValidationError(error)
+
+        transmitter_uuid_station_set = set(transmitter_uuid_station_list)
+        transmitter_station_list = [
+            (transmitters[pair[0]], pair[1]) for pair in transmitter_uuid_station_set
+        ]
+        try:
+            check_transmitter_station_pairs(transmitter_station_list)
+        except OutOfRangeError as error:
+            raise forms.ValidationError(error, code='invalid')
 
 
 class StationForm(forms.ModelForm):
+    """Model Form class for Station objects"""
     class Meta:
         model = Station
-        fields = ['name', 'image', 'alt', 'lat', 'lng', 'qthlocator',
-                  'horizon', 'antenna', 'testing', 'description',
-                  'target_utilization']
+        fields = [
+            'name', 'image', 'alt', 'lat', 'lng', 'qthlocator', 'horizon', 'antenna', 'testing',
+            'description', 'target_utilization'
+        ]
         image = forms.ImageField(required=False)
 
 
 class SatelliteFilterForm(forms.Form):
+    """Form class for Satellite objects"""
     norad = forms.IntegerField(required=False)
-    start_date = forms.CharField(required=False)
-    end_date = forms.CharField(required=False)
+    start = forms.CharField(required=False)
+    end = forms.CharField(required=False)
     ground_station = forms.IntegerField(required=False)
     transmitter = forms.CharField(required=False)

network/base/helpers.py

@@ -1,5 +0,0 @@
-def downlink_low_is_in_range(antenna, transmitter):
-    if transmitter['downlink_low'] is not None:
-        return antenna.frequency <= transmitter['downlink_low'] <= antenna.frequency_max
-    else:
-        return False

network/base/management/commands/fetch_data.py

@@ -1,52 +1,19 @@
-import requests
+"""SatNOGS Network django management command to fetch data (Satellites and Transmitters)"""
+from __future__ import absolute_import
 
 from django.core.management.base import BaseCommand, CommandError
-from django.conf import settings
+# Remove this import after Python 3 migration
+from requests.exceptions import ConnectionError  # pylint: disable=W0622
 
-from network.base.models import Satellite, Transmitter
+from network.base.tasks import fetch_data
 
 
 class Command(BaseCommand):
-    help = 'Fetch Modes, Satellites and Transmitters from satnogs-db'
+    """Django management command to fetch Satellites and Transmitters from SatNOGS DB"""
+    help = 'Fetches Satellites and Transmitters from SaTNOGS DB'
 
     def handle(self, *args, **options):
-        db_api_url = settings.DB_API_ENDPOINT
-        if len(db_api_url) == 0:
-            self.stdout.write("Zero length api url, fetching is stopped")
-            return
-        satellites_url = "{}satellites".format(db_api_url)
-        transmitters_url = "{}transmitters".format(db_api_url)
-
         try:
-            self.stdout.write("Fetching Satellites from {}".format(satellites_url))
-            r_satellites = requests.get(satellites_url)
-
-            self.stdout.write("Fetching Transmitters from {}".format(transmitters_url))
-            r_transmitters = requests.get(transmitters_url)
-        except requests.exceptions.ConnectionError:
-            raise CommandError('API is unreachable')
-
-        # Fetch Satellites
-        for satellite in r_satellites.json():
-            norad_cat_id = satellite['norad_cat_id']
-            name = satellite['name']
-            satellite.pop('decayed', None)
-            try:
-                existing_satellite = Satellite.objects.get(norad_cat_id=norad_cat_id)
-                existing_satellite.__dict__.update(satellite)
-                existing_satellite.save()
-                self.stdout.write('Satellite {0}-{1} updated'.format(norad_cat_id, name))
-            except Satellite.DoesNotExist:
-                Satellite.objects.create(**satellite)
-                self.stdout.write('Satellite {0}-{1} added'.format(norad_cat_id, name))
-
-        # Fetch Transmitters
-        for transmitter in r_transmitters.json():
-            uuid = transmitter['uuid']
-
-            try:
-                Transmitter.objects.get(uuid=uuid)
-                self.stdout.write('Transmitter {0} already exists'.format(uuid))
-            except Transmitter.DoesNotExist:
-                Transmitter.objects.create(uuid=uuid)
-                self.stdout.write('Transmitter {0} created'.format(uuid))
+            fetch_data()
+        except ConnectionError as exception:
+            raise CommandError(exception)

network/base/management/commands/fetch_tle.py

@@ -1,47 +0,0 @@
-from optparse import make_option
-from orbit import satellite
-
-from django.core.management.base import BaseCommand, CommandError
-
-from network.base.models import Satellite
-
-
-class Command(BaseCommand):
-    option_list = BaseCommand.option_list + (
-        make_option('--delete',
-                    action='store_true',
-                    dest='delete',
-                    default=False,
-                    help='Delete Satellite'),
-    )
-    args = '<Satellite Identifiers>'
-    help = 'Updates/Inserts TLEs for certain Satellites'
-
-    def handle(self, *args, **options):
-        for item in args:
-            if options['delete']:
-                try:
-                    Satellite.objects.get(norad_cat_id=item).delete()
-                    self.stdout.write('Satellite {}: deleted'.format(item))
-                    continue
-                except Satellite.DoesNotExist:
-                    raise CommandError('Satellite with Identifier {} does not exist'.format(item))
-
-            try:
-                sat = satellite(item)
-            except IndexError:
-                raise CommandError('Satellite with Identifier {} does not exist'.format(item))
-
-            try:
-                obj = Satellite.objects.get(norad_cat_id=item)
-            except Satellite.DoesNotExist:
-                obj = Satellite(norad_cat_id=item)
-
-            obj.name = sat.name()
-            tle = sat.tle()
-            obj.tle0 = tle[0]
-            obj.tle1 = tle[1]
-            obj.tle2 = tle[2]
-            obj.save()
-
-            self.stdout.write('fetched data for {}: {}'.format(obj.norad_cat_id, obj.name))

network/base/management/commands/initialize.py

@@ -1,15 +1,23 @@
-from django.core.management.base import BaseCommand
+"""SatNOGS Network django management command to initialize a new database"""
+from __future__ import absolute_import
+
 from django.core.management import call_command
+from django.core.management.base import BaseCommand
 
 from network.base.models import Antenna
-from network.base.tests import (generate_payload, generate_payload_name,
-                                DemodDataFactory, StationFactory, ObservationFactory)
+from network.base.tests import DemodDataFactory, RealisticObservationFactory, \
+    StationFactory, generate_payload, generate_payload_name
 
 
 class Command(BaseCommand):
+    """Django management command to initialize a new database"""
     help = 'Create initial fixtures'
 
     def handle(self, *args, **options):
+        station_fixture_count = 40
+        observation_fixture_count = 200
+        demoddata_fixture_count = 40
+
         # Migrate
         self.stdout.write("Creating database...")
         call_command('migrate')
@@ -18,18 +26,24 @@ class Command(BaseCommand):
         call_command('loaddata', 'antennas')
         call_command('fetch_data')
 
-        # Create random fixtures for remaining models
-        self.stdout.write("Creating fixtures...")
-        StationFactory.create_batch(40,
-                                    antennas=(Antenna.objects.all().values_list('id', flat=True)))
-        ObservationFactory.create_batch(200)
-        for _ in range(40):
-            DemodDataFactory.create(payload_demod__data=generate_payload(),
-                                    payload_demod__filename=generate_payload_name())
-
         # Update TLEs
         call_command('update_all_tle')
 
+        # Create random fixtures for remaining models
+        self.stdout.write("Creating fixtures...")
+        StationFactory.create_batch(
+            station_fixture_count, antennas=(Antenna.objects.all().values_list('id', flat=True))
+        )
+        self.stdout.write("Added {} stations.".format(station_fixture_count))
+        RealisticObservationFactory.create_batch(observation_fixture_count)
+        self.stdout.write("Added {} observations.".format(observation_fixture_count))
+        for _ in range(demoddata_fixture_count):
+            DemodDataFactory.create(
+                payload_demod__data=generate_payload(),
+                payload_demod__filename=generate_payload_name()
+            )
+        self.stdout.write("Added {} DemodData objects.".format(demoddata_fixture_count))
+
         # Create superuser
         self.stdout.write("Creating a superuser...")
         call_command('createsuperuser')

network/base/management/commands/update_all_tle.py

@@ -1,9 +1,13 @@
+"""SatNOGS Network django management command to update TLE entries"""
+from __future__ import absolute_import
+
 from django.core.management.base import BaseCommand
 
 from network.base.tasks import update_all_tle
 
 
 class Command(BaseCommand):
+    """Django management command to update TLE entries"""
     help = 'Update TLEs for existing Satellites'
 
     def handle(self, *args, **options):

network/base/management/commands/update_station_last_seen.py

@@ -1,21 +1,30 @@
-from django.core.management.base import LabelCommand
+"""SatNOGS Network django management command to set last seen value on stations entries"""
+from __future__ import absolute_import
+
+from django.core.management.base import BaseCommand
 from django.utils.timezone import now
 
 from network.base.models import Station
 
 
-class Command(LabelCommand):
-    args = '<Station IDs>'
+class Command(BaseCommand):
+    """Django management command to set last seen value on stations entries"""
     help = 'Updates Last_Seen Timestamp for given Stations'
 
-    def handle_label(self, label, **options):
-        try:
-            gs = Station.objects.get(id=label)
-        except Station.DoesNotExist:
-            self.stderr.write('Station with ID {} does not exist'.format(label))
-            return
+    def add_arguments(self, parser):
+        parser.add_argument('station_id', nargs='+', type=int)
 
-        timestamp = now()
-        gs.last_seen = timestamp
-        gs.save()
-        self.stdout.write('Updated Last_Seen for Station {} to {}'.format(label, timestamp))
+    def handle(self, *args, **options):
+        for station_id in options['station_id']:
+            try:
+                ground_station = Station.objects.get(id=station_id)
+            except Station.DoesNotExist:
+                self.stderr.write('Station with ID {} does not exist'.format(station_id))
+                return
+
+            timestamp = now()
+            ground_station.last_seen = timestamp
+            ground_station.save()
+            self.stdout.write(
+                'Updated Last_Seen for Station {} to {}'.format(station_id, timestamp)
+            )

network/base/managers.py

@@ -1,7 +1,12 @@
+"""Django base manager for SatNOGS Network"""
+from __future__ import absolute_import
+
 from django.db import models
 from django.utils.timezone import now
 
 
 class ObservationManager(models.QuerySet):
+    """Observation Manager with extra functionality"""
     def is_future(self):
+        """Return future observations"""
         return self.filter(end__gte=now())

network/base/migrations/0001_initial.py

@@ -20,8 +20,8 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('frequency', models.FloatField(validators=[django.core.validators.MinValueValidator(0)])),
-                ('band', models.CharField(max_length=5, choices=[(b'HF', b'HF'), (b'VHF', b'VHF'), (b'UHF', b'UHF'), (b'L', b'L'), (b'S', b'S'), (b'C', b'C'), (b'X', b'X'), (b'KU', b'KU')])),
-                ('antenna_type', models.CharField(max_length=15, choices=[(b'dipole', b'Dipole'), (b'yagi', b'Yagi'), (b'helical', b'Helical'), (b'parabolic', b'Parabolic')])),
+                ('band', models.CharField(max_length=5, choices=[('HF', 'HF'), ('VHF', 'VHF'), ('UHF', 'UHF'), ('L', 'L'), ('S', 'S'), ('C', 'C'), ('X', 'X'), ('KU', 'KU')])),
+                ('antenna_type', models.CharField(max_length=15, choices=[('dipole', 'Dipole'), ('yagi', 'Yagi'), ('helical', 'Helical'), ('parabolic', 'Parabolic')])),
             ],
         ),
         migrations.CreateModel(
@@ -30,7 +30,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('start', models.DateTimeField()),
                 ('end', models.DateTimeField()),
-                ('payload', models.FileField(null=True, upload_to=b'data_payloads', blank=True)),
+                ('payload', models.FileField(null=True, upload_to='data_payloads', blank=True)),
             ],
             options={
                 'ordering': ['-start', '-end'],
@@ -62,7 +62,7 @@ class Migration(migrations.Migration):
                 ('norad_cat_id', models.PositiveIntegerField()),
                 ('name', models.CharField(max_length=45)),
                 ('names', models.TextField(blank=True)),
-                ('image', models.ImageField(upload_to=b'satellites', blank=True)),
+                ('image', models.ImageField(upload_to='satellites', blank=True)),
                 ('tle0', models.CharField(max_length=100, blank=True)),
                 ('tle1', models.CharField(max_length=200, blank=True)),
                 ('tle2', models.CharField(max_length=200, blank=True)),
@@ -77,8 +77,8 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('name', models.CharField(max_length=45)),
-                ('image', models.ImageField(upload_to=b'ground_stations', blank=True)),
-                ('alt', models.PositiveIntegerField(help_text=b'In meters above ground')),
+                ('image', models.ImageField(upload_to='ground_stations', blank=True)),
+                ('alt', models.PositiveIntegerField(help_text='In meters above ground')),
                 ('lat', models.FloatField(validators=[django.core.validators.MaxValueValidator(90), django.core.validators.MinValueValidator(-90)])),
                 ('lng', models.FloatField(validators=[django.core.validators.MaxValueValidator(180), django.core.validators.MinValueValidator(-180)])),
                 ('qthlocator', models.CharField(max_length=255, blank=True)),
@@ -87,7 +87,7 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(auto_now_add=True)),
                 ('active', models.BooleanField(default=False)),
                 ('last_seen', models.DateTimeField(null=True, blank=True)),
-                ('antenna', models.ManyToManyField(help_text=b'If you want to add a new Antenna contact SatNOGS Team', to='base.Antenna', blank=True)),
+                ('antenna', models.ManyToManyField(help_text='If you want to add a new Antenna contact SatNOGS Team', to='base.Antenna', blank=True)),
                 ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
             ],
             options={

network/base/migrations/0002_auto_20151011_1406.py

@@ -14,6 +14,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='antenna',
             name='antenna_type',
-            field=models.CharField(max_length=15, choices=[(b'dipole', b'Dipole'), (b'yagi', b'Yagi'), (b'helical', b'Helical'), (b'parabolic', b'Parabolic'), (b'vertical', b'Verical')]),
+            field=models.CharField(max_length=15, choices=[('dipole', 'Dipole'), ('yagi', 'Yagi'), ('helical', 'Helical'), ('parabolic', 'Parabolic'), ('vertical', 'Verical')]),
         ),
     ]

network/base/migrations/0004_station_horizon.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='station',
             name='horizon',
-            field=models.PositiveIntegerField(default=10, help_text=b'In degrees above 0'),
+            field=models.PositiveIntegerField(default=10, help_text='In degrees above 0'),
         ),
     ]

network/base/migrations/0005_auto_20160320_1619.py

@@ -17,7 +17,7 @@ class Migration(migrations.Migration):
             name='Rig',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('name', models.CharField(choices=[(b'Radio', b'Radio'), (b'SDR', b'SDR')], max_length=10)),
+                ('name', models.CharField(choices=[('Radio', 'Radio'), ('SDR', 'SDR')], max_length=10)),
                 ('rictld_number', models.PositiveIntegerField(blank=True, null=True)),
             ],
         ),

network/base/migrations/0006_auto_20160325_0126.py

@@ -23,7 +23,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='data',
             name='vetted_status',
-            field=models.CharField(choices=[(b'unknown', b'Unknown'), (b'verified', b'Verified'), (b'data_not_verified', b'Has Data, Not Verified'), (b'no_data', b'No Data')], default=b'unknown', max_length=10),
+            field=models.CharField(choices=[('unknown', 'Unknown'), ('verified', 'Verified'), ('data_not_verified', 'Has Data, Not Verified'), ('no_data', 'No Data')], default='unknown', max_length=10),
         ),
         migrations.AddField(
             model_name='data',

network/base/migrations/0007_data_payload_demode.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='data',
             name='payload_demode',
-            field=models.FileField(blank=True, null=True, upload_to=b'data_payloads'),
+            field=models.FileField(blank=True, null=True, upload_to='data_payloads'),
         ),
     ]

network/base/migrations/0010_auto_20160507_1550.py

@@ -17,7 +17,7 @@ class Migration(migrations.Migration):
             name='DemodData',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('payload_demod', models.FileField(blank=True, null=True, upload_to=b'data_payloads')),
+                ('payload_demod', models.FileField(blank=True, null=True, upload_to='data_payloads')),
             ],
         ),
         migrations.RemoveField(

network/base/migrations/0012_auto_20160508_1516.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='station',
             name='antenna',
-            field=models.ManyToManyField(blank=True, help_text=b'If you want to add a new Antenna contact <a href="https://community.satnogs.org/" target="_blank">SatNOGS Team</a>', to='base.Antenna'),
+            field=models.ManyToManyField(blank=True, help_text='If you want to add a new Antenna contact <a href="https://community.satnogs.org/" target="_blank">SatNOGS Team</a>', to='base.Antenna'),
         ),
     ]

network/base/migrations/0013_data_waterfall.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='data',
             name='waterfall',
-            field=models.ImageField(blank=True, null=True, upload_to=b'data_waterfalls'),
+            field=models.ImageField(blank=True, null=True, upload_to='data_waterfalls'),
         ),
     ]

network/base/migrations/0014_auto_20170205_0059.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='data',
             name='vetted_status',
-            field=models.CharField(choices=[(b'unknown', b'Unknown'), (b'verified', b'Verified'), (b'data_not_verified', b'Has Data, Not Verified'), (b'no_data', b'No Data')], default=b'unknown', max_length=20),
+            field=models.CharField(choices=[('unknown', 'Unknown'), ('verified', 'Verified'), ('data_not_verified', 'Has Data, Not Verified'), ('no_data', 'No Data')], default='unknown', max_length=20),
         ),
     ]

network/base/migrations/0018_auto_20170519_1955.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='antenna',
             name='antenna_type',
-            field=models.CharField(choices=[(b'dipole', b'Dipole'), (b'yagi', b'Yagi'), (b'helical', b'Helical'), (b'parabolic', b'Parabolic'), (b'vertical', b'Verical'), (b'turnstile', b'Turnstile'), (b'quadrafilar', b'Quadrafilar'), (b'eggbeater', b'Eggbeater'), (b'lindenblad', b'Lindenblad')], max_length=15),
+            field=models.CharField(choices=[('dipole', 'Dipole'), ('yagi', 'Yagi'), ('helical', 'Helical'), ('parabolic', 'Parabolic'), ('vertical', 'Verical'), ('turnstile', 'Turnstile'), ('quadrafilar', 'Quadrafilar'), ('eggbeater', 'Eggbeater'), ('lindenblad', 'Lindenblad')], max_length=15),
         ),
     ]

network/base/migrations/0019_satellite_status.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='satellite',
             name='status',
-            field=models.CharField(choices=[(b'alive', b'alive'), (b'dead', b'dead'), (b're-entered', b're-entered')], default=b'alive', max_length=10),
+            field=models.CharField(choices=[('alive', 'alive'), ('dead', 'dead'), ('re-entered', 're-entered')], default='alive', max_length=10),
         ),
     ]

network/base/migrations/0022_auto_20170909_2103.py

@@ -43,7 +43,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='observation',
             name='payload',
-            field=models.FileField(blank=True, null=True, upload_to=b'data_payloads'),
+            field=models.FileField(blank=True, null=True, upload_to='data_payloads'),
         ),
         migrations.AddField(
             model_name='observation',
@@ -63,7 +63,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='observation',
             name='vetted_status',
-            field=models.CharField(choices=[(b'unknown', b'Unknown'), (b'verified', b'Verified'), (b'data_not_verified', b'Has Data, Not Verified'), (b'no_data', b'No Data')], default=b'unknown', max_length=20),
+            field=models.CharField(choices=[('unknown', 'Unknown'), ('verified', 'Verified'), ('data_not_verified', 'Has Data, Not Verified'), ('no_data', 'No Data')], default='unknown', max_length=20),
         ),
         migrations.AddField(
             model_name='observation',
@@ -73,6 +73,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='observation',
             name='waterfall',
-            field=models.ImageField(blank=True, null=True, upload_to=b'data_waterfalls'),
+            field=models.ImageField(blank=True, null=True, upload_to='data_waterfalls'),
         ),
     ]

network/base/migrations/0028_auto_20171127_2154.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='antenna',
             name='antenna_type',
-            field=models.CharField(choices=[(b'dipole', b'Dipole'), (b'yagi', b'Yagi'), (b'helical', b'Helical'), (b'parabolic', b'Parabolic'), (b'vertical', b'Verical'), (b'turnstile', b'Turnstile'), (b'quadrafilar', b'Quadrafilar'), (b'eggbeater', b'Eggbeater'), (b'lindenblad', b'Lindenblad'), (b'paralindy', b'Parasitic Lindenblad')], max_length=15),
+            field=models.CharField(choices=[('dipole', 'Dipole'), ('yagi', 'Yagi'), ('helical', 'Helical'), ('parabolic', 'Parabolic'), ('vertical', 'Verical'), ('turnstile', 'Turnstile'), ('quadrafilar', 'Quadrafilar'), ('eggbeater', 'Eggbeater'), ('lindenblad', 'Lindenblad'), ('paralindy', 'Parasitic Lindenblad')], max_length=15),
         ),
     ]

network/base/migrations/0030_auto_20171224_1701.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='observation',
             name='vetted_status',
-            field=models.CharField(choices=[(b'unknown', b'Unknown'), (b'verified', b'Verified'), (b'no_data', b'No Data'), (b'good', b'Good'), (b'bad', b'Bad'), (b'failed', b'Failed')], default=b'unknown', max_length=20),
+            field=models.CharField(choices=[('unknown', 'Unknown'), ('verified', 'Verified'), ('no_data', 'No Data'), ('good', 'Good'), ('bad', 'Bad'), ('failed', 'Failed')], default='unknown', max_length=20),
         ),
     ]

network/base/migrations/0032_auto_20171224_1703.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='observation',
             name='vetted_status',
-            field=models.CharField(choices=[(b'unknown', b'Unknown'), (b'good', b'Good'), (b'bad', b'Bad'), (b'failed', b'Failed')], default=b'unknown', max_length=20),
+            field=models.CharField(choices=[('unknown', 'Unknown'), ('good', 'Good'), ('bad', 'Bad'), ('failed', 'Failed')], default='unknown', max_length=20),
         ),
     ]

network/base/migrations/0033_auto_20171228_1515.py

@@ -28,7 +28,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='station',
             name='status',
-            field=models.IntegerField(choices=[(2, b'Online'), (1, b'Testing'), (0, b'Offline')], default=0),
+            field=models.IntegerField(choices=[(2, 'Online'), (1, 'Testing'), (0, 'Offline')], default=0),
         ),
         migrations.AddField(
             model_name='station',

network/base/migrations/0036_auto_20180316_1216.py

@@ -16,16 +16,16 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='station',
             name='description',
-            field=models.TextField(blank=True, help_text=b'Max 500 characters', max_length=500),
+            field=models.TextField(blank=True, help_text='Max 500 characters', max_length=500),
         ),
         migrations.AlterField(
             model_name='station',
             name='lat',
-            field=models.FloatField(help_text=b'eg. 38.01697', validators=[django.core.validators.MaxValueValidator(90), django.core.validators.MinValueValidator(-90)]),
+            field=models.FloatField(help_text='eg. 38.01697', validators=[django.core.validators.MaxValueValidator(90), django.core.validators.MinValueValidator(-90)]),
         ),
         migrations.AlterField(
             model_name='station',
             name='lng',
-            field=models.FloatField(help_text=b'eg. 23.7314', validators=[django.core.validators.MaxValueValidator(180), django.core.validators.MinValueValidator(-180)]),
+            field=models.FloatField(help_text='eg. 23.7314', validators=[django.core.validators.MaxValueValidator(180), django.core.validators.MinValueValidator(-180)]),
         ),
     ]

network/base/migrations/0037_stationstatuslog.py

@@ -17,7 +17,7 @@ class Migration(migrations.Migration):
             name='StationStatusLog',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('status', models.IntegerField(choices=[(2, b'Online'), (1, b'Testing'), (0, b'Offline')], default=0)),
+                ('status', models.IntegerField(choices=[(2, 'Online'), (1, 'Testing'), (0, 'Offline')], default=0)),
                 ('changed', models.DateTimeField(auto_now_add=True)),
                 ('station', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='station_logs', to='base.Station')),
             ],

network/base/migrations/0040_auto_20180811_1350.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='station',
             name='alt',
-            field=models.PositiveIntegerField(help_text=b'In meters above sea level'),
+            field=models.PositiveIntegerField(help_text='In meters above sea level'),
         ),
     ]

network/base/migrations/0041_auto_20180812_0915.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='antenna',
             name='antenna_type',
-            field=models.CharField(choices=[(b'dipole', b'Dipole'), (b'v-dipole', b'V-Dipole'), (b'yagi', b'Yagi'), (b'helical', b'Helical'), (b'parabolic', b'Parabolic'), (b'vertical', b'Verical'), (b'turnstile', b'Turnstile'), (b'quadrafilar', b'Quadrafilar'), (b'eggbeater', b'Eggbeater'), (b'lindenblad', b'Lindenblad'), (b'paralindy', b'Parasitic Lindenblad')], max_length=15),
+            field=models.CharField(choices=[('dipole', 'Dipole'), ('v-dipole', 'V-Dipole'), ('yagi', 'Yagi'), ('helical', 'Helical'), ('parabolic', 'Parabolic'), ('vertical', 'Verical'), ('turnstile', 'Turnstile'), ('quadrafilar', 'Quadrafilar'), ('eggbeater', 'Eggbeater'), ('lindenblad', 'Lindenblad'), ('paralindy', 'Parasitic Lindenblad')], max_length=15),
         ),
     ]

network/base/migrations/0042_auto_20180814_1325.py

@@ -16,6 +16,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='station',
             name='image',
-            field=models.ImageField(blank=True, upload_to=b'ground_stations', validators=[network.base.models.validate_image]),
+            field=models.ImageField(blank=True, upload_to='ground_stations', validators=[network.base.models.validate_image]),
         ),
     ]

network/base/migrations/0043_auto_20180817_0923.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='station',
             name='antenna',
-            field=models.ManyToManyField(blank=True, help_text=b'If you want to add a new Antenna contact <a href="https://community.satnogs.org/" target="_blank">SatNOGS Team</a>', related_name='stations', to='base.Antenna'),
+            field=models.ManyToManyField(blank=True, help_text='If you want to add a new Antenna contact <a href="https://community.satnogs.org/" target="_blank">SatNOGS Team</a>', related_name='stations', to='base.Antenna'),
         ),
     ]

network/base/migrations/0044_auto_20180817_1351.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='antenna',
             name='antenna_type',
-            field=models.CharField(choices=[(b'dipole', b'Dipole'), (b'v-dipole', b'V-Dipole'), (b'discone', b'Discone'), (b'yagi', b'Yagi'), (b'helical', b'Helical'), (b'parabolic', b'Parabolic'), (b'vertical', b'Verical'), (b'turnstile', b'Turnstile'), (b'quadrafilar', b'Quadrafilar'), (b'eggbeater', b'Eggbeater'), (b'lindenblad', b'Lindenblad'), (b'paralindy', b'Parasitic Lindenblad')], max_length=15),
+            field=models.CharField(choices=[('dipole', 'Dipole'), ('v-dipole', 'V-Dipole'), ('discone', 'Discone'), ('yagi', 'Yagi'), ('helical', 'Helical'), ('parabolic', 'Parabolic'), ('vertical', 'Verical'), ('turnstile', 'Turnstile'), ('quadrafilar', 'Quadrafilar'), ('eggbeater', 'Eggbeater'), ('lindenblad', 'Lindenblad'), ('paralindy', 'Parasitic Lindenblad')], max_length=15),
         ),
     ]

network/base/migrations/0045_auto_20180822_1947.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='antenna',
             name='antenna_type',
-            field=models.CharField(choices=[(b'dipole', b'Dipole'), (b'v-dipole', b'V-Dipole'), (b'discone', b'Discone'), (b'ground', b'Ground Plane'), (b'yagi', b'Yagi'), (b'helical', b'Helical'), (b'parabolic', b'Parabolic'), (b'vertical', b'Verical'), (b'turnstile', b'Turnstile'), (b'quadrafilar', b'Quadrafilar'), (b'eggbeater', b'Eggbeater'), (b'lindenblad', b'Lindenblad'), (b'paralindy', b'Parasitic Lindenblad')], max_length=15),
+            field=models.CharField(choices=[('dipole', 'Dipole'), ('v-dipole', 'V-Dipole'), ('discone', 'Discone'), ('ground', 'Ground Plane'), ('yagi', 'Yagi'), ('helical', 'Helical'), ('parabolic', 'Parabolic'), ('vertical', 'Verical'), ('turnstile', 'Turnstile'), ('quadrafilar', 'Quadrafilar'), ('eggbeater', 'Eggbeater'), ('lindenblad', 'Lindenblad'), ('paralindy', 'Parasitic Lindenblad')], max_length=15),
         ),
     ]

network/base/migrations/0049_auto_20180915_1503.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='antenna',
             name='antenna_type',
-            field=models.CharField(choices=[(b'dipole', b'Dipole'), (b'v-dipole', b'V-Dipole'), (b'discone', b'Discone'), (b'ground', b'Ground Plane'), (b'yagi', b'Yagi'), (b'cross-yagi', b'Cross Yagi'), (b'helical', b'Helical'), (b'parabolic', b'Parabolic'), (b'vertical', b'Verical'), (b'turnstile', b'Turnstile'), (b'quadrafilar', b'Quadrafilar'), (b'eggbeater', b'Eggbeater'), (b'lindenblad', b'Lindenblad'), (b'paralindy', b'Parasitic Lindenblad')], max_length=15),
+            field=models.CharField(choices=[('dipole', 'Dipole'), ('v-dipole', 'V-Dipole'), ('discone', 'Discone'), ('ground', 'Ground Plane'), ('yagi', 'Yagi'), ('cross-yagi', 'Cross Yagi'), ('helical', 'Helical'), ('parabolic', 'Parabolic'), ('vertical', 'Verical'), ('turnstile', 'Turnstile'), ('quadrafilar', 'Quadrafilar'), ('eggbeater', 'Eggbeater'), ('lindenblad', 'Lindenblad'), ('paralindy', 'Parasitic Lindenblad')], max_length=15),
         ),
     ]

network/base/migrations/0052_auto_20181230_2113.py

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='transmitter',
             name='type',
-            field=models.CharField(choices=[(b'Transmitter', b'Transmitter'), (b'Transceiver', b'Transceiver'), (b'Transponder', b'Transponder')], default=b'Transmitter', max_length=11),
+            field=models.CharField(choices=[('Transmitter', 'Transmitter'), ('Transceiver', 'Transceiver'), ('Transponder', 'Transponder')], default='Transmitter', max_length=11),
         ),
         migrations.AddField(
             model_name='transmitter',

network/base/migrations/0054_station_target_utilization.py

@@ -16,6 +16,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='station',
             name='target_utilization',
-            field=models.IntegerField(blank=True, help_text=b'Target utilization factor for your station', null=True, validators=[django.core.validators.MaxValueValidator(100), django.core.validators.MinValueValidator(0)]),
+            field=models.IntegerField(blank=True, help_text='Target utilization factor for your station', null=True, validators=[django.core.validators.MaxValueValidator(100), django.core.validators.MinValueValidator(0)]),
         ),
     ]

network/base/migrations/0055_add_new_antenna_type.py

@@ -15,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='antenna',
             name='antenna_type',
-            field=models.CharField(choices=[(b'dipole', b'Dipole'), (b'v-dipole', b'V-Dipole'), (b'discone', b'Discone'), (b'ground', b'Ground Plane'), (b'yagi', b'Yagi'), (b'cross-yagi', b'Cross Yagi'), (b'helical', b'Helical'), (b'parabolic', b'Parabolic'), (b'vertical', b'Vertical'), (b'turnstile', b'Turnstile'), (b'quadrafilar', b'Quadrafilar'), (b'eggbeater', b'Eggbeater'), (b'lindenblad', b'Lindenblad'), (b'paralindy', b'Parasitic Lindenblad'), (b'patch', b'Patch')], max_length=15),
+            field=models.CharField(choices=[('dipole', 'Dipole'), ('v-dipole', 'V-Dipole'), ('discone', 'Discone'), ('ground', 'Ground Plane'), ('yagi', 'Yagi'), ('cross-yagi', 'Cross Yagi'), ('helical', 'Helical'), ('parabolic', 'Parabolic'), ('vertical', 'Vertical'), ('turnstile', 'Turnstile'), ('quadrafilar', 'Quadrafilar'), ('eggbeater', 'Eggbeater'), ('lindenblad', 'Lindenblad'), ('paralindy', 'Parasitic Lindenblad'), ('patch', 'Patch')], max_length=15),
         ),
     ]

network/base/migrations/0058_add_transmitter_into_observation_model.py

@@ -49,7 +49,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='observation',
             name='transmitter_description',
-            field=models.TextField(default=b''),
+            field=models.TextField(default=''),
         ),
         migrations.AddField(
             model_name='observation',
@@ -79,7 +79,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='observation',
             name='transmitter_type',
-            field=models.CharField(choices=[(b'Transmitter', b'Transmitter'), (b'Transceiver', b'Transceiver'), (b'Transponder', b'Transponder')], default=b'Transmitter', max_length=11),
+            field=models.CharField(choices=[('Transmitter', 'Transmitter'), ('Transceiver', 'Transceiver'), ('Transponder', 'Transponder')], default='Transmitter', max_length=11),
         ),
         migrations.AddField(
             model_name='observation',

network/base/migrations/0060_add_latest_tle_proxy_model.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-07-24 09:23
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0059_remove_mode_model'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LatestTle',
+            fields=[
+            ],
+            options={
+                'proxy': True,
+                'indexes': [],
+            },
+            bases=('base.tle',),
+        ),
+    ]

network/base/migrations/0061_create_indexes.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.25 on 2019-11-25 18:23
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0060_add_latest_tle_proxy_model'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='observation',
+            name='end',
+            field=models.DateTimeField(db_index=True),
+        ),
+        migrations.AlterField(
+            model_name='observation',
+            name='start',
+            field=models.DateTimeField(db_index=True),
+        ),
+        migrations.AlterField(
+            model_name='satellite',
+            name='norad_cat_id',
+            field=models.PositiveIntegerField(db_index=True),
+        ),
+        migrations.AlterField(
+            model_name='tle',
+            name='tle0',
+            field=models.CharField(blank=True, db_index=True, max_length=100),
+        ),
+        migrations.AddIndex(
+            model_name='observation',
+            index=models.Index(fields=['-start', '-end'], name='base_observ_start_bbb297_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='stationstatuslog',
+            index=models.Index(fields=['-changed'], name='base_statio_changed_71df65_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='station',
+            index=models.Index(fields=['-status', 'id'], name='base_statio_status_797b1c_idx'),
+        ),
+    ]

network/base/models.py

@@ -1,24 +1,30 @@
-import os
-from datetime import datetime, timedelta
-from PIL import Image
-import requests
-from shortuuidfield import ShortUUIDField
-import logging
+"""Django database base model for SatNOGS Network"""
+from __future__ import absolute_import, division
 
+import codecs
+import logging
+import os
+from datetime import timedelta
+
+import requests
 from django.conf import settings
 from django.core.cache import cache
-from django.core.validators import MaxValueValidator, MinValueValidator
 from django.core.exceptions import ValidationError
-from django.dispatch import receiver
+from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
+from django.db.models import OuterRef, Subquery
 from django.db.models.signals import post_save
+from django.dispatch import receiver
 from django.urls import reverse
+from django.utils.encoding import python_2_unicode_compatible
 from django.utils.html import format_html
 from django.utils.timezone import now
-
-from network.users.models import User
-from network.base.managers import ObservationManager
+from PIL import Image
 from rest_framework.authtoken.models import Token
+from shortuuidfield import ShortUUIDField
+
+from network.base.managers import ObservationManager
+from network.users.models import User
 
 ANTENNA_BANDS = ['HF', 'VHF', 'UHF', 'L', 'S', 'C', 'X', 'KU']
 ANTENNA_TYPES = (
@@ -36,7 +42,7 @@ ANTENNA_TYPES = (
     ('eggbeater', 'Eggbeater'),
     ('lindenblad', 'Lindenblad'),
     ('paralindy', 'Parasitic Lindenblad'),
-    ('patch', 'Patch')
+    ('patch', 'Patch')  # yapf: disable
 )
 OBSERVATION_STATUSES = (
     ('unknown', 'Unknown'),
@@ -54,19 +60,28 @@ TRANSMITTER_STATUS = ['active', 'inactive', 'invalid']
 TRANSMITTER_TYPE = ['Transmitter', 'Transceiver', 'Transponder']
 
 
+def _decode_pretty_hex(binary_data):
+    """Return the binary data as hex dump of the following form: `DE AD C0 DE`"""
+
+    data = codecs.encode(binary_data, 'hex').decode('ascii').upper()
+    return ' '.join(data[i:i + 2] for i in range(0, len(data), 2))
+
+
 def _name_obs_files(instance, filename):
+    """Return a filepath formatted by Observation ID"""
     return 'data_obs/{0}/{1}'.format(instance.id, filename)
 
 
 def _name_obs_demoddata(instance, filename):
+    """Return a filepath for DemodData formatted by Observation ID"""
     # On change of the string bellow, change it also at api/views.py
     return 'data_obs/{0}/{1}'.format(instance.observation.id, filename)
 
 
-def _observation_post_save(sender, instance, created, **kwargs):
+def _observation_post_save(sender, instance, created, **kwargs):  # pylint: disable=W0613
     """
     Post save Observation operations
-    * Auto vet as good obserfvation with Demod Data
+    * Auto vet as good observation with DemodData
     * Mark Observations from testing stations
     * Update client version for ground station
     """
@@ -81,7 +96,7 @@ def _observation_post_save(sender, instance, created, **kwargs):
     post_save.connect(_observation_post_save, sender=Observation)
 
 
-def _station_post_save(sender, instance, created, **kwargs):
+def _station_post_save(sender, instance, created, **kwargs):  # pylint: disable=W0613
     """
     Post save Station operations
     * Store current status
@@ -103,56 +118,66 @@ def _station_post_save(sender, instance, created, **kwargs):
     post_save.connect(_station_post_save, sender=Station)
 
 
-def _tle_post_save(sender, instance, created, **kwargs):
+def _tle_post_save(sender, instance, created, **kwargs):  # pylint: disable=W0613
     """
     Post save Tle operations
     * Update TLE for future observations
     """
     if created:
-        start_time = now() + timedelta(minutes=10)
-        Observation.objects.filter(satellite=instance.satellite, start__gt=start_time) \
+        start = now() + timedelta(minutes=10)
+        Observation.objects.filter(satellite=instance.satellite, start__gt=start) \
                            .update(tle=instance.id)
 
 
 def validate_image(fieldfile_obj):
+    """Validates image size"""
     filesize = fieldfile_obj.file.size
     megabyte_limit = 2.0
     if filesize > megabyte_limit * 1024 * 1024:
         raise ValidationError("Max file size is %sMB" % str(megabyte_limit))
 
 
+@python_2_unicode_compatible
 class Antenna(models.Model):
     """Model for antennas tracked with SatNOGS."""
     frequency = models.PositiveIntegerField()
     frequency_max = models.PositiveIntegerField()
-    band = models.CharField(choices=zip(ANTENNA_BANDS, ANTENNA_BANDS),
-                            max_length=5)
+    band = models.CharField(choices=list(zip(ANTENNA_BANDS, ANTENNA_BANDS)), max_length=5)
     antenna_type = models.CharField(choices=ANTENNA_TYPES, max_length=15)
 
-    def __unicode__(self):
-        return '{0} - {1} - {2} - {3}'.format(self.band, self.antenna_type,
-                                              self.frequency,
-                                              self.frequency_max)
+    def __str__(self):
+        return '{0} - {1} - {2} - {3}'.format(
+            self.band, self.antenna_type, self.frequency, self.frequency_max
+        )
 
 
+@python_2_unicode_compatible
 class Station(models.Model):
     """Model for SatNOGS ground stations."""
-    owner = models.ForeignKey(User, related_name="ground_stations",
-                              on_delete=models.SET_NULL, null=True, blank=True)
+    owner = models.ForeignKey(
+        User, related_name="ground_stations", on_delete=models.SET_NULL, null=True, blank=True
+    )
     name = models.CharField(max_length=45)
-    image = models.ImageField(upload_to='ground_stations', blank=True,
-                              validators=[validate_image])
+    image = models.ImageField(upload_to='ground_stations', blank=True, validators=[validate_image])
     alt = models.PositiveIntegerField(help_text='In meters above sea level')
-    lat = models.FloatField(validators=[MaxValueValidator(90), MinValueValidator(-90)],
-                            help_text='eg. 38.01697')
-    lng = models.FloatField(validators=[MaxValueValidator(180), MinValueValidator(-180)],
-                            help_text='eg. 23.7314')
+    lat = models.FloatField(
+        validators=[MaxValueValidator(90), MinValueValidator(-90)], help_text='eg. 38.01697'
+    )
+    lng = models.FloatField(
+        validators=[MaxValueValidator(180), MinValueValidator(-180)], help_text='eg. 23.7314'
+    )
     qthlocator = models.CharField(max_length=255, blank=True)
     location = models.CharField(max_length=255, blank=True)
-    antenna = models.ManyToManyField(Antenna, blank=True, related_name="stations",
-                                     help_text=('If you want to add a new Antenna contact '
-                                                '<a href="https://community.satnogs.org/" '
-                                                'target="_blank">SatNOGS Team</a>'))
+    antenna = models.ManyToManyField(
+        Antenna,
+        blank=True,
+        related_name="stations",
+        help_text=(
+            'If you want to add a new Antenna contact '
+            '<a href="https://community.satnogs.org/" '
+            'target="_blank">SatNOGS Team</a>'
+        )
+    )
     featured_date = models.DateField(null=True, blank=True)
     created = models.DateTimeField(auto_now_add=True)
     testing = models.BooleanField(default=True)
@@ -161,23 +186,27 @@ class Station(models.Model):
     horizon = models.PositiveIntegerField(help_text='In degrees above 0', default=10)
     description = models.TextField(max_length=500, blank=True, help_text='Max 500 characters')
     client_version = models.CharField(max_length=45, blank=True)
-    target_utilization = models.IntegerField(validators=[MaxValueValidator(100),
-                                                         MinValueValidator(0)],
-                                             help_text='Target utilization factor for '
-                                                       'your station',
-                                             null=True, blank=True)
+    target_utilization = models.IntegerField(
+        validators=[MaxValueValidator(100), MinValueValidator(0)],
+        help_text='Target utilization factor for '
+        'your station',
+        null=True,
+        blank=True
+    )
 
     class Meta:
         ordering = ['-status']
+        indexes = [models.Index(fields=['-status', 'id'])]
 
     def get_image(self):
+        """Return the image of the station or the default image if there is a defined one"""
         if self.image and hasattr(self.image, 'url'):
             return self.image.url
-        else:
-            return settings.STATION_DEFAULT_IMAGE
+        return settings.STATION_DEFAULT_IMAGE
 
     @property
     def is_online(self):
+        """Return true if station is online"""
         try:
             heartbeat = self.last_seen + timedelta(minutes=int(settings.STATION_HEARTBEAT_TIME))
             return heartbeat > now()
@@ -186,16 +215,19 @@ class Station(models.Model):
 
     @property
     def is_offline(self):
+        """Return true if station is offline"""
         return not self.is_online
 
     @property
     def is_testing(self):
+        """Return true if station is online and in testing mode"""
         if self.is_online:
             if self.status == 1:
                 return True
         return False
 
     def state(self):
+        """Return the station status in html format"""
         if not self.status:
             return format_html('<span style="color:red;">Offline</span>')
         if self.status == 1:
@@ -204,11 +236,13 @@ class Station(models.Model):
 
     @property
     def success_rate(self):
+        """Return the success rate of the station - successful observation over failed ones"""
         rate = cache.get('station-{0}-rate'.format(self.id))
         if not rate:
             observations = self.observations.exclude(testing=True).exclude(vetted_status="unknown")
-            success = observations.filter(id__in=(o.id for o in observations
-                                                  if o.is_good or o.is_bad)).count()
+            success = observations.filter(
+                id__in=(o.id for o in observations if o.is_good or o.is_bad)
+            ).count()
             if observations:
                 rate = int(100 * (float(success) / float(observations.count())))
                 cache.set('station-{0}-rate'.format(self.id), rate)
@@ -218,156 +252,98 @@ class Station(models.Model):
 
     @property
     def observations_count(self):
+        """Return the number of station's observations"""
         count = self.observations.all().count()
         return count
 
     @property
     def observations_future_count(self):
-        count = self.observations.is_future().count()
+        """Return the number of future station's observations"""
+        # False-positive no-member (E1101) pylint error:
+        # Instance of 'Station' has 'observations' member due to the
+        # Observation.station ForeignKey related_name
+        count = self.observations.is_future().count()  # pylint: disable=E1101
         return count
 
     @property
     def apikey(self):
+        """Return station owner API key"""
         try:
             token = Token.objects.get(user=self.owner)
         except Token.DoesNotExist:
             token = Token.objects.create(user=self.owner)
         return token
 
-    def __unicode__(self):
+    def __str__(self):
         return "%d - %s" % (self.pk, self.name)
 
 
 post_save.connect(_station_post_save, sender=Station)
 
 
+@python_2_unicode_compatible
 class StationStatusLog(models.Model):
-    station = models.ForeignKey(Station, related_name='station_logs',
-                                on_delete=models.CASCADE, null=True, blank=True)
+    """Model for keeping Status log for Station."""
+    station = models.ForeignKey(
+        Station, related_name='station_logs', on_delete=models.CASCADE, null=True, blank=True
+    )
     status = models.IntegerField(choices=STATION_STATUSES, default=0)
     changed = models.DateTimeField(auto_now_add=True)
 
     class Meta:
         ordering = ['-changed']
+        indexes = [models.Index(fields=['-changed'])]
 
-    def __unicode__(self):
+    def __str__(self):
         return '{0} - {1}'.format(self.station, self.status)
 
 
+@python_2_unicode_compatible
 class Satellite(models.Model):
     """Model for SatNOGS satellites."""
-    norad_cat_id = models.PositiveIntegerField()
+    norad_cat_id = models.PositiveIntegerField(db_index=True)
     norad_follow_id = models.PositiveIntegerField(blank=True, null=True)
     name = models.CharField(max_length=45)
     names = models.TextField(blank=True)
     image = models.CharField(max_length=100, blank=True, null=True)
     manual_tle = models.BooleanField(default=False)
-    status = models.CharField(choices=zip(SATELLITE_STATUS, SATELLITE_STATUS),
-                              max_length=10, default='alive')
+    status = models.CharField(
+        choices=list(zip(SATELLITE_STATUS, SATELLITE_STATUS)), max_length=10, default='alive'
+    )
 
     class Meta:
         ordering = ['norad_cat_id']
 
     def get_image(self):
+        """Return the station image or the default if doesn't exist one"""
         if self.image:
             return self.image
-        else:
-            return settings.SATELLITE_DEFAULT_IMAGE
+        return settings.SATELLITE_DEFAULT_IMAGE
 
-    @property
-    def latest_tle(self):
-        try:
-            latest_tle = Tle.objects.filter(satellite=self).latest('updated')
-            return latest_tle
-        except Tle.DoesNotExist:
-            return False
-
-    @property
-    def tle_no(self):
-        try:
-            line = self.latest_tle.tle1
-            return line[64:68]
-        except AttributeError:
-            return False
-
-    @property
-    def tle_epoch(self):
-        try:
-            line = self.latest_tle.tle1
-        except AttributeError:
-            return False
-        yd, s = line[18:32].split('.')
-        epoch = (datetime.strptime(yd, "%y%j") +
-                 timedelta(seconds=float("." + s) * 24 * 60 * 60))
-        return epoch
-
-    @property
-    def data_count(self):
-        return Observation.objects.filter(satellite=self).exclude(vetted_status='failed').count()
-
-    @property
-    def good_count(self):
-        data = Observation.objects.filter(satellite=self)
-        return data.filter(vetted_status='good').count()
-
-    @property
-    def bad_count(self):
-        data = Observation.objects.filter(satellite=self)
-        return data.filter(vetted_status='bad').count()
-
-    @property
-    def unvetted_count(self):
-        data = Observation.objects.filter(satellite=self)
-        return data.filter(vetted_status='unknown',
-                           end__lte=now()).count()
-
-    @property
-    def future_count(self):
-        data = Observation.objects.filter(satellite=self)
-        return data.filter(end__gt=now()).count()
-
-    @property
-    def success_rate(self):
-        try:
-            return int(100 * (float(self.good_count) / float(self.data_count)))
-        except (ZeroDivisionError, TypeError):
-            return 0
-
-    @property
-    def bad_rate(self):
-        try:
-            return int(100 * (float(self.bad_count) / float(self.data_count)))
-        except (ZeroDivisionError, TypeError):
-            return 0
-
-    @property
-    def unvetted_rate(self):
-        try:
-            return int(100 * (float(self.unvetted_count) / float(self.data_count)))
-        except (ZeroDivisionError, TypeError):
-            return 0
-
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 
+@python_2_unicode_compatible
 class Tle(models.Model):
-    tle0 = models.CharField(max_length=100, blank=True)
+    """Model for TLEs."""
+    tle0 = models.CharField(max_length=100, blank=True, db_index=True)
     tle1 = models.CharField(max_length=200, blank=True)
     tle2 = models.CharField(max_length=200, blank=True)
     updated = models.DateTimeField(auto_now=True, blank=True)
-    satellite = models.ForeignKey(Satellite, related_name='tles',
-                                  on_delete=models.CASCADE, null=True, blank=True)
+    satellite = models.ForeignKey(
+        Satellite, related_name='tles', on_delete=models.CASCADE, null=True, blank=True
+    )
 
     class Meta:
         ordering = ['tle0']
 
-    def __unicode__(self):
-        uni_name = "%d - %s" % (self.id, self.tle0)
-        return uni_name
+    def __str__(self):
+        return '{:d} - {:s}'.format(self.id, self.tle0)
 
     @property
     def str_array(self):
+        """Return TLE in string array format"""
         # tle fields are unicode, pyephem and others expect python strings
         return [str(self.tle0), str(self.tle1), str(self.tle2)]
 
@@ -375,33 +351,69 @@ class Tle(models.Model):
 post_save.connect(_tle_post_save, sender=Tle)
 
 
+class LatestTleManager(models.Manager):  # pylint: disable=R0903
+    """Django Manager for latest Tle objects"""
+    def get_queryset(self):
+        """Returns query of latest Tle
+
+        :returns: the latest Tle for each Satellite
+        """
+        subquery = Tle.objects.filter(satellite=OuterRef('satellite')).order_by('-updated')
+        return super(LatestTleManager,
+                     self).get_queryset().filter(updated=Subquery(subquery.values('updated')[:1]))
+
+
+@python_2_unicode_compatible
+class LatestTle(Tle):
+    """LatestTle is the latest entry of a Satellite Tle objects
+    """
+    objects = LatestTleManager()
+
+    class Meta:
+        proxy = True
+
+    def __str__(self):
+        return '{:d} - {:s}'.format(self.id, self.tle0)
+
+
+@python_2_unicode_compatible
 class Transmitter(models.Model):
     """Model for antennas transponders."""
     uuid = ShortUUIDField(db_index=True)
     sync_to_db = models.BooleanField(default=False)
 
+    def __str__(self):
+        return self.uuid
 
+
+@python_2_unicode_compatible
 class Observation(models.Model):
     """Model for SatNOGS observations."""
-    satellite = models.ForeignKey(Satellite, related_name='observations',
-                                  on_delete=models.SET_NULL, null=True, blank=True)
-    tle = models.ForeignKey(Tle, related_name='observations',
-                            on_delete=models.SET_NULL, null=True, blank=True)
-    author = models.ForeignKey(User, related_name='observations',
-                               on_delete=models.SET_NULL, null=True, blank=True)
-    start = models.DateTimeField()
-    end = models.DateTimeField()
-    ground_station = models.ForeignKey(Station, related_name='observations',
-                                       on_delete=models.SET_NULL, null=True, blank=True)
+    satellite = models.ForeignKey(
+        Satellite, related_name='observations', on_delete=models.SET_NULL, null=True, blank=True
+    )
+    tle = models.ForeignKey(
+        Tle, related_name='observations', on_delete=models.SET_NULL, null=True, blank=True
+    )
+    author = models.ForeignKey(
+        User, related_name='observations', on_delete=models.SET_NULL, null=True, blank=True
+    )
+    start = models.DateTimeField(db_index=True)
+    end = models.DateTimeField(db_index=True)
+    ground_station = models.ForeignKey(
+        Station, related_name='observations', on_delete=models.SET_NULL, null=True, blank=True
+    )
     client_version = models.CharField(max_length=255, blank=True)
     client_metadata = models.TextField(blank=True)
     payload = models.FileField(upload_to=_name_obs_files, blank=True, null=True)
     waterfall = models.ImageField(upload_to=_name_obs_files, blank=True, null=True)
     vetted_datetime = models.DateTimeField(null=True, blank=True)
-    vetted_user = models.ForeignKey(User, related_name='observations_vetted',
-                                    on_delete=models.SET_NULL, null=True, blank=True)
-    vetted_status = models.CharField(choices=OBSERVATION_STATUSES,
-                                     max_length=20, default='unknown')
+    vetted_user = models.ForeignKey(
+        User, related_name='observations_vetted', on_delete=models.SET_NULL, null=True, blank=True
+    )
+    vetted_status = models.CharField(
+        choices=OBSERVATION_STATUSES, max_length=20, default='unknown'
+    )
     testing = models.BooleanField(default=False)
     rise_azimuth = models.FloatField(blank=True, null=True)
     max_altitude = models.FloatField(blank=True, null=True)
@@ -409,10 +421,13 @@ class Observation(models.Model):
     archived = models.BooleanField(default=False)
     archive_identifier = models.CharField(max_length=255, blank=True)
     archive_url = models.URLField(blank=True, null=True)
-    transmitter_uuid = ShortUUIDField(db_index=True)
+    transmitter_uuid = ShortUUIDField(auto=False, db_index=True)
     transmitter_description = models.TextField(default='')
-    transmitter_type = models.CharField(choices=zip(TRANSMITTER_TYPE, TRANSMITTER_TYPE),
-                                        max_length=11, default='Transmitter')
+    transmitter_type = models.CharField(
+        choices=list(zip(TRANSMITTER_TYPE, TRANSMITTER_TYPE)),
+        max_length=11,
+        default='Transmitter'
+    )
     transmitter_uplink_low = models.BigIntegerField(blank=True, null=True)
     transmitter_uplink_high = models.BigIntegerField(blank=True, null=True)
     transmitter_uplink_drift = models.IntegerField(blank=True, null=True)
@@ -428,30 +443,41 @@ class Observation(models.Model):
 
     @property
     def is_past(self):
+        """Return true if observation is in the past (end time is in the past)"""
         return self.end < now()
 
     @property
     def is_future(self):
+        """Return true if observation is in the future (end time is in the future)"""
         return self.end > now()
 
+    @property
+    def is_started(self):
+        """Return true if observation has started (start time is in the past)"""
+        return self.start < now()
+
     # this payload has been vetted good/bad/failed by someone
     @property
     def is_vetted(self):
+        """Return true if observation is vetted"""
         return not self.vetted_status == 'unknown'
 
     # this payload has been vetted as good by someone
     @property
     def is_good(self):
+        """Return true if observation is vetted as good"""
         return self.vetted_status == 'good'
 
     # this payload has been vetted as bad by someone
     @property
     def is_bad(self):
+        """Return true if observation is vetted as bad"""
         return self.vetted_status == 'bad'
 
     # this payload has been vetted as failed by someone
     @property
     def is_failed(self):
+        """Return true if observation is vetted as failed"""
         return self.vetted_status == 'failed'
 
     @property
@@ -487,16 +513,18 @@ class Observation(models.Model):
 
     @property
     def audio_url(self):
+        """Return url for observation's audio file"""
         if self.has_audio:
             if self.archive_url:
-                r = requests.get(self.archive_url, allow_redirects=False)
                 try:
-                    url = r.headers['Location']
+                    request = requests.get(self.archive_url, allow_redirects=False)
+                    request.raise_for_status()
+
+                    url = request.headers['Location']
                     return url
-                except Exception:
+                except requests.exceptions.RequestException as error:
                     logger = logging.getLogger(__name__)
-                    logger.warning("Request to '%s' returned status code: %s",
-                                   r.url, r.status_code)
+                    logger.warning("Error in request to '%s'. Error: %s", self.archive_url, error)
                     return ''
             else:
                 return self.payload.url
@@ -504,16 +532,19 @@ class Observation(models.Model):
 
     class Meta:
         ordering = ['-start', '-end']
+        indexes = [models.Index(fields=['-start', '-end'])]
 
-    def __unicode__(self):
+    def __str__(self):
         return str(self.id)
 
     def get_absolute_url(self):
-        return reverse('base:observation_view', kwargs={'id': self.id})
+        """Return absolute url of the model object"""
+        return reverse('base:observation_view', kwargs={'observation_id': self.id})
 
 
 @receiver(models.signals.post_delete, sender=Observation)
-def observation_remove_files(sender, instance, **kwargs):
+def observation_remove_files(sender, instance, **kwargs):  # pylint: disable=W0613
+    """Remove audio and waterfall files of an observation if the observation is deleted"""
     if instance.payload:
         if os.path.isfile(instance.payload.path):
             os.remove(instance.payload.path)
@@ -525,33 +556,54 @@ def observation_remove_files(sender, instance, **kwargs):
 post_save.connect(_observation_post_save, sender=Observation)
 
 
+@python_2_unicode_compatible
 class DemodData(models.Model):
-    observation = models.ForeignKey(Observation, related_name='demoddata',
-                                    on_delete=models.CASCADE)
+    """Model for DemodData."""
+    observation = models.ForeignKey(
+        Observation, related_name='demoddata', on_delete=models.CASCADE
+    )
     payload_demod = models.FileField(upload_to=_name_obs_demoddata, unique=True)
     copied_to_db = models.BooleanField(default=False)
 
     def is_image(self):
-        with open(self.payload_demod.path) as fp:
+        """Return true if data file is an image"""
+        with open(self.payload_demod.path) as file_path:
             try:
-                Image.open(fp)
+                Image.open(file_path)
             except (IOError, TypeError):
                 return False
             else:
                 return True
 
-    def display_payload(self):
-        with open(self.payload_demod.path) as fp:
-            payload = fp.read()
-            try:
-                return unicode(payload)
-            except UnicodeDecodeError:
-                data = payload.encode('hex').upper()
-                return ' '.join(data[i:i + 2] for i in xrange(0, len(data), 2))
+    def display_payload_hex(self):
+        """
+        Return the content of the data file as hex dump of the following form: `DE AD C0 DE`.
+        """
+        with open(self.payload_demod.path) as data_file:
+            payload = data_file.read()
+
+        return _decode_pretty_hex(payload)
+
+    def display_payload_utf8(self):
+        """
+        Return the content of the data file decoded as UTF-8. If this fails,
+        show as hex dump.
+        """
+        with open(self.payload_demod.path) as data_file:
+            payload = data_file.read()
+
+        try:
+            return payload.decode('utf-8')
+        except UnicodeDecodeError:
+            return _decode_pretty_hex(payload)
+
+    def __str__(self):
+        return '{} - {}'.format(self.id, self.payload_demod)
 
 
 @receiver(models.signals.post_delete, sender=DemodData)
-def demoddata_remove_files(sender, instance, **kwargs):
+def demoddata_remove_files(sender, instance, **kwargs):  # pylint: disable=W0613
+    """Remove data file of an observation if the observation is deleted"""
     if instance.payload_demod:
         if os.path.isfile(instance.payload_demod.path):
             os.remove(instance.payload_demod.path)

network/base/perms.py

@@ -1,6 +1,13 @@
+"""SatNOGS Network base permissions"""
+from __future__ import absolute_import
+
 from django.core.exceptions import ObjectDoesNotExist
 
 
+class UserNoPermissionError(Exception):
+    """Error when user has not persmission"""
+
+
 def schedule_perms(user):
     """
     This context flag will determine if user can schedule an observation.
@@ -8,11 +15,12 @@ def schedule_perms(user):
     see: https://wiki.satnogs.org/Operation#Network_permissions_matrix
     """
     if user.is_authenticated():
+        stations_statuses = user.ground_stations.values_list('status', flat=True)
         # User has online station (status=2)
-        if user.ground_stations.filter(status=2).exists():
+        if 2 in stations_statuses:
             return True
         # User has testing station (status=1)
-        if user.ground_stations.filter(status=1).exists():
+        if 1 in stations_statuses:
             return True
         # User has special permissions
         if user.groups.filter(name='Moderators').exists():
@@ -34,14 +42,11 @@ def schedule_station_perms(user, station):
         try:
             if user.ground_stations.filter(status=2).exists() and station.status == 2:
                 return True
-        except (AttributeError, ObjectDoesNotExist):
+        except ObjectDoesNotExist:
             pass
         # If the station is testing (status=1) and user is its owner
-        try:
-            if station.status == 1 and station.owner == user:
-                return True
-        except (AttributeError, ObjectDoesNotExist):
-            pass
+        if station.status == 1 and station.owner == user:
+            return True
         # User has special permissions
         if user.groups.filter(name='Moderators').exists():
             return True
@@ -51,13 +56,60 @@ def schedule_station_perms(user, station):
     return False
 
 
+def schedule_stations_perms(user, stations):
+    """
+    This context flag will determine if user can schedule an observation.
+    That includes station owners, moderators, admins.
+    see: https://wiki.satnogs.org/Operation#Network_permissions_matrix
+    """
+    if user.is_authenticated():
+        # User has special permissions
+        if user.groups.filter(name='Moderators').exists():
+            return {station.id: True for station in stations}
+        if user.is_superuser:
+            return {station.id: True for station in stations}
+        # User has online station (status=2) and station is online
+        try:
+            if user.ground_stations.filter(status=2).exists():
+                return {
+                    s.id: s.status == 2 or (s.owner == user and s.status == 1)
+                    for s in stations
+                }
+
+        except ObjectDoesNotExist:
+            pass
+        # If the station is testing (status=1) and user is its owner
+        return {station.id: station.owner == user and station.status == 1 for station in stations}
+
+    return {station.id: False for station in stations}
+
+
+def check_schedule_perms_per_station(user, station_list):
+    """Checks if user has permissions to schedule on stations"""
+    stations_perms = schedule_stations_perms(user, station_list)
+    stations_without_permissions = [
+        int(station_id) for station_id in stations_perms.keys() if not stations_perms[station_id]
+    ]
+    if stations_without_permissions:
+        if len(stations_without_permissions) == 1:
+            raise UserNoPermissionError(
+                'No permission to schedule observations on station: {0}'.format(
+                    stations_without_permissions[0]
+                )
+            )
+        raise UserNoPermissionError(
+            'No permission to schedule observations on stations: {0}'.
+            format(stations_without_permissions)
+        )
+
+
 def delete_perms(user, observation):
     """
     This context flag will determine if a delete button appears for the observation.
     That includes observer, station owner involved, moderators, admins.
     see: https://wiki.satnogs.org/Operation#Network_permissions_matrix
     """
-    if user.is_authenticated():
+    if not observation.is_started and user.is_authenticated():
         # User owns the observation
         try:
             if observation.author == user:

network/base/scheduling.py

@@ -1,19 +1,25 @@
+"""SatNOGS Network scheduling functions"""
+# ephem is missing lon, lat, elevation and horizon attributes in Observer class slots,
+# Disable assigning-non-slot pylint error:
+# pylint: disable=E0237
+from __future__ import absolute_import, division
+
 import math
 from datetime import timedelta
 
-from django.conf import settings
-from django.utils.timezone import now, make_aware, utc
-from network.base.models import Satellite, Station, Tle, Observation
-from network.base.perms import schedule_station_perms
-
 import ephem
+from django.conf import settings
+from django.utils.timezone import make_aware, now, utc
+from past.utils import old_div
+
+from network.base.models import LatestTle, Observation, Satellite
+from network.base.perms import schedule_stations_perms
+from network.base.validators import NegativeElevationError, \
+    ObservationOverlapError, SinglePassError
 
 
-class ObservationOverlapError(Exception):
-    pass
-
-
-def get_elevation(observer, satellite, date):
+def get_altitude(observer, satellite, date):
+    """Returns altitude of satellite in a specific date for a specific observer"""
     observer = observer.copy()
     satellite = satellite.copy()
     observer.date = date
@@ -22,6 +28,7 @@ def get_elevation(observer, satellite, date):
 
 
 def get_azimuth(observer, satellite, date):
+    """Returns azimuth of satellite in a specific date for a specific observer"""
     observer = observer.copy()
     satellite = satellite.copy()
     observer.date = date
@@ -30,22 +37,23 @@ def get_azimuth(observer, satellite, date):
 
 
 def over_min_duration(duration):
+    """Returns if duration is bigger than the minimum one set in settings"""
     return duration > settings.OBSERVATION_DURATION_MIN
 
 
-def max_elevation_in_window(observer, satellite, pass_tca, window_start, window_end):
+def max_altitude_in_window(observer, satellite, pass_tca, window_start, window_end):
+    """Finds the maximum altitude of a satellite during a certain observation window"""
     # In this case this is an overlapped observation
-    # re-calculate elevation and start/end azimuth
+    # re-calculate altitude and start/end azimuth
     if window_start > pass_tca:
         # Observation window in the second half of the pass
-        # Thus highest elevation right at the beginning of the window
-        return get_elevation(observer, satellite, window_start)
-    elif window_end < pass_tca:
+        # Thus highest altitude right at the beginning of the window
+        return get_altitude(observer, satellite, window_start)
+    if window_end < pass_tca:
         # Observation window in the first half of the pass
-        # Thus highest elevation right at the end of the window
-        return get_elevation(observer, satellite, window_end)
-    else:
-        return get_elevation(observer, satellite, pass_tca)
+        # Thus highest altitude right at the end of the window
+        return get_altitude(observer, satellite, window_end)
+    return get_altitude(observer, satellite, pass_tca)
 
 
 def resolve_overlaps(scheduled_obs, start, end):
@@ -71,10 +79,12 @@ def resolve_overlaps(scheduled_obs, start, end):
                 if start < datum.start and end > datum.end:
                     # In case of splitting the window  to two we
                     # check for overlaps for each generated window.
-                    window1 = resolve_overlaps(scheduled_obs,
-                                               start, datum.start - timedelta(seconds=30))
-                    window2 = resolve_overlaps(scheduled_obs,
-                                               datum.end + timedelta(seconds=30), end)
+                    window1 = resolve_overlaps(
+                        scheduled_obs, start, datum.start - timedelta(seconds=30)
+                    )
+                    window2 = resolve_overlaps(
+                        scheduled_obs, datum.end + timedelta(seconds=30), end
+                    )
                     return (window1[0] + window2[0], True)
                 if datum.start <= start:
                     start = datum.end + timedelta(seconds=30)
@@ -83,19 +93,24 @@ def resolve_overlaps(scheduled_obs, start, end):
     return ([(start, end)], overlapped)
 
 
-def create_station_window(window_start, window_end, azr, azs, elevation, tle,
-                          valid_duration, overlapped, overlap_ratio=0):
-    return {'start': window_start.strftime("%Y-%m-%d %H:%M:%S.%f"),
-            'end': window_end.strftime("%Y-%m-%d %H:%M:%S.%f"),
-            'az_start': azr,
-            'az_end': azs,
-            'elev_max': elevation,
-            'tle0': tle.tle0,
-            'tle1': tle.tle1,
-            'tle2': tle.tle2,
-            'valid_duration': valid_duration,
-            'overlapped': overlapped,
-            'overlap_ratio': overlap_ratio}
+def create_station_window(
+        window_start, window_end, azr, azs, altitude, tle, valid_duration, overlapped,
+        overlap_ratio=0
+):
+    """Creates an observation window"""
+    return {
+        'start': window_start.strftime("%Y-%m-%d %H:%M:%S.%f"),
+        'end': window_end.strftime("%Y-%m-%d %H:%M:%S.%f"),
+        'az_start': azr,
+        'az_end': azs,
+        'elev_max': altitude,
+        'tle0': tle.tle0,
+        'tle1': tle.tle1,
+        'tle2': tle.tle2,
+        'valid_duration': valid_duration,
+        'overlapped': overlapped,
+        'overlap_ratio': overlap_ratio
+    }
 
 
 def create_station_windows(scheduled_obs, overlapped, pass_params, observer, satellite, tle):
@@ -108,39 +123,36 @@ def create_station_windows(scheduled_obs, overlapped, pass_params, observer, sat
     """
     station_windows = []
 
-    windows, windows_changed = resolve_overlaps(scheduled_obs,
-                                                pass_params['rise_time'],
-                                                pass_params['set_time'])
+    windows, windows_changed = resolve_overlaps(
+        scheduled_obs, pass_params['rise_time'], pass_params['set_time']
+    )
 
-    if len(windows) == 0:
-        # No nonrlapping windows found
+    if not windows:
+        # No overlapping windows found
         return []
     if windows_changed:
         # Windows changed due to overlap, recalculate observation parameters
         if overlapped == 0:
             return []
-        elif overlapped == 1:
+
+        if overlapped == 1:
             initial_duration = (pass_params['set_time'] - pass_params['rise_time']).total_seconds()
             for window_start, window_end in windows:
-                elevation = max_elevation_in_window(observer, satellite,
-                                                    pass_params['tca_time'],
-                                                    window_start, window_end)
+                altitude = max_altitude_in_window(
+                    observer, satellite, pass_params['tca_time'], window_start, window_end
+                )
                 window_duration = (window_end - window_start).total_seconds()
                 if not over_min_duration(window_duration):
                     continue
 
                 # Add a window for a partial pass
-                station_windows.append(create_station_window(
-                    window_start,
-                    window_end,
-                    get_azimuth(observer, satellite, window_start),
-                    get_azimuth(observer, satellite, window_end),
-                    elevation,
-                    tle,
-                    True,
-                    True,
-                    min(1, 1 - (window_duration / initial_duration))
-                ))
+                station_windows.append(
+                    create_station_window(
+                        window_start, window_end, get_azimuth(observer, satellite, window_start),
+                        get_azimuth(observer, satellite, window_end), altitude, tle, True, True,
+                        min(1, 1 - (float(window_duration) / float(initial_duration)))
+                    )
+                )
         elif overlapped == 2:
             initial_duration = (pass_params['set_time'] - pass_params['rise_time']).total_seconds()
             total_window_duration = 0
@@ -152,67 +164,50 @@ def create_station_windows(scheduled_obs, overlapped, pass_params, observer, sat
                 total_window_duration += window_duration
 
             # Add a window for the overlapped pass
-            station_windows.append(create_station_window(
-                pass_params['rise_time'],
-                pass_params['set_time'],
-                pass_params['rise_az'],
-                pass_params['set_az'],
-                pass_params['tca_alt'],
-                tle,
-                duration_validity,
-                True,
-                min(1, 1 - (window_duration / initial_duration))
-            ))
+            station_windows.append(
+                create_station_window(
+                    pass_params['rise_time'], pass_params['set_time'], pass_params['rise_az'],
+                    pass_params['set_az'], pass_params['tca_alt'], tle, duration_validity, True,
+                    min(1, 1 - (float(window_duration) / float(initial_duration)))
+                )
+            )
     else:
         window_duration = (windows[0][1] - windows[0][0]).total_seconds()
         if over_min_duration(window_duration):
             # Add a window for a full pass
-            station_windows.append(create_station_window(
-                pass_params['rise_time'],
-                pass_params['set_time'],
-                pass_params['rise_az'],
-                pass_params['set_az'],
-                pass_params['tca_alt'],
-                tle,
-                True,
-                False,
-                0
-            ))
+            station_windows.append(
+                create_station_window(
+                    pass_params['rise_time'], pass_params['set_time'], pass_params['rise_az'],
+                    pass_params['set_az'], pass_params['tca_alt'], tle, True, False, 0
+                )
+            )
     return station_windows
 
 
-def next_pass(observer, satellite, issue105=False):
-    tr, azr, tt, altt, ts, azs = observer.next_pass(satellite)
+def next_pass(observer, satellite, singlepass=True):
+    """Returns the next pass of the satellite above the observer"""
+    rise_time, rise_az, tca_time, tca_alt, set_time, set_az = observer.next_pass(
+        satellite, singlepass
+    )
     # Convert output of pyephems.next_pass into processible values
-    pass_start = make_aware(ephem.Date(tr).datetime(), utc)
-    pass_end = make_aware(ephem.Date(ts).datetime(), utc)
-    pass_tca = make_aware(ephem.Date(tt).datetime(), utc)
-    pass_azr = float(format(math.degrees(azr), '.0f'))
-    pass_azs = float(format(math.degrees(azs), '.0f'))
-    pass_elevation = float(format(math.degrees(altt), '.0f'))
+    pass_start = make_aware(ephem.Date(rise_time).datetime(), utc)
+    pass_end = make_aware(ephem.Date(set_time).datetime(), utc)
+    pass_tca = make_aware(ephem.Date(tca_time).datetime(), utc)
+    pass_azr = float(format(math.degrees(rise_az), '.0f'))
+    pass_azs = float(format(math.degrees(set_az), '.0f'))
+    pass_altitude = float(format(math.degrees(tca_alt), '.0f'))
 
-    if ephem.Date(tr).datetime() > ephem.Date(ts).datetime():
-        # set time before rise time (bug in pyephem)
-        # https://github.com/brandon-rhodes/pyephem/issues/105
-        # move observer time after the current pass end
-        time_start_new = pass_end + timedelta(minutes=1)
-        observer.date = time_start_new.strftime("%Y-%m-%d %H:%M:%S.%f")
-        if issue105:
-            raise ValueError('Recursion of next_pass() due to\
-                              https://github.com/brandon-rhodes/pyephem/issues/105')
-        else:
-            return next_pass(observer, satellite, True)
-
-    return {'rise_time': pass_start,
-            'set_time': pass_end,
-            'tca_time': pass_tca,
-            'rise_az': pass_azr,
-            'set_az': pass_azs,
-            'tca_alt': pass_elevation}
+    return {
+        'rise_time': pass_start,
+        'set_time': pass_end,
+        'tca_time': pass_tca,
+        'rise_az': pass_azr,
+        'set_az': pass_azs,
+        'tca_alt': pass_altitude
+    }
 
 
-def predict_available_observation_windows(station, min_horizon, overlapped, tle,
-                                          start_date, end_date, sat):
+def predict_available_observation_windows(station, min_horizon, overlapped, tle, start, end):
     '''Calculate available observation windows for a certain station and satellite during
     the given time period.
 
@@ -223,12 +218,11 @@ def predict_available_observation_windows(station, min_horizon, overlapped, tle,
     :param overlapped: Calculate and return overlapped observations fully, truncated or not at all
     :type overlapped: integer values: 0 (no return), 1(truncated overlaps), 2(full overlaps)
     :param tle: Satellite current TLE
-    :param tle: Satellite current TLE
-    :type tle: array of 3 strings
-    :param start_date: Start datetime of scheduling period
-    :type start_date: datetime string in '%Y-%m-%d %H:%M'
-    :param end_date: End datetime of scheduling period
-    :type end_date: datetime string in '%Y-%m-%d %H:%M'
+    :type tle: Tle django.db.model.Model
+    :param start: Start datetime of scheduling period
+    :type start: datetime string in '%Y-%m-%d %H:%M'
+    :param end: End datetime of scheduling period
+    :type end: datetime string in '%Y-%m-%d %H:%M'
     :param sat: Satellite for scheduling
     :type sat: Satellite django.db.model.Model
 
@@ -236,14 +230,15 @@ def predict_available_observation_windows(station, min_horizon, overlapped, tle,
     '''
     passes_found = []
     station_windows = []
+    tle_as_str_array = tle.str_array
     # Initialize pyehem Satellite for propagation
-    satellite = ephem.readtle(*tle)
+    satellite = ephem.readtle(*tle_as_str_array)
     # Initialize pyephem Observer for propagation
     observer = ephem.Observer()
     observer.lon = str(station.lng)
     observer.lat = str(station.lat)
     observer.elevation = station.alt
-    observer.date = ephem.Date(start_date)
+    observer.date = ephem.Date(start)
     if min_horizon is not None:
         observer.horizon = str(min_horizon)
     else:
@@ -261,11 +256,11 @@ def predict_available_observation_windows(station, min_horizon, overlapped, tle,
             break
 
         # no match if the sat will not rise above the configured min horizon
-        if pass_params['rise_time'] >= end_date:
+        if pass_params['rise_time'] >= end:
             # start of next pass outside of window bounds
             break
 
-        if pass_params['set_time'] > end_date:
+        if pass_params['set_time'] > end:
             # end of next pass outside of window bounds
             break
 
@@ -280,68 +275,124 @@ def predict_available_observation_windows(station, min_horizon, overlapped, tle,
 
         # Check if overlaps with existing scheduled observations
         # Adjust or discard window if overlaps exist
-        scheduled_obs = Observation.objects \
-            .filter(ground_station=station) \
-            .filter(end__gt=now())
+        scheduled_obs = station.scheduled_obs
 
-        station_windows.extend(create_station_windows(scheduled_obs, overlapped, pass_params,
-                                                      observer, satellite, sat.latest_tle))
+        station_windows.extend(
+            create_station_windows(
+                scheduled_obs, overlapped, pass_params, observer, satellite, tle
+            )
+        )
     return passes_found, station_windows
 
 
-def create_new_observation(station_id, sat_id, transmitter, start_time, end_time, author):
-    ground_station = Station.objects.get(id=station_id)
-    scheduled_obs = Observation.objects.filter(ground_station=ground_station).filter(end__gt=now())
-    window = resolve_overlaps(scheduled_obs, start_time, end_time)
+def create_new_observation(station, transmitter, start, end, author):
+    """
+    Creates and returns a new Observation object
+
+    Arguments:
+    station - network.base.models.Station
+    transmitter - network.base.models.Transmitter
+    start - datetime
+    end - datetime
+    author - network.base.models.User
+
+    Returns network.base.models.Observation
+    Raises NegativeElevationError, ObservationOverlapError, SinglePassError or more
+    """
+    scheduled_obs = Observation.objects.filter(ground_station=station).filter(end__gt=now())
+    window = resolve_overlaps(scheduled_obs, start, end)
 
     if window[1]:
-        raise ObservationOverlapError
+        raise ObservationOverlapError(
+            'One or more observations of station {0} overlap with the already scheduled ones.'.
+            format(station.id)
+        )
 
-    sat = Satellite.objects.get(norad_cat_id=sat_id)
-    tle = Tle.objects.get(id=sat.latest_tle.id)
+    sat = Satellite.objects.get(norad_cat_id=transmitter['norad_cat_id'])
+    tle = LatestTle.objects.get(satellite_id=sat.id)
 
-    sat_ephem = ephem.readtle(str(sat.latest_tle.tle0),
-                              str(sat.latest_tle.tle1),
-                              str(sat.latest_tle.tle2))
+    sat_ephem = ephem.readtle(str(tle.tle0), str(tle.tle1), str(tle.tle2))
     observer = ephem.Observer()
-    observer.lon = str(ground_station.lng)
-    observer.lat = str(ground_station.lat)
-    observer.elevation = ground_station.alt
+    observer.lon = str(station.lng)
+    observer.lat = str(station.lat)
+    observer.elevation = station.alt
 
-    mid_pass_time = start_time + (end_time - start_time) / 2
+    # Replace with the following after Python 3 migration:
+    # mid_pass_time = start + (end - start) / 2
+    mid_pass_time = start + old_div((end - start), 2)
 
-    rise_azimuth = get_azimuth(observer, sat_ephem, start_time)
-    max_altitude = get_elevation(observer, sat_ephem, mid_pass_time)
-    set_azimuth = get_azimuth(observer, sat_ephem, end_time)
+    rise_azimuth = get_azimuth(observer, sat_ephem, start)
+    rise_altitude = get_altitude(observer, sat_ephem, start)
+    max_altitude = get_altitude(observer, sat_ephem, mid_pass_time)
+    set_azimuth = get_azimuth(observer, sat_ephem, end)
+    set_altitude = get_altitude(observer, sat_ephem, end)
+
+    if rise_altitude < 0:
+        raise NegativeElevationError(
+            "Satellite with transmitter {} has negative altitude ({})"
+            " for station {} at start datetime: {}".format(
+                transmitter['uuid'], rise_altitude, station.id, start
+            )
+        )
+    if set_altitude < 0:
+        raise NegativeElevationError(
+            "Satellite with transmitter {} has negative altitude ({})"
+            " for station {} at end datetime: {}".format(
+                transmitter['uuid'], set_altitude, station.id, end
+            )
+        )
+    # Using a short time (1min later) after start for finding the next pass of the satellite to
+    # check that end datetime is before the start datetime of the next pass, in other words that
+    # end time belongs to the same single pass.
+    observer.date = start + timedelta(minutes=1)
+    next_satellite_pass = next_pass(observer, sat_ephem, False)
+    if next_satellite_pass['rise_time'] < end:
+        raise SinglePassError(
+            "Observation should include only one pass of the satellite with transmitter {}"
+            " on station {}, please check start({}) and end({}) datetimes and try again".format(
+                transmitter['uuid'], station.id, start, end
+            )
+        )
 
     return Observation(
-        satellite=sat, tle=tle, author=author, start=start_time, end=end_time,
-        ground_station=ground_station, rise_azimuth=rise_azimuth, max_altitude=max_altitude,
-        set_azimuth=set_azimuth, transmitter_uuid=transmitter['uuid'],
-        transmitter_description=transmitter['description'], transmitter_type=transmitter['type'],
+        satellite=sat,
+        tle=tle,
+        author=author,
+        start=start,
+        end=end,
+        ground_station=station,
+        rise_azimuth=rise_azimuth,
+        max_altitude=max_altitude,
+        set_azimuth=set_azimuth,
+        transmitter_uuid=transmitter['uuid'],
+        transmitter_description=transmitter['description'],
+        transmitter_type=transmitter['type'],
         transmitter_uplink_low=transmitter['uplink_low'],
         transmitter_uplink_high=transmitter['uplink_high'],
         transmitter_uplink_drift=transmitter['uplink_drift'],
         transmitter_downlink_low=transmitter['downlink_low'],
         transmitter_downlink_high=transmitter['downlink_high'],
         transmitter_downlink_drift=transmitter['downlink_drift'],
-        transmitter_mode=transmitter['mode'], transmitter_invert=transmitter['invert'],
-        transmitter_baud=transmitter['baud'], transmitter_created=transmitter['updated']
+        transmitter_mode=transmitter['mode'],
+        transmitter_invert=transmitter['invert'],
+        transmitter_baud=transmitter['baud'],
+        transmitter_created=transmitter['updated']
     )
 
 
 def get_available_stations(stations, downlink, user):
+    """Returns stations for scheduling filtered by a specific downlink and user's permissions"""
     available_stations = []
-    for station in stations:
-        if not schedule_station_perms(user, station):
-            continue
+    stations_perms = schedule_stations_perms(user, stations)
+    stations_with_permissions = [station for station in stations if stations_perms[station.id]]
+    for station in stations_with_permissions:
 
         # Skip if this station is not capable of receiving the frequency
         if not downlink:
             continue
         frequency_supported = False
         for gs_antenna in station.antenna.all():
-            if (gs_antenna.frequency <= downlink <= gs_antenna.frequency_max):
+            if gs_antenna.frequency <= downlink <= gs_antenna.frequency_max:
                 frequency_supported = True
         if not frequency_supported:
             continue

network/base/stats.py

@@ -1,93 +1,105 @@
+"""Module for calculating and keep in cache satellite and transmitter statistics"""
+from __future__ import absolute_import, division
+
+import math
+
 from django.core.cache import cache
+from django.db.models import Case, IntegerField, Sum, When
+from django.utils.timezone import now
 
 from network.base.models import Observation
 
 
-def transmitter_total_count(uuid):
-    return Observation.objects.filter(transmitter_uuid=uuid) \
-                              .exclude(vetted_status='failed') \
-                              .count()
-
-
-def transmitter_good_count(uuid):
-    data = cache.get('tr-{0}-suc-count'.format(uuid))
-    if data is None:
-        obs = Observation.objects.filter(transmitter_uuid=uuid)
-        data = obs.filter(vetted_status='good').count()
-        cache.set('tr-{0}-suc-count'.format(uuid), data, 3600)
-        return data
-    return data
-
-
-def transmitter_bad_count(uuid):
-    data = cache.get('tr-{0}-bad-count'.format(uuid))
-    if data is None:
-        obs = Observation.objects.filter(transmitter_uuid=uuid)
-        data = obs.filter(vetted_status='bad').count()
-        cache.set('tr-{0}-bad-count'.format(uuid), data, 3600)
-        return data
-    return data
-
-
-def transmitter_unvetted_count(uuid):
-    data = cache.get('tr-{0}-unk-count'.format(uuid))
-    if data is None:
-        obs = Observation.objects.filter(transmitter_uuid=uuid)
-        data = obs.filter(vetted_status='unknown').count()
-        cache.set('tr-{0}-unk-count'.format(uuid), data, 3600)
-        return data
-    return data
-
-
-def transmitter_success_rate(uuid):
-    rate = cache.get('tr-{0}-suc-rate'.format(uuid))
-    if rate is None:
-        try:
-            ratio = float(transmitter_good_count(uuid)) / float(transmitter_total_count(uuid))
-            rate = int(100 * ratio)
-            cache.set('tr-{0}-suc-rate'.format(uuid), rate, 3600)
-            return rate
-        except (ZeroDivisionError, TypeError):
-            cache.set('tr-{0}-suc-rate'.format(uuid), 0, 3600)
-            return 0
-    return rate
-
-
-def transmitter_bad_rate(uuid):
-    rate = cache.get('tr-{0}-bad-rate'.format(uuid))
-    if rate is None:
-        try:
-            ratio = float(transmitter_bad_count(uuid)) / float(transmitter_total_count(uuid))
-            rate = int(100 * ratio)
-            cache.set('tr-{0}-bad-rate'.format(uuid), rate, 3600)
-            return rate
-        except (ZeroDivisionError, TypeError):
-            cache.set('tr-{0}-bad-rate'.format(uuid), 0, 3600)
-            return 0
-    return rate
-
-
-def transmitter_unvetted_rate(uuid):
-    rate = cache.get('tr-{0}-unk-rate'.format(uuid))
-    if rate is None:
-        try:
-            ratio = float(transmitter_unvetted_count(uuid)) / float(transmitter_total_count(uuid))
-            rate = int(100 * ratio)
-            cache.set('tr-{0}-unk-rate'.format(uuid), rate, 3600)
-            return rate
-        except (ZeroDivisionError, TypeError):
-            cache.set('tr-{0}-unk-rate'.format(uuid), 0, 3600)
-            return 0
-    return rate
-
-
 def transmitter_stats_by_uuid(uuid):
+    """Calculate and put in cache transmitter statistics"""
+    stats = cache.get('tr-{0}-stats'.format(uuid))
+    if stats is None:
+        # Sum - Case - When should be replaced with Count and filter when we move to Django 2.*
+        # more at https://docs.djangoproject.com/en/2.2/ref/models/conditional-expressions in
+        # "Conditional aggregation" section.
+        stats = Observation.objects.filter(transmitter_uuid=uuid).exclude(
+            vetted_status='failed'
+        ).aggregate(
+            good=Sum(Case(When(vetted_status='good', then=1), output_field=IntegerField())),
+            bad=Sum(Case(When(vetted_status='bad', then=1), output_field=IntegerField())),
+            unvetted=Sum(
+                Case(
+                    When(vetted_status='unknown', end__lte=now(), then=1),
+                    output_field=IntegerField()
+                )
+            ),
+            future=Sum(
+                Case(
+                    When(vetted_status='unknown', end__gt=now(), then=1),
+                    output_field=IntegerField()
+                )
+            )
+        )
+        cache.set('tr-{0}-stats'.format(uuid), stats, 3600)
+
+    total_count = 0
+    unvetted_count = 0 if stats['unvetted'] is None else stats['unvetted']
+    future_count = 0 if stats['future'] is None else stats['future']
+    good_count = 0 if stats['good'] is None else stats['good']
+    bad_count = 0 if stats['bad'] is None else stats['bad']
+    total_count = unvetted_count + future_count + good_count + bad_count
+    unvetted_rate = 0
+    future_rate = 0
+    success_rate = 0
+    bad_rate = 0
+
+    if total_count:
+        unvetted_rate = math.trunc(10000 * (float(unvetted_count) / float(total_count))) / 100.0
+        future_rate = math.trunc(10000 * (float(future_count) / float(total_count))) / 100.0
+        success_rate = math.trunc(10000 * (float(good_count) / float(total_count))) / 100.0
+        bad_rate = math.trunc(10000 * (float(bad_count) / float(total_count))) / 100.0
+
     return {
-        'total_count': transmitter_total_count(uuid),
-        'unvetted_count': transmitter_unvetted_count(uuid),
-        'good_count': transmitter_good_count(uuid),
-        'bad_count': transmitter_bad_count(uuid),
-        'success_rate': transmitter_success_rate(uuid),
-        'bad_rate': transmitter_bad_rate(uuid),
-        'unvetted_rate': transmitter_unvetted_rate(uuid)
+        'total_count': total_count,
+        'unvetted_count': unvetted_count,
+        'future_count': future_count,
+        'good_count': good_count,
+        'bad_count': bad_count,
+        'unvetted_rate': unvetted_rate,
+        'future_rate': future_rate,
+        'success_rate': success_rate,
+        'bad_rate': bad_rate
+    }
+
+
+def satellite_stats_by_transmitter_list(transmitter_list):
+    """Calculate satellite statistics"""
+    total_count = 0
+    unvetted_count = 0
+    future_count = 0
+    good_count = 0
+    bad_count = 0
+    unvetted_rate = 0
+    future_rate = 0
+    success_rate = 0
+    bad_rate = 0
+    for transmitter in transmitter_list:
+        transmitter_stats = transmitter_stats_by_uuid(transmitter['uuid'])
+        total_count += transmitter_stats['total_count']
+        unvetted_count += transmitter_stats['unvetted_count']
+        future_count += transmitter_stats['future_count']
+        good_count += transmitter_stats['good_count']
+        bad_count += transmitter_stats['bad_count']
+
+    if total_count:
+        unvetted_rate = math.trunc(10000 * (float(unvetted_count) / float(total_count))) / 100.0
+        future_rate = math.trunc(10000 * (float(future_count) / float(total_count))) / 100.0
+        success_rate = math.trunc(10000 * (float(good_count) / float(total_count))) / 100.0
+        bad_rate = math.trunc(10000 * (float(bad_count) / float(total_count))) / 100.0
+
+    return {
+        'total_count': total_count,
+        'unvetted_count': unvetted_count,
+        'future_count': future_count,
+        'good_count': good_count,
+        'bad_count': bad_count,
+        'unvetted_rate': unvetted_rate,
+        'future_rate': future_rate,
+        'success_rate': success_rate,
+        'bad_rate': bad_rate
     }

network/base/tasks.py

@@ -1,97 +1,133 @@
-from datetime import timedelta
-import json
+"""SatNOGS Network Celery task functions"""
+from __future__ import absolute_import, division, print_function
+
 import os
-from requests.exceptions import ReadTimeout, HTTPError
-import urllib2
-
-from internetarchive import upload
-from orbit import satellite
+from datetime import timedelta
 
+import requests
+from celery import shared_task
 from django.conf import settings
 from django.contrib.sites.models import Site
 from django.core.cache import cache
 from django.core.mail import send_mail
+from django.db.models import Prefetch
 from django.utils.timezone import now
+from internetarchive import upload
+from requests.exceptions import HTTPError, ReadTimeout  # pylint: disable=C0412
+from satellite_tle import fetch_tles
 
-from network.base.models import Satellite, Tle, Transmitter, Observation, Station, DemodData
-from network.celery import app
-from network.base.utils import demod_to_db
+from network.base.models import DemodData, LatestTle, Observation, Satellite, \
+    Station, Tle, Transmitter
+from network.base.utils import sync_demoddata_to_db
 
 
-@app.task(ignore_result=True)
+@shared_task
 def update_all_tle():
     """Task to update all satellite TLEs"""
-    satellites = Satellite.objects.exclude(manual_tle=True,
-                                           norad_follow_id__isnull=True)
+    latest_tle_queryset = LatestTle.objects.all()
+    satellites = Satellite.objects.exclude(status='re-entered').exclude(
+        manual_tle=True, norad_follow_id__isnull=True
+    ).prefetch_related(Prefetch('tles', queryset=latest_tle_queryset, to_attr='tle'))
 
-    print "==Fetching TLEs=="
+    # Collect all norad ids we are interested in
+    norad_ids = set()
+    for obj in satellites:
+        norad_id = obj.norad_cat_id
+        if obj.manual_tle:
+            norad_id = obj.norad_follow_id
+        norad_ids.add(int(norad_id))
+
+    # Filter only officially announced NORAD IDs
+    catalog_norad_ids = {norad_id for norad_id in norad_ids if norad_id < 99000}
+
+    print("==Fetching TLEs==")
+    tles = fetch_tles(catalog_norad_ids)
 
     for obj in satellites:
         norad_id = obj.norad_cat_id
-        if (obj.manual_tle):
+        if obj.manual_tle:
             norad_id = obj.norad_follow_id
 
-        try:
-            # Fetch latest satellite TLE
-            sat = satellite(norad_id)
-        except IndexError:
-            print '{} - {}: TLE not found [error]'.format(obj.name, norad_id)
+        if norad_id not in list(tles.keys()):
+            # No TLE available for this satellite
+            print('{} - {}: NORAD ID not found [error]'.format(obj.name, norad_id))
             continue
 
-        tle = sat.tle()
+        source, tle = tles[norad_id]
 
-        try:
-            latest_tle = obj.latest_tle.tle1
-            if latest_tle == tle[1]:
-                # Stored TLE is already the latest available for this satellite
-                print '{} - {}: TLE already exists [defer]'.format(obj.name, norad_id)
-                continue
-        except AttributeError:
-            # Satellite in DB has no associated TLE yet
-            pass
+        if obj.tle and obj.tle[0].tle1 == tle[1]:
+            # Stored TLE is already the latest available for this satellite
+            print('{} - {}: TLE already exists [defer]'.format(obj.name, norad_id))
+            continue
 
         Tle.objects.create(tle0=tle[0], tle1=tle[1], tle2=tle[2], satellite=obj)
-        print '{} - {}: new TLE found [updated]'.format(obj.name, norad_id)
+        print('{} - {} - {}: new TLE found [updated]'.format(obj.name, norad_id, source))
 
 
-@app.task(ignore_result=True)
+@shared_task
 def fetch_data():
-    """Task to fetch all data from DB"""
-    apiurl = settings.DB_API_ENDPOINT
-    if len(apiurl) == 0:
-        return
-    satellites_url = "{0}satellites".format(apiurl)
-    transmitters_url = "{0}transmitters".format(apiurl)
+    """Fetch all satellites and transmitters from SatNOGS DB
 
-    try:
-        satellites = urllib2.urlopen(satellites_url).read()
-        transmitters = urllib2.urlopen(transmitters_url).read()
-    except urllib2.URLError:
-        raise Exception('API is unreachable')
+       Throws: requests.exceptions.ConectionError"""
+
+    db_api_url = settings.DB_API_ENDPOINT
+    if not db_api_url:
+        print("Zero length api url, fetching is stopped")
+        return
+    satellites_url = "{}satellites".format(db_api_url)
+    transmitters_url = "{}transmitters".format(db_api_url)
+
+    print("Fetching Satellites from {}".format(satellites_url))
+    r_satellites = requests.get(satellites_url)
+
+    print("Fetching Transmitters from {}".format(transmitters_url))
+    r_transmitters = requests.get(transmitters_url)
 
     # Fetch Satellites
-    for sat in json.loads(satellites):
-        norad_cat_id = sat['norad_cat_id']
-        sat.pop('decayed', None)
+    satellites_added = 0
+    satellites_updated = 0
+    for satellite in r_satellites.json():
+        norad_cat_id = satellite['norad_cat_id']
+        satellite.pop('decayed', None)
         try:
+            # Update Satellite
             existing_satellite = Satellite.objects.get(norad_cat_id=norad_cat_id)
-            existing_satellite.__dict__.update(sat)
+            existing_satellite.__dict__.update(satellite)
             existing_satellite.save()
+            satellites_updated += 1
         except Satellite.DoesNotExist:
-            Satellite.objects.create(**sat)
+            # Add Satellite
+            satellite.pop('telemetries', None)
+            Satellite.objects.create(**satellite)
+            satellites_added += 1
+
+    print('Added/Updated {}/{} satellites from db.'.format(satellites_added, satellites_updated))
 
     # Fetch Transmitters
-    for transmitter in json.loads(transmitters):
+    transmitters_added = 0
+    transmitters_skipped = 0
+    for transmitter in r_transmitters.json():
         uuid = transmitter['uuid']
 
         try:
+            # Transmitter already exists, skip
             Transmitter.objects.get(uuid=uuid)
+            transmitters_skipped += 1
         except Transmitter.DoesNotExist:
+            # Create Transmitter
             Transmitter.objects.create(uuid=uuid)
+            transmitters_added += 1
+
+    print(
+        'Added/Skipped {}/{} transmitters from db.'.format(
+            transmitters_added, transmitters_skipped
+        )
+    )
 
 
-@app.task(ignore_result=True)
+@shared_task
 def archive_audio(obs_id):
+    """Upload audio of observation in archive.org"""
     obs = Observation.objects.get(id=obs_id)
     suffix = '-{0}'.format(settings.ENVIRONMENT)
     if settings.ENVIRONMENT == 'production':
@@ -101,18 +137,27 @@ def archive_audio(obs_id):
     ogg = obs.payload.path
     filename = obs.payload.name.split('/')[-1]
     site = Site.objects.get_current()
-    description = ('<p>Audio file from SatNOGS{0} <a href="{1}/observations/{2}">'
-                   'Observation {3}</a>.</p>').format(suffix, site.domain,
-                                                      obs.id, obs.id)
-    md = dict(collection=settings.ARCHIVE_COLLECTION,
-              title=identifier,
-              mediatype='audio',
-              licenseurl='http://creativecommons.org/licenses/by-sa/4.0/',
-              description=description)
+    description = (
+        '<p>Audio file from SatNOGS{0} <a href="{1}/observations/{2}">'
+        'Observation {3}</a>.</p>'
+    ).format(suffix, site.domain, obs.id, obs.id)
+    metadata = dict(
+        collection=settings.ARCHIVE_COLLECTION,
+        title=identifier,
+        mediatype='audio',
+        licenseurl='http://creativecommons.org/licenses/by-sa/4.0/',
+        description=description
+    )
     try:
-        res = upload(identifier, files=[ogg], metadata=md,
-                     access_key=settings.S3_ACCESS_KEY,
-                     secret_key=settings.S3_SECRET_KEY)
+        res = upload(
+            identifier,
+            files=[ogg],
+            metadata=metadata,
+            access_key=settings.S3_ACCESS_KEY,
+            secret_key=settings.S3_SECRET_KEY,
+            retries=settings.S3_RETRIES_ON_SLOW_DOWN,
+            retries_sleep=settings.S3_RETRIES_SLEEP
+        )
     except (ReadTimeout, HTTPError):
         return
     if res[0].status_code == 200:
@@ -123,7 +168,7 @@ def archive_audio(obs_id):
         obs.payload.delete()
 
 
-@app.task(ignore_result=True)
+@shared_task
 def clean_observations():
     """Task to clean up old observations that lack actual data."""
     threshold = now() - timedelta(days=int(settings.OBSERVATION_OLD_RANGE))
@@ -133,29 +178,31 @@ def clean_observations():
         if settings.ENVIRONMENT == 'stage':
             if not obs.is_good:
                 obs.delete()
-                return
+                continue
         if os.path.isfile(obs.payload.path):
             archive_audio.delay(obs.id)
 
 
-@app.task
+@shared_task
 def sync_to_db():
-    """Task to send demod data to db / SiDS"""
-    q = now() - timedelta(days=1)
+    """Task to send demod data to SatNOGS DB / SiDS"""
     transmitters = Transmitter.objects.filter(sync_to_db=True).values_list('uuid', flat=True)
-    frames = DemodData.objects.filter(observation__end__gte=q,
-                                      copied_to_db=False,
-                                      observation__transmitter_uuid__in=transmitters)
+
+    frames = DemodData.objects.filter(
+        copied_to_db=False, observation__transmitter_uuid__in=transmitters
+    )
     for frame in frames:
+        if frame.is_image() or frame.copied_to_db or not os.path.isfile(frame.payload_demod.path):
+            continue
+
         try:
-            if not frame.is_image() and not frame.copied_to_db:
-                if os.path.isfile(frame.payload_demod.path):
-                    demod_to_db(frame.id)
-        except Exception:
+            sync_demoddata_to_db(frame.id)
+        except requests.exceptions.RequestException:
+            # Sync to db failed, skip this frame for a future task instance
             continue
 
 
-@app.task(ignore_result=True)
+@shared_task
 def station_status_update():
     """Task to update Station status."""
     for station in Station.objects.all():
@@ -168,18 +215,19 @@ def station_status_update():
         station.save()
 
 
-@app.task(ignore_result=True)
+@shared_task
 def notify_for_stations_without_results():
     """Task to send email for stations with observations without results."""
     email_to = settings.EMAIL_FOR_STATIONS_ISSUES
-    if email_to is not None and len(email_to) > 0:
+    if email_to:
         stations = ''
         obs_limit = settings.OBS_NO_RESULTS_MIN_COUNT
         time_limit = now() - timedelta(seconds=settings.OBS_NO_RESULTS_IGNORE_TIME)
         last_check = time_limit - timedelta(seconds=settings.OBS_NO_RESULTS_CHECK_PERIOD)
         for station in Station.objects.filter(status=2):
-            last_obs = Observation.objects.filter(ground_station=station,
-                                                  end__lt=time_limit).order_by("-end")[:obs_limit]
+            last_obs = Observation.objects.filter(
+                ground_station=station, end__lt=time_limit
+            ).order_by("-end")[:obs_limit]
             obs_without_results = 0
             obs_after_last_check = False
             for observation in last_obs:
@@ -189,20 +237,24 @@ def notify_for_stations_without_results():
                     obs_after_last_check = True
             if obs_without_results == obs_limit and obs_after_last_check:
                 stations += ' ' + str(station.id)
-        if len(stations) > 0:
+        if stations:
             # Notify user
             subject = '[satnogs] Station with observations without results'
-            send_mail(subject, stations, settings.DEFAULT_FROM_EMAIL,
-                      [settings.EMAIL_FOR_STATIONS_ISSUES], False)
+            send_mail(
+                subject, stations, settings.DEFAULT_FROM_EMAIL,
+                [settings.EMAIL_FOR_STATIONS_ISSUES], False
+            )
 
 
-@app.task(ignore_result=True)
+@shared_task
 def stations_cache_rates():
+    """Cache the success rate of the stations"""
     stations = Station.objects.all()
     for station in stations:
         observations = station.observations.exclude(testing=True).exclude(vetted_status="unknown")
-        success = observations.filter(id__in=(o.id for o in observations
-                                              if o.is_good or o.is_bad)).count()
+        success = observations.filter(
+            id__in=(o.id for o in observations if o.is_good or o.is_bad)
+        ).count()
         if observations:
             rate = int(100 * (float(success) / float(observations.count())))
             cache.set('station-{0}-rate'.format(station.id), rate, 60 * 60 * 2)

network/base/templatetags/paginator.py

@@ -2,19 +2,21 @@
 Tag to load a smart paginator that does not iterate over every page but has
 ellipses to truncate pages outside an adjacency
 """
-from django import template
+from __future__ import absolute_import
 
+from django import template
 
 register = template.Library()
 
 
+@register.inclusion_tag('base/paginator.html', takes_context=True)
 def paginator(context, request, adjacent_pages=2):
     """
     Adds pagination context variables for use in displaying first, adjacent and
     last page links.
     """
-    paginator = context['paginator']
     page_obj = context['page_obj']
+    num_pages = context['paginator'].num_pages
 
     # Widen the adjacency if near the start or near the end of the page set
     start_page = max(page_obj.number - adjacent_pages, 1)
@@ -22,21 +24,17 @@ def paginator(context, request, adjacent_pages=2):
         start_page = 1
 
     end_page = page_obj.number + adjacent_pages + 1
-    if end_page >= paginator.num_pages - 1:
-        end_page = paginator.num_pages + 1
+    if end_page >= num_pages - 1:
+        end_page = num_pages + 1
 
     # Generate a list of pages to include in the paginator template
-    page_numbers = [n for n in range(start_page, end_page)
-                    if n > 0 and n <= paginator.num_pages]
+    page_numbers = [n for n in range(start_page, end_page) if 0 < n <= num_pages]
 
     return {
         'page_obj': page_obj,
-        'paginator': paginator,
+        'paginator': context['paginator'],
         'page_numbers': page_numbers,
         'show_first': 1 not in page_numbers,
-        'show_last': paginator.num_pages not in page_numbers,
+        'show_last': num_pages not in page_numbers,
         'request': request
     }
-
-
-register.inclusion_tag('base/paginator.html', takes_context=True)(paginator)

network/base/templatetags/tags.py

@@ -1,18 +1,30 @@
+"""Django template tags for SatNOGS Network"""
+from __future__ import absolute_import, division
+
 from django import template
 from django.core.urlresolvers import reverse
+from future.builtins import round
 
 register = template.Library()
 
 
 @register.simple_tag
 def active(request, urls):
+    """Returns if this is an active URL"""
     if request.path in (reverse(url) for url in urls.split()):
         return 'active'
     return None
 
 
+@register.simple_tag
+def drifted_frq(value, drift):
+    """Returns drifred frequency"""
+    return int(round(value + ((value * drift) / 1e9)))
+
+
 @register.filter
 def frq(value):
+    """Returns Hz formatted frequency html string"""
     try:
         to_format = float(value)
     except (TypeError, ValueError):
@@ -24,6 +36,7 @@ def frq(value):
 
 @register.filter
 def percentagerest(value):
+    """Returns the rest of percentage from a given (percentage) value"""
     try:
         return 100 - value
     except (TypeError, ValueError):
@@ -31,15 +44,16 @@ def percentagerest(value):
 
 
 @register.filter
-def truncatesecs(value):
-    try:
-        return value[:-3]
-    except (TypeError, ValueError):
-        return value
+def get_count_from_id(dictionary, key):
+    """Returns observations count from dictionary"""
+    if key in dictionary.keys():
+        return dictionary[key]
+    return 0
 
 
 @register.filter
 def sortdemoddata(demoddata):
+    """Returns a date sorted list of DemodData"""
     try:
         return sorted(list(demoddata), key=lambda x: str(x.payload_demod).split('/', 2)[2:])
     except (TypeError, ValueError):

network/base/templatetags/url_replace.py

@@ -1,5 +1,7 @@
-from django import template
+"""Tag to replace field in GET request"""
+from __future__ import absolute_import
 
+from django import template
 
 register = template.Library()
 

network/base/test_utils.py

@@ -0,0 +1,67 @@
+"""Tests for miscellaneous functions for SatNOGS Network"""
+# pylint: disable=W0621
+
+from __future__ import absolute_import
+
+import pytest
+from network.base.utils import community_get_discussion_details
+
+
+@pytest.fixture
+def observation_with_discussion():
+    '''Parameters describing an observation for which a discussion exists in the forum'''
+    return {
+        'observation_id': 1445404,
+        'satellite_name': 'OSCAR 7',
+        'norad_cat_id': 7530,
+        'observation_url': 'https://network.satnogs.org/observations/1445404/'
+    }
+
+
+@pytest.fixture
+def observation_without_discussion():
+    '''Parameters describing an observation for which no discussion exists in the forum'''
+    return {
+        'observation_id': 1445405,
+        'satellite_name': 'CAS-4B',
+        'norad_cat_id': 42759,
+        'observation_url': 'https://network.satnogs.org/observations/1445405/'
+    }
+
+
+def test_community_get_discussion_details_with_discussion(observation_with_discussion):
+    '''
+    Test community_get_discussion_details returns details.has_comments=True and the proper urls
+    for the existing discussion in community (details.slug)
+    '''
+    details = community_get_discussion_details(**observation_with_discussion)
+
+    assert details == {
+        'url':
+        'https://community.libre.space/new-topic?title=Observation 1445404: OSCAR 7 (7530)&'
+        'body=Regarding [Observation 1445404](https://network.satnogs.org/observations/1445404/)'
+        '...&category=observations',
+        'slug':
+        'https://community.libre.space/t/observation-1445404-oscar-7-7530',
+        'has_comments':
+        True
+    }
+
+
+def test_community_get_discussion_details_without_discussion(observation_without_discussion):
+    '''
+    Test community_get_discussion_details returns details.has_comments=False and the proper urls
+    for creating a new discussion in community (details.url)
+    '''
+    details = community_get_discussion_details(**observation_without_discussion)
+
+    assert details == {
+        'url':
+        'https://community.libre.space/new-topic?title=Observation 1445405: CAS-4B (42759)&'
+        'body=Regarding [Observation 1445405](https://network.satnogs.org/observations/1445405/)'
+        '...&category=observations',
+        'slug':
+        'https://community.libre.space/t/observation-1445405-cas-4b-42759',
+        'has_comments':
+        False
+    }

network/base/tests.py

@@ -1,26 +1,31 @@
+"""SatNOGS Network base test suites"""
+# pylint: disable=R0903
+from __future__ import absolute_import
+
 import random
 from datetime import datetime, timedelta
-import pytest
 
 import factory
-from factory import fuzzy
-
-from django.db import transaction
 from django.contrib.auth.models import Group
-from django.test import TestCase, Client
+from django.db import transaction
+from django.test import Client, TestCase
 from django.utils.timezone import now
+# C0412 below clashes with isort
+from factory import fuzzy  # pylint: disable=C0412
 
-from network.base.models import (ANTENNA_BANDS, ANTENNA_TYPES, OBSERVATION_STATUSES,
-                                 Antenna, Satellite, Tle, Station, Observation, DemodData)
+import pytest
+from network.base.models import ANTENNA_BANDS, ANTENNA_TYPES, \
+    OBSERVATION_STATUSES, Antenna, DemodData, Observation, Satellite, \
+    Station, Tle
 from network.users.tests import UserFactory
 
-
 ANTENNA_BAND_IDS = [c[0] for c in ANTENNA_BANDS]
 ANTENNA_TYPE_IDS = [c[0] for c in ANTENNA_TYPES]
 OBSERVATION_STATUS_IDS = [c[0] for c in OBSERVATION_STATUSES]
 
 
 def generate_payload():
+    """Create data payloads"""
     payload = '{0:b}'.format(random.randint(500000000, 510000000))
     digits = 1824
     while digits:
@@ -31,8 +36,10 @@ def generate_payload():
 
 
 def generate_payload_name():
-    filename = datetime.strftime(fuzzy.FuzzyDateTime(now() - timedelta(days=10), now()).fuzz(),
-                                 '%Y%m%dT%H%M%SZ')
+    """Create payload names"""
+    filename = datetime.strftime(
+        fuzzy.FuzzyDateTime(now() - timedelta(days=10), now()).fuzz(), '%Y%m%dT%H%M%SZ'
+    )
     return filename
 
 
@@ -61,7 +68,8 @@ class StationFactory(factory.django.DjangoModelFactory):
     horizon = fuzzy.FuzzyInteger(10, 20)
 
     @factory.post_generation
-    def antennas(self, create, extracted, **kwargs):
+    def antennas(self, create, extracted, **kwargs):  # pylint: disable=W0613
+        """Generate antennas"""
         if not create:
             return
 
@@ -100,11 +108,10 @@ class ObservationFactory(factory.django.DjangoModelFactory):
     satellite = factory.SubFactory(SatelliteFactory)
     tle = factory.SubFactory(TleFactory)
     author = factory.SubFactory(UserFactory)
-    start = fuzzy.FuzzyDateTime(now() - timedelta(days=3),
-                                now() + timedelta(days=3))
-    end = factory.LazyAttribute(
-        lambda x: x.start + timedelta(hours=random.randint(1, 8))
+    start = fuzzy.FuzzyDateTime(
+        now() - timedelta(days=3), now() + timedelta(days=3), force_microsecond=0
     )
+    end = factory.LazyAttribute(lambda x: x.start + timedelta(hours=random.randint(1, 8)))
     ground_station = factory.Iterator(Station.objects.all())
     payload = factory.django.FileField(filename='data.ogg')
     vetted_datetime = factory.LazyAttribute(
@@ -118,17 +125,26 @@ class ObservationFactory(factory.django.DjangoModelFactory):
     transmitter_uplink_high = fuzzy.FuzzyInteger(200000000, 500000000, step=10000)
     transmitter_downlink_low = fuzzy.FuzzyInteger(200000000, 500000000, step=10000)
     transmitter_downlink_high = fuzzy.FuzzyInteger(200000000, 500000000, step=10000)
-    transmitter_mode = fuzzy.FuzzyText()
+    transmitter_mode = fuzzy.FuzzyText(length=10)
     transmitter_invert = fuzzy.FuzzyChoice(choices=[True, False])
     transmitter_baud = fuzzy.FuzzyInteger(4000, 22000, step=1000)
-    transmitter_created = fuzzy.FuzzyDateTime(now() - timedelta(days=100),
-                                              now() - timedelta(days=3))
+    transmitter_created = fuzzy.FuzzyDateTime(
+        now() - timedelta(days=100),
+        now() - timedelta(days=3)
+    )
 
     class Meta:
         model = Observation
 
 
+class RealisticObservationFactory(ObservationFactory):
+    """Observation model factory which uses existing satellites and tles."""
+    satellite = factory.Iterator(Satellite.objects.all())
+    tle = factory.Iterator(Tle.objects.all())
+
+
 class DemodDataFactory(factory.django.DjangoModelFactory):
+    """DemodData model factory."""
     observation = factory.Iterator(Observation.objects.all())
     payload_demod = factory.django.FileField()
 
@@ -142,6 +158,7 @@ class HomeViewTest(TestCase):
     Simple test to make sure the home page is working
     """
     def test_home_page(self):
+        """Test for string in home page"""
         response = self.client.get('/')
         self.assertContains(response, 'Crowd-sourced satellite operations')
 
@@ -152,6 +169,7 @@ class AboutViewTest(TestCase):
     Simple test to make sure the about page is working
     """
     def test_about_page(self):
+        """Test for string in about page"""
         response = self.client.get('/about/')
         self.assertContains(response, 'SatNOGS Network is a global management interface')
 
@@ -165,14 +183,15 @@ class StationListViewTest(TestCase):
     stations = []
 
     def setUp(self):
-        for x in xrange(1, 10):
+        for _ in range(1, 10):
             self.stations.append(StationFactory())
 
     def test_station_list(self):
+        """Test for owners and station names in station page"""
         response = self.client.get('/stations/')
-        for x in self.stations:
-            self.assertContains(response, x.owner)
-            self.assertContains(response, x.name)
+        for station in self.stations:
+            self.assertContains(response, station.owner)
+            self.assertContains(response, station.name)
 
 
 @pytest.mark.django_db(transaction=True)
@@ -196,45 +215,49 @@ class ObservationsListViewTest(TestCase):
         self.observations_unvetted = []
         self.observations = []
         with transaction.atomic():
-            for x in xrange(1, 10):
+            for _ in range(1, 10):
                 self.satellites.append(SatelliteFactory())
-            for x in xrange(1, 10):
+            for _ in range(1, 10):
                 self.stations.append(StationFactory())
-            for x in xrange(1, 5):
+            for _ in range(1, 5):
                 obs = ObservationFactory(vetted_status='bad')
                 self.observations_bad.append(obs)
                 self.observations.append(obs)
-            for x in xrange(1, 5):
+            for _ in range(1, 5):
                 obs = ObservationFactory(vetted_status='good')
                 self.observations_good.append(obs)
                 self.observations.append(obs)
-            for x in xrange(1, 5):
+            for _ in range(1, 5):
                 obs = ObservationFactory(vetted_status='unknown')
                 self.observations_unvetted.append(obs)
                 self.observations.append(obs)
 
     def test_observations_list(self):
+        """Test for transmitter modes of each observation in observations page"""
         response = self.client.get('/observations/')
-        for x in self.observations:
-            self.assertContains(response, x.transmitter_mode)
+        for observation in self.observations:
+            self.assertContains(response, observation.transmitter_mode)
 
     def test_observations_list_select_bad(self):
+        """Test for transmitter modes of each bad observation in observations page"""
         response = self.client.get('/observations/?bad=1')
 
-        for x in self.observations_bad:
-            self.assertContains(response, x.transmitter_mode)
+        for observation in self.observations_bad:
+            self.assertContains(response, observation.transmitter_mode)
 
     def test_observations_list_select_good(self):
+        """Test for transmitter modes of each good observation in observations page"""
         response = self.client.get('/observations/?good=1')
 
-        for x in self.observations_good:
-            self.assertContains(response, x.transmitter_mode)
+        for observation in self.observations_good:
+            self.assertContains(response, observation.transmitter_mode)
 
     def test_observations_list_select_unvetted(self):
+        """Test for transmitter modes of each unvetted observation in observations page"""
         response = self.client.get('/observations/?unvetted=1')
 
-        for x in self.observations_unvetted:
-            self.assertContains(response, x.transmitter_mode)
+        for observation in self.observations_unvetted:
+            self.assertContains(response, observation.transmitter_mode)
 
 
 class NotFoundErrorTest(TestCase):
@@ -244,8 +267,9 @@ class NotFoundErrorTest(TestCase):
     client = Client()
 
     def test_404_not_found(self):
+        """Test for "404" html status code in response for requesting a non-existed page"""
         response = self.client.get('/blah')
-        self.assertEquals(response.status_code, 404)
+        self.assertEqual(response.status_code, 404)
 
 
 class RobotsViewTest(TestCase):
@@ -255,6 +279,7 @@ class RobotsViewTest(TestCase):
     client = Client()
 
     def test_robots(self):
+        """Test for "Disallow" string in response for requesting robots.txt"""
         response = self.client.get('/robots.txt')
         self.assertContains(response, 'Disallow: /')
 
@@ -272,15 +297,16 @@ class ObservationViewTest(TestCase):
 
     def setUp(self):
         self.user = UserFactory()
-        g = Group.objects.get(name='Moderators')
-        g.user_set.add(self.user)
-        for x in xrange(1, 10):
+        moderators = Group.objects.get(name='Moderators')
+        moderators.user_set.add(self.user)
+        for _ in range(1, 10):
             self.satellites.append(SatelliteFactory())
-        for x in xrange(1, 10):
+        for _ in range(1, 10):
             self.stations.append(StationFactory())
         self.observation = ObservationFactory()
 
     def test_observation(self):
+        """Test for observer and transmitter mode in observation page"""
         response = self.client.get('/observations/%d/' % self.observation.id)
         self.assertContains(response, self.observation.author.username)
         self.assertContains(response, self.observation.transmitter_mode)
@@ -293,37 +319,63 @@ class ObservationDeleteTest(TestCase):
     """
     client = Client()
     user = None
-    observation = None
+    future_observation = None
+    past_observation = None
     satellites = []
 
     def setUp(self):
         self.user = UserFactory()
         self.client.force_login(self.user)
-        for x in xrange(1, 10):
+        for _ in range(1, 10):
             self.satellites.append(SatelliteFactory())
-        self.observation = ObservationFactory()
-        self.observation.author = self.user
-        self.observation.save()
+        self.future_observation = ObservationFactory()
+        self.future_observation.author = self.user
+        self.future_observation.start = now() + timedelta(days=1)
+        self.future_observation.end = self.future_observation.start + timedelta(minutes=15)
+        self.future_observation.save()
+        self.past_observation = ObservationFactory()
+        self.past_observation.author = self.user
+        self.past_observation.start = now() - timedelta(days=1)
+        self.past_observation.end = self.past_observation.start + timedelta(minutes=15)
+        self.past_observation.save()
 
-    def test_observation_delete_author(self):
-        """Deletion OK when user is the author of the observation"""
-        response = self.client.get('/observations/%d/delete/' % self.observation.id)
+    def test_future_observation_delete_author(self):
+        """Deletion OK when user is the author of the observation and observation is in future"""
+        response = self.client.get('/observations/%d/delete/' % self.future_observation.id)
         self.assertRedirects(response, '/observations/')
         response = self.client.get('/observations/')
         with self.assertRaises(Observation.DoesNotExist):
-            _lookup = Observation.objects.get(pk=self.observation.id)  # noqa:F841
+            _lookup = Observation.objects.get(pk=self.future_observation.id)  # noqa:F841
 
-    def test_observation_delete_moderator(self):
-        """Deletion OK when user is moderator and there is no data"""
+    def test_future_observation_delete_moderator(self):
+        """Deletion OK when user is moderator and observation is in future"""
         self.user = UserFactory()
-        g = Group.objects.get(name='Moderators')
-        g.user_set.add(self.user)
+        moderators = Group.objects.get(name='Moderators')
+        moderators.user_set.add(self.user)
         self.client.force_login(self.user)
-        response = self.client.get('/observations/%d/delete/' % self.observation.id)
+        response = self.client.get('/observations/%d/delete/' % self.future_observation.id)
         self.assertRedirects(response, '/observations/')
         response = self.client.get('/observations/')
         with self.assertRaises(Observation.DoesNotExist):
-            _lookup = Observation.objects.get(pk=self.observation.id)  # noqa:F841
+            _lookup = Observation.objects.get(pk=self.future_observation.id)  # noqa:F841
+
+    def test_past_observation_delete_author(self):
+        """Deletion NOT OK when user is the author of the observation and observation is in past"""
+        response = self.client.get('/observations/%d/delete/' % self.past_observation.id)
+        self.assertRedirects(response, '/observations/')
+        response = self.client.get('/observations/')
+        self.assertContains(response, self.past_observation.id)
+
+    def test_past_observation_delete_moderator(self):
+        """Deletion NOT OK when user is moderator and observation is in past"""
+        self.user = UserFactory()
+        moderators = Group.objects.get(name='Moderators')
+        moderators.user_set.add(self.user)
+        self.client.force_login(self.user)
+        response = self.client.get('/observations/%d/delete/' % self.past_observation.id)
+        self.assertRedirects(response, '/observations/')
+        response = self.client.get('/observations/')
+        self.assertContains(response, self.past_observation.id)
 
 
 @pytest.mark.django_db(transaction=True)
@@ -338,6 +390,7 @@ class StationViewTest(TestCase):
         self.station = StationFactory()
 
     def test_observation(self):
+        """Test for owner, elevation and min horizon in station page"""
         response = self.client.get('/stations/%d/' % self.station.id)
         self.assertContains(response, self.station.owner.username)
         self.assertContains(response, self.station.alt)
@@ -361,10 +414,11 @@ class StationDeleteTest(TestCase):
         self.station.save()
 
     def test_station_delete(self):
+        """Test deletion of station"""
         response = self.client.get('/stations/%d/delete/' % self.station.id)
         self.assertRedirects(response, '/users/%s/' % self.user.username)
         with self.assertRaises(Station.DoesNotExist):
-            _lookup = Station.objects.get(pk=self.station.id)       # noqa:F841
+            _lookup = Station.objects.get(pk=self.station.id)  # noqa:F841
 
 
 @pytest.mark.django_db(transaction=True)
@@ -382,30 +436,11 @@ class SettingsSiteViewTest(TestCase):
         self.client.force_login(self.user)
 
     def test_get(self):
+        """Test for "Fetch Data" in Settings Site page"""
         response = self.client.get('/settings_site/')
         self.assertContains(response, 'Fetch Data')
 
 
-@pytest.mark.django_db(transaction=True)
-class SatelliteModelTest(TestCase):
-    """
-    Tests various methods of the Satellite model
-    """
-    satellite = None
-
-    def setUp(self):
-        self.satellite = SatelliteFactory()
-
-    def test_latest_tle(self):
-        self.assertFalse(self.satellite.latest_tle)
-
-    def test_tle_epoch(self):
-        self.assertFalse(self.satellite.tle_epoch)
-
-    def test_tle_no(self):
-        self.assertFalse(self.satellite.tle_no)
-
-
 @pytest.mark.django_db(transaction=True)
 class ObservationModelTest(TestCase):
     """
@@ -417,11 +452,12 @@ class ObservationModelTest(TestCase):
     admin = None
 
     def setUp(self):
-        for x in xrange(1, 10):
+        for _ in range(1, 10):
             self.satellites.append(SatelliteFactory())
         self.observation = ObservationFactory()
         self.observation.end = now()
         self.observation.save()
 
     def test_is_passed(self):
+        """Test for observation be in past"""
         self.assertTrue(self.observation.is_past)

network/base/urls.py

@@ -1,41 +1,64 @@
+"""Django base URL routings for SatNOGS Network"""
+from __future__ import absolute_import
+
 from django.conf.urls import url
 from django.views.generic import TemplateView
 
 from network.base import views
 
-base_urlpatterns = ([
-    url(r'^$', views.index, name='home'),
-    url(r'^about/$', TemplateView.as_view(template_name='base/about.html'), name='about'),
-    url(r'^robots\.txt$', views.robots, name='robots'),
-    url(r'^settings_site/$', views.settings_site, name='settings_site'),
+BASE_URLPATTERNS = (
+    [
+        url(r'^$', views.index, name='home'),
+        url(r'^about/$', TemplateView.as_view(template_name='base/about.html'), name='about'),
+        url(r'^robots\.txt$', views.robots, name='robots'),
+        url(r'^settings_site/$', views.settings_site, name='settings_site'),
 
-    # Observations
-    url(r'^observations/$', views.ObservationListView.as_view(), name='observations_list'),
-    url(r'^observations/(?P<id>[0-9]+)/$', views.observation_view,
-        name='observation_view'),
-    url(r'^observations/(?P<id>[0-9]+)/delete/$', views.observation_delete,
-        name='observation_delete'),
-    url(r'^observations/new/$', views.observation_new, name='observation_new'),
-    url(r'^prediction_windows/$', views.prediction_windows, name='prediction_windows'),
-    url(r'^pass_predictions/(?P<id>[\w.@+-]+)/$',
-        views.pass_predictions, name='pass_predictions'),
-    url(r'^observation_vet/(?P<id>[0-9]+)/$', views.observation_vet, name='observation_vet'),
+        # Observations
+        url(r'^observations/$', views.ObservationListView.as_view(), name='observations_list'),
+        url(
+            r'^observations/(?P<observation_id>[0-9]+)/$',
+            views.observation_view,
+            name='observation_view'
+        ),
+        url(
+            r'^observations/(?P<observation_id>[0-9]+)/delete/$',
+            views.observation_delete,
+            name='observation_delete'
+        ),
+        url(r'^observations/new/$', views.observation_new, name='observation_new'),
+        url(r'^prediction_windows/$', views.prediction_windows, name='prediction_windows'),
+        url(
+            r'^pass_predictions/(?P<station_id>[\w.@+-]+)/$',
+            views.pass_predictions,
+            name='pass_predictions'
+        ),
+        url(
+            r'^observation_vet/(?P<observation_id>[0-9]+)/$',
+            views.observation_vet,
+            name='observation_vet'
+        ),
 
-    # Stations
-    url(r'^stations/$', views.stations_list, name='stations_list'),
-    url(r'^stations/(?P<id>[0-9]+)/$', views.station_view, name='station_view'),
-    url(r'^stations/(?P<id>[0-9]+)/log/$', views.station_log, name='station_log'),
-    url(r'^stations/(?P<id>[0-9]+)/delete/$', views.station_delete, name='station_delete'),
-    url(r'^stations/edit/$', views.station_edit, name='station_edit'),
-    url(r'^stations/edit/(?P<id>[0-9]+)/$', views.station_edit, name='station_edit'),
-    url(r'^stations_all/$', views.StationAllView.as_view({'get': 'list'}), name='stations_all'),
-    url(r'^scheduling_stations/$', views.scheduling_stations, name='scheduling_stations'),
+        # Stations
+        url(r'^stations/$', views.stations_list, name='stations_list'),
+        url(r'^stations/(?P<station_id>[0-9]+)/$', views.station_view, name='station_view'),
+        url(r'^stations/(?P<station_id>[0-9]+)/log/$', views.station_log_view, name='station_log'),
+        url(
+            r'^stations/(?P<station_id>[0-9]+)/delete/$',
+            views.station_delete,
+            name='station_delete'
+        ),
+        url(r'^stations/edit/$', views.station_edit, name='station_edit'),
+        url(r'^stations/edit/(?P<station_id>[0-9]+)/$', views.station_edit, name='station_edit'),
+        url(
+            r'^stations_all/$', views.StationAllView.as_view({'get': 'list'}), name='stations_all'
+        ),
+        url(r'^scheduling_stations/$', views.scheduling_stations, name='scheduling_stations'),
 
-    # Satellites
-    url(r'^satellites/(?P<id>[0-9]+)/$', views.satellite_view, name='satellite_view'),
-    url(r'^satellite_position/(?P<sat_id>[0-9]+)/$', views.satellite_position,
-        name='satellite_position'),
+        # Satellites
+        url(r'^satellites/(?P<norad_id>[0-9]+)/$', views.satellite_view, name='satellite_view'),
 
-    # Transmitters
-    url(r'^transmitters/', views.transmitters_view, name='transmitters_view'),
-], 'base')
+        # Transmitters
+        url(r'^transmitters/', views.transmitters_view, name='transmitters_view'),
+    ],
+    'base'
+)

network/base/utils.py

@@ -1,26 +1,33 @@
+"""Miscellaneous functions for SatNOGS Network"""
+from __future__ import absolute_import, division
+
 import csv
-import urllib
-import urllib2
-from django.http import HttpResponse
-from django.core.exceptions import PermissionDenied
-from django.contrib.admin.helpers import label_for_field
-from django.conf import settings
-from requests.exceptions import ReadTimeout, HTTPError
-from network.base.models import DemodData
+from builtins import str
 from datetime import datetime
 
+import requests  # pylint: disable=C0412
+from django.conf import settings
+from django.contrib.admin.helpers import label_for_field
+from django.core.exceptions import PermissionDenied
+from django.http import HttpResponse
+from django.utils.text import slugify
+from requests.exceptions import RequestException
+
+from network.base.models import DemodData
+
 
 def export_as_csv(modeladmin, request, queryset):
+    """Exports admin panel table in csv format"""
     if not request.user.is_staff:
         raise PermissionDenied
-    opts = modeladmin.model._meta
     field_names = modeladmin.list_display
     if 'action_checkbox' in field_names:
         field_names.remove('action_checkbox')
 
     response = HttpResponse(content_type="text/csv")
-    response['Content-Disposition'] = 'attachment; filename=%s.csv' % unicode(opts)\
-                                      .replace('.', '_')
+    response['Content-Disposition'] = 'attachment; filename={}.csv'.format(
+        str(modeladmin.model._meta).replace('.', '_')
+    )
 
     writer = csv.writer(response)
     headers = []
@@ -41,17 +48,18 @@ def export_as_csv(modeladmin, request, queryset):
                 try:
                     # get value from model
                     value = value()
-                except Exception:
+                except TypeError:
                     # get value from modeladmin e.g: admin_method_1
                     value = value(row)
             if value is None:
                 value = ''
-            values.append(unicode(value).encode('utf-8'))
+            values.append(str(value).encode('utf-8'))
         writer.writerow(values)
     return response
 
 
 def export_station_status(self, request, queryset):
+    """Exports status of selected stations in csv format"""
     meta = self.model._meta
     field_names = ["id", "status"]
 
@@ -66,8 +74,11 @@ def export_station_status(self, request, queryset):
     return response
 
 
-def demod_to_db(frame_id):
-    """Task to send a frame from SatNOGS network to SatNOGS db"""
+def sync_demoddata_to_db(frame_id):
+    """
+    Task to send a frame from SatNOGS Network to SatNOGS DB
+
+    Raises requests.exceptions.RequestException if sync fails."""
     frame = DemodData.objects.get(id=frame_id)
     obs = frame.observation
     sat = obs.satellite
@@ -76,28 +87,54 @@ def demod_to_db(frame_id):
     # need to abstract the timestamp from the filename. hacky..
     file_datetime = frame.payload_demod.name.split('/')[2].split('_')[2]
     frame_datetime = datetime.strptime(file_datetime, '%Y-%m-%dT%H-%M-%S')
-    submit_datetime = datetime.strftime(frame_datetime,
-                                        '%Y-%m-%dT%H:%M:%S.000Z')
+    submit_datetime = datetime.strftime(frame_datetime, '%Y-%m-%dT%H:%M:%S.000Z')
 
     # SiDS parameters
-    params = {}
-    params['noradID'] = sat.norad_cat_id
-    params['source'] = ground_station.name
-    params['timestamp'] = submit_datetime
-    params['locator'] = 'longLat'
-    params['longitude'] = ground_station.lng
-    params['latitude'] = ground_station.lat
-    params['frame'] = frame.display_payload().replace(' ', '')
-    params['satnogs_network'] = 'True'  # NOT a part of SiDS
+    params = {
+        'noradID': sat.norad_cat_id,
+        'source': ground_station.name,
+        'timestamp': submit_datetime,
+        'locator': 'longLat',
+        'longitude': ground_station.lng,
+        'latitude': ground_station.lat,
+        'frame': frame.display_payload_hex().replace(' ', ''),
+        'satnogs_network': 'True'  # NOT a part of SiDS
+    }
 
-    apiurl = settings.DB_API_ENDPOINT
-    telemetry_url = "{0}telemetry/".format(apiurl)
+    telemetry_url = "{}telemetry/".format(settings.DB_API_ENDPOINT)
+
+    response = requests.post(telemetry_url, data=params, timeout=settings.DB_API_TIMEOUT)
+    response.raise_for_status()
+
+    frame.copied_to_db = True
+    frame.save()
+
+
+def community_get_discussion_details(
+        observation_id, satellite_name, norad_cat_id, observation_url
+):
+    """
+    Return the details of a discussion of the observation (if existent) in the
+    satnogs community (discourse)
+    """
+
+    discussion_url = ('https://community.libre.space/new-topic?title=Observation {0}: {1}'
+                      ' ({2})&body=Regarding [Observation {0}]({3}) ...'
+                      '&category=observations') \
+        .format(observation_id, satellite_name, norad_cat_id, observation_url)
+
+    discussion_slug = 'https://community.libre.space/t/observation-{0}-{1}-{2}' \
+        .format(observation_id, slugify(satellite_name),
+                norad_cat_id)
 
     try:
-        res = urllib2.urlopen(telemetry_url, urllib.urlencode(params))
-        code = str(res.getcode())
-        if code.startswith('2'):
-            frame.copied_to_db = True
-            frame.save()
-    except (ReadTimeout, HTTPError):
-        return
+        response = requests.get(
+            '{}.json'.format(discussion_slug), timeout=settings.COMMUNITY_TIMEOUT
+        )
+        response.raise_for_status()
+        has_comments = (response.status_code == 200)
+    except RequestException:
+        # Community is unreachable
+        has_comments = False
+
+    return {'url': discussion_url, 'slug': discussion_slug, 'has_comments': has_comments}

network/base/validators.py

@@ -0,0 +1,111 @@
+"""Django base validators for SatNOGS Network"""
+from __future__ import absolute_import
+
+from datetime import datetime, timedelta
+
+from django.conf import settings
+from django.utils.timezone import make_aware, utc
+
+
+class ObservationOverlapError(Exception):
+    """Error when observation overlaps with already scheduled one"""
+
+
+class OutOfRangeError(Exception):
+    """Error when transmitter is our of station's antenna frequency range"""
+
+
+class NegativeElevationError(Exception):
+    """Error when satellite doesn't raise above station's horizon"""
+
+
+class SinglePassError(Exception):
+    """Error when between given start and end datetimes there are more than one satellite passes"""
+
+
+def check_start_datetime(start):
+    """Validate start datetime"""
+    if start < make_aware(datetime.now(), utc):
+        raise ValueError("Start datetime should be in the future!")
+    if start < make_aware(datetime.now() + timedelta(minutes=settings.OBSERVATION_DATE_MIN_START),
+                          utc):
+        raise ValueError(
+            "Start datetime should be in the future, at least {0} minutes from now".format(
+                settings.OBSERVATION_DATE_MIN_START
+            )
+        )
+
+
+def check_end_datetime(end):
+    """Validate end datetime"""
+    if end < make_aware(datetime.now(), utc):
+        raise ValueError("End datetime should be in the future!")
+    max_duration = settings.OBSERVATION_DATE_MIN_START + settings.OBSERVATION_DATE_MAX_RANGE
+    if end > make_aware(datetime.now() + timedelta(minutes=max_duration), utc):
+        raise ValueError(
+            "End datetime should be in the future, at most {0} minutes from now".
+            format(max_duration)
+        )
+
+
+def check_start_end_datetimes(start, end):
+    """Validate the pair of start and end datetimes"""
+    if start > end:
+        raise ValueError("End datetime should be after Start datetime!")
+    if (end - start) < timedelta(seconds=settings.OBSERVATION_DURATION_MIN):
+        raise ValueError(
+            "Duration of observation should be at least {0} seconds".format(
+                settings.OBSERVATION_DURATION_MIN
+            )
+        )
+
+
+def downlink_low_is_in_range(antenna, transmitter):
+    """Return true if transmitter frequency is in station's antenna frequency range"""
+    if transmitter['downlink_low'] is not None:
+        return antenna.frequency <= transmitter['downlink_low'] <= antenna.frequency_max
+    return False
+
+
+def is_transmitter_in_station_range(transmitter, station):
+    """Return true if transmitter frequency is in one of the station's antennas frequency ranges"""
+    for gs_antenna in station.antenna.all():
+        if downlink_low_is_in_range(gs_antenna, transmitter):
+            return True
+    return False
+
+
+def check_transmitter_station_pairs(transmitter_station_list):
+    """Validate the pairs of transmitter and stations"""
+    out_of_range_pairs = [
+        (str(pair[0]['uuid']), int(pair[1].id)) for pair in transmitter_station_list
+        if not is_transmitter_in_station_range(pair[0], pair[1])
+    ]
+    if out_of_range_pairs:
+        if len(out_of_range_pairs) == 1:
+            raise OutOfRangeError(
+                'Transmitter out of station frequency range.'
+                ' Transmitter-Station pair: {0}'.format(out_of_range_pairs[0])
+            )
+        raise OutOfRangeError(
+            'Transmitter out of station frequency range. '
+            'Transmitter-Station pairs: {0}'.format(out_of_range_pairs)
+        )
+
+
+def check_overlaps(stations_dict):
+    """Check for overlaps among requested observations"""
+    for station in stations_dict.keys():
+        periods = stations_dict[station]
+        total_periods = len(periods)
+        for i in range(0, total_periods):
+            start_i = periods[i][0]
+            end_i = periods[i][1]
+            for j in range(i + 1, total_periods):
+                start_j = periods[j][0]
+                end_j = periods[j][1]
+                if ((start_j <= start_i <= end_j) or (start_j <= end_i <= end_j)
+                        or (start_i <= start_j and end_i >= end_j)):  # noqa: W503
+                    raise ObservationOverlapError(
+                        'Observations of station {0} overlap'.format(station)
+                    )

network/celery.py

@@ -1,50 +1,85 @@
+"""SatNOGS Network celery task workers"""
 from __future__ import absolute_import
 
 import os
 
 from celery import Celery
-
+from django.conf import settings  # noqa
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'network.settings')
 
-from django.conf import settings  # noqa
-
 RUN_DAILY = 60 * 60 * 24
 RUN_EVERY_TWO_HOURS = 2 * 60 * 60
 RUN_HOURLY = 60 * 60
 RUN_EVERY_MINUTE = 60
 RUN_TWICE_HOURLY = 60 * 30
 
-app = Celery('network')
+APP = Celery('network')
 
-app.config_from_object('django.conf:settings', namespace='CELERY')
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+APP.config_from_object('django.conf:settings', namespace='CELERY')
+APP.autodiscover_tasks()
 
 
-@app.on_after_finalize.connect
-def setup_periodic_tasks(sender, **kwargs):
-    from network.base.tasks import (update_all_tle, fetch_data, clean_observations,
-                                    station_status_update, stations_cache_rates,
-                                    notify_for_stations_without_results, sync_to_db)
+# Wrapper tasks as workaround for registering shared tasks to beat scheduler
+# See https://github.com/celery/celery/issues/5059
+# and https://github.com/celery/celery/issues/3797#issuecomment-422656038
+@APP.task
+def update_all_tle():
+    """Wrapper task for 'update_all_tle' shared task"""
+    from network.base.tasks import update_all_tle as periodic_task
+    periodic_task()
 
-    sender.add_periodic_task(RUN_EVERY_TWO_HOURS, update_all_tle.s(),
-                             name='update-all-tle')
 
-    sender.add_periodic_task(RUN_HOURLY, fetch_data.s(),
-                             name='fetch-data')
+@APP.task
+def fetch_data():
+    """Wrapper task for 'fetch_data' shared task"""
+    from network.base.tasks import fetch_data as periodic_task
+    periodic_task()
 
-    sender.add_periodic_task(RUN_HOURLY, station_status_update.s(),
-                             name='station-status-update')
 
-    sender.add_periodic_task(RUN_HOURLY, clean_observations.s(),
-                             name='clean-observations')
+@APP.task
+def clean_observations():
+    """Wrapper task for 'clean_observations' shared task"""
+    from network.base.tasks import clean_observations as periodic_task
+    periodic_task()
 
-    sender.add_periodic_task(RUN_HOURLY, stations_cache_rates.s(),
-                             name='stations-cache-rates')
 
-    sender.add_periodic_task(settings.OBS_NO_RESULTS_CHECK_PERIOD,
-                             notify_for_stations_without_results.s(),
-                             name='notify_for_stations_without_results')
+@APP.task
+def station_status_update():
+    """Wrapper task for 'station_status_update' shared task"""
+    from network.base.tasks import station_status_update as periodic_task
+    periodic_task()
 
-    sender.add_periodic_task(RUN_TWICE_HOURLY, sync_to_db.s(),
-                             name='sync-to-db')
+
+@APP.task
+def stations_cache_rates():
+    """Wrapper task for 'stations_cache_rates' shared task"""
+    from network.base.tasks import stations_cache_rates as periodic_task
+    periodic_task()
+
+
+@APP.task
+def notify_for_stations_without_results():
+    """Wrapper task for 'notify_for_stations_without_results' shared task"""
+    from network.base.tasks import notify_for_stations_without_results as periodic_task
+    periodic_task()
+
+
+@APP.on_after_finalize.connect
+def setup_periodic_tasks(sender, **kwargs):  # pylint: disable=W0613
+    """Initializes celery tasks that need to run on a scheduled basis"""
+    sender.add_periodic_task(RUN_EVERY_TWO_HOURS, update_all_tle.s(), name='update-all-tle')
+
+    sender.add_periodic_task(RUN_HOURLY, fetch_data.s(), name='fetch-data')
+
+    sender.add_periodic_task(RUN_HOURLY, station_status_update.s(), name='station-status-update')
+
+    sender.add_periodic_task(RUN_HOURLY, clean_observations.s(), name='clean-observations')
+
+    sender.add_periodic_task(RUN_HOURLY, stations_cache_rates.s(), name='stations-cache-rates')
+
+    sender.add_periodic_task(
+        settings.OBS_NO_RESULTS_CHECK_PERIOD,
+        notify_for_stations_without_results.s(),
+        name='notify_for_stations_without_results'
+    )

network/settings.py

@@ -1,9 +1,16 @@
-from decouple import config, Csv
-from dj_database_url import parse as db_url
-from unipath import Path
-import sentry_sdk
-from sentry_sdk.integrations.django import DjangoIntegration
+"""SatNOGS Network Application django settings
 
+For local installation settings please copy .env-dist to .env and edit
+the appropriate settings in that file. You should not need to edit this
+file for local settings!
+"""
+from __future__ import absolute_import
+
+from decouple import Csv, config
+from dj_database_url import parse as db_url
+from sentry_sdk import init as sentry_sdk_init
+from sentry_sdk.integrations.django import DjangoIntegration
+from unipath import Path
 
 ROOT = Path(__file__).parent
 
@@ -40,9 +47,15 @@ LOCAL_APPS = (
     'network.api',
 )
 
+if DEBUG:
+    DJANGO_APPS += ('debug_toolbar', )
+    DEBUG_TOOLBAR_CONFIG = {
+        'SHOW_TOOLBAR_CALLBACK':
+        lambda request: request.environ.get('SERVER_NAME', None) != 'testserver',
+    }
 if AUTH0:
-    THIRD_PARTY_APPS += ('social_django',)
-    LOCAL_APPS += ('auth0login',)
+    THIRD_PARTY_APPS += ('social_django', )
+    LOCAL_APPS += ('auth0login', )
 
 INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
 
@@ -58,6 +71,9 @@ MIDDLEWARE = (
     'csp.middleware.CSPMiddleware',
 )
 
+if DEBUG:
+    MIDDLEWARE = ('debug_toolbar.middleware.DebugToolbarMiddleware', ) + MIDDLEWARE
+
 # Email
 if DEBUG:
     EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
@@ -71,22 +87,19 @@ EMAIL_HOST_USER = config('EMAIL_HOST_USER', default='')
 EMAIL_HOST_PASSWORD = config('EMAIL_HOST_PASSWORD', default='')
 DEFAULT_FROM_EMAIL = config('DEFAULT_FROM_EMAIL', default='noreply@satnogs.org')
 EMAIL_FOR_STATIONS_ISSUES = config('EMAIL_FOR_STATIONS_ISSUES', default='')
-ADMINS = [
-    ('SatNOGS Admins', DEFAULT_FROM_EMAIL)
-]
+ADMINS = [('SatNOGS Admins', DEFAULT_FROM_EMAIL)]
 MANAGERS = ADMINS
 SERVER_EMAIL = DEFAULT_FROM_EMAIL
 
 # Cache
 CACHES = {
     'default': {
-        'BACKEND': config('CACHE_BACKEND',
-                          default='django.core.cache.backends.locmem.LocMemCache'),
+        'BACKEND':
+        config('CACHE_BACKEND', default='django.core.cache.backends.locmem.LocMemCache'),
         'LOCATION': config('CACHE_LOCATION', default='unique-location'),
         'OPTIONS': {
             'MAX_ENTRIES': 5000,
-            'CLIENT_CLASS': config('CACHE_CLIENT_CLASS',
-                                   default=''),
+            'CLIENT_CLASS': config('CACHE_CLIENT_CLASS', default=''),
         },
         'KEY_PREFIX': 'network-{0}'.format(ENVIRONMENT),
     }
@@ -122,15 +135,17 @@ TEMPLATES = [
                 'network.base.context_processors.user_processor',
                 'network.base.context_processors.auth_block',
                 'network.base.context_processors.logout_block',
+                'network.base.context_processors.version'
             ],
             'loaders': [
-                ('django.template.loaders.cached.Loader', [
-                    'django.template.loaders.filesystem.Loader',
-                    'django.template.loaders.app_directories.Loader',
-                ]),
+                (
+                    'django.template.loaders.cached.Loader', [
+                        'django.template.loaders.filesystem.Loader',
+                        'django.template.loaders.app_directories.Loader',
+                    ]
+                ),
             ],
         },
-
     },
 ]
 
@@ -154,12 +169,9 @@ COMPRESS_ENABLED = config('COMPRESS_ENABLED', default=False, cast=bool)
 COMPRESS_OFFLINE = config('COMPRESS_OFFLINE', default=False, cast=bool)
 COMPRESS_CACHE_BACKEND = config('COMPRESS_CACHE_BACKEND', default='default')
 COMPRESS_CSS_FILTERS = [
-    'compressor.filters.css_default.CssAbsoluteFilter',
-    'compressor.filters.cssmin.rCSSMinFilter'
+    'compressor.filters.css_default.CssAbsoluteFilter', 'compressor.filters.cssmin.rCSSMinFilter'
 ]
-COMPRESS_PRECOMPILERS = (
-    ('text/scss', 'sass --scss {infile} {outfile}'),
-)
+COMPRESS_PRECOMPILERS = (('text/scss', 'sass --scss {infile} {outfile}'), )
 
 # App conf
 ROOT_URLCONF = 'network.urls'
@@ -167,11 +179,9 @@ WSGI_APPLICATION = 'network.wsgi.application'
 TEST_RUNNER = 'django.test.runner.DiscoverRunner'
 
 # Auth
-AUTHENTICATION_BACKENDS = (
-    'django.contrib.auth.backends.ModelBackend',
-)
+AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend', )
 if AUTH0:
-    AUTHENTICATION_BACKENDS += ('auth0login.auth0backend.Auth0',)
+    AUTHENTICATION_BACKENDS += ('auth0login.auth0backend.Auth0', )
 
 ACCOUNT_ADAPTER = 'allauth.account.adapter.DefaultAccountAdapter'
 ACCOUNT_AUTHENTICATION_METHOD = 'username'
@@ -231,10 +241,7 @@ LOGGING = {
 # Sentry
 SENTRY_ENABLED = config('SENTRY_ENABLED', default=False, cast=bool)
 if SENTRY_ENABLED:
-    sentry_sdk.init(
-        dsn=config('SENTRY_DSN', default=''),
-        integrations=[DjangoIntegration()]
-    )
+    sentry_sdk_init(dsn=config('SENTRY_DSN', default=''), integrations=[DjangoIntegration()])
 
 # Celery
 CELERY_ENABLE_UTC = USE_TZ
@@ -256,15 +263,10 @@ CELERY_BROKER_TRANSPORT_OPTIONS = {
 
 # API
 REST_FRAMEWORK = {
-    'DEFAULT_PERMISSION_CLASSES': (
-        'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly',
-    ),
-    'DEFAULT_AUTHENTICATION_CLASSES': (
-        'rest_framework.authentication.TokenAuthentication',
-    ),
-    'DEFAULT_FILTER_BACKENDS': (
-        'django_filters.rest_framework.DjangoFilterBackend',
-    )
+    'DEFAULT_PERMISSION_CLASSES':
+    ('rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly', ),
+    'DEFAULT_AUTHENTICATION_CLASSES': ('rest_framework.authentication.TokenAuthentication', ),
+    'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend', )
 }
 
 # Security
@@ -297,12 +299,8 @@ CSP_STYLE_SRC = (
     "'self'",
     "'unsafe-inline'",
 )
-CSP_WORKER_SRC = (
-    'blob:',
-)
-CSP_CHILD_SRC = (
-    'blob:',
-)
+CSP_WORKER_SRC = ('blob:', )
+CSP_CHILD_SRC = ('blob:', )
 
 # Database
 DATABASE_URL = config('DATABASE_URL', default='sqlite:///db.sqlite3')
@@ -349,8 +347,16 @@ OBS_NO_RESULTS_MIN_COUNT = config('OBS_NO_RESULTS_MIN_COUNT', default=3, cast=in
 OBS_NO_RESULTS_IGNORE_TIME = config('OBS_NO_RESULTS_IGNORE_TIME', default=1800, cast=int)
 
 # DB API
+# Set DB_API_ENDOINT to '' to disable the data fetching from DB
 DB_API_ENDPOINT = config('DB_API_ENDPOINT', default='https://db.satnogs.org/api/')
 
+# API timeout in seconds
+DB_API_TIMEOUT = config('DB_API_TIMEOUT', default=2.0, cast=float)
+
+# Timeout in seconds for the community forum
+# (used e.g. when checking for the existance of certain threads)
+COMMUNITY_TIMEOUT = config('COMMUNITY_TIMEOUT', default=2.0, cast=float)
+
 # ListView pagination
 ITEMS_PER_PAGE = config('ITEMS_PER_PAGE', default=25, cast=int)
 
@@ -360,11 +366,13 @@ AVATAR_GRAVATAR_DEFAULT = config('AVATAR_GRAVATAR_DEFAULT', default='mm')
 # Archive.org
 S3_ACCESS_KEY = config('S3_ACCESS_KEY', default='')
 S3_SECRET_KEY = config('S3_SECRET_KEY', default='')
+S3_RETRIES_ON_SLOW_DOWN = config('S3_RETRIES_ON_SLOW_DOWN', default=1, cast=int)
+S3_RETRIES_SLEEP = config('S3_RETRIES_SLEEP', default=30, cast=int)
 ARCHIVE_COLLECTION = config('ARCHIVE_COLLECTION', default='test_collection')
 ARCHIVE_URL = 'https://archive.org/download/'
 
 if AUTH0:
-    SOCIAL_AUTH_TRAILING_SLASH = False             # Remove end slash from routes
+    SOCIAL_AUTH_TRAILING_SLASH = False  # Remove end slash from routes
     SOCIAL_AUTH_AUTH0_DOMAIN = config('SOCIAL_AUTH_AUTH0_DOMAIN', default='YOUR_AUTH0_DOMAIN')
     SOCIAL_AUTH_AUTH0_KEY = config('SOCIAL_AUTH_AUTH0_KEY', default='YOUR_CLIENT_ID')
     SOCIAL_AUTH_AUTH0_SECRET = config('SOCIAL_AUTH_AUTH0_SECRET', default='YOUR_CLIENT_SECRET')
@@ -397,4 +405,4 @@ if ENVIRONMENT == 'dev':
         backend['APP_DIRS'] = True
 
 # needed to ensure data_obs files can be read by nginx
-FILE_UPLOAD_PERMISSIONS = 0644
+FILE_UPLOAD_PERMISSIONS = 0o0644

network/static/css/app.scss

@@ -18,3 +18,4 @@
 @import 'pages/station';
 @import 'pages/stations';
 @import 'pages/user';
+@import 'pages/map_overrides';

network/static/css/common/_status.scss

@@ -77,13 +77,19 @@
     filter: opacity(45%) saturate(75%);
 }
 
+.loading-field {
+    display: none;
+    text-align: center;
+    color: #666;
+}
+
 #loading {
     display: none;
     text-align: center;
     color: #666;
 
     .spinner {
-       margin: 50px auto 0;
+        margin: 50px auto 0;
     }
 }
 
@@ -118,4 +124,3 @@
         }
     }
 }
-

network/static/css/pages/_map_overrides.scss

@@ -0,0 +1,17 @@
+// Temporary file with overrides until issue https://github.com/mapbox/mapbox-gl-js/issues/7896 is resolved
+
+/* stylelint-disable declaration-no-important  */
+
+.mapboxgl-ctrl-icon.mapboxgl-ctrl-zoom-out {
+    background-image: url("data:image/svg+xml;charset=utf-8,%3Csvg%20viewBox='0%200%2020%2020'%20xmlns='http://www.w3.org/2000/svg'%3E%20%3Cpath%20style='fill:%23333333;'%20d='m%207,9%20c%20-0.554,0%20-1,0.446%20-1,1%200,0.554%200.446,1%201,1%20l%206,0%20c%200.554,0%201,-0.446%201,-1%200,-0.554%20-0.446,-1%20-1,-1%20z'/%3E%20%3C/svg%3E") !important;
+}
+
+.mapboxgl-ctrl-icon.mapboxgl-ctrl-zoom-in {
+    background-image: url("data:image/svg+xml;charset=utf-8,%3Csvg%20viewBox='0%200%2020%2020'%20xmlns='http://www.w3.org/2000/svg'%3E%20%3Cpath%20style='fill:%23333333;'%20d='M%2010%206%20C%209.446%206%209%206.4459904%209%207%20L%209%209%20L%207%209%20C%206.446%209%206%209.446%206%2010%20C%206%2010.554%206.446%2011%207%2011%20L%209%2011%20L%209%2013%20C%209%2013.55401%209.446%2014%2010%2014%20C%2010.554%2014%2011%2013.55401%2011%2013%20L%2011%2011%20L%2013%2011%20C%2013.554%2011%2014%2010.554%2014%2010%20C%2014%209.446%2013.554%209%2013%209%20L%2011%209%20L%2011%207%20C%2011%206.4459904%2010.554%206%2010%206%20z'/%3E%20%3C/svg%3E") !important;
+}
+
+.mapboxgl-ctrl-icon.mapboxgl-ctrl-compass > .mapboxgl-ctrl-compass-arrow {
+    background-image: url("data:image/svg+xml;charset=utf-8,%3Csvg%20viewBox='0%200%2020%2020'%20xmlns='http://www.w3.org/2000/svg'%3E%20%3Cpolygon%20fill='%23333333'%20points='6,9%2010,1%2014,9'/%3E%20%3Cpolygon%20fill='%23CCCCCC'%20points='6,11%2010,19%2014,11%20'/%3E%20%3C/svg%3E") !important;
+}
+
+/* stylelint-enable */

network/static/css/pages/_observation.scss

@@ -3,8 +3,8 @@
 }
 
 #min-horizon-slider {
-  width: calc(100% - 20px);
-  margin: 10px 0 10px 10px;
+    width: calc(100% - 20px);
+    margin: 10px 0 10px 10px;
 }
 
 .axis {

network/static/css/pages/_satellites.scss

@@ -3,7 +3,7 @@
 }
 
 .satellite-title-wrap {
-    margin-top: 0px;
+    margin-top: 0;
 }
 
 .satellite-title {

network/static/js/observation_new.js

@@ -55,9 +55,17 @@ $(document).ready( function(){
             dataType: 'json',
             beforeSend: function(xhr) {
                 xhr.setRequestHeader('X-CSRFToken', $('[name="csrfmiddlewaretoken"]').val());
+                $('#station-field').hide();
+                $('#station-field-loading').show();
             }
         }).done(function(data) {
-            if (data.stations.length > 0) {
+            if (data.length == 1 && data[0].error) {
+                $('#station-selection').html(`<option id="no-station"
+                                                          value="" selected>
+                                                    No station available
+                                                  </option>`).prop('disabled', true);
+                $('#station-selection').selectpicker('refresh');
+            } else if (data.stations.length > 0) {
                 var stations_options = '';
                 if (filters.station && data.stations.findIndex(st => st.id == filters.station) > -1) {
                     var station = data.stations.find(st => st.id == filters.station);
@@ -83,6 +91,8 @@ $(document).ready( function(){
             if (callback) {
                 callback();
             }
+            $('#station-field-loading').hide();
+            $('#station-field').show();
         });
     }
 
@@ -108,13 +118,17 @@ $(document).ready( function(){
                                         data-toggle="tooltip" data-placement="bottom"
                                         title="` + transmitter.bad_rate + '% (' + transmitter.bad_count + `) Bad"
                                         style="width:` + transmitter.bad_rate + `%"></div>
+                                      <div class="progress-bar progress-bar-info transmitter-future"
+                                        data-toggle="tooltip" data-placement="bottom"
+                                        title="` + transmitter.future_rate + '% (' + transmitter.future_count + `) Future"
+                                        style="width:` + transmitter.future_rate + `%"></div>
                                     </div>
                                   </div>'>
             </option>
         `;
     }
 
-    function select_proper_transmitters(filters, callback){
+    function select_proper_transmitters(filters){
         var url = '/transmitters/';
         var data = {'satellite': filters.satellite};
         if (filters.station) {
@@ -128,9 +142,19 @@ $(document).ready( function(){
             dataType: 'json',
             beforeSend: function(xhr) {
                 xhr.setRequestHeader('X-CSRFToken', $('[name="csrfmiddlewaretoken"]').val());
+                $('#transmitter-field').hide();
+                $('#transmitter-field-loading').show();
+                $('#station-field').hide();
+                $('#station-field-loading').show();
             }
         }).done(function(data) {
-            if (data.transmitters.length > 0) {
+            if (data.length == 1 && data[0].error) {
+                $('#transmitter-selection').html(`<option id="no-transmitter"
+                                                          value="" selected>
+                                                    No transmitter available
+                                                  </option>`).prop('disabled', true);
+                $('#transmitter-selection').selectpicker('refresh');
+            } else if (data.transmitters.length > 0) {
                 var transmitters_options = '';
                 if (filters.transmitter){
                     var is_transmitter_available = (data.transmitters.findIndex(tr => tr.uuid == filters.transmitter) > -1);
@@ -173,11 +197,8 @@ $(document).ready( function(){
                                                   </option>`).prop('disabled', true);
                 $('#transmitter-selection').selectpicker('refresh');
             }
-            if (callback) {
-                select_proper_stations(filters,callback);
-            } else {
-                select_proper_stations(filters);
-            }
+            $('#transmitter-field-loading').hide();
+            $('#transmitter-field').show();
         });
     }
 
@@ -245,20 +266,28 @@ $(document).ready( function(){
         update_schedule_button_status();
     });
 
+    $('#modal-schedule-observation').on('click', function() {
+        $(this).prop('disabled', true);
+        $('#schedule-observation').prop('disabled', true);
+        $('#calculate-observation').prop('disabled', true);
+    });
+
     $('#schedule-observation').on('click', function() {
         $('#windows-data').empty();
         var obs_counter = 0;
         var station_counter = 0;
         var warn_min_obs = parseInt(this.dataset.warnMinObs);
+        var transmitter_uuid = $('#transmitter-selection').find(':selected').val();
         $.each(suggested_data, function(i, station){
             let obs_counted = obs_counter;
             $.each(station.times, function(j, observation){
                 if(observation.selected){
                     var start = moment.utc(observation.starting_time).format('YYYY-MM-DD HH:mm:ss.SSS');
                     var end = moment.utc(observation.ending_time).format('YYYY-MM-DD HH:mm:ss.SSS');
-                    $('#windows-data').append('<input type="hidden" name="' + obs_counter + '-starting_time" value="' + start + '">');
-                    $('#windows-data').append('<input type="hidden" name="' + obs_counter + '-ending_time" value="' + end + '">');
-                    $('#windows-data').append('<input type="hidden" name="' + obs_counter + '-station" value="' + station.id + '">');
+                    $('#windows-data').append('<input type="hidden" name="obs-' + obs_counter + '-start" value="' + start + '">');
+                    $('#windows-data').append('<input type="hidden" name="obs-' + obs_counter + '-end" value="' + end + '">');
+                    $('#windows-data').append('<input type="hidden" name="obs-' + obs_counter + '-ground_station" value="' + station.id + '">');
+                    $('#windows-data').append('<input type="hidden" name="obs-' + obs_counter + '-transmitter_uuid" value="' + transmitter_uuid + '">');
                     obs_counter += 1;
                 }
             });
@@ -266,12 +295,15 @@ $(document).ready( function(){
                 station_counter += 1;
             }
         });
-        $('#windows-data').append('<input type="hidden" name="total" value="' + obs_counter + '">');
+        $('#windows-data').append('<input type="hidden" name="obs-TOTAL_FORMS" value="' + obs_counter + '">');
+        $('#windows-data').append('<input type="hidden" name="obs-INITIAL_FORMS" value="0">');
         if(obs_counter > warn_min_obs){
             $('#confirm-modal .counted-obs').text(obs_counter);
             $('#confirm-modal .counted-stations').text(station_counter);
             $('#confirm-modal').modal('show');
         } else if (obs_counter != 0){
+            $(this).prop('disabled', true);
+            $('#calculate-observation').prop('disabled', true);
             $('#form-obs').submit();
         }
     });
@@ -323,6 +355,7 @@ $(document).ready( function(){
         $('#hover-obs').hide();
         $('#windows-data').empty();
         $('#schedule-observation').prop('disabled', true);
+        $('#calculate-observation').prop('disabled', false);
     }
 
     $('#satellite-selection').on('changed.bs.select', function() {
@@ -341,6 +374,12 @@ $(document).ready( function(){
         select_proper_stations({
             transmitter: transmitter,
             station: station
+        }, function(){
+            if (obs_filter && obs_filter_dates && obs_filter_station && obs_filter_satellite){
+                $('#obs-selection-tools').hide();
+                $('#truncate-overlapped').click();
+                calculate_observation();
+            }
         });
         initiliaze_calculation(false);
     });
@@ -354,12 +393,17 @@ $(document).ready( function(){
     }
 
     function calculate_observation(){
-        $('.calculation-result').show();
         initiliaze_calculation(true);
         var url = '/prediction_windows/';
         var data = {};
-        data.start_time = $('#datetimepicker-start input').val();
-        data.end_time = $('#datetimepicker-end input').val();
+        // Add 5min for giving time to user schedule and avoid frequent start time errors.
+        // More on issue #686: https://gitlab.com/librespacefoundation/satnogs/satnogs-network/issues/686
+        if (!obs_filter_dates){
+            data.start = moment($('#datetimepicker-start input').val()).add(5,'minute').format('YYYY-MM-DD HH:mm');
+        } else {
+            data.start = $('#datetimepicker-start input').val();
+        }
+        data.end = $('#datetimepicker-end input').val();
         data.transmitter = $('#transmitter-selection').find(':selected').val();
         data.satellite = $('#satellite-selection').val();
         data.stations = $('#station-selection').val();
@@ -372,10 +416,10 @@ $(document).ready( function(){
         } else if (data.stations.length == 0 || (data.stations.length == 1 && data.stations[0] == '')) {
             $('#windows-data').html('<span class="text-danger">You should select a Station first.</span>');
             return;
-        } else if (data.start_time.length == 0) {
+        } else if (data.start.length == 0) {
             $('#windows-data').html('<span class="text-danger">You should select a Start Time first.</span>');
             return;
-        } else if (data.end_time.length == 0) {
+        } else if (data.end.length == 0) {
             $('#windows-data').html('<span class="text-danger">You should select an End Time first.</span>');
             return;
         }
@@ -449,7 +493,7 @@ $(document).ready( function(){
                 });
 
                 if (dc > 0) {
-                    timeline_init(data.start_time, data.end_time, suggested_data);
+                    timeline_init(data.start, data.end, suggested_data);
                 } else {
                     var empty_msg = 'No Ground Station available for this observation window';
                     $('#windows-data').html('<span class="text-danger">' + empty_msg + '</span>');
@@ -459,9 +503,9 @@ $(document).ready( function(){
     }
 
     function timeline_init(start, end, payload){
-        var start_time_timeline = moment.utc(start).valueOf();
-        var end_time_timeline = moment.utc(end).valueOf();
-        var period = end_time_timeline - start_time_timeline;
+        var start_timeline = moment.utc(start).valueOf();
+        var end_timeline = moment.utc(end).valueOf();
+        var period = end_timeline - start_timeline;
         var tick_interval = 15;
         var tick_time = d3.time.minutes;
 
@@ -479,8 +523,8 @@ $(document).ready( function(){
         $('#timeline').empty();
 
         var chart = d3.timeline()
-            .beginning(start_time_timeline)
-            .ending(end_time_timeline)
+            .beginning(start_timeline)
+            .ending(end_timeline)
             .mouseout(function () {
                 $('#hover-obs').fadeOut(100);
             })
@@ -491,8 +535,8 @@ $(document).ready( function(){
                     var colors = chart.colors();
                     div.find('.coloredDiv').css('background-color', colors(i));
                     div.find('#name').text(datum.label);
-                    div.find('#start-time').text(moment.utc(d.starting_time).format('YYYY-MM-DD HH:mm:ss'));
-                    div.find('#end-time').text(moment.utc(d.ending_time).format('YYYY-MM-DD HH:mm:ss'));
+                    div.find('#start').text(moment.utc(d.starting_time).format('YYYY-MM-DD HH:mm:ss'));
+                    div.find('#end').text(moment.utc(d.ending_time).format('YYYY-MM-DD HH:mm:ss'));
                     div.find('#details').text('⤉ ' + d.az_start + '° ⇴ ' + d.elev_max + '° ⤈ ' + d.az_end + '°');
                     const groundstation = {
                         lat: datum.lat,
@@ -591,13 +635,11 @@ $(document).ready( function(){
             satellite: obs_filter_satellite,
             transmitter: obs_filter_transmitter,
             station: obs_filter_station
-        }, function(){
-            if (obs_filter_dates && obs_filter_station && obs_filter_satellite){
-                $('#obs-selection-tools').hide();
-                $('#truncate-overlapped').click();
-                calculate_observation();
-            }
         });
+    } else {
+        // Focus on satellite field
+        $('#satellite-selection').selectpicker('refresh');
+        $('#satellite-selection').selectpicker('toggle');
     }
 
     // Hotkeys bindings

network/static/js/observation_view.js

@@ -1,4 +1,4 @@
-/* global WaveSurfer URI calcPolarPlotSVG */
+/* global WaveSurfer calcPolarPlotSVG */
 
 $(document).ready(function() {
     'use strict';
@@ -20,7 +20,6 @@ $(document).ready(function() {
     $('.wave').each(function(){
         var $this = $(this);
         var wid = $this.data('id');
-        var wavesurfer = Object.create(WaveSurfer);
         var data_audio_url = $this.data('audio');
         var container_el = '#data-' + wid;
         $(container_el).css('opacity', '0');
@@ -42,10 +41,18 @@ $(document).ready(function() {
             progressDiv.css('display', 'none');
         };
 
-        wavesurfer.init({
+        var wavesurfer = WaveSurfer.create({
             container: container_el,
             waveColor: '#bf7fbf',
-            progressColor: 'purple'
+            progressColor: 'purple',
+            plugins: [
+                WaveSurfer.spectrogram.create({
+                    wavesurfer: wavesurfer,
+                    container: '#wave-spectrogram',
+                    fftSamples: 256,
+                    windowFunc: 'hann'
+                })
+            ]
         });
 
         wavesurfer.on('destroy', hideProgress);
@@ -67,13 +74,6 @@ $(document).ready(function() {
 
         wavesurfer.on('ready', function() {
             hideProgress();
-            var spectrogram = Object.create(WaveSurfer.Spectrogram);
-            spectrogram.init({
-                wavesurfer: wavesurfer,
-                container: '#wave-spectrogram',
-                fftSamples: 256,
-                windowFunc: 'hann'
-            });
 
             //$playbackTime.text(formatTime(wavesurfer.getCurrentTime()));
             $playbackTime.text(formatTime(wavesurfer.getCurrentTime()));
@@ -90,8 +90,8 @@ $(document).ready(function() {
     });
 
     // Handle Observation tabs
-    var uri = new URI(location.href);
-    var tab = uri.hash();
+    var uri = new URL(location.href);
+    var tab = uri.hash;
     $('.observation-tabs li a[href="' + tab + '"]').tab('show');
 
     // Delete confirmation
@@ -150,7 +150,7 @@ $(document).ready(function() {
                 $('#vetting-spinner').show();
             }
         }).done(function(results) {
-            if (results.hasOwnProperty('error')) {
+            if (Object.prototype.hasOwnProperty.call(results, 'error')) {
                 var error_msg = results.error;
                 show_alert('danger',error_msg);
             } else {

network/static/js/satellite.js

@@ -20,7 +20,7 @@ $(document).ready(function() {
             modal.find('#bad-sat-obs').attr('href', '/observations/?future=0&good=0&bad=1&unvetted=0&failed=0&norad=' + satlink.data('id'));
             modal.find('#future-sat-obs').attr('href', '/observations/?future=1&good=0&bad=0&unvetted=0&failed=0&norad=' + satlink.data('id'));
             modal.find('.satellite-success-rate').text(data.success_rate + '%');
-            modal.find('.satellite-total-obs').text(data.data_count);
+            modal.find('.satellite-total-obs').text(data.total_count);
             modal.find('.satellite-good').text(data.good_count);
             modal.find('.satellite-unvetted').text(data.unvetted_count);
             modal.find('.satellite-bad').text(data.bad_count);
@@ -31,14 +31,6 @@ $(document).ready(function() {
                 if(transmitter.alive){
                     transmitter_status = '-success';
                 }
-                var good_percentage = 0;
-                var unvetted_percentage = 0;
-                var bad_percentage = 0;
-                if(transmitter.total_count > 0){
-                    good_percentage = Math.round((transmitter.good_count / transmitter.total_count) * 100);
-                    unvetted_percentage = Math.round((transmitter.unvetted_count / transmitter.total_count) * 100);
-                    bad_percentage = Math.round((transmitter.bad_count / transmitter.total_count) * 100);
-                }
                 modal.find('#transmitters').append(`
                     <div class="col-md-12 transmitter">
                       <div class="panel panel` + transmitter_status + `">
@@ -52,16 +44,20 @@ $(document).ready(function() {
                             <div class="progress pull-right">
                               <div class="progress-bar progress-bar-success transmitter-good"
                                           data-toggle="tooltip" data-placement="bottom"
-                                          title="` + good_percentage  + '% (' + transmitter.good_count + `) Good"
-                                          style="width:` + good_percentage + `%"></div>
+                                          title="` + transmitter.success_rate  + '% (' + transmitter.good_count + `) Good"
+                                          style="width:` + transmitter.success_rate + `%"></div>
                               <div class="progress-bar progress-bar-warning transmitter-unvetted"
                                           data-toggle="tooltip" data-placement="bottom"
-                                          title="` + unvetted_percentage  + '% (' + transmitter.unvetted_count + `) Unvetted"
-                                          style="width:` + unvetted_percentage + `%"></div>
+                                          title="` + transmitter.unvetted_rate  + '% (' + transmitter.unvetted_count + `) Unvetted"
+                                          style="width:` + transmitter.unvetted_rate + `%"></div>
                               <div class="progress-bar progress-bar-danger transmitter-bad"
                                           data-toggle="tooltip" data-placement="bottom"
-                                          title="` + bad_percentage  + '% (' + transmitter.bad_count + `) Bad"
-                                          style="width:` + bad_percentage + `%"></div>
+                                          title="` + transmitter.bad_rate  + '% (' + transmitter.bad_count + `) Bad"
+                                          style="width:` + transmitter.bad_rate + `%"></div>
+                              <div class="progress-bar progress-bar-info transmitter-info"
+                                          data-toggle="tooltip" data-placement="bottom"
+                                          title="` + transmitter.future_rate  + '% (' + transmitter.future_count + `) Future"
+                                          style="width:` + transmitter.future_rate + `%"></div>
                             </div>
                           </span>
                         </div>

network/static/js/station_edit.js

@@ -20,8 +20,12 @@ $(document).ready(function() {
         value: utilization_value});
 
     var station_image = $('#station-image').data('existing');
-    $('#station-image').fileinput({
-        initialPreview: station_image,
-        initialPreviewAsData: true,
-    });
+    if(station_image === undefined){
+        $('#station-image').fileinput();
+    } else {
+        $('#station-image').fileinput({
+            initialPreview: station_image,
+            initialPreviewAsData: true,
+        });
+    }
 });