From 06ff9469c21ec72095500749cb7493d7e69490b1 Mon Sep 17 00:00:00 2001 From: Corey Shields Date: Fri, 25 Mar 2016 15:52:45 -0400 Subject: [PATCH] Initial data vetting/verification system Model change (with migration 0006) adds 3 fields to Data: vetted_status (charfield with options for data status, default "unknown") vetted_user (who vetted the data) vetted_datetime (when it was vetted) In addition, various boolean functions are added for the Data model to check statuses. More functions are added to the Observation model to check status of verification within an observation as well, assuming multiple data entries in an Observation. With these, I also changed "has_data" to "has_submitted_data" to be more specific alongside the others. For UX, we add a green check sign or red removal sign to the data header in Observation view (along with green/red datetime in the footer) if a data is verified good or bad, respectively. If there is an unknown status, the data header is given a thumbs-up and thumbs-down button to verify the data good or bad. These icons are only offered to is_staff, the observation requestor, and any station owner in the observation. These buttons trigger new URLs/functions in view: data_verify(id) data_mark_bad(id) Returning the user back to the originating Observation page. In the observation lists I changed the coloring of the ID button to be: Future: light blue (same) No uploaded data and/or all vetted bad data: red Some or all unvetted data with no verified good data: orange Some or all verified good data: green These changes are reflected in the observations.html, home.html, and user_detail.html templates. solves satnogs/satnogs-network#171 --- .../migrations/0006_auto_20160325_0126.py | 33 ++++++++++++++ network/base/models.py | 43 ++++++++++++++++++- network/base/urls.py | 2 + network/base/views.py | 35 ++++++++++++++- network/static/css/app.css | 8 ++++ network/templates/base/home.html | 14 +++--- network/templates/base/observation_view.html | 21 ++++++++- network/templates/base/observations.html | 12 +++--- network/templates/users/user_detail.html | 14 +++--- 9 files changed, 163 insertions(+), 19 deletions(-) create mode 100644 network/base/migrations/0006_auto_20160325_0126.py diff --git a/network/base/migrations/0006_auto_20160325_0126.py b/network/base/migrations/0006_auto_20160325_0126.py new file mode 100644 index 0000000..a1f486e --- /dev/null +++ b/network/base/migrations/0006_auto_20160325_0126.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2016-03-25 01:26 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('base', '0005_auto_20160320_1619'), + ] + + operations = [ + migrations.AddField( + model_name='data', + name='vetted_datetime', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='data', + name='vetted_status', + field=models.CharField(choices=[(b'unknown', b'Unknown'), (b'verified', b'Verified'), (b'data_not_verified', b'Has Data, Not Verified'), (b'no_data', b'No Data')], default=b'unknown', max_length=10), + ), + migrations.AddField( + model_name='data', + name='vetted_user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='vetted_user_set', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/network/base/models.py b/network/base/models.py index 766065d..00d3b5b 100644 --- a/network/base/models.py +++ b/network/base/models.py @@ -20,6 +20,12 @@ ANTENNA_TYPES = ( ('parabolic', 'Parabolic'), ('vertical', 'Verical'), ) +OBSERVATION_STATUSES = ( + ('unknown', 'Unknown'), + ('verified', 'Verified'), + ('data_not_verified', 'Has Data, Not Verified'), + ('no_data', 'No Data'), +) class Rig(models.Model): @@ -216,10 +222,26 @@ class Observation(models.Model): deletion = self.start - timedelta(minutes=int(settings.OBSERVATION_MAX_DELETION_RANGE)) return deletion > now() + # observation has at least 1 payload submitted, no verification taken into account @property - def has_data(self): + def has_submitted_data(self): return self.data_set.exclude(payload='').count() + # observaton has at least 1 payload that has been verified good + @property + def has_verified_data(self): + return self.data_set.filter(vetted_status='verified').count() + + # observation is vetted to be all bad data + @property + def has_no_data(self): + return self.data_set.filter(vetted_status='verified').count() == self.data_set.count() + + # observation has at least 1 payload left unvetted + @property + def has_unvetted_data(self): + return self.data_set.filter(vetted_status='unknown').count() + def __unicode__(self): return "%d" % self.id @@ -231,10 +253,29 @@ class Data(models.Model): observation = models.ForeignKey(Observation) ground_station = models.ForeignKey(Station) payload = models.FileField(upload_to='data_payloads', blank=True, null=True) + vetted_datetime = models.DateTimeField(null=True, blank=True) + vetted_user = models.ForeignKey(User, related_name="vetted_user_set", null=True, blank=True) + vetted_status = models.CharField(choices=OBSERVATION_STATUSES, + max_length=10, default='unknown') @property def is_past(self): return self.end < now() + # this payload has been vetted good/bad by someone + @property + def is_vetted(self): + return not self.vetted_status == 'unknown' + + # this payload has been vetted as good by someone + @property + def is_verified(self): + return self.vetted_status == 'verified' + + # this payload has been vetted as bad by someone + @property + def is_no_data(self): + return self.vetted_status == 'no_data' + class Meta: ordering = ['-start', '-end'] diff --git a/network/base/urls.py b/network/base/urls.py index bde1de7..b087216 100644 --- a/network/base/urls.py +++ b/network/base/urls.py @@ -17,6 +17,8 @@ base_urlpatterns = ([ url(r'^observations/new/$', views.observation_new, name='observation_new'), url(r'^prediction_windows/(?P[\w.@+-]+)/(?P.+)/(?P.+)/$', views.prediction_windows, name='prediction_windows'), + url(r'^data_verify/(?P[0-9]+)/$', views.data_verify, name='data_verify'), + url(r'^data_mark_bad/(?P[0-9]+)/$', views.data_mark_bad, name='data_mark_bad'), # Stations url(r'^stations/$', views.stations_list, name='stations_list'), diff --git a/network/base/views.py b/network/base/views.py index 73903aa..97c4545 100644 --- a/network/base/views.py +++ b/network/base/views.py @@ -274,6 +274,14 @@ def observation_view(request, id): observation = get_object_or_404(Observation, id=id) data = Data.objects.filter(observation=observation) + # not all users will be able to vet data within an observation, allow + # staff, observation requestors, and station owners + is_vetting_user = False + if request.user == observation.author or \ + data.filter(ground_station__in=Station.objects.filter(owner=request.user)).count or \ + request.user.is_staff: + is_vetting_user = True + if settings.ENVIRONMENT == 'production': discuss_slug = 'https://community.satnogs.org/t/observation-{0}-{1}-{2}' \ .format(observation.id, slugify(observation.satellite.name), @@ -292,10 +300,11 @@ def observation_view(request, id): return render(request, 'base/observation_view.html', {'observation': observation, 'data': data, 'has_comments': has_comments, - 'discuss_url': discuss_url, 'discuss_slug': discuss_slug}) + 'discuss_url': discuss_url, 'discuss_slug': discuss_slug, + 'is_vetting_user': is_vetting_user}) return render(request, 'base/observation_view.html', - {'observation': observation, 'data': data}) + {'observation': observation, 'data': data, 'is_vetting_user': is_vetting_user}) @login_required @@ -311,6 +320,28 @@ def observation_delete(request, id): return redirect(reverse('base:observations_list')) +@login_required +def data_verify(request, id): + me = request.user + data = get_object_or_404(Data, id=id) + data.vetted_status = 'verified' + data.vetted_user = me + data.vetted_datetime = datetime.today() + data.save(update_fields=['vetted_status', 'vetted_user', 'vetted_datetime']) + return redirect(reverse('base:observation_view', kwargs={'id': data.observation})) + + +@login_required +def data_mark_bad(request, id): + me = request.user + data = get_object_or_404(Data, id=id) + data.vetted_status = 'no_data' + data.vetted_user = me + data.vetted_datetime = datetime.today() + data.save(update_fields=['vetted_status', 'vetted_user', 'vetted_datetime']) + return redirect(reverse('base:observation_view', kwargs={'id': data.observation})) + + def stations_list(request): """View to render Stations page.""" stations = Station.objects.all() diff --git a/network/static/css/app.css b/network/static/css/app.css index b7c7295..a34df09 100644 --- a/network/static/css/app.css +++ b/network/static/css/app.css @@ -310,3 +310,11 @@ code.log p { .panel-title.link { color: #337ab7; } + +.verified { + color: #009933; +} + +.no_data { + color: #cc3300; +} \ No newline at end of file diff --git a/network/templates/base/home.html b/network/templates/base/home.html index d0ce536..2220beb 100644 --- a/network/templates/base/home.html +++ b/network/templates/base/home.html @@ -118,11 +118,15 @@ + {% if observation.has_verified_data %} + label-success" title="There is known good data in this observation" + {% elif observation.is_future %} + label-info" title="This observation is in the future" + {% elif observation.has_unvetted_data %} + label-warning" title="There is data that needs vetting in this observation" + {% else %} + label-danger" title="No good data in this observation" + {% endif %}> {{ observation.id }} diff --git a/network/templates/base/observation_view.html b/network/templates/base/observation_view.html index 181cf21..f303a34 100644 --- a/network/templates/base/observation_view.html +++ b/network/templates/base/observation_view.html @@ -110,6 +110,25 @@ {{ data.ground_station }} + {% if not data.is_vetted and is_vetting_user %} + + + + + + + + {% elif data.is_verified %} + + {% elif data.is_no_data %} + + {% endif %}
@@ -124,7 +143,7 @@
{% endif %} -
+
Timeframe {{ data.start|date:"Y-m-d H:i:s" }} ~ {{ data.end|date:"Y-m-d H:i:s" }} + label-danger" title="No good data in this observation" + {% endif %}> {{ observation.id }} diff --git a/network/templates/users/user_detail.html b/network/templates/users/user_detail.html index 82e4600..4c36b1a 100644 --- a/network/templates/users/user_detail.html +++ b/network/templates/users/user_detail.html @@ -118,11 +118,15 @@ + {% if observation.has_verified_data %} + label-success" title="There is known good data in this observation" + {% elif observation.is_future %} + label-info" title="This observation is in the future" + {% elif observation.has_unvetted_data %} + label-warning" title="There is data that needs vetting in this observation" + {% else %} + label-danger" title="No good data in this observation" + {% endif %}> {{ observation.id }}