From 67ad4e0d99b8657ad6c06b271bcc7b1476fdcdf3 Mon Sep 17 00:00:00 2001 From: Nikos Roussos Date: Fri, 24 Nov 2017 23:38:25 +0100 Subject: [PATCH] Add experimental support for archive.org --- env-dist | 6 ++-- .../migrations/0029_auto_20171216_1712.py | 30 +++++++++++++++++++ network/base/models.py | 3 ++ network/base/tasks.py | 28 +++++++++++++++++ network/settings.py | 5 ++++ requirements/production.txt | 22 ++++++++++++++ 6 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 network/base/migrations/0029_auto_20171216_1712.py diff --git a/env-dist b/env-dist index 4e050c9..25d20f0 100644 --- a/env-dist +++ b/env-dist @@ -8,6 +8,6 @@ DATABASE_URL='sqlite:///db.sqlite3' DB_API_ENDPOINT='https://db.satnogs.org/api/' # Cache -CACHE_BACKEND='django.core.cache.backends.locmem.LocMemCache' -CACHE_LOCATION='unique-snowflake' -CACHE_CLIENT_CLASS=None +CACHE_BACKEND = 'django.core.cache.backends.locmem.LocMemCache' +CACHE_LOCATION = 'unique-snowflake' +CACHE_CLIENT_CLASS = None diff --git a/network/base/migrations/0029_auto_20171216_1712.py b/network/base/migrations/0029_auto_20171216_1712.py new file mode 100644 index 0000000..4f913eb --- /dev/null +++ b/network/base/migrations/0029_auto_20171216_1712.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2017-12-16 17:12 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0028_auto_20171127_2154'), + ] + + operations = [ + migrations.AddField( + model_name='observation', + name='archive_identifier', + field=models.CharField(blank=True, max_length=255), + ), + migrations.AddField( + model_name='observation', + name='archive_url', + field=models.URLField(blank=True, null=True), + ), + migrations.AddField( + model_name='observation', + name='archived', + field=models.BooleanField(default=False), + ), + ] diff --git a/network/base/models.py b/network/base/models.py index 91de922..0969313 100644 --- a/network/base/models.py +++ b/network/base/models.py @@ -295,6 +295,9 @@ class Observation(models.Model): rise_azimuth = models.FloatField(blank=True, null=True) max_altitude = models.FloatField(blank=True, null=True) set_azimuth = models.FloatField(blank=True, null=True) + archived = models.BooleanField(default=False) + archive_identifier = models.CharField(max_length=255, blank=True) + archive_url = models.URLField(blank=True, null=True) @property def is_past(self): diff --git a/network/base/tasks.py b/network/base/tasks.py index cd14a96..45cb3b8 100644 --- a/network/base/tasks.py +++ b/network/base/tasks.py @@ -1,7 +1,9 @@ from datetime import timedelta import json +import os import urllib2 +from internetarchive import upload from orbit import satellite from django.conf import settings @@ -97,6 +99,30 @@ def fetch_data(): new_transmitter.save() +@app.task +def archive_audio(obs_id): + obs = Observation.objects.get(id=obs_id) + suffix = '-{0}'.format(settings.ENVIRONMENT) + if settings.ENVIRONMENT == 'production': + suffix = '' + identifier = 'satnogs{0}-observation-{1}'.format(suffix, obs.id) + if not obs.archived and obs.payload: + if os.path.isfile(obs.payload.path): + ogg = obs.payload.path + md = dict(collection=settings.ARCHIVE_COLLECTION, + title=identifier, + mediatype='audio') + res = upload(identifier, files=[ogg], metadata=md, + access_key=settings.S3_ACCESS_KEY, + secret_key=settings.S3_SECRET_KEY) + if res[0].status_code == 200: + obs.archived = True + obs.archive_url = res[0].url + obs.archive_identifier = identifier + obs.save() + os.remove(obs.payload.path) + + @app.task def clean_observations(): """Task to clean up old observations that lack actual data.""" @@ -106,3 +132,5 @@ def clean_observations(): for obs in observations: if not obs.is_verified: obs.delete() + else: + archive_audio.delay(obs.id) diff --git a/network/settings.py b/network/settings.py index b950a69..815783a 100644 --- a/network/settings.py +++ b/network/settings.py @@ -332,6 +332,11 @@ ITEMS_PER_PAGE = 25 # User settings AVATAR_GRAVATAR_DEFAULT = config('AVATAR_GRAVATAR_DEFAULT', default='mm') +# Archive.org +S3_ACCESS_KEY = config('S3_ACCESS_KEY', default='') +S3_SECRET_KEY = config('S3_SECRET_KEY', default='') +ARCHIVE_COLLECTION = config('ARCHIVE_COLLECTION', default='test_collection') + if ENVIRONMENT == 'dev': # Disable template caching for backend in TEMPLATES: diff --git a/requirements/production.txt b/requirements/production.txt index 3f914f6..3995cf3 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -7,6 +7,9 @@ django-shortuuidfield==0.1.3 \ celery==4.1.0 \ --hash=sha256:81a67f0d53a688ec2bc8557bd5d6d7218f925a6f2e6df80e01560de9e28997ec \ --hash=sha256:77ff3730198d6a17b3c1f05579ebe570b579efb35f6d7e13dba3b1368d068b35 +internetarchive==1.7.4 \ + --hash=sha256:87c781464889d44bd496247b99cdff4392a86438ac48515a09a6e52f526e5025 \ + --hash=sha256:41c5cc3cbc143bb789a055cbc3c12f162f186af63c70490f48688a83136ff1c0 # Configuration unicode-slugify==0.1.3 \ @@ -217,3 +220,22 @@ kombu==4.1.0 \ vine==1.1.4 \ --hash=sha256:6849544be74ec3638e84d90bc1cf2e1e9224cc10d96cd4383ec3f69e9bce077b \ --hash=sha256:52116d59bc45392af9fdd3b75ed98ae48a93e822cee21e5fda249105c59a7a72 +args==0.1.0 \ + --hash=sha256:a785b8d837625e9b61c39108532d95b85274acd679693b71ebb5156848fcf814 +docopt==0.6.2 \ + --hash=sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491 +schema==0.6.6 \ + --hash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ + --hash=sha256:758b59ffc654d3794f6edd6e67db948f26164ddcd3cd7dde8bbeefc4934482d3 \ + --hash=sha256:0e1b5453ea462a10744c612bc8989e2a058609beac7deba9edbac8c5b1a2e2b0 +jsonpointer==1.14 \ + --hash=sha256:381b613fd1afd65376fb28948c4744f035e47ab049a9fdde0c48cc1c30b68559 \ + --hash=sha256:c681af823545c731b7b38aedd5d4eee4c5eff87bc0f25e0ff25444a4189eac4d +jsonpatch==1.20 \ + --hash=sha256:4b084a9c3f4b256d7e1fab66efb13c3560f107d9b21939ef3ee6d5858fa636ca \ + --hash=sha256:b12594a0cfe634bdd2a5c027250af2dd84932d493568e88ae722871edc3eb02d +clint==0.5.1 \ + --hash=sha256:05224c32b1075563d0b16d0015faaf9da43aa214e4a2140e51f08789e7a4c5aa +backports.csv==1.0.5 \ + --hash=sha256:d3b0cefaaca92be3d2d4ceec140827cae1d871da7fff5db70697d72328357d65 \ + --hash=sha256:8c421385cbc6042ba90c68c871c5afc13672acaf91e1508546d6cda6725ebfc6