Various download Internet Archive cruft

wut-aria-active

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+import time
+import xmlrpc.client as xmlrpclib
+from pathlib import Path
+from pprint import pprint
+
+s = xmlrpclib.ServerProxy('http://localhost:4800/rpc')
+path=Path('/srv/dl')
+
+active=s.aria2.tellActive("token:yajnuAdCemNathNojdi")
+
+pprint(active)
+

wut-aria-add

@@ -12,15 +12,16 @@ path=Path('/srv/dl')
 
 # Added torrents
 # dt-10
-torrents=sorted(list(path.glob('**/satnogs-observations-0001?0001-000??0000/satnogs-observations-*_archive.torrent')))
-torrents=sorted(list(path.glob('**/satnogs-observations-0002?0001-000??0000/satnogs-observations-*_archive.torrent')))
-torrents=sorted(list(path.glob('**/satnogs-observations-0003?0001-000??0000/satnogs-observations-*_archive.torrent')))
-torrents=sorted(list(path.glob('**/satnogs-observations-0004?0001-000??0000/satnogs-observations-*_archive.torrent')))
-torrents=sorted(list(path.glob('**/satnogs-observations-0005?0001-000??0000/satnogs-observations-*_archive.torrent')))
-torrents=sorted(list(path.glob('**/satnogs-observations-0006?0001-000??0000/satnogs-observations-*_archive.torrent')))
-torrents=sorted(list(path.glob('**/satnogs-observations-0007?0001-000??0000/satnogs-observations-*_archive.torrent')))
-torrents=sorted(list(path.glob('**/satnogs-observations-0008?0001-000??0000/satnogs-observations-*_archive.torrent')))
-torrents=sorted(list(path.glob('**/satnogs-observations-0009?0001-000??0000/satnogs-observations-*_archive.torrent')))
+torrents=sorted(list(path.glob('**/satnogs-observations-000000001-000010000/satnogs-observations-*_archive.torrent')))
+#torrents=sorted(list(path.glob('**/satnogs-observations-0001?0001-000??0000/satnogs-observations-*_archive.torrent')))
+#torrents=sorted(list(path.glob('**/satnogs-observations-0002?0001-000??0000/satnogs-observations-*_archive.torrent')))
+#torrents=sorted(list(path.glob('**/satnogs-observations-0003?0001-000??0000/satnogs-observations-*_archive.torrent')))
+#torrents=sorted(list(path.glob('**/satnogs-observations-0004?0001-000??0000/satnogs-observations-*_archive.torrent')))
+#torrents=sorted(list(path.glob('**/satnogs-observations-0005?0001-000??0000/satnogs-observations-*_archive.torrent')))
+#torrents=sorted(list(path.glob('**/satnogs-observations-0006?0001-000??0000/satnogs-observations-*_archive.torrent')))
+#torrents=sorted(list(path.glob('**/satnogs-observations-0007?0001-000??0000/satnogs-observations-*_archive.torrent')))
+#torrents=sorted(list(path.glob('**/satnogs-observations-0008?0001-000??0000/satnogs-observations-*_archive.torrent')))
+#torrents=sorted(list(path.glob('**/satnogs-observations-0009?0001-000??0000/satnogs-observations-*_archive.torrent')))
 
 for i in torrents:
     print(i.name)

wut-aria-info

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+import time
+import xmlrpc.client as xmlrpclib
+from pathlib import Path
+from pprint import pprint
+
+s = xmlrpclib.ServerProxy('http://localhost:4800/rpc')
+path=Path('/srv/dl')
+
+info=s.aria2.getSessionInfo("token:yajnuAdCemNathNojdi")
+
+pprint(info)
+

wut-aria-methods

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+import time
+import xmlrpc.client as xmlrpclib
+from pathlib import Path
+from pprint import pprint
+
+s = xmlrpclib.ServerProxy('http://localhost:4800/rpc')
+path=Path('/srv/dl')
+
+methods=s.system.listMethods()
+
+pprint((sorted)(methods))
+

wut-aria-shutdown

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+import time
+import xmlrpc.client as xmlrpclib
+from pathlib import Path
+from pprint import pprint
+
+s = xmlrpclib.ServerProxy('http://localhost:4800/rpc')
+path=Path('/srv/dl')
+
+shutdown=s.aria2.shutdown("token:yajnuAdCemNathNojdi")
+
+pprint(shutdown)
+

wut-aria-stat

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+import time
+import xmlrpc.client as xmlrpclib
+from pathlib import Path
+from pprint import pprint
+
+s = xmlrpclib.ServerProxy('http://localhost:4800/rpc')
+path=Path('/srv/dl')
+
+stat=s.aria2.getGlobalStat("token:yajnuAdCemNathNojdi")
+
+pprint(stat)
+

wut-aria-stopped

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+import time
+import xmlrpc.client as xmlrpclib
+from pathlib import Path
+from pprint import pprint
+
+s = xmlrpclib.ServerProxy('http://localhost:4800/rpc')
+path=Path('/srv/dl')
+
+stopped=s.aria2.tellStopped("token:yajnuAdCemNathNojdi", 0, 9999)
+
+pprint(stopped)
+

wut-aria-waiting

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+import time
+import xmlrpc.client as xmlrpclib
+from pathlib import Path
+from pprint import pprint
+
+s = xmlrpclib.ServerProxy('http://localhost:4800/rpc')
+path=Path('/srv/dl')
+
+waiting=s.aria2.tellWaiting("token:yajnuAdCemNathNojdi", 0, 9999)
+
+pprint(waiting)
+

wut-ia

@@ -1,8 +0,0 @@
-#!/usr/bin/env python3
-#
-# wut-ia --- Download SatNOGS data from the Internet Archive at archive.org.
-#
-# https://archive.org/details/satnogs
-
-import internetarchive
-

wut-ia-sha1

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+#
+# wut-ia-sha1 --- Verify downloaded files checksums
+#
+# XXX uses both ET and xml.parsers.expat
+
+import argparse
+import os
+from xml.parsers.expat import ParserCreate, ExpatError, errors
+from pathlib import Path
+import hashlib
+import xml.etree.ElementTree as ET
+
+dl_dir=Path('/srv/dl')
+
+def convertxml(xmlfile, xml_attribs=True):
+    with open(xmlfile, "rb") as f:
+        d = xmltodict.parse(f, xml_attribs=xml_attribs, process_namespaces=False)
+        return d
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='sha1 check Internet Archive downloads')
+    parser.add_argument('observations',
+			type=str,
+			help='Observation set. Example: 006050001-006060000')
+    args = parser.parse_args()
+    obs_set = 'satnogs-observations-' + args.observations
+    obs_dir = Path(dl_dir, obs_set)
+    filename_xml = obs_set + '_files.xml'
+    print('filename XML:', filename_xml)
+    xmlfile = Path(obs_dir, filename_xml)
+    p = ParserCreate()
+    try:
+        p.ParseFile(open(xmlfile, 'rb'))
+    except:
+        print('No XML file to process')
+        exit()
+
+    return(xmlfile, obs_dir)
+
+def get_sha1(filename):
+    sha1 = hashlib.sha1()
+    try:
+        with open(filename, 'rb') as f:
+            while True:
+                data = f.read(1048576)
+                if not data:
+                    break
+                sha1.update(data)
+        return sha1.hexdigest()
+
+    except:
+         status='EXCEPTION'
+
+def process_set(xmlfile, obs_dir):
+    root_node = ET.parse(xmlfile).getroot()
+    for tag in root_node.findall('file'):
+        name = tag.get('name')
+        for file_sha1 in tag.iter('sha1'):
+            filename = Path(obs_dir, name)
+            sha1_hash=get_sha1(filename)
+            if sha1_hash == file_sha1.text:
+                print('OK ', end='')
+            else:
+                print('FAIL ', end='')
+            print(name)
+
+def main():
+    xmlfile, obs_dir = parse_args()
+    process_set(xmlfile, obs_dir)
+
+if __name__ == "__main__":
+    main();
+

wut-ia-torrents

@@ -23,7 +23,7 @@ for i in search_items('identifier:satnogs-observations-*'):
     checksum=True, destdir=obs_dl,
     retries=4, ignore_errors=True)
 
-    download(obs_id, verbose=True, glob_pattern='*._files.xml',
+    download(obs_id, verbose=True, glob_pattern='*_files.xml',
     checksum=True, destdir=obs_dl,
     retries=4, ignore_errors=True)