Compare commits
2 Commits
bc590a65bf
...
e244a8279e
Author | SHA1 | Date |
---|---|---|
![]() |
e244a8279e | |
![]() |
d492d4ef46 |
|
@ -69,10 +69,14 @@ black gbif-cruft*
|
||||||
# TODO
|
# TODO
|
||||||
Perhaps:
|
Perhaps:
|
||||||
|
|
||||||
* rclone GBIF snapshot, approximately 250 gigs of parquet files.
|
* rclone GBIF snapshot, approximately 210 gigs of parquet files.
|
||||||
* Read parquet files with local tools.
|
* Read parquet files with local tools. Perhaps pyarrow, pqv, dask,
|
||||||
|
* Parquet with GraphQL? graphique.
|
||||||
* Perhaps import parquet files into sota db.
|
* Perhaps import parquet files into sota db.
|
||||||
* Test on ppc64le.
|
* Test on ppc64le.
|
||||||
|
* Type hinted Python dataclass for GBIF API.
|
||||||
|
* Mypyc or similar.
|
||||||
|
* sota media storage.
|
||||||
|
|
||||||
|
|
||||||
# Upstream
|
# Upstream
|
||||||
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
import click
|
||||||
|
from gbif_cruft.__about__ import __version__
|
||||||
|
from gbif_cruft.lazy_group import LazyGroup
|
||||||
|
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
@click.argument('filename', nargs=1, default="/srv/gbif/gbif_2023-10-01/occurrence.parquet/000000", type=str)
|
||||||
|
def cli(filename):
|
||||||
|
print(filename)
|
|
@ -4,13 +4,14 @@
|
||||||
import click
|
import click
|
||||||
|
|
||||||
from gbif_cruft.__about__ import __version__
|
from gbif_cruft.__about__ import __version__
|
||||||
from pygbif import species
|
from gbif_cruft.lazy_group import LazyGroup
|
||||||
import json
|
|
||||||
|
|
||||||
@click.group(context_settings={"help_option_names": ["-h", "--help"]}, invoke_without_command=True)
|
|
||||||
@click.option("--search", default="axolotl", help="Search GBIF.")
|
|
||||||
@click.version_option(version=__version__, prog_name="gbif-cruft")
|
@click.version_option(version=__version__, prog_name="gbif-cruft")
|
||||||
def gbif_cruft(search):
|
@click.group(
|
||||||
foo = species.name_suggest(q=search)
|
cls=LazyGroup,
|
||||||
foo = json.dumps(foo)
|
lazy_subcommands={"search": "search.cli", "filedb": "filedb.cli"},
|
||||||
print(foo)
|
context_settings={"help_option_names": ["-h", "--help"]},
|
||||||
|
)
|
||||||
|
|
||||||
|
def gbif_cruft():
|
||||||
|
pass
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
../lazy_group.py
|
|
@ -0,0 +1,37 @@
|
||||||
|
import importlib
|
||||||
|
import click
|
||||||
|
|
||||||
|
class LazyGroup(click.Group):
|
||||||
|
def __init__(self, *args, lazy_subcommands=None, **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
# lazy_subcommands is a map of the form:
|
||||||
|
#
|
||||||
|
# {command-name} -> {module-name}.{command-object-name}
|
||||||
|
#
|
||||||
|
self.lazy_subcommands = lazy_subcommands or {}
|
||||||
|
|
||||||
|
def list_commands(self, ctx):
|
||||||
|
base = super().list_commands(ctx)
|
||||||
|
lazy = sorted(self.lazy_subcommands.keys())
|
||||||
|
return base + lazy
|
||||||
|
|
||||||
|
def get_command(self, ctx, cmd_name):
|
||||||
|
if cmd_name in self.lazy_subcommands:
|
||||||
|
return self._lazy_load(cmd_name)
|
||||||
|
return super().get_command(ctx, cmd_name)
|
||||||
|
|
||||||
|
def _lazy_load(self, cmd_name):
|
||||||
|
# lazily loading a command, first get the module name and attribute name
|
||||||
|
import_path = self.lazy_subcommands[cmd_name]
|
||||||
|
modname, cmd_object_name = import_path.rsplit(".", 1)
|
||||||
|
# do the import
|
||||||
|
mod = importlib.import_module(modname)
|
||||||
|
# get the Command object from that module
|
||||||
|
cmd_object = getattr(mod, cmd_object_name)
|
||||||
|
# check the result to make debugging easier
|
||||||
|
if not isinstance(cmd_object, click.BaseCommand):
|
||||||
|
raise ValueError(
|
||||||
|
f"Lazy loading of {import_path} failed by returning "
|
||||||
|
"a non-command object"
|
||||||
|
)
|
||||||
|
return cmd_object
|
|
@ -0,0 +1,14 @@
|
||||||
|
import click
|
||||||
|
from gbif_cruft.__about__ import __version__
|
||||||
|
from gbif_cruft.lazy_group import LazyGroup
|
||||||
|
from pygbif import species
|
||||||
|
import json
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
@click.argument('name', nargs=1, default="axolotl", type=str)
|
||||||
|
|
||||||
|
def cli(name):
|
||||||
|
foo = species.name_suggest(q=name)
|
||||||
|
foo = json.dumps(foo)
|
||||||
|
print(foo)
|
||||||
|
|
Loading…
Reference in New Issue