Lazy loading cli subcommands for search, filedb stub

README.md

@@ -69,10 +69,14 @@ black gbif-cruft*
 # TODO
 Perhaps:
 
-* rclone GBIF snapshot, approximately 250 gigs of parquet files.
-* Read parquet files with local tools.
+* rclone GBIF snapshot, approximately 210 gigs of parquet files.
+* Read parquet files with local tools. Perhaps pyarrow, pqv, dask, 
+* Parquet with GraphQL? graphique.
 * Perhaps import parquet files into sota db.
 * Test on ppc64le.
+* Type hinted Python dataclass for GBIF API.
+* Mypyc or similar.
+* sota media storage.
 
 
 # Upstream

src/filedb.py

@@ -0,0 +1,9 @@
+import click
+from gbif_cruft.__about__ import __version__
+from gbif_cruft.lazy_group import LazyGroup
+
+
+@click.command()
+@click.argument('filename', nargs=1, default="/srv/gbif/gbif_2023-10-01/occurrence.parquet/000000", type=str)
+def cli(filename):
+    print(filename)

src/gbif_cruft/cli/__init__.py

@@ -4,13 +4,14 @@
 import click
 
 from gbif_cruft.__about__ import __version__
-from pygbif import species
-import json
+from gbif_cruft.lazy_group import LazyGroup
 
-@click.group(context_settings={"help_option_names": ["-h", "--help"]}, invoke_without_command=True)
-@click.option("--search", default="axolotl", help="Search GBIF.")
 @click.version_option(version=__version__, prog_name="gbif-cruft")
-def gbif_cruft(search):
-    foo = species.name_suggest(q=search)
-    foo = json.dumps(foo)
-    print(foo)
+@click.group(
+    cls=LazyGroup,
+    lazy_subcommands={"search": "search.cli", "filedb": "filedb.cli"},
+    context_settings={"help_option_names": ["-h", "--help"]},
+)
+
+def gbif_cruft():
+    pass

src/gbif_cruft/cli/lazy_group.py

@@ -0,0 +1 @@
+../lazy_group.py

src/gbif_cruft/lazy_group.py

@@ -0,0 +1,37 @@
+import importlib
+import click
+
+class LazyGroup(click.Group):
+    def __init__(self, *args, lazy_subcommands=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        # lazy_subcommands is a map of the form:
+        #
+        #   {command-name} -> {module-name}.{command-object-name}
+        #
+        self.lazy_subcommands = lazy_subcommands or {}
+
+    def list_commands(self, ctx):
+        base = super().list_commands(ctx)
+        lazy = sorted(self.lazy_subcommands.keys())
+        return base + lazy
+
+    def get_command(self, ctx, cmd_name):
+        if cmd_name in self.lazy_subcommands:
+            return self._lazy_load(cmd_name)
+        return super().get_command(ctx, cmd_name)
+
+    def _lazy_load(self, cmd_name):
+        # lazily loading a command, first get the module name and attribute name
+        import_path = self.lazy_subcommands[cmd_name]
+        modname, cmd_object_name = import_path.rsplit(".", 1)
+        # do the import
+        mod = importlib.import_module(modname)
+        # get the Command object from that module
+        cmd_object = getattr(mod, cmd_object_name)
+        # check the result to make debugging easier
+        if not isinstance(cmd_object, click.BaseCommand):
+            raise ValueError(
+                f"Lazy loading of {import_path} failed by returning "
+                "a non-command object"
+            )
+        return cmd_object

src/search.py

@@ -0,0 +1,14 @@
+import click
+from gbif_cruft.__about__ import __version__
+from gbif_cruft.lazy_group import LazyGroup
+from pygbif import species
+import json
+
+@click.command()
+@click.argument('name', nargs=1, default="axolotl", type=str)
+
+def cli(name):
+    foo = species.name_suggest(q=name)
+    foo = json.dumps(foo)
+    print(foo)
+