Merge branch 'staging' of github.com:FarmBot/farmbot_os into staging

farmbot_ext/config/test.exs

@@ -1,5 +1,16 @@
 use Mix.Config
 
 if Mix.env() == :test do
-  config :farmbot_ext, FarmbotExt, children: []
+  config :ex_unit, capture_logs: true
+  mapper = fn mod -> config :farmbot_ext, mod, children: [] end
+
+  list = [
+    FarmbotExt,
+    FarmbotExt.AMQP.ChannelSupervisor,
+    FarmbotExt.API.DirtyWorker.Supervisor,
+    FarmbotExt.API.EagerLoader.Supervisor,
+    FarmbotExt.Bootstrap.Supervisor
+  ]
+
+  Enum.map(list, mapper)
 end

farmbot_ext/lib/farmbot_ext.ex

@@ -4,13 +4,11 @@ defmodule FarmbotExt do
   use Application
 
   def start(_type, _args) do
-    opts = [strategy: :one_for_one, name: __MODULE__]
-    Supervisor.start_link(children(), opts)
+    Supervisor.start_link(children(), opts())
   end
 
-  # This only exists because I was getting too many crashed
-  # supervisor reports in the test suite (distraction from
-  # real test failures).
+  def opts, do: [strategy: :one_for_one, name: __MODULE__]
+
   def children do
     config = Application.get_env(:farmbot_ext, __MODULE__) || []
     Keyword.get(config, :children, [FarmbotExt.Bootstrap])

farmbot_ext/lib/farmbot_ext/amqp/channel_supervisor.ex

@@ -19,17 +19,19 @@ defmodule FarmbotExt.AMQP.ChannelSupervisor do
   end
 
   def init([token]) do
-    jwt = JWT.decode!(token)
+    Supervisor.init(children(JWT.decode!(token)), strategy: :one_for_one)
+  end
 
-    children = [
+  def children(jwt) do
+    config = Application.get_env(:farmbot_ext, __MODULE__) || []
+
+    Keyword.get(config, :children, [
       {TelemetryChannel, [jwt: jwt]},
       {LogChannel, [jwt: jwt]},
       {PingPongChannel, [jwt: jwt]},
       {BotStateChannel, [jwt: jwt]},
       {AutoSyncChannel, [jwt: jwt]},
       {CeleryScriptChannel, [jwt: jwt]}
-    ]
-
-    Supervisor.init(children, strategy: :one_for_one)
+    ])
   end
 end

farmbot_ext/lib/farmbot_ext/amqp/supervisor.ex

@@ -10,14 +10,17 @@ defmodule FarmbotExt.AMQP.Supervisor do
   end
 
   def init([]) do
+    Supervisor.init(children(), strategy: :one_for_all)
+  end
+
+  def children do
     token = get_config_value(:string, "authorization", "token")
     email = get_config_value(:string, "authorization", "email")
+    config = Application.get_env(:farmbot_ext, __MODULE__) || []
 
-    children = [
+    Keyword.get(config, :children, [
       {FarmbotExt.AMQP.ConnectionWorker, [token: token, email: email]},
       {FarmbotExt.AMQP.ChannelSupervisor, [token]}
-    ]
-
-    Supervisor.init(children, strategy: :one_for_all)
+    ])
   end
 end

farmbot_ext/lib/farmbot_ext/api/dirty_worker/supervisor.ex

@@ -33,7 +33,13 @@ defmodule FarmbotExt.API.DirtyWorker.Supervisor do
 
   @impl Supervisor
   def init(_args) do
-    children = [
+    Supervisor.init(children(), strategy: :one_for_one)
+  end
+
+  def children do
+    config = Application.get_env(:farmbot_ext, __MODULE__) || []
+
+    Keyword.get(config, :children, [
       {DirtyWorker, Device},
       {DirtyWorker, DeviceCert},
       {DirtyWorker, FbosConfig},
@@ -50,8 +56,6 @@ defmodule FarmbotExt.API.DirtyWorker.Supervisor do
       {DirtyWorker, Sensor},
       {DirtyWorker, Sequence},
       {DirtyWorker, Tool}
-    ]
-
-    Supervisor.init(children, strategy: :one_for_one)
+    ])
   end
 end

farmbot_ext/lib/farmbot_ext/api/eager_loader/supervisor.ex

@@ -39,7 +39,13 @@ defmodule FarmbotExt.API.EagerLoader.Supervisor do
 
   @impl Supervisor
   def init(_args) do
-    children = [
+    Supervisor.init(children(), strategy: :one_for_one)
+  end
+
+  def children do
+    config = Application.get_env(:farmbot_ext, __MODULE__) || []
+
+    Keyword.get(config, :children, [
       {EagerLoader, Device},
       {EagerLoader, FarmEvent},
       {EagerLoader, FarmwareEnv},
@@ -56,8 +62,6 @@ defmodule FarmbotExt.API.EagerLoader.Supervisor do
       {EagerLoader, Sensor},
       {EagerLoader, Sequence},
       {EagerLoader, Tool}
-    ]
-
-    Supervisor.init(children, strategy: :one_for_one)
+    ])
   end
 end

farmbot_ext/lib/farmbot_ext/api/image_uploader.ex

@@ -16,7 +16,7 @@ defmodule FarmbotExt.API.ImageUploader do
     GenServer.start_link(__MODULE__, args, name: __MODULE__)
   end
 
-  def force_checkup do
+  def force_checkup() do
     GenServer.cast(__MODULE__, :force_checkup)
   end
 
@@ -45,19 +45,23 @@ defmodule FarmbotExt.API.ImageUploader do
 
   def handle_continue([], state), do: {:noreply, state, @checkup_time_ms}
 
-  # the meta here is likely inaccurate here because of
-  # pulling the location data from the cache instead of from the firmware
-  # directly. It's close enough and getting data from the firmware directly
-  # would require more work than it is worth
+  # the meta here is likely inaccurate here because of pulling the location data
+  # from the cache instead of from the firmware directly. It's close enough and
+  # getting data from the firmware directly would require more work than it is
+  # worth
   defp try_upload(image_filename) do
     %{x: x, y: y, z: z} = BotState.fetch().location_data.position
     meta = %{x: x, y: y, z: z, name: Path.rootname(image_filename)}
+    finalize(image_filename, API.upload_image(image_filename, meta))
+  end
 
-    with {:ok, %{status: s, body: _body}} when s > 199 and s < 300 <-
-           API.upload_image(image_filename, meta) do
-      FarmbotCore.Logger.success(3, "Uploaded image: #{image_filename}")
-      File.rm(image_filename)
-    end
+  defp finalize(file, {:ok, %{status: s, body: _}}) when s > 199 and s < 300 do
+    FarmbotCore.Logger.success(3, "Uploaded image: #{file}")
+    File.rm(file)
+  end
+
+  defp finalize(fname, other) do
+    FarmbotCore.Logger.error(3, "Upload Error (#{fname}): #{inspect(other)}")
   end
 
   # Stolen from

farmbot_ext/lib/farmbot_ext/bootstrap/supervisor.ex

@@ -12,14 +12,18 @@ defmodule FarmbotExt.Bootstrap.Supervisor do
 
   @impl Supervisor
   def init([]) do
-    children = [
+    Supervisor.init(children(), strategy: :one_for_one)
+  end
+
+  def children() do
+    config = Application.get_env(:farmbot_ext, __MODULE__) || []
+
+    Keyword.get(config, :children, [
       FarmbotExt.API.EagerLoader.Supervisor,
       FarmbotExt.API.DirtyWorker.Supervisor,
       FarmbotExt.AMQP.Supervisor,
       FarmbotExt.API.ImageUploader,
       FarmbotExt.Bootstrap.DropPasswordTask
-    ]
-
-    Supervisor.init(children, strategy: :one_for_one)
+    ])
   end
 end

farmbot_ext/test/farmbot_ext/amqp/auto_sync_asset_handler_test.exs

@@ -1,5 +1,6 @@
 defmodule AutoSyncAssetHandlerTest do
-  use ExUnit.Case, async: true
+  require Helpers
+  use ExUnit.Case, async: false
   use Mimic
 
   setup :verify_on_exit!
@@ -8,7 +9,10 @@ defmodule AutoSyncAssetHandlerTest do
   alias FarmbotExt.AMQP.AutoSyncAssetHandler
   alias FarmbotCore.{Asset, BotState, Leds}
 
+  import ExUnit.CaptureLog
+
   def auto_sync_off, do: expect(Asset.Query, :auto_sync?, fn -> false end)
+  def auto_sync_on, do: expect(Asset.Query, :auto_sync?, fn -> true end)
 
   def expect_sync_status_to_be(status),
     do: expect(BotState, :set_sync_status, fn ^status -> :ok end)
@@ -22,4 +26,41 @@ defmodule AutoSyncAssetHandlerTest do
     expect_green_leds(:slow_blink)
     AutoSyncAssetHandler.handle_asset("Point", 23, nil)
   end
+
+  test "Handles @no_cache_kinds" do
+    id = 64
+    params = %{}
+
+    kind =
+      ~w(Device FbosConfig FirmwareConfig FarmwareEnv FarmwareInstallation)
+      |> Enum.shuffle()
+      |> Enum.at(0)
+
+    expect(Asset.Command, :update, 1, fn ^kind, ^id, ^params -> :ok end)
+    assert :ok = AutoSyncAssetHandler.cache_sync(kind, id, params)
+  end
+
+  test "handling of deleted assets when auto_sync is enabled" do
+    auto_sync_on()
+    expect_sync_status_to_be("syncing")
+    expect_sync_status_to_be("synced")
+    expect_green_leds(:really_fast_blink)
+    expect_green_leds(:solid)
+    AutoSyncAssetHandler.handle_asset("Point", 32, nil)
+  end
+
+  test "cache sync" do
+    id = 64
+    params = %{}
+    kind = "Point"
+    # Helpers.expect_log("Autocaching sync #{kind} #{id} #{inspect(params)}")
+    changeset = %{ab: :cd}
+    changesetfaker = fn ^kind, ^id, ^params -> changeset end
+    expect(FarmbotCore.Asset.Command, :new_changeset, 1, changesetfaker)
+    expect(FarmbotExt.API.EagerLoader, :cache, 1, fn ^changeset -> :ok end)
+    expect_sync_status_to_be("sync_now")
+    expect_green_leds(:slow_blink)
+    do_it = fn -> AutoSyncAssetHandler.cache_sync(kind, id, params) end
+    assert capture_log(do_it) =~ "Autocaching sync Point 64 %{}"
+  end
 end

farmbot_ext/test/farmbot_ext/amqp/auto_sync_channel_test.exs

@@ -1,6 +1,6 @@
 defmodule AutoSyncChannelTest do
   require Helpers
-  use ExUnit.Case, async: true
+  use ExUnit.Case, async: false
   use Mimic
   alias FarmbotExt.AMQP.AutoSyncChannel
 
@@ -55,7 +55,7 @@ defmodule AutoSyncChannelTest do
     expect(Preloader, :preload_all, 1, fn -> :ok end)
     pid = generate_pid()
     send(pid, msg)
-    Process.sleep(5)
+    Helpers.wait_for(pid)
   end
 
   test "basic_cancel", do: ensure_response_to({:basic_cancel, :anything})
@@ -153,6 +153,6 @@ defmodule AutoSyncChannelTest do
       :ok
     end)
 
-    Process.sleep(1200)
+    Helpers.wait_for(pid)
   end
 end

farmbot_ext/test/farmbot_ext/amqp/bot_state_channel_test.exs

@@ -1,5 +1,5 @@
 defmodule FarmbotExt.AMQP.BotStateChannelTest do
-  use ExUnit.Case
+  use ExUnit.Case, async: false
   use Mimic
 
   # alias FarmbotExt.AMQP.BotStateChannel

farmbot_ext/test/farmbot_ext/api/image_uploader_test.exs

@@ -0,0 +1,43 @@
+defmodule FarmbotExt.API.ImageUploaderTest do
+  require Helpers
+  use ExUnit.Case, async: false
+  use Mimic
+  alias FarmbotExt.API.ImageUploader
+  setup :verify_on_exit!
+  setup :set_mimic_global
+
+  test "force checkup" do
+    pid =
+      if Process.whereis(ImageUploader) do
+        Process.whereis(ImageUploader)
+      else
+        {:ok, p} = ImageUploader.start_link([])
+        p
+      end
+
+    ["a.jpg", "b.jpeg", "c.png", "d.gif"]
+    |> Enum.map(fn fname ->
+      f = "/tmp/images/#{fname}"
+      File.touch!(f)
+      File.write(f, "X")
+    end)
+
+    expect(FarmbotExt.API, :upload_image, 4, fn
+      "/tmp/images/d.gif", _meta -> {:error, %{status: 401, body: %{}}}
+      _image_filename, _meta -> {:ok, %{status: 201, body: %{}}}
+    end)
+
+    err_msg =
+      "Upload Error (/tmp/images/d.gif): " <>
+        "{:error, %{body: %{}, status: 401}}"
+
+    Helpers.expect_log("Uploaded image: /tmp/images/a.jpg")
+    Helpers.expect_log("Uploaded image: /tmp/images/b.jpeg")
+    Helpers.expect_log("Uploaded image: /tmp/images/c.png")
+    Helpers.expect_log(err_msg)
+
+    ImageUploader.force_checkup()
+    send(pid, :timeout)
+    Helpers.wait_for(pid)
+  end
+end

farmbot_ext/test/test_helper.exs

@@ -1,28 +1,54 @@
 Application.ensure_all_started(:farmbot)
 
-Mimic.copy(AMQP.Channel)
-Mimic.copy(FarmbotCeleryScript.SysCalls.Stubs)
-Mimic.copy(FarmbotCore.Asset.Command)
-Mimic.copy(FarmbotCore.Asset.Query)
-Mimic.copy(FarmbotCore.BotState)
-Mimic.copy(FarmbotCore.Leds)
-Mimic.copy(FarmbotCore.LogExecutor)
-Mimic.copy(FarmbotExt.AMQP.ConnectionWorker)
-Mimic.copy(FarmbotExt.API.EagerLoader.Supervisor)
-Mimic.copy(FarmbotExt.API.Preloader)
-Mimic.copy(FarmbotExt.API)
-Mimic.copy(FarmbotExt.AMQP.AutoSyncAssetHandler)
+[
+  AMQP.Channel,
+  FarmbotCeleryScript.SysCalls.Stubs,
+  FarmbotCore.Asset.Command,
+  FarmbotCore.Asset.Query,
+  FarmbotCore.BotState,
+  FarmbotCore.Leds,
+  FarmbotCore.LogExecutor,
+  FarmbotExt.AMQP.AutoSyncAssetHandler,
+  FarmbotExt.AMQP.ConnectionWorker,
+  FarmbotExt.API,
+  FarmbotExt.API.EagerLoader,
+  FarmbotExt.API.EagerLoader.Supervisor,
+  FarmbotExt.API.Preloader
+]
+|> Enum.map(&Mimic.copy/1)
 
-timeout = System.get_env("EXUNIT_TIMEOUT")
+timeout = System.get_env("EXUNIT_TIMEOUT") || "5000"
 System.put_env("LOG_SILENCE", "true")
 
-if timeout do
-  ExUnit.start(assert_receive_timeout: String.to_integer(timeout))
-else
-  ExUnit.start()
-end
+ExUnit.start(assert_receive_timeout: String.to_integer(timeout))
 
 defmodule Helpers do
+  # Maybe I don't need this?
+  # Maybe I could use `start_supervised`?
+  # https://hexdocs.pm/ex_unit/ExUnit.Callbacks.html#start_supervised/2
+
+  @wait_time 60
+  # Base case: We have a pid
+  def wait_for(pid) when is_pid(pid), do: check_on_mbox(pid)
+  # Failure case: We failed to find a pid for a module.
+  def wait_for(nil), do: raise("Attempted to wait on bad module/pid")
+  # Edge case: We have a module and need to try finding its pid.
+  def wait_for(mod), do: wait_for(Process.whereis(mod))
+
+  # Enter recursive loop
+  defp check_on_mbox(pid) do
+    Process.sleep(@wait_time)
+    wait(pid, Process.info(pid, :message_queue_len))
+  end
+
+  # Exit recursive loop (mbox is clear)
+  defp wait(_, {:message_queue_len, 0}), do: Process.sleep(@wait_time * 3)
+  # Exit recursive loop (pid is dead)
+  defp wait(_, nil), do: Process.sleep(@wait_time * 3)
+
+  # Continue recursive loop
+  defp wait(pid, {:message_queue_len, _n}), do: check_on_mbox(pid)
+
   defmacro expect_log(message) do
     quote do
       expect(FarmbotCore.LogExecutor, :execute, fn log ->