Introduce a "Flat IR" for sequence nodes (#655)

This is the first release of the Flat IR storage mechanism.

Gemfile

@@ -28,6 +28,7 @@ gem "skylight", "1.4.0"
 gem "bunny"
 gem "request_store"
 gem "secure_headers"
+gem "hashdiff"
 
 group :development, :test do
   gem "codecov", require: false

Gemfile.lock

@@ -142,6 +142,7 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
+    hashdiff (0.3.7)
     httpclient (2.8.3)
     i18n (0.9.1)
       concurrent-ruby (~> 1.0)
@@ -324,6 +325,7 @@ DEPENDENCIES
   figaro
   fog-google!
   foreman
+  hashdiff
   jwt
   letter_opener
   mutations
@@ -353,4 +355,4 @@ RUBY VERSION
    ruby 2.4.2p198
 
 BUNDLED WITH
-   1.16.0
+   1.16.1

app/controllers/api/abstract_controller.rb

@@ -6,7 +6,8 @@ module Api
     # endpoint that requires JSON.
     class OnlyJson < Exception; end;
     CONSENT_REQUIRED = "all device users must agree to terms of service."
-
+    NOT_JSON = "That request was not valid JSON. Consider checking the request"\
+               " body with a JSON validator.."
     respond_to :json
     before_action :check_fbos_version
     before_action :set_default_stuff
@@ -19,10 +20,7 @@ module Api
 
     rescue_from(JWT::VerificationError) { |e| auth_err }
 
-    rescue_from(ActionDispatch::Http::Parameters::ParseError) do
-      sorry "That request was not valid JSON. Consider checking the request " +
-            "body with a JSON validator..", 422
-    end
+    rescue_from(ActionDispatch::Http::Parameters::ParseError) { sorry NOT_JSON, 422 }
 
     rescue_from(ActiveRecord::ValueTooLong) do
       sorry "Please use reasonable lengths on string inputs", 422

app/controllers/api/farm_events_controller.rb

@@ -29,14 +29,5 @@ module Api
     def farm_event
       @farm_event ||= FarmEvent.find(params[:id])
     end
-
-    # Probably safe to remove this endpoint now. This is from the pre-launch era
-    # when we were still on Angular 1.0.
-    # TODO: Remove this dead code?
-    def default_serializer_options
-      # For some strange reason, angular-data crashes if we don't call super()
-      # here. Rails doesn't care, though.
-      super.merge(start: params[:start], finish: params[:finish])
-    end
   end
 end

app/controllers/api/peripherals_controller.rb

@@ -19,11 +19,7 @@ module Api
     end
 
     def destroy
-      if (peripheral.device_id == current_device.id) && peripheral.destroy!
-        render json: ""
-      else
-        raise Errors::Forbidden, 'Not your Peripheral.'
-      end
+      peripheral.destroy! && (render json: "")
     end
 
     private

app/controllers/api/sequences_controller.rb

@@ -1,18 +1,15 @@
 module Api
   class SequencesController < Api::AbstractController
-    before_action :authorize_user, except: [:index, :create]
     before_action :clean_expired_farm_events, only: [:destroy]
 
     def index
-      query = { device: current_device }
-      # TODO: This is a legacy API from the Angular 1.0 days, I think. Remove.
-      query.merge!(farm_event_id: params[:farm_event_id]) if params[:farm_event_id]
-      sequences = Sequence.where(query)
       render json: sequences
+        .to_a
+        .map { |s| CeleryScript::FetchCelery.run!(sequence: s) }
     end
 
     def show
-      render json: sequence
+      render json: CeleryScript::FetchCelery.run!(sequence: sequence)
     end
 
     def create
@@ -20,7 +17,7 @@ module Api
     end
 
     def update
-      mutate Sequences::Update.run(sequence_params, # params[:sequence].as_json,
+      mutate Sequences::Update.run(sequence_params,
                                     device: current_device,
                                     sequence: sequence)
     end
@@ -31,20 +28,16 @@ module Api
 
     private
 
-    def maybe_migrate(sequences)
-    end
-
     def sequence_params
         @sequence_params ||= raw_json[:sequence] || raw_json || {}
     end
 
-    def sequence
-      @sequence ||= Sequence.find(params[:id])
+    def sequences
+      @sequences ||= Sequence.where(device: current_device)
     end
 
-    def authorize_user
-      raise Errors::Forbidden,
-            "Not your Sequence object." if sequence.device != current_device
+    def sequence
+      @sequence ||= sequences.find(params[:id])
     end
   end
 end

app/lib/celery_script/corpus.rb

@@ -52,8 +52,8 @@ module CeleryScript
       fetchArg(name).additional_validation || CeleryScript::NOOP
     end
 
-    def as_json(optns)
-      { "tag": SequenceMigration::Base.latest_version,
+    def as_json(*)
+      { "tag": Sequence::LATEST_VERSION,
         "args": @arg_def_list.to_a.map(&:last).map{|x| x.as_json({}) },
         "nodes": @node_def_list.to_a.map(&:last).map{|x| x.as_json({}) }}
     end

app/lib/celery_script/csheap.rb

@@ -0,0 +1,153 @@
+# A heap-ish data structure required when converting canonical CeleryScript AST
+# nodes into the Flat IR form.
+# This data structure is useful because it addresses each node in the
+# CeleryScript tree via a unique numerical index, rather than using mutable
+# references.
+# MORE INFO: https://github.com/FarmBot-Labs/Celery-Slicer
+module CeleryScript
+  # Supporting class for CSHeap (below this class)
+  # PROBLEM:  CSHeap uses numbers to address sibling/parent nodes.
+  # PROBLEM:  Numbers are very easy to mix up. Is it an array index? A SQL
+  #           primary key? A primitive value? It's not always easy to say.
+  # SOLUTION: Create a `HeapAddress` value type to remove ambiguity.
+  #           Prevents confusion between index IDs and SQL IDs.
+  class HeapAddress
+    attr_reader :value
+
+    def initialize(value)
+      raise "BAD INPUT" unless value.is_a?(Integer)
+      @value = value.to_i
+    end
+
+    def self.[](value)
+      return self.new(value)
+    end
+
+    def inspect
+      "HeapAddress(#{value})"
+    end
+
+    def hash
+      self.value
+    end
+
+    def ==(other)
+      self.value == other.value
+    end
+
+    def eql?(other)
+      self.value == other.value
+    end
+
+    def +(val)
+      HeapAddress[@value + 1]
+    end
+
+    def -(val)
+      HeapAddress[@value - 1]
+    end
+
+    def is_address?
+      true
+    end
+
+    def to_i
+      @value
+    end
+
+    def to_s
+      @value.to_s
+    end
+  end
+
+  class CSHeap
+    class BadAddress < Exception; end;
+    BAD_ADDR = "Bad node address: "
+    # Nodes that point to other nodes rather than primitive data types (eg:
+    # `locals` and friends) will be prepended with a "🔗".
+    LINK    = "__"
+    # Points to the originator (parent) of an `arg` or `body` node.
+    PARENT  = (LINK + "parent").to_sym
+    # Points to the first element in the `body``
+    BODY    = (LINK + "body").to_sym
+    # (Broke?) Points to the next node in the body chain. Pointing to NOTHING
+    # indicates the end of the body linked list.
+    NEXT    = (LINK + "next").to_sym
+    # Unique key name. See `celery_script_settings_bag.rb`
+    KIND    = :__KIND__
+    COMMENT = :__COMMENT__
+
+    # Keys that primary nodes must have
+    PRIMARY_FIELDS = [PARENT, BODY, KIND, NEXT, COMMENT]
+
+    # Index 0 of the heap represents a null pointer of sorts.
+    # If a field points to this address, it is considered empty.
+    NULL    = HeapAddress[0]
+
+    # What you will find at index 0 of the heap:
+    NOTHING = {
+      KIND   => "nothing",
+      PARENT => NULL,
+      BODY   => NULL,
+      NEXT   => NULL
+    }
+
+
+    # A dictionary of nodes in the CeleryScript tree, as stored in the heap.
+    # Nodes will have:
+    #   * A `KIND` field   - What kind of node is it?
+    #                        `send_message`, `move_rel`, etc..
+    #   * A `PARENT` field - The node directly above the current node.
+    #   * A `BODY` field   - If a node has a body member (and it might not!),
+    #                        this field will point to the first node in the
+    #                        chain. NOTHING pointer indicates that the node has
+    #                        no body.
+    #   * A `NEXT` field   - If you are inside a node's BODY, the next node in
+    #                        chain is denoted by the address of NEXT. A NEXT
+    #                        value of NOTHING means you hit the end of the chain
+    attr_accessor :entries
+
+    # "here" represents the last item added to the heap and, often, the item that
+    # is currently being edited.
+    attr_accessor :here
+
+    # Set "here" to "null". Prepopulates "here" with an empty entry.
+    def initialize
+      @here    = NULL
+      @entries = { @here => NOTHING }
+    end
+
+    # Grow the heap and fill it was a CS node of type `__KIND__`.
+    # Returns the new value of `@here` after expansion.
+    # "Create a new empty heap object and return its address for access later"
+    def allot(__KIND__)
+      entries[@here += 1] = { KIND => __KIND__ }
+      return @here
+    end
+
+    # augment a heap entry with a new key/value pair.
+    # Throws an exception when given a bad heap index.
+    # "Put this VALUE into this ADDRESS and annotate it with the KEY provided"
+    def put(address, key, value)
+      address.is_address?
+      block = entries[address]
+      if (block)
+        block[key.to_sym] = value
+        return
+      else
+        raise BadAddress, BAD_ADDR + address.inspect
+      end
+    end
+
+    # Just an alias
+    def values
+      entries.values
+    end
+
+    # Dump the heap as an easy-to-traverse JSON object.
+    # We need this to reconstruct the node from its IR form to its canonical form.
+    def dump
+      return values
+    end
+  end
+end

app/lib/celery_script/fetch_celery.rb

@@ -0,0 +1,117 @@
+require_relative "./csheap"
+
+# Service object that:
+# 1. Pulls out all PrimaryNodes and EdgeNodes for a sequence node (AST Flat IR form)
+# 2. Stitches the nodes back together in their "canonical" (nested) AST
+#    representation
+module CeleryScript
+  class FetchCelery < Mutations::Command
+  private  # = = = = = = =
+
+    # This class is too CPU intensive to make multiple SQL requests.
+    # To speed up querying, we create an in-memory index for frequently
+    # looked up attributes such as :id, :kind, :parent_id, :primary_node_id
+    def edge_nodes
+      @edge_nodes ||= Indexer.new(EdgeNode.where(sequence: sequence))
+    end
+
+    # See docs for #edge_nodes()
+    def primary_nodes
+      @primary_nodes ||= Indexer.new(PrimaryNode.where(sequence: sequence))
+    end
+
+    # The topmost node is always `NOTHING`. The term "root node" refers to
+    # the node where (kind == "sequence") in a tree of nodes. We start recursion
+    # here and move down.
+    def entry_node
+      @entry_node ||= primary_nodes.by.kind["sequence"].try(:first)
+    end
+
+    # Create a hash and attach all the EdgeNodes to it. Creates a partial "args"
+    # property when converting from flat IR to canonical form.
+    # Does not attach primary (fully formed CeleryScript) nodes.
+    def attach_edges(node)
+      output = {}
+      (edge_nodes.by.primary_node_id[node.id] || [])
+        .map { |edge| output[edge.kind] = edge.value }
+      output
+    end
+
+    # Similar to attach_edges(node) but for fully formed CS nodes.
+    # Eg: Will attach a `coordinate` node to a `location` arg.
+    def attach_primary_nodes(node)
+      output = {}
+      (primary_nodes.by.parent_id[node.id] || []).select(&:parent_arg_name)
+        .map { |x| output[x.parent_arg_name] = recurse_into_node(x) }
+      output
+    end
+
+    def recurse_into_args(node)
+      {}.merge!(attach_edges(node)).merge!(attach_primary_nodes(node))
+    end
+
+    # Pass this method a PrimaryNode and it will return an array filled with
+    # that node's children (or an empty array, since body is always optional).
+    def get_body_elements(node)
+      next_node = node.body
+      results = []
+      until next_node.kind == "nothing"
+        results.push(next_node)
+        next_node = next_node.next
+      end
+      results
+    end
+
+    # Top level function call for converting a single EdgeNode into a JSON
+    # document. Returns Ruby hash that conforms to CeleryScript semantics.
+    def recurse_into_node(node)
+      out  = { kind: node.kind, args: recurse_into_args(node) }
+      body = get_body_elements(node)
+      if body.empty?
+        # Legacy sequences *must* have body on sequence.
+        # Others are fine.
+        out[:body] = [] if node.kind == "sequence"
+      else
+        out[:body] = body.map { |x| recurse_into_node(x) }
+      end
+      out[:comment] = node.comment if node.comment
+      return out
+    end
+
+    # Generates a hash that has all the other fields that API users expect,
+    # Eg: color, id, etc.
+    def misc_fields
+      return {
+        id:         sequence.id,
+        name:       sequence.name,
+        color:      sequence.color,
+        created_at: sequence.created_at,
+        updated_at: sequence.updated_at,
+        args:       { is_outdated: false }
+      }
+    end
+
+  public # = = = = = = =
+    NO_SEQUENCE = "You must have a root node `sequence` at a minimum."
+
+    required do
+      model :sequence, class: Sequence
+    end
+
+    def validate
+      # Legacy sequences won't have EdgeNode/PrimaryNode relations.
+      # We need to run the conversion before we can continue.
+      CeleryScript::StoreCelery
+        .run!(sequence: sequence) unless sequence.migrated_nodes
+      # A sequence lacking a `sequence` node is a syntax error.
+      # This should never show up in the frontend, but *is* helpful for devs
+      # when debugging.
+      add_error :bad_sequence, :bad, NO_SEQUENCE unless entry_node
+    end
+
+    def execute
+      canonical_form = misc_fields.merge!(recurse_into_node(entry_node))
+      return HashWithIndifferentAccess.new(canonical_form)
+    end
+  end
+end

app/lib/celery_script/first_pass.rb

@@ -0,0 +1,106 @@
+require_relative "./csheap"
+
+# ABOUT THIS CLASS:
+#   CSHeap creates an in memory representation of a Flat IR tree using array
+#   indexes (HeapAddress instances, really). This is fine when dealing with the
+#   nodes in memory. Users will need to store these nodes in the DB, though.
+#   This class takes a flat IR tree from memory and converts `HeapAddress`es
+#   to SQL primary/foreign keys.
+module CeleryScript
+  class FirstPass < Mutations::Command
+    # The following constants are abbreviations of the full name, since the
+    # full name is quite long and they are referenced frequently in the code.
+    # Just remember that "B" is "BODY", "K" is "KIND", etc...
+    B    = CeleryScript::CSHeap::BODY
+    C    = CeleryScript::CSHeap::COMMENT
+    K    = CeleryScript::CSHeap::KIND
+    L    = CeleryScript::CSHeap::LINK
+    N    = CeleryScript::CSHeap::NEXT
+    P    = CeleryScript::CSHeap::PARENT
+    NULL = CeleryScript::CSHeap::NULL
+    I    = :instance
+
+    required do
+      model :sequence, class: Sequence
+    end
+
+    def execute
+      Sequence.transaction do
+        flat_ir
+          .each do |node|
+            # Step 1- instantiate records.
+            # TODO: Switch create!() to new() once things are atleast working
+            #   - RC
+            node[I] = PrimaryNode.create!(kind:     node[K],
+                                          sequence: sequence,
+                                          comment:  node[C] || nil)
+          end
+          .each_with_index do |node, index|
+            # Step 2- Assign SQL ids (not to be confused with array index IDs or
+            #         instances of HeapAddress), also sets arent_arg_name
+            model                 = node[I]
+            model.parent_arg_name = parent_arg_name_for(node, index)
+            model.body_id         = fetch_sql_id_for(B, node)
+            model.parent_id       = fetch_sql_id_for(P, node)
+            model.next_id         = fetch_sql_id_for(N, node)
+            node
+          end
+          .map do |node|
+            # Step 3- Set edge nodes
+            pairs = node
+              .to_a
+              .select do |x|
+                key = x.first.to_s
+                (x.first != I) && !key.starts_with?(L)
+              end
+              .map do |(key, value)|
+                EdgeNode.create!(kind:            key,
+                                 value:           value,
+                                 sequence_id:     sequence.id,
+                                 primary_node_id: node[:instance].id)
+              end
+              node[:instance]
+          end
+          .tap { |x| sequence.update_attributes(migrated_nodes: true) unless sequence.migrated_nodes }
+          .map { |x|
+            x.save! if x.changed?
+            x
+          }
+      end
+    end
+
+private
+
+    # Index every primary node in memory by its `HeapAddress`.
+    # We need this info in order to fill out the `parent_arg_name` of a node.
+    def every_primary_link
+      @every_primary_link ||= flat_ir
+        .map do |x|
+          x
+            .except(B,C,I,K,L,N,P)
+            .invert
+            .to_a
+            .select{|(k,v)| k.is_a?(HeapAddress)}
+        end
+        .map(&:to_h)
+        .reduce({}, :merge)
+    end
+
+    def parent_arg_name_for(node, index)
+      resides_in_args  = (node[N] == NULL) && (node[P] != NULL)
+      link_symbol      = every_primary_link[HeapAddress[index]]
+      needs_p_arg_name = (resides_in_args && link_symbol)
+      parent_arg_name  = (needs_p_arg_name ? link_symbol.to_s.gsub(L, "") : nil)
+      return parent_arg_name
+    end
+
+    def fetch_sql_id_for(node_key, node)
+      index = node[node_key].to_i
+      flat_ir[index][I].id
+    end
+
+    def flat_ir
+      @flat_ir ||= Slicer.new.run!(sequence.as_json.deep_symbolize_keys)
+    end
+  end
+end

app/lib/celery_script/indexer.rb

@@ -0,0 +1,36 @@
+  # Re-combingind EdgeNodes and PrimaryNodes is very query intensive.
+  # To avoid excess DB calls, we will index the nodes in memory by field type.
+  module CeleryScript
+    # EXAMPLE USAGE:
+    #  nodes      = Index.new(MY_LIST_OF_NODES)
+    #  my_results = nodes.by.primary_node_id(5)
+    class Indexer
+    # Fields that need indexing in both cases (Edge vs. Primary node)
+    COMMON_FIELDS  = [:kind, :id]
+    # Fields that will be indexed if an EdgeNode collection is passed in.
+    EDGE_FIELDS    = COMMON_FIELDS + [:primary_node_id]
+    # Fields that will be indexed if an PrimaryNode collection is passed in.
+    PRIMARY_FIELDS = COMMON_FIELDS + [:parent_id]
+    # We pick the correct struct based on the class of the collection passed to
+    # the constructor.
+    KLASS_LOOKUP = { PrimaryNode => Struct.new(*PRIMARY_FIELDS),
+                     EdgeNode    => Struct.new(*EDGE_FIELDS) }
+
+    # Example: index_object.by.primary_node_id[6]
+    attr_reader :by
+
+    # Pass in a collection of EdgeNode or PrimaryNode objects.
+    def initialize(collection)
+      struct_class = KLASS_LOOKUP[collection.klass]
+      struct       = struct_class.new()
+      struct
+        .members
+        .each do |key|
+          setter = "#{key}="
+          values = collection.group_by { |record| record.send(key) } || []
+          struct.send(setter, values)
+        end
+      @by = struct
+    end
+  end
+end

app/lib/celery_script/slicer.rb

@@ -0,0 +1,79 @@
+require_relative "./csheap.rb"
+# ORIGINAL IMPLEMENTATION HERE: https://github.com/FarmBot-Labs/Celery-Slicer
+# Take a nested ("canonical") representation of a CeleryScript sequence and
+# transofrms it to a flat/homogenous intermediate representation which is better
+# suited for storage in a relational database.
+module CeleryScript
+  class Slicer
+    attr_reader :root_node
+
+    def run!(node)
+      raise "Not a hash" unless node.is_a?(Hash)
+      @nesting_level = 0
+      @root_node = node
+      heap = CSHeap.new()
+      allocate(heap, node, CSHeap::NULL)
+      @heap_values = heap.values
+      @heap_values.map do |x|
+        x[CSHeap::BODY] ||= CSHeap::NULL
+        x[CSHeap::NEXT] ||= CSHeap::NULL
+      end
+      heap.dump()
+    end
+
+    def is_celery_script(node)
+      node && node.is_a?(Hash) && node[:args] && node[:kind]
+    end
+
+    def heap_values
+      @heap_values
+    end
+
+    def allocate(h, s, parentAddr)
+      addr = h.allot(s[:kind])
+      h.put(addr, CSHeap::PARENT, parentAddr)
+      h.put(addr, CSHeap::COMMENT, s[:comment]) if s[:comment]
+      iterate_over_body(h, s, addr)
+      iterate_over_args(h, s, addr)
+      addr
+    end
+
+    def iterate_over_args(h, s, parentAddr)
+      (s[:args] || {})
+        .keys
+        .map do |key|
+          v = s[:args][key]
+          if (is_celery_script(v))
+            k = CSHeap::LINK + key.to_s
+            h.put(parentAddr, k, allocate(h, v, parentAddr))
+          else
+            h.put(parentAddr, key, v)
+          end
+        end
+    end
+
+    def iterate_over_body(heap, canonical_node, parentAddr)
+      body = (canonical_node[:body] || []).map(&:deep_symbolize_keys)
+      # !body.none? && heap.put(parentAddr, CSHeap::BODY, parentAddr + 1)
+      @nesting_level += 1
+      recurse_into_body(heap, body, parentAddr)
+      @nesting_level -= 1
+    end
+
+    def recurse_into_body(heap, canonical_list, previous_address, index = 0)
+      if canonical_list[index]
+        is_head         = index == 0
+        # BE CAREFUL EDITING THIS LINE, YOU MIGHT BREAK `BODY` NODES:
+        heap # See note above!
+          .put(previous_address, CSHeap::BODY, previous_address + 1) if is_head
+
+        my_heap_address = allocate(heap, canonical_list[index], previous_address)
+
+        prev_next_key = is_head ? CSHeap::NULL : my_heap_address
+        heap.put(previous_address, CSHeap::NEXT, prev_next_key)
+
+        recurse_into_body(heap, canonical_list, my_heap_address, index + 1)
+      end
+    end
+  end
+end

app/lib/celery_script/store_celery.rb

@@ -0,0 +1,22 @@
+# API users hand us sequences as a JSON object with CeleryScript nodes nested
+# within other nodes. We call this the "canonical representation". It's easy to
+# traverse over, but tree structures are not well suited to the app's storage
+# mechanism (SQL).
+# To get around the limitation, we must convert sequence JSON from canonical to
+# flat forms. `StoreCelery` handles the conversion and storage of CS Nodes.
+module CeleryScript
+  class StoreCelery < Mutations::Command
+    required do
+      model :sequence, class: Sequence
+    end
+
+    def execute
+      Sequence.transaction do
+        sequence.primary_nodes.destroy_all
+        sequence.edge_nodes.destroy_all
+        FirstPass.run!(sequence: sequence)
+        sequence.reload
+      end
+    end
+  end
+end

app/lib/sequence_migration.rb

@@ -1,83 +0,0 @@
-# PROBLEM: As time passes, we must update the format and structure of sequences
-#          created by users. Backwards incompatibilities and breaking changes
-#          over time will cause sequences to become outdated. Forcing users to
-#          update their sequences manually after updates is tedious and error
-#           prone.
-# SOLUTION: Every time we make a breaking change to the way Sequences work, we
-#           write a migration to go along with it. Migrations run one-at-a-time
-#           and ensure that sequences never become incompatible with new
-#           features.
-# HOW:      To create a new migration, create a subclass of SequenceMigration.
-#           Give it a VERSION number and a CREATED_ON date. Add the class name
-#           to the array in SequenceMigration::Base.descendants. Perform all
-#           transformations inside of the #up() method. The migration will
-#           automagically run if the API determines a sequence is out of date.
-module SequenceMigration
-  class Base
-    UP_IS_REQUIRED = "You forgot to implement an `up()`" +
-                     " method on your migration"
-    # When we last ran a compaction (22 NOV 17), the highest version number was
-    # this:
-    HIGHEST_VERSION_AT_TIME_OF_LAST_COMPACTION = 4
-    # MAGIC NUMBER. Assume that versionless sequences are "legacy" sequences
-    # from a time before versioning. Since the lowest migration version is 0, a
-    # version of -1 will require all migrations to run.
-    LEGACY_VERSION                             = -1
-    VERSION                                    = "YOU MUST CHANGE THIS!!!"
-    # I shouldn't need to do this, as this method comes with ActiveSupport, but
-    # it's acting weird with autoloading right now :shipit:. TODO: See if there
-    # is a way to automatically infer all classes
-    def self.descendants
-      [SequenceSpeedChange, AddLocalsToSequenceArgs]
-    end
-
-    def self.latest_version
-      self
-        .descendants
-        .map  { |k| k::VERSION }
-        .max
-    end
-
-    attr_accessor :sequence
-
-    def initialize(sequence)
-      @sequence = sequence
-    end
-
-    def before
-      expected_version = self.class::VERSION - 1
-      incorrect_version = sequence_version != expected_version
-      if incorrect_version
-        raise "Version must be #{expected_version} to run #{self.class}. Got: #{sequence_version}"
-      end
-      Rollbar.info "RUNNING MIGRATION #{sequence_version} on #{sequence.id}"
-    end
-
-    def after
-      sequence.args["version"] ||= LEGACY_VERSION
-      sequence.args["version"] += 1
-    end
-
-    def up
-      raise UP_IS_REQUIRED
-    end
-
-    def run
-      before
-      up
-      after
-    end
-
-    def self.generate_list(sequence)
-      theirs = sequence.args["version"] || LEGACY_VERSION
-      descendants
-        .select { |x| x::VERSION > theirs }
-        .sort   { |a, b| a::VERSION <=> b::VERSION }
-        .map    { |x| x.new(sequence) }
-    end
-
-    def sequence_version
-      sequence.args["version"] || LEGACY_VERSION
-    end
-  end
-end

app/lib/sequence_migration/add_locals_to_sequence_args.rb

@@ -1,12 +0,0 @@
-# This migration:
-#   Adds a `locals` arg to all v5 or better sequences.
-module SequenceMigration
-  class AddLocalsToSequenceArgs < Base
-      VERSION     = 6
-      CREATED_ON  = "NOVEMBER 30 2017"
-
-      def up
-        sequence.args["locals"] ||= Sequence::SCOPE_DECLARATION
-      end
-  end
-end

app/lib/sequence_migration/sequence_speed_change.rb

@@ -1,21 +0,0 @@
-# This migration:
-#   Changes the `speed` setting of all v4 sequence to 100 in anticipation of new
-#   movement parameter semantics.
-module SequenceMigration
-  class SequenceSpeedChange < Base
-      VERSION     = 5
-      CREATED_ON  = "NOVEMBER 21 2017"
-      MUST_BE_100 = [ "move_absolute",
-                      "move_relative",
-                      "home",
-                      "find_home" ]
-
-      def up
-        sequence
-          .body
-          .map do |x|
-            x[:args][:speed] = 100 if MUST_BE_100.include?(x.try(:[], :kind))
-          end
-      end
-  end
-end

app/lib/session_token.rb

@@ -5,14 +5,12 @@ class SessionToken < AbstractJwtToken
   MQTT         = ENV.fetch("MQTT_HOST")
   # If you are not using the standard MQTT broker (eg: you use a 3rd party
   # MQTT vendor), you will need to change this line.
-  MQTT_WS      = ENV.fetch("MQTT_WS") do
-    protocol =  ENV["FORCE_SSL"] ? "wss://" : "ws://"
-    host     =  ENV.fetch("MQTT_HOST")
-    "#{protocol}#{host}:3002/ws"
-  end
-  EXPIRY       = 40.days
-  VHOST        = ENV.fetch("MQTT_VHOST") { "/" }
-  BETA_OS_URL  = ENV["BETA_OTA_URL"] || "NOT_SET"
+  DEFAULT_MQTT_WS = \
+    "#{ENV["FORCE_SSL"] ? "wss://" : "ws://"}#{ENV.fetch("MQTT_HOST")}:3002/ws"
+  MQTT_WS         = ENV["MQTT_WS"] || DEFAULT_MQTT_WS
+  EXPIRY          = 40.days
+  VHOST           = ENV.fetch("MQTT_VHOST") { "/" }
+  BETA_OS_URL     = ENV["BETA_OTA_URL"] || DEFAULT_MQTT_WS
   def self.issue_to(user,
                     iat: Time.now.to_i,
                     exp: EXPIRY.from_now.to_i,

app/models/celery_script_settings_bag.rb

@@ -194,6 +194,10 @@ module CeleryScriptSettingsBag
       .defineNode(:parameter_declaration, [:label, :data_type], [])
       .defineNode(:set_servo_angle,   [:pin_number, :pin_value], [])
       .defineNode(:install_first_party_farmware, [])
+
+  ANY_ARG_NAME  = Corpus.as_json[:args].pluck("name").map(&:to_s)
+  ANY_NODE_NAME = Corpus.as_json[:nodes].pluck("name").map(&:to_s)
+
   # Given an array of allowed values and a CeleryScript AST node, will DETERMINE
   # if the node contains a legal value. Throws exception and invalidates if not.
   def self.within(array, node)

app/models/edge_node.rb

@@ -0,0 +1,20 @@
+# In a CeleryScript flat IR tree, primitive values are stored as edge nodes.
+# Canonical representation:
+#  `args: {speed: 100}`
+# IR representation:
+#   `{sequence_id: 6, primary_node_id: 7 kind: "speed" value: "100"}`
+class EdgeNode < ApplicationRecord
+  belongs_to :sequence
+  validates_presence_of :sequence
+  belongs_to :primary_node
+  serialize  :value, JSON
+  # BAD_KIND = "must be a valid CeleryScript node name"
+  # validates :kind, inclusion: { in: CeleryScriptSettingsBag::ANY_NODE_NAME,
+  #                                   message: BAD_KIND,
+  #                                   allow_nil: false }
+
+
+  def broadcast?
+    false
+  end
+end

app/models/primary_node.rb

@@ -0,0 +1,39 @@
+# If a node in the sequence node tree has a `kind` and `args` property, it is
+# said to be a properly formed "PrimaryNode". Everything else is an `EdgeNode`.
+# CeleryScript is a tree of PrimaryNode objects in the center and primitive
+# "EdgeNode" types on the edge of the tree.
+class PrimaryNode < ApplicationRecord
+  belongs_to            :sequence
+  validates_presence_of :sequence
+  has_many   :edge_nodes
+  BAD_KIND = "must be a valid CeleryScript argument name"
+  validates :kind, inclusion: { in: CeleryScriptSettingsBag::ANY_NODE_NAME,
+                                message: BAD_KIND,
+                                allow_nil: false }
+  validates :parent_arg_name,
+    inclusion: {in:        CeleryScriptSettingsBag::ANY_ARG_NAME,
+                message:   BAD_KIND,
+                allow_nil: true}
+
+  before_save :next_must_be_body_node
+
+  def next_must_be_body_node
+    raise "NO!" if(next_id && self.class.find(next_id).parent_arg_name)
+  end
+
+  def parent
+    self.class.find_by(id: parent_id)
+    end
+
+  def body
+    self.class.find_by(id: body_id)
+  end
+
+  def next
+    self.class.find_by(id: next_id)
+  end
+
+  def broadcast?
+    false
+  end
+end

app/models/sequence.rb

@@ -3,10 +3,14 @@
 # most of the functionality of a programming language such a variables and
 # conditional logic.
 class Sequence < ApplicationRecord
+  # This number (YYYYMMDD) helps us prepare for the future by keeping things
+  # versioned. We can use it as a means of identifying legacy sequences when
+  # breaking changes happen.
+  LATEST_VERSION    = 20180209
   NOTHING           = { kind: "nothing", args: {} }
   SCOPE_DECLARATION = { kind: "scope_declaration", args: {} }
-  DEFAULT_ARGS      = { locals: SCOPE_DECLARATION,
-                        version: SequenceMigration::Base.latest_version }
+  DEFAULT_ARGS      = { locals:  SCOPE_DECLARATION,
+                        version: LATEST_VERSION  }
   # Does some extra magic for serialized columns for us, such as providing a
   # default value and making hashes have indifferent access.
   class CustomSerializer
@@ -35,6 +39,8 @@ class Sequence < ApplicationRecord
   has_many  :farm_events, as: :executable
   has_many  :regimen_items
   has_many  :sequence_dependencies, dependent: :destroy
+  has_many  :primary_nodes,         dependent: :destroy
+  has_many  :edge_nodes,            dependent: :destroy
   serialize :body, CustomSerializer.new(Array)
   serialize :args, CustomSerializer.new(Hash)
 
@@ -44,22 +50,15 @@ class Sequence < ApplicationRecord
   validates :name, uniqueness: { scope: :device }
   validates  :device, presence: true
 
-  after_find :maybe_migrate
-
   # http://stackoverflow.com/a/5127684/1064917
   before_validation :set_defaults
-
+  around_destroy :delete_nodes_too
   def set_defaults
     self.args              = {}.merge(DEFAULT_ARGS).merge(self.args)
     self.color           ||= "gray"
     self.kind            ||= "sequence"
   end
 
-  def maybe_migrate
-    # spot check with Sequence.order("RANDOM()").first.maybe_migrate
-    Sequences::Migrate.run!(sequence: self, device: self.device)
-  end
-
   def self.random
     Sequence.order("RANDOM()").first
   end
@@ -71,4 +70,13 @@ class Sequence < ApplicationRecord
       .slice(:kind, :args, :body)
     CeleryScript::JSONClimber.climb(hash, &blk)
   end
+
+  def delete_nodes_too
+    Sequence.transaction do
+      PrimaryNode.where(sequence_id: self.id).destroy_all
+      EdgeNode.where(sequence_id: self.id).destroy_all
+      yield
+    end
+  end
+
 end

app/models/tool.rb

@@ -8,8 +8,4 @@ class Tool < ApplicationRecord
   validates  :device, presence: true
   validates  :name, uniqueness: { scope: :device }
   has_many  :sequence_dependencies, dependent: :destroy, as: :dependency
-
-  def slot
-    tool_slot # I kept forgetting. It's just an alias for when I forget.
-  end
 end

app/mutations/auth/create_token.rb

@@ -18,9 +18,11 @@ module Auth
     def validate
       @user = User.where(email: email.downcase).first
       whoops! unless @user && @user.valid_password?(password)
-      if @user && @user.must_consent? && !agree_to_terms && @user.valid_password?(password)
-        @user.require_consent!
-      end
+      must_consent = @user &&
+                     @user.must_consent? &&
+                     !agree_to_terms &&
+                     @user.valid_password?(password)
+      @user.require_consent! if must_consent
     end
 
     def execute

app/mutations/regimens/update.rb

@@ -1,5 +1,6 @@
 module Regimens
   class Update < Mutations::Command
+    BAD_RECORD = "Failed to instantiate nested RegimenItem. Offending item: "
 
     required do
       model :device, class: Device
@@ -27,9 +28,7 @@ module Regimens
       regimen
     rescue ActiveRecord::RecordInvalid => e
       offender = e.record.as_json.slice("time_offset", "sequence_id").to_s
-      add_error :regimen_items,
-                :probably_bad,
-                "Failed to instantiate nested RegimenItem. Offending item: " + offender
+      add_error :regimen_items, :probably_bad, BAD_RECORD + offender
     end
   end
 end

app/mutations/sequences/celery_script_validators.rb

@@ -45,7 +45,7 @@ module Sequences
         kind: "sequence",
         body: symbolized_input[:body],
         args: {
-          version: SequenceMigration::Base.latest_version,
+          version: Sequence::LATEST_VERSION,
           locals:  symbolized_input
             .deep_symbolize_keys
             .dig(:args, :locals) || Sequence::SCOPE_DECLARATION

app/mutations/sequences/create.rb

@@ -30,13 +30,15 @@ module Sequences
       seq.args["is_outdated"] = false
       # version is never user definable!
       # IF YOU REMOVE THIS BAD STUFF WILL HAPPEN:
-      seq.args["version"]     = SequenceMigration::Base.latest_version
+      seq.args["version"]     = Sequence::LATEST_VERSION
       # See comment above ^
       ActiveRecord::Base.transaction do
+        seq.migrated_nodes = true
         seq.save!
         reload_dependencies(seq)
+        CeleryScript::StoreCelery.run!(sequence: seq)
       end
-      seq
+      CeleryScript::FetchCelery.run!(sequence: seq.reload) # Perf nightmare?
     end
   end
 end

app/mutations/sequences/migrate.rb

@@ -1,25 +0,0 @@
-module Sequences
-  class Migrate < Mutations::Command
-    required do
-      model :device, class: Device
-      model :sequence, class: Sequence
-    end
-
-    optional do
-      boolean :save, default: false
-    end
-
-    def validate
-    end
-
-    def execute
-      theirs = sequence.args["version"]
-      ours   = SequenceMigration::Base.latest_version
-      if theirs != ours
-        SequenceMigration::Base.generate_list(sequence).map(&:run)
-        sequence.args["is_outdated"] = true
-      end
-      sequence
-    end
-  end
-end

app/mutations/sequences/update.rb

@@ -1,7 +1,7 @@
 module Sequences
   class Update < Mutations::Command
     include CeleryScriptValidators
-
+    UNKNOWN = "Unknown validation issues."
     required do
       model :device, class: Device
       model :sequence, class: Sequence
@@ -28,13 +28,14 @@ module Sequences
     def execute
       ActiveRecord::Base.transaction do
         sequence.args["is_outdated"] = false
+        sequence.migrated_nodes = true
         sequence.update_attributes!(inputs.except(:sequence, :device))
         reload_dependencies(sequence)
+        CeleryScript::StoreCelery.run!(sequence: sequence)
       end
-      sequence
+      CeleryScript::FetchCelery.run!(sequence: sequence.reload)
     rescue ActiveRecord::RecordInvalid => e
-      m = (e.try(:message) || "Unknown validation issues.")
-      add_error :other, :unknown, m
+      add_error :other, :unknown, (e.try(:message) || UNKNOWN)
     end
   end
 end

config/application.rb

@@ -1,7 +1,8 @@
 require_relative "../app/models/transport.rb"
 require File.expand_path('../boot', __FILE__)
-
+require_relative "../app/lib/celery_script/csheap"
 require "rails/all"
+
 # Require the gems listed in Gemfile, including any gems
 # you've limited to :test, :development, or :production.
 Bundler.require(:default, Rails.env)

db/migrate/20180126141955_new_sequence_schema.rb

@@ -0,0 +1,26 @@
+class NewSequenceSchema < ActiveRecord::Migration[5.1]
+  def change
+    create_table :primary_nodes do |t|
+      t.timestamps null: false
+      t.references  :sequence, index: true, null: false
+      t.string      :kind,     limit: 50
+      t.references  :child,    index: true, null: true
+      t.references  :parent,   index: true, null: true
+      t.string      :parent_arg_name, limit: 50
+      # 👆 Longest CS Node kind: 28 characters. 50 is plenty. -RC
+    end
+
+    create_table :edge_nodes do |t|
+      t.timestamps null: false
+      t.references :sequence,     index: true, null: false
+      t.references :primary_node, index: true, null: false
+      # Longest CS Node kind: 28 characters.
+      t.string     :kind,         limit: 50
+      # Serialized String, Integer or Boolean.
+      t.string     :value,        limit: 300
+    end
+
+    add_foreign_key :edge_nodes,    :sequences
+    add_foreign_key :primary_nodes, :sequences
+  end
+end

db/migrate/20180201153221_add_migrated_nodes_to_sequences.rb

@@ -0,0 +1,5 @@
+class AddMigratedNodesToSequences < ActiveRecord::Migration[5.1]
+  def change
+    add_column :sequences, :migrated_nodes, :boolean, default: false
+  end
+end

db/migrate/20180202165503_change_primary_node_column_names.rb

@@ -0,0 +1,6 @@
+class ChangePrimaryNodeColumnNames < ActiveRecord::Migration[5.1]
+  def change
+    add_reference :primary_nodes, :next, index: true
+    add_reference :primary_nodes, :body, index: true
+  end
+end

db/migrate/20180209134752_add_comment_column_to_primary_nodes.rb

@@ -0,0 +1,8 @@
+class AddCommentColumnToPrimaryNodes < ActiveRecord::Migration[5.1]
+  def change
+    add_column :primary_nodes,
+               :comment,
+               :string,
+               limit: 80
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180205173255) do
+ActiveRecord::Schema.define(version: 20180209134752) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -256,11 +256,14 @@ ActiveRecord::Schema.define(version: 20180205173255) do
     t.datetime "updated_at", null: false
     t.bigint "sequence_id", null: false
     t.string "kind", limit: 50
-    t.bigint "body_id"
+    t.bigint "child_id"
     t.bigint "parent_id"
     t.string "parent_arg_name", limit: 50
     t.bigint "next_id"
+    t.bigint "body_id"
+    t.string "comment", limit: 80
     t.index ["body_id"], name: "index_primary_nodes_on_body_id"
+    t.index ["child_id"], name: "index_primary_nodes_on_child_id"
     t.index ["next_id"], name: "index_primary_nodes_on_next_id"
     t.index ["parent_id"], name: "index_primary_nodes_on_parent_id"
     t.index ["sequence_id"], name: "index_primary_nodes_on_sequence_id"

db/seeds.rb

@@ -78,7 +78,7 @@ unless Rails.env == "production"
     y:0, z:0}}, offset:{kind:"coordinate", args:{x:0, y:0, z:0}}, speed:100}}])
     t  = Tools::Create.run!(name: "Trench Digging Tool", device: u.device)
     body_txt = File.read("spec/lib/celery_script/ast_fixture4.json")
-                   .gsub("__SEQUENCE_ID__", s.id.to_s)
+                   .gsub("__SEQUENCE_ID__", s[:id].to_s)
                    .gsub("__TOOL_ID__", t.id.to_s)
     Sequences::Create.run!(device: u.device,
         name: "Every Node",
@@ -87,9 +87,9 @@ unless Rails.env == "production"
                          name:"Test Regimen 456",
                          color:"gray",
                          regimen_items: [
-                           {time_offset:300000, sequence_id:s.id},
-                           {time_offset:173100000, sequence_id:s.id},
-                           {time_offset:345900000, sequence_id:s.id}
+                           {time_offset:300000, sequence_id:s[:id]},
+                           {time_offset:173100000, sequence_id:s[:id]},
+                           {time_offset:345900000, sequence_id:s[:id]}
                          ])
     Peripherals::Create.run!(device: u.device, pin: 13, label: "LED")
     2.times do

latest_corpus.rb

@@ -114,7 +114,7 @@ class CorpusEmitter
     */
     """)
     result.push(enum_type :CeleryNode, NODES.map(&:name).map(&:camelize), false)
-    result.push(const(:LATEST_VERSION, SequenceMigration::Base.latest_version))
+    result.push(const(:LATEST_VERSION, Sequence::LATEST_VERSION))
     result.push(const :DIGITAL, CeleryScriptSettingsBag::DIGITAL)
     result.push(const :ANALOG, CeleryScriptSettingsBag::ANALOG)
     result.push(enum_type :ALLOWED_PIN_MODES,

migrate_old_sequences.rb

@@ -0,0 +1,19 @@
+
+def find_next_seq
+  Sequence.where(migrated_nodes: false).order("created_at").last
+end
+next_seq   = find_next_seq
+count      = 0
+total_time = Time.now
+until next_seq == nil
+  t = Time.now
+  count += 1
+  puts "=== Migrating sequence #{next_seq.id}: #{next_seq.name}"
+  Sequence.transaction { CeleryScript::StoreCelery.run!(sequence: next_seq) }
+  puts "=== migrated in #{Time.now - t} seconds"
+  next_seq = find_next_seq
+end
+
+t2 = Time.now - total_time
+
+puts "=== DONE MIGRATING #{count} sequences in #{t2} seconds! (#{count/t2} per second)"

spec/controllers/api/sequences/sequences_create_spec.rb

@@ -190,8 +190,8 @@ describe Api::SequencesController do
 
     it 'tracks Points' do
       point = FactoryBot.create(:point, device: user.device)
-      SequenceDependency.delete_all
-      Sequence.delete_all
+      SequenceDependency.destroy_all
+      Sequence.destroy_all
       old_count = SequenceDependency.count
       HAS_POINTS["body"][0]["args"]["location"]["args"]["pointer_id"] =
         point.id

spec/controllers/api/sequences/sequences_destroy_spec.rb

@@ -26,29 +26,31 @@ describe Api::SequencesController do
       other_persons = FactoryBot.create(:sequence)
       input = { id: other_persons.id }
       delete :destroy, params: input
-      expect(response.status).to eq(403)
+      expect(response.status).to eq(404)
     end
 
-   it 'allows deletion of recurive sequences' do
-     sign_in user
-     s = Sequences::Create.run!({ device: user.device,
+    it 'allows deletion of recurive sequences' do
+      sign_in user
+      s = Sequences::Create.run!({device: user.device,
                                   name: "Rick-cursion", body: [] })
-     patch :update,
-           params: {id: s.id },
-           body: {"sequence" => {
-                  "body" => [{"kind"=>"execute",
-                              "args"=>{ "sequence_id"=>s.id }}]}}.to_json,
-           as: :json
+      body = {
+        sequence: { body: [{ kind: "execute", args: { sequence_id: s[:id] } }] }
+      }.to_json
 
-     sequence.reload
-     input = { id: sequence.id }
-     before = Sequence.count
-     delete :destroy, params: input
-     after  = Sequence.count
-     expect(response.status).to eq(200)
-     expect(after).to be < before
-     expect { s.reload }.to(raise_error(ActiveRecord::RecordNotFound))
-   end
+      patch :update,
+            params: {id: s[:id] },
+            body: body,
+            as: :json
+
+      sequence.reload
+      input = { id: sequence.id }
+      before = Sequence.count
+      delete :destroy, params: input
+      after  = Sequence.count
+      expect(response.status).to eq(200)
+      expect(after).to be < before
+      expect { Sequence.find(s[:id]) }.to(raise_error(ActiveRecord::RecordNotFound))
+    end
 
     it 'does not destroy a sequence when in use by a sequence' do
       before = SequenceDependency.count

spec/factories/sequences.rb

@@ -6,9 +6,7 @@ FactoryBot.define do
     color { Sequence::COLORS.sample }
     device
     kind "sequence"
-    args({
-      version: 4 # Hard coding it for now - RC Nov 22
-    })
+    args({ version: 4 })
     body([])
   end
 end

spec/lib/celery_script/corpus_spec.rb

@@ -101,7 +101,7 @@ describe CeleryScript::Corpus do
   it "serializes into JSON" do
       result = JSON.parse(corpus.to_json)
 
-      expect(result["tag"]).to eq(6)
+      expect(result["tag"]).to eq(Sequence::LATEST_VERSION)
       expect(result["args"]).to be_kind_of(Array)
       expect(result["nodes"]).to be_kind_of(Array)
       expect(result["nodes"].sample.keys.sort).to eq(["allowed_args",

spec/lib/sequence_migration_spec.rb

@@ -1,96 +0,0 @@
-require 'spec_helper'
-
-FIXTURE = JSON.parse('[
-  {
-     "kind":"move_absolute",
-     "args":{
-        "location":{
-           "kind":"coordinate",
-           "args":{
-              "x":0,
-              "y":0,
-              "z":0
-           }
-        },
-        "offset":{
-           "kind":"coordinate",
-           "args":{
-              "x":0,
-              "y":0,
-              "z":0
-           }
-        },
-        "speed":200
-     },
-     "uuid":"c1d740a1-5807-42dd-949f-3f28993d96bc"
-  },
-  {
-     "kind":"move_relative",
-     "args":{
-        "x":0,
-        "y":0,
-        "z":0,
-        "speed":200
-     },
-     "uuid":"003101ee-63b6-4a41-bfe8-cdd6b95480ea"
-  },
-  {
-     "kind":"find_home",
-     "args":{
-        "axis":"all",
-        "speed":200
-     },
-     "uuid":"7be663f7-56ff-4b77-9356-49e965a8fa87"
-  }
-]')
-
-class MockMigration < SequenceMigration::Base
-  VERSION = -9
-end
-
-class MockMigrationTwo < SequenceMigration::Base
-  VERSION = 1
-end
-
-describe SequenceMigration do
-  it 'has a latest version' do
-      expect(SequenceMigration::Base.latest_version).to eq(6)
-  end
-
-  it 'updates speed on all the things < v5' do
-    s = FactoryBot.create(:sequence)
-    s.args["version"] = 4
-    s.body            = FIXTURE
-    expect(s.body[0]["args"]["speed"]).to eq(200)
-    expect(s.body[1]["args"]["speed"]).to eq(200)
-    expect(s.body[2]["args"]["speed"]).to eq(200)
-
-    s.maybe_migrate
-
-    expect(s.body[0]["args"]["speed"]).to eq(100)
-    expect(s.body[1]["args"]["speed"]).to eq(100)
-    expect(s.body[2]["args"]["speed"]).to eq(100)
-    expect(s.args.dig("locals","kind")).to eq("scope_declaration")
-  end
-
-  it 'warns developers that `up()` is required' do
-    base = SequenceMigration::Base.new(FactoryBot.create(:sequence))
-    expect { base.up }.to raise_error(SequenceMigration::Base::UP_IS_REQUIRED)
-  end
-
-  it 'checks for appropriate version number when running `before()`' do
-    base = MockMigration.new(FactoryBot.build(:sequence))
-    expect { base.before }
-      .to raise_error("Version must be -10 to run MockMigration. Got: 4")
-  end
-
-  it 'sorts migrations by version number' do
-    allow(SequenceMigration::Base)
-      .to receive(:descendants) { [MockMigration, MockMigrationTwo] }
-    sequence                 = FactoryBot.build(:sequence)
-    sequence.args["version"] = -10
-    result                   = SequenceMigration::Base.generate_list(sequence)
-    expect(result.first).to be_kind_of(MockMigration)
-    expect(result.last).to  be_kind_of(MockMigrationTwo)
-  end
-end

spec/mutations/flat_ir_stuff/cs_heap_spec.rb

@@ -0,0 +1,9 @@
+require "spec_helper"
+
+describe CeleryScript::CSHeap do
+  it "raises if address is bad" do
+    expect do
+      CeleryScript::CSHeap.new.put(CeleryScript::HeapAddress[99], "no", "no")
+    end.to raise_error(CeleryScript::CSHeap::BadAddress)
+  end
+end

spec/mutations/flat_ir_stuff/fetch_celery_spec.rb

@@ -0,0 +1,53 @@
+require "spec_helper"
+require_relative "./flat_ir_helpers"
+
+describe CeleryScript::FetchCelery do
+  let(:user) { FactoryBot.create(:user) }
+  let(:device) { user.device }
+
+  __NOTHING______ = "nothing"
+
+  it "Makes JSON that is identical to the legacy implementation - part 1" do
+    Sequence.all.destroy_all
+    expect(Sequence.count).to eq(0)
+    expect(PrimaryNode.count).to eq(0)
+    expect(EdgeNode.count).to eq(0)
+    params          = CeleryScript::FlatIrHelpers.typical_sequence
+    params[:device] = device
+    hash            = Sequences::Create.run!(params)
+    actual          = CeleryScript::FetchCelery
+      .run!(sequence: Sequence.find(hash[:id]))
+    expected        = hash.without(:device_id, :migrated_nodes)
+    expect(actual[:body]).to be_kind_of(Array)
+    nodes = Sequence.find(actual[:id]).primary_nodes
+    # This table came from the JS implementation, which is "known good".
+    [
+      #KIND                 PARENT           NEXT             BODY
+      ["nothing",           __NOTHING______, __NOTHING______, __NOTHING______],
+      ["sequence",        __NOTHING______, __NOTHING______, "move_absolute"],
+      ["move_absolute",     "sequence",      "move_relative", __NOTHING______],
+      ["coordinate",        "move_absolute", __NOTHING______, __NOTHING______],
+      ["move_relative",     "move_absolute", "write_pin",     __NOTHING______],
+      ["write_pin",         "move_relative", __NOTHING______, __NOTHING______],
+      ["scope_declaration", "sequence",      __NOTHING______, __NOTHING______],
+    ].map do |(me, expect_parent, expect_next, expect_body)|
+      inspected = nodes.find_by(kind: me)
+      expect(inspected.parent.kind)
+        .to(eq(expect_parent), "BAD PARENT_ID: #{inspected.kind}")
+      expect(inspected.next.kind)
+        .to(eq(expect_next), "BAD NEXT_ID: #{inspected.kind}")
+      expect(inspected.body.kind)
+        .to(eq(expect_body), "BAD BODY_ID: #{inspected.kind}")
+    end
+
+    expected[:body]
+      .each_with_index do |item, index|
+        x = actual[:body][index]
+        y = expected[:body][index]
+        expect(HashDiff.diff(x, y)).to eq([])
+      end
+      expected[:args][:locals][:body] ||= []
+      actual[:args][:locals][:body]   ||= []
+      expect(HashDiff.diff(actual, expected)).to eq([])
+  end
+end

spec/mutations/flat_ir_stuff/first_pass_spec.rb

@@ -0,0 +1,87 @@
+require "spec_helper"
+require_relative "./flat_ir_helpers"
+require_relative "../../../app/lib/celery_script/slicer"
+
+describe CeleryScript::FirstPass do
+  let :result do
+    Sequence.all.destroy_all
+    expect(EdgeNode.count).to eq(0)
+    expect(PrimaryNode.count).to eq(0)
+    CeleryScript::FlatIrHelpers.fake_first_pass
+  end
+
+  kind   = CeleryScript::CSHeap::KIND
+  parent = CeleryScript::CSHeap::PARENT
+  next_  = CeleryScript::CSHeap::NEXT
+  body   = CeleryScript::CSHeap::BODY
+
+  EXPECTATIONS = { # Came from the JS implementation which is known good.
+    0 => { kind => "nothing",           parent => 0, next_ => 0 },
+    1 => { kind => "sequence",          parent => 0, body  => 2 },
+    2 => { kind => "move_absolute",     parent => 1, next_ => 3 },
+    3 => { kind => "coordinate",        parent => 2, next_ => 0 },
+    4 => { kind => "coordinate",        parent => 2, next_ => 0 },
+    5 => { kind => "move_relative",     parent => 2, next_ => 6 },
+    6 => { kind => "write_pin",         parent => 5, next_ => 0 },
+    7 => { kind => "scope_declaration", parent => 1, next_ => 0 }
+  }
+
+  it "sets the correct parent" do
+    parent_look_up = {
+      "sequence"     => "nothing",
+      "_if"          => "take_photo",
+      "take_photo"   => "send_message",
+      "send_message" => "sequence",
+    }
+
+    result
+      .map do |node|
+        expected_parent = parent_look_up[node.kind]
+        expect(node.parent.kind).to eq(expected_parent) if expected_parent
+      end
+  end
+
+  it "sets the correct next node" do
+    next_node_look_up = { "nothing"      => "nothing",
+                          "sequence"     => "nothing",
+                          "send_message" => "take_photo",
+                          "take_photo"   => "_if" }
+
+    result
+      .map do |node|
+        xpected_next = next_node_look_up[node.kind]
+        expect(node.next.kind).to eq(xpected_next) if xpected_next
+      end
+  end
+
+  it "set the correct body nodes" do
+    body_lookup = { "nothing"      => "nothing",
+                          "sequence"     => "nothing",
+                          "send_message" => "take_photo",
+                          "take_photo"   => "_if" }
+
+    result
+      .map do |node|
+        xpected_next = body_lookup[node.kind]
+        expect(node.next.kind).to eq(xpected_next) if xpected_next
+      end
+  end
+
+  it "saves nodes" do
+    Sequence.destroy_all
+    result
+    {
+      "coordinate"        => 2,
+      "move_absolute"     => 1,
+      "move_relative"     => 1,
+      "nothing"           => 1,
+      "scope_declaration" => 1,
+      "sequence"          => 1,
+      "write_pin"         => 1,
+    }.to_a.map do |(kind, count)|
+      real_count = PrimaryNode.where(kind: kind).count
+      msg = "Expected #{count} #{kind} nodes. Got #{real_count}"
+      expect(real_count).to(eq(count), msg)
+    end
+  end
+end

spec/mutations/flat_ir_stuff/flat_ir_helpers.rb

@@ -0,0 +1,75 @@
+module CeleryScript
+  # Some helpers to DRY up Flat IR unit tests.
+  class FlatIrHelpers
+    def self.typical_sequence
+      {
+        kind: "sequence",
+        name: "move_abs(1,2,3), move_rel(4,5,6), write_pin(13, off, digital)",
+        color: "gray",
+        args: {
+          locals: { kind: "scope_declaration", args: {}, body: [] },
+          version: 6,
+          label: "move_abs(1,2,3), move_rel(4,5,6), write_pin(13, off, digital)"
+        },
+        body: [
+          {
+            kind: "move_absolute",
+            args: {
+              location: { kind: "coordinate", args: { x: 0, y: 0, z: 0 } },
+              offset: { kind: "coordinate", args: { x: 0, y: 0, z: 0 } },
+              speed: 100
+            }
+          },
+          {
+            kind: "move_relative",
+            args: { x: 0, y: 0, z: 0, speed: 100 }
+          },
+          {
+            kind: "write_pin",
+            args: { pin_number: 0, pin_value: 0, pin_mode: 0 }
+          }
+        ]
+      }
+    end
+
+    def self.typical_sequence2
+      {
+        kind: 'sequence',
+        args: { locals: { kind: 'scope_declaration', args: {}, body: [] } },
+        body: [
+          { kind: 'take_photo', args: {} },
+          {
+            kind: 'send_message',
+            args: { message: 'test case 1', message_type: 'success' },
+            body: [
+              { kind: 'channel', args: { channel_name: 'toast' } },
+            ],
+          },
+          {
+            kind: '_if',
+            args: {
+              lhs: 'x',
+              op: 'is',
+              rhs: 0,
+              _then: { kind: 'execute', args: { sequence_id: 10 } },
+              _else: { kind: 'nothing', args: {} }
+            }
+          },
+        ]
+      }
+    end
+
+    def self.flattened_heap
+      slicer = CeleryScript::Slicer.new
+      slicer.run!(typical_sequence2)
+      slicer.heap_values
+    end
+
+    def self.fake_first_pass
+      sequence      = FactoryBot.create(:sequence)
+      sequence.args = typical_sequence[:args]
+      sequence.body = typical_sequence[:body]
+      FirstPass.run!(sequence: sequence)
+    end
+  end
+end

spec/mutations/flat_ir_stuff/heap_address_spec.rb

@@ -0,0 +1,13 @@
+require "spec_helper"
+
+describe CeleryScript::HeapAddress do
+  EXAMPLE = CeleryScript::HeapAddress[42]
+
+  it "stringifies" do
+    expect(EXAMPLE.to_s).to eq("42")
+  end
+
+  it "inspects" do
+    expect(EXAMPLE.inspect).to eq("HeapAddress(42)")
+  end
+end

spec/mutations/flat_ir_stuff/slicer_spec.rb

@@ -0,0 +1,132 @@
+require "spec_helper"
+require_relative "./flat_ir_helpers"
+
+describe CeleryScript::Slicer do
+  kind    = CeleryScript::CSHeap::KIND
+  parent  = CeleryScript::CSHeap::PARENT
+  next_   = CeleryScript::CSHeap::NEXT
+  body    = CeleryScript::CSHeap::BODY
+  comment = CeleryScript::CSHeap::COMMENT
+
+  n = "nothing"
+
+  CENTIPEDE_SEQUENCE = {
+    kind: "ROOT",
+    args: { a: "b" },
+    body: [
+      {
+        kind: "ROOT[0]",
+        args: {c: "d"},
+        body: [ { kind: "ROOT[0][0]", args: {e: "f"} } ]
+      },
+      {
+        kind: "ROOT[1]",
+        args: {c: "d"},
+        body: [
+          { kind: "ROOT[1][0]", args: {g: "H"} },
+          { kind: "ROOT[1][1]", args: {i: "j"} },
+          { kind: "ROOT[1][2]", args: {k: "l"} }
+        ]
+      },
+      {
+        kind: "ROOT[2]",
+        args: {c: "d"},
+        body: [
+          { kind: "ROOT[2][0]", args: {m: "n"} },
+          { kind: "ROOT[2][1]", args: {o: "p"} },
+          {
+            kind: "ROOT[2][2]",
+            args: {q: "r"},
+            body: [
+              {
+                kind: "ROOT[2][2][0]",
+                args: {
+                  g: "H"
+                },
+                comment: "very deep node"
+              },
+            ]
+          }
+        ]
+      }
+    ]
+  }
+
+  CORRECT_LINKAGE = { # Came from the JS implementation which is known good.
+    "sequence"          => { p: n,              b: "take_photo", n: n },
+    "scope_declaration" => { p: "sequence",     b: n,            n: n, },
+    "take_photo"        => { p: "sequence",     b: n,            n: "send_message", },
+    "send_message"      => { p: "take_photo",   b: "channel",    n: "_if",},
+    "channel"           => { p: "send_message", b: n,            n: n, },
+    "_if"               => { p: "send_message", b: n,            n: n, },
+    "execute"           => { p: "_if",          b: n,            n: n, },
+  }
+
+
+  it "handles even the most heavily nested nodes" do
+    slicer = CeleryScript::Slicer.new
+    slicer.run!(CENTIPEDE_SEQUENCE)
+    results = slicer
+      .heap_values
+      .entries
+      .to_a
+      .map{ |x| x.slice(kind, body, parent, next_) }
+      .map
+      .with_index(0) do |x, index|
+        [index, x]
+      end.to_h
+    first_comment = slicer.heap_values.map{|x| x[comment] }.compact.first
+    expect(first_comment).to include("deep node")
+    addr = CeleryScript::HeapAddress
+    expectations = {
+      0  => {kind => "nothing",       body => addr[0],  parent => addr[0],  next_ => addr[0] },
+      1  => {kind => "ROOT",          body => addr[2],  parent => addr[0],  next_ => addr[0] },
+      2  => {kind => "ROOT[0]",       body => addr[3],  parent => addr[1],  next_ => addr[4] },
+      3  => {kind => "ROOT[0][0]",    body => addr[0],  parent => addr[2],  next_ => addr[0] },
+      4  => {kind => "ROOT[1]",       body => addr[5],  parent => addr[2],  next_ => addr[8] },
+      5  => {kind => "ROOT[1][0]",    body => addr[0],  parent => addr[4],  next_ => addr[6] },
+      6  => {kind => "ROOT[1][1]",    body => addr[0],  parent => addr[5],  next_ => addr[7] },
+      7  => {kind => "ROOT[1][2]",    body => addr[0],  parent => addr[6],  next_ => addr[0] },
+      8  => {kind => "ROOT[2]",       body => addr[9],  parent => addr[4],  next_ => addr[0] },
+      9  => {kind => "ROOT[2][0]",    body => addr[0],  parent => addr[8],  next_ => addr[10]},
+      10 => {kind => "ROOT[2][1]",    body => addr[0],  parent => addr[9],  next_ => addr[11]},
+      11 => {kind => "ROOT[2][2]",    body => addr[12], parent => addr[10], next_ => addr[0] },
+      12 => {kind => "ROOT[2][2][0]", body => addr[0],  parent => addr[11], next_ => addr[0] },
+    }
+
+    results
+      .to_a
+      .each_with_index do |(index, item)|
+        expect(expectations[index]).to eq(item)
+      end
+  end
+
+  it "attaches `body`, `next` and `parent`" do
+    heap = CeleryScript::FlatIrHelpers.flattened_heap
+    nothing_node = heap[0]
+    expect(nothing_node[kind]).to         eq(n)
+    expect(nothing_node[body].value).to   eq(0)
+    expect(nothing_node[parent].value).to eq(0)
+    expect(nothing_node[next_].value).to  eq(0)
+    output = heap.index_by { |x| x[kind] }
+    CORRECT_LINKAGE.keys.map do |target_kind|
+        actual      = output[target_kind]
+        expectation = CORRECT_LINKAGE[actual[kind]]
+        expect(actual[kind]).to eq(target_kind)
+
+        parent_addr = output[target_kind][parent].value
+        parent_kind = heap[parent_addr][kind]
+        expect(parent_kind).to eq(expectation[:p])
+
+        next_addr = output[target_kind][next_].value
+        next_kind = heap[next_addr][kind]
+        expect(next_kind).to eq(expectation[:n])
+
+        body_addr = output[target_kind][body].value
+        body_kind = heap[body_addr][kind]
+        failure = "`#{target_kind}` needs a body of `#{expectation[:b]}`. "\
+                  "Got `#{body_kind}`."
+        expect(body_kind).to(eq(expectation[:b]), failure)
+    end
+  end
+end

spec/mutations/points/destroy_spec.rb

@@ -7,6 +7,6 @@ describe Points::Destroy do
     result = Points::Destroy.run(points: [s.tool_slot], device: s.device)
     expect(result.success?).to be(false)
     expect(result.errors.message_list)
-      .to include(Points::Destroy::STILL_IN_USE % s.sequence.name)
+      .to include(Points::Destroy::STILL_IN_USE % s.sequence[:name])
   end
 end

spec/mutations/points/update_spec.rb

@@ -9,6 +9,6 @@ describe Points::Update do
                                 tool_id: nil)
     expect(result.success?).to be(false)
     expect(result.errors.message_list)
-      .to include(Points::ToolRemovalCheck::IN_USE % s.sequence.name)
+      .to include(Points::ToolRemovalCheck::IN_USE % s.sequence[:name])
   end
 end

spec/mutations/sequences/create_spec.rb

@@ -14,8 +14,8 @@ describe Sequences::Create do
 
   it 'Builds a `sequence`' do
     seq = Sequences::Create.run!(sequence_params)
-    expect(seq.name).to eq(name)
-    expect(seq.device).to eq(device)
+    expect(seq[:name]).to eq(name)
+    expect(Sequence.find(seq[:id]).device).to eq(device)
   end
 
   it 'Gives validation errors for malformed AST nodes' do
@@ -85,8 +85,8 @@ describe Sequences::Create do
       ]
     }
     seq = Sequences::Create.run!(app)
-    expect(seq.body.first[:body].first["kind"]).to eq("channel")
-    expect(seq.body.dig(0, :args, :message)).to eq("Hello, world!")
+    expect(seq[:body].first[:body].first["kind"]).to eq("channel")
+    expect(seq[:body].dig(0, :args, :message)).to eq("Hello, world!")
   end
 
   it "Strips UUIDs and other 'noises', leaves other attributes in tact. " do
@@ -114,10 +114,10 @@ describe Sequences::Create do
       device: device,
       color:  "gray",
       name:   "New Sequence",
-    }).reload
-    expected    = result.body.dig(0, "args", "location", "args")
+    })
+    expected    = result[:body].dig(0, "args", "location", "args")
     actual      = body.dig(0, "args", "location", "args")
-    extra_stuff = result.body.map{|x| x["uuid"]}.compact
+    extra_stuff = result[:body].map{|x| x["uuid"]}.compact
     expect(extra_stuff.length).to eq(0)
     expect(expected).to eq(actual)
   end