UNSTABLE: Factor down service runners.

pull/1214/head
Rick Carlino 2019-05-29 15:48:58 -05:00
parent 2633643fb9
commit 4b8739c18e
7 changed files with 79 additions and 117 deletions

View File

@ -0,0 +1,19 @@
class AbstractServiceRunner
WAIT_TIME = Rails.env.test? ? 0.01 : 5
OFFLINE_ERROR = Bunny::TCPConnectionFailedForAllHosts
CRASH_MSG = Rails.env.test? ?
"\e[32m.\e[0m" : "Something caused the broker to crash...\n"
def go!(channel)
channel.subscribe(block: true) do |info, _, payl|
self.process(info, payl.force_encoding("UTF-8"))
end
rescue OFFLINE_ERROR, StandardError => e
unless e.is_a?(OFFLINE_ERROR)
Rollbar.error(e)
print CRASH_MSG
end
sleep WAIT_TIME
retry
end
end

View File

@ -1,13 +1,13 @@
# A singleton that runs on a separate process than the web server. # A singleton that runs on a separate process than the web server.
# Listens to *ALL* incoming logs and stores them to the DB. # Listens to *ALL* incoming logs and stores them to the DB.
# Also handles throttling. # Also handles throttling.
class LogService class LogService < AbstractServiceRunner
T = ThrottlePolicy::TimePeriod T = ThrottlePolicy::TimePeriod
THROTTLE_POLICY = ThrottlePolicy.new T.new(1.minute) => 0.5 * 1_000, THROTTLE_POLICY = ThrottlePolicy.new T.new(1.minute) => 0.5 * 1_000,
T.new(1.hour) => 0.5 * 10_000, T.new(1.hour) => 0.5 * 10_000,
T.new(1.day) => 0.5 * 100_000 T.new(1.day) => 0.5 * 100_000
def self.process(delivery_info, payload) def process(delivery_info, payload)
params = { routing_key: delivery_info.routing_key, payload: payload } params = { routing_key: delivery_info.routing_key, payload: payload }
m = AmqpLogParser.run!(params) m = AmqpLogParser.run!(params)
puts "#{m.device_id}: #{m.payload["message"]}" if Rails.env.production? puts "#{m.device_id}: #{m.payload["message"]}" if Rails.env.production?
@ -15,7 +15,7 @@ class LogService
maybe_deliver(m) maybe_deliver(m)
end end
def self.maybe_deliver(data) def maybe_deliver(data)
violation = THROTTLE_POLICY.is_throttled(data.device_id) violation = THROTTLE_POLICY.is_throttled(data.device_id)
ok = data.valid? && !violation ok = data.valid? && !violation
@ -24,13 +24,13 @@ class LogService
end end
end end
def self.deliver(data) def deliver(data)
dev, log = [data.device, data.payload] dev, log = [data.device, data.payload]
dev.maybe_unthrottle dev.maybe_unthrottle
Log.deliver(dev, Logs::Create.run!(log, device: dev)) Log.deliver(dev, Logs::Create.run!(log, device: dev))
end end
def self.warn_user(data, violation) def warn_user(data, violation)
data.device.maybe_throttle(violation) data.device.maybe_throttle(violation)
end end
end end

View File

@ -1,31 +1,30 @@
module Resources module Resources
MQTT_CHAN = "from_api" MQTT_CHAN = "from_api"
CHANNEL_TPL = CHANNEL_TPL =
"bot.device_%{device_id}.resources_v0.%{action}.%{klass}.%{uuid}.%{id}" "bot.device_%{device_id}.resources_v0.%{action}.%{klass}.%{uuid}.%{id}"
INDEX_OF_USERNAME = 1 INDEX_OF_USERNAME = 1
INDEX_OF_OP = 3 INDEX_OF_OP = 3
INDEX_OF_KIND = 4 INDEX_OF_KIND = 4
INDEX_OF_UUID = 5 INDEX_OF_UUID = 5
INDEX_OF_ID = 6 INDEX_OF_ID = 6
class Service class Service < AbstractServiceRunner
def ok(uuid)
def self.ok(uuid)
{ kind: "rpc_ok", args: { label: uuid } }.to_json { kind: "rpc_ok", args: { label: uuid } }.to_json
end end
def self.rpc_err(uuid, error) def rpc_err(uuid, error)
{ {
kind: "rpc_error", kind: "rpc_error",
args: { label: uuid }, args: { label: uuid },
body: (error body: (error
.errors .errors
.values .values
.map { |err| { kind: "explanation", args: { message: err.message }} }) .map { |err| { kind: "explanation", args: { message: err.message } } }),
}.to_json }.to_json
end end
def self.step1(delivery_info, body) # Returns params or nil def step1(delivery_info, body) # Returns params or nil
Preprocessor.from_amqp(delivery_info, body) Preprocessor.from_amqp(delivery_info, body)
rescue Mutations::ValidationException => q rescue Mutations::ValidationException => q
# AUTHORS NOTE: Some of the Bunny data structures have circular # AUTHORS NOTE: Some of the Bunny data structures have circular
@ -36,26 +35,26 @@ module Resources
x = delivery_info.to_h.slice(*safe_attrs).merge(body: body) x = delivery_info.to_h.slice(*safe_attrs).merge(body: body)
Rollbar.error(q, x) Rollbar.error(q, x)
raw_chan = delivery_info&.routing_key&.split(".") || [] raw_chan = delivery_info&.routing_key&.split(".") || []
id = raw_chan[INDEX_OF_USERNAME]&.gsub("device_", "")&.to_i id = raw_chan[INDEX_OF_USERNAME]&.gsub("device_", "")&.to_i
uuid = raw_chan[INDEX_OF_UUID] || "NONE" uuid = raw_chan[INDEX_OF_UUID] || "NONE"
Transport.current.amqp_send(rpc_err(uuid, q), id, MQTT_CHAN) if id Transport.current.amqp_send(rpc_err(uuid, q), id, MQTT_CHAN) if id
nil nil
end end
def self.step2(params) def step2(params)
dev = params[:device] dev = params[:device]
dev.auto_sync_transaction do dev.auto_sync_transaction do
Job.run!(params) Job.run!(params)
uuid = (params[:uuid] || "NONE") uuid = (params[:uuid] || "NONE")
Transport.current.amqp_send(ok(uuid), dev.id, MQTT_CHAN) Transport.current.amqp_send(ok(uuid), dev.id, MQTT_CHAN)
end end
rescue Mutations::ValidationException => q rescue Mutations::ValidationException => q
device = params.fetch(:device) device = params.fetch(:device)
Rollbar.info("device_#{device.id} using AMQP resource mgmt") Rollbar.info("device_#{device.id} using AMQP resource mgmt")
uuid = params.fetch(:uuid) uuid = params.fetch(:uuid)
errors = q.errors.values.map do |err| errors = q.errors.values.map do |err|
{ kind: "explanation", args: { message: err.message }} { kind: "explanation", args: { message: err.message } }
end end
message = { kind: "rpc_error", message = { kind: "rpc_error",
args: { label: uuid }, args: { label: uuid },
@ -63,7 +62,7 @@ module Resources
Transport.current.amqp_send(message, device.id, MQTT_CHAN) Transport.current.amqp_send(message, device.id, MQTT_CHAN)
end end
def self.process(delivery_info, body) def process(delivery_info, body)
params = step1(delivery_info, body) params = step1(delivery_info, body)
params && step2(params) params && step2(params)
end end

View File

@ -1,28 +0,0 @@
class ServiceRunner
WAIT_TIME = Rails.env.test? ? 0.01 : 5
OFFLINE_ERROR = Bunny::TCPConnectionFailedForAllHosts
CRASH_MSG = Rails.env.test? ?
"\e[32m.\e[0m" : "Something caused the broker to crash...\n"
def self.go!(channel, worker_klass)
self.new(channel, worker_klass).run!
end
def initialize(channel, worker_klass)
@channel = channel
@worker = worker_klass
end
def run!
@channel.subscribe(block: true) do |info, _, payl|
@worker.process(info, payl.force_encoding("UTF-8"))
end
rescue OFFLINE_ERROR, StandardError => e
unless e.is_a?(OFFLINE_ERROR)
Rollbar.error(e)
print CRASH_MSG
end
sleep WAIT_TIME
retry
end
end

View File

@ -3,12 +3,12 @@ require "bunny"
# A wrapper around AMQP to stay DRY. Will make life easier if we ever need to # A wrapper around AMQP to stay DRY. Will make life easier if we ever need to
# change protocols # change protocols
class Transport class Transport
OPTS = { read_timeout: 10, heartbeat: 10, log_level: "info" } OPTS = { read_timeout: 10, heartbeat: 10, log_level: "warn" }
RESOURCE_ROUTING_KEY = "bot.*.resources_v0.*.*.*.*" RESOURCE_ROUTING_KEY = "bot.*.resources_v0.*.*.*.*"
def self.amqp_url def self.amqp_url
@amqp_url ||= ENV['CLOUDAMQP_URL'] || @amqp_url ||= ENV["CLOUDAMQP_URL"] ||
ENV['RABBITMQ_URL'] || ENV["RABBITMQ_URL"] ||
"amqp://admin:#{ENV.fetch("ADMIN_PASSWORD")}@mqtt:5672" "amqp://admin:#{ENV.fetch("ADMIN_PASSWORD")}@mqtt:5672"
end end
@ -32,7 +32,7 @@ class Transport
def connection def connection
@connection ||= Transport @connection ||= Transport
.default_amqp_adapter.new(Transport.amqp_url, OPTS).start .default_amqp_adapter.new(Transport.amqp_url, OPTS).start
end end
def log_channel def log_channel
@ -44,10 +44,10 @@ class Transport
def resource_channel def resource_channel
@resource_channel ||= self @resource_channel ||= self
.connection .connection
.create_channel .create_channel
.queue("resource_workers") .queue("resource_workers")
.bind("amq.topic", routing_key: RESOURCE_ROUTING_KEY) .bind("amq.topic", routing_key: RESOURCE_ROUTING_KEY)
end end
# def ping_channel # def ping_channel
@ -59,9 +59,9 @@ class Transport
def amqp_topic def amqp_topic
@amqp_topic ||= self @amqp_topic ||= self
.connection .connection
.create_channel .create_channel
.topic("amq.topic", auto_delete: true) .topic("amq.topic", auto_delete: true)
end end
def amqp_send(message, id, channel) def amqp_send(message, id, channel)
@ -97,10 +97,10 @@ class Transport
end end
def self.api_url def self.api_url
uri = URI(Transport.amqp_url) uri = URI(Transport.amqp_url)
uri.scheme = ENV["FORCE_SSL"] ? "https" : "http" uri.scheme = ENV["FORCE_SSL"] ? "https" : "http"
uri.user = nil uri.user = nil
uri.port = 15672 uri.port = 15672
uri.to_s uri.to_s
end end

View File

@ -6,59 +6,32 @@ require_relative "../app/lib/resources.rb"
require_relative "../app/lib/resources/job.rb" require_relative "../app/lib/resources/job.rb"
require_relative "../app/lib/resources/preprocessor.rb" require_relative "../app/lib/resources/preprocessor.rb"
require_relative "../app/lib/resources/service.rb" require_relative "../app/lib/resources/service.rb"
require_relative "../app/lib/service_runner_base.rb"
require_relative "../app/lib/service_runner_base.rb"
class RabbitWorker class RabbitWorker
# You migiht need this to debug ping stuff RC: WAIT = 3
# class FakePing def self.thread
# def self.process(info, payl) Thread.new do
# puts "=====================================" yield
# arry = info.routing_key.split(".") rescue => e
# arry[2] = "pong" puts "Connecting to broker in #{WAIT} seconds. (#{e.inspect})"
# chan = arry.join(".") sleep WAIT
# puts chan retry
# Transport.current.raw_amqp_send(arry[3], chan) end
# puts "====================================="
# end
# end
WAIT = 3
SERVICES = {
log_channel: LogService,
resource_channel: Resources::Service,
# ping_channel: FakePing
}
def run_it!(chan, service)
puts " Attempting to connect #{service} to #{chan}"
ServiceRunner.go!(Transport.current.send(chan), service)
rescue
puts "Connecting to broker in #{WAIT} seconds."
sleep WAIT
retry
end
def thread(channel, service)
Thread.new { run_it!(channel, service) }
end
def threads
@threads ||= SERVICES.map { |(c,s)| thread(c, s) }
end end
def self.go! def self.go!
loop do loop do
ThreadsWait.all_waits(self.new.threads) ThreadsWait.all_waits([
thread { LogService.new.go!(Transport.current.log_channel) },
thread { Resources::Service.new.go!(Transport.current.resource_channel) },
])
end end
rescue
sleep RabbitWorker::WAIT
retry
end end
end end
sleep(RabbitWorker::WAIT * 2) sleep(RabbitWorker::WAIT * 2)
begin RabbitWorker.go!
RabbitWorker.go!
rescue
sleep RabbitWorker::WAIT
retry
end

View File

@ -1,15 +1,14 @@
require "spec_helper" require "spec_helper"
require "service_runner_base"
describe ServiceRunner do describe AbstractServiceRunner do
class ServiceRunnerStub class ServiceRunnerStub < AbstractServiceRunner
attr_accessor :subscribe_call_count, :process_calls attr_accessor :subscribe_call_count, :process_calls
MSG = RuntimeError.new("First attempt will fail, expect a retry") MSG = RuntimeError.new("First attempt will fail, expect a retry")
def initialize def initialize
@subscribe_call_count = 0 @subscribe_call_count = 0
@process_calls = [] @process_calls = []
end end
def subscribe(*) def subscribe(*)