rm -rf modules/monitor, send data to kamon
This commit is contained in:
parent
a9b8e6817f
commit
fe7151e3f8
|
@ -127,7 +127,6 @@ object Env {
|
|||
def ai = lila.ai.Env.current
|
||||
def analyse = lila.analyse.Env.current
|
||||
def mod = lila.mod.Env.current
|
||||
def monitor = lila.monitor.Env.current
|
||||
def site = lila.site.Env.current
|
||||
def round = lila.round.Env.current
|
||||
def lobby = lila.lobby.Env.current
|
||||
|
|
|
@ -18,10 +18,7 @@ object Global extends GlobalSettings {
|
|||
else if (Env.ai.ServerOnly) {
|
||||
Action(NotFound("I am an AI server")).some
|
||||
}
|
||||
else {
|
||||
Env.monitor.reporting ! AddRequest
|
||||
Env.i18n.requestHandler(req) orElse super.onRouteRequest(req)
|
||||
}
|
||||
else Env.i18n.requestHandler(req) orElse super.onRouteRequest(req)
|
||||
|
||||
private def niceError(req: RequestHeader): Boolean =
|
||||
req.method == "GET" &&
|
||||
|
|
|
@ -1,57 +0,0 @@
|
|||
package controllers
|
||||
|
||||
import akka.pattern.ask
|
||||
import play.api.libs.json._
|
||||
import play.api.mvc._
|
||||
|
||||
import lila.app._
|
||||
import lila.monitor.actorApi._
|
||||
import lila.socket.actorApi.PopulationGet
|
||||
import makeTimeout.short
|
||||
|
||||
object Monitor extends LilaController {
|
||||
|
||||
private def env = Env.monitor
|
||||
|
||||
def index = Secure(_.Admin) { ctx =>
|
||||
me =>
|
||||
Ok(views.html.monitor.monitor()).fuccess
|
||||
}
|
||||
|
||||
def websocket = SocketOption[JsValue] { implicit ctx =>
|
||||
get("sri") ?? { sri =>
|
||||
env.socketHandler(sri) map some
|
||||
}
|
||||
}
|
||||
|
||||
def statusParam = Action.async { implicit req =>
|
||||
handleStatus(~get("key", req))
|
||||
}
|
||||
|
||||
def status(key: String) = Action.async { implicit req =>
|
||||
handleStatus(key)
|
||||
}
|
||||
|
||||
private def handleStatus(key: String) = key match {
|
||||
case "threads" => Ok(java.lang.management.ManagementFactory.getThreadMXBean.getThreadCount).fuccess
|
||||
case "moves" => (env.reporting ? GetNbMoves).mapTo[Int] map { Ok(_) }
|
||||
case "moveLatency" => (env.reporting ? GetMoveLatency).mapTo[Int] map { Ok(_) }
|
||||
case "players" => {
|
||||
(env.reporting ? PopulationGet).mapTo[Int] map { "%d %d".format(_, Env.user.onlineUserIdMemo.count) }
|
||||
} map { Ok(_) }
|
||||
case "uptime" => fuccess {
|
||||
val up = lila.common.PlayApp.uptime
|
||||
Ok {
|
||||
s"""${prop("java.vm.name")} ${prop("java.vendor")} ${prop("java.version")}
|
||||
${prop("user.name")} @ ${prop("os.arch")} ${prop("os.name")} ${prop("os.version")}
|
||||
uptime: ${org.joda.time.format.PeriodFormat.wordBased(new java.util.Locale("en")).print(up)}
|
||||
uptime seconds: ${up.toStandardSeconds.getSeconds}
|
||||
last deploy: ${lila.common.PlayApp.startedAt}"""
|
||||
}
|
||||
}
|
||||
case "locale" => Ok(java.util.Locale.getDefault.toString).fuccess
|
||||
case key => BadRequest(s"Unknown monitor status key: $key").fuccess
|
||||
}
|
||||
|
||||
private def prop(name: String) = System.getProperty(name)
|
||||
}
|
|
@ -333,12 +333,6 @@ GET /import/master/$id<\w{8}>/:color controllers.Importer.masterGame(id: Strin
|
|||
GET /editor/*urlFen controllers.Editor.load(urlFen: String)
|
||||
GET /editor controllers.Editor.index
|
||||
|
||||
# Monitor
|
||||
GET /monitor controllers.Monitor.index
|
||||
GET /monitor/socket controllers.Monitor.websocket
|
||||
GET /monitor/status/:key controllers.Monitor.status(key: String)
|
||||
GET /monitor/status controllers.Monitor.statusParam
|
||||
|
||||
# Report
|
||||
GET /report controllers.Report.form
|
||||
POST /report controllers.Report.create
|
||||
|
|
|
@ -11,8 +11,8 @@ final class Env(
|
|||
config: Config,
|
||||
db: lila.db.Env,
|
||||
renderer: ActorSelection,
|
||||
router: ActorSelection,
|
||||
system: ActorSystem,
|
||||
scheduler: lila.common.Scheduler,
|
||||
roundJsonView: lila.round.JsonView,
|
||||
noteApi: lila.round.NoteApi,
|
||||
forecastApi: lila.round.ForecastApi,
|
||||
|
@ -117,6 +117,8 @@ final class Env(
|
|||
private def makeUrl(path: String): String = s"${Net.BaseUrl}/$path"
|
||||
|
||||
lazy val cli = new Cli(system.lilaBus, renderer)
|
||||
|
||||
system.actorOf(Props(new KamonPusher))
|
||||
}
|
||||
|
||||
object Env {
|
||||
|
@ -125,7 +127,6 @@ object Env {
|
|||
config = lila.common.PlayApp.loadConfig,
|
||||
db = lila.db.Env.current,
|
||||
renderer = lila.hub.Env.current.actor.renderer,
|
||||
router = lila.hub.Env.current.actor.router,
|
||||
userEnv = lila.user.Env.current,
|
||||
analyseEnv = lila.analyse.Env.current,
|
||||
lobbyEnv = lila.lobby.Env.current,
|
||||
|
@ -143,5 +144,6 @@ object Env {
|
|||
prefApi = lila.pref.Env.current.api,
|
||||
gamePgnDump = lila.game.Env.current.pgnDump,
|
||||
system = lila.common.PlayApp.system,
|
||||
scheduler = lila.common.PlayApp.scheduler,
|
||||
isProd = lila.common.PlayApp.isProd)
|
||||
}
|
||||
|
|
32
modules/api/src/main/KamonPusher.scala
Normal file
32
modules/api/src/main/KamonPusher.scala
Normal file
|
@ -0,0 +1,32 @@
|
|||
package lila.api
|
||||
|
||||
import akka.actor._
|
||||
import java.lang.management.ManagementFactory
|
||||
import kamon.Kamon.metrics
|
||||
import scala.concurrent.duration._
|
||||
|
||||
import lila.socket.actorApi.NbMembers
|
||||
|
||||
private final class KamonPusher extends Actor {
|
||||
|
||||
override def preStart() {
|
||||
context.system.lilaBus.subscribe(self, 'nbMembers)
|
||||
context.system.scheduler.schedule(1 second, 1 second, self, Tick)
|
||||
}
|
||||
|
||||
private case object Tick
|
||||
|
||||
private val threadStats = ManagementFactory.getThreadMXBean
|
||||
private val app = lila.common.PlayApp
|
||||
|
||||
def receive = {
|
||||
|
||||
case NbMembers(nb) =>
|
||||
metrics.histogram("socket.member") record nb
|
||||
|
||||
case Tick =>
|
||||
metrics.histogram("jvm.thread") record threadStats.getThreadCount
|
||||
metrics.histogram("jvm.daemon") record threadStats.getDaemonThreadCount
|
||||
metrics.histogram("jvm.uptime") record app.uptime.toStandardSeconds.getSeconds
|
||||
}
|
||||
}
|
|
@ -1,42 +0,0 @@
|
|||
package lila.monitor;
|
||||
|
||||
import com.sun.management.OperatingSystemMXBean;
|
||||
import java.lang.management.*;
|
||||
|
||||
public class CPU {
|
||||
|
||||
private int availableProcessors = ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors();
|
||||
|
||||
private long lastSystemTime = 0;
|
||||
private long lastProcessCpuTime = 0;
|
||||
|
||||
public synchronized double getCpuUsage() {
|
||||
if(lastSystemTime == 0) {
|
||||
baselineCounters();
|
||||
return 0;
|
||||
}
|
||||
|
||||
long systemTime = System.nanoTime();
|
||||
long processCpuTime = 0;
|
||||
|
||||
if(ManagementFactory.getOperatingSystemMXBean() instanceof OperatingSystemMXBean) {
|
||||
processCpuTime = ((OperatingSystemMXBean)ManagementFactory.getOperatingSystemMXBean()).getProcessCpuTime();
|
||||
}
|
||||
|
||||
double cpuUsage = (double) (processCpuTime - lastProcessCpuTime) / (systemTime - lastSystemTime);
|
||||
|
||||
lastSystemTime = systemTime;
|
||||
lastProcessCpuTime = processCpuTime;
|
||||
|
||||
return cpuUsage / availableProcessors;
|
||||
}
|
||||
|
||||
private void baselineCounters() {
|
||||
lastSystemTime = System.nanoTime();
|
||||
|
||||
if (ManagementFactory.getOperatingSystemMXBean() instanceof OperatingSystemMXBean) {
|
||||
lastProcessCpuTime = ( (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean() ).getProcessCpuTime();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -1,60 +0,0 @@
|
|||
package lila.monitor
|
||||
|
||||
import akka.actor._
|
||||
import com.typesafe.config.Config
|
||||
import play.api.libs.json.Json
|
||||
import scala.concurrent.duration._
|
||||
|
||||
import lila.common.PimpedConfig._
|
||||
import lila.common.WindowCount
|
||||
import lila.socket.Channel
|
||||
|
||||
final class Env(
|
||||
config: Config,
|
||||
db: lila.db.Env,
|
||||
hub: lila.hub.Env,
|
||||
system: ActorSystem,
|
||||
scheduler: lila.common.Scheduler) {
|
||||
|
||||
private val ActorName = config getString "actor.name"
|
||||
private val SocketName = config getString "socket.name"
|
||||
private val SocketUidTtl = config duration "socket.uid.ttl"
|
||||
|
||||
private lazy val socket = system.actorOf(
|
||||
Props(new Socket(timeout = SocketUidTtl)), name = SocketName)
|
||||
|
||||
lazy val socketHandler = new SocketHandler(socket, hub)
|
||||
|
||||
val reporting = system.actorOf(
|
||||
Props(new Reporting(
|
||||
reqWindowCount = reqWindowCount,
|
||||
moveWindowCount = moveWindowCount,
|
||||
socket = socket,
|
||||
db = db,
|
||||
hub = hub
|
||||
)), name = ActorName)
|
||||
|
||||
{
|
||||
import scala.concurrent.duration._
|
||||
|
||||
scheduler.message(1 seconds) {
|
||||
reporting -> lila.hub.actorApi.monitor.Update
|
||||
}
|
||||
}
|
||||
|
||||
// requests per second
|
||||
private lazy val reqWindowCount = new WindowCount(1 second)
|
||||
|
||||
// moves per second
|
||||
private lazy val moveWindowCount = new WindowCount(1 second)
|
||||
}
|
||||
|
||||
object Env {
|
||||
|
||||
lazy val current = "monitor" boot new Env(
|
||||
config = lila.common.PlayApp loadConfig "monitor",
|
||||
db = lila.db.Env.current,
|
||||
hub = lila.hub.Env.current,
|
||||
system = lila.common.PlayApp.system,
|
||||
scheduler = lila.common.PlayApp.scheduler)
|
||||
}
|
|
@ -1,31 +0,0 @@
|
|||
package lila.monitor
|
||||
|
||||
import play.api.libs.json.JsObject
|
||||
import play.modules.reactivemongo.json.ImplicitBSONHandlers._
|
||||
import reactivemongo.api.DB
|
||||
import reactivemongo.bson._
|
||||
import reactivemongo.core.commands.Status
|
||||
|
||||
import lila.common.PimpedJson._
|
||||
|
||||
private[monitor] case class MongoStatus(
|
||||
memory: Int = 0,
|
||||
connection: Int = 0,
|
||||
query: Long = 0,
|
||||
qps: Int = 0)
|
||||
|
||||
private[monitor] object MongoStatus {
|
||||
|
||||
val default = new MongoStatus()
|
||||
|
||||
def apply(db: DB)(prev: MongoStatus): Fu[MongoStatus] =
|
||||
db.command(Status) map { bsonMap =>
|
||||
val status = JsObjectReader read BSONDocument(bsonMap)
|
||||
val query = ~(status obj "network" flatMap (_ long "numRequests"))
|
||||
new MongoStatus(
|
||||
memory = ~(status \ "mem" \ "resident").asOpt[Int],
|
||||
connection = ~(status \ "connections" \ "current").asOpt[Int],
|
||||
query = query,
|
||||
qps = (query - prev.query).toInt)
|
||||
}
|
||||
}
|
|
@ -1,110 +0,0 @@
|
|||
package lila.monitor
|
||||
|
||||
import java.lang.management.ManagementFactory
|
||||
import scala.concurrent.duration._
|
||||
import scala.util.{ Success, Failure }
|
||||
|
||||
import akka.actor._
|
||||
import akka.pattern.{ ask, pipe }
|
||||
import play.api.libs.concurrent._
|
||||
|
||||
import actorApi._
|
||||
import lila.hub.actorApi.monitor._
|
||||
import lila.socket.actorApi.{ NbMembers, PopulationGet }
|
||||
|
||||
private[monitor] final class Reporting(
|
||||
reqWindowCount: lila.common.WindowCount,
|
||||
moveWindowCount: lila.common.WindowCount,
|
||||
socket: ActorRef,
|
||||
db: lila.db.Env,
|
||||
hub: lila.hub.Env) extends Actor {
|
||||
|
||||
private val bus = context.system.lilaBus
|
||||
|
||||
bus.subscribe(self, 'nbMembers)
|
||||
|
||||
override def postStop() {
|
||||
bus.unsubscribe(self)
|
||||
}
|
||||
|
||||
var nbMembers = 0
|
||||
var nbPlaying = 0
|
||||
var loadAvg = 0f
|
||||
var nbThreads = 0
|
||||
var memory = 0l
|
||||
var latency = 0
|
||||
var rps = 0
|
||||
var mps = 0
|
||||
var cpu = 0
|
||||
var mongoStatus = MongoStatus.default
|
||||
val moveMillis = scala.collection.mutable.ArrayBuffer.empty[Int]
|
||||
var moveAvgMillis = 0
|
||||
|
||||
var idle = true
|
||||
var lastUpdated = nowMillis
|
||||
|
||||
val osStats = ManagementFactory.getOperatingSystemMXBean
|
||||
val threadStats = ManagementFactory.getThreadMXBean
|
||||
val memoryStats = ManagementFactory.getMemoryMXBean
|
||||
val cpuStats = new CPU
|
||||
implicit val timeout = makeTimeout(100 millis)
|
||||
|
||||
def receive = {
|
||||
|
||||
case Move(millis) =>
|
||||
moveWindowCount.add
|
||||
millis foreach { m =>
|
||||
moveMillis += m
|
||||
}
|
||||
|
||||
case AddRequest => reqWindowCount.add
|
||||
|
||||
case PopulationGet => sender ! nbMembers
|
||||
|
||||
case NbMembers(nb) => nbMembers = nb
|
||||
|
||||
case GetNbMoves => sender ! moveWindowCount.get
|
||||
case GetMoveLatency => sender ! moveAvgMillis
|
||||
|
||||
case Update =>
|
||||
if (moveMillis.size > 0) moveAvgMillis = moveMillis.sum / moveMillis.size
|
||||
moveMillis.clear
|
||||
socket ? PopulationGet foreach {
|
||||
case 0 => idle = true
|
||||
case _ => {
|
||||
val before = nowMillis
|
||||
MongoStatus(db.db)(mongoStatus) onComplete {
|
||||
case Failure(e) => logwarn("[reporting] " + e.getMessage)
|
||||
case Success(mongoS) => {
|
||||
latency = (nowMillis - before).toInt
|
||||
mongoStatus = mongoS
|
||||
loadAvg = osStats.getSystemLoadAverage.toFloat
|
||||
nbThreads = threadStats.getThreadCount
|
||||
memory = memoryStats.getHeapMemoryUsage.getUsed / 1024 / 1024
|
||||
mps = moveWindowCount.get
|
||||
rps = reqWindowCount.get
|
||||
cpu = ((cpuStats.getCpuUsage() * 1000).round / 10.0).toInt
|
||||
socket ! MonitorData(monitorData(idle))
|
||||
idle = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private def monitorData(idle: Boolean) = List(
|
||||
"users" -> nbMembers,
|
||||
"lat" -> latency,
|
||||
"thread" -> nbThreads,
|
||||
"cpu" -> cpu,
|
||||
"load" -> loadAvg,
|
||||
"memory" -> memory,
|
||||
"rps" -> rps,
|
||||
"mps" -> mps,
|
||||
"mlat" -> moveAvgMillis,
|
||||
"dbMemory" -> mongoStatus.memory,
|
||||
"dbConn" -> mongoStatus.connection,
|
||||
"dbQps" -> idle.fold("??", mongoStatus.qps.toString)) map {
|
||||
case (name, value) => s"$value:$name"
|
||||
}
|
||||
}
|
|
@ -1,33 +0,0 @@
|
|||
package lila.monitor
|
||||
|
||||
import scala.concurrent.duration.Duration
|
||||
|
||||
import akka.actor._
|
||||
import play.api.libs.iteratee._
|
||||
import play.api.libs.json._
|
||||
|
||||
import actorApi._
|
||||
import lila.socket._
|
||||
import lila.socket.actorApi.{ PopulationGet, Connected, Broom }
|
||||
|
||||
private[monitor] final class Socket(timeout: Duration) extends SocketActor[Member](timeout) {
|
||||
|
||||
override val startsOnApplicationBoot = true
|
||||
|
||||
def receiveSpecific = {
|
||||
|
||||
// don't eject members - they don't ping the monitor socket
|
||||
case Broom =>
|
||||
|
||||
case PopulationGet => sender ! members.size
|
||||
|
||||
case Join(uid) => {
|
||||
val (enumerator, channel) = Concurrent.broadcast[JsValue]
|
||||
val member = Member(channel)
|
||||
addMember(uid, member)
|
||||
sender ! Connected(enumerator, member)
|
||||
}
|
||||
|
||||
case MonitorData(data) => notifyAll("monitor", data mkString ";")
|
||||
}
|
||||
}
|
|
@ -1,23 +0,0 @@
|
|||
package lila.monitor
|
||||
|
||||
import akka.actor._
|
||||
|
||||
import actorApi._
|
||||
import lila.socket.actorApi.Connected
|
||||
import lila.socket.Handler
|
||||
|
||||
private[monitor] final class SocketHandler(
|
||||
socket: ActorRef,
|
||||
hub: lila.hub.Env) {
|
||||
|
||||
def apply(uid: String): Fu[JsSocketHandler] = {
|
||||
|
||||
def controller: Handler.Controller = {
|
||||
case _ =>
|
||||
}
|
||||
|
||||
Handler(hub, socket, uid, Join(uid), none) {
|
||||
case Connected(enum, member) => (controller, enum, member)
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,15 +0,0 @@
|
|||
package lila.monitor
|
||||
package actorApi
|
||||
|
||||
import lila.socket.SocketMember
|
||||
|
||||
case class Member(channel: JsChannel) extends SocketMember {
|
||||
val userId = none
|
||||
val troll = false
|
||||
}
|
||||
|
||||
case object GetNbMoves
|
||||
case object GetMoveLatency
|
||||
|
||||
case class Join(uid: String)
|
||||
case class MonitorData(data: List[String])
|
|
@ -1,5 +0,0 @@
|
|||
package lila
|
||||
|
||||
import lila.socket.WithSocket
|
||||
|
||||
package object monitor extends PackageObject with WithPlay with WithSocket
|
|
@ -8,7 +8,6 @@ private[socket] final class Population extends Actor {
|
|||
|
||||
var nb = 0
|
||||
val bus = context.system.lilaBus
|
||||
val histogram = kamon.Kamon.metrics.histogram("socket.members")
|
||||
|
||||
bus.subscribe(self, 'socketDoor)
|
||||
|
||||
|
@ -21,8 +20,6 @@ private[socket] final class Population extends Actor {
|
|||
case _: SocketEnter[_] => nb = nb + 1
|
||||
case _: SocketLeave[_] => nb = nb - 1
|
||||
|
||||
case PopulationTell =>
|
||||
bus.publish(NbMembers(nb), 'nbMembers)
|
||||
histogram record nb
|
||||
case PopulationTell => bus.publish(NbMembers(nb), 'nbMembers)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -22,7 +22,6 @@ case object GetVersion
|
|||
|
||||
case class SendToFlag(flag: String, message: JsObject)
|
||||
|
||||
case object PopulationGet
|
||||
case object PopulationTell
|
||||
case class NbMembers(nb: Int)
|
||||
|
||||
|
|
|
@ -56,7 +56,7 @@ object ApplicationBuild extends Build {
|
|||
chess, common, db, rating, user, security, wiki, hub, socket,
|
||||
message, notification, i18n, game, bookmark, search,
|
||||
gameSearch, timeline, forum, forumSearch, team, teamSearch,
|
||||
ai, analyse, mod, monitor, site, round, lobby, setup,
|
||||
ai, analyse, mod, site, round, lobby, setup,
|
||||
importer, tournament, simul, relation, report, pref, // simulation,
|
||||
evaluation, chat, puzzle, tv, coordinate, blog, donation, qa,
|
||||
history, worldMap, opening, video, shutup, push,
|
||||
|
@ -68,7 +68,7 @@ object ApplicationBuild extends Build {
|
|||
lazy val api = project("api", moduleCPDeps)
|
||||
.settings(
|
||||
libraryDependencies ++= provided(
|
||||
play.api, hasher, config, apache, jgit, findbugs, RM)
|
||||
play.api, hasher, config, apache, jgit, findbugs, RM, kamon.core)
|
||||
) aggregate (moduleRefs: _*)
|
||||
|
||||
lazy val puzzle = project("puzzle", Seq(
|
||||
|
@ -309,16 +309,12 @@ object ApplicationBuild extends Build {
|
|||
libraryDependencies ++= provided(play.api)
|
||||
)
|
||||
|
||||
lazy val monitor = project("monitor", Seq(common, hub, socket, db)).settings(
|
||||
libraryDependencies ++= provided(play.api, RM, PRM)
|
||||
)
|
||||
|
||||
lazy val site = project("site", Seq(common, socket)).settings(
|
||||
libraryDependencies ++= provided(play.api)
|
||||
)
|
||||
|
||||
lazy val socket = project("socket", Seq(common, hub, memo)).settings(
|
||||
libraryDependencies ++= provided(play.api, kamon.core)
|
||||
libraryDependencies ++= provided(play.api)
|
||||
)
|
||||
|
||||
lazy val hub = project("hub", Seq(common, chess)).settings(
|
||||
|
|
Loading…
Reference in a new issue