configurable crawlable attribute

app/controllers/Main.scala

@@ -127,9 +127,9 @@ object Main extends LilaController {
     }
   }
 
-  def robots = Action { req =>
+  val robots = Action { _ =>
     Ok {
-      if (req.domain endsWith ".lichess.org")
+      if (Env.api.Net.Crawlable)
         "User-agent: *\nAllow: /\nDisallow: /game/export"
       else
         "User-agent: *\nDisallow: /"

conf/base.conf

@@ -13,6 +13,7 @@ net {
     version = 1126
   }
   email = "contact@lichess.org"
+  crawlable = false
 }
 forcedev = false
 play {

modules/api/src/main/Env.scala

@@ -44,6 +44,7 @@ final class Env(
     val AssetDomain = config getString "net.asset.domain"
     val AssetVersion = config getInt "net.asset.version"
     val Email = config getString "net.email"
+    val Crawlable = config getBoolean "net.crawlable"
   }
   val PrismicApiUrl = config getString "prismic.api_url"
   val EditorAnimationDuration = config duration "editor.animation.duration"