Add functionality to fetch guest accounts on demand

nitter.example.conf

@@ -33,6 +33,12 @@ tokenCount = 10
 # always at least `tokenCount` usable tokens. only increase this if you receive
 # major bursts all the time and don't have a rate limiting setup via e.g. nginx
 
+# Instead of guest_accounts.json, fetch guest accounts from an external URL.
+# Nitter will re-fetch accounts if it runs out of valid ones.
+guestAccountsUrl = ""    # https://example.com/download
+guestAccountsHost = ""   # defaults to nitter's hostname
+guestAccountsKey = ""    # a random string that will be used as a secret to authenticate against the URL
+
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]
 theme = "Nitter"

src/auth.nim

@@ -1,6 +1,7 @@
 #SPDX-License-Identifier: AGPL-3.0-only
-import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
-import types
+import std/[httpclient, asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
+import nimcrypto
+import types, http_pool
 import experimental/parser/guestaccount
 
 # max requests at a time per account to avoid race conditions
@@ -24,6 +25,11 @@ const
   }.toTable
 
 var
+  pool: HttpPool
+  guestAccountsUrl = ""
+  guestAccountsHost = ""
+  guestAccountsKey = ""
+  guestAccountsUrlLastFetched = 0
   accountPool: seq[GuestAccount]
   enableLogging = false
 
@@ -158,6 +164,27 @@ proc release*(account: GuestAccount) =
   dec account.pending
 
 proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} =
+  let now = epochTime().int
+
+  if accountPool.len == 0 and guestAccountsUrl != "" and guestAccountsUrlLastFetched < now - 3600:
+    once:
+      pool = HttpPool()
+
+    guestAccountsUrlLastFetched = now
+
+    log "fetching more accounts from service"
+    pool.use(newHttpHeaders()):
+      let resp = await c.get("$1?host=$2&key=$3" % [guestAccountsUrl, guestAccountsHost, guestAccountsKey])
+      let guestAccounts = await resp.body
+
+      log "status code from service: ", resp.status
+
+      for line in guestAccounts.splitLines:
+        if line != "":
+          accountPool.add parseGuestAccount(line)
+
+      accountPool.keepItIf(not it.hasExpired)
+
   for i in 0 ..< accountPool.len:
     if result.isReady(api): break
     result = accountPool.sample()
@@ -187,6 +214,9 @@ proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) =
 
 proc initAccountPool*(cfg: Config; path: string) =
   enableLogging = cfg.enableDebug
+  guestAccountsUrl = cfg.guestAccountsUrl
+  guestAccountsHost = cfg.guestAccountsHost
+  guestAccountsKey = cfg.guestAccountsKey
 
   let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path
 
@@ -197,7 +227,7 @@ proc initAccountPool*(cfg: Config; path: string) =
   elif fileExists(path):
     log "Parsing JSON guest accounts file: ", path
     accountPool = parseGuestAccounts(path)
-  else:
+  elif guestAccountsUrl == "":
     echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests."
     quit 1
 
@@ -205,5 +235,12 @@ proc initAccountPool*(cfg: Config; path: string) =
   accountPool.keepItIf(not it.hasExpired)
 
   log "Successfully added ", accountPool.len, " valid accounts."
-  if accountsPrePurge > accountPool.len:
-    log "Purged ", accountsPrePurge - accountPool.len, " expired accounts."
+
+proc getAuthHash*(cfg: Config): string =
+  if cfg.guestAccountsKey == "":
+    log "guestAccountsKey is set to bogus value, responding with empty string"
+    return ""
+
+  let hashStr = $sha_256.digest(cfg.guestAccountsKey)
+
+  return hashStr.toLowerAscii

src/config.nim

@@ -40,7 +40,10 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     enableRss: cfg.get("Config", "enableRSS", true),
     enableDebug: cfg.get("Config", "enableDebug", false),
     proxy: cfg.get("Config", "proxy", ""),
-    proxyAuth: cfg.get("Config", "proxyAuth", "")
+    proxyAuth: cfg.get("Config", "proxyAuth", ""),
+    guestAccountsUrl: cfg.get("Config", "guestAccountsUrl", ""),
+    guestAccountsKey: cfg.get("Config", "guestAccountsKey", ""),
+    guestAccountsHost: cfg.get("Config", "guestAccountsHost", cfg.get("Server", "hostname", ""))
   )
 
   return (conf, cfg)

src/nitter.nim

@@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, rss, list, debug,
-  unsupported, embed, resolver, router_utils]
+  unsupported, embed, resolver, router_utils, auth]
 
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
@@ -54,6 +54,7 @@ createMediaRouter(cfg)
 createEmbedRouter(cfg)
 createRssRouter(cfg)
 createDebugRouter(cfg)
+createAuthRouter(cfg)
 
 settings:
   port = Port(cfg.port)
@@ -108,3 +109,4 @@ routes:
   extend embed, ""
   extend debug, ""
   extend unsupported, ""
+  extend auth, ""

src/routes/auth.nim

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+
+import jester
+
+import router_utils
+import ".."/[types, auth]
+
+proc createAuthRouter*(cfg: Config) =
+  router auth:
+    get "/.well-known/nitter-auth-sha256":
+      cond cfg.guestAccountsUrl != ""
+      resp Http200, {"content-type": "text/plain"}, getAuthHash(cfg)

src/types.nim

@@ -254,6 +254,9 @@ type
     title*: string
     hostname*: string
     staticDir*: string
+    guestAccountsUrl*: string
+    guestAccountsKey*: string
+    guestAccountsHost*: string
 
     hmacKey*: string
     base64Media*: bool