Merge e3b7d6d571 into cdff5e9b1c

* Revert "Fix broken video playback by forcing fmp4"

This reverts commit 52db03b73a.

* Fix audio url in video m3u8

* Upgrade hls.js to 1.5.1 and use full version

nitter.example.conf

@@ -26,12 +26,14 @@ enableRSS = true  # set this to false to disable RSS feeds
 enableDebug = false  # enable request logs and debug endpoints (/.accounts)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
-tokenCount = 10
-# minimum amount of usable tokens. tokens are used to authorize API requests,
-# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
-# the limits reset every 15 minutes, and the pool is filled up so there's
-# always at least `tokenCount` usable tokens. only increase this if you receive
-# major bursts all the time and don't have a rate limiting setup via e.g. nginx
+
+# Instead of guest_accounts.json, fetch guest accounts from an external URL.
+# Nitter will re-fetch accounts if it runs out of valid ones.
+[GuestAccounts]
+usePool = false  # enable fetching accounts from external pool.
+poolUrl = ""     # https://example.com/download
+poolId = ""      # defaults to nitter's hostname
+poolAuth = ""    # random secret string for authentication
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]

src/auth.nim

@@ -1,5 +1,6 @@
 #SPDX-License-Identifier: AGPL-3.0-only
-import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
+import std/[httpclient, asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os, uri]
+import nimcrypto
 import types
 import experimental/parser/guestaccount
 
@@ -197,13 +198,55 @@ proc initAccountPool*(cfg: Config; path: string) =
   elif fileExists(path):
     log "Parsing JSON guest accounts file: ", path
     accountPool = parseGuestAccounts(path)
-  else:
-    echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests."
+  elif not cfg.guestAccountsUsePool:
+    echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests. Alternatively, configure the guest account pool in nitter.conf"
     quit 1
 
-  let accountsPrePurge = accountPool.len
   accountPool.keepItIf(not it.hasExpired)
 
   log "Successfully added ", accountPool.len, " valid accounts."
-  if accountsPrePurge > accountPool.len:
-    log "Purged ", accountsPrePurge - accountPool.len, " expired accounts."
+
+proc updateAccountPool*(cfg: Config) {.async.} =
+  if not cfg.guestAccountsUsePool:
+    return
+
+  # wait for a few seconds before fetching guest accounts, so that
+  # /.well-known/... is served correctly
+  await sleepAsync(10 * 1000)
+
+  while true:
+    if accountPool.len == 0:
+      log "fetching more accounts from service"
+
+      let client = newAsyncHttpClient("nitter-accounts")
+
+      try:
+        let resp = await client.get($(cfg.guestAccountsPoolUrl ? {"id": cfg.guestAccountsPoolId, "auth": cfg.guestAccountsPoolAuth}))
+        let guestAccounts = await resp.body
+
+        log "status code from service: ", resp.status
+
+        for line in guestAccounts.splitLines:
+          if line != "":
+            accountPool.add parseGuestAccount(line)
+
+      except Exception as e:
+        log "failed to fetch from accounts service: ", e.msg
+      finally:
+        client.close()
+
+      accountPool.keepItIf(not it.hasExpired)
+
+    await sleepAsync(3600 * 1000)
+
+proc getAuthHash*(cfg: Config): string =
+  if cfg.guestAccountsPoolAuth.len == 0:
+    # If somebody turns on pool auth and provides a dummy key, we should
+    # prevent third parties from using that mis-configured auth and impersonate
+    # this instance
+    log "poolAuth is empty, authentication with accounts service will fail"
+    return ""
+
+  let hashStr = $sha_256.digest(cfg.guestAccountsPoolAuth)
+
+  return hashStr.toLowerAscii

src/config.nim

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import parsecfg except Config
+import std/parsecfg except Config
+import std/uri
 import types, strutils
 
 proc get*[T](config: parseCfg.Config; section, key: string; default: T): T =
@@ -36,11 +37,16 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     # Config
     hmacKey: cfg.get("Config", "hmacKey", "secretkey"),
     base64Media: cfg.get("Config", "base64Media", false),
-    minTokens: cfg.get("Config", "tokenCount", 10),
     enableRss: cfg.get("Config", "enableRSS", true),
     enableDebug: cfg.get("Config", "enableDebug", false),
     proxy: cfg.get("Config", "proxy", ""),
-    proxyAuth: cfg.get("Config", "proxyAuth", "")
+    proxyAuth: cfg.get("Config", "proxyAuth", ""),
+
+    # GuestAccounts
+    guestAccountsUsePool: cfg.get("GuestAccounts", "usePool", false),
+    guestAccountsPoolUrl: parseUri(cfg.get("GuestAccounts", "poolUrl", "")),
+    guestAccountsPoolAuth: cfg.get("GuestAccounts", "poolAuth", ""),
+    guestAccountsPoolId: cfg.get("GuestAccounts", "poolId", cfg.get("Server", "hostname", ""))
   )
 
   return (conf, cfg)

src/formatters.nim

@@ -82,6 +82,8 @@ proc proxifyVideo*(manifest: string; proxy: bool): string =
   for line in manifest.splitLines:
     let url =
       if line.startsWith("#EXT-X-MAP:URI"): line[16 .. ^2]
+      elif line.startsWith("#EXT-X-MEDIA") and "URI=" in line:
+        line[line.find("URI=") + 5 .. -1 + line.find("\"", start= 5 + line.find("URI="))]
       else: line
     if url.startsWith('/'):
       let path = "https://video.twimg.com" & url

src/nitter.nim

@@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, rss, list, debug,
-  unsupported, embed, resolver, router_utils]
+  unsupported, embed, resolver, router_utils, auth]
 
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
@@ -22,6 +22,7 @@ let
   accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json")
 
 initAccountPool(cfg, accountsPath)
+asyncCheck updateAccountPool(cfg)
 
 if not cfg.enableDebug:
   # Silence Jester's query warning
@@ -54,6 +55,7 @@ createMediaRouter(cfg)
 createEmbedRouter(cfg)
 createRssRouter(cfg)
 createDebugRouter(cfg)
+createAuthRouter(cfg)
 
 settings:
   port = Port(cfg.port)
@@ -108,3 +110,4 @@ routes:
   extend embed, ""
   extend debug, ""
   extend unsupported, ""
+  extend auth, ""

src/routes/auth.nim

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+
+import jester
+
+import router_utils
+import ".."/[types, auth]
+
+proc createAuthRouter*(cfg: Config) =
+  router auth:
+    get "/.well-known/nitter-auth":
+      cond cfg.guestAccountsUsePool
+      resp Http200, {"content-type": "text/plain"}, getAuthHash(cfg)

src/types.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import times, sequtils, options, tables
+import std/[times, sequtils, options, tables, uri]
 import prefs_impl
 
 genPrefsType()
@@ -262,12 +262,16 @@ type
 
     hmacKey*: string
     base64Media*: bool
-    minTokens*: int
     enableRss*: bool
     enableDebug*: bool
     proxy*: string
     proxyAuth*: string
 
+    guestAccountsUsePool*: bool
+    guestAccountsPoolUrl*: Uri
+    guestAccountsPoolId*: string
+    guestAccountsPoolAuth*: string
+
     rssCacheTime*: int
     listCacheTime*: int
 

src/utils.nim

@@ -31,9 +31,7 @@ proc getHmac*(data: string): string =
 
 proc getVidUrl*(link: string): string =
   if link.len == 0: return
-  let 
-    link = link.replace("cmaf", "fmp4")
-    sig = getHmac(link)
+  let sig = getHmac(link)
   if base64Media:
     &"/video/enc/{sig}/{encode(link, safe=true)}"
   else:

src/views/general.nim

@@ -73,7 +73,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
       link(rel="alternate", type="application/rss+xml", href=rss, title="RSS feed")
 
     if prefs.hlsPlayback:
-      script(src="/js/hls.light.min.js", `defer`="")
+      script(src="/js/hls.min.js", `defer`="")
       script(src="/js/hlsPlayback.js", `defer`="")
 
     if prefs.infiniteScroll: