diff --git a/nitter.example.conf b/nitter.example.conf index f0b4214..0069e3b 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -26,12 +26,14 @@ enableRSS = true # set this to false to disable RSS feeds enableDebug = false # enable request logs and debug endpoints (/.accounts) proxy = "" # http/https url, SOCKS proxies are not supported proxyAuth = "" -tokenCount = 10 -# minimum amount of usable tokens. tokens are used to authorize API requests, -# but they expire after ~1 hour, and have a limit of 500 requests per endpoint. -# the limits reset every 15 minutes, and the pool is filled up so there's -# always at least `tokenCount` usable tokens. only increase this if you receive -# major bursts all the time and don't have a rate limiting setup via e.g. nginx + +# Instead of guest_accounts.json, fetch guest accounts from an external URL. +# Nitter will re-fetch accounts if it runs out of valid ones. +[GuestAccounts] +usePool = false # enable fetching accounts from external pool. +poolUrl = "" # https://example.com/download +poolId = "" # defaults to nitter's hostname +poolAuth = "" # random secret string for authentication # Change default preferences here, see src/prefs_impl.nim for a complete list [Preferences] diff --git a/src/auth.nim b/src/auth.nim index b288c50..6d4b0b3 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -1,5 +1,6 @@ #SPDX-License-Identifier: AGPL-3.0-only -import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os] +import std/[httpclient, asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os, uri] +import nimcrypto import types import experimental/parser/guestaccount @@ -197,13 +198,55 @@ proc initAccountPool*(cfg: Config; path: string) = elif fileExists(path): log "Parsing JSON guest accounts file: ", path accountPool = parseGuestAccounts(path) - else: - echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests." + elif not cfg.guestAccountsUsePool: + echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests. Alternatively, configure the guest account pool in nitter.conf" quit 1 - let accountsPrePurge = accountPool.len accountPool.keepItIf(not it.hasExpired) log "Successfully added ", accountPool.len, " valid accounts." - if accountsPrePurge > accountPool.len: - log "Purged ", accountsPrePurge - accountPool.len, " expired accounts." + +proc updateAccountPool*(cfg: Config) {.async.} = + if not cfg.guestAccountsUsePool: + return + + # wait for a few seconds before fetching guest accounts, so that + # /.well-known/... is served correctly + await sleepAsync(10 * 1000) + + while true: + if accountPool.len == 0: + log "fetching more accounts from service" + + let client = newAsyncHttpClient("nitter-accounts") + + try: + let resp = await client.get($(cfg.guestAccountsPoolUrl ? {"id": cfg.guestAccountsPoolId, "auth": cfg.guestAccountsPoolAuth})) + let guestAccounts = await resp.body + + log "status code from service: ", resp.status + + for line in guestAccounts.splitLines: + if line != "": + accountPool.add parseGuestAccount(line) + + except Exception as e: + log "failed to fetch from accounts service: ", e.msg + finally: + client.close() + + accountPool.keepItIf(not it.hasExpired) + + await sleepAsync(3600 * 1000) + +proc getAuthHash*(cfg: Config): string = + if cfg.guestAccountsPoolAuth.len == 0: + # If somebody turns on pool auth and provides a dummy key, we should + # prevent third parties from using that mis-configured auth and impersonate + # this instance + log "poolAuth is empty, authentication with accounts service will fail" + return "" + + let hashStr = $sha_256.digest(cfg.guestAccountsPoolAuth) + + return hashStr.toLowerAscii diff --git a/src/config.nim b/src/config.nim index 1b05ffe..d7917a4 100644 --- a/src/config.nim +++ b/src/config.nim @@ -1,5 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-only -import parsecfg except Config +import std/parsecfg except Config +import std/uri import types, strutils proc get*[T](config: parseCfg.Config; section, key: string; default: T): T = @@ -36,11 +37,16 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = # Config hmacKey: cfg.get("Config", "hmacKey", "secretkey"), base64Media: cfg.get("Config", "base64Media", false), - minTokens: cfg.get("Config", "tokenCount", 10), enableRss: cfg.get("Config", "enableRSS", true), enableDebug: cfg.get("Config", "enableDebug", false), proxy: cfg.get("Config", "proxy", ""), - proxyAuth: cfg.get("Config", "proxyAuth", "") + proxyAuth: cfg.get("Config", "proxyAuth", ""), + + # GuestAccounts + guestAccountsUsePool: cfg.get("GuestAccounts", "usePool", false), + guestAccountsPoolUrl: parseUri(cfg.get("GuestAccounts", "poolUrl", "")), + guestAccountsPoolAuth: cfg.get("GuestAccounts", "poolAuth", ""), + guestAccountsPoolId: cfg.get("GuestAccounts", "poolId", cfg.get("Server", "hostname", "")) ) return (conf, cfg) diff --git a/src/nitter.nim b/src/nitter.nim index dfc1dfd..9f05b2a 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, - unsupported, embed, resolver, router_utils] + unsupported, embed, resolver, router_utils, auth] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" @@ -22,6 +22,7 @@ let accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") initAccountPool(cfg, accountsPath) +asyncCheck updateAccountPool(cfg) if not cfg.enableDebug: # Silence Jester's query warning @@ -54,6 +55,7 @@ createMediaRouter(cfg) createEmbedRouter(cfg) createRssRouter(cfg) createDebugRouter(cfg) +createAuthRouter(cfg) settings: port = Port(cfg.port) @@ -108,3 +110,4 @@ routes: extend embed, "" extend debug, "" extend unsupported, "" + extend auth, "" diff --git a/src/routes/auth.nim b/src/routes/auth.nim new file mode 100644 index 0000000..eee9e38 --- /dev/null +++ b/src/routes/auth.nim @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: AGPL-3.0-only + +import jester + +import router_utils +import ".."/[types, auth] + +proc createAuthRouter*(cfg: Config) = + router auth: + get "/.well-known/nitter-auth": + cond cfg.guestAccountsUsePool + resp Http200, {"content-type": "text/plain"}, getAuthHash(cfg) diff --git a/src/types.nim b/src/types.nim index ddbebdf..4b85024 100644 --- a/src/types.nim +++ b/src/types.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import times, sequtils, options, tables +import std/[times, sequtils, options, tables, uri] import prefs_impl genPrefsType() @@ -262,12 +262,16 @@ type hmacKey*: string base64Media*: bool - minTokens*: int enableRss*: bool enableDebug*: bool proxy*: string proxyAuth*: string + guestAccountsUsePool*: bool + guestAccountsPoolUrl*: Uri + guestAccountsPoolId*: string + guestAccountsPoolAuth*: string + rssCacheTime*: int listCacheTime*: int