diff --git a/nitter.example.conf b/nitter.example.conf index ed432a6..0069e3b 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -26,18 +26,14 @@ enableRSS = true # set this to false to disable RSS feeds enableDebug = false # enable request logs and debug endpoints (/.accounts) proxy = "" # http/https url, SOCKS proxies are not supported proxyAuth = "" -tokenCount = 10 -# minimum amount of usable tokens. tokens are used to authorize API requests, -# but they expire after ~1 hour, and have a limit of 500 requests per endpoint. -# the limits reset every 15 minutes, and the pool is filled up so there's -# always at least `tokenCount` usable tokens. only increase this if you receive -# major bursts all the time and don't have a rate limiting setup via e.g. nginx # Instead of guest_accounts.json, fetch guest accounts from an external URL. # Nitter will re-fetch accounts if it runs out of valid ones. -guestAccountsUrl = "" # https://example.com/download -guestAccountsHost = "" # defaults to nitter's hostname -guestAccountsKey = "" # a random string that will be used as a secret to authenticate against the URL +[GuestAccounts] +usePool = false # enable fetching accounts from external pool. +poolUrl = "" # https://example.com/download +poolId = "" # defaults to nitter's hostname +poolAuth = "" # random secret string for authentication # Change default preferences here, see src/prefs_impl.nim for a complete list [Preferences] diff --git a/src/auth.nim b/src/auth.nim index f8e6d90..1e8f45b 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -25,11 +25,6 @@ const }.toTable var - pool: HttpPool - guestAccountsUrl = "" - guestAccountsHost = "" - guestAccountsKey = "" - guestAccountsUrlLastFetched = 0 accountPool: seq[GuestAccount] enableLogging = false @@ -164,27 +159,6 @@ proc release*(account: GuestAccount) = dec account.pending proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} = - let now = epochTime().int - - if accountPool.len == 0 and guestAccountsUrl != "" and guestAccountsUrlLastFetched < now - 3600: - once: - pool = HttpPool() - - guestAccountsUrlLastFetched = now - - log "fetching more accounts from service" - pool.use(newHttpHeaders()): - let resp = await c.get("$1?host=$2&key=$3" % [guestAccountsUrl, guestAccountsHost, guestAccountsKey]) - let guestAccounts = await resp.body - - log "status code from service: ", resp.status - - for line in guestAccounts.splitLines: - if line != "": - accountPool.add parseGuestAccount(line) - - accountPool.keepItIf(not it.hasExpired) - for i in 0 ..< accountPool.len: if result.isReady(api): break result = accountPool.sample() @@ -214,9 +188,6 @@ proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) = proc initAccountPool*(cfg: Config; path: string) = enableLogging = cfg.enableDebug - guestAccountsUrl = cfg.guestAccountsUrl - guestAccountsHost = cfg.guestAccountsHost - guestAccountsKey = cfg.guestAccountsKey let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path @@ -227,20 +198,45 @@ proc initAccountPool*(cfg: Config; path: string) = elif fileExists(path): log "Parsing JSON guest accounts file: ", path accountPool = parseGuestAccounts(path) - elif guestAccountsUrl == "": - echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests." + elif not cfg.guestAccountsUsePool: + echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests. Alternatively, configure the guest account pool in nitter.conf" quit 1 - let accountsPrePurge = accountPool.len accountPool.keepItIf(not it.hasExpired) log "Successfully added ", accountPool.len, " valid accounts." +proc updateAccountPool*(cfg: Config) {.async.} = + if not cfg.guestAccountsUsePool: + return + + while true: + if accountPool.len == 0: + let pool = HttpPool() + + log "fetching more accounts from service" + pool.use(newHttpHeaders()): + let resp = await c.get("$1?host=$2&key=$3" % [cfg.guestAccountsPoolUrl, cfg.guestAccountsPoolId, cfg.guestAccountsPoolAuth]) + let guestAccounts = await resp.body + + log "status code from service: ", resp.status + + for line in guestAccounts.splitLines: + if line != "": + accountPool.add parseGuestAccount(line) + + accountPool.keepItIf(not it.hasExpired) + + await sleepAsync(3600) + proc getAuthHash*(cfg: Config): string = - if cfg.guestAccountsKey == "": - log "guestAccountsKey is set to bogus value, responding with empty string" + if cfg.guestAccountsPoolAuth == "": + # If somebody turns on pool auth and provides a dummy key, we should + # prevent third parties from using that mis-configured auth and impersonate + # this instance + log "poolAuth is set to bogus value, responding with empty string" return "" - let hashStr = $sha_256.digest(cfg.guestAccountsKey) + let hashStr = $sha_256.digest(cfg.guestAccountsPoolAuth) return hashStr.toLowerAscii diff --git a/src/config.nim b/src/config.nim index 3cf514b..e199d65 100644 --- a/src/config.nim +++ b/src/config.nim @@ -36,14 +36,16 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = # Config hmacKey: cfg.get("Config", "hmacKey", "secretkey"), base64Media: cfg.get("Config", "base64Media", false), - minTokens: cfg.get("Config", "tokenCount", 10), enableRss: cfg.get("Config", "enableRSS", true), enableDebug: cfg.get("Config", "enableDebug", false), proxy: cfg.get("Config", "proxy", ""), proxyAuth: cfg.get("Config", "proxyAuth", ""), - guestAccountsUrl: cfg.get("Config", "guestAccountsUrl", ""), - guestAccountsKey: cfg.get("Config", "guestAccountsKey", ""), - guestAccountsHost: cfg.get("Config", "guestAccountsHost", cfg.get("Server", "hostname", "")) + + # GuestAccounts + guestAccountsUsePool: cfg.get("GuestAccounts", "usePool", false), + guestAccountsPoolUrl: cfg.get("GuestAccounts", "poolUrl", ""), + guestAccountsPoolAuth: cfg.get("GuestAccounts", "poolAuth", ""), + guestAccountsPoolId: cfg.get("GuestAccounts", "poolId", cfg.get("Server", "hostname", "")) ) return (conf, cfg) diff --git a/src/nitter.nim b/src/nitter.nim index 3e4847b..9f05b2a 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -22,6 +22,7 @@ let accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") initAccountPool(cfg, accountsPath) +asyncCheck updateAccountPool(cfg) if not cfg.enableDebug: # Silence Jester's query warning diff --git a/src/routes/auth.nim b/src/routes/auth.nim index c5ad590..2a481d8 100644 --- a/src/routes/auth.nim +++ b/src/routes/auth.nim @@ -7,6 +7,6 @@ import ".."/[types, auth] proc createAuthRouter*(cfg: Config) = router auth: - get "/.well-known/nitter-auth-sha256": - cond cfg.guestAccountsUrl != "" + get "/.well-known/nitter-request-auth": + cond cfg.guestAccountsUsePool resp Http200, {"content-type": "text/plain"}, getAuthHash(cfg) diff --git a/src/types.nim b/src/types.nim index 529f11f..e2abaed 100644 --- a/src/types.nim +++ b/src/types.nim @@ -254,18 +254,19 @@ type title*: string hostname*: string staticDir*: string - guestAccountsUrl*: string - guestAccountsKey*: string - guestAccountsHost*: string hmacKey*: string base64Media*: bool - minTokens*: int enableRss*: bool enableDebug*: bool proxy*: string proxyAuth*: string + guestAccountsUsePool*: bool + guestAccountsPoolUrl*: string + guestAccountsPoolId*: string + guestAccountsPoolAuth*: string + rssCacheTime*: int listCacheTime*: int