From 5bd4e545ce67084e0fc515f54e8caae4e08d3f61 Mon Sep 17 00:00:00 2001 From: drpepper66 Date: Tue, 14 Nov 2023 23:51:08 +0100 Subject: [PATCH 1/9] Add functionality to fetch guest accounts on demand --- nitter.example.conf | 6 ++++++ src/auth.nim | 47 ++++++++++++++++++++++++++++++++++++++++----- src/config.nim | 5 ++++- src/nitter.nim | 4 +++- src/routes/auth.nim | 12 ++++++++++++ src/types.nim | 3 +++ 6 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 src/routes/auth.nim diff --git a/nitter.example.conf b/nitter.example.conf index f0b4214..ed432a6 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -33,6 +33,12 @@ tokenCount = 10 # always at least `tokenCount` usable tokens. only increase this if you receive # major bursts all the time and don't have a rate limiting setup via e.g. nginx +# Instead of guest_accounts.json, fetch guest accounts from an external URL. +# Nitter will re-fetch accounts if it runs out of valid ones. +guestAccountsUrl = "" # https://example.com/download +guestAccountsHost = "" # defaults to nitter's hostname +guestAccountsKey = "" # a random string that will be used as a secret to authenticate against the URL + # Change default preferences here, see src/prefs_impl.nim for a complete list [Preferences] theme = "Nitter" diff --git a/src/auth.nim b/src/auth.nim index b288c50..f8e6d90 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -1,6 +1,7 @@ #SPDX-License-Identifier: AGPL-3.0-only -import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os] -import types +import std/[httpclient, asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os] +import nimcrypto +import types, http_pool import experimental/parser/guestaccount # max requests at a time per account to avoid race conditions @@ -24,6 +25,11 @@ const }.toTable var + pool: HttpPool + guestAccountsUrl = "" + guestAccountsHost = "" + guestAccountsKey = "" + guestAccountsUrlLastFetched = 0 accountPool: seq[GuestAccount] enableLogging = false @@ -158,6 +164,27 @@ proc release*(account: GuestAccount) = dec account.pending proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} = + let now = epochTime().int + + if accountPool.len == 0 and guestAccountsUrl != "" and guestAccountsUrlLastFetched < now - 3600: + once: + pool = HttpPool() + + guestAccountsUrlLastFetched = now + + log "fetching more accounts from service" + pool.use(newHttpHeaders()): + let resp = await c.get("$1?host=$2&key=$3" % [guestAccountsUrl, guestAccountsHost, guestAccountsKey]) + let guestAccounts = await resp.body + + log "status code from service: ", resp.status + + for line in guestAccounts.splitLines: + if line != "": + accountPool.add parseGuestAccount(line) + + accountPool.keepItIf(not it.hasExpired) + for i in 0 ..< accountPool.len: if result.isReady(api): break result = accountPool.sample() @@ -187,6 +214,9 @@ proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) = proc initAccountPool*(cfg: Config; path: string) = enableLogging = cfg.enableDebug + guestAccountsUrl = cfg.guestAccountsUrl + guestAccountsHost = cfg.guestAccountsHost + guestAccountsKey = cfg.guestAccountsKey let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path @@ -197,7 +227,7 @@ proc initAccountPool*(cfg: Config; path: string) = elif fileExists(path): log "Parsing JSON guest accounts file: ", path accountPool = parseGuestAccounts(path) - else: + elif guestAccountsUrl == "": echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests." quit 1 @@ -205,5 +235,12 @@ proc initAccountPool*(cfg: Config; path: string) = accountPool.keepItIf(not it.hasExpired) log "Successfully added ", accountPool.len, " valid accounts." - if accountsPrePurge > accountPool.len: - log "Purged ", accountsPrePurge - accountPool.len, " expired accounts." + +proc getAuthHash*(cfg: Config): string = + if cfg.guestAccountsKey == "": + log "guestAccountsKey is set to bogus value, responding with empty string" + return "" + + let hashStr = $sha_256.digest(cfg.guestAccountsKey) + + return hashStr.toLowerAscii diff --git a/src/config.nim b/src/config.nim index 1b05ffe..3cf514b 100644 --- a/src/config.nim +++ b/src/config.nim @@ -40,7 +40,10 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = enableRss: cfg.get("Config", "enableRSS", true), enableDebug: cfg.get("Config", "enableDebug", false), proxy: cfg.get("Config", "proxy", ""), - proxyAuth: cfg.get("Config", "proxyAuth", "") + proxyAuth: cfg.get("Config", "proxyAuth", ""), + guestAccountsUrl: cfg.get("Config", "guestAccountsUrl", ""), + guestAccountsKey: cfg.get("Config", "guestAccountsKey", ""), + guestAccountsHost: cfg.get("Config", "guestAccountsHost", cfg.get("Server", "hostname", "")) ) return (conf, cfg) diff --git a/src/nitter.nim b/src/nitter.nim index dfc1dfd..3e4847b 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, - unsupported, embed, resolver, router_utils] + unsupported, embed, resolver, router_utils, auth] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" @@ -54,6 +54,7 @@ createMediaRouter(cfg) createEmbedRouter(cfg) createRssRouter(cfg) createDebugRouter(cfg) +createAuthRouter(cfg) settings: port = Port(cfg.port) @@ -108,3 +109,4 @@ routes: extend embed, "" extend debug, "" extend unsupported, "" + extend auth, "" diff --git a/src/routes/auth.nim b/src/routes/auth.nim new file mode 100644 index 0000000..c5ad590 --- /dev/null +++ b/src/routes/auth.nim @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: AGPL-3.0-only + +import jester + +import router_utils +import ".."/[types, auth] + +proc createAuthRouter*(cfg: Config) = + router auth: + get "/.well-known/nitter-auth-sha256": + cond cfg.guestAccountsUrl != "" + resp Http200, {"content-type": "text/plain"}, getAuthHash(cfg) diff --git a/src/types.nim b/src/types.nim index 9ddf283..529f11f 100644 --- a/src/types.nim +++ b/src/types.nim @@ -254,6 +254,9 @@ type title*: string hostname*: string staticDir*: string + guestAccountsUrl*: string + guestAccountsKey*: string + guestAccountsHost*: string hmacKey*: string base64Media*: bool From 08604ac8d19d126392967cf3531157e462d02c3f Mon Sep 17 00:00:00 2001 From: drpepper66 Date: Fri, 17 Nov 2023 20:07:14 +0100 Subject: [PATCH 2/9] apply review feedback --- nitter.example.conf | 14 ++++------ src/auth.nim | 66 +++++++++++++++++++++------------------------ src/config.nim | 10 ++++--- src/nitter.nim | 1 + src/routes/auth.nim | 4 +-- src/types.nim | 9 ++++--- 6 files changed, 50 insertions(+), 54 deletions(-) diff --git a/nitter.example.conf b/nitter.example.conf index ed432a6..0069e3b 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -26,18 +26,14 @@ enableRSS = true # set this to false to disable RSS feeds enableDebug = false # enable request logs and debug endpoints (/.accounts) proxy = "" # http/https url, SOCKS proxies are not supported proxyAuth = "" -tokenCount = 10 -# minimum amount of usable tokens. tokens are used to authorize API requests, -# but they expire after ~1 hour, and have a limit of 500 requests per endpoint. -# the limits reset every 15 minutes, and the pool is filled up so there's -# always at least `tokenCount` usable tokens. only increase this if you receive -# major bursts all the time and don't have a rate limiting setup via e.g. nginx # Instead of guest_accounts.json, fetch guest accounts from an external URL. # Nitter will re-fetch accounts if it runs out of valid ones. -guestAccountsUrl = "" # https://example.com/download -guestAccountsHost = "" # defaults to nitter's hostname -guestAccountsKey = "" # a random string that will be used as a secret to authenticate against the URL +[GuestAccounts] +usePool = false # enable fetching accounts from external pool. +poolUrl = "" # https://example.com/download +poolId = "" # defaults to nitter's hostname +poolAuth = "" # random secret string for authentication # Change default preferences here, see src/prefs_impl.nim for a complete list [Preferences] diff --git a/src/auth.nim b/src/auth.nim index f8e6d90..1e8f45b 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -25,11 +25,6 @@ const }.toTable var - pool: HttpPool - guestAccountsUrl = "" - guestAccountsHost = "" - guestAccountsKey = "" - guestAccountsUrlLastFetched = 0 accountPool: seq[GuestAccount] enableLogging = false @@ -164,27 +159,6 @@ proc release*(account: GuestAccount) = dec account.pending proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} = - let now = epochTime().int - - if accountPool.len == 0 and guestAccountsUrl != "" and guestAccountsUrlLastFetched < now - 3600: - once: - pool = HttpPool() - - guestAccountsUrlLastFetched = now - - log "fetching more accounts from service" - pool.use(newHttpHeaders()): - let resp = await c.get("$1?host=$2&key=$3" % [guestAccountsUrl, guestAccountsHost, guestAccountsKey]) - let guestAccounts = await resp.body - - log "status code from service: ", resp.status - - for line in guestAccounts.splitLines: - if line != "": - accountPool.add parseGuestAccount(line) - - accountPool.keepItIf(not it.hasExpired) - for i in 0 ..< accountPool.len: if result.isReady(api): break result = accountPool.sample() @@ -214,9 +188,6 @@ proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) = proc initAccountPool*(cfg: Config; path: string) = enableLogging = cfg.enableDebug - guestAccountsUrl = cfg.guestAccountsUrl - guestAccountsHost = cfg.guestAccountsHost - guestAccountsKey = cfg.guestAccountsKey let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path @@ -227,20 +198,45 @@ proc initAccountPool*(cfg: Config; path: string) = elif fileExists(path): log "Parsing JSON guest accounts file: ", path accountPool = parseGuestAccounts(path) - elif guestAccountsUrl == "": - echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests." + elif not cfg.guestAccountsUsePool: + echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests. Alternatively, configure the guest account pool in nitter.conf" quit 1 - let accountsPrePurge = accountPool.len accountPool.keepItIf(not it.hasExpired) log "Successfully added ", accountPool.len, " valid accounts." +proc updateAccountPool*(cfg: Config) {.async.} = + if not cfg.guestAccountsUsePool: + return + + while true: + if accountPool.len == 0: + let pool = HttpPool() + + log "fetching more accounts from service" + pool.use(newHttpHeaders()): + let resp = await c.get("$1?host=$2&key=$3" % [cfg.guestAccountsPoolUrl, cfg.guestAccountsPoolId, cfg.guestAccountsPoolAuth]) + let guestAccounts = await resp.body + + log "status code from service: ", resp.status + + for line in guestAccounts.splitLines: + if line != "": + accountPool.add parseGuestAccount(line) + + accountPool.keepItIf(not it.hasExpired) + + await sleepAsync(3600) + proc getAuthHash*(cfg: Config): string = - if cfg.guestAccountsKey == "": - log "guestAccountsKey is set to bogus value, responding with empty string" + if cfg.guestAccountsPoolAuth == "": + # If somebody turns on pool auth and provides a dummy key, we should + # prevent third parties from using that mis-configured auth and impersonate + # this instance + log "poolAuth is set to bogus value, responding with empty string" return "" - let hashStr = $sha_256.digest(cfg.guestAccountsKey) + let hashStr = $sha_256.digest(cfg.guestAccountsPoolAuth) return hashStr.toLowerAscii diff --git a/src/config.nim b/src/config.nim index 3cf514b..e199d65 100644 --- a/src/config.nim +++ b/src/config.nim @@ -36,14 +36,16 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = # Config hmacKey: cfg.get("Config", "hmacKey", "secretkey"), base64Media: cfg.get("Config", "base64Media", false), - minTokens: cfg.get("Config", "tokenCount", 10), enableRss: cfg.get("Config", "enableRSS", true), enableDebug: cfg.get("Config", "enableDebug", false), proxy: cfg.get("Config", "proxy", ""), proxyAuth: cfg.get("Config", "proxyAuth", ""), - guestAccountsUrl: cfg.get("Config", "guestAccountsUrl", ""), - guestAccountsKey: cfg.get("Config", "guestAccountsKey", ""), - guestAccountsHost: cfg.get("Config", "guestAccountsHost", cfg.get("Server", "hostname", "")) + + # GuestAccounts + guestAccountsUsePool: cfg.get("GuestAccounts", "usePool", false), + guestAccountsPoolUrl: cfg.get("GuestAccounts", "poolUrl", ""), + guestAccountsPoolAuth: cfg.get("GuestAccounts", "poolAuth", ""), + guestAccountsPoolId: cfg.get("GuestAccounts", "poolId", cfg.get("Server", "hostname", "")) ) return (conf, cfg) diff --git a/src/nitter.nim b/src/nitter.nim index 3e4847b..9f05b2a 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -22,6 +22,7 @@ let accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") initAccountPool(cfg, accountsPath) +asyncCheck updateAccountPool(cfg) if not cfg.enableDebug: # Silence Jester's query warning diff --git a/src/routes/auth.nim b/src/routes/auth.nim index c5ad590..2a481d8 100644 --- a/src/routes/auth.nim +++ b/src/routes/auth.nim @@ -7,6 +7,6 @@ import ".."/[types, auth] proc createAuthRouter*(cfg: Config) = router auth: - get "/.well-known/nitter-auth-sha256": - cond cfg.guestAccountsUrl != "" + get "/.well-known/nitter-request-auth": + cond cfg.guestAccountsUsePool resp Http200, {"content-type": "text/plain"}, getAuthHash(cfg) diff --git a/src/types.nim b/src/types.nim index 529f11f..e2abaed 100644 --- a/src/types.nim +++ b/src/types.nim @@ -254,18 +254,19 @@ type title*: string hostname*: string staticDir*: string - guestAccountsUrl*: string - guestAccountsKey*: string - guestAccountsHost*: string hmacKey*: string base64Media*: bool - minTokens*: int enableRss*: bool enableDebug*: bool proxy*: string proxyAuth*: string + guestAccountsUsePool*: bool + guestAccountsPoolUrl*: string + guestAccountsPoolId*: string + guestAccountsPoolAuth*: string + rssCacheTime*: int listCacheTime*: int From 98cc15e78cb826572c08e558e0ed39088a4fac75 Mon Sep 17 00:00:00 2001 From: drpepper66 Date: Fri, 17 Nov 2023 20:09:10 +0100 Subject: [PATCH 3/9] rename query parameters for consistency -- requires changes in twitterminator anyway --- src/auth.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auth.nim b/src/auth.nim index 1e8f45b..f8210b8 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -216,7 +216,7 @@ proc updateAccountPool*(cfg: Config) {.async.} = log "fetching more accounts from service" pool.use(newHttpHeaders()): - let resp = await c.get("$1?host=$2&key=$3" % [cfg.guestAccountsPoolUrl, cfg.guestAccountsPoolId, cfg.guestAccountsPoolAuth]) + let resp = await c.get("$1?id=$2&auth=$3" % [cfg.guestAccountsPoolUrl, cfg.guestAccountsPoolId, cfg.guestAccountsPoolAuth]) let guestAccounts = await resp.body log "status code from service: ", resp.status From ab321c9b105ac1bec011fec0028d6b1ec66467fd Mon Sep 17 00:00:00 2001 From: drpepper66 Date: Fri, 17 Nov 2023 22:48:51 +0100 Subject: [PATCH 4/9] fix sleeps, better exception handling --- src/auth.nim | 27 ++++++++++++++++++--------- src/routes/auth.nim | 2 +- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/auth.nim b/src/auth.nim index f8210b8..e9c4cbf 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -210,24 +210,33 @@ proc updateAccountPool*(cfg: Config) {.async.} = if not cfg.guestAccountsUsePool: return + # wait for a few seconds before fetching guest accounts, so that + # /.well-known/... is served correctly + await sleepAsync(10 * 1000) + while true: if accountPool.len == 0: let pool = HttpPool() log "fetching more accounts from service" - pool.use(newHttpHeaders()): - let resp = await c.get("$1?id=$2&auth=$3" % [cfg.guestAccountsPoolUrl, cfg.guestAccountsPoolId, cfg.guestAccountsPoolAuth]) - let guestAccounts = await resp.body - log "status code from service: ", resp.status + try: + pool.use(newHttpHeaders()): + let resp = await c.get("$1?id=$2&auth=$3" % [cfg.guestAccountsPoolUrl, cfg.guestAccountsPoolId, cfg.guestAccountsPoolAuth]) + let guestAccounts = await resp.body - for line in guestAccounts.splitLines: - if line != "": - accountPool.add parseGuestAccount(line) + log "status code from service: ", resp.status - accountPool.keepItIf(not it.hasExpired) + for line in guestAccounts.splitLines: + if line != "": + accountPool.add parseGuestAccount(line) - await sleepAsync(3600) + except Exception as e: + log "failed to fetch from accounts service: ", e.msg + + accountPool.keepItIf(not it.hasExpired) + + await sleepAsync(3600 * 1000) proc getAuthHash*(cfg: Config): string = if cfg.guestAccountsPoolAuth == "": diff --git a/src/routes/auth.nim b/src/routes/auth.nim index 2a481d8..eee9e38 100644 --- a/src/routes/auth.nim +++ b/src/routes/auth.nim @@ -7,6 +7,6 @@ import ".."/[types, auth] proc createAuthRouter*(cfg: Config) = router auth: - get "/.well-known/nitter-request-auth": + get "/.well-known/nitter-auth": cond cfg.guestAccountsUsePool resp Http200, {"content-type": "text/plain"}, getAuthHash(cfg) From a11332b2b527226f049d0291d94dce4f80e15d3f Mon Sep 17 00:00:00 2001 From: drpepper66 Date: Fri, 17 Nov 2023 22:54:04 +0100 Subject: [PATCH 5/9] apply some review feedback --- src/auth.nim | 8 ++++---- src/config.nim | 3 ++- src/types.nim | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/auth.nim b/src/auth.nim index e9c4cbf..15db751 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -1,5 +1,5 @@ #SPDX-License-Identifier: AGPL-3.0-only -import std/[httpclient, asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os] +import std/[httpclient, asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os, uri] import nimcrypto import types, http_pool import experimental/parser/guestaccount @@ -222,7 +222,7 @@ proc updateAccountPool*(cfg: Config) {.async.} = try: pool.use(newHttpHeaders()): - let resp = await c.get("$1?id=$2&auth=$3" % [cfg.guestAccountsPoolUrl, cfg.guestAccountsPoolId, cfg.guestAccountsPoolAuth]) + let resp = await c.get($(cfg.guestAccountsPoolUrl ? {"id": cfg.guestAccountsPoolId, "auth": cfg.guestAccountsPoolAuth})) let guestAccounts = await resp.body log "status code from service: ", resp.status @@ -239,11 +239,11 @@ proc updateAccountPool*(cfg: Config) {.async.} = await sleepAsync(3600 * 1000) proc getAuthHash*(cfg: Config): string = - if cfg.guestAccountsPoolAuth == "": + if cfg.guestAccountsPoolAuth.len == 0: # If somebody turns on pool auth and provides a dummy key, we should # prevent third parties from using that mis-configured auth and impersonate # this instance - log "poolAuth is set to bogus value, responding with empty string" + log "poolAuth is empty, authentication with accounts service will fail" return "" let hashStr = $sha_256.digest(cfg.guestAccountsPoolAuth) diff --git a/src/config.nim b/src/config.nim index e199d65..c27cfd5 100644 --- a/src/config.nim +++ b/src/config.nim @@ -1,6 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only import parsecfg except Config import types, strutils +import std/uri proc get*[T](config: parseCfg.Config; section, key: string; default: T): T = let val = config.getSectionValue(section, key) @@ -43,7 +44,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = # GuestAccounts guestAccountsUsePool: cfg.get("GuestAccounts", "usePool", false), - guestAccountsPoolUrl: cfg.get("GuestAccounts", "poolUrl", ""), + guestAccountsPoolUrl: parseUri(cfg.get("GuestAccounts", "poolUrl", "")), guestAccountsPoolAuth: cfg.get("GuestAccounts", "poolAuth", ""), guestAccountsPoolId: cfg.get("GuestAccounts", "poolId", cfg.get("Server", "hostname", "")) ) diff --git a/src/types.nim b/src/types.nim index e2abaed..98b12da 100644 --- a/src/types.nim +++ b/src/types.nim @@ -1,6 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only import times, sequtils, options, tables import prefs_impl +import std/uri genPrefsType() @@ -263,7 +264,7 @@ type proxyAuth*: string guestAccountsUsePool*: bool - guestAccountsPoolUrl*: string + guestAccountsPoolUrl*: Uri guestAccountsPoolId*: string guestAccountsPoolAuth*: string From f6ca1a3f2a38e5fe5677b3a0e3150148b9b67838 Mon Sep 17 00:00:00 2001 From: drpepper66 Date: Fri, 17 Nov 2023 23:07:15 +0100 Subject: [PATCH 6/9] better imports --- src/config.nim | 4 ++-- src/types.nim | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/config.nim b/src/config.nim index c27cfd5..d7917a4 100644 --- a/src/config.nim +++ b/src/config.nim @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only -import parsecfg except Config -import types, strutils +import std/parsecfg except Config import std/uri +import types, strutils proc get*[T](config: parseCfg.Config; section, key: string; default: T): T = let val = config.getSectionValue(section, key) diff --git a/src/types.nim b/src/types.nim index 98b12da..1dfee55 100644 --- a/src/types.nim +++ b/src/types.nim @@ -1,7 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-only -import times, sequtils, options, tables +import std/[times, sequtils, options, tables, uri] import prefs_impl -import std/uri genPrefsType() From c8c606f96e8677f1f4ad682e18aae67121f25e9d Mon Sep 17 00:00:00 2001 From: drpepper66 Date: Fri, 17 Nov 2023 23:09:45 +0100 Subject: [PATCH 7/9] use stdlib http client --- src/auth.nim | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/auth.nim b/src/auth.nim index 15db751..4de950b 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -1,7 +1,7 @@ #SPDX-License-Identifier: AGPL-3.0-only import std/[httpclient, asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os, uri] import nimcrypto -import types, http_pool +import types import experimental/parser/guestaccount # max requests at a time per account to avoid race conditions @@ -216,13 +216,12 @@ proc updateAccountPool*(cfg: Config) {.async.} = while true: if accountPool.len == 0: - let pool = HttpPool() - log "fetching more accounts from service" + let client = newAsyncHttpClient() + try: - pool.use(newHttpHeaders()): - let resp = await c.get($(cfg.guestAccountsPoolUrl ? {"id": cfg.guestAccountsPoolId, "auth": cfg.guestAccountsPoolAuth})) + let resp = await client.get($(cfg.guestAccountsPoolUrl ? {"id": cfg.guestAccountsPoolId, "auth": cfg.guestAccountsPoolAuth})) let guestAccounts = await resp.body log "status code from service: ", resp.status @@ -233,6 +232,8 @@ proc updateAccountPool*(cfg: Config) {.async.} = except Exception as e: log "failed to fetch from accounts service: ", e.msg + finally: + client.close() accountPool.keepItIf(not it.hasExpired) From 8f3b5cafd19c0021167fc3d2e60828c0cf81616e Mon Sep 17 00:00:00 2001 From: drpepper66 Date: Fri, 17 Nov 2023 23:17:12 +0100 Subject: [PATCH 8/9] fix indent --- src/auth.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/auth.nim b/src/auth.nim index 4de950b..6f58eec 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -221,14 +221,14 @@ proc updateAccountPool*(cfg: Config) {.async.} = let client = newAsyncHttpClient() try: - let resp = await client.get($(cfg.guestAccountsPoolUrl ? {"id": cfg.guestAccountsPoolId, "auth": cfg.guestAccountsPoolAuth})) - let guestAccounts = await resp.body + let resp = await client.get($(cfg.guestAccountsPoolUrl ? {"id": cfg.guestAccountsPoolId, "auth": cfg.guestAccountsPoolAuth})) + let guestAccounts = await resp.body - log "status code from service: ", resp.status + log "status code from service: ", resp.status - for line in guestAccounts.splitLines: - if line != "": - accountPool.add parseGuestAccount(line) + for line in guestAccounts.splitLines: + if line != "": + accountPool.add parseGuestAccount(line) except Exception as e: log "failed to fetch from accounts service: ", e.msg From e3b7d6d5715b8b45065e4a31edb706961b41b570 Mon Sep 17 00:00:00 2001 From: drpepper66 Date: Fri, 17 Nov 2023 23:20:01 +0100 Subject: [PATCH 9/9] put a basic UA in there --- src/auth.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auth.nim b/src/auth.nim index 6f58eec..6d4b0b3 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -218,7 +218,7 @@ proc updateAccountPool*(cfg: Config) {.async.} = if accountPool.len == 0: log "fetching more accounts from service" - let client = newAsyncHttpClient() + let client = newAsyncHttpClient("nitter-accounts") try: let resp = await client.get($(cfg.guestAccountsPoolUrl ? {"id": cfg.guestAccountsPoolId, "auth": cfg.guestAccountsPoolAuth}))