nitter/src/tokens.nim

103 lines
2.7 KiB
Nim
Raw Normal View History

import asyncdispatch, httpclient, times, sequtils, json, math, random
2021-01-07 21:31:29 +00:00
import strutils, strformat
import types, agents, consts, http_pool
2020-06-01 00:16:24 +00:00
const
expirationTime = 3.hours
maxLastUse = 1.hours
resetPeriod = 15.minutes
failDelay = initDuration(minutes=30)
2020-07-09 07:18:14 +00:00
var
clientPool {.threadvar.}: HttpPool
2020-07-09 07:18:14 +00:00
tokenPool {.threadvar.}: seq[Token]
lastFailed: Time
proc getPoolInfo*: string =
if tokenPool.len == 0: return "token pool empty"
let avg = tokenPool.mapIt(it.remaining).sum() div tokenPool.len
return &"{tokenPool.len} tokens, average remaining: {avg}"
proc rateLimitError*(): ref RateLimitError =
newException(RateLimitError, "rate limited with " & getPoolInfo())
2020-06-01 00:16:24 +00:00
proc fetchToken(): Future[Token] {.async.} =
if getTime() - lastFailed < failDelay:
raise rateLimitError()
2020-07-09 07:18:14 +00:00
let headers = newHttpHeaders({
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"accept-language": "en-US,en;q=0.5",
"connection": "keep-alive",
"user-agent": getAgent(),
"authorization": auth
})
2020-06-01 00:16:24 +00:00
2020-06-24 13:02:34 +00:00
var
resp: string
2021-07-01 12:33:40 +00:00
tokNode: JsonNode
2020-06-24 13:02:34 +00:00
tok: string
2020-06-02 18:37:55 +00:00
try:
resp = clientPool.use(headers): await c.postContent(activate)
2021-07-01 12:33:40 +00:00
tokNode = parseJson(resp)["guest_token"]
tok = tokNode.getStr($(tokNode.getInt))
2020-06-01 00:16:24 +00:00
2020-06-24 13:02:34 +00:00
let time = getTime()
result = Token(tok: tok, remaining: 187, reset: time + resetPeriod,
2020-06-24 13:02:34 +00:00
init: time, lastUse: time)
except Exception as e:
2020-07-09 07:18:14 +00:00
lastFailed = getTime()
2021-01-07 21:31:29 +00:00
echo "fetching token failed: ", e.msg
2020-06-01 00:16:24 +00:00
template expired(token: Token): untyped =
let time = getTime()
token.init < time - expirationTime or
token.lastUse < time - maxLastUse
2020-06-01 00:16:24 +00:00
template isLimited(token: Token): untyped =
2021-12-20 03:19:11 +00:00
token == nil or (token.remaining <= 5 and token.reset > getTime()) or
2020-06-01 00:16:24 +00:00
token.expired
proc release*(token: Token; invalid=false) =
if token != nil and (invalid or token.expired):
let idx = tokenPool.find(token)
if idx > -1: tokenPool.delete(idx)
2020-06-01 00:16:24 +00:00
proc getToken*(): Future[Token] {.async.} =
for i in 0 ..< tokenPool.len:
if not result.isLimited: break
release(result)
result = tokenPool.sample()
2020-06-01 00:16:24 +00:00
if result.isLimited:
release(result)
2020-06-01 00:16:24 +00:00
result = await fetchToken()
tokenPool.add result
if result == nil:
raise rateLimitError()
2020-06-01 00:16:24 +00:00
proc poolTokens*(amount: int) {.async.} =
var futs: seq[Future[Token]]
for i in 0 ..< amount:
futs.add fetchToken()
for token in futs:
var newToken: Token
try: newToken = await token
except: discard
if newToken != nil:
tokenPool.add newToken
2020-06-01 00:16:24 +00:00
proc initTokenPool*(cfg: Config) {.async.} =
clientPool = HttpPool()
2020-06-01 00:16:24 +00:00
while true:
2020-06-01 11:40:26 +00:00
if tokenPool.countIt(not it.isLimited) < cfg.minTokens:
await poolTokens(min(4, cfg.minTokens - tokenPool.len))
await sleepAsync(2000)