nitter/src/redis_cache.nim

133 lines
4.1 KiB
Nim
Raw Normal View History

import asyncdispatch, times, strutils, tables, hashes
2020-11-15 00:21:56 +00:00
import redis, redpool, frosty, supersnappy
2020-06-01 00:16:24 +00:00
import types, api
const redisNil = "\0\0"
var
2020-06-22 01:50:16 +00:00
pool {.threadvar.}: RedisPool
2020-06-01 00:16:24 +00:00
baseCacheTime = 60 * 60
rssCacheTime: int
listCacheTime*: int
proc setCacheTimes*(cfg: Config) =
rssCacheTime = cfg.rssCacheTime * 60
listCacheTime = cfg.listCacheTime * 60
proc migrate*(key, match: string) {.async.} =
pool.withAcquire(r):
let hasKey = await r.get(key)
if hasKey == redisNil:
let list = await r.scan(newCursor(0), match, 100000)
r.startPipelining()
for item in list:
if item != "p:" or item == match:
discard await r.del(item)
await r.setk(key, "true")
discard await r.flushPipeline()
proc initRedisPool*(cfg: Config) {.async.} =
2020-06-01 11:36:20 +00:00
try:
pool = await newRedisPool(cfg.redisConns, maxConns=cfg.redisMaxConns,
host=cfg.redisHost, port=cfg.redisPort)
await migrate("snappyRss", "rss:*")
await migrate("oldFrosty", "*")
await migrate("userBuckets", "p:")
pool.withAcquire(r):
await r.configSet("hash-max-ziplist-entries", "1000")
2020-06-01 11:36:20 +00:00
except OSError:
2020-06-07 07:18:40 +00:00
stdout.write "Failed to connect to Redis.\n"
stdout.flushFile
quit(1)
2020-06-01 00:16:24 +00:00
template pidKey(name: string): string = "pid:" & $(hash(name) div 1_000_000)
template profileKey(name: string): string = "p:" & name
template listKey(l: List): string = toLower("l:" & l.username & '/' & l.name)
2020-06-01 00:16:24 +00:00
proc get(query: string): Future[string] {.async.} =
pool.withAcquire(r):
result = await r.get(query)
2020-06-02 20:36:02 +00:00
proc setex(key: string; time: int; data: string) {.async.} =
2020-06-01 00:16:24 +00:00
pool.withAcquire(r):
2020-06-02 20:36:02 +00:00
discard await r.setex(key, time, data)
2020-06-01 00:16:24 +00:00
2020-06-02 20:36:02 +00:00
proc cache*(data: List) {.async.} =
await setex(data.listKey, listCacheTime, compress(freeze(data)))
2020-06-01 00:16:24 +00:00
2020-06-16 22:20:34 +00:00
proc cache*(data: PhotoRail; name: string) {.async.} =
await setex("pr:" & name, baseCacheTime, compress(freeze(data)))
2020-06-01 00:16:24 +00:00
2020-06-02 20:36:02 +00:00
proc cache*(data: Profile) {.async.} =
if data.username.len == 0 or data.id.len == 0: return
let name = toLower(data.username)
2020-06-01 00:16:24 +00:00
pool.withAcquire(r):
r.startPipelining()
discard await r.setex(name.profileKey, baseCacheTime, compress(freeze(data)))
discard await r.hset(name.pidKey, name, data.id)
2020-06-01 00:16:24 +00:00
discard await r.flushPipeline()
2020-06-02 22:03:41 +00:00
proc cacheProfileId*(username, id: string) {.async.} =
if username.len == 0 or id.len == 0: return
let name = toLower(username)
2020-06-02 22:03:41 +00:00
pool.withAcquire(r):
discard await r.hset(name.pidKey, name, id)
2020-06-02 22:03:41 +00:00
proc cacheRss*(query: string; rss: Rss) {.async.} =
2020-06-01 00:16:24 +00:00
let key = "rss:" & query
pool.withAcquire(r):
r.startPipelining()
discard await r.hset(key, "rss", rss.feed)
discard await r.hset(key, "min", rss.cursor)
2020-06-01 00:16:24 +00:00
discard await r.expire(key, rssCacheTime)
discard await r.flushPipeline()
proc getProfileId*(username: string): Future[string] {.async.} =
let name = toLower(username)
2020-06-01 00:16:24 +00:00
pool.withAcquire(r):
result = await r.hget(name.pidKey, name)
2020-06-01 00:16:24 +00:00
if result == redisNil:
result.setLen(0)
proc getCachedProfile*(username: string; fetch=true): Future[Profile] {.async.} =
2020-06-01 00:16:24 +00:00
let prof = await get("p:" & toLower(username))
if prof != redisNil:
uncompress(prof).thaw(result)
2020-06-02 22:03:41 +00:00
elif fetch:
result = await getProfile(username)
2020-06-01 00:16:24 +00:00
2020-06-16 22:20:34 +00:00
proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
let rail = await get("pr:" & toLower(name))
2020-06-01 00:16:24 +00:00
if rail != redisNil:
uncompress(rail).thaw(result)
2020-06-01 00:16:24 +00:00
else:
2020-06-16 22:20:34 +00:00
result = await getPhotoRail(name)
await cache(result, name)
2020-06-01 00:16:24 +00:00
proc getCachedList*(username=""; name=""; id=""): Future[List] {.async.} =
let list = if id.len > 0: redisNil
else: await get(toLower("l:" & username & '/' & name))
if list != redisNil:
uncompress(list).thaw(result)
2020-06-01 00:16:24 +00:00
else:
if id.len > 0:
result = await getGraphListById(id)
else:
result = await getGraphList(username, name)
2020-06-02 20:36:02 +00:00
await cache(result)
2020-06-01 00:16:24 +00:00
proc getCachedRss*(key: string): Future[Rss] {.async.} =
let k = "rss:" & key
2020-06-01 00:16:24 +00:00
pool.withAcquire(r):
result.cursor = await r.hget(k, "min")
if result.cursor.len > 2:
result.feed = await r.hget(k, "rss")
else:
result.cursor.setLen 0