Proxy media instead of using file cache

src/nitter.nim

@@ -1,4 +1,4 @@
-import asyncdispatch, mimetypes
+import asyncdispatch
 from net import Port
 
 import jester

src/routes/media.nim

@@ -1,4 +1,5 @@
-import asyncfile, uri, strutils, httpclient, os, mimetypes
+import uri, strutils, httpclient, os, hashes
+import asynchttpserver, asyncstreams, asyncfile, asyncnet
 
 import jester, regex
 
@@ -6,12 +7,59 @@ import router_utils
 import ".."/[types, formatters, agents]
 import ../views/general
 
-export asyncfile, httpclient, os, strutils
+export asynchttpserver, asyncstreams, asyncfile, asyncnet
-export regex
+export httpclient, os, strutils, asyncstreams, regex
+
+const
+  m3u8Regex* = re"""url="(.+.m3u8)""""
+  m3u8Mime* = "application/vnd.apple.mpegurl"
+  maxAge* = "max-age=604800"
 
-const m3u8Regex* = re"""url="(.+.m3u8)""""
 let mediaAgent* = getAgent()
 
+template respond*(req: asynchttpserver.Request; headers) =
+  var msg = "HTTP/1.1 200 OK\c\L"
+  for k, v in headers:
+    msg.add(k & ": " & v & "\c\L")
+
+  msg.add "\c\L"
+  yield req.client.send(msg)
+
+proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
+  result = Http200
+  let
+    request = req.getNativeReq()
+    client = newAsyncHttpClient(userAgent=mediaAgent)
+
+  try:
+    let res = await client.get(url)
+    if res.status != "200 OK":
+      return Http404
+
+    let hashed = $hash(url)
+    if request.headers.getOrDefault("If-None-Match") == hashed:
+      return Http304
+
+    let headers = newHttpHeaders({
+      "Content-Type": res.headers["content-type", 0],
+      "Content-Length": res.headers["content-length", 0],
+      "Cache-Control": maxAge,
+      "ETag": hashed
+    })
+
+    respond(request, headers)
+
+    var (hasValue, data) = (true, "")
+    while hasValue:
+      (hasValue, data) = await res.bodyStream.read()
+      if hasValue:
+        await request.client.send(data)
+    data.setLen 0
+  except HttpRequestError, OSError:
+    result = Http404
+  finally:
+    client.safeClose()
+
 proc createMediaRouter*(cfg: Config) =
   router media:
     get "/pic/?":
@@ -24,48 +72,13 @@ proc createMediaRouter*(cfg: Config) =
       let uri = parseUri(decodeUrl(@"url"))
       cond isTwitterUrl($uri) == true
 
-      let path = uri.path.split("/")[2 .. ^1].join("/")
+      enableRawMode()
-      let filename = cfg.cacheDir / cleanFilename(path & uri.query)
+      let code = await proxyMedia(request, $uri)
-
+      if code == Http200:
-      if path.len == 0:
+        enableRawMode()
-        resp Http404
+        break route
-
+      else:
-      if not existsDir(cfg.cacheDir):
+        resp code
-        createDir(cfg.cacheDir)
-
-      if not existsFile(filename):
-        let client = newAsyncHttpClient(userAgent=mediaAgent)
-        var failed = false
-
-        try:
-          await client.downloadFile($uri, filename)
-        except HttpRequestError:
-          failed = true
-          removeFile(filename)
-        except OSError as e:
-          echo "Disk full, or network error: ", e.msg
-          failed = true
-        finally:
-          client.safeClose()
-
-        if failed:
-          resp Http404
-
-      sendFile(filename)
-
-    get "/gif/@url":
-      cond "http" in @"url"
-      cond "twimg" in @"url"
-      cond "mp4" in @"url" or "gif" in @"url"
-
-      let url = decodeUrl(@"url")
-      cond isTwitterUrl(url) == true
-
-      let content = await safeFetch(url, mediaAgent)
-      if content.len == 0: resp Http404
-
-      let filename = parseUri(url).path.split(".")[^1]
-      resp content, settings.mimes.getMimetype(filename)
 
     get "/video/@sig/@url":
       cond "http" in @"url"
@@ -75,17 +88,26 @@ proc createMediaRouter*(cfg: Config) =
       if getHmac(url) != @"sig":
         resp showError("Failed to verify signature", cfg)
 
-      var content = await safeFetch(url, mediaAgent)
+      if ".mp4" in url or ".ts" in url:
-      if content.len == 0: resp Http404
+        let code = await proxyMedia(request, url)
+        if code == Http200:
+          enableRawMode()
+          break route
+        else:
+          resp code
 
+      var content: string
       if ".vmap" in url:
         var m: RegexMatch
-        discard content.find(m3u8Regex, m)
-        url = decodeUrl(content[m.group(0)[0]])
         content = await safeFetch(url, mediaAgent)
+        if content.find(m3u8Regex, m):
+          url = decodeUrl(content[m.group(0)[0]])
+          content = await safeFetch(url, mediaAgent)
+        else:
+          resp Http404
 
       if ".m3u8" in url:
-        content = proxifyVideo(content, prefs.proxyVideos)
+        let vid = await safeFetch(url, mediaAgent)
+        content = proxifyVideo(vid, prefs.proxyVideos)
 
-      let ext = parseUri(url).path.split(".")[^1]
+      resp content, m3u8Mime
-      resp content, settings.mimes.getMimetype(ext)

src/routes/status.nim

@@ -45,7 +45,7 @@ proc createStatusRouter*(cfg: Config) =
         video = getVideoEmbed(cfg, conv.tweet.id)
       elif conv.tweet.gif.isSome():
         images = @[get(conv.tweet.gif).thumb]
-        video = getGifUrl(get(conv.tweet.gif).url)
+        video = getPicUrl(get(conv.tweet.gif).url)
 
       let html = renderConversation(conv, prefs, getPath() & "#m")
       resp renderMain(html, request, cfg, title, desc,

src/tokens.nim

@@ -35,7 +35,7 @@ proc expired(token: Token): bool {.inline.} =
   result = token.init < getTime() - expirationTime
 
 proc isLimited(token: Token): bool {.inline.} =
-  token == nil or token.remaining <= 1 and token.reset > getTime() or
+  token == nil or (token.remaining <= 1 and token.reset > getTime()) or
     token.expired
 
 proc release*(token: Token) =

src/utils.nim

@@ -28,9 +28,6 @@ proc getVidUrl*(link: string): string =
     url = encodeUrl(link)
   &"/video/{sig}/{url}"
 
-proc getGifUrl*(link: string): string =
-  &"/gif/{encodeUrl(link)}"
-
 proc getPicUrl*(link: string): string =
   &"/pic/{encodeUrl(link)}"
 

src/views/rss.nimf

@@ -36,7 +36,7 @@
 <img src="https://${hostname}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
 #elif tweet.gif.isSome:
 #let thumb = &"https://{hostname}{getPicUrl(get(tweet.gif).thumb)}"
-#let url = &"https://{hostname}{getGifUrl(get(tweet.gif).url)}"
+#let url = &"https://{hostname}{getPicUrl(get(tweet.gif).url)}"
 <video poster="${thumb}" autoplay muted loop style="max-width:250px;">
   <source src="${url}" type="video/mp4"</source></video>
 #end if

src/views/tweet.nim

@@ -107,7 +107,7 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode =
     tdiv(class="gallery-gif", style={maxHeight: "unset"}):
       tdiv(class="attachment"):
         let thumb = getPicUrl(gif.thumb)
-        let url = getGifUrl(gif.url)
+        let url = getPicUrl(gif.url)
         if prefs.autoplayGifs:
           video(class="gif", poster=thumb, controls="", autoplay="", muted="", loop=""):
             source(src=url, `type`="video/mp4")