Use media endpoint for profile media tab

This bypasses "search" rate limits. It now includes media beyond
images and videos (eg. YouTube links are "media"), but the old
behaviour can be restored by clicking search, then filtering "Media"
and excluding retweets and replies.

src/api/list.nim

@@ -21,10 +21,7 @@ proc getListTimeline*(username, list, agent, after: string; media=true): Future[
   if result.content.len == 0:
     return
 
-  let last = result.content[^1]
+  result.minId = getLastId(result)
-  result.minId =
-    if last.retweet.isNone: $last.id
-    else: $(get(last.retweet).id)
 
 proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
   let

src/api/media.nim

@@ -123,9 +123,10 @@ proc getCard*(tweet: Tweet; agent: string) {.async.} =
   if html == nil: return
   parseCard(get(tweet.card), html)
 
-proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} =
+proc getPhotoRail*(username, agent: string; skip=false): Future[seq[GalleryPhoto]] {.async.} =
+  if skip: return
   let
-    headers = genHeaders({"x-requested-with": "XMLHttpRequest"}, agent, base / username)
+    headers = genHeaders(agent, base / username, xml=true)
     params = {"for_photo_rail": "true", "oldest_unread_id": "0"}
     url = base / (timelineMediaUrl % username) ? params
     html = await fetchHtml(url, headers, jsonKey="items_html")

src/api/timeline.nim

@@ -18,10 +18,24 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string;
   if not json.hasKey("items_html"): return
 
   let html = parseHtml(json["items_html"].to(string))
-  let thread = parseChain(html)
+  let timeline = parseChain(html)
 
-  if media: await getMedia(thread, agent)
+  if media: await getMedia(timeline, agent)
-  result.content = thread.content
+  result.content = timeline.content
+
+proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} =
+  var url = base / username
+  if after.len > 0:
+    url = url ? {"max_position": after}
+
+  let
+    headers = genHeaders(agent, base / username, auth=true)
+    html = await fetchHtml(url, headers)
+    timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
+    profile = parseTimelineProfile(html)
+
+  if media: await getMedia(timeline, agent)
+  result = (profile, timeline)
 
 proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
   var params = toSeq({
@@ -39,16 +53,19 @@ proc getTimeline*(username, after, agent: string; media=true): Future[Timeline]
 
   result = await finishTimeline(json, Query(), after, agent, media)
 
-proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} =
+proc getMediaTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
-  var url = base / username
+  echo "mediaTimeline"
+  var params = toSeq({
+    "include_available_features": "1",
+    "include_entities": "1",
+    "reset_error_state": "false"
+  })
+
   if after.len > 0:
-    url = url ? {"max_position": after}
+    params.add {"max_position": after}
 
-  let
+  let headers = genHeaders(agent, base / username, xml=true)
-    headers = genHeaders(agent, base / username, auth=true)
+  let json = await fetchJson(base / (timelineMediaUrl % username) ? params, headers)
-    html = await fetchHtml(url, headers)
-    timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
-    profile = parseTimelineProfile(html)
 
-  if media: await getMedia(timeline, agent)
+  result = await finishTimeline(json, Query(kind: QueryKind.media), after, agent, media)
-  result = (profile, timeline)
+  result.minId = getLastId(result)

src/api/utils.nim

@@ -1,6 +1,7 @@
 import httpclient, asyncdispatch, htmlparser
 import strutils, json, xmltree, uri
 
+import ../types
 import consts
 
 proc genHeaders*(headers: openArray[tuple[key: string, val: string]];
@@ -52,3 +53,11 @@ proc fetchJson*(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
     result = parseJson(resp)
   except:
     return nil
+
+proc getLastId*(tweets: Result[Tweet]): string =
+  if tweets.content.len == 0: return
+  let last = tweets.content[^1]
+  if last.retweet.isNone:
+    $last.id
+  else:
+    $(get(last.retweet).id)

src/routes/rss.nim

@@ -9,7 +9,7 @@ import ../views/general
 include "../views/rss.nimf"
 
 proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} =
-  let (profile, timeline, _) =
+  let (profile, timeline) =
     await fetchSingleTimeline(name, "", getAgent(), query, media=false)
 
   if timeline != nil:

src/routes/timeline.nim

@@ -11,12 +11,8 @@ export router_utils
 export api, cache, formatters, query, agents
 export profile, timeline, status
 
-type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
-
 proc fetchSingleTimeline*(name, after, agent: string; query: Query;
-                          media=true): Future[ProfileTimeline] {.async.} =
+                          media=true): Future[(Profile, Timeline)] {.async.} =
-  let railFut = getPhotoRail(name, agent)
-
   var timeline: Timeline
   var profile: Profile
   var cachedProfile = hasCachedProfile(name)
@@ -31,13 +27,17 @@ proc fetchSingleTimeline*(name, after, agent: string; query: Query;
       (profile, timeline) = await getProfileAndTimeline(name, agent, after, media)
       cache(profile)
   else:
-    var timelineFut = getSearch[Tweet](query, after, agent, media)
+    var timelineFut =
+      if query.kind == QueryKind.media:
+        getMediaTimeline(name, after, agent, media)
+      else:
+        getSearch[Tweet](query, after, agent, media)
     if cachedProfile.isNone:
       profile = await getCachedProfile(name, agent)
     timeline = await timelineFut
 
   if profile.username.len == 0: return
-  return (profile, timeline, await railFut)
+  return (profile, timeline)
 
 proc fetchMultiTimeline*(names: seq[string]; after, agent: string;
                          query: Query): Future[Timeline] {.async.} =
@@ -60,7 +60,10 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; rss: string): Fu
     names = name.strip(chars={'/'}).split(",").filterIt(it.len > 0)
 
   if names.len == 1:
-    let (p, t, r) = await fetchSingleTimeline(names[0], after, agent, query)
+    let
+      rail = getPhotoRail(names[0], agent, skip=(query.kind == media))
+      (p, t) = await fetchSingleTimeline(names[0], after, agent, query)
+      r = await rail
     if p.username.len == 0: return
     let pHtml = renderProfile(p, t, r, prefs, getPath())
     return renderMain(pHtml, request, cfg, pageTitle(p), pageDesc(p),