Add list support

This commit is contained in:
Zed 2019-09-21 01:08:30 +02:00
parent d1fbcef64d
commit 9e3138e51b
25 changed files with 224 additions and 39 deletions

View file

@ -1,2 +1,2 @@
import api/[profile, timeline, tweet, search, media] import api/[profile, timeline, tweet, search, media, list]
export profile, timeline, tweet, search, media export profile, timeline, tweet, search, media, list

View file

@ -11,6 +11,8 @@ const
timelineUrl* = "i/profiles/show/$1/timeline/tweets" timelineUrl* = "i/profiles/show/$1/timeline/tweets"
timelineMediaUrl* = "i/profiles/show/$1/media_timeline" timelineMediaUrl* = "i/profiles/show/$1/media_timeline"
listUrl* = "$1/lists/$2/timeline"
listMembersUrl* = "$1/lists/$2/members"
profilePopupUrl* = "i/profiles/popup" profilePopupUrl* = "i/profiles/popup"
profileIntentUrl* = "intent/user" profileIntentUrl* = "intent/user"
searchUrl* = "i/search/timeline" searchUrl* = "i/search/timeline"

83
src/api/list.nim Normal file
View file

@ -0,0 +1,83 @@
import httpclient, asyncdispatch, htmlparser, strformat
import sequtils, strutils, json, uri
import ".."/[types, parser, parserutils, query]
import utils, consts, timeline, search
proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} =
let url = base / (listUrl % [username, list])
let headers = newHttpHeaders({
"Accept": jsonAccept,
"Referer": $url,
"User-Agent": agent,
"X-Twitter-Active-User": "yes",
"X-Requested-With": "XMLHttpRequest",
"Accept-Language": lang
})
var params = toSeq({
"include_available_features": "1",
"include_entities": "1",
"reset_error_state": "false"
})
if after.len > 0:
params.add {"max_position": after}
let json = await fetchJson(url ? params, headers)
result = await finishTimeline(json, Query(), after, agent)
if result.content.len > 0:
result.minId = result.content[^1].id
proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
let url = base / (listMembersUrl % [username, list])
let headers = newHttpHeaders({
"Accept": htmlAccept,
"Referer": $(base / &"{username}/lists/{list}/members"),
"User-Agent": agent,
"Accept-Language": lang
})
let html = await fetchHtml(url, headers)
result = Result[Profile](
minId: html.selectAttr(".stream-container", "data-min-position"),
hasMore: html.select(".has-more-items") != nil,
beginning: true,
query: Query(kind: users),
content: html.selectAll(".account").map(parseListProfile)
)
proc getListMembersSearch*(username, list, agent, after: string): Future[Result[Profile]] {.async.} =
let url = base / ((listMembersUrl & "/timeline") % [username, list])
let headers = newHttpHeaders({
"Accept": jsonAccept,
"Referer": $(base / &"{username}/lists/{list}/members"),
"User-Agent": agent,
"X-Twitter-Active-User": "yes",
"X-Requested-With": "XMLHttpRequest",
"X-Push-With": "XMLHttpRequest",
"Accept-Language": lang
})
var params = toSeq({
"include_available_features": "1",
"include_entities": "1",
"reset_error_state": "false"
})
if after.len > 0:
params.add {"max_position": after}
let json = await fetchJson(url ? params, headers)
result = getResult[Profile](json, Query(kind: users), after)
if json == nil or not json.hasKey("items_html"): return
let html = json["items_html"].to(string)
result.hasMore = html != "\n"
for p in parseHtml(html).selectAll(".account"):
result.content.add parseListProfile(p)

View file

@ -7,7 +7,7 @@ import utils, consts, timeline
proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] = proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
if json == nil: return Result[T](beginning: true, query: query) if json == nil: return Result[T](beginning: true, query: query)
Result[T]( Result[T](
hasMore: json["has_more_items"].to(bool), hasMore: json.getOrDefault("has_more_items").getBool(false),
maxId: json.getOrDefault("max_position").getStr(""), maxId: json.getOrDefault("max_position").getStr(""),
minId: json.getOrDefault("min_position").getStr("").cleanPos(), minId: json.getOrDefault("min_position").getStr("").cleanPos(),
query: query, query: query,
@ -16,7 +16,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} = proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
let let
kind = if query.kind == users: "users" else: "tweets" kind = if query.kind == userSearch: "users" else: "tweets"
pos = when T is Tweet: genPos(after) else: after pos = when T is Tweet: genPos(after) else: after
param = genQueryParam(query) param = genQueryParam(query)
@ -46,10 +46,9 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn
return Result[T](query: query, beginning: true) return Result[T](query: query, beginning: true)
let json = await fetchJson(base / searchUrl ? params, headers) let json = await fetchJson(base / searchUrl ? params, headers)
if json == nil: return Result[T](query: query, beginning: true)
result = getResult[T](json, query, after) result = getResult[T](json, query, after)
if not json.hasKey("items_html"): return if json == nil or not json.hasKey("items_html"): return
when T is Tweet: when T is Tweet:
result = await finishTimeline(json, query, after, agent) result = await finishTimeline(json, query, after, agent)

View file

@ -1,4 +1,4 @@
import httpclient, asyncdispatch, htmlparser import httpclient, asyncdispatch, htmlparser, strformat
import sequtils, strutils, json, xmltree, uri import sequtils, strutils, json, xmltree, uri
import ".."/[types, parser, parserutils, formatters, query] import ".."/[types, parser, parserutils, formatters, query]

View file

@ -5,13 +5,14 @@ import jester
import types, config, prefs import types, config, prefs
import views/[general, about] import views/[general, about]
import routes/[preferences, timeline, status, media, search, rss] import routes/[preferences, timeline, status, media, search, rss, list]
const configPath {.strdefine.} = "./nitter.conf" const configPath {.strdefine.} = "./nitter.conf"
let cfg = getConfig(configPath) let cfg = getConfig(configPath)
createPrefRouter(cfg) createPrefRouter(cfg)
createTimelineRouter(cfg) createTimelineRouter(cfg)
createListRouter(cfg)
createStatusRouter(cfg) createStatusRouter(cfg)
createSearchRouter(cfg) createSearchRouter(cfg)
createMediaRouter(cfg) createMediaRouter(cfg)
@ -24,15 +25,16 @@ settings:
routes: routes:
get "/": get "/":
resp renderMain(renderSearch(), Prefs(), cfg.title) resp renderMain(renderSearch(), request, cfg.title)
get "/about": get "/about":
resp renderMain(renderAbout(), Prefs(), cfg.title) resp renderMain(renderAbout(), request, cfg.title)
extend preferences, "" extend preferences, ""
extend rss, "" extend rss, ""
extend search, "" extend search, ""
extend timeline, "" extend timeline, ""
extend list, ""
extend status, "" extend status, ""
extend media, "" extend media, ""

View file

@ -39,6 +39,16 @@ proc parsePopupProfile*(node: XmlNode; selector=".profile-card"): Profile =
result.getPopupStats(profile) result.getPopupStats(profile)
proc parseListProfile*(profile: XmlNode): Profile =
result = Profile(
fullname: profile.getName(".fullname"),
username: profile.getUsername(".username"),
bio: profile.getBio(".bio"),
userpic: profile.getAvatar(".avatar"),
verified: isVerified(profile),
protected: isProtected(profile),
)
proc parseIntentProfile*(profile: XmlNode): Profile = proc parseIntentProfile*(profile: XmlNode): Profile =
result = Profile( result = Profile(
fullname: profile.getName("a.fn.url.alternate-context"), fullname: profile.getName("a.fn.url.alternate-context"),

View file

@ -58,7 +58,7 @@ proc genQueryParam*(query: Query): string =
var filters: seq[string] var filters: seq[string]
var param: string var param: string
if query.kind == users: if query.kind == userSearch:
return query.text return query.text
for i, user in query.fromUser: for i, user in query.fromUser:
@ -84,7 +84,7 @@ proc genQueryParam*(query: Query): string =
result &= " " & query.text result &= " " & query.text
proc genQueryUrl*(query: Query): string = proc genQueryUrl*(query: Query): string =
if query.kind notin {custom, users}: return if query.kind notin {custom, userSearch}: return
var params = @[&"kind={query.kind}"] var params = @[&"kind={query.kind}"]
if query.text.len > 0: if query.text.len > 0:

34
src/routes/list.nim Normal file
View file

@ -0,0 +1,34 @@
import strutils
import jester
import router_utils
import ".."/[query, types, api, agents]
import ../views/[general, timeline, list]
template respList*(list, timeline: typed) =
if list.minId.len == 0:
resp Http404, showError("List \"" & @"list" & "\" not found", cfg.title)
let html = renderList(timeline, list.query, @"name", @"list")
let rss = "/$1/lists/$2/rss" % [@"name", @"list"]
resp renderMain(html, request, cfg.title, rss=rss)
proc createListRouter*(cfg: Config) =
router list:
get "/@name/lists/@list":
cond '.' notin @"name"
let
list = await getListTimeline(@"name", @"list", getAgent(), @"after")
tweets = renderTimelineTweets(list, cookiePrefs(), request.path)
respList list, tweets
get "/@name/lists/@list/members":
cond '.' notin @"name"
let list =
if @"after".len == 0:
await getListMembers(@"name", @"list", getAgent())
else:
await getListMembersSearch(@"name", @"list", getAgent(), @"after")
let users = renderTimelineUsers(list, cookiePrefs(), request.path)
respList list, users

View file

@ -3,7 +3,7 @@ import asyncfile, uri, strutils, httpclient, os
import jester, regex import jester, regex
import router_utils import router_utils
import ".."/[types, formatters, prefs] import ".."/[types, formatters]
import ../views/general import ../views/general
export asyncfile, httpclient, os, strutils export asyncfile, httpclient, os, strutils

View file

@ -3,7 +3,7 @@ import strutils, uri
import jester import jester
import router_utils import router_utils
import ".."/[prefs, types] import ".."/[types]
import ../views/[general, preferences] import ../views/[general, preferences]
export preferences export preferences

View file

@ -1,5 +1,5 @@
import ../utils import ../utils, ../prefs
export utils export utils, prefs
template cookiePrefs*(): untyped {.dirty.} = template cookiePrefs*(): untyped {.dirty.} =
getPrefs(request.cookies.getOrDefault("preferences")) getPrefs(request.cookies.getOrDefault("preferences"))

View file

@ -34,3 +34,8 @@ proc createRssRouter*(cfg: Config) =
get "/@name/search/rss": get "/@name/search/rss":
cond '.' notin @"name" cond '.' notin @"name"
respRss(await showRss(@"name", initQuery(params(request), name=(@"name")))) respRss(await showRss(@"name", initQuery(params(request), name=(@"name"))))
get "/@name/lists/@list/rss":
cond '.' notin @"name"
let list = await getListTimeline(@"name", @"list", getAgent(), "")
respRss(renderListRss(list.content, @"name", @"list"))

View file

@ -3,7 +3,7 @@ import strutils, sequtils, uri
import jester import jester
import router_utils import router_utils
import ".."/[query, types, api, agents, prefs] import ".."/[query, types, api, agents]
import ../views/[general, search] import ../views/[general, search]
export search export search
@ -18,7 +18,7 @@ proc createSearchRouter*(cfg: Config) =
let query = initQuery(params(request)) let query = initQuery(params(request))
case query.kind case query.kind
of users: of userSearch:
if "," in @"text": if "," in @"text":
redirect("/" & @"text") redirect("/" & @"text")
let users = await getSearch[Profile](query, @"after", getAgent()) let users = await getSearch[Profile](query, @"after", getAgent())

View file

@ -3,7 +3,7 @@ import asyncdispatch, strutils, sequtils, uri
import jester import jester
import router_utils import router_utils
import ".."/[api, prefs, types, formatters, agents] import ".."/[api, types, formatters, agents]
import ../views/[general, status] import ../views/[general, status]
export uri, sequtils export uri, sequtils

View file

@ -3,7 +3,7 @@ import asyncdispatch, strutils, sequtils, uri
import jester import jester
import router_utils import router_utils
import ".."/[api, prefs, types, cache, formatters, agents, query] import ".."/[api, types, cache, formatters, agents, query]
import ../views/[general, profile, timeline, status, search] import ../views/[general, profile, timeline, status, search]
export uri, sequtils export uri, sequtils

View file

@ -85,7 +85,10 @@
.replying-to { .replying-to {
color: $fg_dark; color: $fg_dark;
margin: -2px 0 4px; margin: -2px 0 4px;
pointer-events: all;
a {
pointer-events: all;
}
} }
.retweet, .pinned, .tweet-stats { .retweet, .pinned, .tweet-stats {

View file

@ -57,7 +57,7 @@ dbFromTypes("cache.db", "", "", "", [Profile, Video])
type type
QueryKind* = enum QueryKind* = enum
posts, replies, media, users, custom posts, replies, media, users, userSearch, custom
Query* = object Query* = object
kind*: QueryKind kind*: QueryKind

View file

@ -41,7 +41,7 @@ proc cleanFilename*(filename: string): string =
filename.replace(reg, "_") filename.replace(reg, "_")
proc filterParams*(params: Table): seq[(string, string)] = proc filterParams*(params: Table): seq[(string, string)] =
let filter = ["name", "id"] let filter = ["name", "id", "list"]
toSeq(params.pairs()).filterIt(it[0] notin filter and it[1].len > 0) toSeq(params.pairs()).filterIt(it[0] notin filter and it[1].len > 0)
proc isTwitterUrl*(url: string): bool = proc isTwitterUrl*(url: string): bool =

View file

@ -71,5 +71,5 @@ proc renderError*(error: string): VNode =
tdiv(class="error-panel"): tdiv(class="error-panel"):
span: text error span: text error
proc showError*(error, title: string): string = template showError*(error, title: string): string =
renderMain(renderError(error), Request(), title, "Error") renderMain(renderError(error), request, title, "Error")

20
src/views/list.nim Normal file
View file

@ -0,0 +1,20 @@
import strformat
import karax/[karaxdsl, vdom]
import renderutils
import ".."/[types]
proc renderListTabs*(query: Query; path: string): VNode =
buildHtml(ul(class="tab")):
li(class=query.getTabClass(posts)):
a(href=(path)): text "Tweets"
li(class=query.getTabClass(users)):
a(href=(path & "/members")): text "Members"
proc renderList*(body: VNode; query: Query; name, list: string): VNode =
buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header"):
text &"\"{list}\" by @{name}"
renderListTabs(query, &"/{name}/lists/{list}")
body

View file

@ -30,10 +30,6 @@ proc linkUser*(profile: Profile, class=""): VNode =
text " " text " "
icon "lock-circled", title="Protected account" icon "lock-circled", title="Protected account"
proc genImg*(url: string; class=""): VNode =
buildHtml():
img(src=getPicUrl(url), class=class, alt="Image")
proc linkText*(text: string; class=""): VNode = proc linkText*(text: string; class=""): VNode =
let url = if "http" notin text: "http://" & text else: text let url = if "http" notin text: "http://" & text else: text
buildHtml(): buildHtml():
@ -91,3 +87,12 @@ proc genDate*(pref, state: string): VNode =
else: else:
verbatim &"<input name={pref} type=\"date\"/>" verbatim &"<input name={pref} type=\"date\"/>"
icon "calendar" icon "calendar"
proc genImg*(url: string; class=""): VNode =
buildHtml():
img(src=getPicUrl(url), class=class, alt="Image")
proc getTabClass*(query: Query; tab: QueryKind): string =
result = "tab-item"
if query.kind == tab:
result &= " active"

View file

@ -71,3 +71,29 @@
</channel> </channel>
</rss> </rss>
#end proc #end proc
#
#proc renderListRss*(tweets: seq[Tweet]; name, list: string): string =
#let prefs = Prefs(replaceTwitter: hostname)
#result = ""
<?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
<channel>
<atom:link href="https://${hostname}/${name}/lists/${list}/rss" rel="self" type="application/rss+xml" />
<title>${list} / @${name}</title>
<link>https://${hostname}/${name}/lists/${list}</link>
<description>Twitter feed for: ${list} by @${name}. Generated by ${hostname}</description>
<language>en-us</language>
<ttl>40</ttl>
#for tweet in tweets:
<item>
<title>${getTitle(tweet, prefs)}</title>
<dc:creator>@${tweet.profile.username}</dc:creator>
<description><![CDATA[${renderRssTweet(tweet, prefs).strip(chars={'\n'})}]]></description>
<pubDate>${getRfc822Time(tweet)}</pubDate>
<guid>https://${hostname}${getLink(tweet)}</guid>
<link>https://${hostname}${getLink(tweet)}</link>
</item>
#end for
</channel>
</rss>
#end proc

View file

@ -23,15 +23,10 @@ proc renderSearch*(): VNode =
buildHtml(tdiv(class="panel-container")): buildHtml(tdiv(class="panel-container")):
tdiv(class="search-bar"): tdiv(class="search-bar"):
form(`method`="get", action="/search"): form(`method`="get", action="/search"):
hiddenField("kind", "users") hiddenField("kind", "userSearch")
input(`type`="text", name="text", autofocus="", placeholder="Enter username...") input(`type`="text", name="text", autofocus="", placeholder="Enter username...")
button(`type`="submit"): icon "search" button(`type`="submit"): icon "search"
proc getTabClass(query: Query; tab: QueryKind): string =
result = "tab-item"
if query.kind == tab:
result &= " active"
proc renderProfileTabs*(query: Query; username: string): VNode = proc renderProfileTabs*(query: Query; username: string): VNode =
let link = "/" & username let link = "/" & username
buildHtml(ul(class="tab")): buildHtml(ul(class="tab")):
@ -50,8 +45,8 @@ proc renderSearchTabs*(query: Query): VNode =
li(class=query.getTabClass(custom)): li(class=query.getTabClass(custom)):
q.kind = custom q.kind = custom
a(href=("?" & genQueryUrl(q))): text "Tweets" a(href=("?" & genQueryUrl(q))): text "Tweets"
li(class=query.getTabClass(users)): li(class=query.getTabClass(userSearch)):
q.kind = users q.kind = userSearch
a(href=("?" & genQueryUrl(q))): text "Users" a(href=("?" & genQueryUrl(q))): text "Users"
proc isPanelOpen(q: Query): bool = proc isPanelOpen(q: Query): bool =
@ -114,7 +109,7 @@ proc renderUserSearch*(users: Result[Profile]; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-container")): buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header"): tdiv(class="timeline-header"):
form(`method`="get", action="/search", class="search-field"): form(`method`="get", action="/search", class="search-field"):
hiddenField("kind", "users") hiddenField("kind", "userSearch")
genInput("text", "", users.query.text, "Enter username...", class="pref-inline") genInput("text", "", users.query.text, "Enter username...", class="pref-inline")
button(`type`="submit"): icon "search" button(`type`="submit"): icon "search"

View file

@ -64,7 +64,8 @@ proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs; path=""): VNod
if results.content.len > 0: if results.content.len > 0:
for user in results.content: for user in results.content:
renderUser(user, prefs) renderUser(user, prefs)
renderMore(results.query, results.minId) if results.minId != "0":
renderMore(results.query, results.minId)
elif results.beginning: elif results.beginning:
renderNoneFound() renderNoneFound()
else: else: