nitter/src/parser.nim

125 lines
3.9 KiB
Nim
Raw Normal View History

2019-06-24 03:14:14 +00:00
import xmltree, sequtils, strtabs, strutils, strformat, json
2019-06-23 23:34:30 +00:00
import nimquery
2019-06-20 14:16:20 +00:00
2019-06-24 06:07:36 +00:00
import ./types, ./parserutils, ./formatters
2019-06-20 14:16:20 +00:00
2019-06-21 00:15:46 +00:00
proc parsePopupProfile*(node: XmlNode): Profile =
2019-06-20 14:16:20 +00:00
let profile = node.querySelector(".profile-card")
2019-06-21 00:15:46 +00:00
if profile.isNil: return
result = Profile(
2019-06-24 00:09:32 +00:00
fullname: profile.getName(".fullname"),
username: profile.getUsername(".username"),
bio: profile.getBio(".bio"),
userpic: profile.getAvatar(".ProfileCard-avatarImage"),
verified: isVerified(profile),
protected: isProtected(profile),
banner: getBanner(profile)
2019-06-21 00:15:46 +00:00
)
2019-06-24 06:07:36 +00:00
2019-06-23 23:34:30 +00:00
result.getPopupStats(profile)
2019-06-20 14:16:20 +00:00
2019-06-21 00:15:46 +00:00
proc parseIntentProfile*(profile: XmlNode): Profile =
result = Profile(
2019-06-24 00:09:32 +00:00
fullname: profile.getName("a.fn.url.alternate-context"),
username: profile.getUsername(".nickname"),
bio: profile.getBio("p.note"),
userpic: profile.querySelector(".profile.summary").getAvatar("img.photo"),
verified: not profile.querySelector("li.verified").isNil,
protected: not profile.querySelector("li.protected").isNil,
banner: getBanner(profile)
2019-06-21 00:15:46 +00:00
)
2019-06-24 06:07:36 +00:00
2019-06-23 23:34:30 +00:00
result.getIntentStats(profile)
2019-06-21 00:15:46 +00:00
proc parseTweetProfile*(profile: XmlNode): Profile =
2019-06-20 14:16:20 +00:00
result = Profile(
2019-06-25 00:38:18 +00:00
fullname: profile.getAttr("data-name").stripText(),
2019-06-21 00:15:46 +00:00
username: profile.getAttr("data-screen-name"),
2019-06-23 23:34:30 +00:00
userpic: profile.getAvatar(".avatar"),
verified: isVerified(profile)
)
2019-06-24 06:07:36 +00:00
proc parseQuote*(quote: XmlNode): Quote =
result = Quote(
id: quote.getAttr("data-item-id"),
link: quote.getAttr("href"),
2019-06-25 02:52:38 +00:00
text: getQuoteText(quote)
2019-06-23 23:34:30 +00:00
)
result.profile = Profile(
2019-06-25 00:38:18 +00:00
fullname: quote.selectText(".QuoteTweet-fullname").stripText(),
2019-06-24 06:07:36 +00:00
username: quote.getAttr("data-screen-name"),
verified: isVerified(quote)
2019-06-20 14:16:20 +00:00
)
2019-06-24 06:07:36 +00:00
result.getQuoteMedia(quote)
2019-06-20 14:16:20 +00:00
proc parseTweet*(tweet: XmlNode): Tweet =
2019-06-21 00:30:57 +00:00
result = Tweet(
2019-06-23 23:34:30 +00:00
id: tweet.getAttr("data-item-id"),
link: tweet.getAttr("data-permalink-path"),
profile: parseTweetProfile(tweet),
text: getTweetText(tweet),
time: getTimestamp(tweet),
shortTime: getShortTime(tweet),
pinned: "pinned" in tweet.getAttr("class")
2019-06-21 00:30:57 +00:00
)
2019-06-20 14:16:20 +00:00
2019-06-23 23:34:30 +00:00
result.getTweetStats(tweet)
result.getTweetMedia(tweet)
2019-06-20 14:16:20 +00:00
2019-06-21 00:30:57 +00:00
let by = tweet.selectText(".js-retweet-text > a > b")
if by.len > 0:
2019-06-25 00:38:18 +00:00
result.retweetBy = some(by.stripText())
result.retweetId = some(tweet.getAttr("data-retweet-id"))
2019-06-21 00:30:57 +00:00
2019-06-24 06:07:36 +00:00
let quote = tweet.querySelector(".QuoteTweet-innerContainer")
if not quote.isNil:
result.quote = some(parseQuote(quote))
2019-06-20 14:16:20 +00:00
proc parseTweets*(node: XmlNode): Tweets =
if node.isNil: return
node.querySelectorAll(".tweet").map(parseTweet)
proc parseConversation*(node: XmlNode): Conversation =
2019-06-24 03:14:14 +00:00
result = Conversation(
tweet: parseTweet(node.querySelector(".permalink-tweet-container > .tweet")),
before: parseTweets(node.querySelector(".in-reply-to"))
)
2019-06-20 14:16:20 +00:00
let replies = node.querySelector(".replies-to")
if replies.isNil: return
2019-06-21 00:30:57 +00:00
result.after = parseTweets(replies.querySelector(".ThreadedConversation--selfThread"))
2019-06-20 14:16:20 +00:00
for reply in replies.querySelectorAll("li > .stream-items"):
let thread = parseTweets(reply)
if not thread.anyIt(it in result.after):
result.replies.add thread
2019-06-24 03:14:14 +00:00
proc parseVideo*(node: JsonNode): Video =
let track = node{"track"}
2019-06-25 05:37:44 +00:00
let contentType = track["contentType"].to(string)
case contentType
of "media_entity":
result = Video(
contentType: m3u8,
thumb: node["posterImage"].to(string),
id: track["contentId"].to(string),
length: track["durationMs"].to(int),
views: track["viewCount"].to(string),
url: track["playbackUrl"].to(string),
available: track{"mediaAvailability"}["status"].to(string) == "available"
)
of "vmap":
result = Video(
contentType: vmap,
thumb: node["posterImage"].to(string),
url: track["vmapUrl"].to(string),
length: track["durationMs"].to(int),
)
else:
echo "Can't parse video of type ", contentType