nitter/src/parser.nim

449 lines
14 KiB
Nim
Raw Normal View History

import strutils, options, tables, times, math
import packedjson
import packedjson / deserialiser
import types, parserutils, utils
2020-06-01 00:16:24 +00:00
proc parseProfile(js: JsonNode; id=""): Profile =
if js.isNull: return
2020-06-01 00:16:24 +00:00
result = Profile(
id: if id.len > 0: id else: js{"id_str"}.getStr,
username: js{"screen_name"}.getStr,
fullname: js{"name"}.getStr,
location: js{"location"}.getStr,
bio: js{"description"}.getStr,
userpic: js{"profile_image_url_https"}.getImageStr.replace("_normal", ""),
2020-06-01 00:16:24 +00:00
banner: js.getBanner,
following: $js{"friends_count"}.getInt,
followers: $js{"followers_count"}.getInt,
tweets: $js{"statuses_count"}.getInt,
likes: $js{"favourites_count"}.getInt,
media: $js{"media_count"}.getInt,
verified: js{"verified"}.getBool,
protected: js{"protected"}.getBool,
joinDate: js{"created_at"}.getTime
)
result.expandProfileEntities(js)
2020-06-17 18:46:45 +00:00
proc parseUserShow*(js: JsonNode; username: string): Profile =
if js.isNull:
return Profile(username: username)
2020-06-17 18:46:45 +00:00
with error, js{"errors"}:
result = Profile(username: username)
if error.getError == suspended:
result.suspended = true
return
result = parseProfile(js)
2020-06-01 00:16:24 +00:00
proc parseGraphProfile*(js: JsonNode; username: string): Profile =
if js.isNull: return
2020-06-01 07:45:38 +00:00
with error, js{"errors"}:
result = Profile(username: username)
2020-06-01 19:53:21 +00:00
if error.getError == suspended:
2020-06-01 07:45:38 +00:00
result.suspended = true
return
2020-06-01 00:16:24 +00:00
let user = js{"data", "user", "legacy"}
let id = js{"data", "user", "rest_id"}.getStr
2020-11-10 13:04:01 +00:00
result = parseProfile(user, id)
2020-06-01 00:16:24 +00:00
proc parseGraphList*(js: JsonNode): List =
if js.isNull: return
2020-06-01 00:16:24 +00:00
var list = js{"data", "user_by_screen_name", "list"}
if list.isNull:
2020-06-01 00:16:24 +00:00
list = js{"data", "list"}
if list.isNull:
2020-06-01 00:16:24 +00:00
return
result = List(
id: list{"id_str"}.getStr,
2020-11-10 11:53:56 +00:00
name: list{"name"}.getStr.replace(' ', '-'),
2020-06-01 00:16:24 +00:00
username: list{"user", "legacy", "screen_name"}.getStr,
userId: list{"user", "legacy", "id_str"}.getStr,
description: list{"description"}.getStr,
members: list{"member_count"}.getInt,
banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
2020-06-01 00:16:24 +00:00
)
proc parseListMembers*(js: JsonNode; cursor: string): Result[Profile] =
result = Result[Profile](
beginning: cursor.len == 0,
query: Query(kind: userList)
)
if js.isNull: return
2020-06-01 00:16:24 +00:00
result.top = js{"previous_cursor_str"}.getStr
result.bottom = js{"next_cursor_str"}.getStr
if result.bottom.len == 1:
result.bottom.setLen 0
for u in js{"users"}:
result.content.add parseProfile(u)
proc parsePoll(js: JsonNode): Poll =
let vals = js{"binding_values"}
# name format is pollNchoice_*
for i in '1' .. js{"name"}.getStr[4]:
let choice = "choice" & i
result.values.add parseInt(vals{choice & "_count"}.getStrVal("0"))
result.options.add vals{choice & "_label"}.getStrVal
let time = vals{"end_datetime_utc", "string_value"}.getDateTime
if time > getTime():
let timeLeft = $(time - getTime())
result.status = timeLeft[0 ..< timeLeft.find(",")]
else:
result.status = "Final results"
result.leader = result.values.find(max(result.values))
result.votes = result.values.sum
proc parseGif(js: JsonNode): Gif =
result = Gif(
url: js{"video_info", "variants"}[0]{"url"}.getImageStr,
thumb: js{"media_url_https"}.getImageStr
2020-06-01 00:16:24 +00:00
)
proc parseVideo(js: JsonNode): Video =
result = Video(
videoId: js{"id_str"}.getStr,
thumb: js{"media_url_https"}.getImageStr,
2020-06-01 00:16:24 +00:00
views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
available: js{"ext_media_availability", "status"}.getStr == "available",
title: js{"ext_alt_text"}.getStr,
2020-11-07 21:48:30 +00:00
durationMs: js{"duration_millis"}.getInt
# playbackType: mp4
2020-06-01 00:16:24 +00:00
)
2020-06-10 15:09:38 +00:00
with title, js{"additional_media_info", "title"}:
result.title = title.getStr
2020-06-01 00:16:24 +00:00
for v in js{"video_info", "variants"}:
result.variants.add VideoVariant(
videoType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
2020-06-01 00:16:24 +00:00
bitrate: v{"bitrate"}.getInt,
url: v{"url"}.getStr
)
proc parsePromoVideo(js: JsonNode): Video =
result = Video(
thumb: js{"player_image_large"}.getImageVal,
2020-06-01 00:16:24 +00:00
available: true,
durationMs: js{"content_duration_seconds"}.getStrVal("0").parseInt * 1000,
playbackType: vmap,
videoId: js{"player_content_id"}.getStrVal(js{"card_id"}.getStrVal(
js{"amplify_content_id"}.getStrVal())),
2020-06-01 00:16:24 +00:00
)
var variant = VideoVariant(
videoType: vmap,
url: js{"player_hls_url"}.getStrVal(js{"player_stream_url"}.getStrVal(
js{"amplify_url_vmap"}.getStrVal()))
2020-06-01 00:16:24 +00:00
)
if "m3u8" in variant.url:
variant.videoType = m3u8
result.playbackType = m3u8
2020-06-01 00:16:24 +00:00
result.variants.add variant
proc parseBroadcast(js: JsonNode): Card =
let image = js{"broadcast_thumbnail_large"}.getImageVal
2020-06-01 00:16:24 +00:00
result = Card(
kind: broadcast,
url: js{"broadcast_url"}.getStrVal,
title: js{"broadcaster_display_name"}.getStrVal,
text: js{"broadcast_title"}.getStrVal,
image: image,
video: some Video(videoId: js{"broadcast_media_id"}.getStrVal, thumb: image)
)
proc parseCard(js: JsonNode; urls: JsonNode): Card =
const imageTypes = ["summary_photo_image", "player_image", "promo_image",
"photo_image_full_size", "thumbnail_image", "thumbnail",
2020-06-10 14:13:40 +00:00
"event_thumbnail", "image"]
2020-06-01 00:16:24 +00:00
let
vals = ? js{"binding_values"}
name = js{"name"}.getStr
kind = parseEnum[CardKind](name[(name.find(":") + 1) ..< name.len], unknown)
2020-06-01 00:16:24 +00:00
result = Card(
kind: kind,
url: vals.getCardUrl(kind),
dest: vals.getCardDomain(kind),
title: vals.getCardTitle(kind),
text: vals{"description"}.getStrVal
)
if result.url.len == 0:
result.url = js{"url"}.getStr
case kind
2020-06-10 14:13:40 +00:00
of promoVideo, promoVideoConvo, appPlayer, videoDirectMessage:
2020-06-01 00:16:24 +00:00
result.video = some parsePromoVideo(vals)
if kind == appPlayer:
result.text = vals{"app_category"}.getStrVal(result.text)
2020-06-01 00:16:24 +00:00
of broadcast:
result = parseBroadcast(vals)
of liveEvent:
result.text = vals{"event_title"}.getStrVal
2020-06-01 00:16:24 +00:00
of player:
result.url = vals{"player_url"}.getStrVal
if "youtube.com" in result.url:
result.url = result.url.replace("/embed/", "/watch?v=")
of unified, unknown:
result.title = "This card type is not supported."
2020-06-01 00:16:24 +00:00
else: discard
for typ in imageTypes:
with img, vals{typ & "_large"}:
result.image = img.getImageVal
2020-06-01 00:16:24 +00:00
break
for u in ? urls:
if u{"url"}.getStr == result.url:
result.url = u{"expanded_url"}.getStr
break
2020-06-10 14:13:40 +00:00
if kind in {videoDirectMessage, imageDirectMessage}:
result.url.setLen 0
if kind in {promoImageConvo, promoImageApp, imageDirectMessage} and
result.url.len == 0 or result.url.startsWith("card://"):
result.url = getPicUrl(result.image)
2020-06-01 00:16:24 +00:00
proc parseTweet(js: JsonNode): Tweet =
if js.isNull: return
2020-06-01 00:16:24 +00:00
result = Tweet(
id: js{"id_str"}.getId,
threadId: js{"conversation_id_str"}.getId,
replyId: js{"in_reply_to_status_id_str"}.getId,
text: js{"full_text"}.getStr,
time: js{"created_at"}.getTime,
hasThread: js{"self_thread"}.notNull,
2020-06-01 00:16:24 +00:00
available: true,
profile: Profile(id: js{"user_id_str"}.getStr),
stats: TweetStats(
replies: js{"reply_count"}.getInt,
retweets: js{"retweet_count"}.getInt,
likes: js{"favorite_count"}.getInt,
quotes: js{"quote_count"}.getInt
2020-06-01 00:16:24 +00:00
)
)
result.expandTweetEntities(js)
if js{"is_quote_status"}.getBool:
result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
with rt, js{"retweeted_status_id_str"}:
result.retweet = some Tweet(id: rt.getId)
return
with jsCard, js{"card"}:
let name = jsCard{"name"}.getStr
if "poll" in name:
if "image" in name:
result.photos.add jsCard{"binding_values", "image_large"}.getImageVal
2020-06-01 00:16:24 +00:00
result.poll = some parsePoll(jsCard)
elif name == "amplify":
result.video = some(parsePromoVideo(jsCard{"binding_values"}))
2020-06-01 00:16:24 +00:00
else:
result.card = some parseCard(jsCard, js{"entities", "urls"})
with jsMedia, js{"extended_entities", "media"}:
for m in jsMedia:
case m{"type"}.getStr
of "photo":
result.photos.add m{"media_url_https"}.getImageStr
2020-06-01 00:16:24 +00:00
of "video":
result.video = some(parseVideo(m))
2020-06-10 15:05:55 +00:00
with user, m{"additional_media_info", "source_user"}:
result.attribution = some(parseProfile(user))
2020-06-01 00:16:24 +00:00
of "animated_gif":
result.gif = some(parseGif(m))
else: discard
let withheldInCountries = (
if js{"withheld_in_countries"}.kind == JArray:
js{"withheld_in_countries"}.to(seq[string])
else:
newSeq[string]()
)
if js{"withheld_copyright"}.getBool or
# XX - Content is withheld in all countries
"XX" in withheldInCountries or
# XY - Content is withheld due to a DMCA request.
"XY" in withheldInCountries or
(withheldInCountries.len > 0 and "withheld" in result.text):
result.available = false
2020-06-01 00:16:24 +00:00
proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
2020-06-04 03:03:25 +00:00
let intId = if id.len > 0: parseBiggestInt(id) else: 0
2020-06-01 00:16:24 +00:00
result = global.tweets.getOrDefault(id, Tweet(id: intId))
if result.quote.isSome:
let quote = get(result.quote).id
if $quote in global.tweets:
result.quote = some global.tweets[$quote]
else:
result.quote = some Tweet()
if result.retweet.isSome:
let rt = get(result.retweet).id
if $rt in global.tweets:
result.retweet = some finalizeTweet(global, $rt)
else:
result.retweet = some Tweet()
proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
let pin = js{"pinEntry", "entry", "entryId"}.getStr
if pin.len == 0: return
let id = pin.getId
if id notin global.tweets: return
global.tweets[id].pinned = true
return finalizeTweet(global, id)
proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result = GlobalObjects()
let
tweets = ? js{"globalObjects", "tweets"}
users = ? js{"globalObjects", "users"}
for k, v in users:
result.users[k] = parseProfile(v, k)
for k, v in tweets:
var tweet = parseTweet(v)
if tweet.profile.id in result.users:
tweet.profile = result.users[tweet.profile.id]
result.tweets[k] = tweet
proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
result.thread = Chain()
for t in js{"content", "timelineModule", "items"}:
let content = t{"item", "content"}
if "Self" in content{"tweet", "displayType"}.getStr:
result.self = true
let entry = t{"entryId"}.getStr
if "show_more" in entry:
let
cursor = content{"timelineCursor"}
more = cursor{"displayTreatment", "actionText"}.getStr
result.thread.cursor = cursor{"value"}.getStr
2020-10-01 20:08:25 +00:00
if more.len > 0 and more[0].isDigit():
result.thread.more = parseInt(more[0 ..< more.find(" ")])
else:
result.thread.more = -1
2020-06-01 00:16:24 +00:00
else:
2020-11-07 21:48:49 +00:00
var tweet = finalizeTweet(global, t.getEntryId)
2020-06-01 00:16:24 +00:00
if not tweet.available:
tweet.tombstone = getTombstone(content{"tombstone"})
result.thread.content.add tweet
proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))
let global = parseGlobalObjects(? js)
let instructions = ? js{"timeline", "instructions"}
if instructions.len == 0:
return
2020-06-01 00:16:24 +00:00
for e in instructions[0]{"addEntries", "entries"}:
let entry = e{"entryId"}.getStr
2020-11-07 21:48:49 +00:00
if "tweet" in entry or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId)
2020-06-01 00:16:24 +00:00
if $tweet.id != tweetId:
result.before.content.add tweet
else:
result.tweet = tweet
elif "conversationThread" in entry:
let (thread, self) = parseThread(e, global)
if thread.content.len > 0:
if self:
result.after = thread
else:
result.replies.content.add thread
elif "cursor-showMore" in entry:
result.replies.bottom = e.getCursor
elif "cursor-bottom" in entry:
result.replies.bottom = e.getCursor
2020-06-07 05:53:40 +00:00
proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
if js.kind != JArray or js.len == 0:
return
for i in js:
when T is Tweet:
if res.beginning and i{"pinEntry"}.notNull:
with pin, parsePin(i, global):
res.content.add pin
with r, i{"replaceEntry", "entry"}:
if "top" in r{"entryId"}.getStr:
res.top = r.getCursor
elif "bottom" in r{"entryId"}.getStr:
res.bottom = r.getCursor
2020-06-01 00:16:24 +00:00
proc parseUsers*(js: JsonNode; after=""): Result[Profile] =
result = Result[Profile](beginning: after.len == 0)
let global = parseGlobalObjects(? js)
let instructions = ? js{"timeline", "instructions"}
2020-06-04 22:11:05 +00:00
if instructions.len == 0: return
2020-06-07 05:53:40 +00:00
result.parseInstructions(global, instructions)
2020-06-04 22:11:05 +00:00
2020-06-01 00:16:24 +00:00
for e in instructions[0]{"addEntries", "entries"}:
let entry = e{"entryId"}.getStr
2021-12-19 20:28:13 +00:00
if "user-" in entry:
2020-06-01 00:16:24 +00:00
let id = entry.getId
if id in global.users:
result.content.add global.users[id]
elif "cursor-top" in entry:
result.top = e.getCursor
elif "cursor-bottom" in entry:
result.bottom = e.getCursor
proc parseTimeline*(js: JsonNode; after=""): Timeline =
result = Timeline(beginning: after.len == 0)
let global = parseGlobalObjects(? js)
let instructions = ? js{"timeline", "instructions"}
if instructions.len == 0: return
2020-06-07 05:53:40 +00:00
result.parseInstructions(global, instructions)
2020-06-01 00:16:24 +00:00
for e in instructions[0]{"addEntries", "entries"}:
let entry = e{"entryId"}.getStr
2020-11-07 21:48:49 +00:00
if "tweet" in entry or "sq-I-t" in entry or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId)
2020-06-01 00:16:24 +00:00
if not tweet.available: continue
result.content.add tweet
elif "cursor-top" in entry:
result.top = e.getCursor
elif "cursor-bottom" in entry:
result.bottom = e.getCursor
2020-06-07 05:54:20 +00:00
proc parsePhotoRail*(js: JsonNode): PhotoRail =
2020-06-16 22:20:34 +00:00
for tweet in js:
let
t = parseTweet(tweet)
url = if t.photos.len > 0: t.photos[0]
elif t.video.isSome: get(t.video).thumb
elif t.gif.isSome: get(t.gif).thumb
elif t.card.isSome: get(t.card).image
else: ""
if url.len == 0: continue
result.add GalleryPhoto(url: url, tweetId: $t.id)