import xmltree, strtabs, strformat, strutils, times, uri, options, json import regex import types, formatters from q import nil from htmlgen import a const thumbRegex = re".+:url\('([^']+)'\)" gifRegex = re".+thumb/([^\.']+)\.[jpng].*" proc selectAll*(node: XmlNode; selector: string): seq[XmlNode] = if node == nil: return q.select(node, selector) proc select*(node: XmlNode; selector: string): XmlNode = if node == nil: return let nodes = node.selectAll(selector) if nodes.len > 0: nodes[0] else: nil proc selectAttr*(node: XmlNode; selector: string; attr: string): string = let res = node.select(selector) if res == nil: "" else: res.attr(attr) proc selectText*(node: XmlNode; selector: string): string = let res = node.select(selector) result = if res == nil: "" else: res.innerText() proc getHeader(profile: XmlNode): XmlNode = result = profile.select(".permalink-header") if result == nil: result = profile.select(".stream-item-header") if result == nil: result = profile.select(".ProfileCard-userFields") if result == nil: result = profile proc isVerified*(profile: XmlNode): bool = getHeader(profile).select(".Icon.Icon--verified") != nil proc isProtected*(profile: XmlNode): bool = getHeader(profile).select(".Icon.Icon--protected") != nil proc parseText*(text: XmlNode; skipLink=""): string = if text == nil: return for el in text: case el.kind of xnText: result.add el of xnElement: if el.attrs == nil: if el.tag == "strong": result.add $el continue let class = el.attr("class") if "data-expanded-url" in el.attrs: let url = el.attr("data-expanded-url") if url == skipLink: continue if "u-hidden" in class and result.len > 0: result.add "\n" result.add a(shortLink(url), href=url) elif "ashtag" in class or "hashflag" in class: let hash = el.innerText() result.add a(hash, href=("/search?q=" & encodeUrl(hash))) elif "atreply" in class: result.add a(el.innerText(), href=el.attr("href")) elif "Emoji" in class: result.add el.attr("alt") else: discard proc getQuoteText*(tweet: XmlNode): string = parseText(tweet.select(".QuoteTweet-text")) proc getTweetText*(tweet: XmlNode): string = let quote = tweet.select(".QuoteTweet") text = tweet.select(".tweet-text") link = text.selectAttr("a.twitter-timeline-link.u-hidden", "data-expanded-url") parseText(text, if quote != nil: link else: "") proc getTimestamp*(tweet: XmlNode): Time = let time = tweet.selectAttr(".js-short-timestamp", "data-time") fromUnix(if time.len > 0: parseBiggestInt(time) else: 0) proc getShortTime*(tweet: XmlNode): string = tweet.selectText(".js-short-timestamp") proc getDate*(node: XmlNode; selector: string): Time = let date = node.select(selector) if date == nil: return parseTime(date.attr("title"), "h:mm tt - d MMM YYYY", utc()) proc getName*(profile: XmlNode; selector: string): string = profile.selectText(selector).stripText() proc getUsername*(profile: XmlNode; selector: string): string = profile.selectText(selector).strip(chars={'@', ' ', '\n'}) proc getBio*(profile: XmlNode; selector: string; fallback=""): string = var bio = profile.select(selector) if bio == nil and fallback.len > 0: bio = profile.select(fallback) parseText(bio) proc getLocation*(profile: XmlNode): string = let sel = ".ProfileHeaderCard-locationText" result = profile.selectText(sel).stripText() let link = profile.selectAttr(sel & " a", "data-place-id") if link.len > 0: result &= ":" & link proc getAvatar*(profile: XmlNode; selector: string): string = profile.selectAttr(selector, "src").getUserpic() proc getBanner*(node: XmlNode): string = let url = node.selectAttr("svg > image", "xlink:href") if url.len > 0: result = url.replace("600x200", "1500x500") else: result = node.selectAttr(".ProfileCard-bg", "style") result = result.replace("background-color: ", "") if result.len == 0: result = "#161616" proc getTimelineBanner*(node: XmlNode): string = let banner = node.select(".ProfileCanopy-headerBg img") let img = banner.attr("src") if img.len > 0: return img let style = node.select("style").innerText() var m: RegexMatch if style.find(re"a:active \{\n +color: (#[A-Z0-9]+)", m): return style[m.group(0)[0]] proc getMediaCount*(node: XmlNode): string = let text = node.selectText(".PhotoRail-headingWithCount") return text.stripText().split(" ")[0] proc getProfileStats*(profile: var Profile; node: XmlNode) = for s in node.selectAll( ".ProfileNav-stat"): let text = s.attr("title").split(" ")[0] case s.attr("data-nav") of "followers": profile.followers = text of "following": profile.following = text of "favorites": profile.likes = text of "tweets": profile.tweets = text proc getPopupStats*(profile: var Profile; node: XmlNode) = for s in node.selectAll( ".ProfileCardStats-statLink"): let text = s.attr("title").split(" ")[0] case s.attr("href").split("/")[^1] of "followers": profile.followers = text of "following": profile.following = text else: profile.tweets = text proc getIntentStats*(profile: var Profile; node: XmlNode) = profile.tweets = "?" for s in node.selectAll( "dd.count > a"): let text = s.innerText() case s.attr("href").split("/")[^1] of "followers": profile.followers = text of "following": profile.following = text proc parseTweetStats*(node: XmlNode): TweetStats = result = TweetStats(replies: "0", retweets: "0", likes: "0") for action in node.selectAll(".ProfileTweet-actionCountForAria"): let text = action.innerText.split() case text[1][0 .. 2] of "ret": result.retweets = text[0] of "rep": result.replies = text[0] of "lik": result.likes = text[0] proc parseTweetReply*(node: XmlNode): seq[string] = let reply = node.select(".ReplyingToContextBelowAuthor") if reply == nil: return let selector = if "Quote" in node.attr("class"): "b" else: "a b" for username in reply.selectAll(selector): result.add username.innerText() proc getGif(player: XmlNode): Gif = let thumb = player.attr("style").replace(thumbRegex, "$1") id = thumb.replace(gifRegex, "$1") url = &"https://video.twimg.com/tweet_video/{id}.mp4" Gif(url: url, thumb: thumb) proc getTweetMedia*(tweet: Tweet; node: XmlNode) = for photo in node.selectAll(".AdaptiveMedia-photoContainer"): tweet.photos.add photo.attrs["data-image-url"] let player = node.select(".PlayableMedia") if player == nil: return let attrib = player.select(".PlayableMedia-attribution") if attrib != nil: tweet.attribution = some Profile( username: attrib.attr("href").strip(chars={'/'}), fullname: attrib.selectText(".fullname"), userpic: attrib.selectAttr(".avatar", "src") ) if "gif" in player.attr("class"): tweet.gif = some getGif(player.select(".PlayableMedia-player")) elif "video" in player.attr("class"): let thumb = player.selectAttr(".PlayableMedia-player", "style").split("'") desc = player.selectText(".PlayableMedia-description") title = player.selectText(".PlayableMedia-title") var video = Video(title: title, description: desc) if thumb.len > 1: video.thumb = thumb[^2] tweet.video = some video proc getQuoteMedia*(quote: var Quote; node: XmlNode) = if node.select(".QuoteTweet--sensitive") != nil: quote.sensitive = true return let media = node.select(".QuoteMedia") if media != nil: quote.thumb = media.selectAttr("img", "src") let badge = node.select(".AdaptiveMedia-badgeText") let gifBadge = node.select(".Icon--gifBadge") if badge != nil: quote.badge = badge.innerText() elif gifBadge != nil: quote.badge = "GIF" proc getTweetCard*(tweet: Tweet; node: XmlNode) = if node.attr("data-has-cards") == "false": return var cardType = node.attr("data-card2-type") if ":" in cardType: cardType = cardType.split(":")[^1] if "poll" in cardType: tweet.poll = some Poll() return if "message_me" in cardType: return let cardDiv = node.select(".card2 > .js-macaw-cards-iframe-container") if cardDiv == nil: return var card = Card( id: $tweet.id, query: cardDiv.attr("data-src") ) try: card.kind = parseEnum[CardKind](cardType) except ValueError: card.kind = summary let cardUrl = cardDiv.attr("data-card-url") for n in node.selectAll(".tweet-text a"): if n.attr("href") == cardUrl: card.url = n.attr("data-expanded-url") tweet.card = some card proc getMoreReplies*(node: XmlNode): int64 = let text = node.innerText().strip() try: result = parseBiggestInt(text.split(" ")[0]) except: result = -1 proc getMediaTags*(node: XmlNode): seq[Profile] = let usernames = node.attr("data-tagged") if usernames.len == 0: return let users = parseJson(node.attr("data-reply-to-users-json")) for user in users: let un = user["screen_name"].getStr if un notin usernames: continue result.add Profile(username: un, fullname: user["name"].getStr) proc getTweetLocation*(node: XmlNode): string = let geo = node.select(".js-geo-pivot-link") if geo == nil: return result = geo.innerText().stripText() result &= ":" & geo.attr("data-place-id")