mirror of
https://github.com/zedeus/nitter.git
synced 2024-12-13 03:26:30 +00:00
Add fallback for sensitive profiles
This commit is contained in:
parent
da03515695
commit
abe21e3ebf
2 changed files with 75 additions and 55 deletions
70
src/api.nim
70
src/api.nim
|
@ -8,9 +8,31 @@ const base = parseUri("https://twitter.com/")
|
||||||
const agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
|
const agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
|
||||||
|
|
||||||
const timelineUrl = "i/profiles/show/$1/timeline/tweets?include_available_features=1&include_entities=1&include_new_items_bar=true"
|
const timelineUrl = "i/profiles/show/$1/timeline/tweets?include_available_features=1&include_entities=1&include_new_items_bar=true"
|
||||||
const profileUrl = "i/profiles/popup"
|
const profilePopupUrl = "i/profiles/popup"
|
||||||
|
const profileIntentUrl = "intent/user"
|
||||||
const tweetUrl = "i/status/"
|
const tweetUrl = "i/status/"
|
||||||
|
|
||||||
|
proc fetchHtml(client: AsyncHttpClient; url: Uri; jsonKey = ""): Future[XmlNode] {.async.} =
|
||||||
|
var resp = ""
|
||||||
|
try:
|
||||||
|
resp = await client.getContent($url)
|
||||||
|
except:
|
||||||
|
return nil
|
||||||
|
|
||||||
|
if jsonKey.len > 0:
|
||||||
|
let json = parseJson(resp)[jsonKey].str
|
||||||
|
return parseHtml(json)
|
||||||
|
else:
|
||||||
|
return parseHtml(resp)
|
||||||
|
|
||||||
|
proc getProfileFallback(username: string; client: AsyncHttpClient): Future[Profile] {.async.} =
|
||||||
|
let
|
||||||
|
params = {"screen_name": username}
|
||||||
|
url = base / profileIntentUrl ? params
|
||||||
|
html = await client.fetchHtml(url)
|
||||||
|
|
||||||
|
result = parseIntentProfile(html)
|
||||||
|
|
||||||
proc getProfile*(username: string): Future[Profile] {.async.} =
|
proc getProfile*(username: string): Future[Profile] {.async.} =
|
||||||
let client = newAsyncHttpClient()
|
let client = newAsyncHttpClient()
|
||||||
defer: client.close()
|
defer: client.close()
|
||||||
|
@ -24,25 +46,19 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
|
||||||
"Accept-Language": "en-US,en;q=0.9"
|
"Accept-Language": "en-US,en;q=0.9"
|
||||||
})
|
})
|
||||||
|
|
||||||
let params = {
|
let
|
||||||
|
params = {
|
||||||
"screen_name": username,
|
"screen_name": username,
|
||||||
"wants_hovercard": "true",
|
"wants_hovercard": "true",
|
||||||
"_": $(epochTime().int)
|
"_": $(epochTime().int)
|
||||||
}
|
}
|
||||||
|
url = base / profilePopupUrl ? params
|
||||||
|
html = await client.fetchHtml(url, jsonKey="html")
|
||||||
|
|
||||||
let url = base / profileUrl ? params
|
if not html.querySelector(".ProfileCard-sensitiveWarningContainer").isNil:
|
||||||
var resp = ""
|
return await getProfileFallback(username, client)
|
||||||
|
|
||||||
try:
|
result = parsePopupProfile(html)
|
||||||
resp = await client.getContent($url)
|
|
||||||
except:
|
|
||||||
return Profile()
|
|
||||||
|
|
||||||
let
|
|
||||||
json = parseJson(resp)["html"].str
|
|
||||||
html = parseHtml(json)
|
|
||||||
|
|
||||||
result = parseProfile(html)
|
|
||||||
|
|
||||||
proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} =
|
proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} =
|
||||||
let client = newAsyncHttpClient()
|
let client = newAsyncHttpClient()
|
||||||
|
@ -61,18 +77,7 @@ proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} =
|
||||||
if after != "":
|
if after != "":
|
||||||
url &= "&max_position=" & after
|
url &= "&max_position=" & after
|
||||||
|
|
||||||
var resp = ""
|
let html = await client.fetchHtml(base / url, jsonKey="items_html")
|
||||||
try:
|
|
||||||
resp = await client.getContent($(base / url))
|
|
||||||
except:
|
|
||||||
return
|
|
||||||
|
|
||||||
var json: string = ""
|
|
||||||
var html: XmlNode
|
|
||||||
json = parseJson(resp)["items_html"].str
|
|
||||||
html = parseHtml(json)
|
|
||||||
|
|
||||||
writeFile("epic.html", $html)
|
|
||||||
|
|
||||||
result = parseTweets(html)
|
result = parseTweets(html)
|
||||||
|
|
||||||
|
@ -91,15 +96,8 @@ proc getTweet*(id: string): Future[Conversation] {.async.} =
|
||||||
"x-previous-page-name": "profile"
|
"x-previous-page-name": "profile"
|
||||||
})
|
})
|
||||||
|
|
||||||
let url = base / tweetUrl / id
|
let
|
||||||
|
url = base / tweetUrl / id
|
||||||
var resp: string = ""
|
html = await client.fetchHtml(url)
|
||||||
try:
|
|
||||||
resp = await client.getContent($url)
|
|
||||||
except:
|
|
||||||
return Conversation()
|
|
||||||
|
|
||||||
var html: XmlNode
|
|
||||||
html = parseHtml(resp)
|
|
||||||
|
|
||||||
result = parseConversation(html)
|
result = parseConversation(html)
|
||||||
|
|
|
@ -4,7 +4,7 @@ import nimquery, regex
|
||||||
import ./types, ./formatters
|
import ./types, ./formatters
|
||||||
|
|
||||||
proc getAttr(node: XmlNode; attr: string; default=""): string =
|
proc getAttr(node: XmlNode; attr: string; default=""): string =
|
||||||
if node.isNIl or node.attrs.isNil: return default
|
if node.isNil or node.attrs.isNil: return default
|
||||||
return node.attrs.getOrDefault(attr)
|
return node.attrs.getOrDefault(attr)
|
||||||
|
|
||||||
proc selectAttr(node: XmlNode; selector: string; attr: string; default=""): string =
|
proc selectAttr(node: XmlNode; selector: string; attr: string; default=""): string =
|
||||||
|
@ -15,16 +15,21 @@ proc selectText(node: XmlNode; selector: string): string =
|
||||||
let res = node.querySelector(selector)
|
let res = node.querySelector(selector)
|
||||||
result = if res == nil: "" else: res.innerText()
|
result = if res == nil: "" else: res.innerText()
|
||||||
|
|
||||||
proc parseProfile*(node: XmlNode): Profile =
|
proc parsePopupProfile*(node: XmlNode): Profile =
|
||||||
let profile = node.querySelector(".profile-card")
|
let profile = node.querySelector(".profile-card")
|
||||||
result.fullname = profile.selectText(".fullname").strip()
|
if profile.isNil: return
|
||||||
result.username = profile.selectText(".username").strip(chars={'@', ' '})
|
|
||||||
result.description = profile.selectText(".bio")
|
result = Profile(
|
||||||
result.verified = profile.selectText(".Icon.Icon--verified").len > 0
|
fullname: profile.selectText(".fullname").strip(),
|
||||||
result.protected = profile.selectText(".Icon.Icon--protected").len > 0
|
username: profile.selectText(".username").strip(chars={'@', ' '}),
|
||||||
result.userpic = profile.selectAttr(".ProfileCard-avatarImage", "src").getUserpic()
|
description: profile.selectText(".bio"),
|
||||||
result.banner = profile.selectAttr("svg > image", "xlink:href").replace("600x200", "1500x500")
|
verified: profile.selectText(".Icon.Icon--verified").len > 0,
|
||||||
if result.banner == "":
|
protected: profile.selectText(".Icon.Icon--protected").len > 0,
|
||||||
|
userpic: profile.selectAttr(".ProfileCard-avatarImage", "src").getUserpic(),
|
||||||
|
banner: profile.selectAttr("svg > image", "xlink:href").replace("600x200", "1500x500")
|
||||||
|
)
|
||||||
|
|
||||||
|
if result.banner.len == 0:
|
||||||
result.banner = profile.selectAttr(".ProfileCard-bg", "style")
|
result.banner = profile.selectAttr(".ProfileCard-bg", "style")
|
||||||
|
|
||||||
let stats = profile.querySelectorAll(".ProfileCardStats-statLink")
|
let stats = profile.querySelectorAll(".ProfileCardStats-statLink")
|
||||||
|
@ -35,12 +40,29 @@ proc parseProfile*(node: XmlNode): Profile =
|
||||||
of "following": result.following = text
|
of "following": result.following = text
|
||||||
else: result.tweets = text
|
else: result.tweets = text
|
||||||
|
|
||||||
proc parseTweetProfile*(tweet: XmlNode): Profile =
|
proc parseIntentProfile*(profile: XmlNode): Profile =
|
||||||
result = Profile(
|
result = Profile(
|
||||||
fullname: tweet.getAttr("data-name"),
|
fullname: profile.selectText("a.fn.url.alternate-context").strip(),
|
||||||
username: tweet.getAttr("data-screen-name"),
|
username: profile.selectText(".nickname").strip(chars={'@', ' '}),
|
||||||
userpic: tweet.selectAttr(".avatar", "src").getUserpic(),
|
userpic: profile.querySelector(".profile.summary").selectAttr("img.photo", "src").getUserPic(),
|
||||||
verified: tweet.selectText(".Icon.Icon--verified").len > 0
|
description: profile.selectText("p.note").strip(),
|
||||||
|
verified: not profile.querySelector("li.verified").isNil,
|
||||||
|
protected: not profile.querySelector("li.protected").isNil,
|
||||||
|
banner: "background-color: #161616",
|
||||||
|
tweets: "?"
|
||||||
|
)
|
||||||
|
|
||||||
|
for stat in profile.querySelectorAll("dd.count > a"):
|
||||||
|
case stat.getAttr("href").split("/")[^1]
|
||||||
|
of "followers": result.followers = stat.innerText()
|
||||||
|
of "following": result.following = stat.innerText()
|
||||||
|
|
||||||
|
proc parseTweetProfile*(profile: XmlNode): Profile =
|
||||||
|
result = Profile(
|
||||||
|
fullname: profile.getAttr("data-name"),
|
||||||
|
username: profile.getAttr("data-screen-name"),
|
||||||
|
userpic: profile.selectAttr(".avatar", "src").getUserpic(),
|
||||||
|
verified: profile.selectText(".Icon.Icon--verified").len > 0
|
||||||
)
|
)
|
||||||
|
|
||||||
proc parseTweet*(tweet: XmlNode): Tweet =
|
proc parseTweet*(tweet: XmlNode): Tweet =
|
||||||
|
|
Loading…
Reference in a new issue