fix(nitter): add graphql user search (#1047)

* fix(nitter): add graphql user search

* fix(nitter): rm gitignore 2nd guest_accounts

* fix(nitter): keep query from user search in result. remove personal mods

* fix(nitter): removce useless line gitignore
This commit is contained in:
LS 2023-10-30 13:13:06 +01:00 committed by GitHub
parent 537af7fd5e
commit 735b30c2da
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 33 additions and 26 deletions

View file

@ -112,25 +112,29 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
if after.len > 0: if after.len > 0:
variables["cursor"] = % after variables["cursor"] = % after
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphSearch(await fetch(url, Api.search), after) result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
result.query = query result.query = query
proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} = proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
if query.text.len == 0: if query.text.len == 0:
return Result[User](query: query, beginning: true) return Result[User](query: query, beginning: true)
let var
page = if page.len == 0: "1" else: page variables = %*{
url = userSearch ? genParams({"q": query.text, "skip_status": "1", "page": page}) "rawQuery": query.text,
js = await fetchRaw(url, Api.userSearch) "count": 20,
"product": "People",
result = parseUsers(js) "withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
}
if after.len > 0:
variables["cursor"] = % after
result.beginning = false
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphSearch[User](await fetch(url, Api.search), after)
result.query = query result.query = query
if page.len == 0:
result.bottom = "2"
elif page.allCharsInSet(Digits):
result.bottom = $(parseInt(page) + 1)
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} = proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return if name.len == 0: return

View file

@ -9,7 +9,6 @@ const
activate* = $(api / "1.1/guest/activate.json") activate* = $(api / "1.1/guest/activate.json")
photoRail* = api / "1.1/statuses/media_timeline.json" photoRail* = api / "1.1/statuses/media_timeline.json"
userSearch* = api / "1.1/users/search.json"
graphql = api / "graphql" graphql = api / "graphql"
graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
@ -35,7 +34,7 @@ const
"include_user_entities": "1", "include_user_entities": "1",
"include_ext_reply_count": "1", "include_ext_reply_count": "1",
"include_ext_is_blue_verified": "1", "include_ext_is_blue_verified": "1",
#"include_ext_verified_type": "1", # "include_ext_verified_type": "1",
"include_ext_media_color": "0", "include_ext_media_color": "0",
"cards_platform": "Web-13", "cards_platform": "Web-13",
"tweet_mode": "extended", "tweet_mode": "extended",

View file

@ -443,8 +443,8 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
tweet.id = parseBiggestInt(entryId) tweet.id = parseBiggestInt(entryId)
result.pinned = some tweet result.pinned = some tweet
proc parseGraphSearch*(js: JsonNode; after=""): Timeline = proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
result = Timeline(beginning: after.len == 0) result = Result[T](beginning: after.len == 0)
let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"} let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
if instructions.len == 0: if instructions.len == 0:
@ -455,13 +455,19 @@ proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
if typ == "TimelineAddEntries": if typ == "TimelineAddEntries":
for e in instruction{"entries"}: for e in instruction{"entries"}:
let entryId = e{"entryId"}.getStr let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"): when T is Tweets:
with tweetRes, e{"content", "itemContent", "tweet_results", "result"}: if entryId.startsWith("tweet"):
let tweet = parseGraphTweet(tweetRes, true) with tweetRes, e{"content", "itemContent", "tweet_results", "result"}:
if not tweet.available: let tweet = parseGraphTweet(tweetRes)
tweet.id = parseBiggestInt(entryId.getId()) if not tweet.available:
result.content.add tweet tweet.id = parseBiggestInt(entryId.getId())
elif entryId.startsWith("cursor-bottom"): result.content.add tweet
elif T is User:
if entryId.startsWith("user"):
with userRes, e{"content", "itemContent"}:
result.content.add parseGraphUser(userRes)
if entryId.startsWith("cursor-bottom"):
result.bottom = e{"content", "value"}.getStr result.bottom = e{"content", "value"}.getStr
elif typ == "TimelineReplaceEntry": elif typ == "TimelineReplaceEntry":
if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"): if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):

View file

@ -29,7 +29,7 @@ proc createSearchRouter*(cfg: Config) =
redirect("/" & q) redirect("/" & q)
var users: Result[User] var users: Result[User]
try: try:
users = await getUserSearch(query, getCursor()) users = await getGraphUserSearch(query, getCursor())
except InternalError: except InternalError:
users = Result[User](beginning: true, query: query) users = Result[User](beginning: true, query: query)
resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title) resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)

View file

@ -62,7 +62,6 @@ proc getPoolJson*(): JsonNode =
Api.userRestId, Api.userScreenName, Api.userRestId, Api.userScreenName,
Api.tweetResult, Api.tweetResult,
Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500 Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500
of Api.userSearch: 900
reqs = maxReqs - apiStatus.remaining reqs = maxReqs - apiStatus.remaining
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs

View file

@ -19,7 +19,6 @@ type
tweetResult tweetResult
photoRail photoRail
search search
userSearch
list list
listBySlug listBySlug
listMembers listMembers