From ae9fa02bf5f8459e59f0f18120fee9f06404aef5 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Fri, 25 Aug 2023 16:28:30 +0200
Subject: [PATCH] Switch to TweetDetail for tweets
---
src/consts.nim | 10 +++++++---
src/parser.nim | 25 +++++++++++++++----------
src/tokens.nim | 3 ++-
tests/test_card.py | 10 +++++-----
tests/test_timeline.py | 2 +-
tests/test_tweet_media.py | 2 +-
6 files changed, 31 insertions(+), 21 deletions(-)
diff --git a/src/consts.nim b/src/consts.nim
index 8bf6422..96cea47 100644
--- a/src/consts.nim
+++ b/src/consts.nim
@@ -17,7 +17,7 @@ const
graphUserTweets* = graphql / "3JNH4e9dq1BifLxAa3UMWg/UserWithProfileTweetsQueryV2"
graphUserTweetsAndReplies* = graphql / "8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2"
graphUserMedia* = graphql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
- graphTweet* = graphql / "83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2"
+ graphTweet* = graphql / "q94uRCEn65LZThakYcPT6g/TweetDetail"
graphTweetResult* = graphql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
@@ -89,8 +89,12 @@ const
tweetVariables* = """{
"focalTweetId": "$1",
$2
- "includeHasBirdwatchNotes": false
-}"""
+ "includeHasBirdwatchNotes": false,
+ "includePromotedContent": false,
+ "withBirdwatchNotes": false,
+ "withVoice": false,
+ "withV2Timeline": true
+}""".replace(" ", "").replace("\n", "")
# oldUserTweetsVariables* = """{
# "userId": "$1", $2
diff --git a/src/parser.nim b/src/parser.nim
index 03242c1..d5190a3 100644
--- a/src/parser.nim
+++ b/src/parser.nim
@@ -324,7 +324,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
of "TweetWithVisibilityResults":
return parseGraphTweet(js{"tweet"})
- var jsCard = copy(js{"tweet_card", "legacy"})
+ var jsCard = copy(js{"card", "legacy"})
if jsCard.kind != JNull:
var values = newJObject()
for val in jsCard["binding_values"]:
@@ -342,7 +342,6 @@ proc parseGraphTweet(js: JsonNode): Tweet =
result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
- let thread = js{"content", "items"}
for t in js{"content", "items"}:
let entryId = t{"entryId"}.getStr
if "cursor-showmore" in entryId:
@@ -350,11 +349,16 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
result.thread.cursor = cursor.getStr
result.thread.hasMore = true
elif "tweet" in entryId:
- let tweet = parseGraphTweet(t{"item", "content", "tweetResult", "result"})
- result.thread.content.add tweet
+ let
+ isLegacy = t{"item"}.hasKey("itemContent")
+ (contentKey, resultKey) = if isLegacy: ("itemContent", "tweet_results")
+ else: ("content", "tweetResult")
- if t{"item", "content", "tweetDisplayType"}.getStr == "SelfThread":
- result.self = true
+ with content, t{"item", contentKey}:
+ result.thread.content.add parseGraphTweet(content{resultKey, "result"})
+
+ if content{"tweetDisplayType"}.getStr == "SelfThread":
+ result.self = true
proc parseGraphTweetResult*(js: JsonNode): Tweet =
with tweet, js{"data", "tweet_result", "result"}:
@@ -363,14 +367,14 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet =
proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))
- let instructions = ? js{"data", "timeline_response", "instructions"}
+ let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
if instructions.len == 0:
return
for e in instructions[0]{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
- with tweetResult, e{"content", "content", "tweetResult", "result"}:
+ with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
@@ -385,7 +389,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
let tweet = Tweet(
id: parseBiggestInt(id),
available: false,
- text: e{"content", "content", "tombstoneInfo", "richText"}.getTombstone
+ text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
)
if id == tweetId:
@@ -397,9 +401,10 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
if self:
result.after = thread
else:
+ echo "adding thread: ", thread.content.len
result.replies.content.add thread
elif entryId.startsWith("cursor-bottom"):
- result.replies.bottom = e{"content", "content", "value"}.getStr
+ result.replies.bottom = e{"content", "itemContent", "value"}.getStr
proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
result = Profile(tweets: Timeline(beginning: after.len == 0))
diff --git a/src/tokens.nim b/src/tokens.nim
index a3ed78a..bb9696c 100644
--- a/src/tokens.nim
+++ b/src/tokens.nim
@@ -52,10 +52,11 @@ proc getPoolJson*(): JsonNode =
maxReqs =
case api
of Api.search: 50
+ of Api.tweetDetail: 150
of Api.photoRail: 180
of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
Api.userRestId, Api.userScreenName,
- Api.tweetResult, Api.tweetDetail,
+ Api.tweetResult,
Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500
of Api.userSearch: 900
reqs = maxReqs - apiStatus.remaining
diff --git a/tests/test_card.py b/tests/test_card.py
index f84ddca..733bd40 100644
--- a/tests/test_card.py
+++ b/tests/test_card.py
@@ -13,11 +13,6 @@ card = [
'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
'gist.github.com', True],
- ['FluentAI/status/1116417904831029248',
- 'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
- 'One of the only ways to improve Alexa is to have human beings check it for errors',
- 'theverge.com', True],
-
['nim_lang/status/1082989146040340480',
'Nim in 2018: A short recap',
'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.',
@@ -25,6 +20,11 @@ card = [
]
no_thumb = [
+ ['FluentAI/status/1116417904831029248',
+ 'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
+ 'One of the only ways to improve Alexa is to have human beings check it for errors',
+ 'theverge.com'],
+
['Thom_Wolf/status/1122466524860702729',
'facebookresearch/fairseq',
'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
diff --git a/tests/test_timeline.py b/tests/test_timeline.py
index b56d6ad..919aa70 100644
--- a/tests/test_timeline.py
+++ b/tests/test_timeline.py
@@ -6,7 +6,7 @@ normal = [['jack'], ['elonmusk']]
after = [['jack', '1681686036294803456'],
['elonmusk', '1681686036294803456']]
-no_more = [['mobile_test_8?cursor=1000']]
+no_more = [['mobile_test_8?cursor=DAABCgABF4YVAqN___kKAAICNn_4msIQAAgAAwAAAAIAAA']]
empty = [['emptyuser'], ['mobile_test_10']]
diff --git a/tests/test_tweet_media.py b/tests/test_tweet_media.py
index 7a00983..f54cea7 100644
--- a/tests/test_tweet_media.py
+++ b/tests/test_tweet_media.py
@@ -14,7 +14,7 @@ poll = [
image = [
['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'],
- ['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
+ #['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
]
gif = [