Replace tweet endpoint with GraphQL

zedeus · Feb 2, 2023 · 19adc65 · 19adc65
1 parent 22b51b4
commit 19adc65
Show file tree

Hide file tree

Showing 7 changed files with 111 additions and 67 deletions.
diff --git a/src/api.nim b/src/api.nim
@@ -101,16 +101,21 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
  except InternalError:
  return Result[T](beginning: true, query: query)
 
-proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
- let url = tweet / (id & ".json") ? genParams(cursor=after)
- result = parseConversation(await fetch(url, Api.tweet), id)
+proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
+ if id.len == 0: return
+ let
+ cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+ variables = tweetVariables % [id, cursor]
+ params = {"variables": variables, "features": tweetFeatures}
+ js = await fetch(graphTweet ? params, Api.tweetDetail)
+ result = parseGraphConversation(js, id)
 
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
- result = (await getTweetImpl(id, after)).replies
+ result = (await getGraphTweet(id, after)).replies
  result.beginning = after.len == 0
 
 proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
- result = await getTweetImpl(id)
+ result = await getGraphTweet(id)
  if after.len > 0:
  result.replies = await getReplies(id, after)
 

diff --git a/src/consts.nim b/src/consts.nim
@@ -19,6 +19,7 @@ const
  tweet* = timelineApi / "conversation"
 
  graphql = api / "graphql"
+ graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
  graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
  graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
  graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
@@ -58,3 +59,34 @@ const
  ## user: "result_filter: user"
  ## photos: "result_filter: photos"
  ## videos: "result_filter: videos"
+
+ tweetVariables* = """{
+ "focalTweetId": "$1",
+ $2
+ "includePromotedContent": false,
+ "withBirdwatchNotes": false,
+ "withDownvotePerspective": false,
+ "withReactionsMetadata": false,
+ "withReactionsPerspective": false,
+ "withSuperFollowsTweetFields": false,
+ "withSuperFollowsUserFields": false,
+ "withVoice": false,
+ "withV2Timeline": true
+}"""
+
+ tweetFeatures* = """{
+ "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+ "responsive_web_graphql_timeline_navigation_enabled": false,
+ "standardized_nudges_misinfo": false,
+ "verified_phone_label_enabled": false,
+ "responsive_web_twitter_blue_verified_badge_is_enabled": false,
+ "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+ "view_counts_everywhere_api_enabled": false,
+ "responsive_web_edit_tweet_api_enabled": false,
+ "tweetypie_unmention_optimization_enabled": false,
+ "vibe_api_enabled": false,
+ "longform_notetweets_consumption_enabled": false,
+ "responsive_web_text_conversations_enabled": false,
+ "responsive_web_enhance_cards_enabled": false,
+ "interactive_text_enabled": false
+}"""
diff --git a/src/parser.nim b/src/parser.nim
@@ -72,8 +72,8 @@ proc parseGif(js: JsonNode): Gif =
 proc parseVideo(js: JsonNode): Video =
  result = Video(
  thumb: js{"media_url_https"}.getImageStr,
- views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
- available: js{"ext_media_availability", "status"}.getStr == "available",
+ views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt),
+ available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available",
  title: js{"ext_alt_text"}.getStr,
  durationMs: js{"video_info", "duration_millis"}.getInt
  # playbackType: mp4
@@ -185,15 +185,14 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
  result.url.len == 0 or result.url.startsWith("card://"):
  result.url = getPicUrl(result.image)
 
-proc parseTweet(js: JsonNode): Tweet =
+proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
  if js.isNull: return
  result = Tweet(
  id: js{"id_str"}.getId,
  threadId: js{"conversation_id_str"}.getId,
  replyId: js{"in_reply_to_status_id_str"}.getId,
  text: js{"full_text"}.getStr,
  time: js{"created_at"}.getTime,
- source: getSource(js),
  hasThread: js{"self_thread"}.notNull,
  available: true,
  user: User(id: js{"user_id_str"}.getStr),
@@ -218,7 +217,7 @@ proc parseTweet(js: JsonNode): Tweet =
  result.retweet = some Tweet(id: rt.getId)
  return
 
- with jsCard, js{"card"}:
+ if jsCard.kind != JNull:
  let name = jsCard{"name"}.getStr
  if "poll" in name:
  if "image" in name:
@@ -295,64 +294,17 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
  result.users[k] = parseUser(v, k)
 
  for k, v in tweets:
- var tweet = parseTweet(v)
+ var tweet = parseTweet(v, v{"card"})
  if tweet.user.id in result.users:
  tweet.user = result.users[tweet.user.id]
  result.tweets[k] = tweet
 
-proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
- result.thread = Chain()
-
- let thread = js{"content", "item", "content", "conversationThread"}
- with cursor, thread{"showMoreCursor"}:
- result.thread.cursor = cursor{"value"}.getStr
- result.thread.hasMore = true
-
- for t in thread{"conversationComponents"}:
- let content = t{"conversationTweetComponent", "tweet"}
-
- if content{"displayType"}.getStr == "SelfThread":
- result.self = true
-
- var tweet = finalizeTweet(global, content{"id"}.getStr)
- if not tweet.available:
- tweet.tombstone = getTombstone(content{"tombstone"})
- result.thread.content.add tweet
-
-proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
- result = Conversation(replies: Result[Chain](beginning: true))
- let global = parseGlobalObjects(? js)
-
- let instructions = ? js{"timeline", "instructions"}
- if instructions.len == 0:
- return
-
- for e in instructions[0]{"addEntries", "entries"}:
- let entry = e{"entryId"}.getStr
- if "tweet" in entry or "tombstone" in entry:
- let tweet = finalizeTweet(global, e.getEntryId)
- if $tweet.id != tweetId:
- result.before.content.add tweet
- else:
- result.tweet = tweet
- elif "conversationThread" in entry:
- let (thread, self) = parseThread(e, global)
- if thread.content.len > 0:
- if self:
- result.after = thread
- else:
- result.replies.content.add thread
- elif "cursor-showMore" in entry:
- result.replies.bottom = e.getCursor
- elif "cursor-bottom" in entry:
- result.replies.bottom = e.getCursor
-
 proc parseStatus*(js: JsonNode): Tweet =
  with e, js{"errors"}:
  if e.getError == tweetNotFound:
  return
 
- result = parseTweet(js)
+ result = parseTweet(js, js{"card"})
  if not result.isNil:
  result.user = parseUser(js{"user"})
 
@@ -409,7 +361,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
 proc parsePhotoRail*(js: JsonNode): PhotoRail =
  for tweet in js:
  let
- t = parseTweet(tweet)
+ t = parseTweet(tweet, js{"card"})
  url = if t.photos.len > 0: t.photos[0]
  elif t.video.isSome: get(t.video).thumb
  elif t.gif.isSome: get(t.gif).thumb
@@ -418,3 +370,61 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =
 
  if url.len == 0: continue
  result.add GalleryPhoto(url: url, tweetId: $t.id)
+
+proc parseGraphTweet(js: JsonNode): Tweet =
+ if js.kind == JNull:
+ return Tweet(available: false)
+
+ var jsCard = copy(js{"card", "legacy"})
+ if jsCard.kind != JNull:
+ var values = newJObject()
+ for val in jsCard["binding_values"]:
+ values[val["key"].getStr] = val["value"]
+ jsCard["binding_values"] = values
+
+ result = parseTweet(js{"legacy"}, jsCard)
+ result.user = parseUser(js{"core", "user_results", "result", "legacy"})
+
+ if result.quote.isSome:
+ result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
+
+proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
+ let thread = js{"content", "items"}
+ for t in js{"content", "items"}:
+ let entryId = t{"entryId"}.getStr
+ if "cursor-showmore" in entryId:
+ let cursor = t{"item", "itemContent", "value"}
+ result.thread.cursor = cursor.getStr
+ result.thread.hasMore = true
+ elif "tweet" in entryId:
+ let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"})
+ result.thread.content.add tweet
+
+ if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
+ result.self = true
+
+proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
+ result = Conversation(replies: Result[Chain](beginning: true))
+
+ let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
+ if instructions.len == 0:
+ return
+
+ for e in instructions[0]{"entries"}:
+ let entryId = e{"entryId"}.getStr
+ # echo entryId
+ if entryId.startsWith("tweet"):
+ let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
+
+ if $tweet.id == tweetId:
+ result.tweet = tweet
+ else:
+ result.before.content.add tweet
+ elif entryId.startsWith("conversationthread"):
+ let (thread, self) = parseGraphThread(e)
+ if self:
+ result.after = thread
+ else:
+ result.replies.content.add thread
+ elif entryId.startsWith("cursor-bottom"):
+ result.replies.bottom = e{"content", "itemContent", "value"}.getStr
diff --git a/src/parserutils.nim b/src/parserutils.nim
@@ -133,10 +133,6 @@ proc getTombstone*(js: JsonNode): string =
  result = js{"tombstoneInfo", "richText", "text"}.getStr
  result.removeSuffix(" Learn more")
 
-proc getSource*(js: JsonNode): string =
- let src = js{"source"}.getStr
- result = src.substr(src.find('>') + 1, src.rfind('<') - 1)
-
 proc getMp4Resolution*(url: string): int =
  # parses the height out of a URL like this one:
  # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4

diff --git a/src/tokens.nim b/src/tokens.nim
@@ -41,7 +41,8 @@ proc getPoolJson*(): JsonNode =
  let
  maxReqs =
  case api
- of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId, Api.userScreenName: 500
+ of Api.listMembers, Api.listBySlug, Api.list,
+ Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
  of Api.timeline: 187
  else: 180
  reqs = maxReqs - token.apis[api].remaining

diff --git a/src/types.nim b/src/types.nim
@@ -9,6 +9,7 @@ type
  InternalError* = object of CatchableError
 
  Api* {.pure.} = enum
+ tweetDetail
  userShow
  timeline
  search
@@ -176,7 +177,6 @@ type
  available*: bool
  tombstone*: string
  location*: string
- source*: string
  stats*: TweetStats
  retweet*: Option[Tweet]
  attribution*: Option[User]

diff --git a/src/views/tweet.nim b/src/views/tweet.nim
@@ -347,7 +347,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
  renderQuote(tweet.quote.get(), prefs, path)
 
  if mainTweet:
- p(class="tweet-published"): text &"{getTime(tweet)} · {tweet.source}"
+ p(class="tweet-published"): text &"{getTime(tweet)}"
 
  if tweet.mediaTags.len > 0:
  renderMediaTags(tweet.mediaTags)