Bring up to date with guest_accounts branch

.github/workflows/run-tests.yml

@@ -10,25 +10,34 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: buildjet-2vcpu-ubuntu-2204
+    strategy:
+      matrix:
+        nim:
+          - "1.6.10"
+          - "1.6.x"
+          - "2.0.x"
+          - "devel"
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Cache nimble
         id: cache-nimble
-        uses: actions/cache@v3
+        uses: buildjet/cache@v3
         with:
           path: ~/.nimble
-          key: nimble-${{ hashFiles('*.nimble') }}
-          restore-keys: "nimble-"
+          key: ${{ matrix.nim }}-nimble-${{ hashFiles('*.nimble') }}
+          restore-keys: |
+            ${{ matrix.nim }}-nimble-
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
           cache: "pip"
       - uses: jiro4989/setup-nim-action@v1
         with:
-          nim-version: "1.x"
+          nim-version: ${{ matrix.nim }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - run: nimble build -d:release -Y
       - run: pip install seleniumbase
       - run: seleniumbase install chromedriver
@@ -37,12 +46,11 @@ jobs:
         run: |
           sudo apt install libsass-dev -y
           cp nitter.example.conf nitter.conf
+          sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
           nimble md
           nimble scss
+          echo '${{ secrets.GUEST_ACCOUNTS }}' > ./guest_accounts.jsonl
       - name: Run tests
-        env:
-          GUEST_ACCOUNTS: ${{ secrets.GUEST_ACCOUNTS }}
         run: |
-          echo $GUEST_ACCOUNTS > ./guest_accounts.json
           ./nitter &
-          pytest -n4 tests
+          pytest -n8 tests

Dockerfile.arm64

@@ -1,7 +1,7 @@
 FROM alpine:3.18 as nim
 LABEL maintainer="setenforce@protonmail.com"
 
-RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.14-r0" nimble pcre
+RUN apk --no-cache add libsass-dev pcre gcc git libc-dev "nim=1.6.14-r0" "nimble=0.13.1-r2"
 
 WORKDIR /src/nitter
 
@@ -15,9 +15,11 @@ RUN nimble build -d:danger -d:lto -d:strip \
 
 FROM alpine:3.18
 WORKDIR /src/
-RUN apk --no-cache add ca-certificates pcre openssl1.1-compat
+RUN apk --no-cache add pcre ca-certificates openssl1.1-compat
 COPY --from=nim /src/nitter/nitter ./
 COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
 COPY --from=nim /src/nitter/public ./public
 EXPOSE 8080
+RUN adduser -h /src/ -D -s /bin/sh nitter
+USER nitter
 CMD ./nitter

src/auth.nim

@@ -1,5 +1,5 @@
 #SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, times, json, random, strutils, tables, intsets, os
+import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
 import types
 import experimental/parser/guestaccount
 
@@ -30,38 +30,50 @@ var
 template log(str: varargs[string, `$`]) =
   if enableLogging: echo "[accounts] ", str.join("")
 
+proc snowflakeToEpoch(flake: int64): int64 =
+  int64(((flake shr 22) + 1288834974657) div 1000)
+
+proc hasExpired(account: GuestAccount): bool =
+  let
+    created = snowflakeToEpoch(account.id)
+    now = epochTime().int64
+    daysOld = int(now - created) div dayInSeconds
+  return daysOld > 30
+
 proc getAccountPoolHealth*(): JsonNode =
   let now = epochTime().int
 
   var
     totalReqs = 0
-    limited: IntSet
+    limited: PackedSet[int64]
     reqsPerApi: Table[string, int]
-    oldest = now
-    newest = 0
-    average = 0
+    oldest = now.int64
+    newest = 0'i64
+    average = 0'i64
 
   for account in accountPool:
-    # Twitter snowflake conversion
-    let created = ((account.id shr 22) + 1288834974657) div 1000
-
+    let created = snowflakeToEpoch(account.id)
     if created > newest:
       newest = created
     if created < oldest:
       oldest = created
-    average.inc created
+    average += created
 
     for api in account.apis.keys:
       let
         apiStatus = account.apis[api]
         reqs = apiMaxReqs[api] - apiStatus.remaining
 
-      reqsPerApi.mgetOrPut($api, 0).inc reqs
-      totalReqs.inc reqs
-
       if apiStatus.limited:
         limited.incl account.id
 
+      # no requests made with this account and endpoint since the limit reset
+      if apiStatus.reset < now:
+        continue
+
+      reqsPerApi.mgetOrPut($api, 0).inc reqs
+      totalReqs.inc reqs
+
   if accountPool.len > 0:
     average = average div accountPool.len
   else:
@@ -71,7 +83,6 @@ proc getAccountPoolHealth*(): JsonNode =
   return %*{
     "accounts": %*{
       "total": accountPool.len,
-      "active": accountPool.len - limited.card,
       "limited": limited.card,
       "oldest": $fromUnix(oldest),
       "newest": $fromUnix(newest),
@@ -189,3 +200,10 @@ proc initAccountPool*(cfg: Config; path: string) =
   else:
     echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests."
     quit 1
+
+  let accountsPrePurge = accountPool.len
+  accountPool.keepItIf(not it.hasExpired)
+
+  log "Successfully added ", accountPool.len, " valid accounts."
+  if accountsPrePurge > accountPool.len:
+    log "Purged ", accountsPrePurge - accountPool.len, " expired accounts."

src/parser.nim

@@ -323,6 +323,8 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
     return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
   of "TweetWithVisibilityResults":
     return parseGraphTweet(js{"tweet"}, isLegacy)
+  else:
+    discard
 
   if not js.hasKey("legacy"):
     return Tweet()

src/sass/profile/card.scss

@@ -115,7 +115,7 @@
     }
 
     .profile-card-tabs-name {
-        @include breakable;
+        flex-shrink: 100;
     }
 
     .profile-card-avatar {

src/types.nim

@@ -36,7 +36,7 @@ type
     limitedAt*: int
 
   GuestAccount* = ref object
-    id*: BiggestInt
+    id*: int64
     oauthToken*: string
     oauthSecret*: string
     pending*: int
@@ -164,7 +164,7 @@ type
     newsletterPublication = "newsletter_publication"
     hidden
     unknown
-    
+
   Card* = object
     kind*: CardKind
     url*: string

tests/test_card.py

@@ -21,9 +21,9 @@ card = [
 
 no_thumb = [
     ['FluentAI/status/1116417904831029248',
-     'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
-     'One of the only ways to improve Alexa is to have human beings check it for errors',
-     'theverge.com'],
+     'LinkedIn',
+     'This link will take you to a page that’s not on LinkedIn',
+     'lnkd.in'],
 
     ['Thom_Wolf/status/1122466524860702729',
      'facebookresearch/fairseq',