diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 37979cb..ee28e33 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -10,25 +10,34 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: buildjet-2vcpu-ubuntu-2204 + strategy: + matrix: + nim: + - "1.6.10" + - "1.6.x" + - "2.0.x" + - "devel" steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Cache nimble id: cache-nimble - uses: actions/cache@v3 + uses: buildjet/cache@v3 with: path: ~/.nimble - key: nimble-${{ hashFiles('*.nimble') }} - restore-keys: "nimble-" + key: ${{ matrix.nim }}-nimble-${{ hashFiles('*.nimble') }} + restore-keys: | + ${{ matrix.nim }}-nimble- - uses: actions/setup-python@v4 with: python-version: "3.10" cache: "pip" - uses: jiro4989/setup-nim-action@v1 with: - nim-version: "1.x" + nim-version: ${{ matrix.nim }} + repo-token: ${{ secrets.GITHUB_TOKEN }} - run: nimble build -d:release -Y - run: pip install seleniumbase - run: seleniumbase install chromedriver @@ -37,12 +46,11 @@ jobs: run: | sudo apt install libsass-dev -y cp nitter.example.conf nitter.conf + sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf nimble md nimble scss + echo '${{ secrets.GUEST_ACCOUNTS }}' > ./guest_accounts.jsonl - name: Run tests - env: - GUEST_ACCOUNTS: ${{ secrets.GUEST_ACCOUNTS }} run: | - echo $GUEST_ACCOUNTS > ./guest_accounts.json ./nitter & - pytest -n4 tests + pytest -n8 tests diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index fbad812..70024b2 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -1,7 +1,7 @@ FROM alpine:3.18 as nim LABEL maintainer="setenforce@protonmail.com" -RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.14-r0" nimble pcre +RUN apk --no-cache add libsass-dev pcre gcc git libc-dev "nim=1.6.14-r0" "nimble=0.13.1-r2" WORKDIR /src/nitter @@ -15,9 +15,11 @@ RUN nimble build -d:danger -d:lto -d:strip \ FROM alpine:3.18 WORKDIR /src/ -RUN apk --no-cache add ca-certificates pcre openssl1.1-compat +RUN apk --no-cache add pcre ca-certificates openssl1.1-compat COPY --from=nim /src/nitter/nitter ./ COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf COPY --from=nim /src/nitter/public ./public EXPOSE 8080 +RUN adduser -h /src/ -D -s /bin/sh nitter +USER nitter CMD ./nitter diff --git a/src/auth.nim b/src/auth.nim index 560fb84..b288c50 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -1,5 +1,5 @@ #SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, times, json, random, strutils, tables, intsets, os +import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os] import types import experimental/parser/guestaccount @@ -30,38 +30,50 @@ var template log(str: varargs[string, `$`]) = if enableLogging: echo "[accounts] ", str.join("") +proc snowflakeToEpoch(flake: int64): int64 = + int64(((flake shr 22) + 1288834974657) div 1000) + +proc hasExpired(account: GuestAccount): bool = + let + created = snowflakeToEpoch(account.id) + now = epochTime().int64 + daysOld = int(now - created) div dayInSeconds + return daysOld > 30 + proc getAccountPoolHealth*(): JsonNode = let now = epochTime().int var totalReqs = 0 - limited: IntSet + limited: PackedSet[int64] reqsPerApi: Table[string, int] - oldest = now - newest = 0 - average = 0 + oldest = now.int64 + newest = 0'i64 + average = 0'i64 for account in accountPool: - # Twitter snowflake conversion - let created = ((account.id shr 22) + 1288834974657) div 1000 - + let created = snowflakeToEpoch(account.id) if created > newest: newest = created if created < oldest: oldest = created - average.inc created + average += created for api in account.apis.keys: let apiStatus = account.apis[api] reqs = apiMaxReqs[api] - apiStatus.remaining - reqsPerApi.mgetOrPut($api, 0).inc reqs - totalReqs.inc reqs - if apiStatus.limited: limited.incl account.id + # no requests made with this account and endpoint since the limit reset + if apiStatus.reset < now: + continue + + reqsPerApi.mgetOrPut($api, 0).inc reqs + totalReqs.inc reqs + if accountPool.len > 0: average = average div accountPool.len else: @@ -71,7 +83,6 @@ proc getAccountPoolHealth*(): JsonNode = return %*{ "accounts": %*{ "total": accountPool.len, - "active": accountPool.len - limited.card, "limited": limited.card, "oldest": $fromUnix(oldest), "newest": $fromUnix(newest), @@ -189,3 +200,10 @@ proc initAccountPool*(cfg: Config; path: string) = else: echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests." quit 1 + + let accountsPrePurge = accountPool.len + accountPool.keepItIf(not it.hasExpired) + + log "Successfully added ", accountPool.len, " valid accounts." + if accountsPrePurge > accountPool.len: + log "Purged ", accountsPrePurge - accountPool.len, " expired accounts." diff --git a/src/parser.nim b/src/parser.nim index cebf6f1..fcee13f 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -323,6 +323,8 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet = return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.") of "TweetWithVisibilityResults": return parseGraphTweet(js{"tweet"}, isLegacy) + else: + discard if not js.hasKey("legacy"): return Tweet() diff --git a/src/sass/profile/card.scss b/src/sass/profile/card.scss index 85878e4..46a9679 100644 --- a/src/sass/profile/card.scss +++ b/src/sass/profile/card.scss @@ -115,7 +115,7 @@ } .profile-card-tabs-name { - @include breakable; + flex-shrink: 100; } .profile-card-avatar { diff --git a/src/types.nim b/src/types.nim index 3b0d55c..9ddf283 100644 --- a/src/types.nim +++ b/src/types.nim @@ -36,7 +36,7 @@ type limitedAt*: int GuestAccount* = ref object - id*: BiggestInt + id*: int64 oauthToken*: string oauthSecret*: string pending*: int @@ -164,7 +164,7 @@ type newsletterPublication = "newsletter_publication" hidden unknown - + Card* = object kind*: CardKind url*: string diff --git a/tests/test_card.py b/tests/test_card.py index 733bd40..8da91a2 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -21,9 +21,9 @@ card = [ no_thumb = [ ['FluentAI/status/1116417904831029248', - 'Amazon’s Alexa isn’t just AI — thousands of humans are listening', - 'One of the only ways to improve Alexa is to have human beings check it for errors', - 'theverge.com'], + 'LinkedIn', + 'This link will take you to a page that’s not on LinkedIn', + 'lnkd.in'], ['Thom_Wolf/status/1122466524860702729', 'facebookresearch/fairseq',