Boost deduplicating (#524)

* Trying something with caching boosts

* Use an actual cache for caching

* Persist cache to documents folder

* Stray debugging variable

* Unpublish seen variable in the ViewModel

* Settings for deduplicating boosts.

* Changes from review / merge conflicts
This commit is contained in:
Gareth Simpson 2023-02-01 17:56:06 +00:00 committed by GitHub
parent 06e219597b
commit fdb402a065
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 264 additions and 19 deletions

View file

@ -7,6 +7,8 @@
objects = {
/* Begin PBXBuildFile section */
065FA1FE29866CD600012EA0 /* LRUCache in Frameworks */ = {isa = PBXBuildFile; productRef = 065FA1FD29866CD600012EA0 /* LRUCache */; };
065FA20A298675BA00012EA0 /* LRUCache in Frameworks */ = {isa = PBXBuildFile; productRef = 065FA209298675BA00012EA0 /* LRUCache */; };
639CDF9C296AC82F00C35E58 /* SafariRouter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 639CDF9B296AC82F00C35E58 /* SafariRouter.swift */; };
7429BCE2297C55D00069A946 /* Localizable.stringsdict in Resources */ = {isa = PBXBuildFile; fileRef = 7429BCE4297C55D00069A946 /* Localizable.stringsdict */; };
7429BCE5297C5A750069A946 /* Localizable.stringsdict in Resources */ = {isa = PBXBuildFile; fileRef = 7429BCE4297C55D00069A946 /* Localizable.stringsdict */; };
@ -249,6 +251,7 @@
9FAD859C2974422700496AB1 /* AppAccount in Frameworks */,
9FAD859A297440CB00496AB1 /* KeychainSwift in Frameworks */,
9FAD85982974405D00496AB1 /* Status in Frameworks */,
065FA20A298675BA00012EA0 /* LRUCache in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@ -259,6 +262,7 @@
9F7335EF29674F7100AFF0BA /* QuickLook.framework in Frameworks */,
9F7335ED2967463400AFF0BA /* AVKit.framework in Frameworks */,
9F2A540C29699705009B2D7C /* RevenueCat in Frameworks */,
065FA1FE29866CD600012EA0 /* LRUCache in Frameworks */,
9F2A540E2969A0B0009B2D7C /* StoreKit.framework in Frameworks */,
9F55C6902955993C00F94077 /* Explore in Frameworks */,
9FAE4ACE29379A5A00772766 /* KeychainSwift in Frameworks */,
@ -511,6 +515,7 @@
9FAD859F297456A100496AB1 /* Models */,
9FAD85A1297456A400496AB1 /* Env */,
9FAD85A3297456A800496AB1 /* DesignSystem */,
065FA209298675BA00012EA0 /* LRUCache */,
);
productName = IceCubesShareExtension;
productReference = 9FAD858829743F7400496AB1 /* IceCubesShareExtension.appex */;
@ -548,6 +553,7 @@
9F2A540929699705009B2D7C /* ReceiptParser */,
9F2A540B29699705009B2D7C /* RevenueCat */,
9FE3DB56296FEFCA00628CB0 /* AppAccount */,
065FA1FD29866CD600012EA0 /* LRUCache */,
);
productName = IceCubesApp;
productReference = 9FBFE639292A715500C250E9 /* IceCubesApp.app */;
@ -624,6 +630,7 @@
packageReferences = (
9FAE4ACC29379A5A00772766 /* XCRemoteSwiftPackageReference "keychain-swift" */,
9F2A540829699705009B2D7C /* XCRemoteSwiftPackageReference "purchases-ios" */,
065FA1FC29866CD600012EA0 /* XCRemoteSwiftPackageReference "LRUCache" */,
);
productRefGroup = 9FBFE63A292A715500C250E9 /* Products */;
projectDirPath = "";
@ -1277,6 +1284,14 @@
/* End XCConfigurationList section */
/* Begin XCRemoteSwiftPackageReference section */
065FA1FC29866CD600012EA0 /* XCRemoteSwiftPackageReference "LRUCache" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/nicklockwood/LRUCache";
requirement = {
kind = upToNextMajorVersion;
minimumVersion = 1.0.0;
};
};
9F2A540829699705009B2D7C /* XCRemoteSwiftPackageReference "purchases-ios" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/RevenueCat/purchases-ios.git";
@ -1296,6 +1311,16 @@
/* End XCRemoteSwiftPackageReference section */
/* Begin XCSwiftPackageProductDependency section */
065FA1FD29866CD600012EA0 /* LRUCache */ = {
isa = XCSwiftPackageProductDependency;
package = 065FA1FC29866CD600012EA0 /* XCRemoteSwiftPackageReference "LRUCache" */;
productName = LRUCache;
};
065FA209298675BA00012EA0 /* LRUCache */ = {
isa = XCSwiftPackageProductDependency;
package = 065FA1FC29866CD600012EA0 /* XCRemoteSwiftPackageReference "LRUCache" */;
productName = LRUCache;
};
9F29553F292B6C3400E0E81B /* Timeline */ = {
isa = XCSwiftPackageProductDependency;
productName = Timeline;

View file

@ -27,6 +27,15 @@
"revision" : "e43f9b99b172ae6a7253047f8ba95c7a0b05b99f"
}
},
{
"identity" : "lrucache",
"kind" : "remoteSourceControl",
"location" : "https://github.com/nicklockwood/LRUCache",
"state" : {
"revision" : "6d2b5246c9c98dcd498552bb22f08d55b12a8371",
"version" : "1.0.4"
}
},
{
"identity" : "nuke",
"kind" : "remoteSourceControl",

View file

@ -13,12 +13,17 @@ struct ContentSettingsView: View {
var body: some View {
Form {
Section {
Section("settings.content.boosts") {
Toggle(isOn: $userPreferences.suppressDupeReblogs) {
Text("settings.content.hide-repeated-boosts")
}
}.listRowBackground(theme.primaryBackgroundColor)
Section("settings.content.instance-settings") {
Toggle(isOn: $userPreferences.useInstanceContentSettings) {
Text("settings.content.use-instance-settings")
}
} footer: {
Text("settings.content.main-toggle.description")
}
.listRowBackground(theme.primaryBackgroundColor)
.onChange(of: userPreferences.useInstanceContentSettings) { newVal in

View file

@ -90,7 +90,9 @@
"settings.system" = "Configuració del sistema";
"settings.content.navigation-title" = "Configuració del contacte";
"settings.content.use-instance-settings" = "Utilitza la configuració del servidor";
"settings.content.main-toggle.description" = "Utilitzeu la configuració de la vostra instància d'inici";
"settings.content.boosts" = "Impulsos";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.expand-spoilers" = "Mostra'm sempre els espòilers";
"settings.content.expand-media" = "Visibilitat del contingut multimèdia";
"settings.content.default-sensitive" = "Marca sempre el contingut com a sensible";

View file

@ -114,8 +114,10 @@
"settings.general.content" = "Inhaltseinstellungen";
"settings.system" = "System Settings";
"settings.content.navigation-title" = "Inhaltseinstellungen";
"settings.content.boosts" = "Boosts";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "Servereinstellungen verwenden";
"settings.content.main-toggle.description" = "Die Einstellungen von der Heiminstanz übernehmen";
"settings.content.expand-spoilers" = "Sensible Inhalte immer zeigen";
"settings.content.expand-media" = "Medienansicht";
"settings.content.default-sensitive" = "Medien immer als sensibel kennzeichnen";

View file

@ -94,8 +94,10 @@
"settings.general.content" = "Content Settings";
"settings.system" = "System Settings";
"settings.content.navigation-title" = "Content Settings";
"settings.content.boosts" = "Boosts";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "Use server settings";
"settings.content.main-toggle.description" = "Use the settings from your home Instance";
"settings.content.expand-spoilers" = "Always show sensitive posts";
"settings.content.expand-media" = "Media display";
"settings.content.default-sensitive" = "Always mark media as sensitive";

View file

@ -114,8 +114,10 @@
"settings.general.content" = "Ajustes de contenido";
"settings.system" = "Ajustes del sistema";
"settings.content.navigation-title" = "Ajustes de contenido";
"settings.content.boosts" = "Boosts";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "Usar ajustes del servidor";
"settings.content.main-toggle.description" = "Usar ajustes de tu instancia inicial";
"settings.content.expand-spoilers" = "Mostrar siempre el contenido sensible";
"settings.content.expand-media" = "Mostrar el contenido multimedia";
"settings.content.default-sensitive" = "Marcar siempre el contenido multimedia como sensible";

View file

@ -90,8 +90,10 @@
"settings.general.content" = "Paramètres de contenu";
"settings.system" = "Paramètres système";
"settings.content.navigation-title" = "Paramètres de contenu";
"settings.content.boosts" = "Boosts";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "Utiliser les paramètres du serveur";
"settings.content.main-toggle.description" = "Utiliser les paramètres de votre instance principale";
"settings.content.expand-spoilers" = "Toujours afficher les messages sensibles";
"settings.content.expand-media" = "Affichage des médias";
"settings.content.default-sensitive" = "Toujours marquer les médias comme sensibles";

View file

@ -115,7 +115,9 @@
"settings.system" = "Vai alle impostazioni di sistema";
"settings.content.navigation-title" = "Impostazioni dei contenuti";
"settings.content.use-instance-settings" = "Utilizza le impostazioni del server";
"settings.content.main-toggle.description" = "Utilizza le impostazioni provenienti dalla tua istanza Mastodon";
"settings.content.boosts" = "Boosts";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.expand-spoilers" = "Visualizza sempre i contenuti sensibili";
"settings.content.expand-media" = "Visualizzazione dei media";
"settings.content.default-sensitive" = "Segnala sempre i contenuti come sensibili";

View file

@ -94,8 +94,10 @@
"settings.general.content" = "コンテンツ設定";
"settings.system" = "システム設定";
"settings.content.navigation-title" = "コンテンツ設定";
"settings.content.boosts" = "ブースト";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "サーバー設定を使用する";
"settings.content.main-toggle.description" = "ホームインスタンスの設定を使用する";
"settings.content.expand-spoilers" = "センシティブな投稿を常に表示する";
"settings.content.expand-media" = "メディア表示";
"settings.content.default-sensitive" = "常にメディアをセンシティブなものとしてマークする";

View file

@ -90,8 +90,10 @@
"settings.general.content" = "콘텐츠 설정";
"settings.system" = "시스템 설정";
"settings.content.navigation-title" = "콘텐츠 설정";
"settings.content.boosts" = "부스트";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "인스턴스 설정에 맞추기";
"settings.content.main-toggle.description" = "기본 인스턴스의 설정을 사용합니다.";
"settings.content.expand-spoilers" = "열람 주의 표시된 글 항상 표시하기";
"settings.content.expand-media" = "표시할 미디어";
"settings.content.default-sensitive" = "내 미디어 항상 민감함으로 표시";

View file

@ -94,8 +94,10 @@
"settings.general.content" = "Innholdsinnstillinger";
"settings.system" = "Systeminnstillinger";
"settings.content.navigation-title" = "Innholdsinnstillinger";
"settings.content.boosts" = "Forsterkninger";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "Bruk serverinnstillinger";
"settings.content.main-toggle.description" = "Bruk innstillingene fra din hjemmeinstans";
"settings.content.expand-spoilers" = "Vis alltid sensitive innlegg";
"settings.content.expand-media" = "Medievisning";
"settings.content.default-sensitive" = "Marker alltid medier som sensitive";

View file

@ -114,8 +114,10 @@
"settings.general.content" = "Inhoud";
"settings.system" = "Systeeminstellingen";
"settings.content.navigation-title" = "Inhoud";
"settings.content.boosts" = "Boosts";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "Gebruik serverinstellingen";
"settings.content.main-toggle.description" = "Gebruik de instellingen van jouw thuisinstantie.";
"settings.content.expand-spoilers" = "Toon gevoelige posts altijd";
"settings.content.expand-media" = "Mediaweergave";
"settings.content.default-sensitive" = "Markeer media standaard als gevoelig";

View file

@ -90,8 +90,10 @@
"settings.general.content" = "Ustawienia treści";
"settings.system" = "Ustawienia systemowe";
"settings.content.navigation-title" = "Ustawienia treści";
"settings.content.boosts" = "Podbicia";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "Zastosuj ustawienia serwera";
"settings.content.main-toggle.description" = "Zastosuj ustawienia z twojego serwera macierzystego";
"settings.content.expand-spoilers" = "Pokazuj wrażliwe posty";
"settings.content.expand-media" = "Multimedia";
"settings.content.default-sensitive" = "Oznaczaj media jako wrażliwe";

View file

@ -90,8 +90,10 @@
"settings.general.content" = "Configurações de Conteúdo";
"settings.system" = "Ajustes do Sistema";
"settings.content.navigation-title" = "Configurações de Conteúdo";
"settings.content.boosts" = "Boosts";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "Usar configurações do servidor";
"settings.content.main-toggle.description" = "Usar configurações da sua Instância principal";
"settings.content.expand-spoilers" = "Sempre exibir posts sensíveis";
"settings.content.expand-media" = "Exibição de mídia";
"settings.content.default-sensitive" = "Sempre marcar mídias como sensíveis";

View file

@ -87,6 +87,22 @@
"settings.push.navigation-title" = "İleti Bildirimleri";
"settings.push.new-posts" = "Yeni Gönderiler";
"settings.push.polls" = "Anket Sonuçları";
"settings.general.content" = "Content Settings";
"settings.system" = "System Settings";
"settings.content.navigation-title" = "Content Settings";
"settings.content.boosts" = "Yükseltmeler";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "Use server settings";
"settings.content.expand-spoilers" = "Always show sensitive posts";
"settings.content.expand-media" = "Media display";
"settings.content.default-sensitive" = "Always mark media as sensitive";
"settings.content.default-visibility" = "Posting visibility";
"settings.content.reading" = "Reading";
"settings.content.posting" = "Posting";
"enum.expand-media.show" = "Show All";
"enum.expand-media.hide" = "Hide All";
"enum.expand-media.hide-sensitive" = "Hide Sensitive";
"settings.section.accounts" = "Hesaplar";
"settings.section.app" = "Uygulama";
"settings.section.app.footer %@" = "Uygulama Versiyonu: %@";

View file

@ -115,8 +115,10 @@
"settings.general.content" = "内容设置";
"settings.system" = "系统设置";
"settings.content.navigation-title" = "内容设置";
"settings.content.boosts" = "转发";
"settings.content.hide-repeated-boosts" = "Hide repeated boosts";
"settings.content.instance-settings" = "Server Content Settings";
"settings.content.use-instance-settings" = "使用服务器设置";
"settings.content.main-toggle.description" = "使用你主服务器的设置";
"settings.content.expand-spoilers" = "始终显示敏感内容";
"settings.content.expand-media" = "媒体显示";
"settings.content.default-sensitive" = "始终将媒体标为敏感内容";

View file

@ -28,6 +28,8 @@ public class UserPreferences: ObservableObject {
@AppStorage("autoplay_video") public var autoPlayVideo = true
@AppStorage("chosen_font") public private(set) var chosenFontData: Data?
@AppStorage("suppress_dupe_reblogs") public var suppressDupeReblogs: Bool = false
public var postVisibility: Models.Visibility {
if useInstanceContentSettings {
return serverPreferences?.postVisibility ?? .pub

View file

@ -0,0 +1,134 @@
import Foundation
import Models
import SwiftUI
import LRUCache
import Env
public class ReblogCache {
struct CacheEntry : Codable {
var reblogId:String
var postId:String
var seen:Bool
}
static public let shared = ReblogCache()
var statusCache = LRUCache<String, CacheEntry>()
private var needsWrite = false
init() {
statusCache.countLimit = 100 // can tune the cache here, 100 is super conservative
// read any existing cache from disk
if FileManager.default.fileExists(atPath: self.cacheFile.path()) {
do {
let data = try Data(contentsOf: self.cacheFile)
let cacheData = try JSONDecoder().decode([CacheEntry].self, from: data)
for entry in cacheData {
self.statusCache.setValue(entry, forKey: entry.reblogId)
}
}
catch {
print("Error reading cache from disc")
}
print("Starting cache has \(statusCache.count) items")
}
DispatchQueue.main.asyncAfter(deadline: .now() + 30.0) { [weak self] in
self?.saveCache()
}
}
private func saveCache() {
if needsWrite {
do {
let data = try JSONEncoder().encode(statusCache.allValues)
try data.write(to: self.cacheFile)
}
catch {
print("Error writing cache to disc")
}
needsWrite = false
}
DispatchQueue.main.asyncAfter(deadline: .now() + 30.0) { [weak self] in
self?.saveCache()
}
}
private var cacheFile:URL {
let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)
let documentsDirectory = paths[0]
return URL(fileURLWithPath: documentsDirectory.path()).appendingPathComponent("reblog.json")
}
@MainActor public func removeDuplicateReblogs(_ statuses: inout [Status]) {
if !UserPreferences.shared.suppressDupeReblogs {
return
}
var i = statuses.count
for status in statuses.reversed() {
// go backwards through the status list
// so that we can remove items without
// borking the array
i -= 1
if let reblog = status.reblog {
if let cached = statusCache.value(forKey: reblog.id) {
// this is already cached
if cached.postId != status.id && cached.seen {
// This was posted by someone other than the person we have in the cache
// and we have seen the items at some point, so we might want to suppress it
if status.account.id != CurrentAccount.shared.account?.id {
// just a quick check to makes sure that this wasn't boosted by the current
// user. Hiding that would be confusing
// But assuming it isn't then we can suppress this boost
print("suppressing: \(reblog.id)/ \(reblog.account.displayName) by \(status.account.displayName)")
statuses.remove(at: i)
// assert(statuses.count == (ct-1))
}
}
}
cache(status, seen:false)
}
}
}
public func cache(_ status:Status, seen:Bool) {
var wasSeen = false
var postToCache = status.id
if let reblog = status.reblog {
// only caching boosts at the moment.
if let cached = statusCache.value(forKey: reblog.id) {
// every time we see it, we refresh it in the list
// so poplular things are kept in the cache
wasSeen = cached.seen
if wasSeen {
postToCache = cached.postId
// if we have seen a particular version of the post
// that's the one we keep
}
}
statusCache.setValue(CacheEntry(reblogId: reblog.id, postId: postToCache, seen: seen || wasSeen), forKey: reblog.id)
needsWrite = true
}
}
}

View file

@ -67,6 +67,7 @@ public struct StatusRowView: View {
}
}
.onAppear {
viewModel.markSeen()
if reasons.isEmpty {
viewModel.client = client
if !viewModel.isCompact, viewModel.embeddedStatus == nil {

View file

@ -3,6 +3,9 @@ import Models
import Network
import SwiftUI
@MainActor
public class StatusRowViewModel: ObservableObject {
let status: Status
@ -26,7 +29,9 @@ public class StatusRowViewModel: ObservableObject {
@Published var translation: String?
@Published var isLoadingTranslation: Bool = false
var seen = false
var filter: Filtered? {
status.reblog?.filtered?.first ?? status.filtered?.first
}
@ -62,6 +67,14 @@ public class StatusRowViewModel: ObservableObject {
isFiltered = filter != nil
}
func markSeen() {
// called in on appear so we can cache that the status has been seen.
if UserPreferences.shared.suppressDupeReblogs && !seen {
ReblogCache.shared.cache(status, seen: true)
seen = true
}
}
func navigateToDetail(routerPath: RouterPath) {
guard !isFocused else { return }

View file

@ -152,6 +152,10 @@ extension TimelineViewModel: StatusesFetcher {
maxId: nil,
minId: nil,
offset: statuses.count))
ReblogCache.shared.removeDuplicateReblogs(&statuses)
await cacheHome()
withAnimation {
statusesState = .display(statuses: statuses, nextPageState: statuses.count < 20 ? .none : .hasNextPage)
@ -168,6 +172,9 @@ extension TimelineViewModel: StatusesFetcher {
newStatuses = newStatuses.filter { status in
!statuses.contains(where: { $0.id == status.id })
}
ReblogCache.shared.removeDuplicateReblogs(&newStatuses)
// If no new statuses, resume streaming and exit.
guard !newStatuses.isEmpty else {
@ -222,7 +229,7 @@ extension TimelineViewModel: StatusesFetcher {
var allStatuses: [Status] = []
var latestMinId = minId
do {
while let newStatuses: [Status] = try await client.get(endpoint: timeline.endpoint(sinceId: nil,
while var newStatuses: [Status] = try await client.get(endpoint: timeline.endpoint(sinceId: nil,
maxId: nil,
minId: latestMinId,
offset: statuses.count)),
@ -230,6 +237,9 @@ extension TimelineViewModel: StatusesFetcher {
pagesLoaded < maxPages
{
pagesLoaded += 1
ReblogCache.shared.removeDuplicateReblogs(&newStatuses)
allStatuses.insert(contentsOf: newStatuses, at: 0)
latestMinId = newStatuses.first?.id ?? ""
}
@ -244,10 +254,14 @@ extension TimelineViewModel: StatusesFetcher {
do {
guard let lastId = statuses.last?.id else { return }
statusesState = .display(statuses: statuses, nextPageState: .loadingNextPage)
let newStatuses: [Status] = try await client.get(endpoint: timeline.endpoint(sinceId: nil,
var newStatuses: [Status] = try await client.get(endpoint: timeline.endpoint(sinceId: nil,
maxId: lastId,
minId: nil,
offset: statuses.count))
ReblogCache.shared.removeDuplicateReblogs(&newStatuses)
statuses.append(contentsOf: newStatuses)
statusesState = .display(statuses: statuses, nextPageState: .hasNextPage)
} catch {