mirrors/metatext
1
0
Fork 0
mirror of https://github.com/metabolist/metatext.git synced 2024-11-22 08:10:59 +00:00
Code Issues Projects Releases Wiki Activity

DB refactoring WIP

Browse source
This commit is contained in:
Justin Mazzocchi 2020-10-03 02:19:05 -07:00
parent 291a320ab8
commit 8d1f94d449
No known key found for this signature in database
GPG key ID: E223E6937AAFB01C
18 changed files with 307 additions and 189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
DB/Sources/DB/Content/AccountPinnedStatusJoin.swift
View file

@ -7,8 +7,6 @@ struct AccountPinnedStatusJoin: Codable, FetchableRecord, PersistableRecord {
let accountId: String
let statusId: String
let index: Int
static let status = belongsTo(StatusRecord.self)
}
extension AccountPinnedStatusJoin {
@ -17,4 +15,6 @@ extension AccountPinnedStatusJoin {
static let statusId = Column(AccountPinnedStatusJoin.CodingKeys.statusId)
static let index = Column(AccountPinnedStatusJoin.CodingKeys.index)
}
static let status = belongsTo(StatusRecord.self)
}

15
DB/Sources/DB/Content/ContentDatabase+Migration.swift
View file

@ -71,7 +71,7 @@ extension ContentDatabase {
t.column("profileCollection", .text)
}
try db.create(table: "loadMore") { t in
try db.create(table: "loadMoreRecord") { t in
t.column("timelineId").notNull().references("timelineRecord", onDelete: .cascade)
t.column("afterStatusId", .text).notNull()
@ -96,12 +96,21 @@ extension ContentDatabase {
t.column("wholeWord", .boolean).notNull()
}
try db.create(table: "statusContextJoin") { t in
try db.create(table: "statusAncestorJoin") { t in
t.column("parentId", .text).indexed().notNull()
.references("statusRecord", onDelete: .cascade)
t.column("statusId", .text).indexed().notNull()
.references("statusRecord", onDelete: .cascade)
t.column("index", .integer).notNull()
t.primaryKey(["parentId", "statusId"], onConflict: .replace)
}
try db.create(table: "statusDescendantJoin") { t in
t.column("parentId", .text).indexed().notNull()
.references("statusRecord", onDelete: .cascade)
t.column("statusId", .text).indexed().notNull()
.references("statusRecord", onDelete: .cascade)
t.column("section", .text).indexed().notNull()
t.column("index", .integer).notNull()
t.primaryKey(["parentId", "statusId"], onConflict: .replace)

111
DB/Sources/DB/Content/ContentDatabase.swift
View file

@ -57,7 +57,7 @@ public extension ContentDatabase {
if let maxIDPresent = maxIDPresent,
let minIDInserted = statuses.map(\.id).min(),
minIDInserted > maxIDPresent {
try LoadMore(timelineId: timeline.id, afterStatusId: minIDInserted).save($0)
try LoadMoreRecord(timelineId: timeline.id, afterStatusId: minIDInserted).save($0)
}
}
.ignoreOutput()
@ -66,27 +66,25 @@ public extension ContentDatabase {
func insert(context: Context, parentID: String) -> AnyPublisher<Never, Error> {
databaseWriter.writePublisher {
for status in context.ancestors + context.descendants {
for (index, status) in context.ancestors.enumerated() {
try status.save($0)
try StatusAncestorJoin(parentId: parentID, statusId: status.id, index: index).save($0)
}
for (section, statuses) in [(StatusContextJoin.Section.ancestors, context.ancestors),
(StatusContextJoin.Section.descendants, context.descendants)] {
for (index, status) in statuses.enumerated() {
try StatusContextJoin(
parentId: parentID,
statusId: status.id,
section: section,
index: index)
.save($0)
}
try StatusContextJoin.filter(
StatusContextJoin.Columns.parentId == parentID
&& StatusContextJoin.Columns.section == section.rawValue
&& !statuses.map(\.id).contains(StatusContextJoin.Columns.statusId))
.deleteAll($0)
for (index, status) in context.descendants.enumerated() {
try status.save($0)
try StatusDescendantJoin(parentId: parentID, statusId: status.id, index: index).save($0)
}
try StatusAncestorJoin.filter(
StatusAncestorJoin.Columns.parentId == parentID
&& !context.ancestors.map(\.id).contains(StatusAncestorJoin.Columns.statusId))
.deleteAll($0)
try StatusDescendantJoin.filter(
StatusDescendantJoin.Columns.parentId == parentID
&& !context.descendants.map(\.id).contains(StatusDescendantJoin.Columns.statusId))
.deleteAll($0)
}
.ignoreOutput()
.eraseToAnyPublisher()
@ -96,7 +94,6 @@ public extension ContentDatabase {
databaseWriter.writePublisher {
for (index, status) in pinnedStatuses.enumerated() {
try status.save($0)
try AccountPinnedStatusJoin(accountId: accountID, statusId: status.id, index: index).save($0)
}
@ -175,80 +172,24 @@ public extension ContentDatabase {
.eraseToAnyPublisher()
}
// Awkward maps explained: https://github.com/groue/GRDB.swift#valueobservation-performance
func observation(timeline: Timeline) -> AnyPublisher<[[Timeline.Item]], Error> {
ValueObservation.tracking { db -> ([StatusInfo], [StatusInfo]?, [LoadMore], [Filter]) in
let timelineRecord = TimelineRecord(timeline: timeline)
let statuses = try timelineRecord.statuses.fetchAll(db)
let loadMores = try timelineRecord.loadMores.fetchAll(db)
let filters = try Filter.active.fetchAll(db)
if case let .profile(accountId, profileCollection) = timeline, profileCollection == .statuses {
let pinnedStatuses = try AccountRecord.filter(AccountRecord.Columns.id == accountId)
.fetchOne(db)?.pinnedStatuses.fetchAll(db)
return (statuses, pinnedStatuses, loadMores, filters)
} else {
return (statuses, nil, loadMores, filters)
}
}
.map { statuses, pinnedStatuses, loadMores, filters -> [[Timeline.Item]] in
var timelineItems = statuses.filtered(filters: filters, context: timeline.filterContext)
.map { Timeline.Item.status(.init(status: .init(info: $0))) }
for loadMore in loadMores {
guard let index = timelineItems.firstIndex(where: {
guard case let .status(configuration) = $0 else { return false }
return loadMore.afterStatusId > configuration.status.id
}) else { continue }
timelineItems.insert(.loadMore(loadMore), at: index)
}
if let pinnedStatuses = pinnedStatuses {
return [pinnedStatuses.filtered(filters: filters, context: timeline.filterContext)
.map { Timeline.Item.status(.init(status: .init(info: $0), pinned: true)) },
timelineItems]
} else {
return [timelineItems]
}
ValueObservation.tracking { db -> (TimelineItemsInfo?, [Filter]) in
(try TimelineItemsInfo.request(
TimelineRecord.filter(TimelineRecord.Columns.id == timeline.id)).fetchOne(db),
try Filter.active.fetchAll(db))
}
.map { $0?.items(filters: $1) ?? [] }
.removeDuplicates()
.publisher(in: databaseWriter)
.eraseToAnyPublisher()
}
func contextObservation(parentID: String) -> AnyPublisher<[[Timeline.Item]], Error> {
ValueObservation.tracking { db -> ([[StatusInfo]], [Filter]) in
guard let parent = try StatusInfo.request(StatusRecord.filter(StatusRecord.Columns.id == parentID))
.fetchOne(db) else {
return ([], [])
}
let ancestors = try parent.record.ancestors.fetchAll(db)
let descendants = try parent.record.descendants.fetchAll(db)
return ([ancestors, [parent], descendants], try Filter.active.fetchAll(db))
}
.map { statusSections, filters in
statusSections.map { section in
section.filtered(filters: filters, context: .thread)
.enumerated()
.map { index, statusInfo in
let isReplyInContext = index > 0
&& section[index - 1].record.id == statusInfo.record.inReplyToId
let hasReplyFollowing = section.count > index + 1
&& section[index + 1].record.inReplyToId == statusInfo.record.id
return Timeline.Item.status(
.init(status: .init(info: statusInfo),
isReplyInContext: isReplyInContext,
hasReplyFollowing: hasReplyFollowing))
}
}
ValueObservation.tracking { db -> (ContextItemsInfo?, [Filter]) in
(try ContextItemsInfo.request(StatusRecord.filter(StatusRecord.Columns.id == parentID)).fetchOne(db),
try Filter.active.fetchAll(db))
}
.map { $0?.items(filters: $1) ?? [] }
.removeDuplicates()
.publisher(in: databaseWriter)
.eraseToAnyPublisher()
@ -259,8 +200,8 @@ public extension ContentDatabase {
.order(TimelineRecord.Columns.listTitle.asc)
.fetchAll)
.removeDuplicates()
.map { $0.map(Timeline.init(record:)).compactMap { $0 } }
.publisher(in: databaseWriter)
.tryMap { $0.map(Timeline.init(record:)).compactMap { $0 } }
.eraseToAnyPublisher()
}

42
DB/Sources/DB/Content/ContextItemsInfo.swift Normal file
View file

@ -0,0 +1,42 @@
// Copyright © 2020 Metabolist. All rights reserved.
import Foundation
import GRDB
import Mastodon
struct ContextItemsInfo: Codable, Hashable, FetchableRecord {
let parent: StatusInfo
let ancestors: [StatusInfo]
let descendants: [StatusInfo]
}
extension ContextItemsInfo {
static func addingIncludes<T: DerivableRequest>(_ request: T) -> T where T.RowDecoder == StatusRecord {
StatusInfo.addingIncludes(request)
.including(all: StatusInfo.addingIncludes(StatusRecord.ancestors).forKey(CodingKeys.ancestors))
.including(all: StatusInfo.addingIncludes(StatusRecord.descendants).forKey(CodingKeys.descendants))
}
static func request(_ request: QueryInterfaceRequest<StatusRecord>) -> QueryInterfaceRequest<Self> {
addingIncludes(request).asRequest(of: self)
}
func items(filters: [Filter]) -> [[Timeline.Item]] {
let regularExpression = filters.regularExpression(context: .thread)
return [ancestors, [parent], descendants].map { section in
section.filtered(regularExpression: regularExpression)
.enumerated()
.map { index, statusInfo in
let isReplyInContext = index > 0
&& section[index - 1].record.id == statusInfo.record.inReplyToId
let hasReplyFollowing = section.count > index + 1
&& section[index + 1].record.inReplyToId == statusInfo.record.id
return .status(.init(status: .init(info: statusInfo),
isReplyInContext: isReplyInContext,
hasReplyFollowing: hasReplyFollowing))
}
}
}
}

27
DB/Sources/DB/Content/LoadMoreRecord.swift Normal file
View file

@ -0,0 +1,27 @@
// Copyright © 2020 Metabolist. All rights reserved.
import Foundation
import GRDB
import Mastodon
struct LoadMoreRecord: Codable, Hashable {
let timelineId: String
let afterStatusId: String
}
extension LoadMoreRecord {
enum Columns {
static let timelineId = Column(LoadMoreRecord.CodingKeys.timelineId)
static let afterStatusId = Column(LoadMoreRecord.CodingKeys.afterStatusId)
}
}
extension LoadMoreRecord: FetchableRecord, PersistableRecord {
static func databaseJSONDecoder(for column: String) -> JSONDecoder {
MastodonDecoder()
}
static func databaseJSONEncoder(for column: String) -> JSONEncoder {
MastodonEncoder()
}
}

20
DB/Sources/DB/Content/StatusAncestorJoin.swift Normal file
View file

@ -0,0 +1,20 @@
// Copyright © 2020 Metabolist. All rights reserved.
import Foundation
import GRDB
struct StatusAncestorJoin: Codable, FetchableRecord, PersistableRecord {
let parentId: String
let statusId: String
let index: Int
static let status = belongsTo(StatusRecord.self, using: ForeignKey([Columns.statusId]))
}
extension StatusAncestorJoin {
enum Columns {
static let parentId = Column(StatusAncestorJoin.CodingKeys.parentId)
static let statusId = Column(StatusAncestorJoin.CodingKeys.statusId)
static let index = Column(StatusAncestorJoin.CodingKeys.index)
}
}

27
DB/Sources/DB/Content/StatusContextJoin.swift
View file

@ -1,27 +0,0 @@
// Copyright © 2020 Metabolist. All rights reserved.
import Foundation
import GRDB
struct StatusContextJoin: Codable, FetchableRecord, PersistableRecord {
enum Section: String, Codable {
case ancestors
case descendants
}
let parentId: String
let statusId: String
let section: Section
let index: Int
static let status = belongsTo(StatusRecord.self, using: ForeignKey([Columns.statusId]))
}
extension StatusContextJoin {
enum Columns {
static let parentId = Column(StatusContextJoin.CodingKeys.parentId)
static let statusId = Column(StatusContextJoin.CodingKeys.statusId)
static let section = Column(StatusContextJoin.CodingKeys.section)
static let index = Column(StatusContextJoin.CodingKeys.index)
}
}

20
DB/Sources/DB/Content/StatusDescendantJoin.swift Normal file
View file

@ -0,0 +1,20 @@
// Copyright © 2020 Metabolist. All rights reserved.
import Foundation
import GRDB
struct StatusDescendantJoin: Codable, FetchableRecord, PersistableRecord {
let parentId: String
let statusId: String
let index: Int
static let status = belongsTo(StatusRecord.self, using: ForeignKey([Columns.statusId]))
}
extension StatusDescendantJoin {
enum Columns {
static let parentId = Column(StatusDescendantJoin.CodingKeys.parentId)
static let statusId = Column(StatusDescendantJoin.CodingKeys.statusId)
static let index = Column(StatusDescendantJoin.CodingKeys.index)
}
}

18
DB/Sources/DB/Content/StatusRecord.swift
View file

@ -93,21 +93,19 @@ extension StatusRecord {
using: AccountRecord.moved)
static let reblog = belongsTo(StatusRecord.self)
static let ancestorJoins = hasMany(
StatusContextJoin.self,
using: ForeignKey([StatusContextJoin.Columns.parentId]))
.filter(StatusContextJoin.Columns.section == StatusContextJoin.Section.ancestors.rawValue)
.order(StatusContextJoin.Columns.index)
StatusAncestorJoin.self,
using: ForeignKey([StatusAncestorJoin.Columns.parentId]))
.order(StatusAncestorJoin.Columns.index)
static let descendantJoins = hasMany(
StatusContextJoin.self,
using: ForeignKey([StatusContextJoin.Columns.parentId]))
.filter(StatusContextJoin.Columns.section == StatusContextJoin.Section.descendants.rawValue)
.order(StatusContextJoin.Columns.index)
StatusDescendantJoin.self,
using: ForeignKey([StatusDescendantJoin.Columns.parentId]))
.order(StatusDescendantJoin.Columns.index)
static let ancestors = hasMany(StatusRecord.self,
through: ancestorJoins,
using: StatusContextJoin.status)
using: StatusAncestorJoin.status)
static let descendants = hasMany(StatusRecord.self,
through: descendantJoins,
using: StatusContextJoin.status)
using: StatusDescendantJoin.status)
var ancestors: QueryInterfaceRequest<StatusInfo> {
StatusInfo.request(request(for: Self.ancestors))

64
DB/Sources/DB/Content/TimelineItemsInfo.swift Normal file
View file

@ -0,0 +1,64 @@
// Copyright © 2020 Metabolist. All rights reserved.
import Foundation
import GRDB
import Mastodon
struct TimelineItemsInfo: Codable, Hashable, FetchableRecord {
let timelineRecord: TimelineRecord
let statusInfos: [StatusInfo]
let pinnedStatusesInfo: PinnedStatusesInfo?
let loadMoreRecords: [LoadMoreRecord]
}
extension TimelineItemsInfo {
struct PinnedStatusesInfo: Codable, Hashable, FetchableRecord {
let accountRecord: AccountRecord
let pinnedStatusInfos: [StatusInfo]
}
static func addingIncludes<T: DerivableRequest>( _ request: T) -> T where T.RowDecoder == TimelineRecord {
request.including(all: StatusInfo.addingIncludes(TimelineRecord.statuses).forKey(CodingKeys.statusInfos))
.including(all: TimelineRecord.loadMores.forKey(CodingKeys.loadMoreRecords))
.including(optional: PinnedStatusesInfo.addingIncludes(TimelineRecord.account)
.forKey(CodingKeys.pinnedStatusesInfo))
}
static func request(_ request: QueryInterfaceRequest<TimelineRecord>) -> QueryInterfaceRequest<Self> {
addingIncludes(request).asRequest(of: self)
}
func items(filters: [Filter]) -> [[Timeline.Item]] {
let timeline = Timeline(record: timelineRecord)!
let filterRegularExpression = filters.regularExpression(context: timeline.filterContext)
var timelineItems = statusInfos.filtered(regularExpression: filterRegularExpression)
.map { Timeline.Item.status(.init(status: .init(info: $0))) }
for loadMoreRecord in loadMoreRecords {
guard let index = timelineItems.firstIndex(where: {
guard case let .status(configuration) = $0 else { return false }
return loadMoreRecord.afterStatusId > configuration.status.id
}) else { continue }
timelineItems.insert(
.loadMore(LoadMore(timeline: timeline, afterStatusId: loadMoreRecord.afterStatusId)),
at: index)
}
if let pinnedStatusInfos = pinnedStatusesInfo?.pinnedStatusInfos {
return [pinnedStatusInfos.filtered(regularExpression: filterRegularExpression)
.map { Timeline.Item.status(.init(status: .init(info: $0), pinned: true)) },
timelineItems]
} else {
return [timelineItems]
}
}
}
extension TimelineItemsInfo.PinnedStatusesInfo {
static func addingIncludes<T: DerivableRequest>(_ request: T) -> T where T.RowDecoder == AccountRecord {
request.including(all: StatusInfo.addingIncludes(AccountRecord.pinnedStatuses)
.forKey(CodingKeys.pinnedStatusInfos))
}
}

5
DB/Sources/DB/Content/TimelineRecord.swift
View file

@ -39,13 +39,14 @@ extension TimelineRecord {
through: statusJoins,
using: TimelineStatusJoin.status)
.order(StatusRecord.Columns.createdAt.desc)
static let loadMores = hasMany(LoadMore.self)
static let account = belongsTo(AccountRecord.self, using: ForeignKey([Columns.accountId]))
static let loadMores = hasMany(LoadMoreRecord.self)
var statuses: QueryInterfaceRequest<StatusInfo> {
StatusInfo.request(request(for: Self.statuses))
}
var loadMores: QueryInterfaceRequest<LoadMore> {
var loadMores: QueryInterfaceRequest<LoadMoreRecord> {
request(for: Self.loadMores)
}

22
DB/Sources/DB/Entities/LoadMore.swift
View file

@ -4,24 +4,14 @@ import Foundation
import GRDB
import Mastodon
public struct LoadMore: Codable, Hashable {
public let timelineId: String
public struct LoadMore: Hashable {
public let timeline: Timeline
public let afterStatusId: String
}
extension LoadMore: FetchableRecord, PersistableRecord {
public static func databaseJSONDecoder(for column: String) -> JSONDecoder {
MastodonDecoder()
}
public static func databaseJSONEncoder(for column: String) -> JSONEncoder {
MastodonEncoder()
}
}
extension LoadMore {
enum Columns {
static let timelineId = Column(LoadMore.CodingKeys.timelineId)
static let belowStatusId = Column(LoadMore.CodingKeys.afterStatusId)
public extension LoadMore {
enum Direction {
case up
case down
}
}

4
DB/Sources/DB/Extensions/Filter+Extensions.swift
View file

@ -30,8 +30,8 @@ extension Filter {
}
extension Array where Element == StatusInfo {
func filtered(filters: [Filter], context: Filter.Context) -> Self {
guard let regEx = filters.filter({ $0.context.contains(context) }).regularExpression() else { return self }
func filtered(regularExpression: String?) -> Self {
guard let regEx = regularExpression else { return self }
return filter { $0.filterableContent.range(of: regEx, options: [.regularExpression, .caseInsensitive]) == nil }
}

8
Mastodon/Sources/Mastodon/Entities/Filter.swift
View file

@ -36,10 +36,12 @@ extension Array where Element == Filter {
// swiftlint:disable line_length
// Adapted from https://github.com/tootsuite/mastodon/blob/bf477cee9f31036ebf3d164ddec1cebef5375513/app/javascript/mastodon/selectors/index.js#L43
// swiftlint:enable line_length
public func regularExpression() -> String? {
guard !isEmpty else { return nil }
public func regularExpression(context: Filter.Context) -> String? {
let inContext = filter { $0.context.contains(context) }
return map {
guard !inContext.isEmpty else { return nil }
return inContext.map {
var expression = NSRegularExpression.escapedPattern(for: $0.phrase)
if $0.wholeWord {

39
ServiceLayer/Sources/ServiceLayer/Entities/Timeline.swift
View file

@ -1,5 +1,44 @@
// Copyright © 2020 Metabolist. All rights reserved.
import DB
import MastodonAPI
public typealias Timeline = DB.Timeline
extension Timeline {
var endpoint: StatusesEndpoint {
switch self {
case .home:
return .timelinesHome
case .local:
return .timelinesPublic(local: true)
case .federated:
return .timelinesPublic(local: false)
case let .list(list):
return .timelinesList(id: list.id)
case let .tag(tag):
return .timelinesTag(tag)
case let .profile(accountId, profileCollection):
let excludeReplies: Bool
let onlyMedia: Bool
switch profileCollection {
case .statuses:
excludeReplies = true
onlyMedia = false
case .statusesAndReplies:
excludeReplies = false
onlyMedia = false
case .media:
excludeReplies = true
onlyMedia = true
}
return .accountsStatuses(
id: accountId,
excludeReplies: excludeReplies,
onlyMedia: onlyMedia,
pinned: false)
}
}
}

28
ServiceLayer/Sources/ServiceLayer/Services/LoadMoreService.swift Normal file
View file

@ -0,0 +1,28 @@
// Copyright © 2020 Metabolist. All rights reserved.
import Combine
import DB
import MastodonAPI
public struct LoadMoreService {
private let loadMore: LoadMore
private let mastodonAPIClient: MastodonAPIClient
private let contentDatabase: ContentDatabase
init(loadMore: LoadMore, mastodonAPIClient: MastodonAPIClient, contentDatabase: ContentDatabase) {
self.loadMore = loadMore
self.mastodonAPIClient = mastodonAPIClient
self.contentDatabase = contentDatabase
}
}
public extension LoadMoreService {
func request(direction: LoadMore.Direction) -> AnyPublisher<Never, Error> {
mastodonAPIClient.pagedRequest(
loadMore.timeline.endpoint,
maxID: direction == .down ? loadMore.afterStatusId : nil,
minID: direction == .up ? loadMore.afterStatusId : nil)
.flatMap { contentDatabase.insert(statuses: $0.result, timeline: loadMore.timeline) }
.eraseToAnyPublisher()
}
}

38
ServiceLayer/Sources/ServiceLayer/Services/StatusListService.swift
View file

@ -76,41 +76,3 @@ public extension StatusListService {
requestClosure(maxID, minID)
}
}
private extension Timeline {
var endpoint: StatusesEndpoint {
switch self {
case .home:
return .timelinesHome
case .local:
return .timelinesPublic(local: true)
case .federated:
return .timelinesPublic(local: false)
case let .list(list):
return .timelinesList(id: list.id)
case let .tag(tag):
return .timelinesTag(tag)
case let .profile(accountId, profileCollection):
let excludeReplies: Bool
let onlyMedia: Bool
switch profileCollection {
case .statuses:
excludeReplies = true
onlyMedia = false
case .statusesAndReplies:
excludeReplies = false
onlyMedia = false
case .media:
excludeReplies = true
onlyMedia = true
}
return .accountsStatuses(
id: accountId,
excludeReplies: excludeReplies,
onlyMedia: onlyMedia,
pinned: false)
}
}
}

4
ViewModels/Sources/ViewModels/LoadMoreViewModel.swift
View file

@ -1,5 +1,7 @@
// Copyright © 2020 Metabolist. All rights reserved.
public struct LoadMoreViewModel {
import Combine
public class LoadMoreViewModel: ObservableObject {
@Published var loading = false
}