From 555ea8edfb2c30d149b3ca6cb0fbe53f2798c7bc Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Thu, 9 Sep 2021 16:15:25 +0200 Subject: [PATCH] Import export (#194) * start with export/import code * messing about with decoding/encoding * some more fiddling * stuff is WORKING * working pretty alright! * go fmt * fix up tests, add docs * start backup/restore doc * tweaks * credits * update advancedVisibility settings * update bun library -> v1.0.4 Signed-off-by: kim (grufwub) * update oauth library -> v4.3.1-SSB Signed-off-by: kim (grufwub) * handle oauth token scope, fix user.SigninCount + token.UserID Signed-off-by: kim (grufwub) * update oauth library --> v4.3.2-SSB Signed-off-by: kim (grufwub) * update sqlite library -> v1.13.0 Signed-off-by: kim (grufwub) * review changes * start with export/import code * messing about with decoding/encoding * some more fiddling * stuff is WORKING * working pretty alright! * go fmt * fix up tests, add docs * start backup/restore doc * tweaks * credits * update advancedVisibility settings * review changes Co-authored-by: kim (grufwub) Co-authored-by: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> --- README.md | 3 +- cmd/gotosocial/admincommands.go | 79 +- docs/admin/backup_and_restore.md | 102 ++ docs/admin/cli.md | 75 + go.mod | 1 + go.sum | 2 + internal/api/s2s/user/repliesget_test.go | 6 +- internal/cache/status.go | 5 +- internal/cliactions/admin/trans/export.go | 52 + internal/cliactions/admin/trans/import.go | 56 + internal/cliactions/server/server.go | 33 +- internal/config/config.go | 8 + internal/db/basic.go | 4 + internal/db/bundb/basic.go | 64 +- internal/db/bundb/basic_test.go | 21 +- internal/db/bundb/status.go | 8 +- internal/db/bundb/status_test.go | 30 +- internal/db/bundb/util.go | 63 + internal/db/params.go | 6 +- internal/federation/dereferencing/announce.go | 5 +- .../federation/dereferencing/status_test.go | 8 + internal/gtsmodel/status.go | 25 +- internal/processing/fromclientapi.go | 2 +- internal/processing/status/boost.go | 2 +- internal/processing/status/fave.go | 2 +- internal/processing/status/util.go | 41 +- internal/timeline/get_test.go | 16 +- internal/timeline/index_test.go | 4 +- internal/timeline/manager_test.go | 16 +- internal/trans/decoders.go | 138 ++ internal/trans/encoders.go | 83 + internal/trans/export.go | 223 +++ internal/trans/exporter.go | 46 + internal/trans/exportminimal.go | 150 ++ internal/trans/exportminimal_test.go | 54 + internal/trans/import.go | 146 ++ internal/trans/import_test.go | 91 + internal/trans/importer.go | 44 + internal/trans/model/account.go | 54 + internal/trans/model/block.go | 31 + internal/trans/model/domainblock.go | 34 + internal/trans/model/follow.go | 31 + internal/trans/model/followrequest.go | 31 + internal/trans/model/instance.go | 43 + internal/trans/model/type.go | 41 + internal/trans/model/user.go | 50 + internal/trans/trans_test.go | 42 + internal/trans/util.go | 32 + internal/typeutils/astointernal.go | 5 + internal/typeutils/internal.go | 5 +- internal/validate/status_test.go | 16 +- testrig/db.go | 30 +- testrig/testmodels.go | 208 ++- .../mitchellh/mapstructure/CHANGELOG.md | 73 + .../github.com/mitchellh/mapstructure/LICENSE | 21 + .../mitchellh/mapstructure/README.md | 46 + .../mitchellh/mapstructure/decode_hooks.go | 256 +++ .../mitchellh/mapstructure/error.go | 50 + .../github.com/mitchellh/mapstructure/go.mod | 3 + .../mitchellh/mapstructure/mapstructure.go | 1462 +++++++++++++++++ vendor/modules.txt | 3 + 61 files changed, 4031 insertions(+), 250 deletions(-) create mode 100644 docs/admin/backup_and_restore.md create mode 100644 internal/cliactions/admin/trans/export.go create mode 100644 internal/cliactions/admin/trans/import.go create mode 100644 internal/trans/decoders.go create mode 100644 internal/trans/encoders.go create mode 100644 internal/trans/export.go create mode 100644 internal/trans/exporter.go create mode 100644 internal/trans/exportminimal.go create mode 100644 internal/trans/exportminimal_test.go create mode 100644 internal/trans/import.go create mode 100644 internal/trans/import_test.go create mode 100644 internal/trans/importer.go create mode 100644 internal/trans/model/account.go create mode 100644 internal/trans/model/block.go create mode 100644 internal/trans/model/domainblock.go create mode 100644 internal/trans/model/follow.go create mode 100644 internal/trans/model/followrequest.go create mode 100644 internal/trans/model/instance.go create mode 100644 internal/trans/model/type.go create mode 100644 internal/trans/model/user.go create mode 100644 internal/trans/trans_test.go create mode 100644 internal/trans/util.go create mode 100644 vendor/github.com/mitchellh/mapstructure/CHANGELOG.md create mode 100644 vendor/github.com/mitchellh/mapstructure/LICENSE create mode 100644 vendor/github.com/mitchellh/mapstructure/README.md create mode 100644 vendor/github.com/mitchellh/mapstructure/decode_hooks.go create mode 100644 vendor/github.com/mitchellh/mapstructure/error.go create mode 100644 vendor/github.com/mitchellh/mapstructure/go.mod create mode 100644 vendor/github.com/mitchellh/mapstructure/mapstructure.go diff --git a/README.md b/README.md index f907e9c05..8703aa657 100644 --- a/README.md +++ b/README.md @@ -134,10 +134,11 @@ The following libraries and frameworks are used by GoToSocial, with gratitude * [gorilla/websocket](https://github.com/gorilla/websocket); Websocket connectivity. [BSD-2-Clause License](https://spdx.org/licenses/BSD-2-Clause.html). * [h2non/filetype](https://github.com/h2non/filetype); filetype checking. [MIT License](https://spdx.org/licenses/MIT.html). * [jackc/pgx](https://github.com/jackc/pgx); Postgres driver. [MIT License](https://spdx.org/licenses/MIT.html). +* [microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday); HTML user-input sanitization. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). +* [mitchellh/mapstructure](https://github.com/mitchellh/mapstructure); Go interface => struct parsing. [MIT License](https://spdx.org/licenses/MIT.html). * [modernc.org/sqlite](https://gitlab.com/cznic/sqlite); cgo-free port of SQLite. [Other License](https://gitlab.com/cznic/sqlite/-/blob/master/LICENSE). * [modernc.org/ccgo](https://gitlab.com/cznic/ccgo); c99 AST -> Go translater. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). * [modernc.org/libc](https://gitlab.com/cznic/libc); C-runtime services. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). -* [microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday); HTML user-input sanitization. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). * [mvdan/xurls](https://github.com/mvdan/xurls); URL parsing regular expressions. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). * [nfnt/resize](https://github.com/nfnt/resize); convenient image resizing. [ISC License](https://spdx.org/licenses/ISC.html). * [oklog/ulid](https://github.com/oklog/ulid); sequential, database-friendly ID generation. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html). diff --git a/cmd/gotosocial/admincommands.go b/cmd/gotosocial/admincommands.go index a777ee525..5d505fe77 100644 --- a/cmd/gotosocial/admincommands.go +++ b/cmd/gotosocial/admincommands.go @@ -20,6 +20,7 @@ package main import ( "github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/account" + "github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/trans" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/urfave/cli/v2" ) @@ -39,16 +40,19 @@ func adminCommands() []*cli.Command { Usage: "create a new account", Flags: []cli.Flag{ &cli.StringFlag{ - Name: config.UsernameFlag, - Usage: config.UsernameUsage, + Name: config.UsernameFlag, + Usage: config.UsernameUsage, + Required: true, }, &cli.StringFlag{ - Name: config.EmailFlag, - Usage: config.EmailUsage, + Name: config.EmailFlag, + Usage: config.EmailUsage, + Required: true, }, &cli.StringFlag{ - Name: config.PasswordFlag, - Usage: config.PasswordUsage, + Name: config.PasswordFlag, + Usage: config.PasswordUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -60,8 +64,9 @@ func adminCommands() []*cli.Command { Usage: "confirm an existing account manually, thereby skipping email confirmation", Flags: []cli.Flag{ &cli.StringFlag{ - Name: config.UsernameFlag, - Usage: config.UsernameUsage, + Name: config.UsernameFlag, + Usage: config.UsernameUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -73,8 +78,9 @@ func adminCommands() []*cli.Command { Usage: "promote an account to admin", Flags: []cli.Flag{ &cli.StringFlag{ - Name: config.UsernameFlag, - Usage: config.UsernameUsage, + Name: config.UsernameFlag, + Usage: config.UsernameUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -86,8 +92,9 @@ func adminCommands() []*cli.Command { Usage: "demote an account from admin to normal user", Flags: []cli.Flag{ &cli.StringFlag{ - Name: config.UsernameFlag, - Usage: config.UsernameUsage, + Name: config.UsernameFlag, + Usage: config.UsernameUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -99,8 +106,9 @@ func adminCommands() []*cli.Command { Usage: "prevent an account from signing in or posting etc, but don't delete anything", Flags: []cli.Flag{ &cli.StringFlag{ - Name: config.UsernameFlag, - Usage: config.UsernameUsage, + Name: config.UsernameFlag, + Usage: config.UsernameUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -112,8 +120,9 @@ func adminCommands() []*cli.Command { Usage: "completely remove an account and all of its posts, media, etc", Flags: []cli.Flag{ &cli.StringFlag{ - Name: config.UsernameFlag, - Usage: config.UsernameUsage, + Name: config.UsernameFlag, + Usage: config.UsernameUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -125,12 +134,14 @@ func adminCommands() []*cli.Command { Usage: "set a new password for the given account", Flags: []cli.Flag{ &cli.StringFlag{ - Name: config.UsernameFlag, - Usage: config.UsernameUsage, + Name: config.UsernameFlag, + Usage: config.UsernameUsage, + Required: true, }, &cli.StringFlag{ - Name: config.PasswordFlag, - Usage: config.PasswordUsage, + Name: config.PasswordFlag, + Usage: config.PasswordUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -139,6 +150,34 @@ func adminCommands() []*cli.Command { }, }, }, + { + Name: "export", + Usage: "export data from the database to file at the given path", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: config.TransPathFlag, + Usage: config.TransPathUsage, + Required: true, + }, + }, + Action: func(c *cli.Context) error { + return runAction(c, trans.Export) + }, + }, + { + Name: "import", + Usage: "import data from a file into the database", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: config.TransPathFlag, + Usage: config.TransPathUsage, + Required: true, + }, + }, + Action: func(c *cli.Context) error { + return runAction(c, trans.Import) + }, + }, }, }, } diff --git a/docs/admin/backup_and_restore.md b/docs/admin/backup_and_restore.md new file mode 100644 index 000000000..d406c529b --- /dev/null +++ b/docs/admin/backup_and_restore.md @@ -0,0 +1,102 @@ +# Backup and Restore + +In certain conditions, it may be desirable to be able to back up a GoToSocial instance, and then to restore it again later, or just save the backup somewhere. + +Some potential scenarios: + +* You want to close down your instance but you might create it again later and you don't want to break federation. +* You need to migrate to a different database for some reason (Postgres => SQLite or vice versa). +* You want to keep regular backups of your data just in case something happens. +* You want to migrate from GoToSocial to a different Fediverse server, or from a different Fediverse server to GoToSocial. +* You're about to hack around on your instance and you want to make a quick backup so you don't lose everything if you mess up. + +There are a few different ways of doing this, most of which require some technical knowledge. + +## Image your disk + +If you're running GoToSocial on a VPS (a remote machine in the cloud), arguably the easiest way to preserve all of your database entries and media is to image the disk attached to the VPS. This will preserve the whole disk. Many VPS providers offer the option of automatically creating backups on a timer, so you'll always be able to restore if your data is lost. + +Advantages: + +* Relatively easy to do. +* Easy to automate (depending on your vps). +* Keep complete media + database entries. + +Disadvantages: + +* Can cost extra depending on your VPS. +* Will probably also preserve stuff you don't need, from other programs running on the same machine. +* Vendor lock-in, difficult to move the data around. + +## Back up your database files + +Regardless of whether you're using Postgres or SQLite as your GoToSocial database, it's possible to simply back up the database files directly by using something like [rclone](https://rclone.org/), or following best practices for [backing up Postgres data](https://www.postgresql.org/docs/9.1/backup.html) or [SQLite data](https://sqlite.org/backup.html). + +Advantages: + +* Backups are relatively portable - you can move data from one machine to another. +* Well-documented procedure with a lot of guides and tooling available. +* Lots of different ways of doing your backups, depending on what you need. + +Disadvantages: + +* Can be a bit fiddly to set up initially. +* You need to figure out where to keep your backups. +* Restoring from backups can be a pain. +* Unless you back up media as well, references to media attachments in your db will be broken. + +## Use the GoToSocial CLI + +The GoToSocial CLI tool also provides commands for backing up and restoring data from your instance, which will preserve the *bare-minimum* necessary data to backup and restore your instance, without breaking federation with other instances. + +What will be **kept**: + +* All local account entries, including private and public keys. +* Followed/following remote accounts, including public keys. +* Follows/follow requests. +* Domain blocks. +* Account blocks. +* Account suspensions. +* User + password entries, email addresses. + +What will be **dropped**: + +* All statuses. +* Media. +* Faves. +* Bookmarks. +* Pins. +* Applications. +* Tokens. + +The backup file produced will be in the form of a line-separated series of JSON objects (not a JSON array!). For example: + +```json +{"type":"account","id":"01F8MH5NBDF2MV7CTC4Q5128HF","createdAt":"2021-08-31T12:00:53.985645Z","username":"1happyturtle","locked":true,"language":"en","uri":"http://localhost:8080/users/1happyturtle","url":"http://localhost:8080/@1happyturtle","inboxURI":"http://localhost:8080/users/1happyturtle/inbox","outboxURI":"http://localhost:8080/users/1happyturtle/outbox","followingUri":"http://localhost:8080/users/1happyturtle/following","followersUri":"http://localhost:8080/users/1happyturtle/followers","featuredCollectionUri":"http://localhost:8080/users/1happyturtle/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjz\nausfsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLz\neUPxdfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFx\njUz9l0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJY\nfKhKn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq\n79WbhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQABAoIBAGF+MxHjD15VV2NY\nKKb1GjMx98i1Xx6TijgoA+zmfha4LGu35e79Lql+0LXFp0zEpa6lAQsMQQhgd0OD\nmKKmSk+pxAvskJ4FxrhIf/yBFA4RMrj5OCaAOocRtdsOJ8n5UtFBrNAF0tzMY9q/\nkgzoq97aVF1mV9iFxaeBx6zT8ozSdqBq1PK/3w1dVg89S5tfKYc7Q0lQ00SfsTnd\niTDClKyqurebo9Pt6M7gXavgg3tvBlmwwr6XHs34Leng3oiN9mW8DVzaBMPzn+rE\nxF2eqs3v9vVpj8es88OwCh5P+ff8vJYvhu7Fcr/bJ8BItBQwfb8QBDATg/MXU2BI\n2ssW6AECgYEA4wmIyYGeu9+hzDa/J3Vh8GnlVNUCohHcChQdOsWsFXUgpVlUIHrX\neKHn42vD4Rzy52/YzJts4NkZTM9sL+kEXIEcpMG/S9xIIud7U0m/hMSAlmnJK/9j\niEXws3o4jo0E77jnRcBdIjpG4K5Eekm0DSR3SFhtZfEdN2DWPvu7K98CgYEA5tER\n/qJwFMc51AobMU87ZjXON7hI2U1WY/pVF62jSl0IcSsnj2riEKWLrs+GRG+HUg+U\naFSqAHcxaVHA0h0AYR8RopAhDdVKh0kvB8biLo+IEzNjPv2vyn0yRN5YSfXdGzyJ\nUjVU6kWdQOwmzy86nHgFaqEx7eofHIaGZzJK/AECgYEAu2VNQHX63TuzQuoVUa5z\nzoq5vhGsALYZF0CO98ndRkDNV22qIL0ESQ/qZS64GYFZhWouWoQXlGfdmCbFN65v\n6SKwz9UT3rvN1vGWO6Ltr9q6AG0EnYpJT1vbV2kUcaU4Y94NFue2d9/+TMnKv91B\n/m8Q/efvNGuWH/WQIaCKV6UCgYBz89WhYMMDfS4M2mLcu5vwddk53qciGxrqMMjs\nkzsz0Va7W12NS7lzeWaZlAE0gf6t98urOdUJVNeKvBoss4sMP0phqxwf0eWV3ur0\ncjIQB+TpGGikLVdRVuGY/UXHKe9AjoHBva8B3aTpB3lbnbNJBXZbIc1uYq3sa5w7\nXWWUAQKBgH3yW73RRpQNcc9hTUssomUsnQQgHxpfWx5tNxqod36Ytd9EKBh3NqUZ\nvPcH6gdh7mcnNaVNTtQOHLHsbPfBK/pqvb3MAsdlokJcQz8MQJ9SGBBPY6PaGw8z\nq/ambaQykER6dwlXTIlU20uXY0bttOL/iYjKmgo3vA66qfzS6nsg\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjzausf\nsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLzeUPx\ndfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFxjUz9\nl0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJYfKhK\nn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq79Wb\nhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/1happyturtle#main-key"} +{"type":"account","id":"01F8MH0BBE4FHXPH513MBVFHB0","createdAt":"2021-09-08T10:00:53.985634Z","username":"weed_lord420","locked":true,"language":"en","uri":"http://localhost:8080/users/weed_lord420","url":"http://localhost:8080/@weed_lord420","inboxURI":"http://localhost:8080/users/weed_lord420/inbox","outboxURI":"http://localhost:8080/users/weed_lord420/outbox","followingUri":"http://localhost:8080/users/weed_lord420/following","followersUri":"http://localhost:8080/users/weed_lord420/followers","featuredCollectionUri":"http://localhost:8080/users/weed_lord420/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0b\nMIyLRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//P\nceYpo5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4\nus6VxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+\nfNyYVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPc\nqwtx0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQABAoIBAEAA4GHNS4k+Ke4j\nx4J0XkUjV5UbuPY0pSpSDjOJHOJmUfLcg85Ds9mYYO6zxwOaqmrC42ieclI5rh84\nTWQUqX9+VAk1J9UKeE4xZ1SSBtnZ3rK9PjrERZ+dmQ0dATaCuEO5Wwgu7Trk++Bg\nIqy8WNGZL94v9tfwALp1jTXW9AvmQoNdCFBP62vcmYW4YLjnggxLCFTA8YKfdePa\nTuxxY6uLkeBbxzWpbRU2+bmlxd5OnCkiRSMHIX+6JdtCu2JdWpUTCnWrFi2n1TZz\nZQx9z5rvowK1O785jGMFum5vBWpjIU8sJcXmPjGMU25zzmrhzfmkJsTXER3CXoUo\nSqSPqgECgYEA78OR7bY5KKQQ7Lyz6dru4Fct5P/OXTQoOg5aS7TKb95LVWj+TANn\n5djwIbLmAUV30z0Id9VgiZOL0Hny8+3VV9eU088Z408pAy5WQrL3dB8tZLUJSq5c\n5k6X15/VjWOOZKppDxShzoV3mcohrnwVwkv4fhPFQQOJJBYz6xurWs0CgYEA3MDE\nsDMd9ahzO0dl62ynojkkA8ZTcn2UdyvLpGj9UxT5j9vWF3CfqitXgcpNiVSIbxqQ\nbo/pBch7c/2Xakv5zkdcrJj5/6gyr+m1/tK2o7+CjDaSE4SYwufXx+qkl03Zpyzt\nKdOi7Hz/b2tdjump7ECEDE45mG2ea8oSnPgXl0cCgYBkGGFzu/9g2B24t47ksmHH\nhp3CXIjqoDurARLxSCi7SzJoFc0ULtfRPSAC8YzUOwwrQ++lF4+V3+MexcqHy2Kl\nqXqYcn18SC/3BAE/Fzf3Yoyw3mNiqihefbEmc7PTsxxfKkVx5ksmzNGBgsFM9sCe\nvNigyeAvpCo8xogmPwbqgQKBgE34mIBTzcUzFmBdu5YH7r3RyPK8XkUWLhZZlbgg\njTmHMw6o61mkIgENBf+F4RUckoQLsfAbTIcKZPB3JcAZzcYaVpVwAv1V/3E671lu\nO6xivE2iCL50GzDcis7GBhSbHsF5kNsxMV6uV9qW5ZjQ13/m2b0u9BDuxwHzgdeH\nmW2JAoGAIUOYniuEwdygxWVnYatpr3NPjT3BOKoV5i9zkeJRu1hFpwQM6vQ4Ds5p\nGC5vbMKAv9Cwuw62e2HvqTun3+U2Y5Uived3XCpgM/50BFrFHCfuqXEnu1bEzk5z\n9mIhp8uXPxzC5N7tRQfb3/eU1IUcb6T6ksbr2P81z0j03J55erg=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0bMIyL\nRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//PceYp\no5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4us6V\nxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+fNyY\nVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPcqwtx\n0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/weed_lord420#main-key"} +{"type":"account","id":"01F8MH17FWEB39HZJ76B6VXSKF","createdAt":"2021-09-05T10:00:53.985641Z","username":"admin","locked":true,"language":"en","uri":"http://localhost:8080/users/admin","url":"http://localhost:8080/@admin","inboxURI":"http://localhost:8080/users/admin/inbox","outboxURI":"http://localhost:8080/users/admin/outbox","followingUri":"http://localhost:8080/users/admin/following","followersUri":"http://localhost:8080/users/admin/followers","featuredCollectionUri":"http://localhost:8080/users/admin/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVq\nhujDhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLR\nBI97qD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wg\nfvtEjEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G\n8kQJDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/Bk\nRhhGp2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQABAoIBAGK0aIADOU4ffJDe\n7sveiih5Fc1PATwx/QIR2QkWM1SREdx6LYclcX44V8xDanAbE44p1SkHY/CsEtYy\nXnyoXnn2FwFDQrdveY7+I6PApOPLAcKWkyLltC+hbVdj92/6YGNrm7EA/a77wruH\nmwjiivLnTG2CLecNiXSl33DA9YU4Yz+2Tza3IpTdjt8c/dz/BKKaxaWV+i9ew5VR\nioo5v51B+J8PrneCM/p8LGiLV148Njr0JqV6eFy1JuzItYMYdc3Fp+YnMzsuMZEA\n1akMcoln/ucVJyOFnCn6jx47nIoPZLl1KxX3aRDRfvrejm6W4yAkkTmR5voSRqax\njPL3rI0CgYEA9Acu4TO8xJ3uGaUad0N9JTYQVSmtAaE/g+df9LGMSzoj8X95S4xE\nQsGPqNGDm2VWADJjK4P05twZ+LfsfSKQ86wbp4/gbgnXpqB1P5Lty/B7KxiTnNwt\nwb1WGWTCukxfUSL3PRyf8uylkrg72RxKiBx4zKO3WVSLWOZWrFtn0qMCgYEA0H2p\nJs9Nv20ADOOX5tQ7+ruS6/B/Fhyj5fhflSYCAtOW7aME7+zQKJyqSQZ4b2Aub3Tp\nGIaUbRIGzjHyuTultFFWvjU3H5aI/0g1G9WKaBhNkyTIYVmMKtYyhXNvouWing8x\noraWx8TTBP8Cdnnk+QgdR2fpug8cghKupp5wvO8CgYA1JFtRL7MsHjh73TimQExA\njkWARlMmx7bNQtXis8eZmk+5h8kiaqly4DQoz3eZn7fa0x5Fm7b5j3UYdPVLSvvG\nFPTwyKRXUk1kPA1MivK+NuCbwf5jao+MYW8emJLPf1JCmRq+dD1g6aglC3n9Dewt\nOAYWipCjI4Y1FfRKFJ3HgQKBgEAb47+DTyzln3ZXJYZdDHR06SCTuwBZnixAy2NZ\nZJTp6yb3UbVU5E0Yn2QFEVNuB9lN4b8g4tMHEACnazN6G+HugPXL9z9HUqjs0yfT\n6dNIZdIxJUyJ9IfXhYFzlYhJhE+F7IVUD9kttJV8tI0pvja1QAuM8Fm9+84jYIDr\nh08RAoGAMYbjKHbtejcHBwt1kIcSss0cDmlZbBleJo8tdmdg4ndf5GE9N4/EL7tq\nm2zYSfr7OVdnOwRhoO+xF/6d1L7+TR1wz+k2fuMsI71aM5Ocp1nYTutjIkBTcldZ\nZzvjOgZWng5icuRLQQiDSKG5uqazqL/xGXkijb4kp4WW6myWY3c=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVqhujD\nhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLRBI97\nqD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wgfvtE\njEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G8kQJ\nDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/BkRhhG\np2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/admin#main-key"} +{"type":"account","id":"01F8MH1H7YV1Z7D2C8K2730QBF","createdAt":"2021-09-06T10:00:53.985643Z","username":"the_mighty_zork","locked":true,"language":"en","uri":"http://localhost:8080/users/the_mighty_zork","url":"http://localhost:8080/@the_mighty_zork","inboxURI":"http://localhost:8080/users/the_mighty_zork/inbox","outboxURI":"http://localhost:8080/users/the_mighty_zork/outbox","followingUri":"http://localhost:8080/users/the_mighty_zork/following","followersUri":"http://localhost:8080/users/the_mighty_zork/followers","featuredCollectionUri":"http://localhost:8080/users/the_mighty_zork/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss\n5mEA/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvC\nC9zt/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZ\nFHptEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1\ntMhsUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlq\nefr58l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQABAoIBAFa+UypbFG1cW2Tr\nNBxPm7ngOEtXl8MicV4dIVKh0TwOo13ZxtNFBbOj7jALmPn/9HrtmbkABPQHDL1U\n/nt9aNSAeTjpwH3RaD5vFX3n0g8n2zJBOZLxxzAjNi4RBLYj5uP1AiKkdvRlsJza\nuSFDkty2zMBqN9mLPHE+RePj5Qa6tjYfIQqQzu/+YnYMlXHoC2yHNKsvz6S5FhVj\nv5zATv2JlJQH3RSmhuPOah73iQnKCLzYYEAHleawKrCg/rZ3ht37Guvabeq7MqQN\nvi9pJdAA+RMxPsboHajskePjOTYJgKQSxEAMRTMfBR40aZxklxQL0EoBd1Y3CHXh\nfMg0xWECgYEA0ORrpJ1A2WNQwKcDDeBBsaJqWF4EraoFzYrugKZrAYEeVyuGD0zq\nARUaWkZTZ1f6wQ10i1WxAuKlBEds7QsLdZzLsA4um4JlBroCZiYfPnmTtb8op1LY\nFqeYTByvAmnfWWTuOI67GX9ruLg8tEGuz38kuQVSxYs51its3tScNPUCgYEAyRst\nwRbqpOqnwoRoS6pxv0Vpc3nUcfaVYwsg/qobJkiwAdlUYeE7alvEY926VW4cvU/X\nhy3L1punAqnyLI7uuqCefXEbNxO0Cebyy4Kv2Ye1uzl0OHsJczSNdfpNqfAIKwtN\nHLCYDGCsluQhz+I/5Pd0dT+JDPPW9hKS2HG7o+kCgYBqugn1VRLo/sEnbS02TbnC\n1ESZWY/yWsgUOEObH2vUnO+vgeFAt/9nBi0sqnm6d0z6jbFZ7zI9UycUhJm2ksoM\nEUxQay6M7ZZIVYkcP6X++YbqePyAYOdey8oYOR+BkC45MkQ0SVh2so+LFTaOsnBq\nO3+7uGiN3ZBzSESbpO0acQKBgQCONrsXZeZO82XpB4tdns3LbgGRWKEkajTgEnml\nvZNvck2NMSwb/5PttbFe0ei4CyMluPV4MamJPQ9Qse+BFR67OWR63uZY/4T8z6X4\nxpUmZnLcUFfgrRlUr+AtgvEy8HxGPDquxC7x6deC6RcEFEIM3/UqCOEZGMJ1x1Ky\n31LLKQKBgGCKwVgQ8+4JyHZFkie3YdHhxJDokgY+Opb0HNnoBY/lZ54UMCCJQPS2\n0XPSu651j/3adr3RQneU04gF6U2/D5JzFEV0kUsqZ4Zy2EEU0LU4ibus0gyomSpK\niWhU4QrC/M4ELxYZinlNu3ThPWNQ/PMNteVWfdgOcV7uUWl0ViFp\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss5mEA\n/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvCC9zt\n/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZFHpt\nEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1tMhs\nUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlqefr5\n8l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/the_mighty_zork#main-key"} +{"type":"block","id":"01FEXXET6XXMF7G2V3ASZP3YQW","createdAt":"2021-09-08T09:00:53.965362Z","uri":"http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW","accountId":"01F8MH5NBDF2MV7CTC4Q5128HF","targetAccountId":"01F8MH5ZK5VRH73AKHQM6Y9VNX"} +{"type":"account","id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","createdAt":"2021-08-31T12:00:53.985646Z","username":"foss_satan","domain":"fossbros-anonymous.io","locked":true,"language":"en","uri":"http://fossbros-anonymous.io/users/foss_satan","url":"http://fossbros-anonymous.io/@foss_satan","inboxURI":"http://fossbros-anonymous.io/users/foss_satan/inbox","outboxURI":"http://fossbros-anonymous.io/users/foss_satan/outbox","followingUri":"http://fossbros-anonymous.io/users/foss_satan/following","followersUri":"http://fossbros-anonymous.io/users/foss_satan/followers","featuredCollectionUri":"http://fossbros-anonymous.io/users/foss_satan/collections/featured","actorType":"Person","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEA2OyVgkaIL9VohXKYTh319j4OouHRX/8QC7piXj71k7q5RDzEyvis\nVZBc5/C1/crCpxt895i0Ai2CiXQx+dISV7s/JBhAGl8s7TQ8jLlMuptrI0+sdkBC\nlu8pU0qQmoeXVnlquOzNmqGufUxIDtLXLZDN17qf/7vWA23q4d0tG5KQhGGGKiVM\n61Ufvr9MmgPBSpyUvYMAulFlz1264L49aGWeVgOz3qUQzqtxjrP0kaIbeyt56miP\nKr5AqkRgSsXci+FAo6suxR5gzo9NgleNkbZWF9MQyKlawukPwZUDSh396vtNQMee\n/4mto7mAXw8iio0IacrYO3F7iyewXnmI/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://fossbros-anonymous.io/users/foss_satan/main-key"} +{"type":"follow","id":"01F8PYDCE8XE23GRE5DPZJDZDP","createdAt":"2021-09-08T09:00:54.749465Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PYDCE8XE23GRE5DPZJDZDP","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH5NBDF2MV7CTC4Q5128HF"} +{"type":"follow","id":"01F8PY8RHWRQZV038T4E8T9YK8","createdAt":"2021-09-06T12:00:54.749459Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PY8RHWRQZV038T4E8T9YK8","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH17FWEB39HZJ76B6VXSKF"} +{"type":"domainBlock","id":"01FF22EQM7X8E3RX1XGPN7S87D","createdAt":"2021-09-08T10:00:53.968971Z","domain":"replyguys.com","createdByAccountID":"01F8MH17FWEB39HZJ76B6VXSKF","privateComment":"i blocked this domain because they keep replying with pushy + unwarranted linux advice","publicComment":"reply-guying to tech posts","obfuscate":false} +{"type":"user","id":"01F8MGYG9E893WRHW0TAEXR8GJ","createdAt":"2021-09-08T10:00:53.97247Z","accountID":"01F8MH0BBE4FHXPH513MBVFHB0","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","locale":"en","lastEmailedAt":"0001-01-01T00:00:00Z","confirmationToken":"a5a280bd-34be-44a3-8330-a57eaf61b8dd","confirmationTokenSentAt":"2021-09-08T10:00:53.972472Z","unconfirmedEmail":"weed_lord420@example.org","moderator":false,"admin":false,"disabled":false,"approved":false} +{"type":"user","id":"01F8MGWYWKVKS3VS8DV1AMYPGE","createdAt":"2021-09-05T10:00:53.972475Z","email":"admin@example.org","accountID":"01F8MH17FWEB39HZJ76B6VXSKF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:50:53.972477Z","lastSignInAt":"2021-09-08T08:00:53.972477Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:30:53.972478Z","confirmedAt":"2021-09-05T10:00:53.972478Z","moderator":true,"admin":true,"disabled":false,"approved":true} +{"type":"user","id":"01F8MGVGPHQ2D3P3X0454H54Z5","createdAt":"2021-09-06T22:00:53.97248Z","email":"zork@example.org","accountID":"01F8MH1H7YV1Z7D2C8K2730QBF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972481Z","lastSignInAt":"2021-09-08T08:00:53.972481Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972482Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972483Z","confirmedAt":"2021-09-07T00:00:53.972482Z","moderator":false,"admin":false,"disabled":false,"approved":true} +{"type":"user","id":"01F8MH1VYJAE00TVVGMM5JNJ8X","createdAt":"2021-09-06T22:00:53.972485Z","email":"tortle.dude@example.org","accountID":"01F8MH5NBDF2MV7CTC4Q5128HF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972485Z","lastSignInAt":"2021-09-08T08:00:53.972486Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972487Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972487Z","confirmedAt":"2021-09-07T00:00:53.972487Z","moderator":false,"admin":false,"disabled":false,"approved":true} +{"type":"instance","id":"01BZDDRPAB8J645ABY31HHF68Y","createdAt":"2021-09-08T10:00:54.763912Z","domain":"localhost:8080","title":"localhost:8080","uri":"http://localhost:8080","reputation":0} +``` + +For information on how to use the commands to import/export, see [here](cli.md#gotosocial-admin-export). + +Advantages: + +* Database agnostic: exported data is in a somewhat generic format, and the `import` command can be used to insert this data into either a Postgres or an SQLite database. +* Lightweight: only what is needed is preserved, so backup files can be quite small (even small enough to send in an email). Backup/import commands just take a few seconds to run. +* Easily readable format: the output is just JSON. + +Disadvantages: + +* Loss of statuses/media/etc: don't do a backup/restore this way unless you're willing to drop stuff. +* You need to use the GtS CLI tool to insert data back into a database, unless you write custom tooling for it. diff --git a/docs/admin/cli.md b/docs/admin/cli.md index 8072a5d7a..e15daf084 100644 --- a/docs/admin/cli.md +++ b/docs/admin/cli.md @@ -213,3 +213,78 @@ Example: ```bash gotosocial admin account password --username some_username --pasword some_really_good_password ``` + +### gotosocial admin export + +This command can be used to export data from your GoToSocial instance into a file, for backup/storage. + +The file format will be a series of newline-separated JSON objects. + +`gotosocial admin export --help`: + +```text +NAME: + gotosocial admin export - export data from the database to file at the given path + +USAGE: + gotosocial admin export [command options] [arguments...] + +OPTIONS: + --path value the path of the file to import from/export to + --help, -h show help (default: false) +``` + +Example: + +```bash +gotosocial admin export --path ./example.json +``` + +`example.json`: + +```json +{"type":"account","id":"01F8MH5NBDF2MV7CTC4Q5128HF","createdAt":"2021-08-31T12:00:53.985645Z","username":"1happyturtle","locked":true,"language":"en","uri":"http://localhost:8080/users/1happyturtle","url":"http://localhost:8080/@1happyturtle","inboxURI":"http://localhost:8080/users/1happyturtle/inbox","outboxURI":"http://localhost:8080/users/1happyturtle/outbox","followingUri":"http://localhost:8080/users/1happyturtle/following","followersUri":"http://localhost:8080/users/1happyturtle/followers","featuredCollectionUri":"http://localhost:8080/users/1happyturtle/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjz\nausfsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLz\neUPxdfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFx\njUz9l0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJY\nfKhKn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq\n79WbhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQABAoIBAGF+MxHjD15VV2NY\nKKb1GjMx98i1Xx6TijgoA+zmfha4LGu35e79Lql+0LXFp0zEpa6lAQsMQQhgd0OD\nmKKmSk+pxAvskJ4FxrhIf/yBFA4RMrj5OCaAOocRtdsOJ8n5UtFBrNAF0tzMY9q/\nkgzoq97aVF1mV9iFxaeBx6zT8ozSdqBq1PK/3w1dVg89S5tfKYc7Q0lQ00SfsTnd\niTDClKyqurebo9Pt6M7gXavgg3tvBlmwwr6XHs34Leng3oiN9mW8DVzaBMPzn+rE\nxF2eqs3v9vVpj8es88OwCh5P+ff8vJYvhu7Fcr/bJ8BItBQwfb8QBDATg/MXU2BI\n2ssW6AECgYEA4wmIyYGeu9+hzDa/J3Vh8GnlVNUCohHcChQdOsWsFXUgpVlUIHrX\neKHn42vD4Rzy52/YzJts4NkZTM9sL+kEXIEcpMG/S9xIIud7U0m/hMSAlmnJK/9j\niEXws3o4jo0E77jnRcBdIjpG4K5Eekm0DSR3SFhtZfEdN2DWPvu7K98CgYEA5tER\n/qJwFMc51AobMU87ZjXON7hI2U1WY/pVF62jSl0IcSsnj2riEKWLrs+GRG+HUg+U\naFSqAHcxaVHA0h0AYR8RopAhDdVKh0kvB8biLo+IEzNjPv2vyn0yRN5YSfXdGzyJ\nUjVU6kWdQOwmzy86nHgFaqEx7eofHIaGZzJK/AECgYEAu2VNQHX63TuzQuoVUa5z\nzoq5vhGsALYZF0CO98ndRkDNV22qIL0ESQ/qZS64GYFZhWouWoQXlGfdmCbFN65v\n6SKwz9UT3rvN1vGWO6Ltr9q6AG0EnYpJT1vbV2kUcaU4Y94NFue2d9/+TMnKv91B\n/m8Q/efvNGuWH/WQIaCKV6UCgYBz89WhYMMDfS4M2mLcu5vwddk53qciGxrqMMjs\nkzsz0Va7W12NS7lzeWaZlAE0gf6t98urOdUJVNeKvBoss4sMP0phqxwf0eWV3ur0\ncjIQB+TpGGikLVdRVuGY/UXHKe9AjoHBva8B3aTpB3lbnbNJBXZbIc1uYq3sa5w7\nXWWUAQKBgH3yW73RRpQNcc9hTUssomUsnQQgHxpfWx5tNxqod36Ytd9EKBh3NqUZ\nvPcH6gdh7mcnNaVNTtQOHLHsbPfBK/pqvb3MAsdlokJcQz8MQJ9SGBBPY6PaGw8z\nq/ambaQykER6dwlXTIlU20uXY0bttOL/iYjKmgo3vA66qfzS6nsg\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjzausf\nsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLzeUPx\ndfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFxjUz9\nl0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJYfKhK\nn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq79Wb\nhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/1happyturtle#main-key"} +{"type":"account","id":"01F8MH0BBE4FHXPH513MBVFHB0","createdAt":"2021-09-08T10:00:53.985634Z","username":"weed_lord420","locked":true,"language":"en","uri":"http://localhost:8080/users/weed_lord420","url":"http://localhost:8080/@weed_lord420","inboxURI":"http://localhost:8080/users/weed_lord420/inbox","outboxURI":"http://localhost:8080/users/weed_lord420/outbox","followingUri":"http://localhost:8080/users/weed_lord420/following","followersUri":"http://localhost:8080/users/weed_lord420/followers","featuredCollectionUri":"http://localhost:8080/users/weed_lord420/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0b\nMIyLRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//P\nceYpo5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4\nus6VxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+\nfNyYVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPc\nqwtx0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQABAoIBAEAA4GHNS4k+Ke4j\nx4J0XkUjV5UbuPY0pSpSDjOJHOJmUfLcg85Ds9mYYO6zxwOaqmrC42ieclI5rh84\nTWQUqX9+VAk1J9UKeE4xZ1SSBtnZ3rK9PjrERZ+dmQ0dATaCuEO5Wwgu7Trk++Bg\nIqy8WNGZL94v9tfwALp1jTXW9AvmQoNdCFBP62vcmYW4YLjnggxLCFTA8YKfdePa\nTuxxY6uLkeBbxzWpbRU2+bmlxd5OnCkiRSMHIX+6JdtCu2JdWpUTCnWrFi2n1TZz\nZQx9z5rvowK1O785jGMFum5vBWpjIU8sJcXmPjGMU25zzmrhzfmkJsTXER3CXoUo\nSqSPqgECgYEA78OR7bY5KKQQ7Lyz6dru4Fct5P/OXTQoOg5aS7TKb95LVWj+TANn\n5djwIbLmAUV30z0Id9VgiZOL0Hny8+3VV9eU088Z408pAy5WQrL3dB8tZLUJSq5c\n5k6X15/VjWOOZKppDxShzoV3mcohrnwVwkv4fhPFQQOJJBYz6xurWs0CgYEA3MDE\nsDMd9ahzO0dl62ynojkkA8ZTcn2UdyvLpGj9UxT5j9vWF3CfqitXgcpNiVSIbxqQ\nbo/pBch7c/2Xakv5zkdcrJj5/6gyr+m1/tK2o7+CjDaSE4SYwufXx+qkl03Zpyzt\nKdOi7Hz/b2tdjump7ECEDE45mG2ea8oSnPgXl0cCgYBkGGFzu/9g2B24t47ksmHH\nhp3CXIjqoDurARLxSCi7SzJoFc0ULtfRPSAC8YzUOwwrQ++lF4+V3+MexcqHy2Kl\nqXqYcn18SC/3BAE/Fzf3Yoyw3mNiqihefbEmc7PTsxxfKkVx5ksmzNGBgsFM9sCe\nvNigyeAvpCo8xogmPwbqgQKBgE34mIBTzcUzFmBdu5YH7r3RyPK8XkUWLhZZlbgg\njTmHMw6o61mkIgENBf+F4RUckoQLsfAbTIcKZPB3JcAZzcYaVpVwAv1V/3E671lu\nO6xivE2iCL50GzDcis7GBhSbHsF5kNsxMV6uV9qW5ZjQ13/m2b0u9BDuxwHzgdeH\nmW2JAoGAIUOYniuEwdygxWVnYatpr3NPjT3BOKoV5i9zkeJRu1hFpwQM6vQ4Ds5p\nGC5vbMKAv9Cwuw62e2HvqTun3+U2Y5Uived3XCpgM/50BFrFHCfuqXEnu1bEzk5z\n9mIhp8uXPxzC5N7tRQfb3/eU1IUcb6T6ksbr2P81z0j03J55erg=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0bMIyL\nRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//PceYp\no5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4us6V\nxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+fNyY\nVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPcqwtx\n0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/weed_lord420#main-key"} +{"type":"account","id":"01F8MH17FWEB39HZJ76B6VXSKF","createdAt":"2021-09-05T10:00:53.985641Z","username":"admin","locked":true,"language":"en","uri":"http://localhost:8080/users/admin","url":"http://localhost:8080/@admin","inboxURI":"http://localhost:8080/users/admin/inbox","outboxURI":"http://localhost:8080/users/admin/outbox","followingUri":"http://localhost:8080/users/admin/following","followersUri":"http://localhost:8080/users/admin/followers","featuredCollectionUri":"http://localhost:8080/users/admin/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVq\nhujDhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLR\nBI97qD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wg\nfvtEjEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G\n8kQJDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/Bk\nRhhGp2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQABAoIBAGK0aIADOU4ffJDe\n7sveiih5Fc1PATwx/QIR2QkWM1SREdx6LYclcX44V8xDanAbE44p1SkHY/CsEtYy\nXnyoXnn2FwFDQrdveY7+I6PApOPLAcKWkyLltC+hbVdj92/6YGNrm7EA/a77wruH\nmwjiivLnTG2CLecNiXSl33DA9YU4Yz+2Tza3IpTdjt8c/dz/BKKaxaWV+i9ew5VR\nioo5v51B+J8PrneCM/p8LGiLV148Njr0JqV6eFy1JuzItYMYdc3Fp+YnMzsuMZEA\n1akMcoln/ucVJyOFnCn6jx47nIoPZLl1KxX3aRDRfvrejm6W4yAkkTmR5voSRqax\njPL3rI0CgYEA9Acu4TO8xJ3uGaUad0N9JTYQVSmtAaE/g+df9LGMSzoj8X95S4xE\nQsGPqNGDm2VWADJjK4P05twZ+LfsfSKQ86wbp4/gbgnXpqB1P5Lty/B7KxiTnNwt\nwb1WGWTCukxfUSL3PRyf8uylkrg72RxKiBx4zKO3WVSLWOZWrFtn0qMCgYEA0H2p\nJs9Nv20ADOOX5tQ7+ruS6/B/Fhyj5fhflSYCAtOW7aME7+zQKJyqSQZ4b2Aub3Tp\nGIaUbRIGzjHyuTultFFWvjU3H5aI/0g1G9WKaBhNkyTIYVmMKtYyhXNvouWing8x\noraWx8TTBP8Cdnnk+QgdR2fpug8cghKupp5wvO8CgYA1JFtRL7MsHjh73TimQExA\njkWARlMmx7bNQtXis8eZmk+5h8kiaqly4DQoz3eZn7fa0x5Fm7b5j3UYdPVLSvvG\nFPTwyKRXUk1kPA1MivK+NuCbwf5jao+MYW8emJLPf1JCmRq+dD1g6aglC3n9Dewt\nOAYWipCjI4Y1FfRKFJ3HgQKBgEAb47+DTyzln3ZXJYZdDHR06SCTuwBZnixAy2NZ\nZJTp6yb3UbVU5E0Yn2QFEVNuB9lN4b8g4tMHEACnazN6G+HugPXL9z9HUqjs0yfT\n6dNIZdIxJUyJ9IfXhYFzlYhJhE+F7IVUD9kttJV8tI0pvja1QAuM8Fm9+84jYIDr\nh08RAoGAMYbjKHbtejcHBwt1kIcSss0cDmlZbBleJo8tdmdg4ndf5GE9N4/EL7tq\nm2zYSfr7OVdnOwRhoO+xF/6d1L7+TR1wz+k2fuMsI71aM5Ocp1nYTutjIkBTcldZ\nZzvjOgZWng5icuRLQQiDSKG5uqazqL/xGXkijb4kp4WW6myWY3c=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVqhujD\nhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLRBI97\nqD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wgfvtE\njEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G8kQJ\nDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/BkRhhG\np2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/admin#main-key"} +{"type":"account","id":"01F8MH1H7YV1Z7D2C8K2730QBF","createdAt":"2021-09-06T10:00:53.985643Z","username":"the_mighty_zork","locked":true,"language":"en","uri":"http://localhost:8080/users/the_mighty_zork","url":"http://localhost:8080/@the_mighty_zork","inboxURI":"http://localhost:8080/users/the_mighty_zork/inbox","outboxURI":"http://localhost:8080/users/the_mighty_zork/outbox","followingUri":"http://localhost:8080/users/the_mighty_zork/following","followersUri":"http://localhost:8080/users/the_mighty_zork/followers","featuredCollectionUri":"http://localhost:8080/users/the_mighty_zork/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss\n5mEA/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvC\nC9zt/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZ\nFHptEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1\ntMhsUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlq\nefr58l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQABAoIBAFa+UypbFG1cW2Tr\nNBxPm7ngOEtXl8MicV4dIVKh0TwOo13ZxtNFBbOj7jALmPn/9HrtmbkABPQHDL1U\n/nt9aNSAeTjpwH3RaD5vFX3n0g8n2zJBOZLxxzAjNi4RBLYj5uP1AiKkdvRlsJza\nuSFDkty2zMBqN9mLPHE+RePj5Qa6tjYfIQqQzu/+YnYMlXHoC2yHNKsvz6S5FhVj\nv5zATv2JlJQH3RSmhuPOah73iQnKCLzYYEAHleawKrCg/rZ3ht37Guvabeq7MqQN\nvi9pJdAA+RMxPsboHajskePjOTYJgKQSxEAMRTMfBR40aZxklxQL0EoBd1Y3CHXh\nfMg0xWECgYEA0ORrpJ1A2WNQwKcDDeBBsaJqWF4EraoFzYrugKZrAYEeVyuGD0zq\nARUaWkZTZ1f6wQ10i1WxAuKlBEds7QsLdZzLsA4um4JlBroCZiYfPnmTtb8op1LY\nFqeYTByvAmnfWWTuOI67GX9ruLg8tEGuz38kuQVSxYs51its3tScNPUCgYEAyRst\nwRbqpOqnwoRoS6pxv0Vpc3nUcfaVYwsg/qobJkiwAdlUYeE7alvEY926VW4cvU/X\nhy3L1punAqnyLI7uuqCefXEbNxO0Cebyy4Kv2Ye1uzl0OHsJczSNdfpNqfAIKwtN\nHLCYDGCsluQhz+I/5Pd0dT+JDPPW9hKS2HG7o+kCgYBqugn1VRLo/sEnbS02TbnC\n1ESZWY/yWsgUOEObH2vUnO+vgeFAt/9nBi0sqnm6d0z6jbFZ7zI9UycUhJm2ksoM\nEUxQay6M7ZZIVYkcP6X++YbqePyAYOdey8oYOR+BkC45MkQ0SVh2so+LFTaOsnBq\nO3+7uGiN3ZBzSESbpO0acQKBgQCONrsXZeZO82XpB4tdns3LbgGRWKEkajTgEnml\nvZNvck2NMSwb/5PttbFe0ei4CyMluPV4MamJPQ9Qse+BFR67OWR63uZY/4T8z6X4\nxpUmZnLcUFfgrRlUr+AtgvEy8HxGPDquxC7x6deC6RcEFEIM3/UqCOEZGMJ1x1Ky\n31LLKQKBgGCKwVgQ8+4JyHZFkie3YdHhxJDokgY+Opb0HNnoBY/lZ54UMCCJQPS2\n0XPSu651j/3adr3RQneU04gF6U2/D5JzFEV0kUsqZ4Zy2EEU0LU4ibus0gyomSpK\niWhU4QrC/M4ELxYZinlNu3ThPWNQ/PMNteVWfdgOcV7uUWl0ViFp\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss5mEA\n/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvCC9zt\n/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZFHpt\nEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1tMhs\nUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlqefr5\n8l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/the_mighty_zork#main-key"} +{"type":"block","id":"01FEXXET6XXMF7G2V3ASZP3YQW","createdAt":"2021-09-08T09:00:53.965362Z","uri":"http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW","accountId":"01F8MH5NBDF2MV7CTC4Q5128HF","targetAccountId":"01F8MH5ZK5VRH73AKHQM6Y9VNX"} +{"type":"account","id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","createdAt":"2021-08-31T12:00:53.985646Z","username":"foss_satan","domain":"fossbros-anonymous.io","locked":true,"language":"en","uri":"http://fossbros-anonymous.io/users/foss_satan","url":"http://fossbros-anonymous.io/@foss_satan","inboxURI":"http://fossbros-anonymous.io/users/foss_satan/inbox","outboxURI":"http://fossbros-anonymous.io/users/foss_satan/outbox","followingUri":"http://fossbros-anonymous.io/users/foss_satan/following","followersUri":"http://fossbros-anonymous.io/users/foss_satan/followers","featuredCollectionUri":"http://fossbros-anonymous.io/users/foss_satan/collections/featured","actorType":"Person","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEA2OyVgkaIL9VohXKYTh319j4OouHRX/8QC7piXj71k7q5RDzEyvis\nVZBc5/C1/crCpxt895i0Ai2CiXQx+dISV7s/JBhAGl8s7TQ8jLlMuptrI0+sdkBC\nlu8pU0qQmoeXVnlquOzNmqGufUxIDtLXLZDN17qf/7vWA23q4d0tG5KQhGGGKiVM\n61Ufvr9MmgPBSpyUvYMAulFlz1264L49aGWeVgOz3qUQzqtxjrP0kaIbeyt56miP\nKr5AqkRgSsXci+FAo6suxR5gzo9NgleNkbZWF9MQyKlawukPwZUDSh396vtNQMee\n/4mto7mAXw8iio0IacrYO3F7iyewXnmI/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://fossbros-anonymous.io/users/foss_satan/main-key"} +{"type":"follow","id":"01F8PYDCE8XE23GRE5DPZJDZDP","createdAt":"2021-09-08T09:00:54.749465Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PYDCE8XE23GRE5DPZJDZDP","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH5NBDF2MV7CTC4Q5128HF"} +{"type":"follow","id":"01F8PY8RHWRQZV038T4E8T9YK8","createdAt":"2021-09-06T12:00:54.749459Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PY8RHWRQZV038T4E8T9YK8","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH17FWEB39HZJ76B6VXSKF"} +{"type":"domainBlock","id":"01FF22EQM7X8E3RX1XGPN7S87D","createdAt":"2021-09-08T10:00:53.968971Z","domain":"replyguys.com","createdByAccountID":"01F8MH17FWEB39HZJ76B6VXSKF","privateComment":"i blocked this domain because they keep replying with pushy + unwarranted linux advice","publicComment":"reply-guying to tech posts","obfuscate":false} +{"type":"user","id":"01F8MGYG9E893WRHW0TAEXR8GJ","createdAt":"2021-09-08T10:00:53.97247Z","accountID":"01F8MH0BBE4FHXPH513MBVFHB0","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","locale":"en","lastEmailedAt":"0001-01-01T00:00:00Z","confirmationToken":"a5a280bd-34be-44a3-8330-a57eaf61b8dd","confirmationTokenSentAt":"2021-09-08T10:00:53.972472Z","unconfirmedEmail":"weed_lord420@example.org","moderator":false,"admin":false,"disabled":false,"approved":false} +{"type":"user","id":"01F8MGWYWKVKS3VS8DV1AMYPGE","createdAt":"2021-09-05T10:00:53.972475Z","email":"admin@example.org","accountID":"01F8MH17FWEB39HZJ76B6VXSKF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:50:53.972477Z","lastSignInAt":"2021-09-08T08:00:53.972477Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:30:53.972478Z","confirmedAt":"2021-09-05T10:00:53.972478Z","moderator":true,"admin":true,"disabled":false,"approved":true} +{"type":"user","id":"01F8MGVGPHQ2D3P3X0454H54Z5","createdAt":"2021-09-06T22:00:53.97248Z","email":"zork@example.org","accountID":"01F8MH1H7YV1Z7D2C8K2730QBF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972481Z","lastSignInAt":"2021-09-08T08:00:53.972481Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972482Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972483Z","confirmedAt":"2021-09-07T00:00:53.972482Z","moderator":false,"admin":false,"disabled":false,"approved":true} +{"type":"user","id":"01F8MH1VYJAE00TVVGMM5JNJ8X","createdAt":"2021-09-06T22:00:53.972485Z","email":"tortle.dude@example.org","accountID":"01F8MH5NBDF2MV7CTC4Q5128HF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972485Z","lastSignInAt":"2021-09-08T08:00:53.972486Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972487Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972487Z","confirmedAt":"2021-09-07T00:00:53.972487Z","moderator":false,"admin":false,"disabled":false,"approved":true} +{"type":"instance","id":"01BZDDRPAB8J645ABY31HHF68Y","createdAt":"2021-09-08T10:00:54.763912Z","domain":"localhost:8080","title":"localhost:8080","uri":"http://localhost:8080","reputation":0} +``` + +### gotosocial admin import + +This command can be used to import data from a file into your GoToSocial database. + +If GoToSocial tables don't yet exist in the database, they will be created. + +If any conflicts occur while importing (an already exists while attempting to import a specific account, for example), then the process will be aborted. + +The file format should be a series of newline-separated JSON objects (see above). + +`gotosocial admin import --help`: + +```text +NAME: + gotosocial admin import - import data from a file into the database + +USAGE: + gotosocial admin import [command options] [arguments...] + +OPTIONS: + --path value the path of the file to import from/export to + --help, -h show help (default: false) +``` + +Example: + +```bash +gotosocial admin import --path ./example.json +``` diff --git a/go.mod b/go.mod index e32767950..2e5e4f984 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/microcosm-cc/bluemonday v1.0.15 + github.com/mitchellh/mapstructure v1.4.1 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 diff --git a/go.sum b/go.sum index 9ce7e6175..4720bd161 100644 --- a/go.sum +++ b/go.sum @@ -346,6 +346,8 @@ github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A github.com/memcachier/mc v2.0.1+incompatible/go.mod h1:7bkvFE61leUBvXz+yxsOnGBQSZpBSPIMUQSmmSHvuXc= github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/internal/api/s2s/user/repliesget_test.go b/internal/api/s2s/user/repliesget_test.go index 75edbc882..a785b2cff 100644 --- a/internal/api/s2s/user/repliesget_test.go +++ b/internal/api/s2s/user/repliesget_test.go @@ -157,7 +157,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() { b, err := ioutil.ReadAll(result.Body) assert.NoError(suite.T(), err) - assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) // should be a Collection m := make(map[string]interface{}) @@ -188,7 +188,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() { // setup request recorder := httptest.NewRecorder() ctx, _ := gin.CreateTestContext(recorder) - ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5", nil) // the endpoint we're hitting + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0", nil) // the endpoint we're hitting ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) ctx.Request.Header.Set("Date", signedRequest.DateHeader) @@ -220,7 +220,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() { assert.NoError(suite.T(), err) fmt.Println(string(b)) - assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) // should be a Collection m := make(map[string]interface{}) diff --git a/internal/cache/status.go b/internal/cache/status.go index 028abc8f7..f6fe45d99 100644 --- a/internal/cache/status.go +++ b/internal/cache/status.go @@ -144,7 +144,10 @@ func copyStatus(status *gtsmodel.Status) *gtsmodel.Status { Sensitive: status.Sensitive, Language: status.Language, CreatedWithApplicationID: status.CreatedWithApplicationID, - VisibilityAdvanced: status.VisibilityAdvanced, + Federated: status.Federated, + Boostable: status.Boostable, + Replyable: status.Replyable, + Likeable: status.Likeable, ActivityStreamsType: status.ActivityStreamsType, Text: status.Text, Pinned: status.Pinned, diff --git a/internal/cliactions/admin/trans/export.go b/internal/cliactions/admin/trans/export.go new file mode 100644 index 000000000..3d9607ea6 --- /dev/null +++ b/internal/cliactions/admin/trans/export.go @@ -0,0 +1,52 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "context" + "errors" + "fmt" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/cliactions" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db/bundb" + "github.com/superseriousbusiness/gotosocial/internal/trans" +) + +// Export exports info from the database into a file +var Export cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error { + dbConn, err := bundb.NewBunDBService(ctx, c, log) + if err != nil { + return fmt.Errorf("error creating dbservice: %s", err) + } + + exporter := trans.NewExporter(dbConn, log) + + path, ok := c.ExportCLIFlags[config.TransPathFlag] + if !ok { + return errors.New("no path set") + } + + if err := exporter.ExportMinimal(ctx, path); err != nil { + return err + } + + return dbConn.Stop(ctx) +} diff --git a/internal/cliactions/admin/trans/import.go b/internal/cliactions/admin/trans/import.go new file mode 100644 index 000000000..7b137eccc --- /dev/null +++ b/internal/cliactions/admin/trans/import.go @@ -0,0 +1,56 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "context" + "errors" + "fmt" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/cliactions" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db/bundb" + "github.com/superseriousbusiness/gotosocial/internal/trans" +) + +// Import imports info from a file into the database +var Import cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error { + dbConn, err := bundb.NewBunDBService(ctx, c, log) + if err != nil { + return fmt.Errorf("error creating dbservice: %s", err) + } + + importer := trans.NewImporter(dbConn, log) + + path, ok := c.ExportCLIFlags[config.TransPathFlag] + if !ok { + return errors.New("no path set") + } + + if err := dbConn.CreateAllTables(ctx); err != nil { + return err + } + + if err := importer.Import(ctx, path); err != nil { + return err + } + + return dbConn.Stop(ctx) +} diff --git a/internal/cliactions/server/server.go b/internal/cliactions/server/server.go index 0769ade82..3ef714fb0 100644 --- a/internal/cliactions/server/server.go +++ b/internal/cliactions/server/server.go @@ -39,7 +39,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/gotosocial" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/oidc" @@ -51,32 +50,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/web" ) -var models []interface{} = []interface{}{ - >smodel.Account{}, - >smodel.Application{}, - >smodel.Block{}, - >smodel.DomainBlock{}, - >smodel.EmailDomainBlock{}, - >smodel.Follow{}, - >smodel.FollowRequest{}, - >smodel.MediaAttachment{}, - >smodel.Mention{}, - >smodel.Status{}, - >smodel.StatusToEmoji{}, - >smodel.StatusToTag{}, - >smodel.StatusFave{}, - >smodel.StatusBookmark{}, - >smodel.StatusMute{}, - >smodel.Tag{}, - >smodel.User{}, - >smodel.Emoji{}, - >smodel.Instance{}, - >smodel.Notification{}, - >smodel.RouterSession{}, - >smodel.Token{}, - >smodel.Client{}, -} - // Start creates and starts a gotosocial server var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error { dbService, err := bundb.NewBunDBService(ctx, c, log) @@ -84,10 +57,8 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log return fmt.Errorf("error creating dbservice: %s", err) } - for _, m := range models { - if err := dbService.CreateTable(ctx, m); err != nil { - return fmt.Errorf("table creation error: %s", err) - } + if err := dbService.CreateAllTables(ctx); err != nil { + return fmt.Errorf("error creating database tables: %s", err) } if err := dbService.CreateInstanceAccount(ctx); err != nil { diff --git a/internal/config/config.go b/internal/config/config.go index 68e958995..bb789b7d2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -36,6 +36,9 @@ const ( PasswordFlag = "password" PasswordUsage = "the password to set for this account" + + TransPathFlag = "path" + TransPathUsage = "the path of the file to import from/export to" ) // Config pulls together all the configuration needed to run gotosocial @@ -65,6 +68,7 @@ type Config struct { Not parsed from .yaml configuration file. */ AccountCLIFlags map[string]string + ExportCLIFlags map[string]string SoftwareVersion string } @@ -92,6 +96,7 @@ func Empty() *Config { LetsEncryptConfig: &LetsEncryptConfig{}, OIDCConfig: &OIDCConfig{}, AccountCLIFlags: make(map[string]string), + ExportCLIFlags: make(map[string]string), } } @@ -320,6 +325,9 @@ func (c *Config) ParseCLIFlags(f KeyedFlags, version string) error { c.AccountCLIFlags[EmailFlag] = f.String(EmailFlag) c.AccountCLIFlags[PasswordFlag] = f.String(PasswordFlag) + // export CLI flags + c.ExportCLIFlags[TransPathFlag] = f.String(TransPathFlag) + c.SoftwareVersion = version return nil } diff --git a/internal/db/basic.go b/internal/db/basic.go index cf65ddc09..2a1141c8d 100644 --- a/internal/db/basic.go +++ b/internal/db/basic.go @@ -26,6 +26,10 @@ type Basic interface { // For implementations that don't use tables, this can just return nil. CreateTable(ctx context.Context, i interface{}) Error + // CreateAllTables creates *all* tables necessary for the running of GoToSocial. + // Because it uses the 'if not exists' parameter it is safe to run against a GtS that's already been initialized. + CreateAllTables(ctx context.Context) Error + // DropTable drops the table for the given interface. // For implementations that don't use tables, this can just return nil. DropTable(ctx context.Context, i interface{}) Error diff --git a/internal/db/bundb/basic.go b/internal/db/bundb/basic.go index a3a8d0ae9..d4de5bb0b 100644 --- a/internal/db/bundb/basic.go +++ b/internal/db/bundb/basic.go @@ -24,6 +24,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/uptrace/bun" ) @@ -53,17 +54,8 @@ func (b *basicDB) GetWhere(ctx context.Context, where []db.Where, i interface{}) } q := b.conn.NewSelect().Model(i) - for _, w := range where { - if w.Value == nil { - q = q.Where("? IS NULL", bun.Ident(w.Key)) - } else { - if w.CaseInsensitive { - q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value) - } else { - q = q.Where("? = ?", bun.Safe(w.Key), w.Value) - } - } - } + + selectWhere(q, where) err := q.Scan(ctx) return b.conn.ProcessError(err) @@ -97,9 +89,7 @@ func (b *basicDB) DeleteWhere(ctx context.Context, where []db.Where, i interface NewDelete(). Model(i) - for _, w := range where { - q = q.Where("? = ?", bun.Safe(w.Key), w.Value) - } + deleteWhere(q, where) _, err := q.Exec(ctx) return b.conn.ProcessError(err) @@ -128,17 +118,7 @@ func (b *basicDB) UpdateOneByID(ctx context.Context, id string, key string, valu func (b *basicDB) UpdateWhere(ctx context.Context, where []db.Where, key string, value interface{}, i interface{}) db.Error { q := b.conn.NewUpdate().Model(i) - for _, w := range where { - if w.Value == nil { - q = q.Where("? IS NULL", bun.Ident(w.Key)) - } else { - if w.CaseInsensitive { - q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value) - } else { - q = q.Where("? = ?", bun.Safe(w.Key), w.Value) - } - } - } + updateWhere(q, where) q = q.Set("? = ?", bun.Safe(key), value) @@ -151,6 +131,40 @@ func (b *basicDB) CreateTable(ctx context.Context, i interface{}) db.Error { return err } +func (b *basicDB) CreateAllTables(ctx context.Context) db.Error { + models := []interface{}{ + >smodel.Account{}, + >smodel.Application{}, + >smodel.Block{}, + >smodel.DomainBlock{}, + >smodel.EmailDomainBlock{}, + >smodel.Follow{}, + >smodel.FollowRequest{}, + >smodel.MediaAttachment{}, + >smodel.Mention{}, + >smodel.Status{}, + >smodel.StatusToEmoji{}, + >smodel.StatusToTag{}, + >smodel.StatusFave{}, + >smodel.StatusBookmark{}, + >smodel.StatusMute{}, + >smodel.Tag{}, + >smodel.User{}, + >smodel.Emoji{}, + >smodel.Instance{}, + >smodel.Notification{}, + >smodel.RouterSession{}, + >smodel.Token{}, + >smodel.Client{}, + } + for _, i := range models { + if err := b.CreateTable(ctx, i); err != nil { + return err + } + } + return nil +} + func (b *basicDB) DropTable(ctx context.Context, i interface{}) db.Error { _, err := b.conn.NewDropTable().Model(i).IfExists().Exec(ctx) return b.conn.ProcessError(err) diff --git a/internal/db/bundb/basic_test.go b/internal/db/bundb/basic_test.go index d8067fb9d..e5f7e159a 100644 --- a/internal/db/bundb/basic_test.go +++ b/internal/db/bundb/basic_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -42,7 +43,25 @@ func (suite *BasicTestSuite) TestGetAllStatuses() { s := []*gtsmodel.Status{} err := suite.db.GetAll(context.Background(), &s) suite.NoError(err) - suite.Len(s, 12) + suite.Len(s, 13) +} + +func (suite *BasicTestSuite) TestGetAllNotNull() { + where := []db.Where{{ + Key: "domain", + Value: nil, + Not: true, + }} + + a := []*gtsmodel.Account{} + + err := suite.db.GetWhere(context.Background(), where, &a) + suite.NoError(err) + suite.NotEmpty(a) + + for _, acct := range a { + suite.NotEmpty(acct.Domain) + } } func TestBasicTestSuite(t *testing.T) { diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index 9464cfadf..2c26a7df9 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -240,11 +240,11 @@ func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status, } } - // only do one loop if we only want direct children - if onlyDirect { - return + // if we're not only looking for direct children of status, then do the same children-finding + // operation for the found child status too. + if !onlyDirect { + s.statusChildren(ctx, child, foundStatuses, false, minID) } - s.statusChildren(ctx, child, foundStatuses, false, minID) } } diff --git a/internal/db/bundb/status_test.go b/internal/db/bundb/status_test.go index 4b4a5aca4..64079c78f 100644 --- a/internal/db/bundb/status_test.go +++ b/internal/db/bundb/status_test.go @@ -43,10 +43,14 @@ func (suite *StatusTestSuite) TestGetStatusByID() { suite.Nil(status.BoostOfAccount) suite.Nil(status.InReplyTo) suite.Nil(status.InReplyToAccount) + suite.True(status.Federated) + suite.True(status.Boostable) + suite.True(status.Replyable) + suite.True(status.Likeable) } func (suite *StatusTestSuite) TestGetStatusByURI() { - status, err := suite.db.GetStatusByURI(context.Background(), suite.testStatuses["local_account_1_status_1"].URI) + status, err := suite.db.GetStatusByURI(context.Background(), suite.testStatuses["local_account_2_status_3"].URI) if err != nil { suite.FailNow(err.Error()) } @@ -57,6 +61,10 @@ func (suite *StatusTestSuite) TestGetStatusByURI() { suite.Nil(status.BoostOfAccount) suite.Nil(status.InReplyTo) suite.Nil(status.InReplyToAccount) + suite.True(status.Federated) + suite.True(status.Boostable) + suite.False(status.Replyable) + suite.False(status.Likeable) } func (suite *StatusTestSuite) TestGetStatusWithExtras() { @@ -70,6 +78,10 @@ func (suite *StatusTestSuite) TestGetStatusWithExtras() { suite.NotEmpty(status.Tags) suite.NotEmpty(status.Attachments) suite.NotEmpty(status.Emojis) + suite.True(status.Federated) + suite.True(status.Boostable) + suite.True(status.Replyable) + suite.True(status.Likeable) } func (suite *StatusTestSuite) TestGetStatusWithMention() { @@ -83,6 +95,10 @@ func (suite *StatusTestSuite) TestGetStatusWithMention() { suite.NotEmpty(status.MentionIDs) suite.NotEmpty(status.InReplyToID) suite.NotEmpty(status.InReplyToAccountID) + suite.True(status.Federated) + suite.True(status.Boostable) + suite.True(status.Replyable) + suite.True(status.Likeable) } func (suite *StatusTestSuite) TestGetStatusTwice() { @@ -104,6 +120,18 @@ func (suite *StatusTestSuite) TestGetStatusTwice() { suite.Less(duration2, duration1) } +func (suite *StatusTestSuite) TestGetStatusChildren() { + targetStatus := suite.testStatuses["local_account_1_status_1"] + children, err := suite.db.GetStatusChildren(context.Background(), targetStatus, true, "") + suite.NoError(err) + suite.Len(children, 2) + for _, c := range children { + suite.Equal(targetStatus.URI, c.InReplyToURI) + suite.Equal(targetStatus.AccountID, c.InReplyToAccountID) + suite.Equal(targetStatus.ID, c.InReplyToID) + } +} + func TestStatusTestSuite(t *testing.T) { suite.Run(t, new(StatusTestSuite)) } diff --git a/internal/db/bundb/util.go b/internal/db/bundb/util.go index 9e1afb87e..459f65d8c 100644 --- a/internal/db/bundb/util.go +++ b/internal/db/bundb/util.go @@ -19,6 +19,7 @@ package bundb import ( + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/uptrace/bun" ) @@ -35,3 +36,65 @@ func whereEmptyOrNull(column string) func(*bun.SelectQuery) *bun.SelectQuery { WhereOr("? = ''", bun.Ident(column)) } } + +// updateWhere parses []db.Where and adds it to the given update query. +func updateWhere(q *bun.UpdateQuery, where []db.Where) { + for _, w := range where { + query, args := parseWhere(w) + q = q.Where(query, args...) + } +} + +// selectWhere parses []db.Where and adds it to the given select query. +func selectWhere(q *bun.SelectQuery, where []db.Where) { + for _, w := range where { + query, args := parseWhere(w) + q = q.Where(query, args...) + } +} + +// deleteWhere parses []db.Where and adds it to the given where query. +func deleteWhere(q *bun.DeleteQuery, where []db.Where) { + for _, w := range where { + query, args := parseWhere(w) + q = q.Where(query, args...) + } +} + +// parseWhere looks through the options on a single db.Where entry, and +// returns the appropriate query string and arguments. +func parseWhere(w db.Where) (query string, args []interface{}) { + if w.Not { + if w.Value == nil { + query = "? IS NOT NULL" + args = []interface{}{bun.Ident(w.Key)} + return + } + + if w.CaseInsensitive { + query = "LOWER(?) != LOWER(?)" + args = []interface{}{bun.Safe(w.Key), w.Value} + return + } + + query = "? != ?" + args = []interface{}{bun.Safe(w.Key), w.Value} + return + } + + if w.Value == nil { + query = "? IS NULL" + args = []interface{}{bun.Ident(w.Key)} + return + } + + if w.CaseInsensitive { + query = "LOWER(?) = LOWER(?)" + args = []interface{}{bun.Safe(w.Key), w.Value} + return + } + + query = "? = ?" + args = []interface{}{bun.Safe(w.Key), w.Value} + return +} diff --git a/internal/db/params.go b/internal/db/params.go index f0c384435..dbbf734a1 100644 --- a/internal/db/params.go +++ b/internal/db/params.go @@ -22,9 +22,13 @@ package db type Where struct { // The table to search on. Key string - // The value that must be set. + // The value to match. Value interface{} // Whether the value (if a string) should be case sensitive or not. // Defaults to false. CaseInsensitive bool + // If set, reverse the where. + // `WHERE k = v` becomes `WHERE k != v`. + // `WHERE k IS NULL` becomes `WHERE k IS NOT NULL` + Not bool } diff --git a/internal/federation/dereferencing/announce.go b/internal/federation/dereferencing/announce.go index d5cc5ad0c..f0b4c9e9a 100644 --- a/internal/federation/dereferencing/announce.go +++ b/internal/federation/dereferencing/announce.go @@ -60,7 +60,10 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat announce.BoostOfID = boostedStatus.ID announce.BoostOfAccountID = boostedStatus.AccountID announce.Visibility = boostedStatus.Visibility - announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced + announce.Federated = boostedStatus.Federated + announce.Boostable = boostedStatus.Boostable + announce.Replyable = boostedStatus.Replyable + announce.Likeable = boostedStatus.Likeable announce.BoostOf = boostedStatus return nil } diff --git a/internal/federation/dereferencing/status_test.go b/internal/federation/dereferencing/status_test.go index aef83f689..636870232 100644 --- a/internal/federation/dereferencing/status_test.go +++ b/internal/federation/dereferencing/status_test.go @@ -58,6 +58,10 @@ func (suite *StatusTestSuite) TestDereferenceSimpleStatus() { dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI) suite.NoError(err) suite.Equal(status.ID, dbStatus.ID) + suite.True(dbStatus.Federated) + suite.True(dbStatus.Boostable) + suite.True(dbStatus.Replyable) + suite.True(dbStatus.Likeable) // account should be in the database now too account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI) @@ -96,6 +100,10 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() { dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI) suite.NoError(err) suite.Equal(status.ID, dbStatus.ID) + suite.True(dbStatus.Federated) + suite.True(dbStatus.Boostable) + suite.True(dbStatus.Replyable) + suite.True(dbStatus.Likeable) // account should be in the database now too account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI) diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index f298e71cd..38cb6e9c1 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -57,10 +57,13 @@ type Status struct { Language string `validate:"-" bun:",nullzero"` // what language is this status written in? CreatedWithApplicationID string `validate:"required_if=Local true,omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Which application was used to create this status? CreatedWithApplication *Application `validate:"-" bun:"rel:belongs-to"` // application corresponding to createdWithApplicationID - VisibilityAdvanced VisibilityAdvanced `validate:"required" bun:",nullzero,notnull" ` // advanced visibility for this status ActivityStreamsType string `validate:"required" bun:",nullzero,notnull"` // What is the activitystreams type of this status? See: https://www.w3.org/TR/activitystreams-vocabulary/#object-types. Will probably almost always be Note but who knows!. Text string `validate:"-" bun:",nullzero"` // Original text of the status without formatting - Pinned bool `validate:"-" bun:",notnull,default:false" ` // Has this status been pinned by its owner? + Pinned bool `validate:"-" bun:",notnull,default:false"` // Has this status been pinned by its owner? + Federated bool `validate:"-" bun:",notnull"` // This status will be federated beyond the local timeline(s) + Boostable bool `validate:"-" bun:",notnull"` // This status can be boosted/reblogged + Replyable bool `validate:"-" bun:",notnull"` // This status can be replied to + Likeable bool `validate:"-" bun:",notnull"` // This status can be liked/faved } // StatusToTag is an intermediate struct to facilitate the many2many relationship between a status and one or more tags. @@ -96,21 +99,3 @@ const ( // VisibilityDefault is used when no other setting can be found. VisibilityDefault Visibility = VisibilityUnlocked ) - -// VisibilityAdvanced models flags for fine-tuning visibility and interactivity of a status. -// -// All flags default to true. -// -// If PUBLIC is selected, flags will all be overwritten to TRUE regardless of what is selected. -// -// If UNLOCKED is selected, any flags can be turned on or off in any combination. -// -// If FOLLOWERS-ONLY or MUTUALS-ONLY are selected, boostable will always be FALSE. Other flags can be turned on or off as desired. -// -// If DIRECT is selected, boostable will be FALSE, and all other flags will be TRUE. -type VisibilityAdvanced struct { - Federated bool `validate:"-" bun:",notnull,default:true"` // This status will be federated beyond the local timeline(s) - Boostable bool `validate:"-" bun:",notnull,default:true"` // This status can be boosted/reblogged - Replyable bool `validate:"-" bun:",notnull,default:true"` // This status can be replied to - Likeable bool `validate:"-" bun:",notnull,default:true"` // This status can be liked/faved -} diff --git a/internal/processing/fromclientapi.go b/internal/processing/fromclientapi.go index 5f1ed325b..5fd88d8e4 100644 --- a/internal/processing/fromclientapi.go +++ b/internal/processing/fromclientapi.go @@ -51,7 +51,7 @@ func (p *processor) ProcessFromClientAPI(ctx context.Context, clientMsg messages return err } - if status.VisibilityAdvanced.Federated { + if status.Federated { return p.federateStatus(ctx, status) } case ap.ActivityFollow: diff --git a/internal/processing/status/boost.go b/internal/processing/status/boost.go index d6c4ada41..4276ca9fa 100644 --- a/internal/processing/status/boost.go +++ b/internal/processing/status/boost.go @@ -46,7 +46,7 @@ func (p *processor) Boost(ctx context.Context, requestingAccount *gtsmodel.Accou if !visible { return nil, gtserror.NewErrorNotFound(errors.New("status is not visible")) } - if !targetStatus.VisibilityAdvanced.Boostable { + if !targetStatus.Boostable { return nil, gtserror.NewErrorForbidden(errors.New("status is not boostable")) } diff --git a/internal/processing/status/fave.go b/internal/processing/status/fave.go index 195bfa56a..f3f10c43c 100644 --- a/internal/processing/status/fave.go +++ b/internal/processing/status/fave.go @@ -49,7 +49,7 @@ func (p *processor) Fave(ctx context.Context, requestingAccount *gtsmodel.Accoun if !visible { return nil, gtserror.NewErrorNotFound(errors.New("status is not visible")) } - if !targetStatus.VisibilityAdvanced.Likeable { + if !targetStatus.Likeable { return nil, gtserror.NewErrorForbidden(errors.New("status is not faveable")) } diff --git a/internal/processing/status/util.go b/internal/processing/status/util.go index 8861a532b..5ed63d919 100644 --- a/internal/processing/status/util.go +++ b/internal/processing/status/util.go @@ -33,12 +33,10 @@ import ( func (p *processor) ProcessVisibility(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error { // by default all flags are set to true - gtsAdvancedVis := gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - } + federated := true + boostable := true + replyable := true + likeable := true var vis gtsmodel.Visibility // If visibility isn't set on the form, then just take the account default. @@ -58,47 +56,50 @@ func (p *processor) ProcessVisibility(ctx context.Context, form *apimodel.Advanc case gtsmodel.VisibilityUnlocked: // for unlocked the user can set any combination of flags they like so look at them all to see if they're set and then apply them if form.Federated != nil { - gtsAdvancedVis.Federated = *form.Federated + federated = *form.Federated } if form.Boostable != nil { - gtsAdvancedVis.Boostable = *form.Boostable + boostable = *form.Boostable } if form.Replyable != nil { - gtsAdvancedVis.Replyable = *form.Replyable + replyable = *form.Replyable } if form.Likeable != nil { - gtsAdvancedVis.Likeable = *form.Likeable + likeable = *form.Likeable } case gtsmodel.VisibilityFollowersOnly, gtsmodel.VisibilityMutualsOnly: // for followers or mutuals only, boostable will *always* be false, but the other fields can be set so check and apply them - gtsAdvancedVis.Boostable = false + boostable = false if form.Federated != nil { - gtsAdvancedVis.Federated = *form.Federated + federated = *form.Federated } if form.Replyable != nil { - gtsAdvancedVis.Replyable = *form.Replyable + replyable = *form.Replyable } if form.Likeable != nil { - gtsAdvancedVis.Likeable = *form.Likeable + likeable = *form.Likeable } case gtsmodel.VisibilityDirect: // direct is pretty easy: there's only one possible setting so return it - gtsAdvancedVis.Federated = true - gtsAdvancedVis.Boostable = false - gtsAdvancedVis.Federated = true - gtsAdvancedVis.Likeable = true + federated = true + boostable = false + replyable = true + likeable = true } status.Visibility = vis - status.VisibilityAdvanced = gtsAdvancedVis + status.Federated = federated + status.Boostable = boostable + status.Replyable = replyable + status.Likeable = likeable return nil } @@ -123,7 +124,7 @@ func (p *processor) ProcessReplyToID(ctx context.Context, form *apimodel.Advance } return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err) } - if !repliedStatus.VisibilityAdvanced.Replyable { + if !repliedStatus.Replyable { return fmt.Errorf("status with id %s is marked as not replyable", form.InReplyToID) } diff --git a/internal/timeline/get_test.go b/internal/timeline/get_test.go index 96c333c5f..6c4a58c76 100644 --- a/internal/timeline/get_test.go +++ b/internal/timeline/get_test.go @@ -73,8 +73,8 @@ func (suite *GetTestSuite) TestGetDefault() { suite.FailNow(err.Error()) } - // we only have 12 statuses in the test suite - suite.Len(statuses, 12) + // we only have 13 statuses in the test suite + suite.Len(statuses, 13) // statuses should be sorted highest to lowest ID var highest string @@ -166,8 +166,8 @@ func (suite *GetTestSuite) TestGetMinID() { suite.FailNow(err.Error()) } - // we should only get 5 statuses back, since we asked for a min ID that excludes some of our entries - suite.Len(statuses, 5) + // we should only get 6 statuses back, since we asked for a min ID that excludes some of our entries + suite.Len(statuses, 6) // statuses should be sorted highest to lowest ID var highest string @@ -188,8 +188,8 @@ func (suite *GetTestSuite) TestGetSinceID() { suite.FailNow(err.Error()) } - // we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries - suite.Len(statuses, 5) + // we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries + suite.Len(statuses, 6) // statuses should be sorted highest to lowest ID var highest string @@ -210,8 +210,8 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() { suite.FailNow(err.Error()) } - // we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries - suite.Len(statuses, 5) + // we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries + suite.Len(statuses, 6) // statuses should be sorted highest to lowest ID var highest string diff --git a/internal/timeline/index_test.go b/internal/timeline/index_test.go index 25565a1de..2a6429b3e 100644 --- a/internal/timeline/index_test.go +++ b/internal/timeline/index_test.go @@ -66,7 +66,7 @@ func (suite *IndexTestSuite) TestIndexBeforeLowID() { // the oldest indexed post should be the lowest one we have in our testrig postID, err := suite.timeline.OldestIndexedPostID(context.Background()) suite.NoError(err) - suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", postID) + suite.Equal("01F8MHAMCHF6Y650WCRSCP4WMY", postID) indexLength := suite.timeline.PostIndexLength(context.Background()) suite.Equal(10, indexLength) @@ -95,7 +95,7 @@ func (suite *IndexTestSuite) TestIndexBehindHighID() { // the newest indexed post should be the highest one we have in our testrig postID, err := suite.timeline.NewestIndexedPostID(context.Background()) suite.NoError(err) - suite.Equal("01FCTA44PW9H1TB328S9AQXKDS", postID) + suite.Equal("01FF25D5Q0DH7CHD57CTRS6WK0", postID) // indexLength should be 10 because that's all this user has hometimelineable indexLength := suite.timeline.PostIndexLength(context.Background()) diff --git a/internal/timeline/manager_test.go b/internal/timeline/manager_test.go index ea4dc4c12..a67a8ae5a 100644 --- a/internal/timeline/manager_test.go +++ b/internal/timeline/manager_test.go @@ -67,9 +67,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { err = suite.manager.PrepareXFromTop(context.Background(), testAccount.ID, 20) suite.NoError(err) - // local_account_1 can see 12 statuses out of the testrig statuses in its home timeline + // local_account_1 can see 13 statuses out of the testrig statuses in its home timeline indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(12, indexedLen) + suite.Equal(13, indexedLen) // oldest should now be set oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID) @@ -79,7 +79,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // get hometimeline statuses, err := suite.manager.HomeTimeline(context.Background(), testAccount.ID, "", "", "", 20, false) suite.NoError(err) - suite.Len(statuses, 12) + suite.Len(statuses, 13) // now wipe the last status from all timelines, as though it had been deleted by the owner err = suite.manager.WipeStatusFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R") @@ -87,7 +87,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(11, indexedLen) + suite.Equal(12, indexedLen) // oldest should now be different oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID) @@ -101,7 +101,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(10, indexedLen) + suite.Equal(11, indexedLen) // oldest should now be different oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID) @@ -112,9 +112,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { err = suite.manager.WipeStatusesFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID) suite.NoError(err) - // timeline should be empty now + // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(5, indexedLen) + suite.Equal(6, indexedLen) // ingest 1 into the timeline status1 := suite.testStatuses["admin_account_status_1"] @@ -130,7 +130,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be longer now indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(7, indexedLen) + suite.Equal(8, indexedLen) // try to ingest status 2 again ingested, err = suite.manager.IngestAndPrepare(context.Background(), status2, testAccount.ID) diff --git a/internal/trans/decoders.go b/internal/trans/decoders.go new file mode 100644 index 000000000..b4f146023 --- /dev/null +++ b/internal/trans/decoders.go @@ -0,0 +1,138 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "time" + + "github.com/mitchellh/mapstructure" + transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" +) + +func newDecoder(target interface{}) (*mapstructure.Decoder, error) { + decoderConfig := &mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), // this is needed to decode time.Time entries serialized as string + Result: target, + } + return mapstructure.NewDecoder(decoderConfig) +} + +func (i *importer) accountDecode(e transmodel.Entry) (*transmodel.Account, error) { + a := &transmodel.Account{} + if err := i.simpleDecode(e, a); err != nil { + return nil, err + } + + // extract public key + publicKeyBlock, _ := pem.Decode([]byte(a.PublicKeyString)) + if publicKeyBlock == nil { + return nil, errors.New("accountDecode: error decoding account public key") + } + publicKey, err := x509.ParsePKCS1PublicKey(publicKeyBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("accountDecode: error parsing account public key: %s", err) + } + a.PublicKey = publicKey + + if a.Domain == "" { + // extract private key (local account) + privateKeyBlock, _ := pem.Decode([]byte(a.PrivateKeyString)) + if privateKeyBlock == nil { + return nil, errors.New("accountDecode: error decoding account private key") + } + privateKey, err := x509.ParsePKCS1PrivateKey(privateKeyBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("accountDecode: error parsing account private key: %s", err) + } + a.PrivateKey = privateKey + } + + return a, nil +} + +func (i *importer) blockDecode(e transmodel.Entry) (*transmodel.Block, error) { + b := &transmodel.Block{} + if err := i.simpleDecode(e, b); err != nil { + return nil, err + } + + return b, nil +} + +func (i *importer) domainBlockDecode(e transmodel.Entry) (*transmodel.DomainBlock, error) { + b := &transmodel.DomainBlock{} + if err := i.simpleDecode(e, b); err != nil { + return nil, err + } + + return b, nil +} + +func (i *importer) followDecode(e transmodel.Entry) (*transmodel.Follow, error) { + f := &transmodel.Follow{} + if err := i.simpleDecode(e, f); err != nil { + return nil, err + } + + return f, nil +} + +func (i *importer) followRequestDecode(e transmodel.Entry) (*transmodel.FollowRequest, error) { + f := &transmodel.FollowRequest{} + if err := i.simpleDecode(e, f); err != nil { + return nil, err + } + + return f, nil +} + +func (i *importer) instanceDecode(e transmodel.Entry) (*transmodel.Instance, error) { + inst := &transmodel.Instance{} + if err := i.simpleDecode(e, inst); err != nil { + return nil, err + } + + return inst, nil +} + +func (i *importer) userDecode(e transmodel.Entry) (*transmodel.User, error) { + u := &transmodel.User{} + if err := i.simpleDecode(e, u); err != nil { + return nil, err + } + + return u, nil +} + +func (i *importer) simpleDecode(entry transmodel.Entry, target interface{}) error { + decoder, err := newDecoder(target) + if err != nil { + return fmt.Errorf("simpleDecode: error creating decoder: %s", err) + } + + if err := decoder.Decode(&entry); err != nil { + return fmt.Errorf("simpleDecode: error decoding: %s", err) + } + + return nil +} diff --git a/internal/trans/encoders.go b/internal/trans/encoders.go new file mode 100644 index 000000000..76c2acadc --- /dev/null +++ b/internal/trans/encoders.go @@ -0,0 +1,83 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "os" + + transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" +) + +// accountEncode handles special fields like private + public keys on accounts +func (e *exporter) accountEncode(ctx context.Context, f *os.File, a *transmodel.Account) error { + a.Type = transmodel.TransAccount + + // marshal public key + encodedPublicKey := x509.MarshalPKCS1PublicKey(a.PublicKey) + if encodedPublicKey == nil { + return errors.New("could not MarshalPKCS1PublicKey") + } + publicKeyBytes := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PUBLIC KEY", + Bytes: encodedPublicKey, + }) + a.PublicKeyString = string(publicKeyBytes) + + if a.Domain == "" { + // marshal private key for local account + encodedPrivateKey := x509.MarshalPKCS1PrivateKey(a.PrivateKey) + if encodedPrivateKey == nil { + return errors.New("could not MarshalPKCS1PrivateKey") + } + privateKeyBytes := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: encodedPrivateKey, + }) + a.PrivateKeyString = string(privateKeyBytes) + } + + return e.simpleEncode(ctx, f, a, a.ID) +} + +// simpleEncode can be used for any type that doesn't have special keys which need handling differently, +// or for types where special keys have already been handled. +// +// Beware, the 'type' key on the passed interface should already have been set, since simpleEncode won't know +// what type it is! If you try to decode stuff you've encoded with a missing type key, you're going to have a bad time. +func (e *exporter) simpleEncode(ctx context.Context, file *os.File, i interface{}, id string) error { + _, alreadyWritten := e.writtenIDs[id] + if alreadyWritten { + // this exporter has already exported an entry with this ID, no need to do it twice + return nil + } + + err := json.NewEncoder(file).Encode(i) + if err != nil { + return fmt.Errorf("simpleEncode: error encoding entry with id %s: %s", id, err) + } + + e.writtenIDs[id] = true + return nil +} diff --git a/internal/trans/export.go b/internal/trans/export.go new file mode 100644 index 000000000..a432975be --- /dev/null +++ b/internal/trans/export.go @@ -0,0 +1,223 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/package trans + +import ( + "context" + "fmt" + "os" + + "github.com/superseriousbusiness/gotosocial/internal/db" + transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" +) + +func (e *exporter) exportAccounts(ctx context.Context, where []db.Where, file *os.File) ([]*transmodel.Account, error) { + // select using the 'where' we've been provided + accounts := []*transmodel.Account{} + if err := e.db.GetWhere(ctx, where, &accounts); err != nil { + return nil, fmt.Errorf("exportAccounts: error selecting accounts: %s", err) + } + + // write any accounts found to file + for _, a := range accounts { + if err := e.accountEncode(ctx, file, a); err != nil { + return nil, fmt.Errorf("exportAccounts: error encoding account: %s", err) + } + } + + return accounts, nil +} + +func (e *exporter) exportBlocks(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.Block, error) { + blocksUnique := make(map[string]*transmodel.Block) + + // for each account we want to export both where it's blocking and where it's blocked + for _, a := range accounts { + // 1. export blocks owned by given account + whereBlocking := []db.Where{{Key: "account_id", Value: a.ID}} + blocking := []*transmodel.Block{} + if err := e.db.GetWhere(ctx, whereBlocking, &blocking); err != nil { + return nil, fmt.Errorf("exportBlocks: error selecting blocks owned by account %s: %s", a.ID, err) + } + for _, b := range blocking { + b.Type = transmodel.TransBlock + if err := e.simpleEncode(ctx, file, b, b.ID); err != nil { + return nil, fmt.Errorf("exportBlocks: error encoding block owned by account %s: %s", a.ID, err) + } + blocksUnique[b.ID] = b + } + + // 2. export blocks that target given account + whereBlocked := []db.Where{{Key: "target_account_id", Value: a.ID}} + blocked := []*transmodel.Block{} + if err := e.db.GetWhere(ctx, whereBlocked, &blocked); err != nil { + return nil, fmt.Errorf("exportBlocks: error selecting blocks targeting account %s: %s", a.ID, err) + } + for _, b := range blocked { + b.Type = transmodel.TransBlock + if err := e.simpleEncode(ctx, file, b, b.ID); err != nil { + return nil, fmt.Errorf("exportBlocks: error encoding block targeting account %s: %s", a.ID, err) + } + blocksUnique[b.ID] = b + } + } + + // now return all the blocks we found + blocks := []*transmodel.Block{} + for _, b := range blocksUnique { + blocks = append(blocks, b) + } + + return blocks, nil +} + +func (e *exporter) exportDomainBlocks(ctx context.Context, file *os.File) ([]*transmodel.DomainBlock, error) { + domainBlocks := []*transmodel.DomainBlock{} + + if err := e.db.GetAll(ctx, &domainBlocks); err != nil { + return nil, fmt.Errorf("exportBlocks: error selecting domain blocks: %s", err) + } + + for _, b := range domainBlocks { + b.Type = transmodel.TransDomainBlock + if err := e.simpleEncode(ctx, file, b, b.ID); err != nil { + return nil, fmt.Errorf("exportBlocks: error encoding domain block: %s", err) + } + } + + return domainBlocks, nil +} + +func (e *exporter) exportFollows(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.Follow, error) { + followsUnique := make(map[string]*transmodel.Follow) + + // for each account we want to export both where it's following and where it's followed + for _, a := range accounts { + // 1. export follows owned by given account + whereFollowing := []db.Where{{Key: "account_id", Value: a.ID}} + following := []*transmodel.Follow{} + if err := e.db.GetWhere(ctx, whereFollowing, &following); err != nil { + return nil, fmt.Errorf("exportFollows: error selecting follows owned by account %s: %s", a.ID, err) + } + for _, follow := range following { + follow.Type = transmodel.TransFollow + if err := e.simpleEncode(ctx, file, follow, follow.ID); err != nil { + return nil, fmt.Errorf("exportFollows: error encoding follow owned by account %s: %s", a.ID, err) + } + followsUnique[follow.ID] = follow + } + + // 2. export follows that target given account + whereFollowed := []db.Where{{Key: "target_account_id", Value: a.ID}} + followed := []*transmodel.Follow{} + if err := e.db.GetWhere(ctx, whereFollowed, &followed); err != nil { + return nil, fmt.Errorf("exportFollows: error selecting follows targeting account %s: %s", a.ID, err) + } + for _, follow := range followed { + follow.Type = transmodel.TransFollow + if err := e.simpleEncode(ctx, file, follow, follow.ID); err != nil { + return nil, fmt.Errorf("exportFollows: error encoding follow targeting account %s: %s", a.ID, err) + } + followsUnique[follow.ID] = follow + } + } + + // now return all the follows we found + follows := []*transmodel.Follow{} + for _, follow := range followsUnique { + follows = append(follows, follow) + } + + return follows, nil +} + +func (e *exporter) exportFollowRequests(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.FollowRequest, error) { + frsUnique := make(map[string]*transmodel.FollowRequest) + + // for each account we want to export both where it's following and where it's followed + for _, a := range accounts { + // 1. export follow requests owned by given account + whereRequesting := []db.Where{{Key: "account_id", Value: a.ID}} + requesting := []*transmodel.FollowRequest{} + if err := e.db.GetWhere(ctx, whereRequesting, &requesting); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests owned by account %s: %s", a.ID, err) + } + for _, fr := range requesting { + fr.Type = transmodel.TransFollowRequest + if err := e.simpleEncode(ctx, file, fr, fr.ID); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error encoding follow request owned by account %s: %s", a.ID, err) + } + frsUnique[fr.ID] = fr + } + + // 2. export follow requests that target given account + whereRequested := []db.Where{{Key: "target_account_id", Value: a.ID}} + requested := []*transmodel.FollowRequest{} + if err := e.db.GetWhere(ctx, whereRequested, &requested); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests targeting account %s: %s", a.ID, err) + } + for _, fr := range requested { + fr.Type = transmodel.TransFollowRequest + if err := e.simpleEncode(ctx, file, fr, fr.ID); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error encoding follow request targeting account %s: %s", a.ID, err) + } + frsUnique[fr.ID] = fr + } + } + + // now return all the followRequests we found + followRequests := []*transmodel.FollowRequest{} + for _, fr := range frsUnique { + followRequests = append(followRequests, fr) + } + + return followRequests, nil +} + +func (e *exporter) exportInstances(ctx context.Context, file *os.File) ([]*transmodel.Instance, error) { + instances := []*transmodel.Instance{} + + if err := e.db.GetAll(ctx, &instances); err != nil { + return nil, fmt.Errorf("exportInstances: error selecting instance: %s", err) + } + + for _, u := range instances { + u.Type = transmodel.TransInstance + if err := e.simpleEncode(ctx, file, u, u.ID); err != nil { + return nil, fmt.Errorf("exportInstances: error encoding instance: %s", err) + } + } + + return instances, nil +} + +func (e *exporter) exportUsers(ctx context.Context, file *os.File) ([]*transmodel.User, error) { + users := []*transmodel.User{} + + if err := e.db.GetAll(ctx, &users); err != nil { + return nil, fmt.Errorf("exportUsers: error selecting users: %s", err) + } + + for _, u := range users { + u.Type = transmodel.TransUser + if err := e.simpleEncode(ctx, file, u, u.ID); err != nil { + return nil, fmt.Errorf("exportUsers: error encoding user: %s", err) + } + } + + return users, nil +} diff --git a/internal/trans/exporter.go b/internal/trans/exporter.go new file mode 100644 index 000000000..3dc0558f5 --- /dev/null +++ b/internal/trans/exporter.go @@ -0,0 +1,46 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "context" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +// Exporter wraps functionality for exporting entries from the database to a file. +type Exporter interface { + ExportMinimal(ctx context.Context, path string) error +} + +type exporter struct { + db db.DB + log *logrus.Logger + writtenIDs map[string]bool +} + +// NewExporter returns a new Exporter that will use the given db and logger. +func NewExporter(db db.DB, log *logrus.Logger) Exporter { + return &exporter{ + db: db, + log: log, + writtenIDs: make(map[string]bool), + } +} diff --git a/internal/trans/exportminimal.go b/internal/trans/exportminimal.go new file mode 100644 index 000000000..a9ace9b91 --- /dev/null +++ b/internal/trans/exportminimal.go @@ -0,0 +1,150 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +func (e *exporter) ExportMinimal(ctx context.Context, path string) error { + if path == "" { + return errors.New("ExportMinimal: path empty") + } + + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("ExportMinimal: couldn't export to %s: %s", path, err) + } + + // export all local accounts we have in the database + localAccounts, err := e.exportAccounts(ctx, []db.Where{{Key: "domain", Value: nil}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting accounts: %s", err) + } + + // export all blocks that relate to local accounts + blocks, err := e.exportBlocks(ctx, localAccounts, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting blocks: %s", err) + } + + // for each block, make sure we've written out the account owning it, or targeted by it -- + // this might include non-local accounts, but we need these so we don't lose anything + for _, b := range blocks { + _, alreadyWritten := e.writtenIDs[b.AccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.AccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting block owner account: %s", err) + } + } + + _, alreadyWritten = e.writtenIDs[b.TargetAccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.TargetAccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting block target account: %s", err) + } + } + } + + // export all follows that relate to local accounts + follows, err := e.exportFollows(ctx, localAccounts, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follows: %s", err) + } + + // for each follow, make sure we've written out the account owning it, or targeted by it -- + // this might include non-local accounts, but we need these so we don't lose anything + for _, follow := range follows { + _, alreadyWritten := e.writtenIDs[follow.AccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.AccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow owner account: %s", err) + } + } + + _, alreadyWritten = e.writtenIDs[follow.TargetAccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.TargetAccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow target account: %s", err) + } + } + } + + // export all follow requests that relate to local accounts + followRequests, err := e.exportFollowRequests(ctx, localAccounts, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow requests: %s", err) + } + + // for each follow request, make sure we've written out the account owning it, or targeted by it -- + // this might include non-local accounts, but we need these so we don't lose anything + for _, fr := range followRequests { + _, alreadyWritten := e.writtenIDs[fr.AccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.AccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow request owner account: %s", err) + } + } + + _, alreadyWritten = e.writtenIDs[fr.TargetAccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.TargetAccountID}}, file) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow request target account: %s", err) + } + } + } + + // export all domain blocks + if _, err := e.exportDomainBlocks(ctx, file); err != nil { + return fmt.Errorf("ExportMinimal: error exporting domain blocks: %s", err) + } + + // export all users + if _, err := e.exportUsers(ctx, file); err != nil { + return fmt.Errorf("ExportMinimal: error exporting users: %s", err) + } + + // export all instances + if _, err := e.exportInstances(ctx, file); err != nil { + return fmt.Errorf("ExportMinimal: error exporting instances: %s", err) + } + + // export all SUSPENDED accounts to make sure the suspension sticks across db migration etc + whereSuspended := []db.Where{{ + Key: "suspended_at", + Not: true, + Value: nil, + }} + if _, err := e.exportAccounts(ctx, whereSuspended, file); err != nil { + return fmt.Errorf("ExportMinimal: error exporting suspended accounts: %s", err) + } + + return neatClose(file) +} diff --git a/internal/trans/exportminimal_test.go b/internal/trans/exportminimal_test.go new file mode 100644 index 000000000..2bffffcfe --- /dev/null +++ b/internal/trans/exportminimal_test.go @@ -0,0 +1,54 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans_test + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/trans" +) + +type ExportMinimalTestSuite struct { + TransTestSuite +} + +func (suite *ExportMinimalTestSuite) TestExportMinimalOK() { + // use a temporary file path that will be cleaned when the test is closed + tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString()) + + // export to the tempFilePath + exporter := trans.NewExporter(suite.db, suite.log) + err := exporter.ExportMinimal(context.Background(), tempFilePath) + suite.NoError(err) + + // we should have some bytes in that file now + b, err := os.ReadFile(tempFilePath) + suite.NoError(err) + suite.NotEmpty(b) + fmt.Println(string(b)) +} + +func TestExportMinimalTestSuite(t *testing.T) { + suite.Run(t, &ExportMinimalTestSuite{}) +} diff --git a/internal/trans/import.go b/internal/trans/import.go new file mode 100644 index 000000000..74624e540 --- /dev/null +++ b/internal/trans/import.go @@ -0,0 +1,146 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + + transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" +) + +func (i *importer) Import(ctx context.Context, path string) error { + if path == "" { + return errors.New("Export: path empty") + } + + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("Import: couldn't export to %s: %s", path, err) + } + + decoder := json.NewDecoder(file) + decoder.UseNumber() + + for { + entry := transmodel.Entry{} + err := decoder.Decode(&entry) + if err != nil { + if err == io.EOF { + i.log.Infof("Import: reached end of file") + return neatClose(file) + } + return fmt.Errorf("Import: error decoding in readLoop: %s", err) + } + if err := i.inputEntry(ctx, entry); err != nil { + return fmt.Errorf("Import: error inputting entry: %s", err) + } + } +} + +func (i *importer) inputEntry(ctx context.Context, entry transmodel.Entry) error { + t, ok := entry[transmodel.TypeKey].(string) + if !ok { + return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json") + } + + switch transmodel.Type(t) { + case transmodel.TransAccount: + account, err := i.accountDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into account: %s", err) + } + if err := i.putInDB(ctx, account); err != nil { + return fmt.Errorf("inputEntry: error adding account to database: %s", err) + } + i.log.Infof("inputEntry: added account with id %s", account.ID) + return nil + case transmodel.TransBlock: + block, err := i.blockDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into block: %s", err) + } + if err := i.putInDB(ctx, block); err != nil { + return fmt.Errorf("inputEntry: error adding block to database: %s", err) + } + i.log.Infof("inputEntry: added block with id %s", block.ID) + return nil + case transmodel.TransDomainBlock: + block, err := i.domainBlockDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into domain block: %s", err) + } + if err := i.putInDB(ctx, block); err != nil { + return fmt.Errorf("inputEntry: error adding domain block to database: %s", err) + } + i.log.Infof("inputEntry: added domain block with id %s", block.ID) + return nil + case transmodel.TransFollow: + follow, err := i.followDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into follow: %s", err) + } + if err := i.putInDB(ctx, follow); err != nil { + return fmt.Errorf("inputEntry: error adding follow to database: %s", err) + } + i.log.Infof("inputEntry: added follow with id %s", follow.ID) + return nil + case transmodel.TransFollowRequest: + fr, err := i.followRequestDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into follow request: %s", err) + } + if err := i.putInDB(ctx, fr); err != nil { + return fmt.Errorf("inputEntry: error adding follow request to database: %s", err) + } + i.log.Infof("inputEntry: added follow request with id %s", fr.ID) + return nil + case transmodel.TransInstance: + inst, err := i.instanceDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into instance: %s", err) + } + if err := i.putInDB(ctx, inst); err != nil { + return fmt.Errorf("inputEntry: error adding instance to database: %s", err) + } + i.log.Infof("inputEntry: added instance with id %s", inst.ID) + return nil + case transmodel.TransUser: + user, err := i.userDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into user: %s", err) + } + if err := i.putInDB(ctx, user); err != nil { + return fmt.Errorf("inputEntry: error adding user to database: %s", err) + } + i.log.Infof("inputEntry: added user with id %s", user.ID) + return nil + } + + i.log.Errorf("inputEntry: didn't recognize transtype '%s', skipping it", t) + return nil +} + +func (i *importer) putInDB(ctx context.Context, entry interface{}) error { + return i.db.Put(ctx, entry) +} diff --git a/internal/trans/import_test.go b/internal/trans/import_test.go new file mode 100644 index 000000000..137a5fae1 --- /dev/null +++ b/internal/trans/import_test.go @@ -0,0 +1,91 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans_test + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/trans" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type ImportMinimalTestSuite struct { + TransTestSuite +} + +func (suite *ImportMinimalTestSuite) TestImportMinimalOK() { + ctx := context.Background() + + // use a temporary file path + tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString()) + + // export to the tempFilePath + exporter := trans.NewExporter(suite.db, suite.log) + err := exporter.ExportMinimal(ctx, tempFilePath) + suite.NoError(err) + + // we should have some bytes in that file now + b, err := os.ReadFile(tempFilePath) + suite.NoError(err) + suite.NotEmpty(b) + fmt.Println(string(b)) + + // create a new database with just the tables created, no entries + testrig.StandardDBTeardown(suite.db) + newDB := testrig.NewTestDB() + testrig.CreateTestTables(newDB) + + importer := trans.NewImporter(newDB, suite.log) + err = importer.Import(ctx, tempFilePath) + suite.NoError(err) + + // we should have some accounts in the database + accounts := []*gtsmodel.Account{} + err = newDB.GetAll(ctx, &accounts) + suite.NoError(err) + suite.NotEmpty(accounts) + + // we should have some blocks in the database + blocks := []*gtsmodel.Block{} + err = newDB.GetAll(ctx, &blocks) + suite.NoError(err) + suite.NotEmpty(blocks) + + // we should have some follows in the database + follows := []*gtsmodel.Follow{} + err = newDB.GetAll(ctx, &follows) + suite.NoError(err) + suite.NotEmpty(follows) + + // we should have some domain blocks in the database + domainBlocks := []*gtsmodel.DomainBlock{} + err = newDB.GetAll(ctx, &domainBlocks) + suite.NoError(err) + suite.NotEmpty(domainBlocks) +} + +func TestImportMinimalTestSuite(t *testing.T) { + suite.Run(t, &ImportMinimalTestSuite{}) +} diff --git a/internal/trans/importer.go b/internal/trans/importer.go new file mode 100644 index 000000000..ea8866f53 --- /dev/null +++ b/internal/trans/importer.go @@ -0,0 +1,44 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "context" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +// Importer wraps functionality for importing entries from a file into the database. +type Importer interface { + Import(ctx context.Context, path string) error +} + +type importer struct { + db db.DB + log *logrus.Logger +} + +// NewImporter returns a new Importer interface that uses the given db and logger. +func NewImporter(db db.DB, log *logrus.Logger) Importer { + return &importer{ + db: db, + log: log, + } +} diff --git a/internal/trans/model/account.go b/internal/trans/model/account.go new file mode 100644 index 000000000..2350048d7 --- /dev/null +++ b/internal/trans/model/account.go @@ -0,0 +1,54 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "crypto/rsa" + "time" +) + +// Account represents the minimum viable representation of an account for export/import. +type Account struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Username string `json:"username" bun:",nullzero"` + DisplayName string `json:"displayName,omitempty" bun:",nullzero"` + Note string `json:"note,omitempty" bun:",nullzero"` + Domain string `json:"domain,omitempty" bun:",nullzero"` + HeaderRemoteURL string `json:"headerRemoteURL,omitempty" bun:",nullzero"` + AvatarRemoteURL string `json:"avatarRemoteURL,omitempty" bun:",nullzero"` + Locked bool `json:"locked"` + Language string `json:"language,omitempty" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + URL string `json:"url" bun:",nullzero"` + InboxURI string `json:"inboxURI" bun:",nullzero"` + OutboxURI string `json:"outboxURI" bun:",nullzero"` + FollowingURI string `json:"followingUri" bun:",nullzero"` + FollowersURI string `json:"followersUri" bun:",nullzero"` + FeaturedCollectionURI string `json:"featuredCollectionUri" bun:",nullzero"` + ActorType string `json:"actorType" bun:",nullzero"` + PrivateKey *rsa.PrivateKey `json:"-" mapstructure:"-"` + PrivateKeyString string `json:"privateKey,omitempty" mapstructure:"privateKey" bun:"-"` + PublicKey *rsa.PublicKey `json:"-" mapstructure:"-"` + PublicKeyString string `json:"publicKey,omitempty" mapstructure:"publicKey" bun:"-"` + PublicKeyURI string `json:"publicKeyUri" bun:",nullzero"` + SuspendedAt *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"` + SuspensionOrigin string `json:"suspensionOrigin,omitempty" bun:",nullzero"` +} diff --git a/internal/trans/model/block.go b/internal/trans/model/block.go new file mode 100644 index 000000000..313e6a7cd --- /dev/null +++ b/internal/trans/model/block.go @@ -0,0 +1,31 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import "time" + +// Block represents an account block as serialized in an exported file. +type Block struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + AccountID string `json:"accountId" bun:",nullzero"` + TargetAccountID string `json:"targetAccountId" bun:",nullzero"` +} diff --git a/internal/trans/model/domainblock.go b/internal/trans/model/domainblock.go new file mode 100644 index 000000000..de2bcd00a --- /dev/null +++ b/internal/trans/model/domainblock.go @@ -0,0 +1,34 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import "time" + +// DomainBlock represents a domain block as serialized in an exported file. +type DomainBlock struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Domain string `json:"domain" bun:",nullzero"` + CreatedByAccountID string `json:"createdByAccountID" bun:",nullzero"` + PrivateComment string `json:"privateComment,omitempty" bun:",nullzero"` + PublicComment string `json:"publicComment,omitempty" bun:",nullzero"` + Obfuscate bool `json:"obfuscate" bun:",nullzero"` + SubscriptionID string `json:"subscriptionID,omitempty" bun:",nullzero"` +} diff --git a/internal/trans/model/follow.go b/internal/trans/model/follow.go new file mode 100644 index 000000000..b94f2600d --- /dev/null +++ b/internal/trans/model/follow.go @@ -0,0 +1,31 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import "time" + +// Follow represents an account follow as serialized in an export file. +type Follow struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + AccountID string `json:"accountId" bun:",nullzero"` + TargetAccountID string `json:"targetAccountId" bun:",nullzero"` +} diff --git a/internal/trans/model/followrequest.go b/internal/trans/model/followrequest.go new file mode 100644 index 000000000..844bcb7af --- /dev/null +++ b/internal/trans/model/followrequest.go @@ -0,0 +1,31 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import "time" + +// FollowRequest represents an account follow request as serialized in an export file. +type FollowRequest struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + AccountID string `json:"accountId" bun:",nullzero"` + TargetAccountID string `json:"targetAccountId" bun:",nullzero"` +} diff --git a/internal/trans/model/instance.go b/internal/trans/model/instance.go new file mode 100644 index 000000000..a75aa65bf --- /dev/null +++ b/internal/trans/model/instance.go @@ -0,0 +1,43 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "time" +) + +// Instance represents an instance entry as serialized in an export file. +type Instance struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Domain string `json:"domain" bun:",nullzero"` + Title string `json:"title,omitempty" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + SuspendedAt *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"` + DomainBlockID string `json:"domainBlockID,omitempty" bun:",nullzero"` + ShortDescription string `json:"shortDescription,omitempty" bun:",nullzero"` + Description string `json:"description,omitempty" bun:",nullzero"` + Terms string `json:"terms,omitempty" bun:",nullzero"` + ContactEmail string `json:"contactEmail,omitempty" bun:",nullzero"` + ContactAccountUsername string `json:"contactAccountUsername,omitempty" bun:",nullzero"` + ContactAccountID string `json:"contactAccountID,omitempty" bun:",nullzero"` + Reputation int64 `json:"reputation"` + Version string `json:"version,omitempty" bun:",nullzero"` +} diff --git a/internal/trans/model/type.go b/internal/trans/model/type.go new file mode 100644 index 000000000..76f57c843 --- /dev/null +++ b/internal/trans/model/type.go @@ -0,0 +1,41 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +// TypeKey should be set on a TransEntry to indicate the type of entry it is. +const TypeKey = "type" + +// Type describes the type of a trans entry, and how it should be read/serialized. +type Type string + +// Type of the trans entry. Describes how it should be read from file. +const ( + TransAccount Type = "account" + TransBlock Type = "block" + TransDomainBlock Type = "domainBlock" + TransEmailDomainBlock Type = "emailDomainBlock" + TransFollow Type = "follow" + TransFollowRequest Type = "followRequest" + TransInstance Type = "instance" + TransUser Type = "user" +) + +// Entry is used for deserializing trans entries into a rough interface so that +// the TypeKey can be fetched, before continuing with full parsing. +type Entry map[string]interface{} diff --git a/internal/trans/model/user.go b/internal/trans/model/user.go new file mode 100644 index 000000000..293b124a2 --- /dev/null +++ b/internal/trans/model/user.go @@ -0,0 +1,50 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "time" +) + +// User represents a local instance user as serialized to an export file. +type User struct { + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Email string `json:"email,omitempty" bun:",nullzero"` + AccountID string `json:"accountID" bun:",nullzero"` + EncryptedPassword string `json:"encryptedPassword" bun:",nullzero"` + CurrentSignInAt *time.Time `json:"currentSignInAt,omitempty" bun:",nullzero"` + LastSignInAt *time.Time `json:"lastSignInAt,omitempty" bun:",nullzero"` + InviteID string `json:"inviteID,omitempty" bun:",nullzero"` + ChosenLanguages []string `json:"chosenLanguages,omitempty" bun:",nullzero"` + FilteredLanguages []string `json:"filteredLanguage,omitempty" bun:",nullzero"` + Locale string `json:"locale" bun:",nullzero"` + LastEmailedAt time.Time `json:"lastEmailedAt,omitempty" bun:",nullzero"` + ConfirmationToken string `json:"confirmationToken,omitempty" bun:",nullzero"` + ConfirmationSentAt *time.Time `json:"confirmationTokenSentAt,omitempty" bun:",nullzero"` + ConfirmedAt *time.Time `json:"confirmedAt,omitempty" bun:",nullzero"` + UnconfirmedEmail string `json:"unconfirmedEmail,omitempty" bun:",nullzero"` + Moderator bool `json:"moderator"` + Admin bool `json:"admin"` + Disabled bool `json:"disabled"` + Approved bool `json:"approved"` + ResetPasswordToken string `json:"resetPasswordToken,omitempty" bun:",nullzero"` + ResetPasswordSentAt *time.Time `json:"resetPasswordSentAt,omitempty" bun:",nullzero"` +} diff --git a/internal/trans/trans_test.go b/internal/trans/trans_test.go new file mode 100644 index 000000000..4a231486b --- /dev/null +++ b/internal/trans/trans_test.go @@ -0,0 +1,42 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans_test + +import ( + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type TransTestSuite struct { + suite.Suite + db db.DB + log *logrus.Logger +} + +func (suite *TransTestSuite) SetupTest() { + suite.db = testrig.NewTestDB() + suite.log = testrig.NewTestLog() + testrig.StandardDBSetup(suite.db, nil) +} + +func (suite *TransTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) +} diff --git a/internal/trans/util.go b/internal/trans/util.go new file mode 100644 index 000000000..4ccc1a4b6 --- /dev/null +++ b/internal/trans/util.go @@ -0,0 +1,32 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "fmt" + "os" +) + +func neatClose(f *os.File) error { + if err := f.Close(); err != nil { + return fmt.Errorf("error closing file: %s", err) + } + + return nil +} diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index 580f999bc..9b87e03d3 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -325,6 +325,11 @@ func (c *converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab // advanced visibility for this status // TODO: a lot of work to be done here -- a new type needs to be created for this in go-fed/activity using ASTOOL + // for now we just set everything to true + status.Federated = true + status.Boostable = true + status.Replyable = true + status.Likeable = true // sensitive // TODO: this is a bool diff --git a/internal/typeutils/internal.go b/internal/typeutils/internal.go index 23839b9a8..b6a425732 100644 --- a/internal/typeutils/internal.go +++ b/internal/typeutils/internal.go @@ -70,7 +70,10 @@ func (c *converter) StatusToBoost(ctx context.Context, s *gtsmodel.Status, boost BoostOfID: s.ID, BoostOfAccountID: s.AccountID, Visibility: s.Visibility, - VisibilityAdvanced: s.VisibilityAdvanced, + Federated: s.Federated, + Boostable: s.Boostable, + Replyable: s.Replyable, + Likeable: s.Likeable, // attach these here for convenience -- the boosted status/account won't go in the DB // but they're needed in the processor and for the frontend. Since we have them, we can diff --git a/internal/validate/status_test.go b/internal/validate/status_test.go index 7c85414b1..333cc408d 100644 --- a/internal/validate/status_test.go +++ b/internal/validate/status_test.go @@ -63,15 +63,13 @@ func happyStatus() *gtsmodel.Status { Language: "en", CreatedWithApplicationID: "01FEBBZHF4GFVRXSJVXD0JTZZ2", CreatedWithApplication: nil, - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, - Text: "Test status! #hello", - Pinned: false, + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, + Text: "Test status! #hello", + Pinned: false, } } diff --git a/testrig/db.go b/testrig/db.go index 7cb4f7645..268ba16b7 100644 --- a/testrig/db.go +++ b/testrig/db.go @@ -72,6 +72,16 @@ func NewTestDB() db.DB { return testDB } +// CreateTestTables creates prerequisite test tables in the database, but doesn't populate them. +func CreateTestTables(db db.DB) { + ctx := context.Background() + for _, m := range testModels { + if err := db.CreateTable(ctx, m); err != nil { + logrus.Panicf("error creating table for %+v: %s", m, err) + } + } +} + // StandardDBSetup populates a given db with all the necessary tables/models for perfoming tests. // // The accounts parameter is provided in case the db should be populated with a certain set of accounts. @@ -85,13 +95,9 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) { logrus.Panic("db setup: db was nil") } - ctx := context.Background() + CreateTestTables(db) - for _, m := range testModels { - if err := db.CreateTable(ctx, m); err != nil { - logrus.Panicf("error creating table for %+v: %s", m, err) - } - } + ctx := context.Background() for _, v := range NewTestTokens() { if err := db.Put(ctx, v); err != nil { @@ -111,6 +117,18 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) { } } + for _, v := range NewTestBlocks() { + if err := db.Put(ctx, v); err != nil { + logrus.Panic(err) + } + } + + for _, v := range NewTestDomainBlocks() { + if err := db.Put(ctx, v); err != nil { + logrus.Panic(err) + } + } + for _, v := range NewTestUsers() { if err := db.Put(ctx, v); err != nil { logrus.Panic(err) diff --git a/testrig/testmodels.go b/testrig/testmodels.go index e4daed12c..91335c2a3 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -736,6 +736,19 @@ func NewTestEmojis() map[string]*gtsmodel.Emoji { } } +func NewTestDomainBlocks() map[string]*gtsmodel.DomainBlock { + return map[string]*gtsmodel.DomainBlock{ + "replyguys.com": { + ID: "01FF22EQM7X8E3RX1XGPN7S87D", + Domain: "replyguys.com", + CreatedByAccountID: "01F8MH17FWEB39HZJ76B6VXSKF", + PrivateComment: "i blocked this domain because they keep replying with pushy + unwarranted linux advice", + PublicComment: "reply-guying to tech posts", + Obfuscate: false, + }, + } +} + type filenames struct { Original string Small string @@ -803,13 +816,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: false, Language: "en", CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, }, "admin_account_status_2": { ID: "01F8MHAAY43M6RJ473VQFCVH37", @@ -828,13 +839,36 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: true, Language: "en", CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, + }, + "admin_account_status_3": { + ID: "01FF25D5Q0DH7CHD57CTRS6WK0", + URI: "http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0", + URL: "http://localhost:8080/@admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0", + Content: "hi @the_mighty_zork welcome to the instance!", + CreatedAt: time.Now().Add(-46 * time.Hour), + UpdatedAt: time.Now().Add(-46 * time.Hour), + Local: true, + AccountURI: "http://localhost:8080/users/admin", + MentionIDs: []string{"01FF26A6BGEKCZFWNEHXB2ZZ6M"}, + AccountID: "01F8MH17FWEB39HZJ76B6VXSKF", + InReplyToID: "01F8MHAMCHF6Y650WCRSCP4WMY", + InReplyToAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", + InReplyToURI: "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY", + BoostOfID: "", + Visibility: gtsmodel.VisibilityPublic, + Sensitive: false, + Language: "en", + CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F", + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, }, "local_account_1_status_1": { ID: "01F8MHAMCHF6Y650WCRSCP4WMY", @@ -853,13 +887,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: true, Language: "en", CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, }, "local_account_1_status_2": { ID: "01F8MHAYFKS4KMXF8K5Y1C0KRN", @@ -878,13 +910,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: false, Language: "en", CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: false, - Boostable: true, - Replyable: true, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: false, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, }, "local_account_1_status_3": { ID: "01F8MHBBN8120SYH7D5S050MGK", @@ -903,13 +933,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: false, Language: "en", CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: false, - Replyable: false, - Likeable: false, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: true, + Boostable: false, + Replyable: false, + Likeable: false, + ActivityStreamsType: ap.ObjectNote, }, "local_account_1_status_4": { ID: "01F8MH82FYRXD2RC6108DAJ5HB", @@ -929,13 +957,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: false, Language: "en", CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, }, "local_account_1_status_5": { ID: "01FCTA44PW9H1TB328S9AQXKDS", @@ -955,13 +981,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: false, Language: "en", CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, }, "local_account_2_status_1": { ID: "01F8MHBQCBTDKN6X5VHGMMN4MA", @@ -980,13 +1004,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: true, Language: "en", CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, }, "local_account_2_status_2": { ID: "01F8MHC0H0A7XHTVH5F596ZKBM", @@ -1005,13 +1027,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: true, Language: "en", CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: false, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: true, + Boostable: true, + Replyable: false, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, }, "local_account_2_status_3": { ID: "01F8MHC8VWDRBQR0N1BATDDEM5", @@ -1030,13 +1050,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: true, Language: "en", CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: false, - Likeable: false, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: true, + Boostable: true, + Replyable: false, + Likeable: false, + ActivityStreamsType: ap.ObjectNote, }, "local_account_2_status_4": { ID: "01F8MHCP5P2NWYQ416SBA0XSEV", @@ -1055,12 +1073,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: true, Language: "en", CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: false, - Boostable: false, - Replyable: true, - Likeable: true, - }, + Federated: false, + Boostable: false, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, }, "local_account_2_status_5": { @@ -1083,13 +1100,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status { Sensitive: false, Language: "en", CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ", - VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - }, - ActivityStreamsType: ap.ObjectNote, + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + ActivityStreamsType: ap.ObjectNote, }, } } @@ -1149,6 +1164,18 @@ func NewTestMentions() map[string]*gtsmodel.Mention { TargetAccountURI: "http://localhost:8080/users/the_mighty_zork", TargetAccountURL: "http://localhost:8080/@the_mighty_zork", }, + "admin_account_mention_zork": { + ID: "01FF26A6BGEKCZFWNEHXB2ZZ6M", + StatusID: "01FF25D5Q0DH7CHD57CTRS6WK0", + CreatedAt: time.Now().Add(-46 * time.Hour), + UpdatedAt: time.Now().Add(-46 * time.Hour), + OriginAccountID: "01F8MH17FWEB39HZJ76B6VXSKF", + OriginAccountURI: "http://localhost:8080/users/admin", + TargetAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", + NameString: "@the_mighty_zork", + TargetAccountURI: "http://localhost:8080/users/the_mighty_zork", + TargetAccountURL: "http://localhost:8080/@the_mighty_zork", + }, } } @@ -1215,6 +1242,19 @@ func NewTestFollows() map[string]*gtsmodel.Follow { } } +func NewTestBlocks() map[string]*gtsmodel.Block { + return map[string]*gtsmodel.Block{ + "local_account_2_block_remote_account_1": { + ID: "01FEXXET6XXMF7G2V3ASZP3YQW", + CreatedAt: time.Now().Add(-1 * time.Hour), + UpdatedAt: time.Now().Add(-1 * time.Hour), + URI: "http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW", + AccountID: "01F8MH5NBDF2MV7CTC4Q5128HF", + TargetAccountID: "01F8MH5ZK5VRH73AKHQM6Y9VNX", + }, + } +} + // ActivityWithSignature wraps a pub.Activity along with its signature headers, for testing. type ActivityWithSignature struct { Activity pub.Activity @@ -1374,7 +1414,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin DateHeader: date, } - target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5") + target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0") sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) fossSatanDereferenceLocalAccount1Status1RepliesLast := ActivityWithSignature{ SignatureHeader: sig, diff --git a/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md b/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md new file mode 100644 index 000000000..1955f2878 --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md @@ -0,0 +1,73 @@ +## unreleased + +* Fix regression where `*time.Time` value would be set to empty and not be sent + to decode hooks properly [GH-232] + +## 1.4.0 + +* A new decode hook type `DecodeHookFuncValue` has been added that has + access to the full values. [GH-183] +* Squash is now supported with embedded fields that are struct pointers [GH-205] +* Empty strings will convert to 0 for all numeric types when weakly decoding [GH-206] + +## 1.3.3 + +* Decoding maps from maps creates a settable value for decode hooks [GH-203] + +## 1.3.2 + +* Decode into interface type with a struct value is supported [GH-187] + +## 1.3.1 + +* Squash should only squash embedded structs. [GH-194] + +## 1.3.0 + +* Added `",omitempty"` support. This will ignore zero values in the source + structure when encoding. [GH-145] + +## 1.2.3 + +* Fix duplicate entries in Keys list with pointer values. [GH-185] + +## 1.2.2 + +* Do not add unsettable (unexported) values to the unused metadata key + or "remain" value. [GH-150] + +## 1.2.1 + +* Go modules checksum mismatch fix + +## 1.2.0 + +* Added support to capture unused values in a field using the `",remain"` value + in the mapstructure tag. There is an example to showcase usage. +* Added `DecoderConfig` option to always squash embedded structs +* `json.Number` can decode into `uint` types +* Empty slices are preserved and not replaced with nil slices +* Fix panic that can occur in when decoding a map into a nil slice of structs +* Improved package documentation for godoc + +## 1.1.2 + +* Fix error when decode hook decodes interface implementation into interface + type. [GH-140] + +## 1.1.1 + +* Fix panic that can happen in `decodePtr` + +## 1.1.0 + +* Added `StringToIPHookFunc` to convert `string` to `net.IP` and `net.IPNet` [GH-133] +* Support struct to struct decoding [GH-137] +* If source map value is nil, then destination map value is nil (instead of empty) +* If source slice value is nil, then destination slice value is nil (instead of empty) +* If source pointer is nil, then destination pointer is set to nil (instead of + allocated zero value of type) + +## 1.0.0 + +* Initial tagged stable release. diff --git a/vendor/github.com/mitchellh/mapstructure/LICENSE b/vendor/github.com/mitchellh/mapstructure/LICENSE new file mode 100644 index 000000000..f9c841a51 --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/mitchellh/mapstructure/README.md b/vendor/github.com/mitchellh/mapstructure/README.md new file mode 100644 index 000000000..0018dc7d9 --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/README.md @@ -0,0 +1,46 @@ +# mapstructure [![Godoc](https://godoc.org/github.com/mitchellh/mapstructure?status.svg)](https://godoc.org/github.com/mitchellh/mapstructure) + +mapstructure is a Go library for decoding generic map values to structures +and vice versa, while providing helpful error handling. + +This library is most useful when decoding values from some data stream (JSON, +Gob, etc.) where you don't _quite_ know the structure of the underlying data +until you read a part of it. You can therefore read a `map[string]interface{}` +and use this library to decode it into the proper underlying native Go +structure. + +## Installation + +Standard `go get`: + +``` +$ go get github.com/mitchellh/mapstructure +``` + +## Usage & Example + +For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/mapstructure). + +The `Decode` function has examples associated with it there. + +## But Why?! + +Go offers fantastic standard libraries for decoding formats such as JSON. +The standard method is to have a struct pre-created, and populate that struct +from the bytes of the encoded format. This is great, but the problem is if +you have configuration or an encoding that changes slightly depending on +specific fields. For example, consider this JSON: + +```json +{ + "type": "person", + "name": "Mitchell" +} +``` + +Perhaps we can't populate a specific structure without first reading +the "type" field from the JSON. We could always do two passes over the +decoding of the JSON (reading the "type" first, and the rest later). +However, it is much simpler to just decode this into a `map[string]interface{}` +structure, read the "type" key, then use something like this library +to decode it into the proper structure. diff --git a/vendor/github.com/mitchellh/mapstructure/decode_hooks.go b/vendor/github.com/mitchellh/mapstructure/decode_hooks.go new file mode 100644 index 000000000..92e6f76ff --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/decode_hooks.go @@ -0,0 +1,256 @@ +package mapstructure + +import ( + "encoding" + "errors" + "fmt" + "net" + "reflect" + "strconv" + "strings" + "time" +) + +// typedDecodeHook takes a raw DecodeHookFunc (an interface{}) and turns +// it into the proper DecodeHookFunc type, such as DecodeHookFuncType. +func typedDecodeHook(h DecodeHookFunc) DecodeHookFunc { + // Create variables here so we can reference them with the reflect pkg + var f1 DecodeHookFuncType + var f2 DecodeHookFuncKind + var f3 DecodeHookFuncValue + + // Fill in the variables into this interface and the rest is done + // automatically using the reflect package. + potential := []interface{}{f1, f2, f3} + + v := reflect.ValueOf(h) + vt := v.Type() + for _, raw := range potential { + pt := reflect.ValueOf(raw).Type() + if vt.ConvertibleTo(pt) { + return v.Convert(pt).Interface() + } + } + + return nil +} + +// DecodeHookExec executes the given decode hook. This should be used +// since it'll naturally degrade to the older backwards compatible DecodeHookFunc +// that took reflect.Kind instead of reflect.Type. +func DecodeHookExec( + raw DecodeHookFunc, + from reflect.Value, to reflect.Value) (interface{}, error) { + + switch f := typedDecodeHook(raw).(type) { + case DecodeHookFuncType: + return f(from.Type(), to.Type(), from.Interface()) + case DecodeHookFuncKind: + return f(from.Kind(), to.Kind(), from.Interface()) + case DecodeHookFuncValue: + return f(from, to) + default: + return nil, errors.New("invalid decode hook signature") + } +} + +// ComposeDecodeHookFunc creates a single DecodeHookFunc that +// automatically composes multiple DecodeHookFuncs. +// +// The composed funcs are called in order, with the result of the +// previous transformation. +func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc { + return func(f reflect.Value, t reflect.Value) (interface{}, error) { + var err error + var data interface{} + newFrom := f + for _, f1 := range fs { + data, err = DecodeHookExec(f1, newFrom, t) + if err != nil { + return nil, err + } + newFrom = reflect.ValueOf(data) + } + + return data, nil + } +} + +// StringToSliceHookFunc returns a DecodeHookFunc that converts +// string to []string by splitting on the given sep. +func StringToSliceHookFunc(sep string) DecodeHookFunc { + return func( + f reflect.Kind, + t reflect.Kind, + data interface{}) (interface{}, error) { + if f != reflect.String || t != reflect.Slice { + return data, nil + } + + raw := data.(string) + if raw == "" { + return []string{}, nil + } + + return strings.Split(raw, sep), nil + } +} + +// StringToTimeDurationHookFunc returns a DecodeHookFunc that converts +// strings to time.Duration. +func StringToTimeDurationHookFunc() DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(time.Duration(5)) { + return data, nil + } + + // Convert it by parsing + return time.ParseDuration(data.(string)) + } +} + +// StringToIPHookFunc returns a DecodeHookFunc that converts +// strings to net.IP +func StringToIPHookFunc() DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(net.IP{}) { + return data, nil + } + + // Convert it by parsing + ip := net.ParseIP(data.(string)) + if ip == nil { + return net.IP{}, fmt.Errorf("failed parsing ip %v", data) + } + + return ip, nil + } +} + +// StringToIPNetHookFunc returns a DecodeHookFunc that converts +// strings to net.IPNet +func StringToIPNetHookFunc() DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(net.IPNet{}) { + return data, nil + } + + // Convert it by parsing + _, net, err := net.ParseCIDR(data.(string)) + return net, err + } +} + +// StringToTimeHookFunc returns a DecodeHookFunc that converts +// strings to time.Time. +func StringToTimeHookFunc(layout string) DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(time.Time{}) { + return data, nil + } + + // Convert it by parsing + return time.Parse(layout, data.(string)) + } +} + +// WeaklyTypedHook is a DecodeHookFunc which adds support for weak typing to +// the decoder. +// +// Note that this is significantly different from the WeaklyTypedInput option +// of the DecoderConfig. +func WeaklyTypedHook( + f reflect.Kind, + t reflect.Kind, + data interface{}) (interface{}, error) { + dataVal := reflect.ValueOf(data) + switch t { + case reflect.String: + switch f { + case reflect.Bool: + if dataVal.Bool() { + return "1", nil + } + return "0", nil + case reflect.Float32: + return strconv.FormatFloat(dataVal.Float(), 'f', -1, 64), nil + case reflect.Int: + return strconv.FormatInt(dataVal.Int(), 10), nil + case reflect.Slice: + dataType := dataVal.Type() + elemKind := dataType.Elem().Kind() + if elemKind == reflect.Uint8 { + return string(dataVal.Interface().([]uint8)), nil + } + case reflect.Uint: + return strconv.FormatUint(dataVal.Uint(), 10), nil + } + } + + return data, nil +} + +func RecursiveStructToMapHookFunc() DecodeHookFunc { + return func(f reflect.Value, t reflect.Value) (interface{}, error) { + if f.Kind() != reflect.Struct { + return f.Interface(), nil + } + + var i interface{} = struct{}{} + if t.Type() != reflect.TypeOf(&i).Elem() { + return f.Interface(), nil + } + + m := make(map[string]interface{}) + t.Set(reflect.ValueOf(m)) + + return f.Interface(), nil + } +} + +// TextUnmarshallerHookFunc returns a DecodeHookFunc that applies +// strings to the UnmarshalText function, when the target type +// implements the encoding.TextUnmarshaler interface +func TextUnmarshallerHookFunc() DecodeHookFuncType { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + result := reflect.New(t).Interface() + unmarshaller, ok := result.(encoding.TextUnmarshaler) + if !ok { + return data, nil + } + if err := unmarshaller.UnmarshalText([]byte(data.(string))); err != nil { + return nil, err + } + return result, nil + } +} diff --git a/vendor/github.com/mitchellh/mapstructure/error.go b/vendor/github.com/mitchellh/mapstructure/error.go new file mode 100644 index 000000000..47a99e5af --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/error.go @@ -0,0 +1,50 @@ +package mapstructure + +import ( + "errors" + "fmt" + "sort" + "strings" +) + +// Error implements the error interface and can represents multiple +// errors that occur in the course of a single decode. +type Error struct { + Errors []string +} + +func (e *Error) Error() string { + points := make([]string, len(e.Errors)) + for i, err := range e.Errors { + points[i] = fmt.Sprintf("* %s", err) + } + + sort.Strings(points) + return fmt.Sprintf( + "%d error(s) decoding:\n\n%s", + len(e.Errors), strings.Join(points, "\n")) +} + +// WrappedErrors implements the errwrap.Wrapper interface to make this +// return value more useful with the errwrap and go-multierror libraries. +func (e *Error) WrappedErrors() []error { + if e == nil { + return nil + } + + result := make([]error, len(e.Errors)) + for i, e := range e.Errors { + result[i] = errors.New(e) + } + + return result +} + +func appendErrors(errors []string, err error) []string { + switch e := err.(type) { + case *Error: + return append(errors, e.Errors...) + default: + return append(errors, e.Error()) + } +} diff --git a/vendor/github.com/mitchellh/mapstructure/go.mod b/vendor/github.com/mitchellh/mapstructure/go.mod new file mode 100644 index 000000000..a03ae9730 --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/go.mod @@ -0,0 +1,3 @@ +module github.com/mitchellh/mapstructure + +go 1.14 diff --git a/vendor/github.com/mitchellh/mapstructure/mapstructure.go b/vendor/github.com/mitchellh/mapstructure/mapstructure.go new file mode 100644 index 000000000..3643901f5 --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/mapstructure.go @@ -0,0 +1,1462 @@ +// Package mapstructure exposes functionality to convert one arbitrary +// Go type into another, typically to convert a map[string]interface{} +// into a native Go structure. +// +// The Go structure can be arbitrarily complex, containing slices, +// other structs, etc. and the decoder will properly decode nested +// maps and so on into the proper structures in the native Go struct. +// See the examples to see what the decoder is capable of. +// +// The simplest function to start with is Decode. +// +// Field Tags +// +// When decoding to a struct, mapstructure will use the field name by +// default to perform the mapping. For example, if a struct has a field +// "Username" then mapstructure will look for a key in the source value +// of "username" (case insensitive). +// +// type User struct { +// Username string +// } +// +// You can change the behavior of mapstructure by using struct tags. +// The default struct tag that mapstructure looks for is "mapstructure" +// but you can customize it using DecoderConfig. +// +// Renaming Fields +// +// To rename the key that mapstructure looks for, use the "mapstructure" +// tag and set a value directly. For example, to change the "username" example +// above to "user": +// +// type User struct { +// Username string `mapstructure:"user"` +// } +// +// Embedded Structs and Squashing +// +// Embedded structs are treated as if they're another field with that name. +// By default, the two structs below are equivalent when decoding with +// mapstructure: +// +// type Person struct { +// Name string +// } +// +// type Friend struct { +// Person +// } +// +// type Friend struct { +// Person Person +// } +// +// This would require an input that looks like below: +// +// map[string]interface{}{ +// "person": map[string]interface{}{"name": "alice"}, +// } +// +// If your "person" value is NOT nested, then you can append ",squash" to +// your tag value and mapstructure will treat it as if the embedded struct +// were part of the struct directly. Example: +// +// type Friend struct { +// Person `mapstructure:",squash"` +// } +// +// Now the following input would be accepted: +// +// map[string]interface{}{ +// "name": "alice", +// } +// +// When decoding from a struct to a map, the squash tag squashes the struct +// fields into a single map. Using the example structs from above: +// +// Friend{Person: Person{Name: "alice"}} +// +// Will be decoded into a map: +// +// map[string]interface{}{ +// "name": "alice", +// } +// +// DecoderConfig has a field that changes the behavior of mapstructure +// to always squash embedded structs. +// +// Remainder Values +// +// If there are any unmapped keys in the source value, mapstructure by +// default will silently ignore them. You can error by setting ErrorUnused +// in DecoderConfig. If you're using Metadata you can also maintain a slice +// of the unused keys. +// +// You can also use the ",remain" suffix on your tag to collect all unused +// values in a map. The field with this tag MUST be a map type and should +// probably be a "map[string]interface{}" or "map[interface{}]interface{}". +// See example below: +// +// type Friend struct { +// Name string +// Other map[string]interface{} `mapstructure:",remain"` +// } +// +// Given the input below, Other would be populated with the other +// values that weren't used (everything but "name"): +// +// map[string]interface{}{ +// "name": "bob", +// "address": "123 Maple St.", +// } +// +// Omit Empty Values +// +// When decoding from a struct to any other value, you may use the +// ",omitempty" suffix on your tag to omit that value if it equates to +// the zero value. The zero value of all types is specified in the Go +// specification. +// +// For example, the zero type of a numeric type is zero ("0"). If the struct +// field value is zero and a numeric type, the field is empty, and it won't +// be encoded into the destination type. +// +// type Source { +// Age int `mapstructure:",omitempty"` +// } +// +// Unexported fields +// +// Since unexported (private) struct fields cannot be set outside the package +// where they are defined, the decoder will simply skip them. +// +// For this output type definition: +// +// type Exported struct { +// private string // this unexported field will be skipped +// Public string +// } +// +// Using this map as input: +// +// map[string]interface{}{ +// "private": "I will be ignored", +// "Public": "I made it through!", +// } +// +// The following struct will be decoded: +// +// type Exported struct { +// private: "" // field is left with an empty string (zero value) +// Public: "I made it through!" +// } +// +// Other Configuration +// +// mapstructure is highly configurable. See the DecoderConfig struct +// for other features and options that are supported. +package mapstructure + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "sort" + "strconv" + "strings" +) + +// DecodeHookFunc is the callback function that can be used for +// data transformations. See "DecodeHook" in the DecoderConfig +// struct. +// +// The type must be one of DecodeHookFuncType, DecodeHookFuncKind, or +// DecodeHookFuncValue. +// Values are a superset of Types (Values can return types), and Types are a +// superset of Kinds (Types can return Kinds) and are generally a richer thing +// to use, but Kinds are simpler if you only need those. +// +// The reason DecodeHookFunc is multi-typed is for backwards compatibility: +// we started with Kinds and then realized Types were the better solution, +// but have a promise to not break backwards compat so we now support +// both. +type DecodeHookFunc interface{} + +// DecodeHookFuncType is a DecodeHookFunc which has complete information about +// the source and target types. +type DecodeHookFuncType func(reflect.Type, reflect.Type, interface{}) (interface{}, error) + +// DecodeHookFuncKind is a DecodeHookFunc which knows only the Kinds of the +// source and target types. +type DecodeHookFuncKind func(reflect.Kind, reflect.Kind, interface{}) (interface{}, error) + +// DecodeHookFuncRaw is a DecodeHookFunc which has complete access to both the source and target +// values. +type DecodeHookFuncValue func(from reflect.Value, to reflect.Value) (interface{}, error) + +// DecoderConfig is the configuration that is used to create a new decoder +// and allows customization of various aspects of decoding. +type DecoderConfig struct { + // DecodeHook, if set, will be called before any decoding and any + // type conversion (if WeaklyTypedInput is on). This lets you modify + // the values before they're set down onto the resulting struct. The + // DecodeHook is called for every map and value in the input. This means + // that if a struct has embedded fields with squash tags the decode hook + // is called only once with all of the input data, not once for each + // embedded struct. + // + // If an error is returned, the entire decode will fail with that error. + DecodeHook DecodeHookFunc + + // If ErrorUnused is true, then it is an error for there to exist + // keys in the original map that were unused in the decoding process + // (extra keys). + ErrorUnused bool + + // ZeroFields, if set to true, will zero fields before writing them. + // For example, a map will be emptied before decoded values are put in + // it. If this is false, a map will be merged. + ZeroFields bool + + // If WeaklyTypedInput is true, the decoder will make the following + // "weak" conversions: + // + // - bools to string (true = "1", false = "0") + // - numbers to string (base 10) + // - bools to int/uint (true = 1, false = 0) + // - strings to int/uint (base implied by prefix) + // - int to bool (true if value != 0) + // - string to bool (accepts: 1, t, T, TRUE, true, True, 0, f, F, + // FALSE, false, False. Anything else is an error) + // - empty array = empty map and vice versa + // - negative numbers to overflowed uint values (base 10) + // - slice of maps to a merged map + // - single values are converted to slices if required. Each + // element is weakly decoded. For example: "4" can become []int{4} + // if the target type is an int slice. + // + WeaklyTypedInput bool + + // Squash will squash embedded structs. A squash tag may also be + // added to an individual struct field using a tag. For example: + // + // type Parent struct { + // Child `mapstructure:",squash"` + // } + Squash bool + + // Metadata is the struct that will contain extra metadata about + // the decoding. If this is nil, then no metadata will be tracked. + Metadata *Metadata + + // Result is a pointer to the struct that will contain the decoded + // value. + Result interface{} + + // The tag name that mapstructure reads for field names. This + // defaults to "mapstructure" + TagName string +} + +// A Decoder takes a raw interface value and turns it into structured +// data, keeping track of rich error information along the way in case +// anything goes wrong. Unlike the basic top-level Decode method, you can +// more finely control how the Decoder behaves using the DecoderConfig +// structure. The top-level Decode method is just a convenience that sets +// up the most basic Decoder. +type Decoder struct { + config *DecoderConfig +} + +// Metadata contains information about decoding a structure that +// is tedious or difficult to get otherwise. +type Metadata struct { + // Keys are the keys of the structure which were successfully decoded + Keys []string + + // Unused is a slice of keys that were found in the raw value but + // weren't decoded since there was no matching field in the result interface + Unused []string +} + +// Decode takes an input structure and uses reflection to translate it to +// the output structure. output must be a pointer to a map or struct. +func Decode(input interface{}, output interface{}) error { + config := &DecoderConfig{ + Metadata: nil, + Result: output, + } + + decoder, err := NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +// WeakDecode is the same as Decode but is shorthand to enable +// WeaklyTypedInput. See DecoderConfig for more info. +func WeakDecode(input, output interface{}) error { + config := &DecoderConfig{ + Metadata: nil, + Result: output, + WeaklyTypedInput: true, + } + + decoder, err := NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +// DecodeMetadata is the same as Decode, but is shorthand to +// enable metadata collection. See DecoderConfig for more info. +func DecodeMetadata(input interface{}, output interface{}, metadata *Metadata) error { + config := &DecoderConfig{ + Metadata: metadata, + Result: output, + } + + decoder, err := NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +// WeakDecodeMetadata is the same as Decode, but is shorthand to +// enable both WeaklyTypedInput and metadata collection. See +// DecoderConfig for more info. +func WeakDecodeMetadata(input interface{}, output interface{}, metadata *Metadata) error { + config := &DecoderConfig{ + Metadata: metadata, + Result: output, + WeaklyTypedInput: true, + } + + decoder, err := NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +// NewDecoder returns a new decoder for the given configuration. Once +// a decoder has been returned, the same configuration must not be used +// again. +func NewDecoder(config *DecoderConfig) (*Decoder, error) { + val := reflect.ValueOf(config.Result) + if val.Kind() != reflect.Ptr { + return nil, errors.New("result must be a pointer") + } + + val = val.Elem() + if !val.CanAddr() { + return nil, errors.New("result must be addressable (a pointer)") + } + + if config.Metadata != nil { + if config.Metadata.Keys == nil { + config.Metadata.Keys = make([]string, 0) + } + + if config.Metadata.Unused == nil { + config.Metadata.Unused = make([]string, 0) + } + } + + if config.TagName == "" { + config.TagName = "mapstructure" + } + + result := &Decoder{ + config: config, + } + + return result, nil +} + +// Decode decodes the given raw interface to the target pointer specified +// by the configuration. +func (d *Decoder) Decode(input interface{}) error { + return d.decode("", input, reflect.ValueOf(d.config.Result).Elem()) +} + +// Decodes an unknown data type into a specific reflection value. +func (d *Decoder) decode(name string, input interface{}, outVal reflect.Value) error { + var inputVal reflect.Value + if input != nil { + inputVal = reflect.ValueOf(input) + + // We need to check here if input is a typed nil. Typed nils won't + // match the "input == nil" below so we check that here. + if inputVal.Kind() == reflect.Ptr && inputVal.IsNil() { + input = nil + } + } + + if input == nil { + // If the data is nil, then we don't set anything, unless ZeroFields is set + // to true. + if d.config.ZeroFields { + outVal.Set(reflect.Zero(outVal.Type())) + + if d.config.Metadata != nil && name != "" { + d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) + } + } + return nil + } + + if !inputVal.IsValid() { + // If the input value is invalid, then we just set the value + // to be the zero value. + outVal.Set(reflect.Zero(outVal.Type())) + if d.config.Metadata != nil && name != "" { + d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) + } + return nil + } + + if d.config.DecodeHook != nil { + // We have a DecodeHook, so let's pre-process the input. + var err error + input, err = DecodeHookExec(d.config.DecodeHook, inputVal, outVal) + if err != nil { + return fmt.Errorf("error decoding '%s': %s", name, err) + } + } + + var err error + outputKind := getKind(outVal) + addMetaKey := true + switch outputKind { + case reflect.Bool: + err = d.decodeBool(name, input, outVal) + case reflect.Interface: + err = d.decodeBasic(name, input, outVal) + case reflect.String: + err = d.decodeString(name, input, outVal) + case reflect.Int: + err = d.decodeInt(name, input, outVal) + case reflect.Uint: + err = d.decodeUint(name, input, outVal) + case reflect.Float32: + err = d.decodeFloat(name, input, outVal) + case reflect.Struct: + err = d.decodeStruct(name, input, outVal) + case reflect.Map: + err = d.decodeMap(name, input, outVal) + case reflect.Ptr: + addMetaKey, err = d.decodePtr(name, input, outVal) + case reflect.Slice: + err = d.decodeSlice(name, input, outVal) + case reflect.Array: + err = d.decodeArray(name, input, outVal) + case reflect.Func: + err = d.decodeFunc(name, input, outVal) + default: + // If we reached this point then we weren't able to decode it + return fmt.Errorf("%s: unsupported type: %s", name, outputKind) + } + + // If we reached here, then we successfully decoded SOMETHING, so + // mark the key as used if we're tracking metainput. + if addMetaKey && d.config.Metadata != nil && name != "" { + d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) + } + + return err +} + +// This decodes a basic type (bool, int, string, etc.) and sets the +// value to "data" of that type. +func (d *Decoder) decodeBasic(name string, data interface{}, val reflect.Value) error { + if val.IsValid() && val.Elem().IsValid() { + elem := val.Elem() + + // If we can't address this element, then its not writable. Instead, + // we make a copy of the value (which is a pointer and therefore + // writable), decode into that, and replace the whole value. + copied := false + if !elem.CanAddr() { + copied = true + + // Make *T + copy := reflect.New(elem.Type()) + + // *T = elem + copy.Elem().Set(elem) + + // Set elem so we decode into it + elem = copy + } + + // Decode. If we have an error then return. We also return right + // away if we're not a copy because that means we decoded directly. + if err := d.decode(name, data, elem); err != nil || !copied { + return err + } + + // If we're a copy, we need to set te final result + val.Set(elem.Elem()) + return nil + } + + dataVal := reflect.ValueOf(data) + + // If the input data is a pointer, and the assigned type is the dereference + // of that exact pointer, then indirect it so that we can assign it. + // Example: *string to string + if dataVal.Kind() == reflect.Ptr && dataVal.Type().Elem() == val.Type() { + dataVal = reflect.Indirect(dataVal) + } + + if !dataVal.IsValid() { + dataVal = reflect.Zero(val.Type()) + } + + dataValType := dataVal.Type() + if !dataValType.AssignableTo(val.Type()) { + return fmt.Errorf( + "'%s' expected type '%s', got '%s'", + name, val.Type(), dataValType) + } + + val.Set(dataVal) + return nil +} + +func (d *Decoder) decodeString(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataKind := getKind(dataVal) + + converted := true + switch { + case dataKind == reflect.String: + val.SetString(dataVal.String()) + case dataKind == reflect.Bool && d.config.WeaklyTypedInput: + if dataVal.Bool() { + val.SetString("1") + } else { + val.SetString("0") + } + case dataKind == reflect.Int && d.config.WeaklyTypedInput: + val.SetString(strconv.FormatInt(dataVal.Int(), 10)) + case dataKind == reflect.Uint && d.config.WeaklyTypedInput: + val.SetString(strconv.FormatUint(dataVal.Uint(), 10)) + case dataKind == reflect.Float32 && d.config.WeaklyTypedInput: + val.SetString(strconv.FormatFloat(dataVal.Float(), 'f', -1, 64)) + case dataKind == reflect.Slice && d.config.WeaklyTypedInput, + dataKind == reflect.Array && d.config.WeaklyTypedInput: + dataType := dataVal.Type() + elemKind := dataType.Elem().Kind() + switch elemKind { + case reflect.Uint8: + var uints []uint8 + if dataKind == reflect.Array { + uints = make([]uint8, dataVal.Len(), dataVal.Len()) + for i := range uints { + uints[i] = dataVal.Index(i).Interface().(uint8) + } + } else { + uints = dataVal.Interface().([]uint8) + } + val.SetString(string(uints)) + default: + converted = false + } + default: + converted = false + } + + if !converted { + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s', value: '%v'", + name, val.Type(), dataVal.Type(), data) + } + + return nil +} + +func (d *Decoder) decodeInt(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataKind := getKind(dataVal) + dataType := dataVal.Type() + + switch { + case dataKind == reflect.Int: + val.SetInt(dataVal.Int()) + case dataKind == reflect.Uint: + val.SetInt(int64(dataVal.Uint())) + case dataKind == reflect.Float32: + val.SetInt(int64(dataVal.Float())) + case dataKind == reflect.Bool && d.config.WeaklyTypedInput: + if dataVal.Bool() { + val.SetInt(1) + } else { + val.SetInt(0) + } + case dataKind == reflect.String && d.config.WeaklyTypedInput: + str := dataVal.String() + if str == "" { + str = "0" + } + + i, err := strconv.ParseInt(str, 0, val.Type().Bits()) + if err == nil { + val.SetInt(i) + } else { + return fmt.Errorf("cannot parse '%s' as int: %s", name, err) + } + case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": + jn := data.(json.Number) + i, err := jn.Int64() + if err != nil { + return fmt.Errorf( + "error decoding json.Number into %s: %s", name, err) + } + val.SetInt(i) + default: + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s', value: '%v'", + name, val.Type(), dataVal.Type(), data) + } + + return nil +} + +func (d *Decoder) decodeUint(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataKind := getKind(dataVal) + dataType := dataVal.Type() + + switch { + case dataKind == reflect.Int: + i := dataVal.Int() + if i < 0 && !d.config.WeaklyTypedInput { + return fmt.Errorf("cannot parse '%s', %d overflows uint", + name, i) + } + val.SetUint(uint64(i)) + case dataKind == reflect.Uint: + val.SetUint(dataVal.Uint()) + case dataKind == reflect.Float32: + f := dataVal.Float() + if f < 0 && !d.config.WeaklyTypedInput { + return fmt.Errorf("cannot parse '%s', %f overflows uint", + name, f) + } + val.SetUint(uint64(f)) + case dataKind == reflect.Bool && d.config.WeaklyTypedInput: + if dataVal.Bool() { + val.SetUint(1) + } else { + val.SetUint(0) + } + case dataKind == reflect.String && d.config.WeaklyTypedInput: + str := dataVal.String() + if str == "" { + str = "0" + } + + i, err := strconv.ParseUint(str, 0, val.Type().Bits()) + if err == nil { + val.SetUint(i) + } else { + return fmt.Errorf("cannot parse '%s' as uint: %s", name, err) + } + case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": + jn := data.(json.Number) + i, err := jn.Int64() + if err != nil { + return fmt.Errorf( + "error decoding json.Number into %s: %s", name, err) + } + if i < 0 && !d.config.WeaklyTypedInput { + return fmt.Errorf("cannot parse '%s', %d overflows uint", + name, i) + } + val.SetUint(uint64(i)) + default: + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s', value: '%v'", + name, val.Type(), dataVal.Type(), data) + } + + return nil +} + +func (d *Decoder) decodeBool(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataKind := getKind(dataVal) + + switch { + case dataKind == reflect.Bool: + val.SetBool(dataVal.Bool()) + case dataKind == reflect.Int && d.config.WeaklyTypedInput: + val.SetBool(dataVal.Int() != 0) + case dataKind == reflect.Uint && d.config.WeaklyTypedInput: + val.SetBool(dataVal.Uint() != 0) + case dataKind == reflect.Float32 && d.config.WeaklyTypedInput: + val.SetBool(dataVal.Float() != 0) + case dataKind == reflect.String && d.config.WeaklyTypedInput: + b, err := strconv.ParseBool(dataVal.String()) + if err == nil { + val.SetBool(b) + } else if dataVal.String() == "" { + val.SetBool(false) + } else { + return fmt.Errorf("cannot parse '%s' as bool: %s", name, err) + } + default: + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s', value: '%v'", + name, val.Type(), dataVal.Type(), data) + } + + return nil +} + +func (d *Decoder) decodeFloat(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataKind := getKind(dataVal) + dataType := dataVal.Type() + + switch { + case dataKind == reflect.Int: + val.SetFloat(float64(dataVal.Int())) + case dataKind == reflect.Uint: + val.SetFloat(float64(dataVal.Uint())) + case dataKind == reflect.Float32: + val.SetFloat(dataVal.Float()) + case dataKind == reflect.Bool && d.config.WeaklyTypedInput: + if dataVal.Bool() { + val.SetFloat(1) + } else { + val.SetFloat(0) + } + case dataKind == reflect.String && d.config.WeaklyTypedInput: + str := dataVal.String() + if str == "" { + str = "0" + } + + f, err := strconv.ParseFloat(str, val.Type().Bits()) + if err == nil { + val.SetFloat(f) + } else { + return fmt.Errorf("cannot parse '%s' as float: %s", name, err) + } + case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": + jn := data.(json.Number) + i, err := jn.Float64() + if err != nil { + return fmt.Errorf( + "error decoding json.Number into %s: %s", name, err) + } + val.SetFloat(i) + default: + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s', value: '%v'", + name, val.Type(), dataVal.Type(), data) + } + + return nil +} + +func (d *Decoder) decodeMap(name string, data interface{}, val reflect.Value) error { + valType := val.Type() + valKeyType := valType.Key() + valElemType := valType.Elem() + + // By default we overwrite keys in the current map + valMap := val + + // If the map is nil or we're purposely zeroing fields, make a new map + if valMap.IsNil() || d.config.ZeroFields { + // Make a new map to hold our result + mapType := reflect.MapOf(valKeyType, valElemType) + valMap = reflect.MakeMap(mapType) + } + + // Check input type and based on the input type jump to the proper func + dataVal := reflect.Indirect(reflect.ValueOf(data)) + switch dataVal.Kind() { + case reflect.Map: + return d.decodeMapFromMap(name, dataVal, val, valMap) + + case reflect.Struct: + return d.decodeMapFromStruct(name, dataVal, val, valMap) + + case reflect.Array, reflect.Slice: + if d.config.WeaklyTypedInput { + return d.decodeMapFromSlice(name, dataVal, val, valMap) + } + + fallthrough + + default: + return fmt.Errorf("'%s' expected a map, got '%s'", name, dataVal.Kind()) + } +} + +func (d *Decoder) decodeMapFromSlice(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { + // Special case for BC reasons (covered by tests) + if dataVal.Len() == 0 { + val.Set(valMap) + return nil + } + + for i := 0; i < dataVal.Len(); i++ { + err := d.decode( + name+"["+strconv.Itoa(i)+"]", + dataVal.Index(i).Interface(), val) + if err != nil { + return err + } + } + + return nil +} + +func (d *Decoder) decodeMapFromMap(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { + valType := val.Type() + valKeyType := valType.Key() + valElemType := valType.Elem() + + // Accumulate errors + errors := make([]string, 0) + + // If the input data is empty, then we just match what the input data is. + if dataVal.Len() == 0 { + if dataVal.IsNil() { + if !val.IsNil() { + val.Set(dataVal) + } + } else { + // Set to empty allocated value + val.Set(valMap) + } + + return nil + } + + for _, k := range dataVal.MapKeys() { + fieldName := name + "[" + k.String() + "]" + + // First decode the key into the proper type + currentKey := reflect.Indirect(reflect.New(valKeyType)) + if err := d.decode(fieldName, k.Interface(), currentKey); err != nil { + errors = appendErrors(errors, err) + continue + } + + // Next decode the data into the proper type + v := dataVal.MapIndex(k).Interface() + currentVal := reflect.Indirect(reflect.New(valElemType)) + if err := d.decode(fieldName, v, currentVal); err != nil { + errors = appendErrors(errors, err) + continue + } + + valMap.SetMapIndex(currentKey, currentVal) + } + + // Set the built up map to the value + val.Set(valMap) + + // If we had errors, return those + if len(errors) > 0 { + return &Error{errors} + } + + return nil +} + +func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { + typ := dataVal.Type() + for i := 0; i < typ.NumField(); i++ { + // Get the StructField first since this is a cheap operation. If the + // field is unexported, then ignore it. + f := typ.Field(i) + if f.PkgPath != "" { + continue + } + + // Next get the actual value of this field and verify it is assignable + // to the map value. + v := dataVal.Field(i) + if !v.Type().AssignableTo(valMap.Type().Elem()) { + return fmt.Errorf("cannot assign type '%s' to map value field of type '%s'", v.Type(), valMap.Type().Elem()) + } + + tagValue := f.Tag.Get(d.config.TagName) + keyName := f.Name + + // If Squash is set in the config, we squash the field down. + squash := d.config.Squash && v.Kind() == reflect.Struct && f.Anonymous + + // Determine the name of the key in the map + if index := strings.Index(tagValue, ","); index != -1 { + if tagValue[:index] == "-" { + continue + } + // If "omitempty" is specified in the tag, it ignores empty values. + if strings.Index(tagValue[index+1:], "omitempty") != -1 && isEmptyValue(v) { + continue + } + + // If "squash" is specified in the tag, we squash the field down. + squash = !squash && strings.Index(tagValue[index+1:], "squash") != -1 + if squash { + // When squashing, the embedded type can be a pointer to a struct. + if v.Kind() == reflect.Ptr && v.Elem().Kind() == reflect.Struct { + v = v.Elem() + } + + // The final type must be a struct + if v.Kind() != reflect.Struct { + return fmt.Errorf("cannot squash non-struct type '%s'", v.Type()) + } + } + keyName = tagValue[:index] + } else if len(tagValue) > 0 { + if tagValue == "-" { + continue + } + keyName = tagValue + } + + switch v.Kind() { + // this is an embedded struct, so handle it differently + case reflect.Struct: + x := reflect.New(v.Type()) + x.Elem().Set(v) + + vType := valMap.Type() + vKeyType := vType.Key() + vElemType := vType.Elem() + mType := reflect.MapOf(vKeyType, vElemType) + vMap := reflect.MakeMap(mType) + + // Creating a pointer to a map so that other methods can completely + // overwrite the map if need be (looking at you decodeMapFromMap). The + // indirection allows the underlying map to be settable (CanSet() == true) + // where as reflect.MakeMap returns an unsettable map. + addrVal := reflect.New(vMap.Type()) + reflect.Indirect(addrVal).Set(vMap) + + err := d.decode(keyName, x.Interface(), reflect.Indirect(addrVal)) + if err != nil { + return err + } + + // the underlying map may have been completely overwritten so pull + // it indirectly out of the enclosing value. + vMap = reflect.Indirect(addrVal) + + if squash { + for _, k := range vMap.MapKeys() { + valMap.SetMapIndex(k, vMap.MapIndex(k)) + } + } else { + valMap.SetMapIndex(reflect.ValueOf(keyName), vMap) + } + + default: + valMap.SetMapIndex(reflect.ValueOf(keyName), v) + } + } + + if val.CanAddr() { + val.Set(valMap) + } + + return nil +} + +func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) (bool, error) { + // If the input data is nil, then we want to just set the output + // pointer to be nil as well. + isNil := data == nil + if !isNil { + switch v := reflect.Indirect(reflect.ValueOf(data)); v.Kind() { + case reflect.Chan, + reflect.Func, + reflect.Interface, + reflect.Map, + reflect.Ptr, + reflect.Slice: + isNil = v.IsNil() + } + } + if isNil { + if !val.IsNil() && val.CanSet() { + nilValue := reflect.New(val.Type()).Elem() + val.Set(nilValue) + } + + return true, nil + } + + // Create an element of the concrete (non pointer) type and decode + // into that. Then set the value of the pointer to this type. + valType := val.Type() + valElemType := valType.Elem() + if val.CanSet() { + realVal := val + if realVal.IsNil() || d.config.ZeroFields { + realVal = reflect.New(valElemType) + } + + if err := d.decode(name, data, reflect.Indirect(realVal)); err != nil { + return false, err + } + + val.Set(realVal) + } else { + if err := d.decode(name, data, reflect.Indirect(val)); err != nil { + return false, err + } + } + return false, nil +} + +func (d *Decoder) decodeFunc(name string, data interface{}, val reflect.Value) error { + // Create an element of the concrete (non pointer) type and decode + // into that. Then set the value of the pointer to this type. + dataVal := reflect.Indirect(reflect.ValueOf(data)) + if val.Type() != dataVal.Type() { + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s', value: '%v'", + name, val.Type(), dataVal.Type(), data) + } + val.Set(dataVal) + return nil +} + +func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataValKind := dataVal.Kind() + valType := val.Type() + valElemType := valType.Elem() + sliceType := reflect.SliceOf(valElemType) + + // If we have a non array/slice type then we first attempt to convert. + if dataValKind != reflect.Array && dataValKind != reflect.Slice { + if d.config.WeaklyTypedInput { + switch { + // Slice and array we use the normal logic + case dataValKind == reflect.Slice, dataValKind == reflect.Array: + break + + // Empty maps turn into empty slices + case dataValKind == reflect.Map: + if dataVal.Len() == 0 { + val.Set(reflect.MakeSlice(sliceType, 0, 0)) + return nil + } + // Create slice of maps of other sizes + return d.decodeSlice(name, []interface{}{data}, val) + + case dataValKind == reflect.String && valElemType.Kind() == reflect.Uint8: + return d.decodeSlice(name, []byte(dataVal.String()), val) + + // All other types we try to convert to the slice type + // and "lift" it into it. i.e. a string becomes a string slice. + default: + // Just re-try this function with data as a slice. + return d.decodeSlice(name, []interface{}{data}, val) + } + } + + return fmt.Errorf( + "'%s': source data must be an array or slice, got %s", name, dataValKind) + } + + // If the input value is nil, then don't allocate since empty != nil + if dataVal.IsNil() { + return nil + } + + valSlice := val + if valSlice.IsNil() || d.config.ZeroFields { + // Make a new slice to hold our result, same size as the original data. + valSlice = reflect.MakeSlice(sliceType, dataVal.Len(), dataVal.Len()) + } + + // Accumulate any errors + errors := make([]string, 0) + + for i := 0; i < dataVal.Len(); i++ { + currentData := dataVal.Index(i).Interface() + for valSlice.Len() <= i { + valSlice = reflect.Append(valSlice, reflect.Zero(valElemType)) + } + currentField := valSlice.Index(i) + + fieldName := name + "[" + strconv.Itoa(i) + "]" + if err := d.decode(fieldName, currentData, currentField); err != nil { + errors = appendErrors(errors, err) + } + } + + // Finally, set the value to the slice we built up + val.Set(valSlice) + + // If there were errors, we return those + if len(errors) > 0 { + return &Error{errors} + } + + return nil +} + +func (d *Decoder) decodeArray(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataValKind := dataVal.Kind() + valType := val.Type() + valElemType := valType.Elem() + arrayType := reflect.ArrayOf(valType.Len(), valElemType) + + valArray := val + + if valArray.Interface() == reflect.Zero(valArray.Type()).Interface() || d.config.ZeroFields { + // Check input type + if dataValKind != reflect.Array && dataValKind != reflect.Slice { + if d.config.WeaklyTypedInput { + switch { + // Empty maps turn into empty arrays + case dataValKind == reflect.Map: + if dataVal.Len() == 0 { + val.Set(reflect.Zero(arrayType)) + return nil + } + + // All other types we try to convert to the array type + // and "lift" it into it. i.e. a string becomes a string array. + default: + // Just re-try this function with data as a slice. + return d.decodeArray(name, []interface{}{data}, val) + } + } + + return fmt.Errorf( + "'%s': source data must be an array or slice, got %s", name, dataValKind) + + } + if dataVal.Len() > arrayType.Len() { + return fmt.Errorf( + "'%s': expected source data to have length less or equal to %d, got %d", name, arrayType.Len(), dataVal.Len()) + + } + + // Make a new array to hold our result, same size as the original data. + valArray = reflect.New(arrayType).Elem() + } + + // Accumulate any errors + errors := make([]string, 0) + + for i := 0; i < dataVal.Len(); i++ { + currentData := dataVal.Index(i).Interface() + currentField := valArray.Index(i) + + fieldName := name + "[" + strconv.Itoa(i) + "]" + if err := d.decode(fieldName, currentData, currentField); err != nil { + errors = appendErrors(errors, err) + } + } + + // Finally, set the value to the array we built up + val.Set(valArray) + + // If there were errors, we return those + if len(errors) > 0 { + return &Error{errors} + } + + return nil +} + +func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + + // If the type of the value to write to and the data match directly, + // then we just set it directly instead of recursing into the structure. + if dataVal.Type() == val.Type() { + val.Set(dataVal) + return nil + } + + dataValKind := dataVal.Kind() + switch dataValKind { + case reflect.Map: + return d.decodeStructFromMap(name, dataVal, val) + + case reflect.Struct: + // Not the most efficient way to do this but we can optimize later if + // we want to. To convert from struct to struct we go to map first + // as an intermediary. + + // Make a new map to hold our result + mapType := reflect.TypeOf((map[string]interface{})(nil)) + mval := reflect.MakeMap(mapType) + + // Creating a pointer to a map so that other methods can completely + // overwrite the map if need be (looking at you decodeMapFromMap). The + // indirection allows the underlying map to be settable (CanSet() == true) + // where as reflect.MakeMap returns an unsettable map. + addrVal := reflect.New(mval.Type()) + + reflect.Indirect(addrVal).Set(mval) + if err := d.decodeMapFromStruct(name, dataVal, reflect.Indirect(addrVal), mval); err != nil { + return err + } + + result := d.decodeStructFromMap(name, reflect.Indirect(addrVal), val) + return result + + default: + return fmt.Errorf("'%s' expected a map, got '%s'", name, dataVal.Kind()) + } +} + +func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) error { + dataValType := dataVal.Type() + if kind := dataValType.Key().Kind(); kind != reflect.String && kind != reflect.Interface { + return fmt.Errorf( + "'%s' needs a map with string keys, has '%s' keys", + name, dataValType.Key().Kind()) + } + + dataValKeys := make(map[reflect.Value]struct{}) + dataValKeysUnused := make(map[interface{}]struct{}) + for _, dataValKey := range dataVal.MapKeys() { + dataValKeys[dataValKey] = struct{}{} + dataValKeysUnused[dataValKey.Interface()] = struct{}{} + } + + errors := make([]string, 0) + + // This slice will keep track of all the structs we'll be decoding. + // There can be more than one struct if there are embedded structs + // that are squashed. + structs := make([]reflect.Value, 1, 5) + structs[0] = val + + // Compile the list of all the fields that we're going to be decoding + // from all the structs. + type field struct { + field reflect.StructField + val reflect.Value + } + + // remainField is set to a valid field set with the "remain" tag if + // we are keeping track of remaining values. + var remainField *field + + fields := []field{} + for len(structs) > 0 { + structVal := structs[0] + structs = structs[1:] + + structType := structVal.Type() + + for i := 0; i < structType.NumField(); i++ { + fieldType := structType.Field(i) + fieldVal := structVal.Field(i) + if fieldVal.Kind() == reflect.Ptr && fieldVal.Elem().Kind() == reflect.Struct { + // Handle embedded struct pointers as embedded structs. + fieldVal = fieldVal.Elem() + } + + // If "squash" is specified in the tag, we squash the field down. + squash := d.config.Squash && fieldVal.Kind() == reflect.Struct && fieldType.Anonymous + remain := false + + // We always parse the tags cause we're looking for other tags too + tagParts := strings.Split(fieldType.Tag.Get(d.config.TagName), ",") + for _, tag := range tagParts[1:] { + if tag == "squash" { + squash = true + break + } + + if tag == "remain" { + remain = true + break + } + } + + if squash { + if fieldVal.Kind() != reflect.Struct { + errors = appendErrors(errors, + fmt.Errorf("%s: unsupported type for squash: %s", fieldType.Name, fieldVal.Kind())) + } else { + structs = append(structs, fieldVal) + } + continue + } + + // Build our field + if remain { + remainField = &field{fieldType, fieldVal} + } else { + // Normal struct field, store it away + fields = append(fields, field{fieldType, fieldVal}) + } + } + } + + // for fieldType, field := range fields { + for _, f := range fields { + field, fieldValue := f.field, f.val + fieldName := field.Name + + tagValue := field.Tag.Get(d.config.TagName) + tagValue = strings.SplitN(tagValue, ",", 2)[0] + if tagValue != "" { + fieldName = tagValue + } + + rawMapKey := reflect.ValueOf(fieldName) + rawMapVal := dataVal.MapIndex(rawMapKey) + if !rawMapVal.IsValid() { + // Do a slower search by iterating over each key and + // doing case-insensitive search. + for dataValKey := range dataValKeys { + mK, ok := dataValKey.Interface().(string) + if !ok { + // Not a string key + continue + } + + if strings.EqualFold(mK, fieldName) { + rawMapKey = dataValKey + rawMapVal = dataVal.MapIndex(dataValKey) + break + } + } + + if !rawMapVal.IsValid() { + // There was no matching key in the map for the value in + // the struct. Just ignore. + continue + } + } + + if !fieldValue.IsValid() { + // This should never happen + panic("field is not valid") + } + + // If we can't set the field, then it is unexported or something, + // and we just continue onwards. + if !fieldValue.CanSet() { + continue + } + + // Delete the key we're using from the unused map so we stop tracking + delete(dataValKeysUnused, rawMapKey.Interface()) + + // If the name is empty string, then we're at the root, and we + // don't dot-join the fields. + if name != "" { + fieldName = name + "." + fieldName + } + + if err := d.decode(fieldName, rawMapVal.Interface(), fieldValue); err != nil { + errors = appendErrors(errors, err) + } + } + + // If we have a "remain"-tagged field and we have unused keys then + // we put the unused keys directly into the remain field. + if remainField != nil && len(dataValKeysUnused) > 0 { + // Build a map of only the unused values + remain := map[interface{}]interface{}{} + for key := range dataValKeysUnused { + remain[key] = dataVal.MapIndex(reflect.ValueOf(key)).Interface() + } + + // Decode it as-if we were just decoding this map onto our map. + if err := d.decodeMap(name, remain, remainField.val); err != nil { + errors = appendErrors(errors, err) + } + + // Set the map to nil so we have none so that the next check will + // not error (ErrorUnused) + dataValKeysUnused = nil + } + + if d.config.ErrorUnused && len(dataValKeysUnused) > 0 { + keys := make([]string, 0, len(dataValKeysUnused)) + for rawKey := range dataValKeysUnused { + keys = append(keys, rawKey.(string)) + } + sort.Strings(keys) + + err := fmt.Errorf("'%s' has invalid keys: %s", name, strings.Join(keys, ", ")) + errors = appendErrors(errors, err) + } + + if len(errors) > 0 { + return &Error{errors} + } + + // Add the unused keys to the list of unused keys if we're tracking metadata + if d.config.Metadata != nil { + for rawKey := range dataValKeysUnused { + key := rawKey.(string) + if name != "" { + key = name + "." + key + } + + d.config.Metadata.Unused = append(d.config.Metadata.Unused, key) + } + } + + return nil +} + +func isEmptyValue(v reflect.Value) bool { + switch getKind(v) { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + return false +} + +func getKind(val reflect.Value) reflect.Kind { + kind := val.Kind() + + switch { + case kind >= reflect.Int && kind <= reflect.Int64: + return reflect.Int + case kind >= reflect.Uint && kind <= reflect.Uint64: + return reflect.Uint + case kind >= reflect.Float32 && kind <= reflect.Float64: + return reflect.Float32 + default: + return kind + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ce1b8eac3..a5a9b5a1d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -338,6 +338,9 @@ github.com/mattn/go-isatty ## explicit github.com/microcosm-cc/bluemonday github.com/microcosm-cc/bluemonday/css +# github.com/mitchellh/mapstructure v1.4.1 +## explicit +github.com/mitchellh/mapstructure # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd ## explicit github.com/modern-go/concurrent