Import export (#194)

* start with export/import code

* messing about with decoding/encoding

* some more fiddling

* stuff is WORKING

* working pretty alright!

* go fmt

* fix up tests, add docs

* start backup/restore doc

* tweaks

* credits

* update advancedVisibility settings

* update bun library -> v1.0.4

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update oauth library -> v4.3.1-SSB

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* handle oauth token scope, fix user.SigninCount + token.UserID

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update oauth library --> v4.3.2-SSB

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update sqlite library -> v1.13.0

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* review changes

* start with export/import code

* messing about with decoding/encoding

* some more fiddling

* stuff is WORKING

* working pretty alright!

* go fmt

* fix up tests, add docs

* start backup/restore doc

* tweaks

* credits

* update advancedVisibility settings

* review changes

Co-authored-by: kim (grufwub) <grufwub@gmail.com>
Co-authored-by: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>
This commit is contained in:
tobi 2021-09-09 16:15:25 +02:00 committed by GitHub
parent a027da0ac9
commit 555ea8edfb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
61 changed files with 4031 additions and 250 deletions

View file

@ -134,10 +134,11 @@ The following libraries and frameworks are used by GoToSocial, with gratitude
* [gorilla/websocket](https://github.com/gorilla/websocket); Websocket connectivity. [BSD-2-Clause License](https://spdx.org/licenses/BSD-2-Clause.html).
* [h2non/filetype](https://github.com/h2non/filetype); filetype checking. [MIT License](https://spdx.org/licenses/MIT.html).
* [jackc/pgx](https://github.com/jackc/pgx); Postgres driver. [MIT License](https://spdx.org/licenses/MIT.html).
* [microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday); HTML user-input sanitization. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
* [mitchellh/mapstructure](https://github.com/mitchellh/mapstructure); Go interface => struct parsing. [MIT License](https://spdx.org/licenses/MIT.html).
* [modernc.org/sqlite](https://gitlab.com/cznic/sqlite); cgo-free port of SQLite. [Other License](https://gitlab.com/cznic/sqlite/-/blob/master/LICENSE).
* [modernc.org/ccgo](https://gitlab.com/cznic/ccgo); c99 AST -> Go translater. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
* [modernc.org/libc](https://gitlab.com/cznic/libc); C-runtime services. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
* [microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday); HTML user-input sanitization. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
* [mvdan/xurls](https://github.com/mvdan/xurls); URL parsing regular expressions. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
* [nfnt/resize](https://github.com/nfnt/resize); convenient image resizing. [ISC License](https://spdx.org/licenses/ISC.html).
* [oklog/ulid](https://github.com/oklog/ulid); sequential, database-friendly ID generation. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).

View file

@ -20,6 +20,7 @@ package main
import (
"github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/account"
"github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/trans"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/urfave/cli/v2"
)
@ -41,14 +42,17 @@ func adminCommands() []*cli.Command {
&cli.StringFlag{
Name: config.UsernameFlag,
Usage: config.UsernameUsage,
Required: true,
},
&cli.StringFlag{
Name: config.EmailFlag,
Usage: config.EmailUsage,
Required: true,
},
&cli.StringFlag{
Name: config.PasswordFlag,
Usage: config.PasswordUsage,
Required: true,
},
},
Action: func(c *cli.Context) error {
@ -62,6 +66,7 @@ func adminCommands() []*cli.Command {
&cli.StringFlag{
Name: config.UsernameFlag,
Usage: config.UsernameUsage,
Required: true,
},
},
Action: func(c *cli.Context) error {
@ -75,6 +80,7 @@ func adminCommands() []*cli.Command {
&cli.StringFlag{
Name: config.UsernameFlag,
Usage: config.UsernameUsage,
Required: true,
},
},
Action: func(c *cli.Context) error {
@ -88,6 +94,7 @@ func adminCommands() []*cli.Command {
&cli.StringFlag{
Name: config.UsernameFlag,
Usage: config.UsernameUsage,
Required: true,
},
},
Action: func(c *cli.Context) error {
@ -101,6 +108,7 @@ func adminCommands() []*cli.Command {
&cli.StringFlag{
Name: config.UsernameFlag,
Usage: config.UsernameUsage,
Required: true,
},
},
Action: func(c *cli.Context) error {
@ -114,6 +122,7 @@ func adminCommands() []*cli.Command {
&cli.StringFlag{
Name: config.UsernameFlag,
Usage: config.UsernameUsage,
Required: true,
},
},
Action: func(c *cli.Context) error {
@ -127,10 +136,12 @@ func adminCommands() []*cli.Command {
&cli.StringFlag{
Name: config.UsernameFlag,
Usage: config.UsernameUsage,
Required: true,
},
&cli.StringFlag{
Name: config.PasswordFlag,
Usage: config.PasswordUsage,
Required: true,
},
},
Action: func(c *cli.Context) error {
@ -139,6 +150,34 @@ func adminCommands() []*cli.Command {
},
},
},
{
Name: "export",
Usage: "export data from the database to file at the given path",
Flags: []cli.Flag{
&cli.StringFlag{
Name: config.TransPathFlag,
Usage: config.TransPathUsage,
Required: true,
},
},
Action: func(c *cli.Context) error {
return runAction(c, trans.Export)
},
},
{
Name: "import",
Usage: "import data from a file into the database",
Flags: []cli.Flag{
&cli.StringFlag{
Name: config.TransPathFlag,
Usage: config.TransPathUsage,
Required: true,
},
},
Action: func(c *cli.Context) error {
return runAction(c, trans.Import)
},
},
},
},
}

View file

@ -0,0 +1,102 @@
# Backup and Restore
In certain conditions, it may be desirable to be able to back up a GoToSocial instance, and then to restore it again later, or just save the backup somewhere.
Some potential scenarios:
* You want to close down your instance but you might create it again later and you don't want to break federation.
* You need to migrate to a different database for some reason (Postgres => SQLite or vice versa).
* You want to keep regular backups of your data just in case something happens.
* You want to migrate from GoToSocial to a different Fediverse server, or from a different Fediverse server to GoToSocial.
* You're about to hack around on your instance and you want to make a quick backup so you don't lose everything if you mess up.
There are a few different ways of doing this, most of which require some technical knowledge.
## Image your disk
If you're running GoToSocial on a VPS (a remote machine in the cloud), arguably the easiest way to preserve all of your database entries and media is to image the disk attached to the VPS. This will preserve the whole disk. Many VPS providers offer the option of automatically creating backups on a timer, so you'll always be able to restore if your data is lost.
Advantages:
* Relatively easy to do.
* Easy to automate (depending on your vps).
* Keep complete media + database entries.
Disadvantages:
* Can cost extra depending on your VPS.
* Will probably also preserve stuff you don't need, from other programs running on the same machine.
* Vendor lock-in, difficult to move the data around.
## Back up your database files
Regardless of whether you're using Postgres or SQLite as your GoToSocial database, it's possible to simply back up the database files directly by using something like [rclone](https://rclone.org/), or following best practices for [backing up Postgres data](https://www.postgresql.org/docs/9.1/backup.html) or [SQLite data](https://sqlite.org/backup.html).
Advantages:
* Backups are relatively portable - you can move data from one machine to another.
* Well-documented procedure with a lot of guides and tooling available.
* Lots of different ways of doing your backups, depending on what you need.
Disadvantages:
* Can be a bit fiddly to set up initially.
* You need to figure out where to keep your backups.
* Restoring from backups can be a pain.
* Unless you back up media as well, references to media attachments in your db will be broken.
## Use the GoToSocial CLI
The GoToSocial CLI tool also provides commands for backing up and restoring data from your instance, which will preserve the *bare-minimum* necessary data to backup and restore your instance, without breaking federation with other instances.
What will be **kept**:
* All local account entries, including private and public keys.
* Followed/following remote accounts, including public keys.
* Follows/follow requests.
* Domain blocks.
* Account blocks.
* Account suspensions.
* User + password entries, email addresses.
What will be **dropped**:
* All statuses.
* Media.
* Faves.
* Bookmarks.
* Pins.
* Applications.
* Tokens.
The backup file produced will be in the form of a line-separated series of JSON objects (not a JSON array!). For example:
```json
{"type":"account","id":"01F8MH5NBDF2MV7CTC4Q5128HF","createdAt":"2021-08-31T12:00:53.985645Z","username":"1happyturtle","locked":true,"language":"en","uri":"http://localhost:8080/users/1happyturtle","url":"http://localhost:8080/@1happyturtle","inboxURI":"http://localhost:8080/users/1happyturtle/inbox","outboxURI":"http://localhost:8080/users/1happyturtle/outbox","followingUri":"http://localhost:8080/users/1happyturtle/following","followersUri":"http://localhost:8080/users/1happyturtle/followers","featuredCollectionUri":"http://localhost:8080/users/1happyturtle/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjz\nausfsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLz\neUPxdfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFx\njUz9l0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJY\nfKhKn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq\n79WbhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQABAoIBAGF+MxHjD15VV2NY\nKKb1GjMx98i1Xx6TijgoA+zmfha4LGu35e79Lql+0LXFp0zEpa6lAQsMQQhgd0OD\nmKKmSk+pxAvskJ4FxrhIf/yBFA4RMrj5OCaAOocRtdsOJ8n5UtFBrNAF0tzMY9q/\nkgzoq97aVF1mV9iFxaeBx6zT8ozSdqBq1PK/3w1dVg89S5tfKYc7Q0lQ00SfsTnd\niTDClKyqurebo9Pt6M7gXavgg3tvBlmwwr6XHs34Leng3oiN9mW8DVzaBMPzn+rE\nxF2eqs3v9vVpj8es88OwCh5P+ff8vJYvhu7Fcr/bJ8BItBQwfb8QBDATg/MXU2BI\n2ssW6AECgYEA4wmIyYGeu9+hzDa/J3Vh8GnlVNUCohHcChQdOsWsFXUgpVlUIHrX\neKHn42vD4Rzy52/YzJts4NkZTM9sL+kEXIEcpMG/S9xIIud7U0m/hMSAlmnJK/9j\niEXws3o4jo0E77jnRcBdIjpG4K5Eekm0DSR3SFhtZfEdN2DWPvu7K98CgYEA5tER\n/qJwFMc51AobMU87ZjXON7hI2U1WY/pVF62jSl0IcSsnj2riEKWLrs+GRG+HUg+U\naFSqAHcxaVHA0h0AYR8RopAhDdVKh0kvB8biLo+IEzNjPv2vyn0yRN5YSfXdGzyJ\nUjVU6kWdQOwmzy86nHgFaqEx7eofHIaGZzJK/AECgYEAu2VNQHX63TuzQuoVUa5z\nzoq5vhGsALYZF0CO98ndRkDNV22qIL0ESQ/qZS64GYFZhWouWoQXlGfdmCbFN65v\n6SKwz9UT3rvN1vGWO6Ltr9q6AG0EnYpJT1vbV2kUcaU4Y94NFue2d9/+TMnKv91B\n/m8Q/efvNGuWH/WQIaCKV6UCgYBz89WhYMMDfS4M2mLcu5vwddk53qciGxrqMMjs\nkzsz0Va7W12NS7lzeWaZlAE0gf6t98urOdUJVNeKvBoss4sMP0phqxwf0eWV3ur0\ncjIQB+TpGGikLVdRVuGY/UXHKe9AjoHBva8B3aTpB3lbnbNJBXZbIc1uYq3sa5w7\nXWWUAQKBgH3yW73RRpQNcc9hTUssomUsnQQgHxpfWx5tNxqod36Ytd9EKBh3NqUZ\nvPcH6gdh7mcnNaVNTtQOHLHsbPfBK/pqvb3MAsdlokJcQz8MQJ9SGBBPY6PaGw8z\nq/ambaQykER6dwlXTIlU20uXY0bttOL/iYjKmgo3vA66qfzS6nsg\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjzausf\nsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLzeUPx\ndfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFxjUz9\nl0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJYfKhK\nn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq79Wb\nhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/1happyturtle#main-key"}
{"type":"account","id":"01F8MH0BBE4FHXPH513MBVFHB0","createdAt":"2021-09-08T10:00:53.985634Z","username":"weed_lord420","locked":true,"language":"en","uri":"http://localhost:8080/users/weed_lord420","url":"http://localhost:8080/@weed_lord420","inboxURI":"http://localhost:8080/users/weed_lord420/inbox","outboxURI":"http://localhost:8080/users/weed_lord420/outbox","followingUri":"http://localhost:8080/users/weed_lord420/following","followersUri":"http://localhost:8080/users/weed_lord420/followers","featuredCollectionUri":"http://localhost:8080/users/weed_lord420/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0b\nMIyLRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//P\nceYpo5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4\nus6VxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+\nfNyYVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPc\nqwtx0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQABAoIBAEAA4GHNS4k+Ke4j\nx4J0XkUjV5UbuPY0pSpSDjOJHOJmUfLcg85Ds9mYYO6zxwOaqmrC42ieclI5rh84\nTWQUqX9+VAk1J9UKeE4xZ1SSBtnZ3rK9PjrERZ+dmQ0dATaCuEO5Wwgu7Trk++Bg\nIqy8WNGZL94v9tfwALp1jTXW9AvmQoNdCFBP62vcmYW4YLjnggxLCFTA8YKfdePa\nTuxxY6uLkeBbxzWpbRU2+bmlxd5OnCkiRSMHIX+6JdtCu2JdWpUTCnWrFi2n1TZz\nZQx9z5rvowK1O785jGMFum5vBWpjIU8sJcXmPjGMU25zzmrhzfmkJsTXER3CXoUo\nSqSPqgECgYEA78OR7bY5KKQQ7Lyz6dru4Fct5P/OXTQoOg5aS7TKb95LVWj+TANn\n5djwIbLmAUV30z0Id9VgiZOL0Hny8+3VV9eU088Z408pAy5WQrL3dB8tZLUJSq5c\n5k6X15/VjWOOZKppDxShzoV3mcohrnwVwkv4fhPFQQOJJBYz6xurWs0CgYEA3MDE\nsDMd9ahzO0dl62ynojkkA8ZTcn2UdyvLpGj9UxT5j9vWF3CfqitXgcpNiVSIbxqQ\nbo/pBch7c/2Xakv5zkdcrJj5/6gyr+m1/tK2o7+CjDaSE4SYwufXx+qkl03Zpyzt\nKdOi7Hz/b2tdjump7ECEDE45mG2ea8oSnPgXl0cCgYBkGGFzu/9g2B24t47ksmHH\nhp3CXIjqoDurARLxSCi7SzJoFc0ULtfRPSAC8YzUOwwrQ++lF4+V3+MexcqHy2Kl\nqXqYcn18SC/3BAE/Fzf3Yoyw3mNiqihefbEmc7PTsxxfKkVx5ksmzNGBgsFM9sCe\nvNigyeAvpCo8xogmPwbqgQKBgE34mIBTzcUzFmBdu5YH7r3RyPK8XkUWLhZZlbgg\njTmHMw6o61mkIgENBf+F4RUckoQLsfAbTIcKZPB3JcAZzcYaVpVwAv1V/3E671lu\nO6xivE2iCL50GzDcis7GBhSbHsF5kNsxMV6uV9qW5ZjQ13/m2b0u9BDuxwHzgdeH\nmW2JAoGAIUOYniuEwdygxWVnYatpr3NPjT3BOKoV5i9zkeJRu1hFpwQM6vQ4Ds5p\nGC5vbMKAv9Cwuw62e2HvqTun3+U2Y5Uived3XCpgM/50BFrFHCfuqXEnu1bEzk5z\n9mIhp8uXPxzC5N7tRQfb3/eU1IUcb6T6ksbr2P81z0j03J55erg=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0bMIyL\nRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//PceYp\no5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4us6V\nxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+fNyY\nVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPcqwtx\n0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/weed_lord420#main-key"}
{"type":"account","id":"01F8MH17FWEB39HZJ76B6VXSKF","createdAt":"2021-09-05T10:00:53.985641Z","username":"admin","locked":true,"language":"en","uri":"http://localhost:8080/users/admin","url":"http://localhost:8080/@admin","inboxURI":"http://localhost:8080/users/admin/inbox","outboxURI":"http://localhost:8080/users/admin/outbox","followingUri":"http://localhost:8080/users/admin/following","followersUri":"http://localhost:8080/users/admin/followers","featuredCollectionUri":"http://localhost:8080/users/admin/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVq\nhujDhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLR\nBI97qD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wg\nfvtEjEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G\n8kQJDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/Bk\nRhhGp2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQABAoIBAGK0aIADOU4ffJDe\n7sveiih5Fc1PATwx/QIR2QkWM1SREdx6LYclcX44V8xDanAbE44p1SkHY/CsEtYy\nXnyoXnn2FwFDQrdveY7+I6PApOPLAcKWkyLltC+hbVdj92/6YGNrm7EA/a77wruH\nmwjiivLnTG2CLecNiXSl33DA9YU4Yz+2Tza3IpTdjt8c/dz/BKKaxaWV+i9ew5VR\nioo5v51B+J8PrneCM/p8LGiLV148Njr0JqV6eFy1JuzItYMYdc3Fp+YnMzsuMZEA\n1akMcoln/ucVJyOFnCn6jx47nIoPZLl1KxX3aRDRfvrejm6W4yAkkTmR5voSRqax\njPL3rI0CgYEA9Acu4TO8xJ3uGaUad0N9JTYQVSmtAaE/g+df9LGMSzoj8X95S4xE\nQsGPqNGDm2VWADJjK4P05twZ+LfsfSKQ86wbp4/gbgnXpqB1P5Lty/B7KxiTnNwt\nwb1WGWTCukxfUSL3PRyf8uylkrg72RxKiBx4zKO3WVSLWOZWrFtn0qMCgYEA0H2p\nJs9Nv20ADOOX5tQ7+ruS6/B/Fhyj5fhflSYCAtOW7aME7+zQKJyqSQZ4b2Aub3Tp\nGIaUbRIGzjHyuTultFFWvjU3H5aI/0g1G9WKaBhNkyTIYVmMKtYyhXNvouWing8x\noraWx8TTBP8Cdnnk+QgdR2fpug8cghKupp5wvO8CgYA1JFtRL7MsHjh73TimQExA\njkWARlMmx7bNQtXis8eZmk+5h8kiaqly4DQoz3eZn7fa0x5Fm7b5j3UYdPVLSvvG\nFPTwyKRXUk1kPA1MivK+NuCbwf5jao+MYW8emJLPf1JCmRq+dD1g6aglC3n9Dewt\nOAYWipCjI4Y1FfRKFJ3HgQKBgEAb47+DTyzln3ZXJYZdDHR06SCTuwBZnixAy2NZ\nZJTp6yb3UbVU5E0Yn2QFEVNuB9lN4b8g4tMHEACnazN6G+HugPXL9z9HUqjs0yfT\n6dNIZdIxJUyJ9IfXhYFzlYhJhE+F7IVUD9kttJV8tI0pvja1QAuM8Fm9+84jYIDr\nh08RAoGAMYbjKHbtejcHBwt1kIcSss0cDmlZbBleJo8tdmdg4ndf5GE9N4/EL7tq\nm2zYSfr7OVdnOwRhoO+xF/6d1L7+TR1wz+k2fuMsI71aM5Ocp1nYTutjIkBTcldZ\nZzvjOgZWng5icuRLQQiDSKG5uqazqL/xGXkijb4kp4WW6myWY3c=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVqhujD\nhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLRBI97\nqD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wgfvtE\njEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G8kQJ\nDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/BkRhhG\np2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/admin#main-key"}
{"type":"account","id":"01F8MH1H7YV1Z7D2C8K2730QBF","createdAt":"2021-09-06T10:00:53.985643Z","username":"the_mighty_zork","locked":true,"language":"en","uri":"http://localhost:8080/users/the_mighty_zork","url":"http://localhost:8080/@the_mighty_zork","inboxURI":"http://localhost:8080/users/the_mighty_zork/inbox","outboxURI":"http://localhost:8080/users/the_mighty_zork/outbox","followingUri":"http://localhost:8080/users/the_mighty_zork/following","followersUri":"http://localhost:8080/users/the_mighty_zork/followers","featuredCollectionUri":"http://localhost:8080/users/the_mighty_zork/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss\n5mEA/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvC\nC9zt/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZ\nFHptEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1\ntMhsUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlq\nefr58l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQABAoIBAFa+UypbFG1cW2Tr\nNBxPm7ngOEtXl8MicV4dIVKh0TwOo13ZxtNFBbOj7jALmPn/9HrtmbkABPQHDL1U\n/nt9aNSAeTjpwH3RaD5vFX3n0g8n2zJBOZLxxzAjNi4RBLYj5uP1AiKkdvRlsJza\nuSFDkty2zMBqN9mLPHE+RePj5Qa6tjYfIQqQzu/+YnYMlXHoC2yHNKsvz6S5FhVj\nv5zATv2JlJQH3RSmhuPOah73iQnKCLzYYEAHleawKrCg/rZ3ht37Guvabeq7MqQN\nvi9pJdAA+RMxPsboHajskePjOTYJgKQSxEAMRTMfBR40aZxklxQL0EoBd1Y3CHXh\nfMg0xWECgYEA0ORrpJ1A2WNQwKcDDeBBsaJqWF4EraoFzYrugKZrAYEeVyuGD0zq\nARUaWkZTZ1f6wQ10i1WxAuKlBEds7QsLdZzLsA4um4JlBroCZiYfPnmTtb8op1LY\nFqeYTByvAmnfWWTuOI67GX9ruLg8tEGuz38kuQVSxYs51its3tScNPUCgYEAyRst\nwRbqpOqnwoRoS6pxv0Vpc3nUcfaVYwsg/qobJkiwAdlUYeE7alvEY926VW4cvU/X\nhy3L1punAqnyLI7uuqCefXEbNxO0Cebyy4Kv2Ye1uzl0OHsJczSNdfpNqfAIKwtN\nHLCYDGCsluQhz+I/5Pd0dT+JDPPW9hKS2HG7o+kCgYBqugn1VRLo/sEnbS02TbnC\n1ESZWY/yWsgUOEObH2vUnO+vgeFAt/9nBi0sqnm6d0z6jbFZ7zI9UycUhJm2ksoM\nEUxQay6M7ZZIVYkcP6X++YbqePyAYOdey8oYOR+BkC45MkQ0SVh2so+LFTaOsnBq\nO3+7uGiN3ZBzSESbpO0acQKBgQCONrsXZeZO82XpB4tdns3LbgGRWKEkajTgEnml\nvZNvck2NMSwb/5PttbFe0ei4CyMluPV4MamJPQ9Qse+BFR67OWR63uZY/4T8z6X4\nxpUmZnLcUFfgrRlUr+AtgvEy8HxGPDquxC7x6deC6RcEFEIM3/UqCOEZGMJ1x1Ky\n31LLKQKBgGCKwVgQ8+4JyHZFkie3YdHhxJDokgY+Opb0HNnoBY/lZ54UMCCJQPS2\n0XPSu651j/3adr3RQneU04gF6U2/D5JzFEV0kUsqZ4Zy2EEU0LU4ibus0gyomSpK\niWhU4QrC/M4ELxYZinlNu3ThPWNQ/PMNteVWfdgOcV7uUWl0ViFp\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss5mEA\n/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvCC9zt\n/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZFHpt\nEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1tMhs\nUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlqefr5\n8l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/the_mighty_zork#main-key"}
{"type":"block","id":"01FEXXET6XXMF7G2V3ASZP3YQW","createdAt":"2021-09-08T09:00:53.965362Z","uri":"http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW","accountId":"01F8MH5NBDF2MV7CTC4Q5128HF","targetAccountId":"01F8MH5ZK5VRH73AKHQM6Y9VNX"}
{"type":"account","id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","createdAt":"2021-08-31T12:00:53.985646Z","username":"foss_satan","domain":"fossbros-anonymous.io","locked":true,"language":"en","uri":"http://fossbros-anonymous.io/users/foss_satan","url":"http://fossbros-anonymous.io/@foss_satan","inboxURI":"http://fossbros-anonymous.io/users/foss_satan/inbox","outboxURI":"http://fossbros-anonymous.io/users/foss_satan/outbox","followingUri":"http://fossbros-anonymous.io/users/foss_satan/following","followersUri":"http://fossbros-anonymous.io/users/foss_satan/followers","featuredCollectionUri":"http://fossbros-anonymous.io/users/foss_satan/collections/featured","actorType":"Person","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEA2OyVgkaIL9VohXKYTh319j4OouHRX/8QC7piXj71k7q5RDzEyvis\nVZBc5/C1/crCpxt895i0Ai2CiXQx+dISV7s/JBhAGl8s7TQ8jLlMuptrI0+sdkBC\nlu8pU0qQmoeXVnlquOzNmqGufUxIDtLXLZDN17qf/7vWA23q4d0tG5KQhGGGKiVM\n61Ufvr9MmgPBSpyUvYMAulFlz1264L49aGWeVgOz3qUQzqtxjrP0kaIbeyt56miP\nKr5AqkRgSsXci+FAo6suxR5gzo9NgleNkbZWF9MQyKlawukPwZUDSh396vtNQMee\n/4mto7mAXw8iio0IacrYO3F7iyewXnmI/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://fossbros-anonymous.io/users/foss_satan/main-key"}
{"type":"follow","id":"01F8PYDCE8XE23GRE5DPZJDZDP","createdAt":"2021-09-08T09:00:54.749465Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PYDCE8XE23GRE5DPZJDZDP","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH5NBDF2MV7CTC4Q5128HF"}
{"type":"follow","id":"01F8PY8RHWRQZV038T4E8T9YK8","createdAt":"2021-09-06T12:00:54.749459Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PY8RHWRQZV038T4E8T9YK8","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH17FWEB39HZJ76B6VXSKF"}
{"type":"domainBlock","id":"01FF22EQM7X8E3RX1XGPN7S87D","createdAt":"2021-09-08T10:00:53.968971Z","domain":"replyguys.com","createdByAccountID":"01F8MH17FWEB39HZJ76B6VXSKF","privateComment":"i blocked this domain because they keep replying with pushy + unwarranted linux advice","publicComment":"reply-guying to tech posts","obfuscate":false}
{"type":"user","id":"01F8MGYG9E893WRHW0TAEXR8GJ","createdAt":"2021-09-08T10:00:53.97247Z","accountID":"01F8MH0BBE4FHXPH513MBVFHB0","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","locale":"en","lastEmailedAt":"0001-01-01T00:00:00Z","confirmationToken":"a5a280bd-34be-44a3-8330-a57eaf61b8dd","confirmationTokenSentAt":"2021-09-08T10:00:53.972472Z","unconfirmedEmail":"weed_lord420@example.org","moderator":false,"admin":false,"disabled":false,"approved":false}
{"type":"user","id":"01F8MGWYWKVKS3VS8DV1AMYPGE","createdAt":"2021-09-05T10:00:53.972475Z","email":"admin@example.org","accountID":"01F8MH17FWEB39HZJ76B6VXSKF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:50:53.972477Z","lastSignInAt":"2021-09-08T08:00:53.972477Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:30:53.972478Z","confirmedAt":"2021-09-05T10:00:53.972478Z","moderator":true,"admin":true,"disabled":false,"approved":true}
{"type":"user","id":"01F8MGVGPHQ2D3P3X0454H54Z5","createdAt":"2021-09-06T22:00:53.97248Z","email":"zork@example.org","accountID":"01F8MH1H7YV1Z7D2C8K2730QBF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972481Z","lastSignInAt":"2021-09-08T08:00:53.972481Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972482Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972483Z","confirmedAt":"2021-09-07T00:00:53.972482Z","moderator":false,"admin":false,"disabled":false,"approved":true}
{"type":"user","id":"01F8MH1VYJAE00TVVGMM5JNJ8X","createdAt":"2021-09-06T22:00:53.972485Z","email":"tortle.dude@example.org","accountID":"01F8MH5NBDF2MV7CTC4Q5128HF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972485Z","lastSignInAt":"2021-09-08T08:00:53.972486Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972487Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972487Z","confirmedAt":"2021-09-07T00:00:53.972487Z","moderator":false,"admin":false,"disabled":false,"approved":true}
{"type":"instance","id":"01BZDDRPAB8J645ABY31HHF68Y","createdAt":"2021-09-08T10:00:54.763912Z","domain":"localhost:8080","title":"localhost:8080","uri":"http://localhost:8080","reputation":0}
```
For information on how to use the commands to import/export, see [here](cli.md#gotosocial-admin-export).
Advantages:
* Database agnostic: exported data is in a somewhat generic format, and the `import` command can be used to insert this data into either a Postgres or an SQLite database.
* Lightweight: only what is needed is preserved, so backup files can be quite small (even small enough to send in an email). Backup/import commands just take a few seconds to run.
* Easily readable format: the output is just JSON.
Disadvantages:
* Loss of statuses/media/etc: don't do a backup/restore this way unless you're willing to drop stuff.
* You need to use the GtS CLI tool to insert data back into a database, unless you write custom tooling for it.

View file

@ -213,3 +213,78 @@ Example:
```bash
gotosocial admin account password --username some_username --pasword some_really_good_password
```
### gotosocial admin export
This command can be used to export data from your GoToSocial instance into a file, for backup/storage.
The file format will be a series of newline-separated JSON objects.
`gotosocial admin export --help`:
```text
NAME:
gotosocial admin export - export data from the database to file at the given path
USAGE:
gotosocial admin export [command options] [arguments...]
OPTIONS:
--path value the path of the file to import from/export to
--help, -h show help (default: false)
```
Example:
```bash
gotosocial admin export --path ./example.json
```
`example.json`:
```json
{"type":"account","id":"01F8MH5NBDF2MV7CTC4Q5128HF","createdAt":"2021-08-31T12:00:53.985645Z","username":"1happyturtle","locked":true,"language":"en","uri":"http://localhost:8080/users/1happyturtle","url":"http://localhost:8080/@1happyturtle","inboxURI":"http://localhost:8080/users/1happyturtle/inbox","outboxURI":"http://localhost:8080/users/1happyturtle/outbox","followingUri":"http://localhost:8080/users/1happyturtle/following","followersUri":"http://localhost:8080/users/1happyturtle/followers","featuredCollectionUri":"http://localhost:8080/users/1happyturtle/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjz\nausfsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLz\neUPxdfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFx\njUz9l0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJY\nfKhKn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq\n79WbhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQABAoIBAGF+MxHjD15VV2NY\nKKb1GjMx98i1Xx6TijgoA+zmfha4LGu35e79Lql+0LXFp0zEpa6lAQsMQQhgd0OD\nmKKmSk+pxAvskJ4FxrhIf/yBFA4RMrj5OCaAOocRtdsOJ8n5UtFBrNAF0tzMY9q/\nkgzoq97aVF1mV9iFxaeBx6zT8ozSdqBq1PK/3w1dVg89S5tfKYc7Q0lQ00SfsTnd\niTDClKyqurebo9Pt6M7gXavgg3tvBlmwwr6XHs34Leng3oiN9mW8DVzaBMPzn+rE\nxF2eqs3v9vVpj8es88OwCh5P+ff8vJYvhu7Fcr/bJ8BItBQwfb8QBDATg/MXU2BI\n2ssW6AECgYEA4wmIyYGeu9+hzDa/J3Vh8GnlVNUCohHcChQdOsWsFXUgpVlUIHrX\neKHn42vD4Rzy52/YzJts4NkZTM9sL+kEXIEcpMG/S9xIIud7U0m/hMSAlmnJK/9j\niEXws3o4jo0E77jnRcBdIjpG4K5Eekm0DSR3SFhtZfEdN2DWPvu7K98CgYEA5tER\n/qJwFMc51AobMU87ZjXON7hI2U1WY/pVF62jSl0IcSsnj2riEKWLrs+GRG+HUg+U\naFSqAHcxaVHA0h0AYR8RopAhDdVKh0kvB8biLo+IEzNjPv2vyn0yRN5YSfXdGzyJ\nUjVU6kWdQOwmzy86nHgFaqEx7eofHIaGZzJK/AECgYEAu2VNQHX63TuzQuoVUa5z\nzoq5vhGsALYZF0CO98ndRkDNV22qIL0ESQ/qZS64GYFZhWouWoQXlGfdmCbFN65v\n6SKwz9UT3rvN1vGWO6Ltr9q6AG0EnYpJT1vbV2kUcaU4Y94NFue2d9/+TMnKv91B\n/m8Q/efvNGuWH/WQIaCKV6UCgYBz89WhYMMDfS4M2mLcu5vwddk53qciGxrqMMjs\nkzsz0Va7W12NS7lzeWaZlAE0gf6t98urOdUJVNeKvBoss4sMP0phqxwf0eWV3ur0\ncjIQB+TpGGikLVdRVuGY/UXHKe9AjoHBva8B3aTpB3lbnbNJBXZbIc1uYq3sa5w7\nXWWUAQKBgH3yW73RRpQNcc9hTUssomUsnQQgHxpfWx5tNxqod36Ytd9EKBh3NqUZ\nvPcH6gdh7mcnNaVNTtQOHLHsbPfBK/pqvb3MAsdlokJcQz8MQJ9SGBBPY6PaGw8z\nq/ambaQykER6dwlXTIlU20uXY0bttOL/iYjKmgo3vA66qfzS6nsg\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjzausf\nsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLzeUPx\ndfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFxjUz9\nl0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJYfKhK\nn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq79Wb\nhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/1happyturtle#main-key"}
{"type":"account","id":"01F8MH0BBE4FHXPH513MBVFHB0","createdAt":"2021-09-08T10:00:53.985634Z","username":"weed_lord420","locked":true,"language":"en","uri":"http://localhost:8080/users/weed_lord420","url":"http://localhost:8080/@weed_lord420","inboxURI":"http://localhost:8080/users/weed_lord420/inbox","outboxURI":"http://localhost:8080/users/weed_lord420/outbox","followingUri":"http://localhost:8080/users/weed_lord420/following","followersUri":"http://localhost:8080/users/weed_lord420/followers","featuredCollectionUri":"http://localhost:8080/users/weed_lord420/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0b\nMIyLRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//P\nceYpo5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4\nus6VxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+\nfNyYVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPc\nqwtx0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQABAoIBAEAA4GHNS4k+Ke4j\nx4J0XkUjV5UbuPY0pSpSDjOJHOJmUfLcg85Ds9mYYO6zxwOaqmrC42ieclI5rh84\nTWQUqX9+VAk1J9UKeE4xZ1SSBtnZ3rK9PjrERZ+dmQ0dATaCuEO5Wwgu7Trk++Bg\nIqy8WNGZL94v9tfwALp1jTXW9AvmQoNdCFBP62vcmYW4YLjnggxLCFTA8YKfdePa\nTuxxY6uLkeBbxzWpbRU2+bmlxd5OnCkiRSMHIX+6JdtCu2JdWpUTCnWrFi2n1TZz\nZQx9z5rvowK1O785jGMFum5vBWpjIU8sJcXmPjGMU25zzmrhzfmkJsTXER3CXoUo\nSqSPqgECgYEA78OR7bY5KKQQ7Lyz6dru4Fct5P/OXTQoOg5aS7TKb95LVWj+TANn\n5djwIbLmAUV30z0Id9VgiZOL0Hny8+3VV9eU088Z408pAy5WQrL3dB8tZLUJSq5c\n5k6X15/VjWOOZKppDxShzoV3mcohrnwVwkv4fhPFQQOJJBYz6xurWs0CgYEA3MDE\nsDMd9ahzO0dl62ynojkkA8ZTcn2UdyvLpGj9UxT5j9vWF3CfqitXgcpNiVSIbxqQ\nbo/pBch7c/2Xakv5zkdcrJj5/6gyr+m1/tK2o7+CjDaSE4SYwufXx+qkl03Zpyzt\nKdOi7Hz/b2tdjump7ECEDE45mG2ea8oSnPgXl0cCgYBkGGFzu/9g2B24t47ksmHH\nhp3CXIjqoDurARLxSCi7SzJoFc0ULtfRPSAC8YzUOwwrQ++lF4+V3+MexcqHy2Kl\nqXqYcn18SC/3BAE/Fzf3Yoyw3mNiqihefbEmc7PTsxxfKkVx5ksmzNGBgsFM9sCe\nvNigyeAvpCo8xogmPwbqgQKBgE34mIBTzcUzFmBdu5YH7r3RyPK8XkUWLhZZlbgg\njTmHMw6o61mkIgENBf+F4RUckoQLsfAbTIcKZPB3JcAZzcYaVpVwAv1V/3E671lu\nO6xivE2iCL50GzDcis7GBhSbHsF5kNsxMV6uV9qW5ZjQ13/m2b0u9BDuxwHzgdeH\nmW2JAoGAIUOYniuEwdygxWVnYatpr3NPjT3BOKoV5i9zkeJRu1hFpwQM6vQ4Ds5p\nGC5vbMKAv9Cwuw62e2HvqTun3+U2Y5Uived3XCpgM/50BFrFHCfuqXEnu1bEzk5z\n9mIhp8uXPxzC5N7tRQfb3/eU1IUcb6T6ksbr2P81z0j03J55erg=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0bMIyL\nRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//PceYp\no5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4us6V\nxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+fNyY\nVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPcqwtx\n0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/weed_lord420#main-key"}
{"type":"account","id":"01F8MH17FWEB39HZJ76B6VXSKF","createdAt":"2021-09-05T10:00:53.985641Z","username":"admin","locked":true,"language":"en","uri":"http://localhost:8080/users/admin","url":"http://localhost:8080/@admin","inboxURI":"http://localhost:8080/users/admin/inbox","outboxURI":"http://localhost:8080/users/admin/outbox","followingUri":"http://localhost:8080/users/admin/following","followersUri":"http://localhost:8080/users/admin/followers","featuredCollectionUri":"http://localhost:8080/users/admin/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVq\nhujDhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLR\nBI97qD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wg\nfvtEjEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G\n8kQJDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/Bk\nRhhGp2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQABAoIBAGK0aIADOU4ffJDe\n7sveiih5Fc1PATwx/QIR2QkWM1SREdx6LYclcX44V8xDanAbE44p1SkHY/CsEtYy\nXnyoXnn2FwFDQrdveY7+I6PApOPLAcKWkyLltC+hbVdj92/6YGNrm7EA/a77wruH\nmwjiivLnTG2CLecNiXSl33DA9YU4Yz+2Tza3IpTdjt8c/dz/BKKaxaWV+i9ew5VR\nioo5v51B+J8PrneCM/p8LGiLV148Njr0JqV6eFy1JuzItYMYdc3Fp+YnMzsuMZEA\n1akMcoln/ucVJyOFnCn6jx47nIoPZLl1KxX3aRDRfvrejm6W4yAkkTmR5voSRqax\njPL3rI0CgYEA9Acu4TO8xJ3uGaUad0N9JTYQVSmtAaE/g+df9LGMSzoj8X95S4xE\nQsGPqNGDm2VWADJjK4P05twZ+LfsfSKQ86wbp4/gbgnXpqB1P5Lty/B7KxiTnNwt\nwb1WGWTCukxfUSL3PRyf8uylkrg72RxKiBx4zKO3WVSLWOZWrFtn0qMCgYEA0H2p\nJs9Nv20ADOOX5tQ7+ruS6/B/Fhyj5fhflSYCAtOW7aME7+zQKJyqSQZ4b2Aub3Tp\nGIaUbRIGzjHyuTultFFWvjU3H5aI/0g1G9WKaBhNkyTIYVmMKtYyhXNvouWing8x\noraWx8TTBP8Cdnnk+QgdR2fpug8cghKupp5wvO8CgYA1JFtRL7MsHjh73TimQExA\njkWARlMmx7bNQtXis8eZmk+5h8kiaqly4DQoz3eZn7fa0x5Fm7b5j3UYdPVLSvvG\nFPTwyKRXUk1kPA1MivK+NuCbwf5jao+MYW8emJLPf1JCmRq+dD1g6aglC3n9Dewt\nOAYWipCjI4Y1FfRKFJ3HgQKBgEAb47+DTyzln3ZXJYZdDHR06SCTuwBZnixAy2NZ\nZJTp6yb3UbVU5E0Yn2QFEVNuB9lN4b8g4tMHEACnazN6G+HugPXL9z9HUqjs0yfT\n6dNIZdIxJUyJ9IfXhYFzlYhJhE+F7IVUD9kttJV8tI0pvja1QAuM8Fm9+84jYIDr\nh08RAoGAMYbjKHbtejcHBwt1kIcSss0cDmlZbBleJo8tdmdg4ndf5GE9N4/EL7tq\nm2zYSfr7OVdnOwRhoO+xF/6d1L7+TR1wz+k2fuMsI71aM5Ocp1nYTutjIkBTcldZ\nZzvjOgZWng5icuRLQQiDSKG5uqazqL/xGXkijb4kp4WW6myWY3c=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVqhujD\nhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLRBI97\nqD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wgfvtE\njEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G8kQJ\nDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/BkRhhG\np2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/admin#main-key"}
{"type":"account","id":"01F8MH1H7YV1Z7D2C8K2730QBF","createdAt":"2021-09-06T10:00:53.985643Z","username":"the_mighty_zork","locked":true,"language":"en","uri":"http://localhost:8080/users/the_mighty_zork","url":"http://localhost:8080/@the_mighty_zork","inboxURI":"http://localhost:8080/users/the_mighty_zork/inbox","outboxURI":"http://localhost:8080/users/the_mighty_zork/outbox","followingUri":"http://localhost:8080/users/the_mighty_zork/following","followersUri":"http://localhost:8080/users/the_mighty_zork/followers","featuredCollectionUri":"http://localhost:8080/users/the_mighty_zork/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss\n5mEA/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvC\nC9zt/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZ\nFHptEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1\ntMhsUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlq\nefr58l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQABAoIBAFa+UypbFG1cW2Tr\nNBxPm7ngOEtXl8MicV4dIVKh0TwOo13ZxtNFBbOj7jALmPn/9HrtmbkABPQHDL1U\n/nt9aNSAeTjpwH3RaD5vFX3n0g8n2zJBOZLxxzAjNi4RBLYj5uP1AiKkdvRlsJza\nuSFDkty2zMBqN9mLPHE+RePj5Qa6tjYfIQqQzu/+YnYMlXHoC2yHNKsvz6S5FhVj\nv5zATv2JlJQH3RSmhuPOah73iQnKCLzYYEAHleawKrCg/rZ3ht37Guvabeq7MqQN\nvi9pJdAA+RMxPsboHajskePjOTYJgKQSxEAMRTMfBR40aZxklxQL0EoBd1Y3CHXh\nfMg0xWECgYEA0ORrpJ1A2WNQwKcDDeBBsaJqWF4EraoFzYrugKZrAYEeVyuGD0zq\nARUaWkZTZ1f6wQ10i1WxAuKlBEds7QsLdZzLsA4um4JlBroCZiYfPnmTtb8op1LY\nFqeYTByvAmnfWWTuOI67GX9ruLg8tEGuz38kuQVSxYs51its3tScNPUCgYEAyRst\nwRbqpOqnwoRoS6pxv0Vpc3nUcfaVYwsg/qobJkiwAdlUYeE7alvEY926VW4cvU/X\nhy3L1punAqnyLI7uuqCefXEbNxO0Cebyy4Kv2Ye1uzl0OHsJczSNdfpNqfAIKwtN\nHLCYDGCsluQhz+I/5Pd0dT+JDPPW9hKS2HG7o+kCgYBqugn1VRLo/sEnbS02TbnC\n1ESZWY/yWsgUOEObH2vUnO+vgeFAt/9nBi0sqnm6d0z6jbFZ7zI9UycUhJm2ksoM\nEUxQay6M7ZZIVYkcP6X++YbqePyAYOdey8oYOR+BkC45MkQ0SVh2so+LFTaOsnBq\nO3+7uGiN3ZBzSESbpO0acQKBgQCONrsXZeZO82XpB4tdns3LbgGRWKEkajTgEnml\nvZNvck2NMSwb/5PttbFe0ei4CyMluPV4MamJPQ9Qse+BFR67OWR63uZY/4T8z6X4\nxpUmZnLcUFfgrRlUr+AtgvEy8HxGPDquxC7x6deC6RcEFEIM3/UqCOEZGMJ1x1Ky\n31LLKQKBgGCKwVgQ8+4JyHZFkie3YdHhxJDokgY+Opb0HNnoBY/lZ54UMCCJQPS2\n0XPSu651j/3adr3RQneU04gF6U2/D5JzFEV0kUsqZ4Zy2EEU0LU4ibus0gyomSpK\niWhU4QrC/M4ELxYZinlNu3ThPWNQ/PMNteVWfdgOcV7uUWl0ViFp\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss5mEA\n/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvCC9zt\n/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZFHpt\nEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1tMhs\nUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlqefr5\n8l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/the_mighty_zork#main-key"}
{"type":"block","id":"01FEXXET6XXMF7G2V3ASZP3YQW","createdAt":"2021-09-08T09:00:53.965362Z","uri":"http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW","accountId":"01F8MH5NBDF2MV7CTC4Q5128HF","targetAccountId":"01F8MH5ZK5VRH73AKHQM6Y9VNX"}
{"type":"account","id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","createdAt":"2021-08-31T12:00:53.985646Z","username":"foss_satan","domain":"fossbros-anonymous.io","locked":true,"language":"en","uri":"http://fossbros-anonymous.io/users/foss_satan","url":"http://fossbros-anonymous.io/@foss_satan","inboxURI":"http://fossbros-anonymous.io/users/foss_satan/inbox","outboxURI":"http://fossbros-anonymous.io/users/foss_satan/outbox","followingUri":"http://fossbros-anonymous.io/users/foss_satan/following","followersUri":"http://fossbros-anonymous.io/users/foss_satan/followers","featuredCollectionUri":"http://fossbros-anonymous.io/users/foss_satan/collections/featured","actorType":"Person","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEA2OyVgkaIL9VohXKYTh319j4OouHRX/8QC7piXj71k7q5RDzEyvis\nVZBc5/C1/crCpxt895i0Ai2CiXQx+dISV7s/JBhAGl8s7TQ8jLlMuptrI0+sdkBC\nlu8pU0qQmoeXVnlquOzNmqGufUxIDtLXLZDN17qf/7vWA23q4d0tG5KQhGGGKiVM\n61Ufvr9MmgPBSpyUvYMAulFlz1264L49aGWeVgOz3qUQzqtxjrP0kaIbeyt56miP\nKr5AqkRgSsXci+FAo6suxR5gzo9NgleNkbZWF9MQyKlawukPwZUDSh396vtNQMee\n/4mto7mAXw8iio0IacrYO3F7iyewXnmI/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://fossbros-anonymous.io/users/foss_satan/main-key"}
{"type":"follow","id":"01F8PYDCE8XE23GRE5DPZJDZDP","createdAt":"2021-09-08T09:00:54.749465Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PYDCE8XE23GRE5DPZJDZDP","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH5NBDF2MV7CTC4Q5128HF"}
{"type":"follow","id":"01F8PY8RHWRQZV038T4E8T9YK8","createdAt":"2021-09-06T12:00:54.749459Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PY8RHWRQZV038T4E8T9YK8","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH17FWEB39HZJ76B6VXSKF"}
{"type":"domainBlock","id":"01FF22EQM7X8E3RX1XGPN7S87D","createdAt":"2021-09-08T10:00:53.968971Z","domain":"replyguys.com","createdByAccountID":"01F8MH17FWEB39HZJ76B6VXSKF","privateComment":"i blocked this domain because they keep replying with pushy + unwarranted linux advice","publicComment":"reply-guying to tech posts","obfuscate":false}
{"type":"user","id":"01F8MGYG9E893WRHW0TAEXR8GJ","createdAt":"2021-09-08T10:00:53.97247Z","accountID":"01F8MH0BBE4FHXPH513MBVFHB0","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","locale":"en","lastEmailedAt":"0001-01-01T00:00:00Z","confirmationToken":"a5a280bd-34be-44a3-8330-a57eaf61b8dd","confirmationTokenSentAt":"2021-09-08T10:00:53.972472Z","unconfirmedEmail":"weed_lord420@example.org","moderator":false,"admin":false,"disabled":false,"approved":false}
{"type":"user","id":"01F8MGWYWKVKS3VS8DV1AMYPGE","createdAt":"2021-09-05T10:00:53.972475Z","email":"admin@example.org","accountID":"01F8MH17FWEB39HZJ76B6VXSKF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:50:53.972477Z","lastSignInAt":"2021-09-08T08:00:53.972477Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:30:53.972478Z","confirmedAt":"2021-09-05T10:00:53.972478Z","moderator":true,"admin":true,"disabled":false,"approved":true}
{"type":"user","id":"01F8MGVGPHQ2D3P3X0454H54Z5","createdAt":"2021-09-06T22:00:53.97248Z","email":"zork@example.org","accountID":"01F8MH1H7YV1Z7D2C8K2730QBF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972481Z","lastSignInAt":"2021-09-08T08:00:53.972481Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972482Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972483Z","confirmedAt":"2021-09-07T00:00:53.972482Z","moderator":false,"admin":false,"disabled":false,"approved":true}
{"type":"user","id":"01F8MH1VYJAE00TVVGMM5JNJ8X","createdAt":"2021-09-06T22:00:53.972485Z","email":"tortle.dude@example.org","accountID":"01F8MH5NBDF2MV7CTC4Q5128HF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972485Z","lastSignInAt":"2021-09-08T08:00:53.972486Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972487Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972487Z","confirmedAt":"2021-09-07T00:00:53.972487Z","moderator":false,"admin":false,"disabled":false,"approved":true}
{"type":"instance","id":"01BZDDRPAB8J645ABY31HHF68Y","createdAt":"2021-09-08T10:00:54.763912Z","domain":"localhost:8080","title":"localhost:8080","uri":"http://localhost:8080","reputation":0}
```
### gotosocial admin import
This command can be used to import data from a file into your GoToSocial database.
If GoToSocial tables don't yet exist in the database, they will be created.
If any conflicts occur while importing (an already exists while attempting to import a specific account, for example), then the process will be aborted.
The file format should be a series of newline-separated JSON objects (see above).
`gotosocial admin import --help`:
```text
NAME:
gotosocial admin import - import data from a file into the database
USAGE:
gotosocial admin import [command options] [arguments...]
OPTIONS:
--path value the path of the file to import from/export to
--help, -h show help (default: false)
```
Example:
```bash
gotosocial admin import --path ./example.json
```

1
go.mod
View file

@ -35,6 +35,7 @@ require (
github.com/leodido/go-urn v1.2.1 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/microcosm-cc/bluemonday v1.0.15
github.com/mitchellh/mapstructure v1.4.1
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646

2
go.sum
View file

@ -346,6 +346,8 @@ github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A
github.com/memcachier/mc v2.0.1+incompatible/go.mod h1:7bkvFE61leUBvXz+yxsOnGBQSZpBSPIMUQSmmSHvuXc=
github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY=
github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

View file

@ -157,7 +157,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() {
b, err := ioutil.ReadAll(result.Body)
assert.NoError(suite.T(), err)
assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
// should be a Collection
m := make(map[string]interface{})
@ -188,7 +188,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
// setup request
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5", nil) // the endpoint we're hitting
ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0", nil) // the endpoint we're hitting
ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
ctx.Request.Header.Set("Date", signedRequest.DateHeader)
@ -220,7 +220,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
assert.NoError(suite.T(), err)
fmt.Println(string(b))
assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
// should be a Collection
m := make(map[string]interface{})

View file

@ -144,7 +144,10 @@ func copyStatus(status *gtsmodel.Status) *gtsmodel.Status {
Sensitive: status.Sensitive,
Language: status.Language,
CreatedWithApplicationID: status.CreatedWithApplicationID,
VisibilityAdvanced: status.VisibilityAdvanced,
Federated: status.Federated,
Boostable: status.Boostable,
Replyable: status.Replyable,
Likeable: status.Likeable,
ActivityStreamsType: status.ActivityStreamsType,
Text: status.Text,
Pinned: status.Pinned,

View file

@ -0,0 +1,52 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"context"
"errors"
"fmt"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/cliactions"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
"github.com/superseriousbusiness/gotosocial/internal/trans"
)
// Export exports info from the database into a file
var Export cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error {
dbConn, err := bundb.NewBunDBService(ctx, c, log)
if err != nil {
return fmt.Errorf("error creating dbservice: %s", err)
}
exporter := trans.NewExporter(dbConn, log)
path, ok := c.ExportCLIFlags[config.TransPathFlag]
if !ok {
return errors.New("no path set")
}
if err := exporter.ExportMinimal(ctx, path); err != nil {
return err
}
return dbConn.Stop(ctx)
}

View file

@ -0,0 +1,56 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"context"
"errors"
"fmt"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/cliactions"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
"github.com/superseriousbusiness/gotosocial/internal/trans"
)
// Import imports info from a file into the database
var Import cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error {
dbConn, err := bundb.NewBunDBService(ctx, c, log)
if err != nil {
return fmt.Errorf("error creating dbservice: %s", err)
}
importer := trans.NewImporter(dbConn, log)
path, ok := c.ExportCLIFlags[config.TransPathFlag]
if !ok {
return errors.New("no path set")
}
if err := dbConn.CreateAllTables(ctx); err != nil {
return err
}
if err := importer.Import(ctx, path); err != nil {
return err
}
return dbConn.Stop(ctx)
}

View file

@ -39,7 +39,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
"github.com/superseriousbusiness/gotosocial/internal/gotosocial"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/oidc"
@ -51,32 +50,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/web"
)
var models []interface{} = []interface{}{
&gtsmodel.Account{},
&gtsmodel.Application{},
&gtsmodel.Block{},
&gtsmodel.DomainBlock{},
&gtsmodel.EmailDomainBlock{},
&gtsmodel.Follow{},
&gtsmodel.FollowRequest{},
&gtsmodel.MediaAttachment{},
&gtsmodel.Mention{},
&gtsmodel.Status{},
&gtsmodel.StatusToEmoji{},
&gtsmodel.StatusToTag{},
&gtsmodel.StatusFave{},
&gtsmodel.StatusBookmark{},
&gtsmodel.StatusMute{},
&gtsmodel.Tag{},
&gtsmodel.User{},
&gtsmodel.Emoji{},
&gtsmodel.Instance{},
&gtsmodel.Notification{},
&gtsmodel.RouterSession{},
&gtsmodel.Token{},
&gtsmodel.Client{},
}
// Start creates and starts a gotosocial server
var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error {
dbService, err := bundb.NewBunDBService(ctx, c, log)
@ -84,10 +57,8 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log
return fmt.Errorf("error creating dbservice: %s", err)
}
for _, m := range models {
if err := dbService.CreateTable(ctx, m); err != nil {
return fmt.Errorf("table creation error: %s", err)
}
if err := dbService.CreateAllTables(ctx); err != nil {
return fmt.Errorf("error creating database tables: %s", err)
}
if err := dbService.CreateInstanceAccount(ctx); err != nil {

View file

@ -36,6 +36,9 @@ const (
PasswordFlag = "password"
PasswordUsage = "the password to set for this account"
TransPathFlag = "path"
TransPathUsage = "the path of the file to import from/export to"
)
// Config pulls together all the configuration needed to run gotosocial
@ -65,6 +68,7 @@ type Config struct {
Not parsed from .yaml configuration file.
*/
AccountCLIFlags map[string]string
ExportCLIFlags map[string]string
SoftwareVersion string
}
@ -92,6 +96,7 @@ func Empty() *Config {
LetsEncryptConfig: &LetsEncryptConfig{},
OIDCConfig: &OIDCConfig{},
AccountCLIFlags: make(map[string]string),
ExportCLIFlags: make(map[string]string),
}
}
@ -320,6 +325,9 @@ func (c *Config) ParseCLIFlags(f KeyedFlags, version string) error {
c.AccountCLIFlags[EmailFlag] = f.String(EmailFlag)
c.AccountCLIFlags[PasswordFlag] = f.String(PasswordFlag)
// export CLI flags
c.ExportCLIFlags[TransPathFlag] = f.String(TransPathFlag)
c.SoftwareVersion = version
return nil
}

View file

@ -26,6 +26,10 @@ type Basic interface {
// For implementations that don't use tables, this can just return nil.
CreateTable(ctx context.Context, i interface{}) Error
// CreateAllTables creates *all* tables necessary for the running of GoToSocial.
// Because it uses the 'if not exists' parameter it is safe to run against a GtS that's already been initialized.
CreateAllTables(ctx context.Context) Error
// DropTable drops the table for the given interface.
// For implementations that don't use tables, this can just return nil.
DropTable(ctx context.Context, i interface{}) Error

View file

@ -24,6 +24,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/uptrace/bun"
)
@ -53,17 +54,8 @@ func (b *basicDB) GetWhere(ctx context.Context, where []db.Where, i interface{})
}
q := b.conn.NewSelect().Model(i)
for _, w := range where {
if w.Value == nil {
q = q.Where("? IS NULL", bun.Ident(w.Key))
} else {
if w.CaseInsensitive {
q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value)
} else {
q = q.Where("? = ?", bun.Safe(w.Key), w.Value)
}
}
}
selectWhere(q, where)
err := q.Scan(ctx)
return b.conn.ProcessError(err)
@ -97,9 +89,7 @@ func (b *basicDB) DeleteWhere(ctx context.Context, where []db.Where, i interface
NewDelete().
Model(i)
for _, w := range where {
q = q.Where("? = ?", bun.Safe(w.Key), w.Value)
}
deleteWhere(q, where)
_, err := q.Exec(ctx)
return b.conn.ProcessError(err)
@ -128,17 +118,7 @@ func (b *basicDB) UpdateOneByID(ctx context.Context, id string, key string, valu
func (b *basicDB) UpdateWhere(ctx context.Context, where []db.Where, key string, value interface{}, i interface{}) db.Error {
q := b.conn.NewUpdate().Model(i)
for _, w := range where {
if w.Value == nil {
q = q.Where("? IS NULL", bun.Ident(w.Key))
} else {
if w.CaseInsensitive {
q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value)
} else {
q = q.Where("? = ?", bun.Safe(w.Key), w.Value)
}
}
}
updateWhere(q, where)
q = q.Set("? = ?", bun.Safe(key), value)
@ -151,6 +131,40 @@ func (b *basicDB) CreateTable(ctx context.Context, i interface{}) db.Error {
return err
}
func (b *basicDB) CreateAllTables(ctx context.Context) db.Error {
models := []interface{}{
&gtsmodel.Account{},
&gtsmodel.Application{},
&gtsmodel.Block{},
&gtsmodel.DomainBlock{},
&gtsmodel.EmailDomainBlock{},
&gtsmodel.Follow{},
&gtsmodel.FollowRequest{},
&gtsmodel.MediaAttachment{},
&gtsmodel.Mention{},
&gtsmodel.Status{},
&gtsmodel.StatusToEmoji{},
&gtsmodel.StatusToTag{},
&gtsmodel.StatusFave{},
&gtsmodel.StatusBookmark{},
&gtsmodel.StatusMute{},
&gtsmodel.Tag{},
&gtsmodel.User{},
&gtsmodel.Emoji{},
&gtsmodel.Instance{},
&gtsmodel.Notification{},
&gtsmodel.RouterSession{},
&gtsmodel.Token{},
&gtsmodel.Client{},
}
for _, i := range models {
if err := b.CreateTable(ctx, i); err != nil {
return err
}
}
return nil
}
func (b *basicDB) DropTable(ctx context.Context, i interface{}) db.Error {
_, err := b.conn.NewDropTable().Model(i).IfExists().Exec(ctx)
return b.conn.ProcessError(err)

View file

@ -23,6 +23,7 @@ import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
@ -42,7 +43,25 @@ func (suite *BasicTestSuite) TestGetAllStatuses() {
s := []*gtsmodel.Status{}
err := suite.db.GetAll(context.Background(), &s)
suite.NoError(err)
suite.Len(s, 12)
suite.Len(s, 13)
}
func (suite *BasicTestSuite) TestGetAllNotNull() {
where := []db.Where{{
Key: "domain",
Value: nil,
Not: true,
}}
a := []*gtsmodel.Account{}
err := suite.db.GetWhere(context.Background(), where, &a)
suite.NoError(err)
suite.NotEmpty(a)
for _, acct := range a {
suite.NotEmpty(acct.Domain)
}
}
func TestBasicTestSuite(t *testing.T) {

View file

@ -240,12 +240,12 @@ func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status,
}
}
// only do one loop if we only want direct children
if onlyDirect {
return
}
// if we're not only looking for direct children of status, then do the same children-finding
// operation for the found child status too.
if !onlyDirect {
s.statusChildren(ctx, child, foundStatuses, false, minID)
}
}
}
func (s *statusDB) CountStatusReplies(ctx context.Context, status *gtsmodel.Status) (int, db.Error) {

View file

@ -43,10 +43,14 @@ func (suite *StatusTestSuite) TestGetStatusByID() {
suite.Nil(status.BoostOfAccount)
suite.Nil(status.InReplyTo)
suite.Nil(status.InReplyToAccount)
suite.True(status.Federated)
suite.True(status.Boostable)
suite.True(status.Replyable)
suite.True(status.Likeable)
}
func (suite *StatusTestSuite) TestGetStatusByURI() {
status, err := suite.db.GetStatusByURI(context.Background(), suite.testStatuses["local_account_1_status_1"].URI)
status, err := suite.db.GetStatusByURI(context.Background(), suite.testStatuses["local_account_2_status_3"].URI)
if err != nil {
suite.FailNow(err.Error())
}
@ -57,6 +61,10 @@ func (suite *StatusTestSuite) TestGetStatusByURI() {
suite.Nil(status.BoostOfAccount)
suite.Nil(status.InReplyTo)
suite.Nil(status.InReplyToAccount)
suite.True(status.Federated)
suite.True(status.Boostable)
suite.False(status.Replyable)
suite.False(status.Likeable)
}
func (suite *StatusTestSuite) TestGetStatusWithExtras() {
@ -70,6 +78,10 @@ func (suite *StatusTestSuite) TestGetStatusWithExtras() {
suite.NotEmpty(status.Tags)
suite.NotEmpty(status.Attachments)
suite.NotEmpty(status.Emojis)
suite.True(status.Federated)
suite.True(status.Boostable)
suite.True(status.Replyable)
suite.True(status.Likeable)
}
func (suite *StatusTestSuite) TestGetStatusWithMention() {
@ -83,6 +95,10 @@ func (suite *StatusTestSuite) TestGetStatusWithMention() {
suite.NotEmpty(status.MentionIDs)
suite.NotEmpty(status.InReplyToID)
suite.NotEmpty(status.InReplyToAccountID)
suite.True(status.Federated)
suite.True(status.Boostable)
suite.True(status.Replyable)
suite.True(status.Likeable)
}
func (suite *StatusTestSuite) TestGetStatusTwice() {
@ -104,6 +120,18 @@ func (suite *StatusTestSuite) TestGetStatusTwice() {
suite.Less(duration2, duration1)
}
func (suite *StatusTestSuite) TestGetStatusChildren() {
targetStatus := suite.testStatuses["local_account_1_status_1"]
children, err := suite.db.GetStatusChildren(context.Background(), targetStatus, true, "")
suite.NoError(err)
suite.Len(children, 2)
for _, c := range children {
suite.Equal(targetStatus.URI, c.InReplyToURI)
suite.Equal(targetStatus.AccountID, c.InReplyToAccountID)
suite.Equal(targetStatus.ID, c.InReplyToID)
}
}
func TestStatusTestSuite(t *testing.T) {
suite.Run(t, new(StatusTestSuite))
}

View file

@ -19,6 +19,7 @@
package bundb
import (
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/uptrace/bun"
)
@ -35,3 +36,65 @@ func whereEmptyOrNull(column string) func(*bun.SelectQuery) *bun.SelectQuery {
WhereOr("? = ''", bun.Ident(column))
}
}
// updateWhere parses []db.Where and adds it to the given update query.
func updateWhere(q *bun.UpdateQuery, where []db.Where) {
for _, w := range where {
query, args := parseWhere(w)
q = q.Where(query, args...)
}
}
// selectWhere parses []db.Where and adds it to the given select query.
func selectWhere(q *bun.SelectQuery, where []db.Where) {
for _, w := range where {
query, args := parseWhere(w)
q = q.Where(query, args...)
}
}
// deleteWhere parses []db.Where and adds it to the given where query.
func deleteWhere(q *bun.DeleteQuery, where []db.Where) {
for _, w := range where {
query, args := parseWhere(w)
q = q.Where(query, args...)
}
}
// parseWhere looks through the options on a single db.Where entry, and
// returns the appropriate query string and arguments.
func parseWhere(w db.Where) (query string, args []interface{}) {
if w.Not {
if w.Value == nil {
query = "? IS NOT NULL"
args = []interface{}{bun.Ident(w.Key)}
return
}
if w.CaseInsensitive {
query = "LOWER(?) != LOWER(?)"
args = []interface{}{bun.Safe(w.Key), w.Value}
return
}
query = "? != ?"
args = []interface{}{bun.Safe(w.Key), w.Value}
return
}
if w.Value == nil {
query = "? IS NULL"
args = []interface{}{bun.Ident(w.Key)}
return
}
if w.CaseInsensitive {
query = "LOWER(?) = LOWER(?)"
args = []interface{}{bun.Safe(w.Key), w.Value}
return
}
query = "? = ?"
args = []interface{}{bun.Safe(w.Key), w.Value}
return
}

View file

@ -22,9 +22,13 @@ package db
type Where struct {
// The table to search on.
Key string
// The value that must be set.
// The value to match.
Value interface{}
// Whether the value (if a string) should be case sensitive or not.
// Defaults to false.
CaseInsensitive bool
// If set, reverse the where.
// `WHERE k = v` becomes `WHERE k != v`.
// `WHERE k IS NULL` becomes `WHERE k IS NOT NULL`
Not bool
}

View file

@ -60,7 +60,10 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat
announce.BoostOfID = boostedStatus.ID
announce.BoostOfAccountID = boostedStatus.AccountID
announce.Visibility = boostedStatus.Visibility
announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
announce.Federated = boostedStatus.Federated
announce.Boostable = boostedStatus.Boostable
announce.Replyable = boostedStatus.Replyable
announce.Likeable = boostedStatus.Likeable
announce.BoostOf = boostedStatus
return nil
}

View file

@ -58,6 +58,10 @@ func (suite *StatusTestSuite) TestDereferenceSimpleStatus() {
dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI)
suite.NoError(err)
suite.Equal(status.ID, dbStatus.ID)
suite.True(dbStatus.Federated)
suite.True(dbStatus.Boostable)
suite.True(dbStatus.Replyable)
suite.True(dbStatus.Likeable)
// account should be in the database now too
account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI)
@ -96,6 +100,10 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() {
dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI)
suite.NoError(err)
suite.Equal(status.ID, dbStatus.ID)
suite.True(dbStatus.Federated)
suite.True(dbStatus.Boostable)
suite.True(dbStatus.Replyable)
suite.True(dbStatus.Likeable)
// account should be in the database now too
account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI)

View file

@ -57,10 +57,13 @@ type Status struct {
Language string `validate:"-" bun:",nullzero"` // what language is this status written in?
CreatedWithApplicationID string `validate:"required_if=Local true,omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Which application was used to create this status?
CreatedWithApplication *Application `validate:"-" bun:"rel:belongs-to"` // application corresponding to createdWithApplicationID
VisibilityAdvanced VisibilityAdvanced `validate:"required" bun:",nullzero,notnull" ` // advanced visibility for this status
ActivityStreamsType string `validate:"required" bun:",nullzero,notnull"` // What is the activitystreams type of this status? See: https://www.w3.org/TR/activitystreams-vocabulary/#object-types. Will probably almost always be Note but who knows!.
Text string `validate:"-" bun:",nullzero"` // Original text of the status without formatting
Pinned bool `validate:"-" bun:",notnull,default:false" ` // Has this status been pinned by its owner?
Pinned bool `validate:"-" bun:",notnull,default:false"` // Has this status been pinned by its owner?
Federated bool `validate:"-" bun:",notnull"` // This status will be federated beyond the local timeline(s)
Boostable bool `validate:"-" bun:",notnull"` // This status can be boosted/reblogged
Replyable bool `validate:"-" bun:",notnull"` // This status can be replied to
Likeable bool `validate:"-" bun:",notnull"` // This status can be liked/faved
}
// StatusToTag is an intermediate struct to facilitate the many2many relationship between a status and one or more tags.
@ -96,21 +99,3 @@ const (
// VisibilityDefault is used when no other setting can be found.
VisibilityDefault Visibility = VisibilityUnlocked
)
// VisibilityAdvanced models flags for fine-tuning visibility and interactivity of a status.
//
// All flags default to true.
//
// If PUBLIC is selected, flags will all be overwritten to TRUE regardless of what is selected.
//
// If UNLOCKED is selected, any flags can be turned on or off in any combination.
//
// If FOLLOWERS-ONLY or MUTUALS-ONLY are selected, boostable will always be FALSE. Other flags can be turned on or off as desired.
//
// If DIRECT is selected, boostable will be FALSE, and all other flags will be TRUE.
type VisibilityAdvanced struct {
Federated bool `validate:"-" bun:",notnull,default:true"` // This status will be federated beyond the local timeline(s)
Boostable bool `validate:"-" bun:",notnull,default:true"` // This status can be boosted/reblogged
Replyable bool `validate:"-" bun:",notnull,default:true"` // This status can be replied to
Likeable bool `validate:"-" bun:",notnull,default:true"` // This status can be liked/faved
}

View file

@ -51,7 +51,7 @@ func (p *processor) ProcessFromClientAPI(ctx context.Context, clientMsg messages
return err
}
if status.VisibilityAdvanced.Federated {
if status.Federated {
return p.federateStatus(ctx, status)
}
case ap.ActivityFollow:

View file

@ -46,7 +46,7 @@ func (p *processor) Boost(ctx context.Context, requestingAccount *gtsmodel.Accou
if !visible {
return nil, gtserror.NewErrorNotFound(errors.New("status is not visible"))
}
if !targetStatus.VisibilityAdvanced.Boostable {
if !targetStatus.Boostable {
return nil, gtserror.NewErrorForbidden(errors.New("status is not boostable"))
}

View file

@ -49,7 +49,7 @@ func (p *processor) Fave(ctx context.Context, requestingAccount *gtsmodel.Accoun
if !visible {
return nil, gtserror.NewErrorNotFound(errors.New("status is not visible"))
}
if !targetStatus.VisibilityAdvanced.Likeable {
if !targetStatus.Likeable {
return nil, gtserror.NewErrorForbidden(errors.New("status is not faveable"))
}

View file

@ -33,12 +33,10 @@ import (
func (p *processor) ProcessVisibility(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error {
// by default all flags are set to true
gtsAdvancedVis := gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
}
federated := true
boostable := true
replyable := true
likeable := true
var vis gtsmodel.Visibility
// If visibility isn't set on the form, then just take the account default.
@ -58,47 +56,50 @@ func (p *processor) ProcessVisibility(ctx context.Context, form *apimodel.Advanc
case gtsmodel.VisibilityUnlocked:
// for unlocked the user can set any combination of flags they like so look at them all to see if they're set and then apply them
if form.Federated != nil {
gtsAdvancedVis.Federated = *form.Federated
federated = *form.Federated
}
if form.Boostable != nil {
gtsAdvancedVis.Boostable = *form.Boostable
boostable = *form.Boostable
}
if form.Replyable != nil {
gtsAdvancedVis.Replyable = *form.Replyable
replyable = *form.Replyable
}
if form.Likeable != nil {
gtsAdvancedVis.Likeable = *form.Likeable
likeable = *form.Likeable
}
case gtsmodel.VisibilityFollowersOnly, gtsmodel.VisibilityMutualsOnly:
// for followers or mutuals only, boostable will *always* be false, but the other fields can be set so check and apply them
gtsAdvancedVis.Boostable = false
boostable = false
if form.Federated != nil {
gtsAdvancedVis.Federated = *form.Federated
federated = *form.Federated
}
if form.Replyable != nil {
gtsAdvancedVis.Replyable = *form.Replyable
replyable = *form.Replyable
}
if form.Likeable != nil {
gtsAdvancedVis.Likeable = *form.Likeable
likeable = *form.Likeable
}
case gtsmodel.VisibilityDirect:
// direct is pretty easy: there's only one possible setting so return it
gtsAdvancedVis.Federated = true
gtsAdvancedVis.Boostable = false
gtsAdvancedVis.Federated = true
gtsAdvancedVis.Likeable = true
federated = true
boostable = false
replyable = true
likeable = true
}
status.Visibility = vis
status.VisibilityAdvanced = gtsAdvancedVis
status.Federated = federated
status.Boostable = boostable
status.Replyable = replyable
status.Likeable = likeable
return nil
}
@ -123,7 +124,7 @@ func (p *processor) ProcessReplyToID(ctx context.Context, form *apimodel.Advance
}
return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err)
}
if !repliedStatus.VisibilityAdvanced.Replyable {
if !repliedStatus.Replyable {
return fmt.Errorf("status with id %s is marked as not replyable", form.InReplyToID)
}

View file

@ -73,8 +73,8 @@ func (suite *GetTestSuite) TestGetDefault() {
suite.FailNow(err.Error())
}
// we only have 12 statuses in the test suite
suite.Len(statuses, 12)
// we only have 13 statuses in the test suite
suite.Len(statuses, 13)
// statuses should be sorted highest to lowest ID
var highest string
@ -166,8 +166,8 @@ func (suite *GetTestSuite) TestGetMinID() {
suite.FailNow(err.Error())
}
// we should only get 5 statuses back, since we asked for a min ID that excludes some of our entries
suite.Len(statuses, 5)
// we should only get 6 statuses back, since we asked for a min ID that excludes some of our entries
suite.Len(statuses, 6)
// statuses should be sorted highest to lowest ID
var highest string
@ -188,8 +188,8 @@ func (suite *GetTestSuite) TestGetSinceID() {
suite.FailNow(err.Error())
}
// we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries
suite.Len(statuses, 5)
// we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries
suite.Len(statuses, 6)
// statuses should be sorted highest to lowest ID
var highest string
@ -210,8 +210,8 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() {
suite.FailNow(err.Error())
}
// we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries
suite.Len(statuses, 5)
// we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries
suite.Len(statuses, 6)
// statuses should be sorted highest to lowest ID
var highest string

View file

@ -66,7 +66,7 @@ func (suite *IndexTestSuite) TestIndexBeforeLowID() {
// the oldest indexed post should be the lowest one we have in our testrig
postID, err := suite.timeline.OldestIndexedPostID(context.Background())
suite.NoError(err)
suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", postID)
suite.Equal("01F8MHAMCHF6Y650WCRSCP4WMY", postID)
indexLength := suite.timeline.PostIndexLength(context.Background())
suite.Equal(10, indexLength)
@ -95,7 +95,7 @@ func (suite *IndexTestSuite) TestIndexBehindHighID() {
// the newest indexed post should be the highest one we have in our testrig
postID, err := suite.timeline.NewestIndexedPostID(context.Background())
suite.NoError(err)
suite.Equal("01FCTA44PW9H1TB328S9AQXKDS", postID)
suite.Equal("01FF25D5Q0DH7CHD57CTRS6WK0", postID)
// indexLength should be 10 because that's all this user has hometimelineable
indexLength := suite.timeline.PostIndexLength(context.Background())

View file

@ -67,9 +67,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
err = suite.manager.PrepareXFromTop(context.Background(), testAccount.ID, 20)
suite.NoError(err)
// local_account_1 can see 12 statuses out of the testrig statuses in its home timeline
// local_account_1 can see 13 statuses out of the testrig statuses in its home timeline
indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(12, indexedLen)
suite.Equal(13, indexedLen)
// oldest should now be set
oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
@ -79,7 +79,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
// get hometimeline
statuses, err := suite.manager.HomeTimeline(context.Background(), testAccount.ID, "", "", "", 20, false)
suite.NoError(err)
suite.Len(statuses, 12)
suite.Len(statuses, 13)
// now wipe the last status from all timelines, as though it had been deleted by the owner
err = suite.manager.WipeStatusFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R")
@ -87,7 +87,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
// timeline should be shorter
indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(11, indexedLen)
suite.Equal(12, indexedLen)
// oldest should now be different
oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
@ -101,7 +101,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
// timeline should be shorter
indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(10, indexedLen)
suite.Equal(11, indexedLen)
// oldest should now be different
oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
@ -112,9 +112,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
err = suite.manager.WipeStatusesFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID)
suite.NoError(err)
// timeline should be empty now
// timeline should be shorter
indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(5, indexedLen)
suite.Equal(6, indexedLen)
// ingest 1 into the timeline
status1 := suite.testStatuses["admin_account_status_1"]
@ -130,7 +130,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
// timeline should be longer now
indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(7, indexedLen)
suite.Equal(8, indexedLen)
// try to ingest status 2 again
ingested, err = suite.manager.IngestAndPrepare(context.Background(), status2, testAccount.ID)

138
internal/trans/decoders.go Normal file
View file

@ -0,0 +1,138 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"time"
"github.com/mitchellh/mapstructure"
transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
)
func newDecoder(target interface{}) (*mapstructure.Decoder, error) {
decoderConfig := &mapstructure.DecoderConfig{
DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), // this is needed to decode time.Time entries serialized as string
Result: target,
}
return mapstructure.NewDecoder(decoderConfig)
}
func (i *importer) accountDecode(e transmodel.Entry) (*transmodel.Account, error) {
a := &transmodel.Account{}
if err := i.simpleDecode(e, a); err != nil {
return nil, err
}
// extract public key
publicKeyBlock, _ := pem.Decode([]byte(a.PublicKeyString))
if publicKeyBlock == nil {
return nil, errors.New("accountDecode: error decoding account public key")
}
publicKey, err := x509.ParsePKCS1PublicKey(publicKeyBlock.Bytes)
if err != nil {
return nil, fmt.Errorf("accountDecode: error parsing account public key: %s", err)
}
a.PublicKey = publicKey
if a.Domain == "" {
// extract private key (local account)
privateKeyBlock, _ := pem.Decode([]byte(a.PrivateKeyString))
if privateKeyBlock == nil {
return nil, errors.New("accountDecode: error decoding account private key")
}
privateKey, err := x509.ParsePKCS1PrivateKey(privateKeyBlock.Bytes)
if err != nil {
return nil, fmt.Errorf("accountDecode: error parsing account private key: %s", err)
}
a.PrivateKey = privateKey
}
return a, nil
}
func (i *importer) blockDecode(e transmodel.Entry) (*transmodel.Block, error) {
b := &transmodel.Block{}
if err := i.simpleDecode(e, b); err != nil {
return nil, err
}
return b, nil
}
func (i *importer) domainBlockDecode(e transmodel.Entry) (*transmodel.DomainBlock, error) {
b := &transmodel.DomainBlock{}
if err := i.simpleDecode(e, b); err != nil {
return nil, err
}
return b, nil
}
func (i *importer) followDecode(e transmodel.Entry) (*transmodel.Follow, error) {
f := &transmodel.Follow{}
if err := i.simpleDecode(e, f); err != nil {
return nil, err
}
return f, nil
}
func (i *importer) followRequestDecode(e transmodel.Entry) (*transmodel.FollowRequest, error) {
f := &transmodel.FollowRequest{}
if err := i.simpleDecode(e, f); err != nil {
return nil, err
}
return f, nil
}
func (i *importer) instanceDecode(e transmodel.Entry) (*transmodel.Instance, error) {
inst := &transmodel.Instance{}
if err := i.simpleDecode(e, inst); err != nil {
return nil, err
}
return inst, nil
}
func (i *importer) userDecode(e transmodel.Entry) (*transmodel.User, error) {
u := &transmodel.User{}
if err := i.simpleDecode(e, u); err != nil {
return nil, err
}
return u, nil
}
func (i *importer) simpleDecode(entry transmodel.Entry, target interface{}) error {
decoder, err := newDecoder(target)
if err != nil {
return fmt.Errorf("simpleDecode: error creating decoder: %s", err)
}
if err := decoder.Decode(&entry); err != nil {
return fmt.Errorf("simpleDecode: error decoding: %s", err)
}
return nil
}

View file

@ -0,0 +1,83 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"context"
"crypto/x509"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"os"
transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
)
// accountEncode handles special fields like private + public keys on accounts
func (e *exporter) accountEncode(ctx context.Context, f *os.File, a *transmodel.Account) error {
a.Type = transmodel.TransAccount
// marshal public key
encodedPublicKey := x509.MarshalPKCS1PublicKey(a.PublicKey)
if encodedPublicKey == nil {
return errors.New("could not MarshalPKCS1PublicKey")
}
publicKeyBytes := pem.EncodeToMemory(&pem.Block{
Type: "RSA PUBLIC KEY",
Bytes: encodedPublicKey,
})
a.PublicKeyString = string(publicKeyBytes)
if a.Domain == "" {
// marshal private key for local account
encodedPrivateKey := x509.MarshalPKCS1PrivateKey(a.PrivateKey)
if encodedPrivateKey == nil {
return errors.New("could not MarshalPKCS1PrivateKey")
}
privateKeyBytes := pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: encodedPrivateKey,
})
a.PrivateKeyString = string(privateKeyBytes)
}
return e.simpleEncode(ctx, f, a, a.ID)
}
// simpleEncode can be used for any type that doesn't have special keys which need handling differently,
// or for types where special keys have already been handled.
//
// Beware, the 'type' key on the passed interface should already have been set, since simpleEncode won't know
// what type it is! If you try to decode stuff you've encoded with a missing type key, you're going to have a bad time.
func (e *exporter) simpleEncode(ctx context.Context, file *os.File, i interface{}, id string) error {
_, alreadyWritten := e.writtenIDs[id]
if alreadyWritten {
// this exporter has already exported an entry with this ID, no need to do it twice
return nil
}
err := json.NewEncoder(file).Encode(i)
if err != nil {
return fmt.Errorf("simpleEncode: error encoding entry with id %s: %s", id, err)
}
e.writtenIDs[id] = true
return nil
}

223
internal/trans/export.go Normal file
View file

@ -0,0 +1,223 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/package trans
import (
"context"
"fmt"
"os"
"github.com/superseriousbusiness/gotosocial/internal/db"
transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
)
func (e *exporter) exportAccounts(ctx context.Context, where []db.Where, file *os.File) ([]*transmodel.Account, error) {
// select using the 'where' we've been provided
accounts := []*transmodel.Account{}
if err := e.db.GetWhere(ctx, where, &accounts); err != nil {
return nil, fmt.Errorf("exportAccounts: error selecting accounts: %s", err)
}
// write any accounts found to file
for _, a := range accounts {
if err := e.accountEncode(ctx, file, a); err != nil {
return nil, fmt.Errorf("exportAccounts: error encoding account: %s", err)
}
}
return accounts, nil
}
func (e *exporter) exportBlocks(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.Block, error) {
blocksUnique := make(map[string]*transmodel.Block)
// for each account we want to export both where it's blocking and where it's blocked
for _, a := range accounts {
// 1. export blocks owned by given account
whereBlocking := []db.Where{{Key: "account_id", Value: a.ID}}
blocking := []*transmodel.Block{}
if err := e.db.GetWhere(ctx, whereBlocking, &blocking); err != nil {
return nil, fmt.Errorf("exportBlocks: error selecting blocks owned by account %s: %s", a.ID, err)
}
for _, b := range blocking {
b.Type = transmodel.TransBlock
if err := e.simpleEncode(ctx, file, b, b.ID); err != nil {
return nil, fmt.Errorf("exportBlocks: error encoding block owned by account %s: %s", a.ID, err)
}
blocksUnique[b.ID] = b
}
// 2. export blocks that target given account
whereBlocked := []db.Where{{Key: "target_account_id", Value: a.ID}}
blocked := []*transmodel.Block{}
if err := e.db.GetWhere(ctx, whereBlocked, &blocked); err != nil {
return nil, fmt.Errorf("exportBlocks: error selecting blocks targeting account %s: %s", a.ID, err)
}
for _, b := range blocked {
b.Type = transmodel.TransBlock
if err := e.simpleEncode(ctx, file, b, b.ID); err != nil {
return nil, fmt.Errorf("exportBlocks: error encoding block targeting account %s: %s", a.ID, err)
}
blocksUnique[b.ID] = b
}
}
// now return all the blocks we found
blocks := []*transmodel.Block{}
for _, b := range blocksUnique {
blocks = append(blocks, b)
}
return blocks, nil
}
func (e *exporter) exportDomainBlocks(ctx context.Context, file *os.File) ([]*transmodel.DomainBlock, error) {
domainBlocks := []*transmodel.DomainBlock{}
if err := e.db.GetAll(ctx, &domainBlocks); err != nil {
return nil, fmt.Errorf("exportBlocks: error selecting domain blocks: %s", err)
}
for _, b := range domainBlocks {
b.Type = transmodel.TransDomainBlock
if err := e.simpleEncode(ctx, file, b, b.ID); err != nil {
return nil, fmt.Errorf("exportBlocks: error encoding domain block: %s", err)
}
}
return domainBlocks, nil
}
func (e *exporter) exportFollows(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.Follow, error) {
followsUnique := make(map[string]*transmodel.Follow)
// for each account we want to export both where it's following and where it's followed
for _, a := range accounts {
// 1. export follows owned by given account
whereFollowing := []db.Where{{Key: "account_id", Value: a.ID}}
following := []*transmodel.Follow{}
if err := e.db.GetWhere(ctx, whereFollowing, &following); err != nil {
return nil, fmt.Errorf("exportFollows: error selecting follows owned by account %s: %s", a.ID, err)
}
for _, follow := range following {
follow.Type = transmodel.TransFollow
if err := e.simpleEncode(ctx, file, follow, follow.ID); err != nil {
return nil, fmt.Errorf("exportFollows: error encoding follow owned by account %s: %s", a.ID, err)
}
followsUnique[follow.ID] = follow
}
// 2. export follows that target given account
whereFollowed := []db.Where{{Key: "target_account_id", Value: a.ID}}
followed := []*transmodel.Follow{}
if err := e.db.GetWhere(ctx, whereFollowed, &followed); err != nil {
return nil, fmt.Errorf("exportFollows: error selecting follows targeting account %s: %s", a.ID, err)
}
for _, follow := range followed {
follow.Type = transmodel.TransFollow
if err := e.simpleEncode(ctx, file, follow, follow.ID); err != nil {
return nil, fmt.Errorf("exportFollows: error encoding follow targeting account %s: %s", a.ID, err)
}
followsUnique[follow.ID] = follow
}
}
// now return all the follows we found
follows := []*transmodel.Follow{}
for _, follow := range followsUnique {
follows = append(follows, follow)
}
return follows, nil
}
func (e *exporter) exportFollowRequests(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.FollowRequest, error) {
frsUnique := make(map[string]*transmodel.FollowRequest)
// for each account we want to export both where it's following and where it's followed
for _, a := range accounts {
// 1. export follow requests owned by given account
whereRequesting := []db.Where{{Key: "account_id", Value: a.ID}}
requesting := []*transmodel.FollowRequest{}
if err := e.db.GetWhere(ctx, whereRequesting, &requesting); err != nil {
return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests owned by account %s: %s", a.ID, err)
}
for _, fr := range requesting {
fr.Type = transmodel.TransFollowRequest
if err := e.simpleEncode(ctx, file, fr, fr.ID); err != nil {
return nil, fmt.Errorf("exportFollowRequests: error encoding follow request owned by account %s: %s", a.ID, err)
}
frsUnique[fr.ID] = fr
}
// 2. export follow requests that target given account
whereRequested := []db.Where{{Key: "target_account_id", Value: a.ID}}
requested := []*transmodel.FollowRequest{}
if err := e.db.GetWhere(ctx, whereRequested, &requested); err != nil {
return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests targeting account %s: %s", a.ID, err)
}
for _, fr := range requested {
fr.Type = transmodel.TransFollowRequest
if err := e.simpleEncode(ctx, file, fr, fr.ID); err != nil {
return nil, fmt.Errorf("exportFollowRequests: error encoding follow request targeting account %s: %s", a.ID, err)
}
frsUnique[fr.ID] = fr
}
}
// now return all the followRequests we found
followRequests := []*transmodel.FollowRequest{}
for _, fr := range frsUnique {
followRequests = append(followRequests, fr)
}
return followRequests, nil
}
func (e *exporter) exportInstances(ctx context.Context, file *os.File) ([]*transmodel.Instance, error) {
instances := []*transmodel.Instance{}
if err := e.db.GetAll(ctx, &instances); err != nil {
return nil, fmt.Errorf("exportInstances: error selecting instance: %s", err)
}
for _, u := range instances {
u.Type = transmodel.TransInstance
if err := e.simpleEncode(ctx, file, u, u.ID); err != nil {
return nil, fmt.Errorf("exportInstances: error encoding instance: %s", err)
}
}
return instances, nil
}
func (e *exporter) exportUsers(ctx context.Context, file *os.File) ([]*transmodel.User, error) {
users := []*transmodel.User{}
if err := e.db.GetAll(ctx, &users); err != nil {
return nil, fmt.Errorf("exportUsers: error selecting users: %s", err)
}
for _, u := range users {
u.Type = transmodel.TransUser
if err := e.simpleEncode(ctx, file, u, u.ID); err != nil {
return nil, fmt.Errorf("exportUsers: error encoding user: %s", err)
}
}
return users, nil
}

View file

@ -0,0 +1,46 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"context"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
)
// Exporter wraps functionality for exporting entries from the database to a file.
type Exporter interface {
ExportMinimal(ctx context.Context, path string) error
}
type exporter struct {
db db.DB
log *logrus.Logger
writtenIDs map[string]bool
}
// NewExporter returns a new Exporter that will use the given db and logger.
func NewExporter(db db.DB, log *logrus.Logger) Exporter {
return &exporter{
db: db,
log: log,
writtenIDs: make(map[string]bool),
}
}

View file

@ -0,0 +1,150 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"context"
"errors"
"fmt"
"os"
"github.com/superseriousbusiness/gotosocial/internal/db"
)
func (e *exporter) ExportMinimal(ctx context.Context, path string) error {
if path == "" {
return errors.New("ExportMinimal: path empty")
}
file, err := os.Create(path)
if err != nil {
return fmt.Errorf("ExportMinimal: couldn't export to %s: %s", path, err)
}
// export all local accounts we have in the database
localAccounts, err := e.exportAccounts(ctx, []db.Where{{Key: "domain", Value: nil}}, file)
if err != nil {
return fmt.Errorf("ExportMinimal: error exporting accounts: %s", err)
}
// export all blocks that relate to local accounts
blocks, err := e.exportBlocks(ctx, localAccounts, file)
if err != nil {
return fmt.Errorf("ExportMinimal: error exporting blocks: %s", err)
}
// for each block, make sure we've written out the account owning it, or targeted by it --
// this might include non-local accounts, but we need these so we don't lose anything
for _, b := range blocks {
_, alreadyWritten := e.writtenIDs[b.AccountID]
if !alreadyWritten {
_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.AccountID}}, file)
if err != nil {
return fmt.Errorf("ExportMinimal: error exporting block owner account: %s", err)
}
}
_, alreadyWritten = e.writtenIDs[b.TargetAccountID]
if !alreadyWritten {
_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.TargetAccountID}}, file)
if err != nil {
return fmt.Errorf("ExportMinimal: error exporting block target account: %s", err)
}
}
}
// export all follows that relate to local accounts
follows, err := e.exportFollows(ctx, localAccounts, file)
if err != nil {
return fmt.Errorf("ExportMinimal: error exporting follows: %s", err)
}
// for each follow, make sure we've written out the account owning it, or targeted by it --
// this might include non-local accounts, but we need these so we don't lose anything
for _, follow := range follows {
_, alreadyWritten := e.writtenIDs[follow.AccountID]
if !alreadyWritten {
_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.AccountID}}, file)
if err != nil {
return fmt.Errorf("ExportMinimal: error exporting follow owner account: %s", err)
}
}
_, alreadyWritten = e.writtenIDs[follow.TargetAccountID]
if !alreadyWritten {
_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.TargetAccountID}}, file)
if err != nil {
return fmt.Errorf("ExportMinimal: error exporting follow target account: %s", err)
}
}
}
// export all follow requests that relate to local accounts
followRequests, err := e.exportFollowRequests(ctx, localAccounts, file)
if err != nil {
return fmt.Errorf("ExportMinimal: error exporting follow requests: %s", err)
}
// for each follow request, make sure we've written out the account owning it, or targeted by it --
// this might include non-local accounts, but we need these so we don't lose anything
for _, fr := range followRequests {
_, alreadyWritten := e.writtenIDs[fr.AccountID]
if !alreadyWritten {
_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.AccountID}}, file)
if err != nil {
return fmt.Errorf("ExportMinimal: error exporting follow request owner account: %s", err)
}
}
_, alreadyWritten = e.writtenIDs[fr.TargetAccountID]
if !alreadyWritten {
_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.TargetAccountID}}, file)
if err != nil {
return fmt.Errorf("ExportMinimal: error exporting follow request target account: %s", err)
}
}
}
// export all domain blocks
if _, err := e.exportDomainBlocks(ctx, file); err != nil {
return fmt.Errorf("ExportMinimal: error exporting domain blocks: %s", err)
}
// export all users
if _, err := e.exportUsers(ctx, file); err != nil {
return fmt.Errorf("ExportMinimal: error exporting users: %s", err)
}
// export all instances
if _, err := e.exportInstances(ctx, file); err != nil {
return fmt.Errorf("ExportMinimal: error exporting instances: %s", err)
}
// export all SUSPENDED accounts to make sure the suspension sticks across db migration etc
whereSuspended := []db.Where{{
Key: "suspended_at",
Not: true,
Value: nil,
}}
if _, err := e.exportAccounts(ctx, whereSuspended, file); err != nil {
return fmt.Errorf("ExportMinimal: error exporting suspended accounts: %s", err)
}
return neatClose(file)
}

View file

@ -0,0 +1,54 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans_test
import (
"context"
"fmt"
"os"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/trans"
)
type ExportMinimalTestSuite struct {
TransTestSuite
}
func (suite *ExportMinimalTestSuite) TestExportMinimalOK() {
// use a temporary file path that will be cleaned when the test is closed
tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString())
// export to the tempFilePath
exporter := trans.NewExporter(suite.db, suite.log)
err := exporter.ExportMinimal(context.Background(), tempFilePath)
suite.NoError(err)
// we should have some bytes in that file now
b, err := os.ReadFile(tempFilePath)
suite.NoError(err)
suite.NotEmpty(b)
fmt.Println(string(b))
}
func TestExportMinimalTestSuite(t *testing.T) {
suite.Run(t, &ExportMinimalTestSuite{})
}

146
internal/trans/import.go Normal file
View file

@ -0,0 +1,146 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
)
func (i *importer) Import(ctx context.Context, path string) error {
if path == "" {
return errors.New("Export: path empty")
}
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("Import: couldn't export to %s: %s", path, err)
}
decoder := json.NewDecoder(file)
decoder.UseNumber()
for {
entry := transmodel.Entry{}
err := decoder.Decode(&entry)
if err != nil {
if err == io.EOF {
i.log.Infof("Import: reached end of file")
return neatClose(file)
}
return fmt.Errorf("Import: error decoding in readLoop: %s", err)
}
if err := i.inputEntry(ctx, entry); err != nil {
return fmt.Errorf("Import: error inputting entry: %s", err)
}
}
}
func (i *importer) inputEntry(ctx context.Context, entry transmodel.Entry) error {
t, ok := entry[transmodel.TypeKey].(string)
if !ok {
return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json")
}
switch transmodel.Type(t) {
case transmodel.TransAccount:
account, err := i.accountDecode(entry)
if err != nil {
return fmt.Errorf("inputEntry: error decoding entry into account: %s", err)
}
if err := i.putInDB(ctx, account); err != nil {
return fmt.Errorf("inputEntry: error adding account to database: %s", err)
}
i.log.Infof("inputEntry: added account with id %s", account.ID)
return nil
case transmodel.TransBlock:
block, err := i.blockDecode(entry)
if err != nil {
return fmt.Errorf("inputEntry: error decoding entry into block: %s", err)
}
if err := i.putInDB(ctx, block); err != nil {
return fmt.Errorf("inputEntry: error adding block to database: %s", err)
}
i.log.Infof("inputEntry: added block with id %s", block.ID)
return nil
case transmodel.TransDomainBlock:
block, err := i.domainBlockDecode(entry)
if err != nil {
return fmt.Errorf("inputEntry: error decoding entry into domain block: %s", err)
}
if err := i.putInDB(ctx, block); err != nil {
return fmt.Errorf("inputEntry: error adding domain block to database: %s", err)
}
i.log.Infof("inputEntry: added domain block with id %s", block.ID)
return nil
case transmodel.TransFollow:
follow, err := i.followDecode(entry)
if err != nil {
return fmt.Errorf("inputEntry: error decoding entry into follow: %s", err)
}
if err := i.putInDB(ctx, follow); err != nil {
return fmt.Errorf("inputEntry: error adding follow to database: %s", err)
}
i.log.Infof("inputEntry: added follow with id %s", follow.ID)
return nil
case transmodel.TransFollowRequest:
fr, err := i.followRequestDecode(entry)
if err != nil {
return fmt.Errorf("inputEntry: error decoding entry into follow request: %s", err)
}
if err := i.putInDB(ctx, fr); err != nil {
return fmt.Errorf("inputEntry: error adding follow request to database: %s", err)
}
i.log.Infof("inputEntry: added follow request with id %s", fr.ID)
return nil
case transmodel.TransInstance:
inst, err := i.instanceDecode(entry)
if err != nil {
return fmt.Errorf("inputEntry: error decoding entry into instance: %s", err)
}
if err := i.putInDB(ctx, inst); err != nil {
return fmt.Errorf("inputEntry: error adding instance to database: %s", err)
}
i.log.Infof("inputEntry: added instance with id %s", inst.ID)
return nil
case transmodel.TransUser:
user, err := i.userDecode(entry)
if err != nil {
return fmt.Errorf("inputEntry: error decoding entry into user: %s", err)
}
if err := i.putInDB(ctx, user); err != nil {
return fmt.Errorf("inputEntry: error adding user to database: %s", err)
}
i.log.Infof("inputEntry: added user with id %s", user.ID)
return nil
}
i.log.Errorf("inputEntry: didn't recognize transtype '%s', skipping it", t)
return nil
}
func (i *importer) putInDB(ctx context.Context, entry interface{}) error {
return i.db.Put(ctx, entry)
}

View file

@ -0,0 +1,91 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans_test
import (
"context"
"fmt"
"os"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/trans"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type ImportMinimalTestSuite struct {
TransTestSuite
}
func (suite *ImportMinimalTestSuite) TestImportMinimalOK() {
ctx := context.Background()
// use a temporary file path
tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString())
// export to the tempFilePath
exporter := trans.NewExporter(suite.db, suite.log)
err := exporter.ExportMinimal(ctx, tempFilePath)
suite.NoError(err)
// we should have some bytes in that file now
b, err := os.ReadFile(tempFilePath)
suite.NoError(err)
suite.NotEmpty(b)
fmt.Println(string(b))
// create a new database with just the tables created, no entries
testrig.StandardDBTeardown(suite.db)
newDB := testrig.NewTestDB()
testrig.CreateTestTables(newDB)
importer := trans.NewImporter(newDB, suite.log)
err = importer.Import(ctx, tempFilePath)
suite.NoError(err)
// we should have some accounts in the database
accounts := []*gtsmodel.Account{}
err = newDB.GetAll(ctx, &accounts)
suite.NoError(err)
suite.NotEmpty(accounts)
// we should have some blocks in the database
blocks := []*gtsmodel.Block{}
err = newDB.GetAll(ctx, &blocks)
suite.NoError(err)
suite.NotEmpty(blocks)
// we should have some follows in the database
follows := []*gtsmodel.Follow{}
err = newDB.GetAll(ctx, &follows)
suite.NoError(err)
suite.NotEmpty(follows)
// we should have some domain blocks in the database
domainBlocks := []*gtsmodel.DomainBlock{}
err = newDB.GetAll(ctx, &domainBlocks)
suite.NoError(err)
suite.NotEmpty(domainBlocks)
}
func TestImportMinimalTestSuite(t *testing.T) {
suite.Run(t, &ImportMinimalTestSuite{})
}

View file

@ -0,0 +1,44 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"context"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
)
// Importer wraps functionality for importing entries from a file into the database.
type Importer interface {
Import(ctx context.Context, path string) error
}
type importer struct {
db db.DB
log *logrus.Logger
}
// NewImporter returns a new Importer interface that uses the given db and logger.
func NewImporter(db db.DB, log *logrus.Logger) Importer {
return &importer{
db: db,
log: log,
}
}

View file

@ -0,0 +1,54 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"crypto/rsa"
"time"
)
// Account represents the minimum viable representation of an account for export/import.
type Account struct {
Type Type `json:"type" bun:"-"`
ID string `json:"id" bun:",nullzero"`
CreatedAt *time.Time `json:"createdAt" bun:",nullzero"`
Username string `json:"username" bun:",nullzero"`
DisplayName string `json:"displayName,omitempty" bun:",nullzero"`
Note string `json:"note,omitempty" bun:",nullzero"`
Domain string `json:"domain,omitempty" bun:",nullzero"`
HeaderRemoteURL string `json:"headerRemoteURL,omitempty" bun:",nullzero"`
AvatarRemoteURL string `json:"avatarRemoteURL,omitempty" bun:",nullzero"`
Locked bool `json:"locked"`
Language string `json:"language,omitempty" bun:",nullzero"`
URI string `json:"uri" bun:",nullzero"`
URL string `json:"url" bun:",nullzero"`
InboxURI string `json:"inboxURI" bun:",nullzero"`
OutboxURI string `json:"outboxURI" bun:",nullzero"`
FollowingURI string `json:"followingUri" bun:",nullzero"`
FollowersURI string `json:"followersUri" bun:",nullzero"`
FeaturedCollectionURI string `json:"featuredCollectionUri" bun:",nullzero"`
ActorType string `json:"actorType" bun:",nullzero"`
PrivateKey *rsa.PrivateKey `json:"-" mapstructure:"-"`
PrivateKeyString string `json:"privateKey,omitempty" mapstructure:"privateKey" bun:"-"`
PublicKey *rsa.PublicKey `json:"-" mapstructure:"-"`
PublicKeyString string `json:"publicKey,omitempty" mapstructure:"publicKey" bun:"-"`
PublicKeyURI string `json:"publicKeyUri" bun:",nullzero"`
SuspendedAt *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"`
SuspensionOrigin string `json:"suspensionOrigin,omitempty" bun:",nullzero"`
}

View file

@ -0,0 +1,31 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import "time"
// Block represents an account block as serialized in an exported file.
type Block struct {
Type Type `json:"type" bun:"-"`
ID string `json:"id" bun:",nullzero"`
CreatedAt *time.Time `json:"createdAt" bun:",nullzero"`
URI string `json:"uri" bun:",nullzero"`
AccountID string `json:"accountId" bun:",nullzero"`
TargetAccountID string `json:"targetAccountId" bun:",nullzero"`
}

View file

@ -0,0 +1,34 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import "time"
// DomainBlock represents a domain block as serialized in an exported file.
type DomainBlock struct {
Type Type `json:"type" bun:"-"`
ID string `json:"id" bun:",nullzero"`
CreatedAt *time.Time `json:"createdAt" bun:",nullzero"`
Domain string `json:"domain" bun:",nullzero"`
CreatedByAccountID string `json:"createdByAccountID" bun:",nullzero"`
PrivateComment string `json:"privateComment,omitempty" bun:",nullzero"`
PublicComment string `json:"publicComment,omitempty" bun:",nullzero"`
Obfuscate bool `json:"obfuscate" bun:",nullzero"`
SubscriptionID string `json:"subscriptionID,omitempty" bun:",nullzero"`
}

View file

@ -0,0 +1,31 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import "time"
// Follow represents an account follow as serialized in an export file.
type Follow struct {
Type Type `json:"type" bun:"-"`
ID string `json:"id" bun:",nullzero"`
CreatedAt *time.Time `json:"createdAt" bun:",nullzero"`
URI string `json:"uri" bun:",nullzero"`
AccountID string `json:"accountId" bun:",nullzero"`
TargetAccountID string `json:"targetAccountId" bun:",nullzero"`
}

View file

@ -0,0 +1,31 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import "time"
// FollowRequest represents an account follow request as serialized in an export file.
type FollowRequest struct {
Type Type `json:"type" bun:"-"`
ID string `json:"id" bun:",nullzero"`
CreatedAt *time.Time `json:"createdAt" bun:",nullzero"`
URI string `json:"uri" bun:",nullzero"`
AccountID string `json:"accountId" bun:",nullzero"`
TargetAccountID string `json:"targetAccountId" bun:",nullzero"`
}

View file

@ -0,0 +1,43 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"time"
)
// Instance represents an instance entry as serialized in an export file.
type Instance struct {
Type Type `json:"type" bun:"-"`
ID string `json:"id" bun:",nullzero"`
CreatedAt *time.Time `json:"createdAt" bun:",nullzero"`
Domain string `json:"domain" bun:",nullzero"`
Title string `json:"title,omitempty" bun:",nullzero"`
URI string `json:"uri" bun:",nullzero"`
SuspendedAt *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"`
DomainBlockID string `json:"domainBlockID,omitempty" bun:",nullzero"`
ShortDescription string `json:"shortDescription,omitempty" bun:",nullzero"`
Description string `json:"description,omitempty" bun:",nullzero"`
Terms string `json:"terms,omitempty" bun:",nullzero"`
ContactEmail string `json:"contactEmail,omitempty" bun:",nullzero"`
ContactAccountUsername string `json:"contactAccountUsername,omitempty" bun:",nullzero"`
ContactAccountID string `json:"contactAccountID,omitempty" bun:",nullzero"`
Reputation int64 `json:"reputation"`
Version string `json:"version,omitempty" bun:",nullzero"`
}

View file

@ -0,0 +1,41 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
// TypeKey should be set on a TransEntry to indicate the type of entry it is.
const TypeKey = "type"
// Type describes the type of a trans entry, and how it should be read/serialized.
type Type string
// Type of the trans entry. Describes how it should be read from file.
const (
TransAccount Type = "account"
TransBlock Type = "block"
TransDomainBlock Type = "domainBlock"
TransEmailDomainBlock Type = "emailDomainBlock"
TransFollow Type = "follow"
TransFollowRequest Type = "followRequest"
TransInstance Type = "instance"
TransUser Type = "user"
)
// Entry is used for deserializing trans entries into a rough interface so that
// the TypeKey can be fetched, before continuing with full parsing.
type Entry map[string]interface{}

View file

@ -0,0 +1,50 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"time"
)
// User represents a local instance user as serialized to an export file.
type User struct {
Type Type `json:"type" bun:"-"`
ID string `json:"id" bun:",nullzero"`
CreatedAt *time.Time `json:"createdAt" bun:",nullzero"`
Email string `json:"email,omitempty" bun:",nullzero"`
AccountID string `json:"accountID" bun:",nullzero"`
EncryptedPassword string `json:"encryptedPassword" bun:",nullzero"`
CurrentSignInAt *time.Time `json:"currentSignInAt,omitempty" bun:",nullzero"`
LastSignInAt *time.Time `json:"lastSignInAt,omitempty" bun:",nullzero"`
InviteID string `json:"inviteID,omitempty" bun:",nullzero"`
ChosenLanguages []string `json:"chosenLanguages,omitempty" bun:",nullzero"`
FilteredLanguages []string `json:"filteredLanguage,omitempty" bun:",nullzero"`
Locale string `json:"locale" bun:",nullzero"`
LastEmailedAt time.Time `json:"lastEmailedAt,omitempty" bun:",nullzero"`
ConfirmationToken string `json:"confirmationToken,omitempty" bun:",nullzero"`
ConfirmationSentAt *time.Time `json:"confirmationTokenSentAt,omitempty" bun:",nullzero"`
ConfirmedAt *time.Time `json:"confirmedAt,omitempty" bun:",nullzero"`
UnconfirmedEmail string `json:"unconfirmedEmail,omitempty" bun:",nullzero"`
Moderator bool `json:"moderator"`
Admin bool `json:"admin"`
Disabled bool `json:"disabled"`
Approved bool `json:"approved"`
ResetPasswordToken string `json:"resetPasswordToken,omitempty" bun:",nullzero"`
ResetPasswordSentAt *time.Time `json:"resetPasswordSentAt,omitempty" bun:",nullzero"`
}

View file

@ -0,0 +1,42 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans_test
import (
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type TransTestSuite struct {
suite.Suite
db db.DB
log *logrus.Logger
}
func (suite *TransTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.log = testrig.NewTestLog()
testrig.StandardDBSetup(suite.db, nil)
}
func (suite *TransTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
}

32
internal/trans/util.go Normal file
View file

@ -0,0 +1,32 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package trans
import (
"fmt"
"os"
)
func neatClose(f *os.File) error {
if err := f.Close(); err != nil {
return fmt.Errorf("error closing file: %s", err)
}
return nil
}

View file

@ -325,6 +325,11 @@ func (c *converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab
// advanced visibility for this status
// TODO: a lot of work to be done here -- a new type needs to be created for this in go-fed/activity using ASTOOL
// for now we just set everything to true
status.Federated = true
status.Boostable = true
status.Replyable = true
status.Likeable = true
// sensitive
// TODO: this is a bool

View file

@ -70,7 +70,10 @@ func (c *converter) StatusToBoost(ctx context.Context, s *gtsmodel.Status, boost
BoostOfID: s.ID,
BoostOfAccountID: s.AccountID,
Visibility: s.Visibility,
VisibilityAdvanced: s.VisibilityAdvanced,
Federated: s.Federated,
Boostable: s.Boostable,
Replyable: s.Replyable,
Likeable: s.Likeable,
// attach these here for convenience -- the boosted status/account won't go in the DB
// but they're needed in the processor and for the frontend. Since we have them, we can

View file

@ -63,12 +63,10 @@ func happyStatus() *gtsmodel.Status {
Language: "en",
CreatedWithApplicationID: "01FEBBZHF4GFVRXSJVXD0JTZZ2",
CreatedWithApplication: nil,
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
},
ActivityStreamsType: ap.ObjectNote,
Text: "Test status! #hello",
Pinned: false,

View file

@ -72,6 +72,16 @@ func NewTestDB() db.DB {
return testDB
}
// CreateTestTables creates prerequisite test tables in the database, but doesn't populate them.
func CreateTestTables(db db.DB) {
ctx := context.Background()
for _, m := range testModels {
if err := db.CreateTable(ctx, m); err != nil {
logrus.Panicf("error creating table for %+v: %s", m, err)
}
}
}
// StandardDBSetup populates a given db with all the necessary tables/models for perfoming tests.
//
// The accounts parameter is provided in case the db should be populated with a certain set of accounts.
@ -85,13 +95,9 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
logrus.Panic("db setup: db was nil")
}
ctx := context.Background()
CreateTestTables(db)
for _, m := range testModels {
if err := db.CreateTable(ctx, m); err != nil {
logrus.Panicf("error creating table for %+v: %s", m, err)
}
}
ctx := context.Background()
for _, v := range NewTestTokens() {
if err := db.Put(ctx, v); err != nil {
@ -111,6 +117,18 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
}
}
for _, v := range NewTestBlocks() {
if err := db.Put(ctx, v); err != nil {
logrus.Panic(err)
}
}
for _, v := range NewTestDomainBlocks() {
if err := db.Put(ctx, v); err != nil {
logrus.Panic(err)
}
}
for _, v := range NewTestUsers() {
if err := db.Put(ctx, v); err != nil {
logrus.Panic(err)

View file

@ -736,6 +736,19 @@ func NewTestEmojis() map[string]*gtsmodel.Emoji {
}
}
func NewTestDomainBlocks() map[string]*gtsmodel.DomainBlock {
return map[string]*gtsmodel.DomainBlock{
"replyguys.com": {
ID: "01FF22EQM7X8E3RX1XGPN7S87D",
Domain: "replyguys.com",
CreatedByAccountID: "01F8MH17FWEB39HZJ76B6VXSKF",
PrivateComment: "i blocked this domain because they keep replying with pushy + unwarranted linux advice",
PublicComment: "reply-guying to tech posts",
Obfuscate: false,
},
}
}
type filenames struct {
Original string
Small string
@ -803,12 +816,10 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: false,
Language: "en",
CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
},
ActivityStreamsType: ap.ObjectNote,
},
"admin_account_status_2": {
@ -828,12 +839,35 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: true,
Language: "en",
CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
ActivityStreamsType: ap.ObjectNote,
},
"admin_account_status_3": {
ID: "01FF25D5Q0DH7CHD57CTRS6WK0",
URI: "http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0",
URL: "http://localhost:8080/@admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0",
Content: "hi @the_mighty_zork welcome to the instance!",
CreatedAt: time.Now().Add(-46 * time.Hour),
UpdatedAt: time.Now().Add(-46 * time.Hour),
Local: true,
AccountURI: "http://localhost:8080/users/admin",
MentionIDs: []string{"01FF26A6BGEKCZFWNEHXB2ZZ6M"},
AccountID: "01F8MH17FWEB39HZJ76B6VXSKF",
InReplyToID: "01F8MHAMCHF6Y650WCRSCP4WMY",
InReplyToAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF",
InReplyToURI: "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
BoostOfID: "",
Visibility: gtsmodel.VisibilityPublic,
Sensitive: false,
Language: "en",
CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F",
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
ActivityStreamsType: ap.ObjectNote,
},
"local_account_1_status_1": {
@ -853,12 +887,10 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: true,
Language: "en",
CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
},
ActivityStreamsType: ap.ObjectNote,
},
"local_account_1_status_2": {
@ -878,12 +910,10 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: false,
Language: "en",
CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: false,
Boostable: true,
Replyable: true,
Likeable: true,
},
ActivityStreamsType: ap.ObjectNote,
},
"local_account_1_status_3": {
@ -903,12 +933,10 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: false,
Language: "en",
CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: false,
Replyable: false,
Likeable: false,
},
ActivityStreamsType: ap.ObjectNote,
},
"local_account_1_status_4": {
@ -929,12 +957,10 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: false,
Language: "en",
CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
},
ActivityStreamsType: ap.ObjectNote,
},
"local_account_1_status_5": {
@ -955,12 +981,10 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: false,
Language: "en",
CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
},
ActivityStreamsType: ap.ObjectNote,
},
"local_account_2_status_1": {
@ -980,12 +1004,10 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: true,
Language: "en",
CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
},
ActivityStreamsType: ap.ObjectNote,
},
"local_account_2_status_2": {
@ -1005,12 +1027,10 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: true,
Language: "en",
CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: false,
Likeable: true,
},
ActivityStreamsType: ap.ObjectNote,
},
"local_account_2_status_3": {
@ -1030,12 +1050,10 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: true,
Language: "en",
CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: false,
Likeable: false,
},
ActivityStreamsType: ap.ObjectNote,
},
"local_account_2_status_4": {
@ -1055,12 +1073,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: true,
Language: "en",
CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: false,
Boostable: false,
Replyable: true,
Likeable: true,
},
ActivityStreamsType: ap.ObjectNote,
},
"local_account_2_status_5": {
@ -1083,12 +1100,10 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Sensitive: false,
Language: "en",
CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
},
ActivityStreamsType: ap.ObjectNote,
},
}
@ -1149,6 +1164,18 @@ func NewTestMentions() map[string]*gtsmodel.Mention {
TargetAccountURI: "http://localhost:8080/users/the_mighty_zork",
TargetAccountURL: "http://localhost:8080/@the_mighty_zork",
},
"admin_account_mention_zork": {
ID: "01FF26A6BGEKCZFWNEHXB2ZZ6M",
StatusID: "01FF25D5Q0DH7CHD57CTRS6WK0",
CreatedAt: time.Now().Add(-46 * time.Hour),
UpdatedAt: time.Now().Add(-46 * time.Hour),
OriginAccountID: "01F8MH17FWEB39HZJ76B6VXSKF",
OriginAccountURI: "http://localhost:8080/users/admin",
TargetAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF",
NameString: "@the_mighty_zork",
TargetAccountURI: "http://localhost:8080/users/the_mighty_zork",
TargetAccountURL: "http://localhost:8080/@the_mighty_zork",
},
}
}
@ -1215,6 +1242,19 @@ func NewTestFollows() map[string]*gtsmodel.Follow {
}
}
func NewTestBlocks() map[string]*gtsmodel.Block {
return map[string]*gtsmodel.Block{
"local_account_2_block_remote_account_1": {
ID: "01FEXXET6XXMF7G2V3ASZP3YQW",
CreatedAt: time.Now().Add(-1 * time.Hour),
UpdatedAt: time.Now().Add(-1 * time.Hour),
URI: "http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW",
AccountID: "01F8MH5NBDF2MV7CTC4Q5128HF",
TargetAccountID: "01F8MH5ZK5VRH73AKHQM6Y9VNX",
},
}
}
// ActivityWithSignature wraps a pub.Activity along with its signature headers, for testing.
type ActivityWithSignature struct {
Activity pub.Activity
@ -1374,7 +1414,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
DateHeader: date,
}
target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5")
target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0")
sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
fossSatanDereferenceLocalAccount1Status1RepliesLast := ActivityWithSignature{
SignatureHeader: sig,

73
vendor/github.com/mitchellh/mapstructure/CHANGELOG.md generated vendored Normal file
View file

@ -0,0 +1,73 @@
## unreleased
* Fix regression where `*time.Time` value would be set to empty and not be sent
to decode hooks properly [GH-232]
## 1.4.0
* A new decode hook type `DecodeHookFuncValue` has been added that has
access to the full values. [GH-183]
* Squash is now supported with embedded fields that are struct pointers [GH-205]
* Empty strings will convert to 0 for all numeric types when weakly decoding [GH-206]
## 1.3.3
* Decoding maps from maps creates a settable value for decode hooks [GH-203]
## 1.3.2
* Decode into interface type with a struct value is supported [GH-187]
## 1.3.1
* Squash should only squash embedded structs. [GH-194]
## 1.3.0
* Added `",omitempty"` support. This will ignore zero values in the source
structure when encoding. [GH-145]
## 1.2.3
* Fix duplicate entries in Keys list with pointer values. [GH-185]
## 1.2.2
* Do not add unsettable (unexported) values to the unused metadata key
or "remain" value. [GH-150]
## 1.2.1
* Go modules checksum mismatch fix
## 1.2.0
* Added support to capture unused values in a field using the `",remain"` value
in the mapstructure tag. There is an example to showcase usage.
* Added `DecoderConfig` option to always squash embedded structs
* `json.Number` can decode into `uint` types
* Empty slices are preserved and not replaced with nil slices
* Fix panic that can occur in when decoding a map into a nil slice of structs
* Improved package documentation for godoc
## 1.1.2
* Fix error when decode hook decodes interface implementation into interface
type. [GH-140]
## 1.1.1
* Fix panic that can happen in `decodePtr`
## 1.1.0
* Added `StringToIPHookFunc` to convert `string` to `net.IP` and `net.IPNet` [GH-133]
* Support struct to struct decoding [GH-137]
* If source map value is nil, then destination map value is nil (instead of empty)
* If source slice value is nil, then destination slice value is nil (instead of empty)
* If source pointer is nil, then destination pointer is set to nil (instead of
allocated zero value of type)
## 1.0.0
* Initial tagged stable release.

21
vendor/github.com/mitchellh/mapstructure/LICENSE generated vendored Normal file
View file

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2013 Mitchell Hashimoto
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

46
vendor/github.com/mitchellh/mapstructure/README.md generated vendored Normal file
View file

@ -0,0 +1,46 @@
# mapstructure [![Godoc](https://godoc.org/github.com/mitchellh/mapstructure?status.svg)](https://godoc.org/github.com/mitchellh/mapstructure)
mapstructure is a Go library for decoding generic map values to structures
and vice versa, while providing helpful error handling.
This library is most useful when decoding values from some data stream (JSON,
Gob, etc.) where you don't _quite_ know the structure of the underlying data
until you read a part of it. You can therefore read a `map[string]interface{}`
and use this library to decode it into the proper underlying native Go
structure.
## Installation
Standard `go get`:
```
$ go get github.com/mitchellh/mapstructure
```
## Usage & Example
For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/mapstructure).
The `Decode` function has examples associated with it there.
## But Why?!
Go offers fantastic standard libraries for decoding formats such as JSON.
The standard method is to have a struct pre-created, and populate that struct
from the bytes of the encoded format. This is great, but the problem is if
you have configuration or an encoding that changes slightly depending on
specific fields. For example, consider this JSON:
```json
{
"type": "person",
"name": "Mitchell"
}
```
Perhaps we can't populate a specific structure without first reading
the "type" field from the JSON. We could always do two passes over the
decoding of the JSON (reading the "type" first, and the rest later).
However, it is much simpler to just decode this into a `map[string]interface{}`
structure, read the "type" key, then use something like this library
to decode it into the proper structure.

View file

@ -0,0 +1,256 @@
package mapstructure
import (
"encoding"
"errors"
"fmt"
"net"
"reflect"
"strconv"
"strings"
"time"
)
// typedDecodeHook takes a raw DecodeHookFunc (an interface{}) and turns
// it into the proper DecodeHookFunc type, such as DecodeHookFuncType.
func typedDecodeHook(h DecodeHookFunc) DecodeHookFunc {
// Create variables here so we can reference them with the reflect pkg
var f1 DecodeHookFuncType
var f2 DecodeHookFuncKind
var f3 DecodeHookFuncValue
// Fill in the variables into this interface and the rest is done
// automatically using the reflect package.
potential := []interface{}{f1, f2, f3}
v := reflect.ValueOf(h)
vt := v.Type()
for _, raw := range potential {
pt := reflect.ValueOf(raw).Type()
if vt.ConvertibleTo(pt) {
return v.Convert(pt).Interface()
}
}
return nil
}
// DecodeHookExec executes the given decode hook. This should be used
// since it'll naturally degrade to the older backwards compatible DecodeHookFunc
// that took reflect.Kind instead of reflect.Type.
func DecodeHookExec(
raw DecodeHookFunc,
from reflect.Value, to reflect.Value) (interface{}, error) {
switch f := typedDecodeHook(raw).(type) {
case DecodeHookFuncType:
return f(from.Type(), to.Type(), from.Interface())
case DecodeHookFuncKind:
return f(from.Kind(), to.Kind(), from.Interface())
case DecodeHookFuncValue:
return f(from, to)
default:
return nil, errors.New("invalid decode hook signature")
}
}
// ComposeDecodeHookFunc creates a single DecodeHookFunc that
// automatically composes multiple DecodeHookFuncs.
//
// The composed funcs are called in order, with the result of the
// previous transformation.
func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc {
return func(f reflect.Value, t reflect.Value) (interface{}, error) {
var err error
var data interface{}
newFrom := f
for _, f1 := range fs {
data, err = DecodeHookExec(f1, newFrom, t)
if err != nil {
return nil, err
}
newFrom = reflect.ValueOf(data)
}
return data, nil
}
}
// StringToSliceHookFunc returns a DecodeHookFunc that converts
// string to []string by splitting on the given sep.
func StringToSliceHookFunc(sep string) DecodeHookFunc {
return func(
f reflect.Kind,
t reflect.Kind,
data interface{}) (interface{}, error) {
if f != reflect.String || t != reflect.Slice {
return data, nil
}
raw := data.(string)
if raw == "" {
return []string{}, nil
}
return strings.Split(raw, sep), nil
}
}
// StringToTimeDurationHookFunc returns a DecodeHookFunc that converts
// strings to time.Duration.
func StringToTimeDurationHookFunc() DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(time.Duration(5)) {
return data, nil
}
// Convert it by parsing
return time.ParseDuration(data.(string))
}
}
// StringToIPHookFunc returns a DecodeHookFunc that converts
// strings to net.IP
func StringToIPHookFunc() DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(net.IP{}) {
return data, nil
}
// Convert it by parsing
ip := net.ParseIP(data.(string))
if ip == nil {
return net.IP{}, fmt.Errorf("failed parsing ip %v", data)
}
return ip, nil
}
}
// StringToIPNetHookFunc returns a DecodeHookFunc that converts
// strings to net.IPNet
func StringToIPNetHookFunc() DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(net.IPNet{}) {
return data, nil
}
// Convert it by parsing
_, net, err := net.ParseCIDR(data.(string))
return net, err
}
}
// StringToTimeHookFunc returns a DecodeHookFunc that converts
// strings to time.Time.
func StringToTimeHookFunc(layout string) DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(time.Time{}) {
return data, nil
}
// Convert it by parsing
return time.Parse(layout, data.(string))
}
}
// WeaklyTypedHook is a DecodeHookFunc which adds support for weak typing to
// the decoder.
//
// Note that this is significantly different from the WeaklyTypedInput option
// of the DecoderConfig.
func WeaklyTypedHook(
f reflect.Kind,
t reflect.Kind,
data interface{}) (interface{}, error) {
dataVal := reflect.ValueOf(data)
switch t {
case reflect.String:
switch f {
case reflect.Bool:
if dataVal.Bool() {
return "1", nil
}
return "0", nil
case reflect.Float32:
return strconv.FormatFloat(dataVal.Float(), 'f', -1, 64), nil
case reflect.Int:
return strconv.FormatInt(dataVal.Int(), 10), nil
case reflect.Slice:
dataType := dataVal.Type()
elemKind := dataType.Elem().Kind()
if elemKind == reflect.Uint8 {
return string(dataVal.Interface().([]uint8)), nil
}
case reflect.Uint:
return strconv.FormatUint(dataVal.Uint(), 10), nil
}
}
return data, nil
}
func RecursiveStructToMapHookFunc() DecodeHookFunc {
return func(f reflect.Value, t reflect.Value) (interface{}, error) {
if f.Kind() != reflect.Struct {
return f.Interface(), nil
}
var i interface{} = struct{}{}
if t.Type() != reflect.TypeOf(&i).Elem() {
return f.Interface(), nil
}
m := make(map[string]interface{})
t.Set(reflect.ValueOf(m))
return f.Interface(), nil
}
}
// TextUnmarshallerHookFunc returns a DecodeHookFunc that applies
// strings to the UnmarshalText function, when the target type
// implements the encoding.TextUnmarshaler interface
func TextUnmarshallerHookFunc() DecodeHookFuncType {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
result := reflect.New(t).Interface()
unmarshaller, ok := result.(encoding.TextUnmarshaler)
if !ok {
return data, nil
}
if err := unmarshaller.UnmarshalText([]byte(data.(string))); err != nil {
return nil, err
}
return result, nil
}
}

50
vendor/github.com/mitchellh/mapstructure/error.go generated vendored Normal file
View file

@ -0,0 +1,50 @@
package mapstructure
import (
"errors"
"fmt"
"sort"
"strings"
)
// Error implements the error interface and can represents multiple
// errors that occur in the course of a single decode.
type Error struct {
Errors []string
}
func (e *Error) Error() string {
points := make([]string, len(e.Errors))
for i, err := range e.Errors {
points[i] = fmt.Sprintf("* %s", err)
}
sort.Strings(points)
return fmt.Sprintf(
"%d error(s) decoding:\n\n%s",
len(e.Errors), strings.Join(points, "\n"))
}
// WrappedErrors implements the errwrap.Wrapper interface to make this
// return value more useful with the errwrap and go-multierror libraries.
func (e *Error) WrappedErrors() []error {
if e == nil {
return nil
}
result := make([]error, len(e.Errors))
for i, e := range e.Errors {
result[i] = errors.New(e)
}
return result
}
func appendErrors(errors []string, err error) []string {
switch e := err.(type) {
case *Error:
return append(errors, e.Errors...)
default:
return append(errors, e.Error())
}
}

3
vendor/github.com/mitchellh/mapstructure/go.mod generated vendored Normal file
View file

@ -0,0 +1,3 @@
module github.com/mitchellh/mapstructure
go 1.14

1462
vendor/github.com/mitchellh/mapstructure/mapstructure.go generated vendored Normal file

File diff suppressed because it is too large Load diff

3
vendor/modules.txt vendored
View file

@ -338,6 +338,9 @@ github.com/mattn/go-isatty
## explicit
github.com/microcosm-cc/bluemonday
github.com/microcosm-cc/bluemonday/css
# github.com/mitchellh/mapstructure v1.4.1
## explicit
github.com/mitchellh/mapstructure
# github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
## explicit
github.com/modern-go/concurrent