[feature] list commands for both attachment and emojis (#2121)

* [feature] list commands for both attachment and emojis

* use fewer commands, provide `local-only` and `remote-only` as filters

* envparsing

---------

Co-authored-by: Romain de Laage <romain.delaage@rdelaage.ovh>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
This commit is contained in:
rdelaage 2023-08-23 18:01:16 +02:00 committed by GitHub
parent 8f38dc2e7f
commit 7b48437f17
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 283 additions and 52 deletions

View file

@ -20,6 +20,7 @@ package media
import ( import (
"bufio" "bufio"
"context" "context"
"errors"
"fmt" "fmt"
"os" "os"
"path" "path"
@ -34,14 +35,17 @@ import (
) )
type list struct { type list struct {
dbService db.DB dbService db.DB
state *state.State state *state.State
maxID string maxID string
limit int limit int
out *bufio.Writer localOnly bool
remoteOnly bool
out *bufio.Writer
} }
func (l *list) GetAllMediaPaths(ctx context.Context, filter func(*gtsmodel.MediaAttachment) string) ([]string, error) { // Get a list of attachment using a custom filter
func (l *list) GetAllAttachmentPaths(ctx context.Context, filter func(*gtsmodel.MediaAttachment) string) ([]string, error) {
res := make([]string, 0, 100) res := make([]string, 0, 100)
for { for {
attachments, err := l.dbService.GetAttachments(ctx, l.maxID, l.limit) attachments, err := l.dbService.GetAttachments(ctx, l.maxID, l.limit)
@ -72,8 +76,52 @@ func (l *list) GetAllMediaPaths(ctx context.Context, filter func(*gtsmodel.Media
return res, nil return res, nil
} }
// Get a list of emojis using a custom filter
func (l *list) GetAllEmojisPaths(ctx context.Context, filter func(*gtsmodel.Emoji) string) ([]string, error) {
res := make([]string, 0, 100)
for {
attachments, err := l.dbService.GetEmojis(ctx, l.maxID, l.limit)
if err != nil {
return nil, fmt.Errorf("failed to retrieve media metadata from database: %w", err)
}
for _, a := range attachments {
v := filter(a)
if v != "" {
res = append(res, v)
}
}
// If we got less results than our limit, we've reached the
// last page to retrieve and we can break the loop. If the
// last batch happens to contain exactly the same amount of
// items as the limit we'll end up doing one extra query.
if len(attachments) < l.limit {
break
}
// Grab the last ID from the batch and set it as the maxID
// that'll be used in the next iteration so we don't get items
// we've already seen.
l.maxID = attachments[len(attachments)-1].ID
}
return res, nil
}
func setupList(ctx context.Context) (*list, error) { func setupList(ctx context.Context) (*list, error) {
var state state.State var (
localOnly = config.GetAdminMediaListLocalOnly()
remoteOnly = config.GetAdminMediaListRemoteOnly()
state state.State
)
// Validate flags.
if localOnly && remoteOnly {
return nil, errors.New(
"local-only and remote-only flags cannot be true at the same time; " +
"choose one or the other, or set neither to list all media",
)
}
state.Caches.Init() state.Caches.Init()
state.Caches.Start() state.Caches.Start()
@ -87,11 +135,13 @@ func setupList(ctx context.Context) (*list, error) {
state.DB = dbService state.DB = dbService
return &list{ return &list{
dbService: dbService, dbService: dbService,
state: &state, state: &state,
limit: 200, limit: 200,
maxID: "", maxID: "",
out: bufio.NewWriter(os.Stdout), localOnly: localOnly,
remoteOnly: remoteOnly,
out: bufio.NewWriter(os.Stdout),
}, nil }, nil
} }
@ -103,7 +153,8 @@ func (l *list) shutdown() error {
return err return err
} }
var ListLocal action.GTSAction = func(ctx context.Context) error { // ListAttachments lists local, remote, or all attachment paths.
var ListAttachments action.GTSAction = func(ctx context.Context) error {
list, err := setupList(ctx) list, err := setupList(ctx)
if err != nil { if err != nil {
return err return err
@ -116,26 +167,53 @@ var ListLocal action.GTSAction = func(ctx context.Context) error {
} }
}() }()
mediaPath := config.GetStorageLocalBasePath() var (
media, err := list.GetAllMediaPaths( mediaPath = config.GetStorageLocalBasePath()
ctx, filter func(*gtsmodel.MediaAttachment) string
func(m *gtsmodel.MediaAttachment) string { )
if m.RemoteURL == "" {
return path.Join(mediaPath, m.File.Path) switch {
case list.localOnly:
filter = func(m *gtsmodel.MediaAttachment) string {
if m.RemoteURL != "" {
// Remote, not
// interested.
return ""
} }
return ""
}) return path.Join(mediaPath, m.File.Path)
}
case list.remoteOnly:
filter = func(m *gtsmodel.MediaAttachment) string {
if m.RemoteURL == "" {
// Local, not
// interested.
return ""
}
return path.Join(mediaPath, m.File.Path)
}
default:
filter = func(m *gtsmodel.MediaAttachment) string {
return path.Join(mediaPath, m.File.Path)
}
}
attachments, err := list.GetAllAttachmentPaths(ctx, filter)
if err != nil { if err != nil {
return err return err
} }
for _, m := range media { for _, a := range attachments {
_, _ = list.out.WriteString(m + "\n") _, _ = list.out.WriteString(a + "\n")
} }
return nil return nil
} }
var ListRemote action.GTSAction = func(ctx context.Context) error { // ListEmojis lists local, remote, or all emoji filepaths.
var ListEmojis action.GTSAction = func(ctx context.Context) error {
list, err := setupList(ctx) list, err := setupList(ctx)
if err != nil { if err != nil {
return err return err
@ -148,17 +226,47 @@ var ListRemote action.GTSAction = func(ctx context.Context) error {
} }
}() }()
media, err := list.GetAllMediaPaths( var (
ctx, mediaPath = config.GetStorageLocalBasePath()
func(m *gtsmodel.MediaAttachment) string { filter func(*gtsmodel.Emoji) string
return m.RemoteURL )
})
switch {
case list.localOnly:
filter = func(e *gtsmodel.Emoji) string {
if e.ImageRemoteURL != "" {
// Remote, not
// interested.
return ""
}
return path.Join(mediaPath, e.ImagePath)
}
case list.remoteOnly:
filter = func(e *gtsmodel.Emoji) string {
if e.ImageRemoteURL == "" {
// Local, not
// interested.
return ""
}
return path.Join(mediaPath, e.ImagePath)
}
default:
filter = func(e *gtsmodel.Emoji) string {
return path.Join(mediaPath, e.ImagePath)
}
}
emojis, err := list.GetAllEmojisPaths(ctx, filter)
if err != nil { if err != nil {
return err return err
} }
for _, m := range media { for _, e := range emojis {
_, _ = list.out.WriteString(m + "\n") _, _ = list.out.WriteString(e + "\n")
} }
return nil return nil
} }

View file

@ -178,29 +178,31 @@ func adminCommands() *cobra.Command {
ADMIN MEDIA LIST COMMANDS ADMIN MEDIA LIST COMMANDS
*/ */
adminMediaListLocalCmd := &cobra.Command{ adminMediaListAttachmentsCmd := &cobra.Command{
Use: "list-local", Use: "list-attachments",
Short: "admin command to list media on local storage", Short: "list local, remote, or all attachments",
PreRunE: func(cmd *cobra.Command, args []string) error { PreRunE: func(cmd *cobra.Command, args []string) error {
return preRun(preRunArgs{cmd: cmd}) return preRun(preRunArgs{cmd: cmd})
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return run(cmd.Context(), media.ListLocal) return run(cmd.Context(), media.ListAttachments)
}, },
} }
config.AddAdminMediaList(adminMediaListAttachmentsCmd)
adminMediaCmd.AddCommand(adminMediaListAttachmentsCmd)
adminMediaListRemoteCmd := &cobra.Command{ adminMediaListEmojisLocalCmd := &cobra.Command{
Use: "list-remote", Use: "list-emojis",
Short: "admin command to list remote media cached on this instance", Short: "list local, remote, or all emojis",
PreRunE: func(cmd *cobra.Command, args []string) error { PreRunE: func(cmd *cobra.Command, args []string) error {
return preRun(preRunArgs{cmd: cmd}) return preRun(preRunArgs{cmd: cmd})
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return run(cmd.Context(), media.ListRemote) return run(cmd.Context(), media.ListEmojis)
}, },
} }
config.AddAdminMediaList(adminMediaListEmojisLocalCmd)
adminMediaCmd.AddCommand(adminMediaListLocalCmd, adminMediaListRemoteCmd) adminMediaCmd.AddCommand(adminMediaListEmojisLocalCmd)
/* /*
ADMIN MEDIA PRUNE COMMANDS ADMIN MEDIA PRUNE COMMANDS

View file

@ -255,17 +255,73 @@ Example:
gotosocial admin import --path example.json --config-path config.yaml gotosocial admin import --path example.json --config-path config.yaml
``` ```
### gotosocial admin media list-local ### gotosocial admin media list-attachments
This command can be used to list local media. Local media is media that belongs to posts by users with an account on the instance. Can be used to list the storage paths of local, remote, or all media attachments on your instance (including headers and avatars).
The output will be a list of files. The list can be used to drive your backups. `local-only` and `remote-only` can be used as filters; they cannot both be set at once.
### gotosocial admin media list-remote If neither `local-only` or `remote-only` are set, all media attachments on your instance will be listed.
This is the corollary to list-local, but instead lists media from remote instances. Remote media belongs to other instances, but was attached to a post we received over federation and have potentially cached locally. You may want to run this with `GTS_LOG_LEVEL` set to `warn` or `error`, otherwise it will log a lot of info messages you probably don't need.
The output will be a list of URLs to retrieve the original content from. GoToSocial automatically retrieves remote media when it needs it, so you should never need to do so yourself. `gotosocial admin media list-attachments --help`:
```text
list local, remote, or all attachments
Usage:
gotosocial admin media list-attachments [flags]
Flags:
-h, --help help for list-attachments
--local-only list only local attachments/emojis; if specified then remote-only cannot also be true
--remote-only list only remote attachments/emojis; if specified then local-only cannot also be true
```
Example output:
```text
/gotosocial/062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpg
/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg
/gotosocial/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg
/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg
/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH7TDVANYKWVE8VVKFPJTJ.gif
/gotosocial/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg
/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH58A357CV5K7R7TJMSH6S.jpg
/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.gif
```
### gotosocial admin media list-emojis
Can be used to list the storage paths of local, remote, or all emojis on your instance.
`local-only` and `remote-only` can be used as filters; they cannot both be set at once.
If neither `local-only` or `remote-only` are set, all emojis on your instance will be listed.
You may want to run this with `GTS_LOG_LEVEL` set to `warn` or `error`, otherwise it will log a lot of info messages you probably don't need.
`gotosocial admin media list-emojis --help`:
```text
list local, remote, or all emojis
Usage:
gotosocial admin media list-emojis [flags]
Flags:
-h, --help help for list-emojis
--local-only list only local attachments/emojis; if specified then remote-only cannot also be true
--remote-only list only remote attachments/emojis; if specified then local-only cannot also be true
```
Example output:
```text
/gotosocial/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01GD5KP5CQEE1R3X43Y1EHS2CW.png
/gotosocial/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png
```
### gotosocial admin media prune orphaned ### gotosocial admin media prune orphaned

View file

@ -161,11 +161,13 @@ type Configuration struct {
Cache CacheConfiguration `name:"cache"` Cache CacheConfiguration `name:"cache"`
// TODO: move these elsewhere, these are more ephemeral vs long-running flags like above // TODO: move these elsewhere, these are more ephemeral vs long-running flags like above
AdminAccountUsername string `name:"username" usage:"the username to create/delete/etc"` AdminAccountUsername string `name:"username" usage:"the username to create/delete/etc"`
AdminAccountEmail string `name:"email" usage:"the email address of this account"` AdminAccountEmail string `name:"email" usage:"the email address of this account"`
AdminAccountPassword string `name:"password" usage:"the password to set for this account"` AdminAccountPassword string `name:"password" usage:"the password to set for this account"`
AdminTransPath string `name:"path" usage:"the path of the file to import from/export to"` AdminTransPath string `name:"path" usage:"the path of the file to import from/export to"`
AdminMediaPruneDryRun bool `name:"dry-run" usage:"perform a dry run and only log number of items eligible for pruning"` AdminMediaPruneDryRun bool `name:"dry-run" usage:"perform a dry run and only log number of items eligible for pruning"`
AdminMediaListLocalOnly bool `name:"local-only" usage:"list only local attachments/emojis; if specified then remote-only cannot also be true"`
AdminMediaListRemoteOnly bool `name:"remote-only" usage:"list only remote attachments/emojis; if specified then local-only cannot also be true"`
RequestIDHeader string `name:"request-id-header" usage:"Header to extract the Request ID from. Eg.,'X-Request-Id'."` RequestIDHeader string `name:"request-id-header" usage:"Header to extract the Request ID from. Eg.,'X-Request-Id'."`
} }

View file

@ -203,6 +203,17 @@ func AddAdminTrans(cmd *cobra.Command) {
} }
} }
// AddAdminMediaList attaches flags pertaining to media list commands.
func AddAdminMediaList(cmd *cobra.Command) {
localOnly := AdminMediaListLocalOnlyFlag()
localOnlyUsage := fieldtag("AdminMediaListLocalOnly", "usage")
cmd.Flags().Bool(localOnly, false, localOnlyUsage)
remoteOnly := AdminMediaListRemoteOnlyFlag()
remoteOnlyUsage := fieldtag("AdminMediaListRemoteOnly", "usage")
cmd.Flags().Bool(remoteOnly, false, remoteOnlyUsage)
}
// AddAdminMediaPrune attaches flags pertaining to media storage prune commands. // AddAdminMediaPrune attaches flags pertaining to media storage prune commands.
func AddAdminMediaPrune(cmd *cobra.Command) { func AddAdminMediaPrune(cmd *cobra.Command) {
name := AdminMediaPruneDryRunFlag() name := AdminMediaPruneDryRunFlag()

View file

@ -3374,6 +3374,56 @@ func GetAdminMediaPruneDryRun() bool { return global.GetAdminMediaPruneDryRun()
// SetAdminMediaPruneDryRun safely sets the value for global configuration 'AdminMediaPruneDryRun' field // SetAdminMediaPruneDryRun safely sets the value for global configuration 'AdminMediaPruneDryRun' field
func SetAdminMediaPruneDryRun(v bool) { global.SetAdminMediaPruneDryRun(v) } func SetAdminMediaPruneDryRun(v bool) { global.SetAdminMediaPruneDryRun(v) }
// GetAdminMediaListLocalOnly safely fetches the Configuration value for state's 'AdminMediaListLocalOnly' field
func (st *ConfigState) GetAdminMediaListLocalOnly() (v bool) {
st.mutex.RLock()
v = st.config.AdminMediaListLocalOnly
st.mutex.RUnlock()
return
}
// SetAdminMediaListLocalOnly safely sets the Configuration value for state's 'AdminMediaListLocalOnly' field
func (st *ConfigState) SetAdminMediaListLocalOnly(v bool) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.AdminMediaListLocalOnly = v
st.reloadToViper()
}
// AdminMediaListLocalOnlyFlag returns the flag name for the 'AdminMediaListLocalOnly' field
func AdminMediaListLocalOnlyFlag() string { return "local-only" }
// GetAdminMediaListLocalOnly safely fetches the value for global configuration 'AdminMediaListLocalOnly' field
func GetAdminMediaListLocalOnly() bool { return global.GetAdminMediaListLocalOnly() }
// SetAdminMediaListLocalOnly safely sets the value for global configuration 'AdminMediaListLocalOnly' field
func SetAdminMediaListLocalOnly(v bool) { global.SetAdminMediaListLocalOnly(v) }
// GetAdminMediaListRemoteOnly safely fetches the Configuration value for state's 'AdminMediaListRemoteOnly' field
func (st *ConfigState) GetAdminMediaListRemoteOnly() (v bool) {
st.mutex.RLock()
v = st.config.AdminMediaListRemoteOnly
st.mutex.RUnlock()
return
}
// SetAdminMediaListRemoteOnly safely sets the Configuration value for state's 'AdminMediaListRemoteOnly' field
func (st *ConfigState) SetAdminMediaListRemoteOnly(v bool) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.AdminMediaListRemoteOnly = v
st.reloadToViper()
}
// AdminMediaListRemoteOnlyFlag returns the flag name for the 'AdminMediaListRemoteOnly' field
func AdminMediaListRemoteOnlyFlag() string { return "remote-only" }
// GetAdminMediaListRemoteOnly safely fetches the value for global configuration 'AdminMediaListRemoteOnly' field
func GetAdminMediaListRemoteOnly() bool { return global.GetAdminMediaListRemoteOnly() }
// SetAdminMediaListRemoteOnly safely sets the value for global configuration 'AdminMediaListRemoteOnly' field
func SetAdminMediaListRemoteOnly(v bool) { global.SetAdminMediaListRemoteOnly(v) }
// GetRequestIDHeader safely fetches the Configuration value for state's 'RequestIDHeader' field // GetRequestIDHeader safely fetches the Configuration value for state's 'RequestIDHeader' field
func (st *ConfigState) GetRequestIDHeader() (v string) { func (st *ConfigState) GetRequestIDHeader() (v string) {
st.mutex.RLock() st.mutex.RLock()

View file

@ -87,6 +87,7 @@ EXPECT=$(cat << "EOF"
"letsencrypt-email-address": "", "letsencrypt-email-address": "",
"letsencrypt-enabled": true, "letsencrypt-enabled": true,
"letsencrypt-port": 80, "letsencrypt-port": 80,
"local-only": false,
"log-client-ip": false, "log-client-ip": false,
"log-db-queries": true, "log-db-queries": true,
"log-level": "info", "log-level": "info",
@ -116,6 +117,7 @@ EXPECT=$(cat << "EOF"
"path": "", "path": "",
"port": 6969, "port": 6969,
"protocol": "http", "protocol": "http",
"remote-only": false,
"request-id-header": "X-Trace-Id", "request-id-header": "X-Trace-Id",
"smtp-disclose-recipients": true, "smtp-disclose-recipients": true,
"smtp-from": "queen.rip.in.piss@terfisland.org", "smtp-from": "queen.rip.in.piss@terfisland.org",