forgejo/modules/storage/minio_test.go

216 lines
6.9 KiB
Go
Raw Normal View History

// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package storage
import (
"context"
"net/http"
"net/http/httptest"
"os"
"testing"
"code.gitea.io/gitea/modules/setting"
"github.com/minio/minio-go/v7"
"github.com/stretchr/testify/assert"
)
func TestMinioStorageIterator(t *testing.T) {
if os.Getenv("CI") == "" {
t.Skip("minioStorage not present outside of CI")
return
}
testStorageIterator(t, setting.MinioStorageType, &setting.Storage{
MinioConfig: setting.MinioStorageConfig{
[CI] Forgejo Actions based CI for PR & branches (cherry picked from commit f9d75d4705ece5c119f2fd5e2bfbaf630d0bf739) (cherry picked from commit 64f76f4ab269daa6a584305164732c68be0161c5) (cherry picked from commit 5d024541551f1be98444923f0fa327a855104725) [CI] Forgejo Actions workflows (cherry picked from commit 3ff59b5379ebf761f32875f9d869a1d18f79741c) (cherry picked from commit 8af826a6f7c70d03079ec68f10230041695017cd) (cherry picked from commit d7c09d9cc80037a28d488da142ae2a2a99f59ac9) [CI] use the docker label instead of ubuntu-latest (cherry picked from commit b6a6470db6c8bae9963e204f9c8c408f309e81e3) [CI] all tests need compliance before proceeding (cherry picked from commit b35c496f2c3034164b9bb3a3550f35026adf9372) (cherry picked from commit 36a4148a8ec654ae9fa5a6925a3b8606b96aebcc) (cherry picked from commit 7ffcffa653808a284f422fdc31f6ea07874b585d) (cherry picked from commit 8a246d296e502c181c6ee779d1773c9e78ca7acf) (cherry picked from commit dd0b6e1826ba7699f967de2c00c1332909a53473) [CI] Forgejo Actions based release process (squash) MySQL optimization Refs: https://codeberg.org/forgejo/forgejo/issues/976 (cherry picked from commit b4b8c489e6e9fe36349eced5d4249467b25df2a7) (cherry picked from commit 1e861db4afdc32549784381a96b16dac508ec1c0) (cherry picked from commit a6c0e00330ded851c2f524039259f0b723d793e4) (cherry picked from commit f97b336465fd6be15079e756c9ee3a580f0aa339) (cherry picked from commit 6d65d5f0d6f798556a8f9e547896be03a5ee2f87) (cherry picked from commit 79bfbadbed3bf84b96f2c027d119a4aa5c60d5e2) (cherry picked from commit e86c40a34a97fc568c724ee5f8ee5b2bf46ebf19) (cherry picked from commit da0c454adbcf1b5dbb95d9d74afaa371e32351a0) (cherry picked from commit b49d892cda968b22ad6c29caade042690d96f2e2) [CI] enable minio tests (cherry picked from commit 4d8f4380311b872061bd20f561424662ecd8d817) (cherry picked from commit c4eeb0a61ef579c21b0382edd61c62adebe93b28) [CI] Forgejo Actions based CI for PR & branches (squash) cleanup (cherry picked from commit 80eb20e84267552c971096693558734f0ec7afdd) (cherry picked from commit d2ff589858961e4f25bc1d903cdf57df1db4c316) (cherry picked from commit f6eedecb67fce992605bb1fcdd93c866ec47a9d4) (cherry picked from commit cf458091e245e1c70944e940a214d5127f304303) (cherry picked from commit ddd322cb2de165126080b222c1b09b2a91fea182) (cherry picked from commit f0f5729b6467bea053d532c4c64c54a0fe737c98) Conflicts: .github/workflows/pull-db-tests.yml https://codeberg.org/forgejo/forgejo/pulls/1573 (cherry picked from commit bb347aedd4e5030eea46ce22176a0444619df6b8) (cherry picked from commit 0f5ebe8c3e78cef6f310ee9097a8413df904c08a) (cherry picked from commit 9101cb4715ed88c6a3f0ad14443758d9bc9c61ed) (cherry picked from commit ab118fe4efd96e9bb7134065643f094a379bb192) [CI] Forgejo Actions based CI for PR & branches (squash) use node:20-bookworm No longer use the custom test-env image, it is unecessary technical debt. Also upgrade to bitnami/minio:2023.8.31 to align with what Gitea tests (cherry picked from commit d9b77fd2735a52043b4f8f1baaaa2e15073db621) (cherry picked from commit ef8f099f9ea03d216324ceff348fe643240341dc) (cherry picked from commit 6d3c675d203ba838de1f0540977f5e7e9f62f055) (cherry picked from commit fc00ff45f2e810cc2a84bf57cdb2765830e5b07f) (cherry picked from commit b7f02b9846b8c7087a4827a9c87f48eafd2ba1a2) (cherry picked from commit d23d86e56728f4a263a0567c14e2dc20407baaff) (cherry picked from commit e68e65460c40d343a3ada858c0c48d03a6a14bbb) (cherry picked from commit 96f93a1fdb9f2c6ecbd05ac24c5375f3e0d631df) (cherry picked from commit 2ff3080018633212075935ec6644654184009a2e) (cherry picked from commit 8ea00bcff471b877e442f7b78bb46880c9e396f0) (cherry picked from commit 85be961970b0d374d9816348a8a1b51289f7c787) (cherry picked from commit 387b279fc33d79be1f0f0b35b32f5c17b0d64c9c) (cherry picked from commit 219be7aa85fbf2bd5cb9daead3544b27f6200d83) (cherry picked from commit 929b75ee39cd6d1a462bbd0c20597c357949c953) (cherry picked from commit dd42a30d6a9e6cad26c9c916d733a0270d94a87e)
2023-05-28 13:31:52 +00:00
Endpoint: "minio:9000",
AccessKeyID: "123456",
SecretAccessKey: "12345678",
Bucket: "gitea",
Location: "us-east-1",
},
})
}
2024-03-04 03:13:59 +00:00
func TestVirtualHostMinioStorage(t *testing.T) {
if os.Getenv("CI") == "" {
t.Skip("minioStorage not present outside of CI")
return
}
testStorageIterator(t, setting.MinioStorageType, &setting.Storage{
MinioConfig: setting.MinioStorageConfig{
Endpoint: "minio:9000",
AccessKeyID: "123456",
SecretAccessKey: "12345678",
Bucket: "gitea",
Location: "us-east-1",
BucketLookup: "dns",
},
})
}
func TestMinioStoragePath(t *testing.T) {
m := &MinioStorage{basePath: ""}
assert.Equal(t, "", m.buildMinioPath("/"))
assert.Equal(t, "", m.buildMinioPath("."))
assert.Equal(t, "a", m.buildMinioPath("/a"))
assert.Equal(t, "a/b", m.buildMinioPath("/a/b/"))
assert.Equal(t, "", m.buildMinioDirPrefix(""))
assert.Equal(t, "a/", m.buildMinioDirPrefix("/a/"))
m = &MinioStorage{basePath: "/"}
assert.Equal(t, "", m.buildMinioPath("/"))
assert.Equal(t, "", m.buildMinioPath("."))
assert.Equal(t, "a", m.buildMinioPath("/a"))
assert.Equal(t, "a/b", m.buildMinioPath("/a/b/"))
assert.Equal(t, "", m.buildMinioDirPrefix(""))
assert.Equal(t, "a/", m.buildMinioDirPrefix("/a/"))
m = &MinioStorage{basePath: "/base"}
assert.Equal(t, "base", m.buildMinioPath("/"))
assert.Equal(t, "base", m.buildMinioPath("."))
assert.Equal(t, "base/a", m.buildMinioPath("/a"))
assert.Equal(t, "base/a/b", m.buildMinioPath("/a/b/"))
assert.Equal(t, "base/", m.buildMinioDirPrefix(""))
assert.Equal(t, "base/a/", m.buildMinioDirPrefix("/a/"))
m = &MinioStorage{basePath: "/base/"}
assert.Equal(t, "base", m.buildMinioPath("/"))
assert.Equal(t, "base", m.buildMinioPath("."))
assert.Equal(t, "base/a", m.buildMinioPath("/a"))
assert.Equal(t, "base/a/b", m.buildMinioPath("/a/b/"))
assert.Equal(t, "base/", m.buildMinioDirPrefix(""))
assert.Equal(t, "base/a/", m.buildMinioDirPrefix("/a/"))
}
func TestS3StorageBadRequest(t *testing.T) {
if os.Getenv("CI") == "" {
t.Skip("S3Storage not present outside of CI")
return
}
cfg := &setting.Storage{
MinioConfig: setting.MinioStorageConfig{
Endpoint: "minio:9000",
AccessKeyID: "123456",
SecretAccessKey: "12345678",
Bucket: "bucket",
Location: "us-east-1",
},
}
message := "ERROR"
old := getBucketVersioning
defer func() { getBucketVersioning = old }()
getBucketVersioning = func(ctx context.Context, minioClient *minio.Client, bucket string) error {
return minio.ErrorResponse{
StatusCode: http.StatusBadRequest,
Code: "FixtureError",
Message: message,
}
}
_, err := NewStorage(setting.MinioStorageType, cfg)
assert.ErrorContains(t, err, message)
}
func TestMinioCredentials(t *testing.T) {
const (
ExpectedAccessKey = "ExampleAccessKeyID"
ExpectedSecretAccessKey = "ExampleSecretAccessKeyID"
// Use a FakeEndpoint for IAM credentials to avoid logging any
// potential real IAM credentials when running in EC2.
FakeEndpoint = "http://localhost"
)
t.Run("Static Credentials", func(t *testing.T) {
cfg := setting.MinioStorageConfig{
AccessKeyID: ExpectedAccessKey,
SecretAccessKey: ExpectedSecretAccessKey,
}
creds := buildMinioCredentials(cfg, FakeEndpoint)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey, v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey, v.SecretAccessKey)
})
t.Run("Chain", func(t *testing.T) {
cfg := setting.MinioStorageConfig{}
t.Run("EnvMinio", func(t *testing.T) {
t.Setenv("MINIO_ACCESS_KEY", ExpectedAccessKey+"Minio")
t.Setenv("MINIO_SECRET_KEY", ExpectedSecretAccessKey+"Minio")
creds := buildMinioCredentials(cfg, FakeEndpoint)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey+"Minio", v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey+"Minio", v.SecretAccessKey)
})
t.Run("EnvAWS", func(t *testing.T) {
t.Setenv("AWS_ACCESS_KEY", ExpectedAccessKey+"AWS")
t.Setenv("AWS_SECRET_KEY", ExpectedSecretAccessKey+"AWS")
creds := buildMinioCredentials(cfg, FakeEndpoint)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey+"AWS", v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey+"AWS", v.SecretAccessKey)
})
t.Run("FileMinio", func(t *testing.T) {
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/minio.json")
// prevent loading any actual credentials files from the user
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake")
creds := buildMinioCredentials(cfg, FakeEndpoint)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey+"MinioFile", v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey+"MinioFile", v.SecretAccessKey)
})
t.Run("FileAWS", func(t *testing.T) {
// prevent loading any actual credentials files from the user
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json")
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/aws_credentials")
creds := buildMinioCredentials(cfg, FakeEndpoint)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey+"AWSFile", v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey+"AWSFile", v.SecretAccessKey)
})
t.Run("IAM", func(t *testing.T) {
// prevent loading any actual credentials files from the user
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json")
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake")
// Spawn a server to emulate the EC2 Instance Metadata
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// The client will actually make 3 requests here,
// first will be to get the IMDSv2 token, second to
// get the role, and third for the actual
// credentials. However, we can return credentials
// every request since we're not emulating a full
// IMDSv2 flow.
w.Write([]byte(`{"Code":"Success","AccessKeyId":"ExampleAccessKeyIDIAM","SecretAccessKey":"ExampleSecretAccessKeyIDIAM"}`))
}))
defer server.Close()
// Use the provided EC2 Instance Metadata server
creds := buildMinioCredentials(cfg, server.URL)
v, err := creds.Get()
assert.NoError(t, err)
assert.Equal(t, ExpectedAccessKey+"IAM", v.AccessKeyID)
assert.Equal(t, ExpectedSecretAccessKey+"IAM", v.SecretAccessKey)
})
})
}