diff --git a/go.mod b/go.mod index cf1f412eb..d66a2b8b2 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/jackc/pgx/v5 v5.4.1 github.com/microcosm-cc/bluemonday v1.0.24 github.com/miekg/dns v1.1.55 - github.com/minio/minio-go/v7 v7.0.58 + github.com/minio/minio-go/v7 v7.0.59 github.com/mitchellh/mapstructure v1.5.0 github.com/oklog/ulid v1.3.1 github.com/spf13/cobra v1.7.0 diff --git a/go.sum b/go.sum index c590394af..61315f771 100644 --- a/go.sum +++ b/go.sum @@ -450,8 +450,8 @@ github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.58 h1:B9/8Az8Om/2kX8Ys2ai2PZbBTokRE5W6P5OaqnAs6po= -github.com/minio/minio-go/v7 v7.0.58/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= +github.com/minio/minio-go/v7 v7.0.59 h1:lxIXwsTIcQkYoEG25rUJbzpmSB/oWeVDmxFo/uWUUsw= +github.com/minio/minio-go/v7 v7.0.59/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go b/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go index d5895dfe0..73c751ff8 100644 --- a/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go @@ -289,3 +289,67 @@ func (c *Client) GetBucketReplicationResyncStatus(ctx context.Context, bucketNam } return rinfo, nil } + +// GetBucketReplicationMetricsV2 fetches bucket replication status metrics +func (c *Client) GetBucketReplicationMetricsV2(ctx context.Context, bucketName string) (s replication.MetricsV2, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return s, err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication-metrics", "2") + + // Execute GET on bucket to get replication metrics. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return s, err + } + + if resp.StatusCode != http.StatusOK { + return s, httpRespToErrorResponse(resp, bucketName, "") + } + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return s, err + } + + if err := json.Unmarshal(respBytes, &s); err != nil { + return s, err + } + return s, nil +} + +// CheckBucketReplication validates if replication is set up properly for a bucket +func (c *Client) CheckBucketReplication(ctx context.Context, bucketName string) (err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication-check", "") + + // Execute GET on bucket to get replication config. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} diff --git a/vendor/github.com/minio/minio-go/v7/api-compose-object.go b/vendor/github.com/minio/minio-go/v7/api-compose-object.go index 1ba68c79e..38c25ab94 100644 --- a/vendor/github.com/minio/minio-go/v7/api-compose-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-compose-object.go @@ -222,6 +222,9 @@ func (c *Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuc if dstOpts.Internal.ReplicationRequest { headers.Set(minIOBucketReplicationRequest, "true") } + if dstOpts.Internal.ReplicationValidityCheck { + headers.Set(minIOBucketReplicationCheck, "true") + } if !dstOpts.Internal.LegalholdTimestamp.IsZero() { headers.Set(minIOBucketReplicationObjectLegalHoldTimestamp, dstOpts.Internal.LegalholdTimestamp.Format(time.RFC3339Nano)) } diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object.go b/vendor/github.com/minio/minio-go/v7/api-put-object.go index b29df17d4..2c4de4f96 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object.go @@ -56,14 +56,15 @@ func (r ReplicationStatus) Empty() bool { // AdvancedPutOptions for internal use - to be utilized by replication, ILM transition // implementation on MinIO server type AdvancedPutOptions struct { - SourceVersionID string - SourceETag string - ReplicationStatus ReplicationStatus - SourceMTime time.Time - ReplicationRequest bool - RetentionTimestamp time.Time - TaggingTimestamp time.Time - LegalholdTimestamp time.Time + SourceVersionID string + SourceETag string + ReplicationStatus ReplicationStatus + SourceMTime time.Time + ReplicationRequest bool + RetentionTimestamp time.Time + TaggingTimestamp time.Time + LegalholdTimestamp time.Time + ReplicationValidityCheck bool } // PutObjectOptions represents options specified by user for PutObject call @@ -188,6 +189,9 @@ func (opts PutObjectOptions) Header() (header http.Header) { if opts.Internal.ReplicationRequest { header.Set(minIOBucketReplicationRequest, "true") } + if opts.Internal.ReplicationValidityCheck { + header.Set(minIOBucketReplicationCheck, "true") + } if !opts.Internal.LegalholdTimestamp.IsZero() { header.Set(minIOBucketReplicationObjectLegalHoldTimestamp, opts.Internal.LegalholdTimestamp.Format(time.RFC3339Nano)) } diff --git a/vendor/github.com/minio/minio-go/v7/api-remove.go b/vendor/github.com/minio/minio-go/v7/api-remove.go index ca6b80a7d..9c0ac449a 100644 --- a/vendor/github.com/minio/minio-go/v7/api-remove.go +++ b/vendor/github.com/minio/minio-go/v7/api-remove.go @@ -112,10 +112,11 @@ func (c *Client) RemoveBucket(ctx context.Context, bucketName string) error { // AdvancedRemoveOptions intended for internal use by replication type AdvancedRemoveOptions struct { - ReplicationDeleteMarker bool - ReplicationStatus ReplicationStatus - ReplicationMTime time.Time - ReplicationRequest bool + ReplicationDeleteMarker bool + ReplicationStatus ReplicationStatus + ReplicationMTime time.Time + ReplicationRequest bool + ReplicationValidityCheck bool // check permissions } // RemoveObjectOptions represents options specified by user for RemoveObject call @@ -168,6 +169,9 @@ func (c *Client) removeObject(ctx context.Context, bucketName, objectName string if opts.Internal.ReplicationRequest { headers.Set(minIOBucketReplicationRequest, "true") } + if opts.Internal.ReplicationValidityCheck { + headers.Set(minIOBucketReplicationCheck, "true") + } if opts.ForceDelete { headers.Set(minIOForceDelete, "true") } diff --git a/vendor/github.com/minio/minio-go/v7/api.go b/vendor/github.com/minio/minio-go/v7/api.go index fa1d60824..9251c3894 100644 --- a/vendor/github.com/minio/minio-go/v7/api.go +++ b/vendor/github.com/minio/minio-go/v7/api.go @@ -124,7 +124,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.58" + libraryVersion = "v7.0.59" ) // User Agent should always following the below style. diff --git a/vendor/github.com/minio/minio-go/v7/constants.go b/vendor/github.com/minio/minio-go/v7/constants.go index 1c3e8e6f6..401d2a74d 100644 --- a/vendor/github.com/minio/minio-go/v7/constants.go +++ b/vendor/github.com/minio/minio-go/v7/constants.go @@ -94,6 +94,8 @@ const ( minIOBucketReplicationDeleteMarker = "X-Minio-Source-DeleteMarker" minIOBucketReplicationProxyRequest = "X-Minio-Source-Proxy-Request" minIOBucketReplicationRequest = "X-Minio-Source-Replication-Request" + minIOBucketReplicationCheck = "X-Minio-Source-Replication-Check" + // Header indicates last tag update time on source minIOBucketReplicationTaggingTimestamp = "X-Minio-Source-Replication-Tagging-Timestamp" // Header indicates last retention update time on source diff --git a/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go b/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go index 645fe18cb..1425a62aa 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go @@ -20,6 +20,7 @@ import ( "bytes" "encoding/xml" "fmt" + "math" "strconv" "strings" "time" @@ -704,6 +705,8 @@ type TargetMetrics struct { BandWidthLimitInBytesPerSecond int64 `json:"limitInBits"` // Current bandwidth used in bytes/sec for this target CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth"` + // Completed count + ReplicatedCount uint64 `json:"replicationCount"` } // Metrics represents inline replication metrics for a bucket. @@ -721,6 +724,10 @@ type Metrics struct { PendingCount uint64 `json:"pendingReplicationCount"` // Total number of failed operations including metadata updates across targets FailedCount uint64 `json:"failedReplicationCount"` + // Total Replica counts + ReplicaCount int64 `json:"replicaCount,omitempty"` + // Total Replicated count + ReplicatedCount int64 `json:"replicationCount,omitempty"` } // ResyncTargetsInfo provides replication target information to resync replicated data. @@ -742,9 +749,114 @@ type ResyncTarget struct { FailedSize int64 `json:"failedReplicationSize,omitempty"` // Total number of failed operations FailedCount int64 `json:"failedReplicationCount,omitempty"` - // Total number of failed operations + // Total number of completed operations ReplicatedCount int64 `json:"replicationCount,omitempty"` // Last bucket/object replicated. Bucket string `json:"bucket,omitempty"` Object string `json:"object,omitempty"` } + +// XferStats holds transfer rate info for uploads/sec +type XferStats struct { + AvgRate float64 `json:"avgRate"` + PeakRate float64 `json:"peakRate"` + CurrRate float64 `json:"currRate"` +} + +// InQueueStats holds stats for objects in replication queue +type InQueueStats struct { + Count int32 `json:"count"` + Bytes int64 `json:"bytes"` +} + +// MetricName name of replication metric +type MetricName string + +const ( + // Large is a metric name for large objects >=128MiB + Large MetricName = "Large" + // Small is a metric name for objects <128MiB size + Small MetricName = "Small" + // Total is a metric name for total objects + Total MetricName = "Total" +) + +// ReplQNodeStats holds stats for a node in replication queue +type ReplQNodeStats struct { + NodeName string `json:"nodeName"` + Uptime int64 `json:"uptime"` + ActiveWorkers int32 `json:"activeWorkers"` + + XferStats map[MetricName]XferStats `json:"xferStats"` + QStats map[MetricName]InQueueStats `json:"qStats"` +} + +// ReplQueueStats holds stats for replication queue across nodes +type ReplQueueStats struct { + Nodes []ReplQNodeStats `json:"nodes"` +} + +// Workers returns number of workers across all nodes +func (q ReplQueueStats) Workers() int64 { + var workers int64 + for _, node := range q.Nodes { + workers += int64(node.ActiveWorkers) + } + return workers +} + +// ReplQStats holds stats for objects in replication queue +type ReplQStats struct { + Uptime int64 `json:"uptime"` + Workers int64 `json:"workers"` + + XferStats map[MetricName]XferStats `json:"xferStats"` + QStats map[MetricName]InQueueStats `json:"qStats"` +} + +// QStats returns cluster level stats for objects in replication queue +func (q ReplQueueStats) QStats() (r ReplQStats) { + r.QStats = make(map[MetricName]InQueueStats) + r.XferStats = make(map[MetricName]XferStats) + for _, node := range q.Nodes { + r.Workers += int64(node.ActiveWorkers) + for k, v := range node.XferStats { + st, ok := r.XferStats[k] + if !ok { + st = XferStats{} + } + st.AvgRate += v.AvgRate + st.CurrRate += v.CurrRate + st.PeakRate = math.Max(st.PeakRate, v.PeakRate) + r.XferStats[k] = st + } + for k, v := range node.QStats { + st, ok := r.QStats[k] + if !ok { + st = InQueueStats{} + } + st.Count += v.Count + st.Bytes += v.Bytes + r.QStats[k] = st + } + r.Uptime += node.Uptime + } + if len(q.Nodes) > 0 { + for k := range r.XferStats { + st := r.XferStats[k] + st.AvgRate /= float64(len(q.Nodes)) + st.CurrRate /= float64(len(q.Nodes)) + r.XferStats[k] = st + } + r.Uptime /= int64(len(q.Nodes)) // average uptime + } + + return +} + +// MetricsV2 represents replication metrics for a bucket. +type MetricsV2 struct { + History Metrics `json:"history"` + CurrentStats Metrics `json:"currStats"` + QueueStats ReplQueueStats `json:"queueStats"` +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2b8537efa..579ed2a5a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -386,7 +386,7 @@ github.com/miekg/dns # github.com/minio/md5-simd v1.1.2 ## explicit; go 1.14 github.com/minio/md5-simd -# github.com/minio/minio-go/v7 v7.0.58 +# github.com/minio/minio-go/v7 v7.0.59 ## explicit; go 1.17 github.com/minio/minio-go/v7 github.com/minio/minio-go/v7/pkg/credentials