forked from mirrors/gotosocial
bump go-store version (includes minio) (#1657)
Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
GitHub
parent
0746ef741a
commit
a5c920a50b
33 changed files with 14876 additions and 8512 deletions
8
go.mod
8
go.mod
|
|
@ -14,7 +14,7 @@ require (
|
|||
codeberg.org/gruf/go-mutexes v1.1.5
|
||||
codeberg.org/gruf/go-runners v1.6.1
|
||||
codeberg.org/gruf/go-sched v1.2.3
|
||||
codeberg.org/gruf/go-store/v2 v2.2.1
|
||||
codeberg.org/gruf/go-store/v2 v2.2.2
|
||||
github.com/KimMachineGun/automemlimit v0.2.4
|
||||
github.com/abema/go-mp4 v0.10.1
|
||||
github.com/buckket/go-blurhash v1.1.0
|
||||
|
|
@ -36,7 +36,7 @@ require (
|
|||
github.com/jackc/pgx/v4 v4.18.1
|
||||
github.com/microcosm-cc/bluemonday v1.0.23
|
||||
github.com/miekg/dns v1.1.52
|
||||
github.com/minio/minio-go/v7 v7.0.49
|
||||
github.com/minio/minio-go/v7 v7.0.50
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/oklog/ulid v1.3.1
|
||||
github.com/spf13/cobra v1.6.1
|
||||
|
|
@ -119,8 +119,8 @@ require (
|
|||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.15.15 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
|
||||
github.com/klauspost/compress v1.16.3 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/leodido/go-urn v1.2.2 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
|
|
|
|||
16
go.sum
16
go.sum
|
|
@ -83,8 +83,8 @@ codeberg.org/gruf/go-runners v1.6.1 h1:0KNiEfGnmNUs9intqxEAWqIKUyxVOmYTtn3kPVOHs
|
|||
codeberg.org/gruf/go-runners v1.6.1/go.mod h1:QRcSExqXX8DM0rm8Xs6qX7baOzyvw0JIe4mu3TsQT+Y=
|
||||
codeberg.org/gruf/go-sched v1.2.3 h1:H5ViDxxzOBR3uIyGBCf0eH8b1L8wMybOXcdtUUTXZHk=
|
||||
codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A=
|
||||
codeberg.org/gruf/go-store/v2 v2.2.1 h1:lbvMjhMLebefiaPNLtWvPySKSYM5xN1aztSxxz+vCzU=
|
||||
codeberg.org/gruf/go-store/v2 v2.2.1/go.mod h1:pxdyfSzau8fFs1TfZlyRzhDYvZWLaj1sXpcjXpzBB6k=
|
||||
codeberg.org/gruf/go-store/v2 v2.2.2 h1:S+OpXKXtPpkxJ7GyFZgJISXjXtGtz70//ee/uOi/6ks=
|
||||
codeberg.org/gruf/go-store/v2 v2.2.2/go.mod h1:QRM3LUAfYyoGMWLTqA1WzohxQgYqPFiVv9cqwL0+Uvs=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
|
|
@ -400,13 +400,13 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
|
|||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw=
|
||||
github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4=
|
||||
github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
|
||||
github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
|
|
@ -446,8 +446,8 @@ github.com/miekg/dns v1.1.52 h1:Bmlc/qsNNULOe6bpXcUTsuOajd0DzRHwup6D9k1An0c=
|
|||
github.com/miekg/dns v1.1.52/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.49 h1:dE5DfOtnXMXCjr/HWI6zN9vCrY6Sv666qhhiwUMvGV4=
|
||||
github.com/minio/minio-go/v7 v7.0.49/go.mod h1:UI34MvQEiob3Cf/gGExGMmzugkM/tNgbFypNDy5LMVc=
|
||||
github.com/minio/minio-go/v7 v7.0.50 h1:4IL4V8m/kI90ZL6GupCARZVrBv8/XrcKcJhaJ3iz68k=
|
||||
github.com/minio/minio-go/v7 v7.0.50/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
|
||||
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
||||
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
||||
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
|
||||
|
|
|
|||
12
vendor/codeberg.org/gruf/go-store/v2/storage/disk.go
generated
vendored
12
vendor/codeberg.org/gruf/go-store/v2/storage/disk.go
generated
vendored
|
|
@ -195,7 +195,7 @@ func (st *DiskStorage) ReadBytes(ctx context.Context, key string) ([]byte, error
|
|||
// ReadStream implements Storage.ReadStream().
|
||||
func (st *DiskStorage) ReadStream(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
// Get file path for key
|
||||
kpath, err := st.filepath(key)
|
||||
kpath, err := st.Filepath(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -235,7 +235,7 @@ func (st *DiskStorage) WriteBytes(ctx context.Context, key string, value []byte)
|
|||
// WriteStream implements Storage.WriteStream().
|
||||
func (st *DiskStorage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) {
|
||||
// Get file path for key
|
||||
kpath, err := st.filepath(key)
|
||||
kpath, err := st.Filepath(key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
|
@ -291,7 +291,7 @@ func (st *DiskStorage) WriteStream(ctx context.Context, key string, r io.Reader)
|
|||
// Stat implements Storage.Stat().
|
||||
func (st *DiskStorage) Stat(ctx context.Context, key string) (bool, error) {
|
||||
// Get file path for key
|
||||
kpath, err := st.filepath(key)
|
||||
kpath, err := st.Filepath(key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
|
@ -313,7 +313,7 @@ func (st *DiskStorage) Stat(ctx context.Context, key string) (bool, error) {
|
|||
// Remove implements Storage.Remove().
|
||||
func (st *DiskStorage) Remove(ctx context.Context, key string) error {
|
||||
// Get file path for key
|
||||
kpath, err := st.filepath(key)
|
||||
kpath, err := st.Filepath(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -384,8 +384,8 @@ func (st *DiskStorage) WalkKeys(ctx context.Context, opts WalkKeysOptions) error
|
|||
})
|
||||
}
|
||||
|
||||
// filepath checks and returns a formatted filepath for given key.
|
||||
func (st *DiskStorage) filepath(key string) (string, error) {
|
||||
// Filepath checks and returns a formatted Filepath for given key.
|
||||
func (st *DiskStorage) Filepath(key string) (string, error) {
|
||||
// Calculate transformed key path
|
||||
key = st.config.Transform.KeyToPath(key)
|
||||
|
||||
|
|
|
|||
8
vendor/codeberg.org/gruf/go-store/v2/storage/s3.go
generated
vendored
8
vendor/codeberg.org/gruf/go-store/v2/storage/s3.go
generated
vendored
|
|
@ -15,6 +15,7 @@ var DefaultS3Config = &S3Config{
|
|||
CoreOpts: minio.Options{},
|
||||
GetOpts: minio.GetObjectOptions{},
|
||||
PutOpts: minio.PutObjectOptions{},
|
||||
PutChunkOpts: minio.PutObjectPartOptions{},
|
||||
PutChunkSize: 4 * 1024 * 1024, // 4MiB
|
||||
StatOpts: minio.StatObjectOptions{},
|
||||
RemoveOpts: minio.RemoveObjectOptions{},
|
||||
|
|
@ -37,6 +38,9 @@ type S3Config struct {
|
|||
// a byte stream reader of unknown size as a multi-part object.
|
||||
PutChunkSize int64
|
||||
|
||||
// PutChunkOpts are S3 client options passed during chunked .Write___() calls.
|
||||
PutChunkOpts minio.PutObjectPartOptions
|
||||
|
||||
// StatOpts are S3 client options passed during .Stat() calls.
|
||||
StatOpts minio.StatObjectOptions
|
||||
|
||||
|
|
@ -251,9 +255,7 @@ loop:
|
|||
index,
|
||||
rbuf,
|
||||
int64(n),
|
||||
"",
|
||||
"",
|
||||
nil,
|
||||
st.config.PutChunkOpts,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
|
|
|
|||
186
vendor/github.com/klauspost/compress/s2/README.md
generated
vendored
186
vendor/github.com/klauspost/compress/s2/README.md
generated
vendored
|
|
@ -20,11 +20,12 @@ This is important, so you don't have to worry about spending CPU cycles on alrea
|
|||
* Concurrent stream compression
|
||||
* Faster decompression, even for Snappy compatible content
|
||||
* Concurrent Snappy/S2 stream decompression
|
||||
* Ability to quickly skip forward in compressed stream
|
||||
* Skip forward in compressed stream
|
||||
* Random seeking with indexes
|
||||
* Compatible with reading Snappy compressed content
|
||||
* Smaller block size overhead on incompressible blocks
|
||||
* Block concatenation
|
||||
* Block Dictionary support
|
||||
* Uncompressed stream mode
|
||||
* Automatic stream size padding
|
||||
* Snappy compatible block compression
|
||||
|
|
@ -594,6 +595,123 @@ Best... 10737418240 -> 4210602774 [39.21%]; 42.96s, 254.4MB/s
|
|||
|
||||
Decompression speed should be around the same as using the 'better' compression mode.
|
||||
|
||||
## Dictionaries
|
||||
|
||||
*Note: S2 dictionary compression is currently at an early implementation stage, with no assembly for
|
||||
neither encoding nor decoding. Performance improvements can be expected in the future.*
|
||||
|
||||
Adding dictionaries allow providing a custom dictionary that will serve as lookup in the beginning of blocks.
|
||||
|
||||
The same dictionary *must* be used for both encoding and decoding.
|
||||
S2 does not keep track of whether the same dictionary is used,
|
||||
and using the wrong dictionary will most often not result in an error when decompressing.
|
||||
|
||||
Blocks encoded *without* dictionaries can be decompressed seamlessly *with* a dictionary.
|
||||
This means it is possible to switch from an encoding without dictionaries to an encoding with dictionaries
|
||||
and treat the blocks similarly.
|
||||
|
||||
Similar to [zStandard dictionaries](https://github.com/facebook/zstd#the-case-for-small-data-compression),
|
||||
the same usage scenario applies to S2 dictionaries.
|
||||
|
||||
> Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no universal dictionary). Hence, deploying one dictionary per type of data will provide the greatest benefits. Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
|
||||
|
||||
S2 further limits the dictionary to only be enabled on the first 64KB of a block.
|
||||
This will remove any negative (speed) impacts of the dictionaries on bigger blocks.
|
||||
|
||||
### Compression
|
||||
|
||||
Using the [github_users_sample_set](https://github.com/facebook/zstd/releases/download/v1.1.3/github_users_sample_set.tar.zst)
|
||||
and a 64KB dictionary trained with zStandard the following sizes can be achieved.
|
||||
|
||||
| | Default | Better | Best |
|
||||
|--------------------|------------------|------------------|-----------------------|
|
||||
| Without Dictionary | 3362023 (44.92%) | 3083163 (41.19%) | 3057944 (40.86%) |
|
||||
| With Dictionary | 921524 (12.31%) | 873154 (11.67%) | 785503 bytes (10.49%) |
|
||||
|
||||
So for highly repetitive content, this case provides an almost 3x reduction in size.
|
||||
|
||||
For less uniform data we will use the Go source code tree.
|
||||
Compressing First 64KB of all `.go` files in `go/src`, Go 1.19.5, 8912 files, 51253563 bytes input:
|
||||
|
||||
| | Default | Better | Best |
|
||||
|--------------------|-------------------|-------------------|-------------------|
|
||||
| Without Dictionary | 22955767 (44.79%) | 20189613 (39.39% | 19482828 (38.01%) |
|
||||
| With Dictionary | 19654568 (38.35%) | 16289357 (31.78%) | 15184589 (29.63%) |
|
||||
| Saving/file | 362 bytes | 428 bytes | 472 bytes |
|
||||
|
||||
|
||||
### Creating Dictionaries
|
||||
|
||||
There are no tools to create dictionaries in S2.
|
||||
However, there are multiple ways to create a useful dictionary:
|
||||
|
||||
#### Using a Sample File
|
||||
|
||||
If your input is very uniform, you can just use a sample file as the dictionary.
|
||||
|
||||
For example in the `github_users_sample_set` above, the average compression only goes up from
|
||||
10.49% to 11.48% by using the first file as dictionary compared to using a dedicated dictionary.
|
||||
|
||||
```Go
|
||||
// Read a sample
|
||||
sample, err := os.ReadFile("sample.json")
|
||||
|
||||
// Create a dictionary.
|
||||
dict := s2.MakeDict(sample, nil)
|
||||
|
||||
// b := dict.Bytes() will provide a dictionary that can be saved
|
||||
// and reloaded with s2.NewDict(b).
|
||||
|
||||
// To encode:
|
||||
encoded := dict.Encode(nil, file)
|
||||
|
||||
// To decode:
|
||||
decoded, err := dict.Decode(nil, file)
|
||||
```
|
||||
|
||||
#### Using Zstandard
|
||||
|
||||
Zstandard dictionaries can easily be converted to S2 dictionaries.
|
||||
|
||||
This can be helpful to generate dictionaries for files that don't have a fixed structure.
|
||||
|
||||
|
||||
Example, with training set files placed in `./training-set`:
|
||||
|
||||
`λ zstd -r --train-fastcover training-set/* --maxdict=65536 -o name.dict`
|
||||
|
||||
This will create a dictionary of 64KB, that can be converted to a dictionary like this:
|
||||
|
||||
```Go
|
||||
// Decode the Zstandard dictionary.
|
||||
insp, err := zstd.InspectDictionary(zdict)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// We are only interested in the contents.
|
||||
// Assume that files start with "// Copyright (c) 2023".
|
||||
// Search for the longest match for that.
|
||||
// This may save a few bytes.
|
||||
dict := s2.MakeDict(insp.Content(), []byte("// Copyright (c) 2023"))
|
||||
|
||||
// b := dict.Bytes() will provide a dictionary that can be saved
|
||||
// and reloaded with s2.NewDict(b).
|
||||
|
||||
// We can now encode using this dictionary
|
||||
encodedWithDict := dict.Encode(nil, payload)
|
||||
|
||||
// To decode content:
|
||||
decoded, err := dict.Decode(nil, encodedWithDict)
|
||||
```
|
||||
|
||||
It is recommended to save the dictionary returned by ` b:= dict.Bytes()`, since that will contain only the S2 dictionary.
|
||||
|
||||
This dictionary can later be loaded using `s2.NewDict(b)`. The dictionary then no longer requires `zstd` to be initialized.
|
||||
|
||||
Also note how `s2.MakeDict` allows you to search for a common starting sequence of your files.
|
||||
This can be omitted, at the expense of a few bytes.
|
||||
|
||||
# Snappy Compatibility
|
||||
|
||||
S2 now offers full compatibility with Snappy.
|
||||
|
|
@ -929,6 +1047,72 @@ The first copy of a block cannot be a repeat offset and the offset is reset on e
|
|||
|
||||
Default streaming block size is 1MB.
|
||||
|
||||
# Dictionary Encoding
|
||||
|
||||
Adding dictionaries allow providing a custom dictionary that will serve as lookup in the beginning of blocks.
|
||||
|
||||
A dictionary provides an initial repeat value that can be used to point to a common header.
|
||||
|
||||
Other than that the dictionary contains values that can be used as back-references.
|
||||
|
||||
Often used data should be placed at the *end* of the dictionary since offsets < 2048 bytes will be smaller.
|
||||
|
||||
## Format
|
||||
|
||||
Dictionary *content* must at least 16 bytes and less or equal to 64KiB (65536 bytes).
|
||||
|
||||
Encoding: `[repeat value (uvarint)][dictionary content...]`
|
||||
|
||||
Before the dictionary content, an unsigned base-128 (uvarint) encoded value specifying the initial repeat offset.
|
||||
This value is an offset into the dictionary content and not a back-reference offset,
|
||||
so setting this to 0 will make the repeat value point to the first value of the dictionary.
|
||||
|
||||
The value must be less than the dictionary length-8
|
||||
|
||||
## Encoding
|
||||
|
||||
From the decoder point of view the dictionary content is seen as preceding the encoded content.
|
||||
|
||||
`[dictionary content][decoded output]`
|
||||
|
||||
Backreferences to the dictionary are encoded as ordinary backreferences that have an offset before the start of the decoded block.
|
||||
|
||||
Matches copying from the dictionary are **not** allowed to cross from the dictionary into the decoded data.
|
||||
However, if a copy ends at the end of the dictionary the next repeat will point to the start of the decoded buffer, which is allowed.
|
||||
|
||||
The first match can be a repeat value, which will use the repeat offset stored in the dictionary.
|
||||
|
||||
When 64KB (65536 bytes) has been en/decoded it is no longer allowed to reference the dictionary,
|
||||
neither by a copy nor repeat operations.
|
||||
If the boundary is crossed while copying from the dictionary, the operation should complete,
|
||||
but the next instruction is not allowed to reference the dictionary.
|
||||
|
||||
Valid blocks encoded *without* a dictionary can be decoded with any dictionary.
|
||||
There are no checks whether the supplied dictionary is the correct for a block.
|
||||
Because of this there is no overhead by using a dictionary.
|
||||
|
||||
## Example
|
||||
|
||||
This is the dictionary content. Elements are separated by `[]`.
|
||||
|
||||
Dictionary: `[0x0a][Yesterday 25 bananas were added to Benjamins brown bag]`.
|
||||
|
||||
Initial repeat offset is set at 10, which is the letter `2`.
|
||||
|
||||
Encoded `[LIT "10"][REPEAT len=10][LIT "hich"][MATCH off=50 len=6][MATCH off=31 len=6][MATCH off=61 len=10]`
|
||||
|
||||
Decoded: `[10][ bananas w][hich][ were ][brown ][were added]`
|
||||
|
||||
Output: `10 bananas which were brown were added`
|
||||
|
||||
|
||||
## Streams
|
||||
|
||||
For streams each block can use the dictionary.
|
||||
|
||||
The dictionary cannot not currently be provided on the stream.
|
||||
|
||||
|
||||
# LICENSE
|
||||
|
||||
This code is based on the [Snappy-Go](https://github.com/golang/snappy) implementation.
|
||||
|
|
|
|||
489
vendor/github.com/klauspost/compress/s2/decode.go
generated
vendored
489
vendor/github.com/klauspost/compress/s2/decode.go
generated
vendored
|
|
@ -13,6 +13,7 @@ import (
|
|||
"io/ioutil"
|
||||
"math"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
)
|
||||
|
||||
|
|
@ -880,15 +881,20 @@ func (r *Reader) Skip(n int64) error {
|
|||
// See Reader.ReadSeeker
|
||||
type ReadSeeker struct {
|
||||
*Reader
|
||||
readAtMu sync.Mutex
|
||||
}
|
||||
|
||||
// ReadSeeker will return an io.ReadSeeker compatible version of the reader.
|
||||
// ReadSeeker will return an io.ReadSeeker and io.ReaderAt
|
||||
// compatible version of the reader.
|
||||
// If 'random' is specified the returned io.Seeker can be used for
|
||||
// random seeking, otherwise only forward seeking is supported.
|
||||
// Enabling random seeking requires the original input to support
|
||||
// the io.Seeker interface.
|
||||
// A custom index can be specified which will be used if supplied.
|
||||
// When using a custom index, it will not be read from the input stream.
|
||||
// The ReadAt position will affect regular reads and the current position of Seek.
|
||||
// So using Read after ReadAt will continue from where the ReadAt stopped.
|
||||
// No functions should be used concurrently.
|
||||
// The returned ReadSeeker contains a shallow reference to the existing Reader,
|
||||
// meaning changes performed to one is reflected in the other.
|
||||
func (r *Reader) ReadSeeker(random bool, index []byte) (*ReadSeeker, error) {
|
||||
|
|
@ -958,42 +964,55 @@ func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
|||
// Reset on EOF
|
||||
r.err = nil
|
||||
}
|
||||
if offset == 0 && whence == io.SeekCurrent {
|
||||
return r.blockStart + int64(r.i), nil
|
||||
}
|
||||
if !r.readHeader {
|
||||
// Make sure we read the header.
|
||||
_, r.err = r.Read([]byte{})
|
||||
}
|
||||
rs, ok := r.r.(io.ReadSeeker)
|
||||
if r.index == nil || !ok {
|
||||
if whence == io.SeekCurrent && offset >= 0 {
|
||||
err := r.Skip(offset)
|
||||
return r.blockStart + int64(r.i), err
|
||||
}
|
||||
if whence == io.SeekStart && offset >= r.blockStart+int64(r.i) {
|
||||
err := r.Skip(offset - r.blockStart - int64(r.i))
|
||||
return r.blockStart + int64(r.i), err
|
||||
}
|
||||
return 0, ErrUnsupported
|
||||
|
||||
}
|
||||
// Calculate absolute offset.
|
||||
absOffset := offset
|
||||
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
case io.SeekCurrent:
|
||||
offset += r.blockStart + int64(r.i)
|
||||
absOffset = r.blockStart + int64(r.i) + offset
|
||||
case io.SeekEnd:
|
||||
if offset > 0 {
|
||||
return 0, errors.New("seek after end of file")
|
||||
if r.index == nil {
|
||||
return 0, ErrUnsupported
|
||||
}
|
||||
offset = r.index.TotalUncompressed + offset
|
||||
absOffset = r.index.TotalUncompressed + offset
|
||||
default:
|
||||
r.err = ErrUnsupported
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
if offset < 0 {
|
||||
if absOffset < 0 {
|
||||
return 0, errors.New("seek before start of file")
|
||||
}
|
||||
|
||||
c, u, err := r.index.Find(offset)
|
||||
if !r.readHeader {
|
||||
// Make sure we read the header.
|
||||
_, r.err = r.Read([]byte{})
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
}
|
||||
|
||||
// If we are inside current block no need to seek.
|
||||
// This includes no offset changes.
|
||||
if absOffset >= r.blockStart && absOffset < r.blockStart+int64(r.j) {
|
||||
r.i = int(absOffset - r.blockStart)
|
||||
return r.blockStart + int64(r.i), nil
|
||||
}
|
||||
|
||||
rs, ok := r.r.(io.ReadSeeker)
|
||||
if r.index == nil || !ok {
|
||||
currOffset := r.blockStart + int64(r.i)
|
||||
if absOffset >= currOffset {
|
||||
err := r.Skip(absOffset - currOffset)
|
||||
return r.blockStart + int64(r.i), err
|
||||
}
|
||||
return 0, ErrUnsupported
|
||||
}
|
||||
|
||||
// We can seek and we have an index.
|
||||
c, u, err := r.index.Find(absOffset)
|
||||
if err != nil {
|
||||
return r.blockStart + int64(r.i), err
|
||||
}
|
||||
|
|
@ -1005,11 +1024,56 @@ func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
|||
}
|
||||
|
||||
r.i = r.j // Remove rest of current block.
|
||||
if u < offset {
|
||||
r.blockStart = u - int64(r.j) // Adjust current block start for accounting.
|
||||
if u < absOffset {
|
||||
// Forward inside block
|
||||
return offset, r.Skip(offset - u)
|
||||
return absOffset, r.Skip(absOffset - u)
|
||||
}
|
||||
return offset, nil
|
||||
if u > absOffset {
|
||||
return 0, fmt.Errorf("s2 seek: (internal error) u (%d) > absOffset (%d)", u, absOffset)
|
||||
}
|
||||
return absOffset, nil
|
||||
}
|
||||
|
||||
// ReadAt reads len(p) bytes into p starting at offset off in the
|
||||
// underlying input source. It returns the number of bytes
|
||||
// read (0 <= n <= len(p)) and any error encountered.
|
||||
//
|
||||
// When ReadAt returns n < len(p), it returns a non-nil error
|
||||
// explaining why more bytes were not returned. In this respect,
|
||||
// ReadAt is stricter than Read.
|
||||
//
|
||||
// Even if ReadAt returns n < len(p), it may use all of p as scratch
|
||||
// space during the call. If some data is available but not len(p) bytes,
|
||||
// ReadAt blocks until either all the data is available or an error occurs.
|
||||
// In this respect ReadAt is different from Read.
|
||||
//
|
||||
// If the n = len(p) bytes returned by ReadAt are at the end of the
|
||||
// input source, ReadAt may return either err == EOF or err == nil.
|
||||
//
|
||||
// If ReadAt is reading from an input source with a seek offset,
|
||||
// ReadAt should not affect nor be affected by the underlying
|
||||
// seek offset.
|
||||
//
|
||||
// Clients of ReadAt can execute parallel ReadAt calls on the
|
||||
// same input source. This is however not recommended.
|
||||
func (r *ReadSeeker) ReadAt(p []byte, offset int64) (int, error) {
|
||||
r.readAtMu.Lock()
|
||||
defer r.readAtMu.Unlock()
|
||||
_, err := r.Seek(offset, io.SeekStart)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n := 0
|
||||
for n < len(p) {
|
||||
n2, err := r.Read(p[n:])
|
||||
if err != nil {
|
||||
// This will include io.EOF
|
||||
return n + n2, err
|
||||
}
|
||||
n += n2
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// ReadByte satisfies the io.ByteReader interface.
|
||||
|
|
@ -1048,3 +1112,370 @@ func (r *Reader) SkippableCB(id uint8, fn func(r io.Reader) error) error {
|
|||
r.skippableCB[id] = fn
|
||||
return nil
|
||||
}
|
||||
|
||||
// s2DecodeDict writes the decoding of src to dst. It assumes that the varint-encoded
|
||||
// length of the decompressed bytes has already been read, and that len(dst)
|
||||
// equals that length.
|
||||
//
|
||||
// It returns 0 on success or a decodeErrCodeXxx error code on failure.
|
||||
func s2DecodeDict(dst, src []byte, dict *Dict) int {
|
||||
if dict == nil {
|
||||
return s2Decode(dst, src)
|
||||
}
|
||||
const debug = false
|
||||
const debugErrs = debug
|
||||
|
||||
if debug {
|
||||
fmt.Println("Starting decode, dst len:", len(dst))
|
||||
}
|
||||
var d, s, length int
|
||||
offset := len(dict.dict) - dict.repeat
|
||||
|
||||
// As long as we can read at least 5 bytes...
|
||||
for s < len(src)-5 {
|
||||
// Removing bounds checks is SLOWER, when if doing
|
||||
// in := src[s:s+5]
|
||||
// Checked on Go 1.18
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
in := src[s : s+3]
|
||||
x = uint32(in[1]) | uint32(in[2])<<8
|
||||
s += 3
|
||||
case x == 62:
|
||||
in := src[s : s+4]
|
||||
// Load as 32 bit and shift down.
|
||||
x = uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24
|
||||
x >>= 8
|
||||
s += 4
|
||||
case x == 63:
|
||||
in := src[s : s+5]
|
||||
x = uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24
|
||||
s += 5
|
||||
}
|
||||
length = int(x) + 1
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debugErrs {
|
||||
fmt.Println("corrupt literal: length:", length, "d-left:", len(dst)-d, "src-left:", len(src)-s)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
length = int(src[s]) + 4
|
||||
s += 1
|
||||
case 6:
|
||||
in := src[s : s+2]
|
||||
length = int(uint32(in[0])|(uint32(in[1])<<8)) + (1 << 8)
|
||||
s += 2
|
||||
case 7:
|
||||
in := src[s : s+3]
|
||||
length = int((uint32(in[2])<<16)|(uint32(in[1])<<8)|uint32(in[0])) + (1 << 16)
|
||||
s += 3
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
in := src[s : s+3]
|
||||
offset = int(uint32(in[1]) | uint32(in[2])<<8)
|
||||
length = 1 + int(in[0])>>2
|
||||
s += 3
|
||||
|
||||
case tagCopy4:
|
||||
in := src[s : s+5]
|
||||
offset = int(uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24)
|
||||
length = 1 + int(in[0])>>2
|
||||
s += 5
|
||||
}
|
||||
|
||||
if offset <= 0 || length > len(dst)-d {
|
||||
if debugErrs {
|
||||
fmt.Println("match error; offset:", offset, "length:", length, "dst-left:", len(dst)-d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
// copy from dict
|
||||
if d < offset {
|
||||
if d > MaxDictSrcOffset {
|
||||
if debugErrs {
|
||||
fmt.Println("dict after", MaxDictSrcOffset, "d:", d, "offset:", offset, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
startOff := len(dict.dict) - offset + d
|
||||
if startOff < 0 || startOff+length > len(dict.dict) {
|
||||
if debugErrs {
|
||||
fmt.Printf("offset (%d) + length (%d) bigger than dict (%d)\n", offset, length, len(dict.dict))
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("dict copy, length:", length, "offset:", offset, "d-after:", d+length, "dict start offset:", startOff)
|
||||
}
|
||||
copy(dst[d:d+length], dict.dict[startOff:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
// Remaining with extra checks...
|
||||
for s < len(src) {
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
case x == 62:
|
||||
s += 4
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
case x == 63:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debugErrs {
|
||||
fmt.Println("corrupt literal: length:", length, "d-left:", len(dst)-d, "src-left:", len(src)-s)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
s += 1
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-1])) + 4
|
||||
case 6:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
|
||||
case 7:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-3])>>2
|
||||
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
|
||||
case tagCopy4:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-5])>>2
|
||||
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
}
|
||||
|
||||
if offset <= 0 || length > len(dst)-d {
|
||||
if debugErrs {
|
||||
fmt.Println("match error; offset:", offset, "length:", length, "dst-left:", len(dst)-d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
// copy from dict
|
||||
if d < offset {
|
||||
if d > MaxDictSrcOffset {
|
||||
if debugErrs {
|
||||
fmt.Println("dict after", MaxDictSrcOffset, "d:", d, "offset:", offset, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
rOff := len(dict.dict) - (offset - d)
|
||||
if debug {
|
||||
fmt.Println("starting dict entry from dict offset", len(dict.dict)-rOff)
|
||||
}
|
||||
if rOff+length > len(dict.dict) {
|
||||
if debugErrs {
|
||||
fmt.Println("err: END offset", rOff+length, "bigger than dict", len(dict.dict), "dict offset:", rOff, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if rOff < 0 {
|
||||
if debugErrs {
|
||||
fmt.Println("err: START offset", rOff, "less than 0", len(dict.dict), "dict offset:", rOff, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
copy(dst[d:d+length], dict.dict[rOff:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
if d != len(dst) {
|
||||
if debugErrs {
|
||||
fmt.Println("wanted length", len(dst), "got", d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
|
|
|||
13
vendor/github.com/klauspost/compress/s2/decode_other.go
generated
vendored
13
vendor/github.com/klauspost/compress/s2/decode_other.go
generated
vendored
|
|
@ -57,6 +57,9 @@ func s2Decode(dst, src []byte) int {
|
|||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debug {
|
||||
fmt.Println("corrupt: lit size", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
|
|
@ -109,6 +112,10 @@ func s2Decode(dst, src []byte) int {
|
|||
}
|
||||
|
||||
if offset <= 0 || d < offset || length > len(dst)-d {
|
||||
if debug {
|
||||
fmt.Println("corrupt: match, length", length, "offset:", offset, "dst avail:", len(dst)-d, "dst pos:", d)
|
||||
}
|
||||
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
|
|
@ -175,6 +182,9 @@ func s2Decode(dst, src []byte) int {
|
|||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debug {
|
||||
fmt.Println("corrupt: lit size", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
|
|
@ -241,6 +251,9 @@ func s2Decode(dst, src []byte) int {
|
|||
}
|
||||
|
||||
if offset <= 0 || d < offset || length > len(dst)-d {
|
||||
if debug {
|
||||
fmt.Println("corrupt: match, length", length, "offset:", offset, "dst avail:", len(dst)-d, "dst pos:", d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
|
|
|
|||
331
vendor/github.com/klauspost/compress/s2/dict.go
generated
vendored
Normal file
331
vendor/github.com/klauspost/compress/s2/dict.go
generated
vendored
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
// Copyright (c) 2022+ Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
// MinDictSize is the minimum dictionary size when repeat has been read.
|
||||
MinDictSize = 16
|
||||
|
||||
// MaxDictSize is the maximum dictionary size when repeat has been read.
|
||||
MaxDictSize = 65536
|
||||
|
||||
// MaxDictSrcOffset is the maximum offset where a dictionary entry can start.
|
||||
MaxDictSrcOffset = 65535
|
||||
)
|
||||
|
||||
// Dict contains a dictionary that can be used for encoding and decoding s2
|
||||
type Dict struct {
|
||||
dict []byte
|
||||
repeat int // Repeat as index of dict
|
||||
|
||||
fast, better, best sync.Once
|
||||
fastTable *[1 << 14]uint16
|
||||
|
||||
betterTableShort *[1 << 14]uint16
|
||||
betterTableLong *[1 << 17]uint16
|
||||
|
||||
bestTableShort *[1 << 16]uint32
|
||||
bestTableLong *[1 << 19]uint32
|
||||
}
|
||||
|
||||
// NewDict will read a dictionary.
|
||||
// It will return nil if the dictionary is invalid.
|
||||
func NewDict(dict []byte) *Dict {
|
||||
if len(dict) == 0 {
|
||||
return nil
|
||||
}
|
||||
var d Dict
|
||||
// Repeat is the first value of the dict
|
||||
r, n := binary.Uvarint(dict)
|
||||
if n <= 0 {
|
||||
return nil
|
||||
}
|
||||
dict = dict[n:]
|
||||
d.dict = dict
|
||||
if cap(d.dict) < len(d.dict)+16 {
|
||||
d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...)
|
||||
}
|
||||
if len(dict) < MinDictSize || len(dict) > MaxDictSize {
|
||||
return nil
|
||||
}
|
||||
d.repeat = int(r)
|
||||
if d.repeat > len(dict) {
|
||||
return nil
|
||||
}
|
||||
return &d
|
||||
}
|
||||
|
||||
// Bytes will return a serialized version of the dictionary.
|
||||
// The output can be sent to NewDict.
|
||||
func (d *Dict) Bytes() []byte {
|
||||
dst := make([]byte, binary.MaxVarintLen16+len(d.dict))
|
||||
return append(dst[:binary.PutUvarint(dst, uint64(d.repeat))], d.dict...)
|
||||
}
|
||||
|
||||
// MakeDict will create a dictionary.
|
||||
// 'data' must be at least MinDictSize.
|
||||
// If data is longer than MaxDictSize only the last MaxDictSize bytes will be used.
|
||||
// If searchStart is set the start repeat value will be set to the last
|
||||
// match of this content.
|
||||
// If no matches are found, it will attempt to find shorter matches.
|
||||
// This content should match the typical start of a block.
|
||||
// If at least 4 bytes cannot be matched, repeat is set to start of block.
|
||||
func MakeDict(data []byte, searchStart []byte) *Dict {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(data) > MaxDictSize {
|
||||
data = data[len(data)-MaxDictSize:]
|
||||
}
|
||||
var d Dict
|
||||
dict := data
|
||||
d.dict = dict
|
||||
if cap(d.dict) < len(d.dict)+16 {
|
||||
d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...)
|
||||
}
|
||||
if len(dict) < MinDictSize {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find the longest match possible, last entry if multiple.
|
||||
for s := len(searchStart); s > 4; s-- {
|
||||
if idx := bytes.LastIndex(data, searchStart[:s]); idx >= 0 && idx <= len(data)-8 {
|
||||
d.repeat = idx
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return &d
|
||||
}
|
||||
|
||||
// Encode returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) Encode(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if cap(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
} else {
|
||||
dst = dst[:n]
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:dstP]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
n := encodeBlockDictGo(dst[dstP:], src, d)
|
||||
if n > 0 {
|
||||
dstP += n
|
||||
return dst[:dstP]
|
||||
}
|
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
|
||||
// EncodeBetter returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// EncodeBetter compresses better than Encode but typically with a
|
||||
// 10-40% speed decrease on both compression and decompression.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) EncodeBetter(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if len(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:dstP]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
n := encodeBlockBetterDict(dst[dstP:], src, d)
|
||||
if n > 0 {
|
||||
dstP += n
|
||||
return dst[:dstP]
|
||||
}
|
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
|
||||
// EncodeBest returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// EncodeBest compresses as good as reasonably possible but with a
|
||||
// big speed decrease.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) EncodeBest(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if len(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:dstP]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
n := encodeBlockBest(dst[dstP:], src, d)
|
||||
if n > 0 {
|
||||
dstP += n
|
||||
return dst[:dstP]
|
||||
}
|
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
|
||||
// Decode returns the decoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire decoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
func (d *Dict) Decode(dst, src []byte) ([]byte, error) {
|
||||
dLen, s, err := decodedLen(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if dLen <= cap(dst) {
|
||||
dst = dst[:dLen]
|
||||
} else {
|
||||
dst = make([]byte, dLen)
|
||||
}
|
||||
if s2DecodeDict(dst, src[s:], d) != 0 {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
func (d *Dict) initFast() {
|
||||
d.fast.Do(func() {
|
||||
const (
|
||||
tableBits = 14
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint16
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8-2; i += 3 {
|
||||
x0 := load64(d.dict, i)
|
||||
h0 := hash6(x0, tableBits)
|
||||
h1 := hash6(x0>>8, tableBits)
|
||||
h2 := hash6(x0>>16, tableBits)
|
||||
table[h0] = uint16(i)
|
||||
table[h1] = uint16(i + 1)
|
||||
table[h2] = uint16(i + 2)
|
||||
}
|
||||
d.fastTable = &table
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Dict) initBetter() {
|
||||
d.better.Do(func() {
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 17
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 14
|
||||
maxSTableSize = 1 << sTableBits
|
||||
)
|
||||
|
||||
var lTable [maxLTableSize]uint16
|
||||
var sTable [maxSTableSize]uint16
|
||||
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8; i++ {
|
||||
cv := load64(d.dict, i)
|
||||
lTable[hash7(cv, lTableBits)] = uint16(i)
|
||||
sTable[hash4(cv, sTableBits)] = uint16(i)
|
||||
}
|
||||
d.betterTableShort = &sTable
|
||||
d.betterTableLong = &lTable
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Dict) initBest() {
|
||||
d.best.Do(func() {
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 19
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 16
|
||||
maxSTableSize = 1 << sTableBits
|
||||
)
|
||||
|
||||
var lTable [maxLTableSize]uint32
|
||||
var sTable [maxSTableSize]uint32
|
||||
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8; i++ {
|
||||
cv := load64(d.dict, i)
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
candidateS := sTable[hashS]
|
||||
lTable[hashL] = uint32(i) | candidateL<<16
|
||||
sTable[hashS] = uint32(i) | candidateS<<16
|
||||
}
|
||||
d.bestTableShort = &sTable
|
||||
d.bestTableLong = &lTable
|
||||
})
|
||||
}
|
||||
46
vendor/github.com/klauspost/compress/s2/encode.go
generated
vendored
46
vendor/github.com/klauspost/compress/s2/encode.go
generated
vendored
|
|
@ -58,6 +58,32 @@ func Encode(dst, src []byte) []byte {
|
|||
return dst[:d]
|
||||
}
|
||||
|
||||
// EstimateBlockSize will perform a very fast compression
|
||||
// without outputting the result and return the compressed output size.
|
||||
// The function returns -1 if no improvement could be achieved.
|
||||
// Using actual compression will most often produce better compression than the estimate.
|
||||
func EstimateBlockSize(src []byte) (d int) {
|
||||
if len(src) < 6 || int64(len(src)) > 0xffffffff {
|
||||
return -1
|
||||
}
|
||||
if len(src) <= 1024 {
|
||||
d = calcBlockSizeSmall(src)
|
||||
} else {
|
||||
d = calcBlockSize(src)
|
||||
}
|
||||
|
||||
if d == 0 {
|
||||
return -1
|
||||
}
|
||||
// Size of the varint encoded block size.
|
||||
d += (bits.Len64(uint64(len(src))) + 7) / 7
|
||||
|
||||
if d >= len(src) {
|
||||
return -1
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// EncodeBetter returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
|
|
@ -132,7 +158,7 @@ func EncodeBest(dst, src []byte) []byte {
|
|||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
n := encodeBlockBest(dst[d:], src)
|
||||
n := encodeBlockBest(dst[d:], src, nil)
|
||||
if n > 0 {
|
||||
d += n
|
||||
return dst[:d]
|
||||
|
|
@ -408,6 +434,7 @@ type Writer struct {
|
|||
randSrc io.Reader
|
||||
writerWg sync.WaitGroup
|
||||
index Index
|
||||
customEnc func(dst, src []byte) int
|
||||
|
||||
// wroteStreamHeader is whether we have written the stream header.
|
||||
wroteStreamHeader bool
|
||||
|
|
@ -773,6 +800,9 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) {
|
|||
}
|
||||
|
||||
func (w *Writer) encodeBlock(obuf, uncompressed []byte) int {
|
||||
if w.customEnc != nil {
|
||||
return w.customEnc(obuf, uncompressed)
|
||||
}
|
||||
if w.snappy {
|
||||
switch w.level {
|
||||
case levelFast:
|
||||
|
|
@ -790,7 +820,7 @@ func (w *Writer) encodeBlock(obuf, uncompressed []byte) int {
|
|||
case levelBetter:
|
||||
return encodeBlockBetter(obuf, uncompressed)
|
||||
case levelBest:
|
||||
return encodeBlockBest(obuf, uncompressed)
|
||||
return encodeBlockBest(obuf, uncompressed, nil)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
|
@ -1339,3 +1369,15 @@ func WriterFlushOnWrite() WriterOption {
|
|||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WriterCustomEncoder allows to override the encoder for blocks on the stream.
|
||||
// The function must compress 'src' into 'dst' and return the bytes used in dst as an integer.
|
||||
// Block size (initial varint) should not be added by the encoder.
|
||||
// Returning value 0 indicates the block could not be compressed.
|
||||
// The function should expect to be called concurrently.
|
||||
func WriterCustomEncoder(fn func(dst, src []byte) int) WriterOption {
|
||||
return func(w *Writer) error {
|
||||
w.customEnc = fn
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
|
|
|||
592
vendor/github.com/klauspost/compress/s2/encode_all.go
generated
vendored
592
vendor/github.com/klauspost/compress/s2/encode_all.go
generated
vendored
|
|
@ -8,6 +8,7 @@ package s2
|
|||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
|
|
@ -455,3 +456,594 @@ emitRemainder:
|
|||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// encodeBlockGo encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockDictGo(dst, src []byte, dict *Dict) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 14
|
||||
maxTableSize = 1 << tableBits
|
||||
maxAhead = 8 // maximum bytes ahead without checking sLimit
|
||||
|
||||
debug = false
|
||||
)
|
||||
dict.initFast()
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
if sLimit > MaxDictSrcOffset-maxAhead {
|
||||
sLimit = MaxDictSrcOffset - maxAhead
|
||||
}
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form can start with a dict entry (copy or repeat).
|
||||
s := 0
|
||||
|
||||
// Convert dict repeat to offset
|
||||
repeat := len(dict.dict) - dict.repeat
|
||||
cv := load64(src, 0)
|
||||
|
||||
// While in dict
|
||||
searchDict:
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
if nextS > sLimit {
|
||||
if debug {
|
||||
fmt.Println("slimit reached", s, nextS)
|
||||
}
|
||||
break searchDict
|
||||
}
|
||||
candidateDict := int(dict.fastTable[hash0])
|
||||
candidateDict2 := int(dict.fastTable[hash1])
|
||||
candidate2 := int(table[hash1])
|
||||
candidate := int(table[hash0])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
|
||||
if repeat > s {
|
||||
candidate := len(dict.dict) - repeat + s
|
||||
if repeat-s >= 4 && uint32(cv) == load32(dict.dict, candidate) {
|
||||
// Extend back
|
||||
base := s
|
||||
for i := candidate; base > nextEmit && i > 0 && dict.dict[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("emitted ", base-nextEmit, "literals")
|
||||
}
|
||||
s += 4
|
||||
candidate += 4
|
||||
for candidate < len(dict.dict)-8 && s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(dict.dict, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
if debug {
|
||||
fmt.Println("emitted dict repeat length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
} else if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("emitted ", base-nextEmit, "literals")
|
||||
}
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
if nextEmit > 0 {
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("emitted reg repeat", s-base, "s:", s)
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue searchDict
|
||||
}
|
||||
if s == 0 {
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
continue searchDict
|
||||
}
|
||||
// Start with table. These matches will always be closer.
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
goto emitMatch
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
goto emitMatch
|
||||
}
|
||||
|
||||
// Check dict. Dicts have longer offsets, so we want longer matches.
|
||||
if cv == load64(dict.dict, candidateDict) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
goto emitDict
|
||||
}
|
||||
|
||||
candidateDict = int(dict.fastTable[hash2])
|
||||
// Check if upper 7 bytes match
|
||||
if candidateDict2 >= 1 {
|
||||
if cv^load64(dict.dict, candidateDict2-1) < (1 << 8) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidateDict = candidateDict2
|
||||
s++
|
||||
goto emitDict
|
||||
}
|
||||
}
|
||||
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
goto emitMatch
|
||||
}
|
||||
if candidateDict >= 2 {
|
||||
// Check if upper 6 bytes match
|
||||
if cv^load64(dict.dict, candidateDict-2) < (1 << 16) {
|
||||
s += 2
|
||||
goto emitDict
|
||||
}
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
continue searchDict
|
||||
|
||||
emitDict:
|
||||
{
|
||||
if debug {
|
||||
if load32(dict.dict, candidateDict) != load32(src, s) {
|
||||
panic("dict emit mismatch")
|
||||
}
|
||||
}
|
||||
// Extend backwards.
|
||||
// The top bytes will be rechecked to get the full match.
|
||||
for candidateDict > 0 && s > nextEmit && dict.dict[candidateDict-1] == src[s-1] {
|
||||
candidateDict--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", s-nextEmit, "literals")
|
||||
}
|
||||
{
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = s + (len(dict.dict)) - candidateDict
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidateDict += 4
|
||||
for s <= len(src)-8 && len(dict.dict)-candidateDict >= 8 {
|
||||
if diff := load64(src, s) ^ load64(dict.dict, candidateDict); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidateDict += 8
|
||||
}
|
||||
|
||||
// Matches longer than 64 are split.
|
||||
if s <= sLimit || s-base < 8 {
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
} else {
|
||||
// Split to ensure we don't start a copy within next block
|
||||
d += emitCopy(dst[d:], repeat, 4)
|
||||
d += emitRepeat(dst[d:], repeat, s-base-4)
|
||||
}
|
||||
if false {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := dict.dict[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("emitted dict copy, length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
|
||||
// Index and continue loop to try new candidate.
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>8, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s - 1)
|
||||
cv = load64(src, s)
|
||||
}
|
||||
continue
|
||||
}
|
||||
emitMatch:
|
||||
|
||||
// Extend backwards.
|
||||
// The top bytes will be rechecked to get the full match.
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", s-nextEmit, "literals")
|
||||
}
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("emitted src copy, length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if debug && s == candidate {
|
||||
panic("s == candidate")
|
||||
}
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Search without dict:
|
||||
if repeat > s {
|
||||
repeat = 0
|
||||
}
|
||||
|
||||
// No more dict
|
||||
sLimit = len(src) - inputMargin
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("non-dict matching at", s, "repeat:", repeat)
|
||||
}
|
||||
cv = load64(src, s)
|
||||
if debug {
|
||||
fmt.Println("now", s, "->", sLimit, "out:", d, "left:", len(src)-s, "nextemit:", nextEmit, "dstLimit:", dstLimit, "s:", s)
|
||||
}
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if repeat > 0 && uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("emitted ", base-nextEmit, "literals")
|
||||
}
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
if nextEmit > 0 {
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("emitted src repeat length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards.
|
||||
// The top bytes will be rechecked to get the full match.
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", s-nextEmit, "literals")
|
||||
}
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("emitted src copy, length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if debug && s == candidate {
|
||||
panic("s == candidate")
|
||||
}
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", len(src)-nextEmit, "literals")
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
|
|
|||
2
vendor/github.com/klauspost/compress/s2/encode_amd64.go
generated
vendored
2
vendor/github.com/klauspost/compress/s2/encode_amd64.go
generated
vendored
|
|
@ -3,6 +3,8 @@
|
|||
|
||||
package s2
|
||||
|
||||
const hasAmd64Asm = true
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
|
|
|
|||
174
vendor/github.com/klauspost/compress/s2/encode_best.go
generated
vendored
174
vendor/github.com/klauspost/compress/s2/encode_best.go
generated
vendored
|
|
@ -7,6 +7,7 @@ package s2
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
|
|
@ -18,7 +19,7 @@ import (
|
|||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBest(dst, src []byte) (d int) {
|
||||
func encodeBlockBest(dst, src []byte, dict *Dict) (d int) {
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
|
|
@ -30,6 +31,8 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
maxSTableSize = 1 << sTableBits
|
||||
|
||||
inputMargin = 8 + 2
|
||||
|
||||
debug = false
|
||||
)
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
|
|
@ -39,6 +42,10 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
sLimitDict := len(src) - inputMargin
|
||||
if sLimitDict > MaxDictSrcOffset-inputMargin {
|
||||
sLimitDict = MaxDictSrcOffset - inputMargin
|
||||
}
|
||||
|
||||
var lTable [maxLTableSize]uint64
|
||||
var sTable [maxSTableSize]uint64
|
||||
|
|
@ -52,10 +59,15 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
repeat := 1
|
||||
if dict != nil {
|
||||
dict.initBest()
|
||||
s = 0
|
||||
repeat = len(dict.dict) - dict.repeat
|
||||
}
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
const lowbitMask = 0xffffffff
|
||||
getCur := func(x uint64) int {
|
||||
return int(x & lowbitMask)
|
||||
|
|
@ -71,7 +83,7 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
s int
|
||||
length int
|
||||
score int
|
||||
rep bool
|
||||
rep, dict bool
|
||||
}
|
||||
var best match
|
||||
for {
|
||||
|
|
@ -85,6 +97,12 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
if dict != nil && s >= MaxDictSrcOffset {
|
||||
dict = nil
|
||||
if repeat > s {
|
||||
repeat = math.MinInt32
|
||||
}
|
||||
}
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
|
|
@ -114,7 +132,15 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
}
|
||||
m := match{offset: offset, s: s, length: 4 + offset, rep: rep}
|
||||
s += 4
|
||||
for s <= sLimit {
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[m.length] {
|
||||
m.length++
|
||||
s++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
|
|
@ -130,6 +156,62 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
}
|
||||
return m
|
||||
}
|
||||
matchDict := func(candidate, s int, first uint32, rep bool) match {
|
||||
// Calculate offset as if in continuous array with s
|
||||
offset := -len(dict.dict) + candidate
|
||||
if best.length != 0 && best.s-best.offset == s-offset && !rep {
|
||||
// Don't retest if we have the same offset.
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
|
||||
if load32(dict.dict, candidate) != first {
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
m := match{offset: offset, s: s, length: 4 + candidate, rep: rep, dict: true}
|
||||
s += 4
|
||||
if !rep {
|
||||
for s < sLimitDict && m.length < len(dict.dict) {
|
||||
if len(src)-s < 8 || len(dict.dict)-m.length < 8 {
|
||||
if src[s] == dict.dict[m.length] {
|
||||
m.length++
|
||||
s++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(dict.dict, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
} else {
|
||||
for s < len(src) && m.length < len(dict.dict) {
|
||||
if len(src)-s < 8 || len(dict.dict)-m.length < 8 {
|
||||
if src[s] == dict.dict[m.length] {
|
||||
m.length++
|
||||
s++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(dict.dict, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
}
|
||||
m.length -= candidate
|
||||
m.score = score(m)
|
||||
if m.score <= -m.s {
|
||||
// Eliminate if no savings, we might find a better one.
|
||||
m.length = 0
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
bestOf := func(a, b match) match {
|
||||
if b.length == 0 {
|
||||
|
|
@ -146,35 +228,82 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
return b
|
||||
}
|
||||
|
||||
if s > 0 {
|
||||
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv), false), matchAt(getPrev(candidateL), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv), false))
|
||||
|
||||
}
|
||||
if dict != nil {
|
||||
candidateL := dict.bestTableLong[hashL]
|
||||
candidateS := dict.bestTableShort[hashS]
|
||||
best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateL>>16), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS>>16), s, uint32(cv), false))
|
||||
}
|
||||
{
|
||||
if (dict == nil || repeat <= s) && repeat > 0 {
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
|
||||
} else if s-repeat < -4 && dict != nil {
|
||||
candidate := len(dict.dict) - (repeat - s)
|
||||
best = bestOf(best, matchDict(candidate, s, uint32(cv), true))
|
||||
candidate++
|
||||
best = bestOf(best, matchDict(candidate, s+1, uint32(cv>>8), true))
|
||||
}
|
||||
|
||||
if best.length > 0 {
|
||||
hashS := hash4(cv>>8, sTableBits)
|
||||
// s+1
|
||||
nextShort := sTable[hash4(cv>>8, sTableBits)]
|
||||
nextShort := sTable[hashS]
|
||||
s := s + 1
|
||||
cv := load64(src, s)
|
||||
nextLong := lTable[hash8(cv, lTableBits)]
|
||||
hashL := hash8(cv, lTableBits)
|
||||
nextLong := lTable[hashL]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
|
||||
// Repeat at + 2
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
|
||||
|
||||
// Dict at + 1
|
||||
if dict != nil {
|
||||
candidateL := dict.bestTableLong[hashL]
|
||||
candidateS := dict.bestTableShort[hashS]
|
||||
|
||||
best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false))
|
||||
}
|
||||
|
||||
// s+2
|
||||
if true {
|
||||
nextShort = sTable[hash4(cv>>8, sTableBits)]
|
||||
hashS := hash4(cv>>8, sTableBits)
|
||||
|
||||
nextShort = sTable[hashS]
|
||||
s++
|
||||
cv = load64(src, s)
|
||||
nextLong = lTable[hash8(cv, lTableBits)]
|
||||
hashL := hash8(cv, lTableBits)
|
||||
nextLong = lTable[hashL]
|
||||
|
||||
if (dict == nil || repeat <= s) && repeat > 0 {
|
||||
// Repeat at + 2
|
||||
best = bestOf(best, matchAt(s-repeat, s, uint32(cv), true))
|
||||
} else if repeat-s > 4 && dict != nil {
|
||||
candidate := len(dict.dict) - (repeat - s)
|
||||
best = bestOf(best, matchDict(candidate, s, uint32(cv), true))
|
||||
}
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
|
||||
|
||||
// Dict at +2
|
||||
// Very small gain
|
||||
if dict != nil {
|
||||
candidateL := dict.bestTableLong[hashL]
|
||||
candidateS := dict.bestTableShort[hashS]
|
||||
|
||||
best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false))
|
||||
}
|
||||
}
|
||||
// Search for a match at best match end, see if that is better.
|
||||
// Allow some bytes at the beginning to mismatch.
|
||||
|
|
@ -227,7 +356,7 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
|
||||
// Extend backwards, not needed for repeats...
|
||||
s = best.s
|
||||
if !best.rep {
|
||||
if !best.rep && !best.dict {
|
||||
for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] {
|
||||
best.offset--
|
||||
best.length++
|
||||
|
|
@ -244,7 +373,6 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
|
||||
base := s
|
||||
offset := s - best.offset
|
||||
|
||||
s += best.length
|
||||
|
||||
if offset > 65535 && s-base <= 5 && !best.rep {
|
||||
|
|
@ -256,16 +384,28 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
|||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("EMIT", base-nextEmit, "literals. base-after:", base)
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if best.rep {
|
||||
if nextEmit > 0 {
|
||||
if nextEmit > 0 || best.dict {
|
||||
if debug {
|
||||
fmt.Println("REPEAT, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best)
|
||||
}
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], offset, best.length)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
// First match without dict cannot be a repeat.
|
||||
if debug {
|
||||
fmt.Println("COPY, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best)
|
||||
}
|
||||
d += emitCopy(dst[d:], offset, best.length)
|
||||
}
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Println("COPY, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best)
|
||||
}
|
||||
d += emitCopy(dst[d:], offset, best.length)
|
||||
}
|
||||
repeat = offset
|
||||
|
|
@ -296,6 +436,9 @@ emitRemainder:
|
|||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", len(src)-nextEmit, "literals")
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
|
|
@ -642,7 +785,6 @@ func emitRepeatSize(offset, length int) int {
|
|||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
if left > 0 {
|
||||
return 5 + emitRepeatSize(offset, left)
|
||||
|
|
|
|||
622
vendor/github.com/klauspost/compress/s2/encode_better.go
generated
vendored
622
vendor/github.com/klauspost/compress/s2/encode_better.go
generated
vendored
|
|
@ -6,6 +6,8 @@
|
|||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
|
|
@ -476,3 +478,623 @@ emitRemainder:
|
|||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// encodeBlockBetterDict encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetterDict(dst, src []byte, dict *Dict) (d int) {
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 17
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 14
|
||||
maxSTableSize = 1 << sTableBits
|
||||
|
||||
maxAhead = 8 // maximum bytes ahead without checking sLimit
|
||||
|
||||
debug = false
|
||||
)
|
||||
|
||||
sLimit := len(src) - inputMargin
|
||||
if sLimit > MaxDictSrcOffset-maxAhead {
|
||||
sLimit = MaxDictSrcOffset - maxAhead
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
dict.initBetter()
|
||||
|
||||
var lTable [maxLTableSize]uint32
|
||||
var sTable [maxSTableSize]uint32
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 6
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 0
|
||||
cv := load64(src, s)
|
||||
|
||||
// We initialize repeat to 0, so we never match on first attempt
|
||||
repeat := len(dict.dict) - dict.repeat
|
||||
|
||||
// While in dict
|
||||
searchDict:
|
||||
for {
|
||||
candidateL := 0
|
||||
nextS := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS = s + (s-nextEmit)>>7 + 1
|
||||
if nextS > sLimit {
|
||||
break searchDict
|
||||
}
|
||||
hashL := hash7(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
candidateS := int(sTable[hashS])
|
||||
dictL := int(dict.betterTableLong[hashL])
|
||||
dictS := int(dict.betterTableShort[hashS])
|
||||
lTable[hashL] = uint32(s)
|
||||
sTable[hashS] = uint32(s)
|
||||
|
||||
valLong := load64(src, candidateL)
|
||||
valShort := load64(src, candidateS)
|
||||
|
||||
// If long matches at least 8 bytes, use that.
|
||||
if s != 0 {
|
||||
if cv == valLong {
|
||||
goto emitMatch
|
||||
}
|
||||
if cv == valShort {
|
||||
candidateL = candidateS
|
||||
goto emitMatch
|
||||
}
|
||||
}
|
||||
|
||||
// Check dict repeat.
|
||||
if repeat >= s+4 {
|
||||
candidate := len(dict.dict) - repeat + s
|
||||
if candidate > 0 && uint32(cv) == load32(dict.dict, candidate) {
|
||||
// Extend back
|
||||
base := s
|
||||
for i := candidate; base > nextEmit && i > 0 && dict.dict[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("emitted ", base-nextEmit, "literals")
|
||||
}
|
||||
s += 4
|
||||
candidate += 4
|
||||
for candidate < len(dict.dict)-8 && s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(dict.dict, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
if debug {
|
||||
fmt.Println("emitted dict repeat length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
cv = load64(src, s)
|
||||
// Index in-between
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv = load64(src, s)
|
||||
for index0 < index1 {
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Don't try to find match at s==0
|
||||
if s == 0 {
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
continue
|
||||
}
|
||||
|
||||
// Long likely matches 7, so take that.
|
||||
if uint32(cv) == uint32(valLong) {
|
||||
goto emitMatch
|
||||
}
|
||||
|
||||
// Long dict...
|
||||
if uint32(cv) == load32(dict.dict, dictL) {
|
||||
candidateL = dictL
|
||||
goto emitDict
|
||||
}
|
||||
|
||||
// Check our short candidate
|
||||
if uint32(cv) == uint32(valShort) {
|
||||
// Try a long candidate at s+1
|
||||
hashL = hash7(cv>>8, lTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
lTable[hashL] = uint32(s + 1)
|
||||
if uint32(cv>>8) == load32(src, candidateL) {
|
||||
s++
|
||||
goto emitMatch
|
||||
}
|
||||
// Use our short candidate.
|
||||
candidateL = candidateS
|
||||
goto emitMatch
|
||||
}
|
||||
if uint32(cv) == load32(dict.dict, dictS) {
|
||||
// Try a long candidate at s+1
|
||||
hashL = hash7(cv>>8, lTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
lTable[hashL] = uint32(s + 1)
|
||||
if uint32(cv>>8) == load32(src, candidateL) {
|
||||
s++
|
||||
goto emitMatch
|
||||
}
|
||||
candidateL = dictS
|
||||
goto emitDict
|
||||
}
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
emitDict:
|
||||
{
|
||||
if debug {
|
||||
if load32(dict.dict, candidateL) != load32(src, s) {
|
||||
panic("dict emit mismatch")
|
||||
}
|
||||
}
|
||||
// Extend backwards.
|
||||
// The top bytes will be rechecked to get the full match.
|
||||
for candidateL > 0 && s > nextEmit && dict.dict[candidateL-1] == src[s-1] {
|
||||
candidateL--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", s-nextEmit, "literals")
|
||||
}
|
||||
{
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
offset := s + (len(dict.dict)) - candidateL
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidateL += 4
|
||||
for s <= len(src)-8 && len(dict.dict)-candidateL >= 8 {
|
||||
if diff := load64(src, s) ^ load64(dict.dict, candidateL); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidateL += 8
|
||||
}
|
||||
|
||||
if repeat == offset {
|
||||
if debug {
|
||||
fmt.Println("emitted dict repeat, length", s-base, "offset:", offset, "s:", s, "dict offset:", candidateL)
|
||||
}
|
||||
d += emitRepeat(dst[d:], offset, s-base)
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Println("emitted dict copy, length", s-base, "offset:", offset, "s:", s, "dict offset:", candidateL)
|
||||
}
|
||||
// Matches longer than 64 are split.
|
||||
if s <= sLimit || s-base < 8 {
|
||||
d += emitCopy(dst[d:], offset, s-base)
|
||||
} else {
|
||||
// Split to ensure we don't start a copy within next block.
|
||||
d += emitCopy(dst[d:], offset, 4)
|
||||
d += emitRepeat(dst[d:], offset, s-base-4)
|
||||
}
|
||||
repeat = offset
|
||||
}
|
||||
if false {
|
||||
// Validate match.
|
||||
if s <= candidateL {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := dict.dict[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
|
||||
// Index short & long
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 1
|
||||
index1 -= 1
|
||||
cv = load64(src, s)
|
||||
|
||||
// index every second long in between.
|
||||
for index0 < index1 {
|
||||
lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
|
||||
lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
emitMatch:
|
||||
|
||||
// Extend backwards
|
||||
for candidateL > 0 && s > nextEmit && src[candidateL-1] == src[s-1] {
|
||||
candidateL--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := base - candidateL
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidateL += 4
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[candidateL] {
|
||||
s++
|
||||
candidateL++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, candidateL); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidateL += 8
|
||||
}
|
||||
|
||||
if offset > 65535 && s-base <= 5 && repeat != offset {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = nextS + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", s-nextEmit, "literals")
|
||||
}
|
||||
if repeat == offset {
|
||||
if debug {
|
||||
fmt.Println("emitted match repeat, length", s-base, "offset:", offset, "s:", s)
|
||||
}
|
||||
d += emitRepeat(dst[d:], offset, s-base)
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Println("emitted match copy, length", s-base, "offset:", offset, "s:", s)
|
||||
}
|
||||
d += emitCopy(dst[d:], offset, s-base)
|
||||
repeat = offset
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
|
||||
// Index short & long
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 1
|
||||
index1 -= 1
|
||||
cv = load64(src, s)
|
||||
|
||||
// index every second long in between.
|
||||
for index0 < index1 {
|
||||
lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
|
||||
lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
}
|
||||
|
||||
// Search without dict:
|
||||
if repeat > s {
|
||||
repeat = 0
|
||||
}
|
||||
|
||||
// No more dict
|
||||
sLimit = len(src) - inputMargin
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
if debug {
|
||||
fmt.Println("now", s, "->", sLimit, "out:", d, "left:", len(src)-s, "nextemit:", nextEmit, "dstLimit:", dstLimit, "s:", s)
|
||||
}
|
||||
for {
|
||||
candidateL := 0
|
||||
nextS := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS = s + (s-nextEmit)>>7 + 1
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hashL := hash7(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
candidateS := int(sTable[hashS])
|
||||
lTable[hashL] = uint32(s)
|
||||
sTable[hashS] = uint32(s)
|
||||
|
||||
valLong := load64(src, candidateL)
|
||||
valShort := load64(src, candidateS)
|
||||
|
||||
// If long matches at least 8 bytes, use that.
|
||||
if cv == valLong {
|
||||
break
|
||||
}
|
||||
if cv == valShort {
|
||||
candidateL = candidateS
|
||||
break
|
||||
}
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
// Minimum length of a repeat. Tested with various values.
|
||||
// While 4-5 offers improvements in some, 6 reduces
|
||||
// regressions significantly.
|
||||
const wantRepeatBytes = 6
|
||||
const repeatMask = ((1 << (wantRepeatBytes * 8)) - 1) << (8 * checkRep)
|
||||
if false && repeat > 0 && cv&repeatMask == load64(src, s-repeat)&repeatMask {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + wantRepeatBytes + checkRep
|
||||
s += wantRepeatBytes + checkRep
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[candidate] {
|
||||
s++
|
||||
candidate++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
// Index in-between
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv = load64(src, s)
|
||||
for index0 < index1 {
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
// Long likely matches 7, so take that.
|
||||
if uint32(cv) == uint32(valLong) {
|
||||
break
|
||||
}
|
||||
|
||||
// Check our short candidate
|
||||
if uint32(cv) == uint32(valShort) {
|
||||
// Try a long candidate at s+1
|
||||
hashL = hash7(cv>>8, lTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
lTable[hashL] = uint32(s + 1)
|
||||
if uint32(cv>>8) == load32(src, candidateL) {
|
||||
s++
|
||||
break
|
||||
}
|
||||
// Use our short candidate.
|
||||
candidateL = candidateS
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidateL > 0 && s > nextEmit && src[candidateL-1] == src[s-1] {
|
||||
candidateL--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := base - candidateL
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidateL += 4
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[candidateL] {
|
||||
s++
|
||||
candidateL++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, candidateL); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidateL += 8
|
||||
}
|
||||
|
||||
if offset > 65535 && s-base <= 5 && repeat != offset {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = nextS + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if repeat == offset {
|
||||
d += emitRepeat(dst[d:], offset, s-base)
|
||||
} else {
|
||||
d += emitCopy(dst[d:], offset, s-base)
|
||||
repeat = offset
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
|
||||
// Index short & long
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 1
|
||||
index1 -= 1
|
||||
cv = load64(src, s)
|
||||
|
||||
// index every second long in between.
|
||||
for index0 < index1 {
|
||||
lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
|
||||
lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
|
|
|||
413
vendor/github.com/klauspost/compress/s2/encode_go.go
generated
vendored
413
vendor/github.com/klauspost/compress/s2/encode_go.go
generated
vendored
|
|
@ -4,9 +4,12 @@
|
|||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
const hasAmd64Asm = false
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
|
|
@ -312,3 +315,413 @@ func matchLen(a []byte, b []byte) int {
|
|||
}
|
||||
return len(a) + checked
|
||||
}
|
||||
|
||||
func calcBlockSize(src []byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 13
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteralSize(src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
if false {
|
||||
// Validate match.
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func calcBlockSizeSmall(src []byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 9
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteralSize(src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
if false {
|
||||
// Validate match.
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
func emitLiteralSize(lit []byte) int {
|
||||
if len(lit) == 0 {
|
||||
return 0
|
||||
}
|
||||
switch {
|
||||
case len(lit) <= 60:
|
||||
return len(lit) + 1
|
||||
case len(lit) <= 1<<8:
|
||||
return len(lit) + 2
|
||||
case len(lit) <= 1<<16:
|
||||
return len(lit) + 3
|
||||
case len(lit) <= 1<<24:
|
||||
return len(lit) + 4
|
||||
default:
|
||||
return len(lit) + 5
|
||||
}
|
||||
}
|
||||
|
||||
func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) {
|
||||
panic("cvtLZ4BlockAsm should be unreachable")
|
||||
}
|
||||
|
||||
func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) {
|
||||
panic("cvtLZ4BlockSnappyAsm should be unreachable")
|
||||
}
|
||||
|
||||
func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) {
|
||||
panic("cvtLZ4sBlockAsm should be unreachable")
|
||||
}
|
||||
|
||||
func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) {
|
||||
panic("cvtLZ4sBlockSnappyAsm should be unreachable")
|
||||
}
|
||||
|
|
|
|||
34
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go
generated
vendored
34
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go
generated
vendored
|
|
@ -146,6 +146,20 @@ func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte) int
|
|||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte) int
|
||||
|
||||
// calcBlockSize encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func calcBlockSize(src []byte) int
|
||||
|
||||
// calcBlockSizeSmall encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 1024 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func calcBlockSizeSmall(src []byte) int
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
|
|
@ -192,3 +206,23 @@ func emitCopyNoRepeat(dst []byte, offset int, length int) int
|
|||
//
|
||||
//go:noescape
|
||||
func matchLen(a []byte, b []byte) int
|
||||
|
||||
// cvtLZ4Block converts an LZ4 block to S2
|
||||
//
|
||||
//go:noescape
|
||||
func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
||||
|
||||
// cvtLZ4sBlock converts an LZ4s block to S2
|
||||
//
|
||||
//go:noescape
|
||||
func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
||||
|
||||
// cvtLZ4Block converts an LZ4 block to Snappy
|
||||
//
|
||||
//go:noescape
|
||||
func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
||||
|
||||
// cvtLZ4sBlock converts an LZ4s block to Snappy
|
||||
//
|
||||
//go:noescape
|
||||
func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
||||
|
|
|
|||
18841
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s
generated
vendored
18841
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s
generated
vendored
File diff suppressed because it is too large
Load diff
585
vendor/github.com/klauspost/compress/s2/lz4convert.go
generated
vendored
Normal file
585
vendor/github.com/klauspost/compress/s2/lz4convert.go
generated
vendored
Normal file
|
|
@ -0,0 +1,585 @@
|
|||
// Copyright (c) 2022 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// LZ4Converter provides conversion from LZ4 blocks as defined here:
|
||||
// https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md
|
||||
type LZ4Converter struct {
|
||||
}
|
||||
|
||||
// ErrDstTooSmall is returned when provided destination is too small.
|
||||
var ErrDstTooSmall = errors.New("s2: destination too small")
|
||||
|
||||
// ConvertBlock will convert an LZ4 block and append it as an S2
|
||||
// block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4Converter) ConvertBlock(dst, src []byte) ([]byte, int, error) {
|
||||
if len(src) == 0 {
|
||||
return dst, 0, nil
|
||||
}
|
||||
const debug = false
|
||||
const inline = true
|
||||
const lz4MinMatch = 4
|
||||
|
||||
s, d := 0, len(dst)
|
||||
dst = dst[:cap(dst)]
|
||||
if !debug && hasAmd64Asm {
|
||||
res, sz := cvtLZ4BlockAsm(dst[d:], src)
|
||||
if res < 0 {
|
||||
const (
|
||||
errCorrupt = -1
|
||||
errDstTooSmall = -2
|
||||
)
|
||||
switch res {
|
||||
case errCorrupt:
|
||||
return nil, 0, ErrCorrupt
|
||||
case errDstTooSmall:
|
||||
return nil, 0, ErrDstTooSmall
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res)
|
||||
}
|
||||
}
|
||||
if d+sz > len(dst) {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
return dst[:d+sz], res, nil
|
||||
}
|
||||
|
||||
dLimit := len(dst) - 10
|
||||
var lastOffset uint16
|
||||
var uncompressed int
|
||||
if debug {
|
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst))
|
||||
}
|
||||
|
||||
for {
|
||||
if s >= len(src) {
|
||||
return dst[:d], 0, ErrCorrupt
|
||||
}
|
||||
// Read literal info
|
||||
token := src[s]
|
||||
ll := int(token >> 4)
|
||||
ml := int(lz4MinMatch + (token & 0xf))
|
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 {
|
||||
for {
|
||||
s++
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return dst[:d], 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
ll += int(val)
|
||||
if val != 255 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Skip past token
|
||||
if s+ll >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
s++
|
||||
if ll > 0 {
|
||||
if d+ll > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit %d literals\n", ll)
|
||||
}
|
||||
d += emitLiteralGo(dst[d:], src[s:s+ll])
|
||||
s += ll
|
||||
uncompressed += ll
|
||||
}
|
||||
|
||||
// Check if we are done...
|
||||
if s == len(src) && ml == lz4MinMatch {
|
||||
break
|
||||
}
|
||||
// 2 byte offset
|
||||
if s >= len(src)-2 {
|
||||
if debug {
|
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
offset := binary.LittleEndian.Uint16(src[s:])
|
||||
s += 2
|
||||
if offset == 0 {
|
||||
if debug {
|
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
if int(offset) > uncompressed {
|
||||
if debug {
|
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
if ml == lz4MinMatch+15 {
|
||||
for {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
s++
|
||||
ml += int(val)
|
||||
if val != 255 {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if offset == lastOffset {
|
||||
if debug {
|
||||
fmt.Printf("emit repeat, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
if !inline {
|
||||
d += emitRepeat16(dst[d:], offset, ml)
|
||||
} else {
|
||||
length := ml
|
||||
dst := dst[d:]
|
||||
for len(dst) > 5 {
|
||||
// Repeat offset, make length cheaper
|
||||
length -= 4
|
||||
if length <= 4 {
|
||||
dst[0] = uint8(length)<<2 | tagCopy1
|
||||
dst[1] = 0
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
if length < 8 && offset < 2048 {
|
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
if length < (1<<8)+4 {
|
||||
length -= 4
|
||||
dst[2] = uint8(length)
|
||||
dst[1] = 0
|
||||
dst[0] = 5<<2 | tagCopy1
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
if length < (1<<16)+(1<<8) {
|
||||
length -= 1 << 8
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 6<<2 | tagCopy1
|
||||
d += 4
|
||||
break
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= 1 << 16
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
dst[4] = uint8(length >> 16)
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 7<<2 | tagCopy1
|
||||
if left > 0 {
|
||||
d += 5 + emitRepeat16(dst[5:], offset, left)
|
||||
break
|
||||
}
|
||||
d += 5
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
if !inline {
|
||||
d += emitCopy16(dst[d:], offset, ml)
|
||||
} else {
|
||||
length := ml
|
||||
dst := dst[d:]
|
||||
for len(dst) > 5 {
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
off := 3
|
||||
if offset < 2048 {
|
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1
|
||||
length -= 8
|
||||
off = 2
|
||||
} else {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
}
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
d += off + emitRepeat16(dst[off:], offset, length)
|
||||
break
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
}
|
||||
lastOffset = offset
|
||||
}
|
||||
uncompressed += ml
|
||||
if d > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
}
|
||||
|
||||
return dst[:d], uncompressed, nil
|
||||
}
|
||||
|
||||
// ConvertBlockSnappy will convert an LZ4 block and append it
|
||||
// as a Snappy block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4Converter) ConvertBlockSnappy(dst, src []byte) ([]byte, int, error) {
|
||||
if len(src) == 0 {
|
||||
return dst, 0, nil
|
||||
}
|
||||
const debug = false
|
||||
const lz4MinMatch = 4
|
||||
|
||||
s, d := 0, len(dst)
|
||||
dst = dst[:cap(dst)]
|
||||
// Use assembly when possible
|
||||
if !debug && hasAmd64Asm {
|
||||
res, sz := cvtLZ4BlockSnappyAsm(dst[d:], src)
|
||||
if res < 0 {
|
||||
const (
|
||||
errCorrupt = -1
|
||||
errDstTooSmall = -2
|
||||
)
|
||||
switch res {
|
||||
case errCorrupt:
|
||||
return nil, 0, ErrCorrupt
|
||||
case errDstTooSmall:
|
||||
return nil, 0, ErrDstTooSmall
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res)
|
||||
}
|
||||
}
|
||||
if d+sz > len(dst) {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
return dst[:d+sz], res, nil
|
||||
}
|
||||
|
||||
dLimit := len(dst) - 10
|
||||
var uncompressed int
|
||||
if debug {
|
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst))
|
||||
}
|
||||
|
||||
for {
|
||||
if s >= len(src) {
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
// Read literal info
|
||||
token := src[s]
|
||||
ll := int(token >> 4)
|
||||
ml := int(lz4MinMatch + (token & 0xf))
|
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 {
|
||||
for {
|
||||
s++
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
ll += int(val)
|
||||
if val != 255 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Skip past token
|
||||
if s+ll >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
s++
|
||||
if ll > 0 {
|
||||
if d+ll > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit %d literals\n", ll)
|
||||
}
|
||||
d += emitLiteralGo(dst[d:], src[s:s+ll])
|
||||
s += ll
|
||||
uncompressed += ll
|
||||
}
|
||||
|
||||
// Check if we are done...
|
||||
if s == len(src) && ml == lz4MinMatch {
|
||||
break
|
||||
}
|
||||
// 2 byte offset
|
||||
if s >= len(src)-2 {
|
||||
if debug {
|
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
offset := binary.LittleEndian.Uint16(src[s:])
|
||||
s += 2
|
||||
if offset == 0 {
|
||||
if debug {
|
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
if int(offset) > uncompressed {
|
||||
if debug {
|
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
if ml == lz4MinMatch+15 {
|
||||
for {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
s++
|
||||
ml += int(val)
|
||||
if val != 255 {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
length := ml
|
||||
// d += emitCopyNoRepeat(dst[d:], int(offset), ml)
|
||||
for length > 0 {
|
||||
if d >= dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit a length 64 copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8)
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = 63<<2 | tagCopy2
|
||||
length -= 64
|
||||
d += 3
|
||||
continue
|
||||
}
|
||||
if length >= 12 || offset >= 2048 || length < 4 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8)
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = uint8(length-1)<<2 | tagCopy2
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
uncompressed += ml
|
||||
if d > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
}
|
||||
|
||||
return dst[:d], uncompressed, nil
|
||||
}
|
||||
|
||||
// emitRepeat writes a repeat chunk and returns the number of bytes written.
|
||||
// Length must be at least 4 and < 1<<24
|
||||
func emitRepeat16(dst []byte, offset uint16, length int) int {
|
||||
// Repeat offset, make length cheaper
|
||||
length -= 4
|
||||
if length <= 4 {
|
||||
dst[0] = uint8(length)<<2 | tagCopy1
|
||||
dst[1] = 0
|
||||
return 2
|
||||
}
|
||||
if length < 8 && offset < 2048 {
|
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
if length < (1<<8)+4 {
|
||||
length -= 4
|
||||
dst[2] = uint8(length)
|
||||
dst[1] = 0
|
||||
dst[0] = 5<<2 | tagCopy1
|
||||
return 3
|
||||
}
|
||||
if length < (1<<16)+(1<<8) {
|
||||
length -= 1 << 8
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 6<<2 | tagCopy1
|
||||
return 4
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= 1 << 16
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
dst[4] = uint8(length >> 16)
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 7<<2 | tagCopy1
|
||||
if left > 0 {
|
||||
return 5 + emitRepeat16(dst[5:], offset, left)
|
||||
}
|
||||
return 5
|
||||
}
|
||||
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint16
|
||||
// 4 <= length && length <= math.MaxUint32
|
||||
func emitCopy16(dst []byte, offset uint16, length int) int {
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
off := 3
|
||||
if offset < 2048 {
|
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1
|
||||
length -= 8
|
||||
off = 2
|
||||
} else {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
}
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return off + emitRepeat16(dst[off:], offset, length)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
func emitLiteralGo(dst, lit []byte) int {
|
||||
if len(lit) == 0 {
|
||||
return 0
|
||||
}
|
||||
i, n := 0, uint(len(lit)-1)
|
||||
switch {
|
||||
case n < 60:
|
||||
dst[0] = uint8(n)<<2 | tagLiteral
|
||||
i = 1
|
||||
case n < 1<<8:
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 60<<2 | tagLiteral
|
||||
i = 2
|
||||
case n < 1<<16:
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 61<<2 | tagLiteral
|
||||
i = 3
|
||||
case n < 1<<24:
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 62<<2 | tagLiteral
|
||||
i = 4
|
||||
default:
|
||||
dst[4] = uint8(n >> 24)
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 63<<2 | tagLiteral
|
||||
i = 5
|
||||
}
|
||||
return i + copy(dst[i:], lit)
|
||||
}
|
||||
467
vendor/github.com/klauspost/compress/s2/lz4sconvert.go
generated
vendored
Normal file
467
vendor/github.com/klauspost/compress/s2/lz4sconvert.go
generated
vendored
Normal file
|
|
@ -0,0 +1,467 @@
|
|||
// Copyright (c) 2022 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// LZ4sConverter provides conversion from LZ4s.
|
||||
// (Intel modified LZ4 Blocks)
|
||||
// https://cdrdv2-public.intel.com/743912/743912-qat-programmers-guide-v2.0.pdf
|
||||
// LZ4s is a variant of LZ4 block format. LZ4s should be considered as an intermediate compressed block format.
|
||||
// The LZ4s format is selected when the application sets the compType to CPA_DC_LZ4S in CpaDcSessionSetupData.
|
||||
// The LZ4s block returned by the Intel® QAT hardware can be used by an external
|
||||
// software post-processing to generate other compressed data formats.
|
||||
// The following table lists the differences between LZ4 and LZ4s block format. LZ4s block format uses
|
||||
// the same high-level formatting as LZ4 block format with the following encoding changes:
|
||||
// For Min Match of 4 bytes, Copy length value 1-15 means length 4-18 with 18 bytes adding an extra byte.
|
||||
// ONLY "Min match of 4 bytes" is supported.
|
||||
type LZ4sConverter struct {
|
||||
}
|
||||
|
||||
// ConvertBlock will convert an LZ4s block and append it as an S2
|
||||
// block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4sConverter) ConvertBlock(dst, src []byte) ([]byte, int, error) {
|
||||
if len(src) == 0 {
|
||||
return dst, 0, nil
|
||||
}
|
||||
const debug = false
|
||||
const inline = true
|
||||
const lz4MinMatch = 3
|
||||
|
||||
s, d := 0, len(dst)
|
||||
dst = dst[:cap(dst)]
|
||||
if !debug && hasAmd64Asm {
|
||||
res, sz := cvtLZ4sBlockAsm(dst[d:], src)
|
||||
if res < 0 {
|
||||
const (
|
||||
errCorrupt = -1
|
||||
errDstTooSmall = -2
|
||||
)
|
||||
switch res {
|
||||
case errCorrupt:
|
||||
return nil, 0, ErrCorrupt
|
||||
case errDstTooSmall:
|
||||
return nil, 0, ErrDstTooSmall
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res)
|
||||
}
|
||||
}
|
||||
if d+sz > len(dst) {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
return dst[:d+sz], res, nil
|
||||
}
|
||||
|
||||
dLimit := len(dst) - 10
|
||||
var lastOffset uint16
|
||||
var uncompressed int
|
||||
if debug {
|
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst))
|
||||
}
|
||||
|
||||
for {
|
||||
if s >= len(src) {
|
||||
return dst[:d], 0, ErrCorrupt
|
||||
}
|
||||
// Read literal info
|
||||
token := src[s]
|
||||
ll := int(token >> 4)
|
||||
ml := int(lz4MinMatch + (token & 0xf))
|
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 {
|
||||
for {
|
||||
s++
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return dst[:d], 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
ll += int(val)
|
||||
if val != 255 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Skip past token
|
||||
if s+ll >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
s++
|
||||
if ll > 0 {
|
||||
if d+ll > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit %d literals\n", ll)
|
||||
}
|
||||
d += emitLiteralGo(dst[d:], src[s:s+ll])
|
||||
s += ll
|
||||
uncompressed += ll
|
||||
}
|
||||
|
||||
// Check if we are done...
|
||||
if ml == lz4MinMatch {
|
||||
if s == len(src) {
|
||||
break
|
||||
}
|
||||
// 0 bytes.
|
||||
continue
|
||||
}
|
||||
// 2 byte offset
|
||||
if s >= len(src)-2 {
|
||||
if debug {
|
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
offset := binary.LittleEndian.Uint16(src[s:])
|
||||
s += 2
|
||||
if offset == 0 {
|
||||
if debug {
|
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
if int(offset) > uncompressed {
|
||||
if debug {
|
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
if ml == lz4MinMatch+15 {
|
||||
for {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
s++
|
||||
ml += int(val)
|
||||
if val != 255 {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if offset == lastOffset {
|
||||
if debug {
|
||||
fmt.Printf("emit repeat, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
if !inline {
|
||||
d += emitRepeat16(dst[d:], offset, ml)
|
||||
} else {
|
||||
length := ml
|
||||
dst := dst[d:]
|
||||
for len(dst) > 5 {
|
||||
// Repeat offset, make length cheaper
|
||||
length -= 4
|
||||
if length <= 4 {
|
||||
dst[0] = uint8(length)<<2 | tagCopy1
|
||||
dst[1] = 0
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
if length < 8 && offset < 2048 {
|
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
if length < (1<<8)+4 {
|
||||
length -= 4
|
||||
dst[2] = uint8(length)
|
||||
dst[1] = 0
|
||||
dst[0] = 5<<2 | tagCopy1
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
if length < (1<<16)+(1<<8) {
|
||||
length -= 1 << 8
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 6<<2 | tagCopy1
|
||||
d += 4
|
||||
break
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= 1 << 16
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
dst[4] = uint8(length >> 16)
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 7<<2 | tagCopy1
|
||||
if left > 0 {
|
||||
d += 5 + emitRepeat16(dst[5:], offset, left)
|
||||
break
|
||||
}
|
||||
d += 5
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
if !inline {
|
||||
d += emitCopy16(dst[d:], offset, ml)
|
||||
} else {
|
||||
length := ml
|
||||
dst := dst[d:]
|
||||
for len(dst) > 5 {
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
off := 3
|
||||
if offset < 2048 {
|
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1
|
||||
length -= 8
|
||||
off = 2
|
||||
} else {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
}
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
d += off + emitRepeat16(dst[off:], offset, length)
|
||||
break
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
}
|
||||
lastOffset = offset
|
||||
}
|
||||
uncompressed += ml
|
||||
if d > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
}
|
||||
|
||||
return dst[:d], uncompressed, nil
|
||||
}
|
||||
|
||||
// ConvertBlockSnappy will convert an LZ4s block and append it
|
||||
// as a Snappy block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4sConverter) ConvertBlockSnappy(dst, src []byte) ([]byte, int, error) {
|
||||
if len(src) == 0 {
|
||||
return dst, 0, nil
|
||||
}
|
||||
const debug = false
|
||||
const lz4MinMatch = 3
|
||||
|
||||
s, d := 0, len(dst)
|
||||
dst = dst[:cap(dst)]
|
||||
// Use assembly when possible
|
||||
if !debug && hasAmd64Asm {
|
||||
res, sz := cvtLZ4sBlockSnappyAsm(dst[d:], src)
|
||||
if res < 0 {
|
||||
const (
|
||||
errCorrupt = -1
|
||||
errDstTooSmall = -2
|
||||
)
|
||||
switch res {
|
||||
case errCorrupt:
|
||||
return nil, 0, ErrCorrupt
|
||||
case errDstTooSmall:
|
||||
return nil, 0, ErrDstTooSmall
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res)
|
||||
}
|
||||
}
|
||||
if d+sz > len(dst) {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
return dst[:d+sz], res, nil
|
||||
}
|
||||
|
||||
dLimit := len(dst) - 10
|
||||
var uncompressed int
|
||||
if debug {
|
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst))
|
||||
}
|
||||
|
||||
for {
|
||||
if s >= len(src) {
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
// Read literal info
|
||||
token := src[s]
|
||||
ll := int(token >> 4)
|
||||
ml := int(lz4MinMatch + (token & 0xf))
|
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 {
|
||||
for {
|
||||
s++
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
ll += int(val)
|
||||
if val != 255 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Skip past token
|
||||
if s+ll >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
s++
|
||||
if ll > 0 {
|
||||
if d+ll > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit %d literals\n", ll)
|
||||
}
|
||||
d += emitLiteralGo(dst[d:], src[s:s+ll])
|
||||
s += ll
|
||||
uncompressed += ll
|
||||
}
|
||||
|
||||
// Check if we are done...
|
||||
if ml == lz4MinMatch {
|
||||
if s == len(src) {
|
||||
break
|
||||
}
|
||||
// 0 bytes.
|
||||
continue
|
||||
}
|
||||
// 2 byte offset
|
||||
if s >= len(src)-2 {
|
||||
if debug {
|
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
offset := binary.LittleEndian.Uint16(src[s:])
|
||||
s += 2
|
||||
if offset == 0 {
|
||||
if debug {
|
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
if int(offset) > uncompressed {
|
||||
if debug {
|
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
if ml == lz4MinMatch+15 {
|
||||
for {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
s++
|
||||
ml += int(val)
|
||||
if val != 255 {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
length := ml
|
||||
// d += emitCopyNoRepeat(dst[d:], int(offset), ml)
|
||||
for length > 0 {
|
||||
if d >= dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit a length 64 copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8)
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = 63<<2 | tagCopy2
|
||||
length -= 64
|
||||
d += 3
|
||||
continue
|
||||
}
|
||||
if length >= 12 || offset >= 2048 || length < 4 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8)
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = uint8(length-1)<<2 | tagCopy2
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
uncompressed += ml
|
||||
if d > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
}
|
||||
|
||||
return dst[:d], uncompressed, nil
|
||||
}
|
||||
13
vendor/github.com/klauspost/cpuid/v2/README.md
generated
vendored
13
vendor/github.com/klauspost/cpuid/v2/README.md
generated
vendored
|
|
@ -19,6 +19,12 @@ Package home: https://github.com/klauspost/cpuid
|
|||
`go get -u github.com/klauspost/cpuid/v2` using modules.
|
||||
Drop `v2` for others.
|
||||
|
||||
Installing binary:
|
||||
|
||||
`go install github.com/klauspost/cpuid/v2/cmd/cpuid@latest`
|
||||
|
||||
Or download binaries from release page: https://github.com/klauspost/cpuid/releases
|
||||
|
||||
### Homebrew
|
||||
|
||||
For macOS/Linux users, you can install via [brew](https://brew.sh/)
|
||||
|
|
@ -302,6 +308,7 @@ Exit Code 1
|
|||
| AVXSLOW | Indicates the CPU performs 2 128 bit operations instead of one |
|
||||
| AVXVNNI | AVX (VEX encoded) VNNI neural network instructions |
|
||||
| AVXVNNIINT8 | AVX-VNNI-INT8 instructions |
|
||||
| BHI_CTRL | Branch History Injection and Intra-mode Branch Target Injection / CVE-2022-0001, CVE-2022-0002 / INTEL-SA-00598 |
|
||||
| BMI1 | Bit Manipulation Instruction Set 1 |
|
||||
| BMI2 | Bit Manipulation Instruction Set 2 |
|
||||
| CETIBT | Intel CET Indirect Branch Tracking |
|
||||
|
|
@ -355,6 +362,7 @@ Exit Code 1
|
|||
| IBS_OPFUSE | AMD: Indicates support for IbsOpFuse |
|
||||
| IBS_PREVENTHOST | Disallowing IBS use by the host supported |
|
||||
| IBS_ZEN4 | Fetch and Op IBS support IBS extensions added with Zen4 |
|
||||
| IDPRED_CTRL | IPRED_DIS |
|
||||
| INT_WBINVD | WBINVD/WBNOINVD are interruptible. |
|
||||
| INVLPGB | NVLPGB and TLBSYNC instruction supported |
|
||||
| LAHF | LAHF/SAHF in long mode |
|
||||
|
|
@ -374,6 +382,7 @@ Exit Code 1
|
|||
| MPX | Intel MPX (Memory Protection Extensions) |
|
||||
| MOVU | MOVU SSE instructions are more efficient and should be preferred to SSE MOVL/MOVH. MOVUPS is more efficient than MOVLPS/MOVHPS. MOVUPD is more efficient than MOVLPD/MOVHPD |
|
||||
| MSRIRC | Instruction Retired Counter MSR available |
|
||||
| MSRLIST | Read/Write List of Model Specific Registers |
|
||||
| MSR_PAGEFLUSH | Page Flush MSR available |
|
||||
| NRIPS | Indicates support for NRIP save on VMEXIT |
|
||||
| NX | NX (No-Execute) bit |
|
||||
|
|
@ -382,11 +391,12 @@ Exit Code 1
|
|||
| POPCNT | POPCNT instruction |
|
||||
| PPIN | AMD: Protected Processor Inventory Number support. Indicates that Protected Processor Inventory Number (PPIN) capability can be enabled |
|
||||
| PREFETCHI | PREFETCHIT0/1 instructions |
|
||||
| PSFD | AMD: Predictive Store Forward Disable |
|
||||
| PSFD | Predictive Store Forward Disable |
|
||||
| RDPRU | RDPRU instruction supported |
|
||||
| RDRAND | RDRAND instruction is available |
|
||||
| RDSEED | RDSEED instruction is available |
|
||||
| RDTSCP | RDTSCP Instruction |
|
||||
| RRSBA_CTRL | Restricted RSB Alternate |
|
||||
| RTM | Restricted Transactional Memory |
|
||||
| RTM_ALWAYS_ABORT | Indicates that the loaded microcode is forcing RTM abort. |
|
||||
| SERIALIZE | Serialize Instruction Execution |
|
||||
|
|
@ -439,6 +449,7 @@ Exit Code 1
|
|||
| VTE | AMD Virtual Transparent Encryption supported |
|
||||
| WAITPKG | TPAUSE, UMONITOR, UMWAIT |
|
||||
| WBNOINVD | Write Back and Do Not Invalidate Cache |
|
||||
| WRMSRNS | Non-Serializing Write to Model Specific Register |
|
||||
| X87 | FPU |
|
||||
| XGETBV1 | Supports XGETBV with ECX = 1 |
|
||||
| XOP | Bulldozer XOP functions |
|
||||
|
|
|
|||
14
vendor/github.com/klauspost/cpuid/v2/cpuid.go
generated
vendored
14
vendor/github.com/klauspost/cpuid/v2/cpuid.go
generated
vendored
|
|
@ -99,6 +99,7 @@ const (
|
|||
AVXSLOW // Indicates the CPU performs 2 128 bit operations instead of one
|
||||
AVXVNNI // AVX (VEX encoded) VNNI neural network instructions
|
||||
AVXVNNIINT8 // AVX-VNNI-INT8 instructions
|
||||
BHI_CTRL // Branch History Injection and Intra-mode Branch Target Injection / CVE-2022-0001, CVE-2022-0002 / INTEL-SA-00598
|
||||
BMI1 // Bit Manipulation Instruction Set 1
|
||||
BMI2 // Bit Manipulation Instruction Set 2
|
||||
CETIBT // Intel CET Indirect Branch Tracking
|
||||
|
|
@ -152,6 +153,7 @@ const (
|
|||
IBS_OPFUSE // AMD: Indicates support for IbsOpFuse
|
||||
IBS_PREVENTHOST // Disallowing IBS use by the host supported
|
||||
IBS_ZEN4 // AMD: Fetch and Op IBS support IBS extensions added with Zen4
|
||||
IDPRED_CTRL // IPRED_DIS
|
||||
INT_WBINVD // WBINVD/WBNOINVD are interruptible.
|
||||
INVLPGB // NVLPGB and TLBSYNC instruction supported
|
||||
LAHF // LAHF/SAHF in long mode
|
||||
|
|
@ -171,6 +173,7 @@ const (
|
|||
MOVU // AMD: MOVU SSE instructions are more efficient and should be preferred to SSE MOVL/MOVH. MOVUPS is more efficient than MOVLPS/MOVHPS. MOVUPD is more efficient than MOVLPD/MOVHPD
|
||||
MPX // Intel MPX (Memory Protection Extensions)
|
||||
MSRIRC // Instruction Retired Counter MSR available
|
||||
MSRLIST // Read/Write List of Model Specific Registers
|
||||
MSR_PAGEFLUSH // Page Flush MSR available
|
||||
NRIPS // Indicates support for NRIP save on VMEXIT
|
||||
NX // NX (No-Execute) bit
|
||||
|
|
@ -179,11 +182,12 @@ const (
|
|||
POPCNT // POPCNT instruction
|
||||
PPIN // AMD: Protected Processor Inventory Number support. Indicates that Protected Processor Inventory Number (PPIN) capability can be enabled
|
||||
PREFETCHI // PREFETCHIT0/1 instructions
|
||||
PSFD // AMD: Predictive Store Forward Disable
|
||||
PSFD // Predictive Store Forward Disable
|
||||
RDPRU // RDPRU instruction supported
|
||||
RDRAND // RDRAND instruction is available
|
||||
RDSEED // RDSEED instruction is available
|
||||
RDTSCP // RDTSCP Instruction
|
||||
RRSBA_CTRL // Restricted RSB Alternate
|
||||
RTM // Restricted Transactional Memory
|
||||
RTM_ALWAYS_ABORT // Indicates that the loaded microcode is forcing RTM abort.
|
||||
SERIALIZE // Serialize Instruction Execution
|
||||
|
|
@ -236,6 +240,7 @@ const (
|
|||
VTE // AMD Virtual Transparent Encryption supported
|
||||
WAITPKG // TPAUSE, UMONITOR, UMWAIT
|
||||
WBNOINVD // Write Back and Do Not Invalidate Cache
|
||||
WRMSRNS // Non-Serializing Write to Model Specific Register
|
||||
X87 // FPU
|
||||
XGETBV1 // Supports XGETBV with ECX = 1
|
||||
XOP // Bulldozer XOP functions
|
||||
|
|
@ -1232,13 +1237,20 @@ func support() flagSet {
|
|||
fs.setIf(edx&(1<<25) != 0, AMXINT8)
|
||||
// eax1 = CPUID.(EAX=7, ECX=1).EAX
|
||||
fs.setIf(eax1&(1<<5) != 0, AVX512BF16)
|
||||
fs.setIf(eax1&(1<<19) != 0, WRMSRNS)
|
||||
fs.setIf(eax1&(1<<21) != 0, AMXFP16)
|
||||
fs.setIf(eax1&(1<<27) != 0, MSRLIST)
|
||||
}
|
||||
}
|
||||
|
||||
// CPUID.(EAX=7, ECX=2)
|
||||
_, _, _, edx = cpuidex(7, 2)
|
||||
fs.setIf(edx&(1<<0) != 0, PSFD)
|
||||
fs.setIf(edx&(1<<1) != 0, IDPRED_CTRL)
|
||||
fs.setIf(edx&(1<<2) != 0, RRSBA_CTRL)
|
||||
fs.setIf(edx&(1<<4) != 0, BHI_CTRL)
|
||||
fs.setIf(edx&(1<<5) != 0, MCDT_NO)
|
||||
|
||||
}
|
||||
|
||||
// Processor Extended State Enumeration Sub-leaf (EAX = 0DH, ECX = 1)
|
||||
|
|
|
|||
347
vendor/github.com/klauspost/cpuid/v2/featureid_string.go
generated
vendored
347
vendor/github.com/klauspost/cpuid/v2/featureid_string.go
generated
vendored
|
|
@ -39,181 +39,186 @@ func _() {
|
|||
_ = x[AVXSLOW-29]
|
||||
_ = x[AVXVNNI-30]
|
||||
_ = x[AVXVNNIINT8-31]
|
||||
_ = x[BMI1-32]
|
||||
_ = x[BMI2-33]
|
||||
_ = x[CETIBT-34]
|
||||
_ = x[CETSS-35]
|
||||
_ = x[CLDEMOTE-36]
|
||||
_ = x[CLMUL-37]
|
||||
_ = x[CLZERO-38]
|
||||
_ = x[CMOV-39]
|
||||
_ = x[CMPCCXADD-40]
|
||||
_ = x[CMPSB_SCADBS_SHORT-41]
|
||||
_ = x[CMPXCHG8-42]
|
||||
_ = x[CPBOOST-43]
|
||||
_ = x[CPPC-44]
|
||||
_ = x[CX16-45]
|
||||
_ = x[EFER_LMSLE_UNS-46]
|
||||
_ = x[ENQCMD-47]
|
||||
_ = x[ERMS-48]
|
||||
_ = x[F16C-49]
|
||||
_ = x[FLUSH_L1D-50]
|
||||
_ = x[FMA3-51]
|
||||
_ = x[FMA4-52]
|
||||
_ = x[FP128-53]
|
||||
_ = x[FP256-54]
|
||||
_ = x[FSRM-55]
|
||||
_ = x[FXSR-56]
|
||||
_ = x[FXSROPT-57]
|
||||
_ = x[GFNI-58]
|
||||
_ = x[HLE-59]
|
||||
_ = x[HRESET-60]
|
||||
_ = x[HTT-61]
|
||||
_ = x[HWA-62]
|
||||
_ = x[HYBRID_CPU-63]
|
||||
_ = x[HYPERVISOR-64]
|
||||
_ = x[IA32_ARCH_CAP-65]
|
||||
_ = x[IA32_CORE_CAP-66]
|
||||
_ = x[IBPB-67]
|
||||
_ = x[IBRS-68]
|
||||
_ = x[IBRS_PREFERRED-69]
|
||||
_ = x[IBRS_PROVIDES_SMP-70]
|
||||
_ = x[IBS-71]
|
||||
_ = x[IBSBRNTRGT-72]
|
||||
_ = x[IBSFETCHSAM-73]
|
||||
_ = x[IBSFFV-74]
|
||||
_ = x[IBSOPCNT-75]
|
||||
_ = x[IBSOPCNTEXT-76]
|
||||
_ = x[IBSOPSAM-77]
|
||||
_ = x[IBSRDWROPCNT-78]
|
||||
_ = x[IBSRIPINVALIDCHK-79]
|
||||
_ = x[IBS_FETCH_CTLX-80]
|
||||
_ = x[IBS_OPDATA4-81]
|
||||
_ = x[IBS_OPFUSE-82]
|
||||
_ = x[IBS_PREVENTHOST-83]
|
||||
_ = x[IBS_ZEN4-84]
|
||||
_ = x[INT_WBINVD-85]
|
||||
_ = x[INVLPGB-86]
|
||||
_ = x[LAHF-87]
|
||||
_ = x[LAM-88]
|
||||
_ = x[LBRVIRT-89]
|
||||
_ = x[LZCNT-90]
|
||||
_ = x[MCAOVERFLOW-91]
|
||||
_ = x[MCDT_NO-92]
|
||||
_ = x[MCOMMIT-93]
|
||||
_ = x[MD_CLEAR-94]
|
||||
_ = x[MMX-95]
|
||||
_ = x[MMXEXT-96]
|
||||
_ = x[MOVBE-97]
|
||||
_ = x[MOVDIR64B-98]
|
||||
_ = x[MOVDIRI-99]
|
||||
_ = x[MOVSB_ZL-100]
|
||||
_ = x[MOVU-101]
|
||||
_ = x[MPX-102]
|
||||
_ = x[MSRIRC-103]
|
||||
_ = x[MSR_PAGEFLUSH-104]
|
||||
_ = x[NRIPS-105]
|
||||
_ = x[NX-106]
|
||||
_ = x[OSXSAVE-107]
|
||||
_ = x[PCONFIG-108]
|
||||
_ = x[POPCNT-109]
|
||||
_ = x[PPIN-110]
|
||||
_ = x[PREFETCHI-111]
|
||||
_ = x[PSFD-112]
|
||||
_ = x[RDPRU-113]
|
||||
_ = x[RDRAND-114]
|
||||
_ = x[RDSEED-115]
|
||||
_ = x[RDTSCP-116]
|
||||
_ = x[RTM-117]
|
||||
_ = x[RTM_ALWAYS_ABORT-118]
|
||||
_ = x[SERIALIZE-119]
|
||||
_ = x[SEV-120]
|
||||
_ = x[SEV_64BIT-121]
|
||||
_ = x[SEV_ALTERNATIVE-122]
|
||||
_ = x[SEV_DEBUGSWAP-123]
|
||||
_ = x[SEV_ES-124]
|
||||
_ = x[SEV_RESTRICTED-125]
|
||||
_ = x[SEV_SNP-126]
|
||||
_ = x[SGX-127]
|
||||
_ = x[SGXLC-128]
|
||||
_ = x[SHA-129]
|
||||
_ = x[SME-130]
|
||||
_ = x[SME_COHERENT-131]
|
||||
_ = x[SPEC_CTRL_SSBD-132]
|
||||
_ = x[SRBDS_CTRL-133]
|
||||
_ = x[SSE-134]
|
||||
_ = x[SSE2-135]
|
||||
_ = x[SSE3-136]
|
||||
_ = x[SSE4-137]
|
||||
_ = x[SSE42-138]
|
||||
_ = x[SSE4A-139]
|
||||
_ = x[SSSE3-140]
|
||||
_ = x[STIBP-141]
|
||||
_ = x[STIBP_ALWAYSON-142]
|
||||
_ = x[STOSB_SHORT-143]
|
||||
_ = x[SUCCOR-144]
|
||||
_ = x[SVM-145]
|
||||
_ = x[SVMDA-146]
|
||||
_ = x[SVMFBASID-147]
|
||||
_ = x[SVML-148]
|
||||
_ = x[SVMNP-149]
|
||||
_ = x[SVMPF-150]
|
||||
_ = x[SVMPFT-151]
|
||||
_ = x[SYSCALL-152]
|
||||
_ = x[SYSEE-153]
|
||||
_ = x[TBM-154]
|
||||
_ = x[TLB_FLUSH_NESTED-155]
|
||||
_ = x[TME-156]
|
||||
_ = x[TOPEXT-157]
|
||||
_ = x[TSCRATEMSR-158]
|
||||
_ = x[TSXLDTRK-159]
|
||||
_ = x[VAES-160]
|
||||
_ = x[VMCBCLEAN-161]
|
||||
_ = x[VMPL-162]
|
||||
_ = x[VMSA_REGPROT-163]
|
||||
_ = x[VMX-164]
|
||||
_ = x[VPCLMULQDQ-165]
|
||||
_ = x[VTE-166]
|
||||
_ = x[WAITPKG-167]
|
||||
_ = x[WBNOINVD-168]
|
||||
_ = x[X87-169]
|
||||
_ = x[XGETBV1-170]
|
||||
_ = x[XOP-171]
|
||||
_ = x[XSAVE-172]
|
||||
_ = x[XSAVEC-173]
|
||||
_ = x[XSAVEOPT-174]
|
||||
_ = x[XSAVES-175]
|
||||
_ = x[AESARM-176]
|
||||
_ = x[ARMCPUID-177]
|
||||
_ = x[ASIMD-178]
|
||||
_ = x[ASIMDDP-179]
|
||||
_ = x[ASIMDHP-180]
|
||||
_ = x[ASIMDRDM-181]
|
||||
_ = x[ATOMICS-182]
|
||||
_ = x[CRC32-183]
|
||||
_ = x[DCPOP-184]
|
||||
_ = x[EVTSTRM-185]
|
||||
_ = x[FCMA-186]
|
||||
_ = x[FP-187]
|
||||
_ = x[FPHP-188]
|
||||
_ = x[GPA-189]
|
||||
_ = x[JSCVT-190]
|
||||
_ = x[LRCPC-191]
|
||||
_ = x[PMULL-192]
|
||||
_ = x[SHA1-193]
|
||||
_ = x[SHA2-194]
|
||||
_ = x[SHA3-195]
|
||||
_ = x[SHA512-196]
|
||||
_ = x[SM3-197]
|
||||
_ = x[SM4-198]
|
||||
_ = x[SVE-199]
|
||||
_ = x[lastID-200]
|
||||
_ = x[BHI_CTRL-32]
|
||||
_ = x[BMI1-33]
|
||||
_ = x[BMI2-34]
|
||||
_ = x[CETIBT-35]
|
||||
_ = x[CETSS-36]
|
||||
_ = x[CLDEMOTE-37]
|
||||
_ = x[CLMUL-38]
|
||||
_ = x[CLZERO-39]
|
||||
_ = x[CMOV-40]
|
||||
_ = x[CMPCCXADD-41]
|
||||
_ = x[CMPSB_SCADBS_SHORT-42]
|
||||
_ = x[CMPXCHG8-43]
|
||||
_ = x[CPBOOST-44]
|
||||
_ = x[CPPC-45]
|
||||
_ = x[CX16-46]
|
||||
_ = x[EFER_LMSLE_UNS-47]
|
||||
_ = x[ENQCMD-48]
|
||||
_ = x[ERMS-49]
|
||||
_ = x[F16C-50]
|
||||
_ = x[FLUSH_L1D-51]
|
||||
_ = x[FMA3-52]
|
||||
_ = x[FMA4-53]
|
||||
_ = x[FP128-54]
|
||||
_ = x[FP256-55]
|
||||
_ = x[FSRM-56]
|
||||
_ = x[FXSR-57]
|
||||
_ = x[FXSROPT-58]
|
||||
_ = x[GFNI-59]
|
||||
_ = x[HLE-60]
|
||||
_ = x[HRESET-61]
|
||||
_ = x[HTT-62]
|
||||
_ = x[HWA-63]
|
||||
_ = x[HYBRID_CPU-64]
|
||||
_ = x[HYPERVISOR-65]
|
||||
_ = x[IA32_ARCH_CAP-66]
|
||||
_ = x[IA32_CORE_CAP-67]
|
||||
_ = x[IBPB-68]
|
||||
_ = x[IBRS-69]
|
||||
_ = x[IBRS_PREFERRED-70]
|
||||
_ = x[IBRS_PROVIDES_SMP-71]
|
||||
_ = x[IBS-72]
|
||||
_ = x[IBSBRNTRGT-73]
|
||||
_ = x[IBSFETCHSAM-74]
|
||||
_ = x[IBSFFV-75]
|
||||
_ = x[IBSOPCNT-76]
|
||||
_ = x[IBSOPCNTEXT-77]
|
||||
_ = x[IBSOPSAM-78]
|
||||
_ = x[IBSRDWROPCNT-79]
|
||||
_ = x[IBSRIPINVALIDCHK-80]
|
||||
_ = x[IBS_FETCH_CTLX-81]
|
||||
_ = x[IBS_OPDATA4-82]
|
||||
_ = x[IBS_OPFUSE-83]
|
||||
_ = x[IBS_PREVENTHOST-84]
|
||||
_ = x[IBS_ZEN4-85]
|
||||
_ = x[IDPRED_CTRL-86]
|
||||
_ = x[INT_WBINVD-87]
|
||||
_ = x[INVLPGB-88]
|
||||
_ = x[LAHF-89]
|
||||
_ = x[LAM-90]
|
||||
_ = x[LBRVIRT-91]
|
||||
_ = x[LZCNT-92]
|
||||
_ = x[MCAOVERFLOW-93]
|
||||
_ = x[MCDT_NO-94]
|
||||
_ = x[MCOMMIT-95]
|
||||
_ = x[MD_CLEAR-96]
|
||||
_ = x[MMX-97]
|
||||
_ = x[MMXEXT-98]
|
||||
_ = x[MOVBE-99]
|
||||
_ = x[MOVDIR64B-100]
|
||||
_ = x[MOVDIRI-101]
|
||||
_ = x[MOVSB_ZL-102]
|
||||
_ = x[MOVU-103]
|
||||
_ = x[MPX-104]
|
||||
_ = x[MSRIRC-105]
|
||||
_ = x[MSRLIST-106]
|
||||
_ = x[MSR_PAGEFLUSH-107]
|
||||
_ = x[NRIPS-108]
|
||||
_ = x[NX-109]
|
||||
_ = x[OSXSAVE-110]
|
||||
_ = x[PCONFIG-111]
|
||||
_ = x[POPCNT-112]
|
||||
_ = x[PPIN-113]
|
||||
_ = x[PREFETCHI-114]
|
||||
_ = x[PSFD-115]
|
||||
_ = x[RDPRU-116]
|
||||
_ = x[RDRAND-117]
|
||||
_ = x[RDSEED-118]
|
||||
_ = x[RDTSCP-119]
|
||||
_ = x[RRSBA_CTRL-120]
|
||||
_ = x[RTM-121]
|
||||
_ = x[RTM_ALWAYS_ABORT-122]
|
||||
_ = x[SERIALIZE-123]
|
||||
_ = x[SEV-124]
|
||||
_ = x[SEV_64BIT-125]
|
||||
_ = x[SEV_ALTERNATIVE-126]
|
||||
_ = x[SEV_DEBUGSWAP-127]
|
||||
_ = x[SEV_ES-128]
|
||||
_ = x[SEV_RESTRICTED-129]
|
||||
_ = x[SEV_SNP-130]
|
||||
_ = x[SGX-131]
|
||||
_ = x[SGXLC-132]
|
||||
_ = x[SHA-133]
|
||||
_ = x[SME-134]
|
||||
_ = x[SME_COHERENT-135]
|
||||
_ = x[SPEC_CTRL_SSBD-136]
|
||||
_ = x[SRBDS_CTRL-137]
|
||||
_ = x[SSE-138]
|
||||
_ = x[SSE2-139]
|
||||
_ = x[SSE3-140]
|
||||
_ = x[SSE4-141]
|
||||
_ = x[SSE42-142]
|
||||
_ = x[SSE4A-143]
|
||||
_ = x[SSSE3-144]
|
||||
_ = x[STIBP-145]
|
||||
_ = x[STIBP_ALWAYSON-146]
|
||||
_ = x[STOSB_SHORT-147]
|
||||
_ = x[SUCCOR-148]
|
||||
_ = x[SVM-149]
|
||||
_ = x[SVMDA-150]
|
||||
_ = x[SVMFBASID-151]
|
||||
_ = x[SVML-152]
|
||||
_ = x[SVMNP-153]
|
||||
_ = x[SVMPF-154]
|
||||
_ = x[SVMPFT-155]
|
||||
_ = x[SYSCALL-156]
|
||||
_ = x[SYSEE-157]
|
||||
_ = x[TBM-158]
|
||||
_ = x[TLB_FLUSH_NESTED-159]
|
||||
_ = x[TME-160]
|
||||
_ = x[TOPEXT-161]
|
||||
_ = x[TSCRATEMSR-162]
|
||||
_ = x[TSXLDTRK-163]
|
||||
_ = x[VAES-164]
|
||||
_ = x[VMCBCLEAN-165]
|
||||
_ = x[VMPL-166]
|
||||
_ = x[VMSA_REGPROT-167]
|
||||
_ = x[VMX-168]
|
||||
_ = x[VPCLMULQDQ-169]
|
||||
_ = x[VTE-170]
|
||||
_ = x[WAITPKG-171]
|
||||
_ = x[WBNOINVD-172]
|
||||
_ = x[WRMSRNS-173]
|
||||
_ = x[X87-174]
|
||||
_ = x[XGETBV1-175]
|
||||
_ = x[XOP-176]
|
||||
_ = x[XSAVE-177]
|
||||
_ = x[XSAVEC-178]
|
||||
_ = x[XSAVEOPT-179]
|
||||
_ = x[XSAVES-180]
|
||||
_ = x[AESARM-181]
|
||||
_ = x[ARMCPUID-182]
|
||||
_ = x[ASIMD-183]
|
||||
_ = x[ASIMDDP-184]
|
||||
_ = x[ASIMDHP-185]
|
||||
_ = x[ASIMDRDM-186]
|
||||
_ = x[ATOMICS-187]
|
||||
_ = x[CRC32-188]
|
||||
_ = x[DCPOP-189]
|
||||
_ = x[EVTSTRM-190]
|
||||
_ = x[FCMA-191]
|
||||
_ = x[FP-192]
|
||||
_ = x[FPHP-193]
|
||||
_ = x[GPA-194]
|
||||
_ = x[JSCVT-195]
|
||||
_ = x[LRCPC-196]
|
||||
_ = x[PMULL-197]
|
||||
_ = x[SHA1-198]
|
||||
_ = x[SHA2-199]
|
||||
_ = x[SHA3-200]
|
||||
_ = x[SHA512-201]
|
||||
_ = x[SM3-202]
|
||||
_ = x[SM4-203]
|
||||
_ = x[SVE-204]
|
||||
_ = x[lastID-205]
|
||||
_ = x[firstID-0]
|
||||
}
|
||||
|
||||
const _FeatureID_name = "firstIDADXAESNIAMD3DNOWAMD3DNOWEXTAMXBF16AMXFP16AMXINT8AMXTILEAVXAVX2AVX512BF16AVX512BITALGAVX512BWAVX512CDAVX512DQAVX512ERAVX512FAVX512FP16AVX512IFMAAVX512PFAVX512VBMIAVX512VBMI2AVX512VLAVX512VNNIAVX512VP2INTERSECTAVX512VPOPCNTDQAVXIFMAAVXNECONVERTAVXSLOWAVXVNNIAVXVNNIINT8BMI1BMI2CETIBTCETSSCLDEMOTECLMULCLZEROCMOVCMPCCXADDCMPSB_SCADBS_SHORTCMPXCHG8CPBOOSTCPPCCX16EFER_LMSLE_UNSENQCMDERMSF16CFLUSH_L1DFMA3FMA4FP128FP256FSRMFXSRFXSROPTGFNIHLEHRESETHTTHWAHYBRID_CPUHYPERVISORIA32_ARCH_CAPIA32_CORE_CAPIBPBIBRSIBRS_PREFERREDIBRS_PROVIDES_SMPIBSIBSBRNTRGTIBSFETCHSAMIBSFFVIBSOPCNTIBSOPCNTEXTIBSOPSAMIBSRDWROPCNTIBSRIPINVALIDCHKIBS_FETCH_CTLXIBS_OPDATA4IBS_OPFUSEIBS_PREVENTHOSTIBS_ZEN4INT_WBINVDINVLPGBLAHFLAMLBRVIRTLZCNTMCAOVERFLOWMCDT_NOMCOMMITMD_CLEARMMXMMXEXTMOVBEMOVDIR64BMOVDIRIMOVSB_ZLMOVUMPXMSRIRCMSR_PAGEFLUSHNRIPSNXOSXSAVEPCONFIGPOPCNTPPINPREFETCHIPSFDRDPRURDRANDRDSEEDRDTSCPRTMRTM_ALWAYS_ABORTSERIALIZESEVSEV_64BITSEV_ALTERNATIVESEV_DEBUGSWAPSEV_ESSEV_RESTRICTEDSEV_SNPSGXSGXLCSHASMESME_COHERENTSPEC_CTRL_SSBDSRBDS_CTRLSSESSE2SSE3SSE4SSE42SSE4ASSSE3STIBPSTIBP_ALWAYSONSTOSB_SHORTSUCCORSVMSVMDASVMFBASIDSVMLSVMNPSVMPFSVMPFTSYSCALLSYSEETBMTLB_FLUSH_NESTEDTMETOPEXTTSCRATEMSRTSXLDTRKVAESVMCBCLEANVMPLVMSA_REGPROTVMXVPCLMULQDQVTEWAITPKGWBNOINVDX87XGETBV1XOPXSAVEXSAVECXSAVEOPTXSAVESAESARMARMCPUIDASIMDASIMDDPASIMDHPASIMDRDMATOMICSCRC32DCPOPEVTSTRMFCMAFPFPHPGPAJSCVTLRCPCPMULLSHA1SHA2SHA3SHA512SM3SM4SVElastID"
|
||||
const _FeatureID_name = "firstIDADXAESNIAMD3DNOWAMD3DNOWEXTAMXBF16AMXFP16AMXINT8AMXTILEAVXAVX2AVX512BF16AVX512BITALGAVX512BWAVX512CDAVX512DQAVX512ERAVX512FAVX512FP16AVX512IFMAAVX512PFAVX512VBMIAVX512VBMI2AVX512VLAVX512VNNIAVX512VP2INTERSECTAVX512VPOPCNTDQAVXIFMAAVXNECONVERTAVXSLOWAVXVNNIAVXVNNIINT8BHI_CTRLBMI1BMI2CETIBTCETSSCLDEMOTECLMULCLZEROCMOVCMPCCXADDCMPSB_SCADBS_SHORTCMPXCHG8CPBOOSTCPPCCX16EFER_LMSLE_UNSENQCMDERMSF16CFLUSH_L1DFMA3FMA4FP128FP256FSRMFXSRFXSROPTGFNIHLEHRESETHTTHWAHYBRID_CPUHYPERVISORIA32_ARCH_CAPIA32_CORE_CAPIBPBIBRSIBRS_PREFERREDIBRS_PROVIDES_SMPIBSIBSBRNTRGTIBSFETCHSAMIBSFFVIBSOPCNTIBSOPCNTEXTIBSOPSAMIBSRDWROPCNTIBSRIPINVALIDCHKIBS_FETCH_CTLXIBS_OPDATA4IBS_OPFUSEIBS_PREVENTHOSTIBS_ZEN4IDPRED_CTRLINT_WBINVDINVLPGBLAHFLAMLBRVIRTLZCNTMCAOVERFLOWMCDT_NOMCOMMITMD_CLEARMMXMMXEXTMOVBEMOVDIR64BMOVDIRIMOVSB_ZLMOVUMPXMSRIRCMSRLISTMSR_PAGEFLUSHNRIPSNXOSXSAVEPCONFIGPOPCNTPPINPREFETCHIPSFDRDPRURDRANDRDSEEDRDTSCPRRSBA_CTRLRTMRTM_ALWAYS_ABORTSERIALIZESEVSEV_64BITSEV_ALTERNATIVESEV_DEBUGSWAPSEV_ESSEV_RESTRICTEDSEV_SNPSGXSGXLCSHASMESME_COHERENTSPEC_CTRL_SSBDSRBDS_CTRLSSESSE2SSE3SSE4SSE42SSE4ASSSE3STIBPSTIBP_ALWAYSONSTOSB_SHORTSUCCORSVMSVMDASVMFBASIDSVMLSVMNPSVMPFSVMPFTSYSCALLSYSEETBMTLB_FLUSH_NESTEDTMETOPEXTTSCRATEMSRTSXLDTRKVAESVMCBCLEANVMPLVMSA_REGPROTVMXVPCLMULQDQVTEWAITPKGWBNOINVDWRMSRNSX87XGETBV1XOPXSAVEXSAVECXSAVEOPTXSAVESAESARMARMCPUIDASIMDASIMDDPASIMDHPASIMDRDMATOMICSCRC32DCPOPEVTSTRMFCMAFPFPHPGPAJSCVTLRCPCPMULLSHA1SHA2SHA3SHA512SM3SM4SVElastID"
|
||||
|
||||
var _FeatureID_index = [...]uint16{0, 7, 10, 15, 23, 34, 41, 48, 55, 62, 65, 69, 79, 91, 99, 107, 115, 123, 130, 140, 150, 158, 168, 179, 187, 197, 215, 230, 237, 249, 256, 263, 274, 278, 282, 288, 293, 301, 306, 312, 316, 325, 343, 351, 358, 362, 366, 380, 386, 390, 394, 403, 407, 411, 416, 421, 425, 429, 436, 440, 443, 449, 452, 455, 465, 475, 488, 501, 505, 509, 523, 540, 543, 553, 564, 570, 578, 589, 597, 609, 625, 639, 650, 660, 675, 683, 693, 700, 704, 707, 714, 719, 730, 737, 744, 752, 755, 761, 766, 775, 782, 790, 794, 797, 803, 816, 821, 823, 830, 837, 843, 847, 856, 860, 865, 871, 877, 883, 886, 902, 911, 914, 923, 938, 951, 957, 971, 978, 981, 986, 989, 992, 1004, 1018, 1028, 1031, 1035, 1039, 1043, 1048, 1053, 1058, 1063, 1077, 1088, 1094, 1097, 1102, 1111, 1115, 1120, 1125, 1131, 1138, 1143, 1146, 1162, 1165, 1171, 1181, 1189, 1193, 1202, 1206, 1218, 1221, 1231, 1234, 1241, 1249, 1252, 1259, 1262, 1267, 1273, 1281, 1287, 1293, 1301, 1306, 1313, 1320, 1328, 1335, 1340, 1345, 1352, 1356, 1358, 1362, 1365, 1370, 1375, 1380, 1384, 1388, 1392, 1398, 1401, 1404, 1407, 1413}
|
||||
var _FeatureID_index = [...]uint16{0, 7, 10, 15, 23, 34, 41, 48, 55, 62, 65, 69, 79, 91, 99, 107, 115, 123, 130, 140, 150, 158, 168, 179, 187, 197, 215, 230, 237, 249, 256, 263, 274, 282, 286, 290, 296, 301, 309, 314, 320, 324, 333, 351, 359, 366, 370, 374, 388, 394, 398, 402, 411, 415, 419, 424, 429, 433, 437, 444, 448, 451, 457, 460, 463, 473, 483, 496, 509, 513, 517, 531, 548, 551, 561, 572, 578, 586, 597, 605, 617, 633, 647, 658, 668, 683, 691, 702, 712, 719, 723, 726, 733, 738, 749, 756, 763, 771, 774, 780, 785, 794, 801, 809, 813, 816, 822, 829, 842, 847, 849, 856, 863, 869, 873, 882, 886, 891, 897, 903, 909, 919, 922, 938, 947, 950, 959, 974, 987, 993, 1007, 1014, 1017, 1022, 1025, 1028, 1040, 1054, 1064, 1067, 1071, 1075, 1079, 1084, 1089, 1094, 1099, 1113, 1124, 1130, 1133, 1138, 1147, 1151, 1156, 1161, 1167, 1174, 1179, 1182, 1198, 1201, 1207, 1217, 1225, 1229, 1238, 1242, 1254, 1257, 1267, 1270, 1277, 1285, 1292, 1295, 1302, 1305, 1310, 1316, 1324, 1330, 1336, 1344, 1349, 1356, 1363, 1371, 1378, 1383, 1388, 1395, 1399, 1401, 1405, 1408, 1413, 1418, 1423, 1427, 1431, 1435, 1441, 1444, 1447, 1450, 1456}
|
||||
|
||||
func (i FeatureID) String() string {
|
||||
if i < 0 || i >= FeatureID(len(_FeatureID_index)-1) {
|
||||
|
|
|
|||
2
vendor/github.com/minio/minio-go/v7/api-datatypes.go
generated
vendored
2
vendor/github.com/minio/minio-go/v7/api-datatypes.go
generated
vendored
|
|
@ -43,7 +43,7 @@ type StringMap map[string]string
|
|||
// if m is nil it can be initialized, which is often the case if m is
|
||||
// nested in another xml structural. This is also why the first thing done
|
||||
// on the first line is initialize it.
|
||||
func (m *StringMap) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
func (m *StringMap) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) error {
|
||||
*m = StringMap{}
|
||||
type Item struct {
|
||||
Key string
|
||||
|
|
|
|||
8
vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go
generated
vendored
8
vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go
generated
vendored
|
|
@ -387,6 +387,12 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
|
|||
return UploadInfo{}, err
|
||||
}
|
||||
|
||||
headers := opts.Header()
|
||||
if s3utils.IsAmazonEndpoint(*c.endpointURL) {
|
||||
headers.Del(encrypt.SseKmsKeyID) // Remove X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id not supported in CompleteMultipartUpload
|
||||
headers.Del(encrypt.SseGenericHeader) // Remove X-Amz-Server-Side-Encryption not supported in CompleteMultipartUpload
|
||||
}
|
||||
|
||||
// Instantiate all the complete multipart buffer.
|
||||
completeMultipartUploadBuffer := bytes.NewReader(completeMultipartUploadBytes)
|
||||
reqMetadata := requestMetadata{
|
||||
|
|
@ -396,7 +402,7 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
|
|||
contentBody: completeMultipartUploadBuffer,
|
||||
contentLength: int64(len(completeMultipartUploadBytes)),
|
||||
contentSHA256Hex: sum256Hex(completeMultipartUploadBytes),
|
||||
customHeader: opts.Header(),
|
||||
customHeader: headers,
|
||||
}
|
||||
|
||||
// Execute POST to complete multipart upload for an objectName.
|
||||
|
|
|
|||
4
vendor/github.com/minio/minio-go/v7/api-remove.go
generated
vendored
4
vendor/github.com/minio/minio-go/v7/api-remove.go
generated
vendored
|
|
@ -235,7 +235,7 @@ func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte {
|
|||
|
||||
// processRemoveMultiObjectsResponse - parse the remove multi objects web service
|
||||
// and return the success/failure result status for each object
|
||||
func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, resultCh chan<- RemoveObjectResult) {
|
||||
func processRemoveMultiObjectsResponse(body io.Reader, resultCh chan<- RemoveObjectResult) {
|
||||
// Parse multi delete XML response
|
||||
rmResult := &deleteMultiObjectsResult{}
|
||||
err := xmlDecoder(body, rmResult)
|
||||
|
|
@ -459,7 +459,7 @@ func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh
|
|||
}
|
||||
|
||||
// Process multiobjects remove xml response
|
||||
processRemoveMultiObjectsResponse(resp.Body, batch, resultCh)
|
||||
processRemoveMultiObjectsResponse(resp.Body, resultCh)
|
||||
|
||||
closeResponse(resp)
|
||||
}
|
||||
|
|
|
|||
2
vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go
generated
vendored
2
vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go
generated
vendored
|
|
@ -110,7 +110,7 @@ type ListVersionsResult struct {
|
|||
// UnmarshalXML is a custom unmarshal code for the response of ListObjectVersions, the custom
|
||||
// code will unmarshal <Version> and <DeleteMarker> tags and save them in Versions field to
|
||||
// preserve the lexical order of the listing.
|
||||
func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
|
||||
func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) (err error) {
|
||||
for {
|
||||
// Read tokens from the XML document in a stream.
|
||||
t, err := d.Token()
|
||||
|
|
|
|||
12
vendor/github.com/minio/minio-go/v7/api.go
generated
vendored
12
vendor/github.com/minio/minio-go/v7/api.go
generated
vendored
|
|
@ -106,6 +106,12 @@ type Options struct {
|
|||
Region string
|
||||
BucketLookup BucketLookupType
|
||||
|
||||
// Allows setting a custom region lookup based on URL pattern
|
||||
// not all URL patterns are covered by this library so if you
|
||||
// have a custom endpoints with many regions you can use this
|
||||
// function to perform region lookups appropriately.
|
||||
CustomRegionViaURL func(u url.URL) string
|
||||
|
||||
// TrailingHeaders indicates server support of trailing headers.
|
||||
// Only supported for v4 signatures.
|
||||
TrailingHeaders bool
|
||||
|
|
@ -118,7 +124,7 @@ type Options struct {
|
|||
// Global constants.
|
||||
const (
|
||||
libraryName = "minio-go"
|
||||
libraryVersion = "v7.0.49"
|
||||
libraryVersion = "v7.0.50"
|
||||
)
|
||||
|
||||
// User Agent should always following the below style.
|
||||
|
|
@ -234,8 +240,12 @@ func privateNew(endpoint string, opts *Options) (*Client, error) {
|
|||
|
||||
// Sets custom region, if region is empty bucket location cache is used automatically.
|
||||
if opts.Region == "" {
|
||||
if opts.CustomRegionViaURL != nil {
|
||||
opts.Region = opts.CustomRegionViaURL(*clnt.endpointURL)
|
||||
} else {
|
||||
opts.Region = s3utils.GetRegionFromURL(*clnt.endpointURL)
|
||||
}
|
||||
}
|
||||
clnt.region = opts.Region
|
||||
|
||||
// Instantiate bucket location cache.
|
||||
|
|
|
|||
5
vendor/github.com/minio/minio-go/v7/bucket-cache.go
generated
vendored
5
vendor/github.com/minio/minio-go/v7/bucket-cache.go
generated
vendored
|
|
@ -190,12 +190,11 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string
|
|||
}
|
||||
}
|
||||
|
||||
isVirtualHost := s3utils.IsVirtualHostSupported(targetURL, bucketName)
|
||||
isVirtualStyle := c.isVirtualHostStyleRequest(targetURL, bucketName)
|
||||
|
||||
var urlStr string
|
||||
|
||||
// only support Aliyun OSS for virtual hosted path, compatible Amazon & Google Endpoint
|
||||
if isVirtualHost && s3utils.IsAliyunOSSEndpoint(targetURL) {
|
||||
if isVirtualStyle {
|
||||
urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + targetURL.Host + "/?location"
|
||||
} else {
|
||||
targetURL.Path = path.Join(bucketName, "") + "/"
|
||||
|
|
|
|||
23
vendor/github.com/minio/minio-go/v7/core.go
generated
vendored
23
vendor/github.com/minio/minio-go/v7/core.go
generated
vendored
|
|
@ -86,19 +86,30 @@ func (c Core) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarke
|
|||
return c.listMultipartUploadsQuery(ctx, bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads)
|
||||
}
|
||||
|
||||
// PutObjectPartOptions contains options for PutObjectPart API
|
||||
type PutObjectPartOptions struct {
|
||||
Md5Base64, Sha256Hex string
|
||||
SSE encrypt.ServerSide
|
||||
CustomHeader, Trailer http.Header
|
||||
}
|
||||
|
||||
// PutObjectPart - Upload an object part.
|
||||
func (c Core) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) {
|
||||
func (c Core) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int,
|
||||
data io.Reader, size int64, opts PutObjectPartOptions,
|
||||
) (ObjectPart, error) {
|
||||
p := uploadPartParams{
|
||||
bucketName: bucket,
|
||||
objectName: object,
|
||||
uploadID: uploadID,
|
||||
reader: data,
|
||||
partNumber: partID,
|
||||
md5Base64: md5Base64,
|
||||
sha256Hex: sha256Hex,
|
||||
md5Base64: opts.Md5Base64,
|
||||
sha256Hex: opts.Sha256Hex,
|
||||
size: size,
|
||||
sse: sse,
|
||||
sse: opts.SSE,
|
||||
streamSha256: true,
|
||||
customHeader: opts.CustomHeader,
|
||||
trailer: opts.Trailer,
|
||||
}
|
||||
return c.uploadPart(ctx, p)
|
||||
}
|
||||
|
|
@ -109,11 +120,11 @@ func (c Core) ListObjectParts(ctx context.Context, bucket, object, uploadID stri
|
|||
}
|
||||
|
||||
// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object.
|
||||
func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart, opts PutObjectOptions) (string, error) {
|
||||
func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart, opts PutObjectOptions) (UploadInfo, error) {
|
||||
res, err := c.completeMultipartUpload(ctx, bucket, object, uploadID, completeMultipartUpload{
|
||||
Parts: parts,
|
||||
}, opts)
|
||||
return res.ETag, err
|
||||
return res, err
|
||||
}
|
||||
|
||||
// AbortMultipartUpload - Abort an incomplete upload.
|
||||
|
|
|
|||
12
vendor/github.com/minio/minio-go/v7/functional_tests.go
generated
vendored
12
vendor/github.com/minio/minio-go/v7/functional_tests.go
generated
vendored
|
|
@ -2053,7 +2053,7 @@ func testPutObjectWithChecksums() {
|
|||
}
|
||||
|
||||
// Enable tracing, write to stderr.
|
||||
//c.TraceOn(os.Stderr)
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
|
||||
|
|
@ -8414,14 +8414,20 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() {
|
|||
|
||||
var completeParts []minio.CompletePart
|
||||
|
||||
part, err := c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption)
|
||||
part, err := c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1,
|
||||
bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024,
|
||||
minio.PutObjectPartOptions{SSE: srcencryption},
|
||||
)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "PutObjectPart call failed", err)
|
||||
return
|
||||
}
|
||||
completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag})
|
||||
|
||||
part, err = c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption)
|
||||
part, err = c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 2,
|
||||
bytes.NewReader(buf[5*1024*1024:]), 1024*1024,
|
||||
minio.PutObjectPartOptions{SSE: srcencryption},
|
||||
)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "PutObjectPart call failed", err)
|
||||
return
|
||||
|
|
|
|||
56
vendor/github.com/minio/minio-go/v7/pkg/encrypt/server-side.go
generated
vendored
56
vendor/github.com/minio/minio-go/v7/pkg/encrypt/server-side.go
generated
vendored
|
|
@ -28,27 +28,27 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
// sseGenericHeader is the AWS SSE header used for SSE-S3 and SSE-KMS.
|
||||
sseGenericHeader = "X-Amz-Server-Side-Encryption"
|
||||
// SseGenericHeader is the AWS SSE header used for SSE-S3 and SSE-KMS.
|
||||
SseGenericHeader = "X-Amz-Server-Side-Encryption"
|
||||
|
||||
// sseKmsKeyID is the AWS SSE-KMS key id.
|
||||
sseKmsKeyID = sseGenericHeader + "-Aws-Kms-Key-Id"
|
||||
// sseEncryptionContext is the AWS SSE-KMS Encryption Context data.
|
||||
sseEncryptionContext = sseGenericHeader + "-Context"
|
||||
// SseKmsKeyID is the AWS SSE-KMS key id.
|
||||
SseKmsKeyID = SseGenericHeader + "-Aws-Kms-Key-Id"
|
||||
// SseEncryptionContext is the AWS SSE-KMS Encryption Context data.
|
||||
SseEncryptionContext = SseGenericHeader + "-Context"
|
||||
|
||||
// sseCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key.
|
||||
sseCustomerAlgorithm = sseGenericHeader + "-Customer-Algorithm"
|
||||
// sseCustomerKey is the AWS SSE-C encryption key HTTP header key.
|
||||
sseCustomerKey = sseGenericHeader + "-Customer-Key"
|
||||
// sseCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key.
|
||||
sseCustomerKeyMD5 = sseGenericHeader + "-Customer-Key-MD5"
|
||||
// SseCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key.
|
||||
SseCustomerAlgorithm = SseGenericHeader + "-Customer-Algorithm"
|
||||
// SseCustomerKey is the AWS SSE-C encryption key HTTP header key.
|
||||
SseCustomerKey = SseGenericHeader + "-Customer-Key"
|
||||
// SseCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key.
|
||||
SseCustomerKeyMD5 = SseGenericHeader + "-Customer-Key-MD5"
|
||||
|
||||
// sseCopyCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key for CopyObject API.
|
||||
sseCopyCustomerAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
|
||||
// sseCopyCustomerKey is the AWS SSE-C encryption key HTTP header key for CopyObject API.
|
||||
sseCopyCustomerKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
|
||||
// sseCopyCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key for CopyObject API.
|
||||
sseCopyCustomerKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-MD5"
|
||||
// SseCopyCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key for CopyObject API.
|
||||
SseCopyCustomerAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
|
||||
// SseCopyCustomerKey is the AWS SSE-C encryption key HTTP header key for CopyObject API.
|
||||
SseCopyCustomerKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
|
||||
// SseCopyCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key for CopyObject API.
|
||||
SseCopyCustomerKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-MD5"
|
||||
)
|
||||
|
||||
// PBKDF creates a SSE-C key from the provided password and salt.
|
||||
|
|
@ -157,9 +157,9 @@ func (s ssec) Type() Type { return SSEC }
|
|||
|
||||
func (s ssec) Marshal(h http.Header) {
|
||||
keyMD5 := md5.Sum(s[:])
|
||||
h.Set(sseCustomerAlgorithm, "AES256")
|
||||
h.Set(sseCustomerKey, base64.StdEncoding.EncodeToString(s[:]))
|
||||
h.Set(sseCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:]))
|
||||
h.Set(SseCustomerAlgorithm, "AES256")
|
||||
h.Set(SseCustomerKey, base64.StdEncoding.EncodeToString(s[:]))
|
||||
h.Set(SseCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:]))
|
||||
}
|
||||
|
||||
type ssecCopy [32]byte
|
||||
|
|
@ -168,16 +168,16 @@ func (s ssecCopy) Type() Type { return SSEC }
|
|||
|
||||
func (s ssecCopy) Marshal(h http.Header) {
|
||||
keyMD5 := md5.Sum(s[:])
|
||||
h.Set(sseCopyCustomerAlgorithm, "AES256")
|
||||
h.Set(sseCopyCustomerKey, base64.StdEncoding.EncodeToString(s[:]))
|
||||
h.Set(sseCopyCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:]))
|
||||
h.Set(SseCopyCustomerAlgorithm, "AES256")
|
||||
h.Set(SseCopyCustomerKey, base64.StdEncoding.EncodeToString(s[:]))
|
||||
h.Set(SseCopyCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:]))
|
||||
}
|
||||
|
||||
type s3 struct{}
|
||||
|
||||
func (s s3) Type() Type { return S3 }
|
||||
|
||||
func (s s3) Marshal(h http.Header) { h.Set(sseGenericHeader, "AES256") }
|
||||
func (s s3) Marshal(h http.Header) { h.Set(SseGenericHeader, "AES256") }
|
||||
|
||||
type kms struct {
|
||||
key string
|
||||
|
|
@ -188,11 +188,11 @@ type kms struct {
|
|||
func (s kms) Type() Type { return KMS }
|
||||
|
||||
func (s kms) Marshal(h http.Header) {
|
||||
h.Set(sseGenericHeader, "aws:kms")
|
||||
h.Set(SseGenericHeader, "aws:kms")
|
||||
if s.key != "" {
|
||||
h.Set(sseKmsKeyID, s.key)
|
||||
h.Set(SseKmsKeyID, s.key)
|
||||
}
|
||||
if s.hasContext {
|
||||
h.Set(sseEncryptionContext, base64.StdEncoding.EncodeToString(s.context))
|
||||
h.Set(SseEncryptionContext, base64.StdEncoding.EncodeToString(s.context))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
6
vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go
generated
vendored
6
vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go
generated
vendored
|
|
@ -339,12 +339,12 @@ func EncodePath(pathName string) string {
|
|||
encodedPathname.WriteRune(s)
|
||||
continue
|
||||
default:
|
||||
len := utf8.RuneLen(s)
|
||||
if len < 0 {
|
||||
l := utf8.RuneLen(s)
|
||||
if l < 0 {
|
||||
// if utf8 cannot convert return the same string as is
|
||||
return pathName
|
||||
}
|
||||
u := make([]byte, len)
|
||||
u := make([]byte, l)
|
||||
utf8.EncodeRune(u, s)
|
||||
for _, r := range u {
|
||||
hex := hex.EncodeToString([]byte{r})
|
||||
|
|
|
|||
1
vendor/github.com/minio/minio-go/v7/s3-endpoints.go
generated
vendored
1
vendor/github.com/minio/minio-go/v7/s3-endpoints.go
generated
vendored
|
|
@ -34,6 +34,7 @@ var awsS3EndpointMap = map[string]string{
|
|||
"eu-south-2": "s3.dualstack.eu-south-2.amazonaws.com",
|
||||
"ap-east-1": "s3.dualstack.ap-east-1.amazonaws.com",
|
||||
"ap-south-1": "s3.dualstack.ap-south-1.amazonaws.com",
|
||||
"ap-south-2": "s3.dualstack.ap-south-2.amazonaws.com",
|
||||
"ap-southeast-1": "s3.dualstack.ap-southeast-1.amazonaws.com",
|
||||
"ap-southeast-2": "s3.dualstack.ap-southeast-2.amazonaws.com",
|
||||
"ap-northeast-1": "s3.dualstack.ap-northeast-1.amazonaws.com",
|
||||
|
|
|
|||
10
vendor/modules.txt
vendored
10
vendor/modules.txt
vendored
|
|
@ -64,7 +64,7 @@ codeberg.org/gruf/go-runners
|
|||
# codeberg.org/gruf/go-sched v1.2.3
|
||||
## explicit; go 1.19
|
||||
codeberg.org/gruf/go-sched
|
||||
# codeberg.org/gruf/go-store/v2 v2.2.1
|
||||
# codeberg.org/gruf/go-store/v2 v2.2.2
|
||||
## explicit; go 1.19
|
||||
codeberg.org/gruf/go-store/v2/kv
|
||||
codeberg.org/gruf/go-store/v2/storage
|
||||
|
|
@ -327,14 +327,14 @@ github.com/json-iterator/go
|
|||
# github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
|
||||
## explicit
|
||||
github.com/kballard/go-shellquote
|
||||
# github.com/klauspost/compress v1.15.15
|
||||
## explicit; go 1.17
|
||||
# github.com/klauspost/compress v1.16.3
|
||||
## explicit; go 1.18
|
||||
github.com/klauspost/compress/flate
|
||||
github.com/klauspost/compress/gzip
|
||||
github.com/klauspost/compress/s2
|
||||
github.com/klauspost/compress/snappy
|
||||
github.com/klauspost/compress/zlib
|
||||
# github.com/klauspost/cpuid/v2 v2.2.3
|
||||
# github.com/klauspost/cpuid/v2 v2.2.4
|
||||
## explicit; go 1.15
|
||||
github.com/klauspost/cpuid/v2
|
||||
# github.com/leodido/go-urn v1.2.2
|
||||
|
|
@ -356,7 +356,7 @@ github.com/miekg/dns
|
|||
# github.com/minio/md5-simd v1.1.2
|
||||
## explicit; go 1.14
|
||||
github.com/minio/md5-simd
|
||||
# github.com/minio/minio-go/v7 v7.0.49
|
||||
# github.com/minio/minio-go/v7 v7.0.50
|
||||
## explicit; go 1.17
|
||||
github.com/minio/minio-go/v7
|
||||
github.com/minio/minio-go/v7/pkg/credentials
|
||||
|
|
|
|||
Loading…
Reference in a new issue