forgejo/tests/integration/lfs_getobject_test.go

228 lines
6.8 KiB
Go
Raw Normal View History

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package integration
import (
"archive/zip"
"bytes"
"io"
"net/http"
"net/http/httptest"
"testing"
"code.gitea.io/gitea/models/auth"
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/tests"
[GITEA] Use maintained gziphandler - https://github.com/NYTimes/gziphandler doesn't seems to be maintained anymore and Forgejo already includes https://github.com/klauspost/compress which provides a maintained and faster gzip handler fork. - Enables Jitter to prevent BREACH attacks, as this *seems* to be possible in the context of Forgejo. (cherry picked from commit cc2847241d82001babd8d40c87d03169f21c14cd) (cherry picked from commit 99ba56a8761dd08e08d9499cab2ded1a6b7b970f) Conflicts: go.sum https://codeberg.org/forgejo/forgejo/pulls/1581 (cherry picked from commit 711638193daa2311e2ead6249a47dcec47b4e335) (cherry picked from commit 9c12a37fde6fa84414bf332ff4a066facdb92d38) (cherry picked from commit d13065345431a499f9e0b7a3c2043d7487b8aa5b) (cherry picked from commit 45a16f8c3c6f7d5e4aab8fdde6a621cf36e4801c) (cherry picked from commit a497acb31f76d580c8b0567f9461274bd78080f4) (cherry picked from commit fe87fd828973945192b98310c5c3b2001c6e0f86) (cherry picked from commit 6ac12e6693cf45cb12109028dabd868957c4b74c) (cherry picked from commit 981ec37e1e72ab19c20067ff4d2a7e20a60d3305) (cherry picked from commit 5d6892ec10086f0ba63f26693faa82d0fd4e3f4a) (cherry picked from commit 9df7968f4fc72de9788d84ca3f349e4c98ee630e) (cherry picked from commit 7d588d183329cd760053663ea2e1e82e62958409) Conflicts: routers/web/web.go https://codeberg.org/forgejo/forgejo/pulls/2075 (cherry picked from commit defb101281f5a6ba410abc763674bafa7b63dffd) (cherry picked from commit 5830f204a17767fda3e45d16dbf3af8c32e7f387)
2023-09-30 22:09:25 +00:00
"github.com/klauspost/compress/gzhttp"
gzipp "github.com/klauspost/compress/gzip"
"github.com/stretchr/testify/assert"
)
func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string {
Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
2021-04-08 22:25:57 +00:00
pointer, err := lfs.GeneratePointer(bytes.NewReader(*content))
assert.NoError(t, err)
_, err = git_model.NewLFSMetaObject(db.DefaultContext, repositoryID, pointer)
assert.NoError(t, err)
Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
2021-04-08 22:25:57 +00:00
contentStore := lfs.NewContentStore()
exist, err := contentStore.Exists(pointer)
assert.NoError(t, err)
if !exist {
Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
2021-04-08 22:25:57 +00:00
err := contentStore.Put(pointer, bytes.NewReader(*content))
assert.NoError(t, err)
}
Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
2021-04-08 22:25:57 +00:00
return pointer.Oid
}
Redesign Scoped Access Tokens (#24767) ## Changes - Adds the following high level access scopes, each with `read` and `write` levels: - `activitypub` - `admin` (hidden if user is not a site admin) - `misc` - `notification` - `organization` - `package` - `issue` - `repository` - `user` - Adds new middleware function `tokenRequiresScopes()` in addition to `reqToken()` - `tokenRequiresScopes()` is used for each high-level api section - _if_ a scoped token is present, checks that the required scope is included based on the section and HTTP method - `reqToken()` is used for individual routes - checks that required authentication is present (but does not check scope levels as this will already have been handled by `tokenRequiresScopes()` - Adds migration to convert old scoped access tokens to the new set of scopes - Updates the user interface for scope selection ### User interface example <img width="903" alt="Screen Shot 2023-05-31 at 1 56 55 PM" src="https://github.com/go-gitea/gitea/assets/23248839/654766ec-2143-4f59-9037-3b51600e32f3"> <img width="917" alt="Screen Shot 2023-05-31 at 1 56 43 PM" src="https://github.com/go-gitea/gitea/assets/23248839/1ad64081-012c-4a73-b393-66b30352654c"> ## tokenRequiresScopes Design Decision - `tokenRequiresScopes()` was added to more reliably cover api routes. For an incoming request, this function uses the given scope category (say `AccessTokenScopeCategoryOrganization`) and the HTTP method (say `DELETE`) and verifies that any scoped tokens in use include `delete:organization`. - `reqToken()` is used to enforce auth for individual routes that require it. If a scoped token is not present for a request, `tokenRequiresScopes()` will not return an error ## TODO - [x] Alphabetize scope categories - [x] Change 'public repos only' to a radio button (private vs public). Also expand this to organizations - [X] Disable token creation if no scopes selected. Alternatively, show warning - [x] `reqToken()` is missing from many `POST/DELETE` routes in the api. `tokenRequiresScopes()` only checks that a given token has the correct scope, `reqToken()` must be used to check that a token (or some other auth) is present. - _This should be addressed in this PR_ - [x] The migration should be reviewed very carefully in order to minimize access changes to existing user tokens. - _This should be addressed in this PR_ - [x] Link to api to swagger documentation, clarify what read/write/delete levels correspond to - [x] Review cases where more than one scope is needed as this directly deviates from the api definition. - _This should be addressed in this PR_ - For example: ```go m.Group("/users/{username}/orgs", func() { m.Get("", reqToken(), org.ListUserOrgs) m.Get("/{org}/permissions", reqToken(), org.GetUserOrgsPermissions) }, tokenRequiresScopes(auth_model.AccessTokenScopeCategoryUser, auth_model.AccessTokenScopeCategoryOrganization), context_service.UserAssignmentAPI()) ``` ## Future improvements - [ ] Add required scopes to swagger documentation - [ ] Redesign `reqToken()` to be opt-out rather than opt-in - [ ] Subdivide scopes like `repository` - [ ] Once a token is created, if it has no scopes, we should display text instead of an empty bullet point - [ ] If the 'public repos only' option is selected, should read categories be selected by default Closes #24501 Closes #24799 Co-authored-by: Jonathan Tran <jon@allspice.io> Co-authored-by: Kyle D <kdumontnu@gmail.com> Co-authored-by: silverwind <me@silverwind.io>
2023-06-04 18:57:16 +00:00
func storeAndGetLfsToken(t *testing.T, content *[]byte, extraHeader *http.Header, expectedStatus int, ts ...auth.AccessTokenScope) *httptest.ResponseRecorder {
repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1")
assert.NoError(t, err)
oid := storeObjectInRepo(t, repo.ID, content)
defer git_model.RemoveLFSMetaObjectByOid(db.DefaultContext, repo.ID, oid)
Redesign Scoped Access Tokens (#24767) ## Changes - Adds the following high level access scopes, each with `read` and `write` levels: - `activitypub` - `admin` (hidden if user is not a site admin) - `misc` - `notification` - `organization` - `package` - `issue` - `repository` - `user` - Adds new middleware function `tokenRequiresScopes()` in addition to `reqToken()` - `tokenRequiresScopes()` is used for each high-level api section - _if_ a scoped token is present, checks that the required scope is included based on the section and HTTP method - `reqToken()` is used for individual routes - checks that required authentication is present (but does not check scope levels as this will already have been handled by `tokenRequiresScopes()` - Adds migration to convert old scoped access tokens to the new set of scopes - Updates the user interface for scope selection ### User interface example <img width="903" alt="Screen Shot 2023-05-31 at 1 56 55 PM" src="https://github.com/go-gitea/gitea/assets/23248839/654766ec-2143-4f59-9037-3b51600e32f3"> <img width="917" alt="Screen Shot 2023-05-31 at 1 56 43 PM" src="https://github.com/go-gitea/gitea/assets/23248839/1ad64081-012c-4a73-b393-66b30352654c"> ## tokenRequiresScopes Design Decision - `tokenRequiresScopes()` was added to more reliably cover api routes. For an incoming request, this function uses the given scope category (say `AccessTokenScopeCategoryOrganization`) and the HTTP method (say `DELETE`) and verifies that any scoped tokens in use include `delete:organization`. - `reqToken()` is used to enforce auth for individual routes that require it. If a scoped token is not present for a request, `tokenRequiresScopes()` will not return an error ## TODO - [x] Alphabetize scope categories - [x] Change 'public repos only' to a radio button (private vs public). Also expand this to organizations - [X] Disable token creation if no scopes selected. Alternatively, show warning - [x] `reqToken()` is missing from many `POST/DELETE` routes in the api. `tokenRequiresScopes()` only checks that a given token has the correct scope, `reqToken()` must be used to check that a token (or some other auth) is present. - _This should be addressed in this PR_ - [x] The migration should be reviewed very carefully in order to minimize access changes to existing user tokens. - _This should be addressed in this PR_ - [x] Link to api to swagger documentation, clarify what read/write/delete levels correspond to - [x] Review cases where more than one scope is needed as this directly deviates from the api definition. - _This should be addressed in this PR_ - For example: ```go m.Group("/users/{username}/orgs", func() { m.Get("", reqToken(), org.ListUserOrgs) m.Get("/{org}/permissions", reqToken(), org.GetUserOrgsPermissions) }, tokenRequiresScopes(auth_model.AccessTokenScopeCategoryUser, auth_model.AccessTokenScopeCategoryOrganization), context_service.UserAssignmentAPI()) ``` ## Future improvements - [ ] Add required scopes to swagger documentation - [ ] Redesign `reqToken()` to be opt-out rather than opt-in - [ ] Subdivide scopes like `repository` - [ ] Once a token is created, if it has no scopes, we should display text instead of an empty bullet point - [ ] If the 'public repos only' option is selected, should read categories be selected by default Closes #24501 Closes #24799 Co-authored-by: Jonathan Tran <jon@allspice.io> Co-authored-by: Kyle D <kdumontnu@gmail.com> Co-authored-by: silverwind <me@silverwind.io>
2023-06-04 18:57:16 +00:00
token := getUserToken(t, "user2", ts...)
// Request OID
req := NewRequest(t, "GET", "/user2/repo1.git/info/lfs/objects/"+oid+"/test")
req.Header.Set("Accept-Encoding", "gzip")
req.SetBasicAuth("user2", token)
if extraHeader != nil {
for key, values := range *extraHeader {
for _, value := range values {
req.Header.Add(key, value)
}
}
}
resp := MakeRequest(t, req, expectedStatus)
return resp
}
func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, expectedStatus int) *httptest.ResponseRecorder {
repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1")
assert.NoError(t, err)
oid := storeObjectInRepo(t, repo.ID, content)
defer git_model.RemoveLFSMetaObjectByOid(db.DefaultContext, repo.ID, oid)
session := loginUser(t, "user2")
// Request OID
req := NewRequest(t, "GET", "/user2/repo1.git/info/lfs/objects/"+oid+"/test")
req.Header.Set("Accept-Encoding", "gzip")
if extraHeader != nil {
for key, values := range *extraHeader {
for _, value := range values {
req.Header.Add(key, value)
}
}
}
resp := session.MakeRequest(t, req, expectedStatus)
return resp
}
func checkResponseTestContentEncoding(t *testing.T, content *[]byte, resp *httptest.ResponseRecorder, expectGzip bool) {
contentEncoding := resp.Header().Get("Content-Encoding")
if !expectGzip || !setting.EnableGzip {
assert.NotContains(t, contentEncoding, "gzip")
result := resp.Body.Bytes()
assert.Equal(t, *content, result)
} else {
assert.Contains(t, contentEncoding, "gzip")
gzippReader, err := gzipp.NewReader(resp.Body)
assert.NoError(t, err)
result, err := io.ReadAll(gzippReader)
assert.NoError(t, err)
assert.Equal(t, *content, result)
}
}
func TestGetLFSSmall(t *testing.T) {
defer tests.PrepareTestEnv(t)()
content := []byte("A very small file\n")
resp := storeAndGetLfs(t, &content, nil, http.StatusOK)
checkResponseTestContentEncoding(t, &content, resp, false)
}
func TestGetLFSSmallToken(t *testing.T) {
defer tests.PrepareTestEnv(t)()
content := []byte("A very small file\n")
Redesign Scoped Access Tokens (#24767) ## Changes - Adds the following high level access scopes, each with `read` and `write` levels: - `activitypub` - `admin` (hidden if user is not a site admin) - `misc` - `notification` - `organization` - `package` - `issue` - `repository` - `user` - Adds new middleware function `tokenRequiresScopes()` in addition to `reqToken()` - `tokenRequiresScopes()` is used for each high-level api section - _if_ a scoped token is present, checks that the required scope is included based on the section and HTTP method - `reqToken()` is used for individual routes - checks that required authentication is present (but does not check scope levels as this will already have been handled by `tokenRequiresScopes()` - Adds migration to convert old scoped access tokens to the new set of scopes - Updates the user interface for scope selection ### User interface example <img width="903" alt="Screen Shot 2023-05-31 at 1 56 55 PM" src="https://github.com/go-gitea/gitea/assets/23248839/654766ec-2143-4f59-9037-3b51600e32f3"> <img width="917" alt="Screen Shot 2023-05-31 at 1 56 43 PM" src="https://github.com/go-gitea/gitea/assets/23248839/1ad64081-012c-4a73-b393-66b30352654c"> ## tokenRequiresScopes Design Decision - `tokenRequiresScopes()` was added to more reliably cover api routes. For an incoming request, this function uses the given scope category (say `AccessTokenScopeCategoryOrganization`) and the HTTP method (say `DELETE`) and verifies that any scoped tokens in use include `delete:organization`. - `reqToken()` is used to enforce auth for individual routes that require it. If a scoped token is not present for a request, `tokenRequiresScopes()` will not return an error ## TODO - [x] Alphabetize scope categories - [x] Change 'public repos only' to a radio button (private vs public). Also expand this to organizations - [X] Disable token creation if no scopes selected. Alternatively, show warning - [x] `reqToken()` is missing from many `POST/DELETE` routes in the api. `tokenRequiresScopes()` only checks that a given token has the correct scope, `reqToken()` must be used to check that a token (or some other auth) is present. - _This should be addressed in this PR_ - [x] The migration should be reviewed very carefully in order to minimize access changes to existing user tokens. - _This should be addressed in this PR_ - [x] Link to api to swagger documentation, clarify what read/write/delete levels correspond to - [x] Review cases where more than one scope is needed as this directly deviates from the api definition. - _This should be addressed in this PR_ - For example: ```go m.Group("/users/{username}/orgs", func() { m.Get("", reqToken(), org.ListUserOrgs) m.Get("/{org}/permissions", reqToken(), org.GetUserOrgsPermissions) }, tokenRequiresScopes(auth_model.AccessTokenScopeCategoryUser, auth_model.AccessTokenScopeCategoryOrganization), context_service.UserAssignmentAPI()) ``` ## Future improvements - [ ] Add required scopes to swagger documentation - [ ] Redesign `reqToken()` to be opt-out rather than opt-in - [ ] Subdivide scopes like `repository` - [ ] Once a token is created, if it has no scopes, we should display text instead of an empty bullet point - [ ] If the 'public repos only' option is selected, should read categories be selected by default Closes #24501 Closes #24799 Co-authored-by: Jonathan Tran <jon@allspice.io> Co-authored-by: Kyle D <kdumontnu@gmail.com> Co-authored-by: silverwind <me@silverwind.io>
2023-06-04 18:57:16 +00:00
resp := storeAndGetLfsToken(t, &content, nil, http.StatusOK, auth.AccessTokenScopePublicOnly, auth.AccessTokenScopeReadRepository)
checkResponseTestContentEncoding(t, &content, resp, false)
}
func TestGetLFSSmallTokenFail(t *testing.T) {
defer tests.PrepareTestEnv(t)()
content := []byte("A very small file\n")
Redesign Scoped Access Tokens (#24767) ## Changes - Adds the following high level access scopes, each with `read` and `write` levels: - `activitypub` - `admin` (hidden if user is not a site admin) - `misc` - `notification` - `organization` - `package` - `issue` - `repository` - `user` - Adds new middleware function `tokenRequiresScopes()` in addition to `reqToken()` - `tokenRequiresScopes()` is used for each high-level api section - _if_ a scoped token is present, checks that the required scope is included based on the section and HTTP method - `reqToken()` is used for individual routes - checks that required authentication is present (but does not check scope levels as this will already have been handled by `tokenRequiresScopes()` - Adds migration to convert old scoped access tokens to the new set of scopes - Updates the user interface for scope selection ### User interface example <img width="903" alt="Screen Shot 2023-05-31 at 1 56 55 PM" src="https://github.com/go-gitea/gitea/assets/23248839/654766ec-2143-4f59-9037-3b51600e32f3"> <img width="917" alt="Screen Shot 2023-05-31 at 1 56 43 PM" src="https://github.com/go-gitea/gitea/assets/23248839/1ad64081-012c-4a73-b393-66b30352654c"> ## tokenRequiresScopes Design Decision - `tokenRequiresScopes()` was added to more reliably cover api routes. For an incoming request, this function uses the given scope category (say `AccessTokenScopeCategoryOrganization`) and the HTTP method (say `DELETE`) and verifies that any scoped tokens in use include `delete:organization`. - `reqToken()` is used to enforce auth for individual routes that require it. If a scoped token is not present for a request, `tokenRequiresScopes()` will not return an error ## TODO - [x] Alphabetize scope categories - [x] Change 'public repos only' to a radio button (private vs public). Also expand this to organizations - [X] Disable token creation if no scopes selected. Alternatively, show warning - [x] `reqToken()` is missing from many `POST/DELETE` routes in the api. `tokenRequiresScopes()` only checks that a given token has the correct scope, `reqToken()` must be used to check that a token (or some other auth) is present. - _This should be addressed in this PR_ - [x] The migration should be reviewed very carefully in order to minimize access changes to existing user tokens. - _This should be addressed in this PR_ - [x] Link to api to swagger documentation, clarify what read/write/delete levels correspond to - [x] Review cases where more than one scope is needed as this directly deviates from the api definition. - _This should be addressed in this PR_ - For example: ```go m.Group("/users/{username}/orgs", func() { m.Get("", reqToken(), org.ListUserOrgs) m.Get("/{org}/permissions", reqToken(), org.GetUserOrgsPermissions) }, tokenRequiresScopes(auth_model.AccessTokenScopeCategoryUser, auth_model.AccessTokenScopeCategoryOrganization), context_service.UserAssignmentAPI()) ``` ## Future improvements - [ ] Add required scopes to swagger documentation - [ ] Redesign `reqToken()` to be opt-out rather than opt-in - [ ] Subdivide scopes like `repository` - [ ] Once a token is created, if it has no scopes, we should display text instead of an empty bullet point - [ ] If the 'public repos only' option is selected, should read categories be selected by default Closes #24501 Closes #24799 Co-authored-by: Jonathan Tran <jon@allspice.io> Co-authored-by: Kyle D <kdumontnu@gmail.com> Co-authored-by: silverwind <me@silverwind.io>
2023-06-04 18:57:16 +00:00
storeAndGetLfsToken(t, &content, nil, http.StatusForbidden, auth.AccessTokenScopeReadNotification)
}
func TestGetLFSLarge(t *testing.T) {
defer tests.PrepareTestEnv(t)()
[GITEA] Use maintained gziphandler - https://github.com/NYTimes/gziphandler doesn't seems to be maintained anymore and Forgejo already includes https://github.com/klauspost/compress which provides a maintained and faster gzip handler fork. - Enables Jitter to prevent BREACH attacks, as this *seems* to be possible in the context of Forgejo. (cherry picked from commit cc2847241d82001babd8d40c87d03169f21c14cd) (cherry picked from commit 99ba56a8761dd08e08d9499cab2ded1a6b7b970f) Conflicts: go.sum https://codeberg.org/forgejo/forgejo/pulls/1581 (cherry picked from commit 711638193daa2311e2ead6249a47dcec47b4e335) (cherry picked from commit 9c12a37fde6fa84414bf332ff4a066facdb92d38) (cherry picked from commit d13065345431a499f9e0b7a3c2043d7487b8aa5b) (cherry picked from commit 45a16f8c3c6f7d5e4aab8fdde6a621cf36e4801c) (cherry picked from commit a497acb31f76d580c8b0567f9461274bd78080f4) (cherry picked from commit fe87fd828973945192b98310c5c3b2001c6e0f86) (cherry picked from commit 6ac12e6693cf45cb12109028dabd868957c4b74c) (cherry picked from commit 981ec37e1e72ab19c20067ff4d2a7e20a60d3305) (cherry picked from commit 5d6892ec10086f0ba63f26693faa82d0fd4e3f4a) (cherry picked from commit 9df7968f4fc72de9788d84ca3f349e4c98ee630e) (cherry picked from commit 7d588d183329cd760053663ea2e1e82e62958409) Conflicts: routers/web/web.go https://codeberg.org/forgejo/forgejo/pulls/2075 (cherry picked from commit defb101281f5a6ba410abc763674bafa7b63dffd) (cherry picked from commit 5830f204a17767fda3e45d16dbf3af8c32e7f387)
2023-09-30 22:09:25 +00:00
content := make([]byte, gzhttp.DefaultMinSize*10)
for i := range content {
content[i] = byte(i % 256)
}
resp := storeAndGetLfs(t, &content, nil, http.StatusOK)
checkResponseTestContentEncoding(t, &content, resp, true)
}
func TestGetLFSGzip(t *testing.T) {
defer tests.PrepareTestEnv(t)()
[GITEA] Use maintained gziphandler - https://github.com/NYTimes/gziphandler doesn't seems to be maintained anymore and Forgejo already includes https://github.com/klauspost/compress which provides a maintained and faster gzip handler fork. - Enables Jitter to prevent BREACH attacks, as this *seems* to be possible in the context of Forgejo. (cherry picked from commit cc2847241d82001babd8d40c87d03169f21c14cd) (cherry picked from commit 99ba56a8761dd08e08d9499cab2ded1a6b7b970f) Conflicts: go.sum https://codeberg.org/forgejo/forgejo/pulls/1581 (cherry picked from commit 711638193daa2311e2ead6249a47dcec47b4e335) (cherry picked from commit 9c12a37fde6fa84414bf332ff4a066facdb92d38) (cherry picked from commit d13065345431a499f9e0b7a3c2043d7487b8aa5b) (cherry picked from commit 45a16f8c3c6f7d5e4aab8fdde6a621cf36e4801c) (cherry picked from commit a497acb31f76d580c8b0567f9461274bd78080f4) (cherry picked from commit fe87fd828973945192b98310c5c3b2001c6e0f86) (cherry picked from commit 6ac12e6693cf45cb12109028dabd868957c4b74c) (cherry picked from commit 981ec37e1e72ab19c20067ff4d2a7e20a60d3305) (cherry picked from commit 5d6892ec10086f0ba63f26693faa82d0fd4e3f4a) (cherry picked from commit 9df7968f4fc72de9788d84ca3f349e4c98ee630e) (cherry picked from commit 7d588d183329cd760053663ea2e1e82e62958409) Conflicts: routers/web/web.go https://codeberg.org/forgejo/forgejo/pulls/2075 (cherry picked from commit defb101281f5a6ba410abc763674bafa7b63dffd) (cherry picked from commit 5830f204a17767fda3e45d16dbf3af8c32e7f387)
2023-09-30 22:09:25 +00:00
b := make([]byte, gzhttp.DefaultMinSize*10)
for i := range b {
b[i] = byte(i % 256)
}
outputBuffer := bytes.NewBuffer([]byte{})
gzippWriter := gzipp.NewWriter(outputBuffer)
gzippWriter.Write(b)
gzippWriter.Close()
content := outputBuffer.Bytes()
resp := storeAndGetLfs(t, &content, nil, http.StatusOK)
checkResponseTestContentEncoding(t, &content, resp, false)
}
func TestGetLFSZip(t *testing.T) {
defer tests.PrepareTestEnv(t)()
[GITEA] Use maintained gziphandler - https://github.com/NYTimes/gziphandler doesn't seems to be maintained anymore and Forgejo already includes https://github.com/klauspost/compress which provides a maintained and faster gzip handler fork. - Enables Jitter to prevent BREACH attacks, as this *seems* to be possible in the context of Forgejo. (cherry picked from commit cc2847241d82001babd8d40c87d03169f21c14cd) (cherry picked from commit 99ba56a8761dd08e08d9499cab2ded1a6b7b970f) Conflicts: go.sum https://codeberg.org/forgejo/forgejo/pulls/1581 (cherry picked from commit 711638193daa2311e2ead6249a47dcec47b4e335) (cherry picked from commit 9c12a37fde6fa84414bf332ff4a066facdb92d38) (cherry picked from commit d13065345431a499f9e0b7a3c2043d7487b8aa5b) (cherry picked from commit 45a16f8c3c6f7d5e4aab8fdde6a621cf36e4801c) (cherry picked from commit a497acb31f76d580c8b0567f9461274bd78080f4) (cherry picked from commit fe87fd828973945192b98310c5c3b2001c6e0f86) (cherry picked from commit 6ac12e6693cf45cb12109028dabd868957c4b74c) (cherry picked from commit 981ec37e1e72ab19c20067ff4d2a7e20a60d3305) (cherry picked from commit 5d6892ec10086f0ba63f26693faa82d0fd4e3f4a) (cherry picked from commit 9df7968f4fc72de9788d84ca3f349e4c98ee630e) (cherry picked from commit 7d588d183329cd760053663ea2e1e82e62958409) Conflicts: routers/web/web.go https://codeberg.org/forgejo/forgejo/pulls/2075 (cherry picked from commit defb101281f5a6ba410abc763674bafa7b63dffd) (cherry picked from commit 5830f204a17767fda3e45d16dbf3af8c32e7f387)
2023-09-30 22:09:25 +00:00
b := make([]byte, gzhttp.DefaultMinSize*10)
for i := range b {
b[i] = byte(i % 256)
}
outputBuffer := bytes.NewBuffer([]byte{})
zipWriter := zip.NewWriter(outputBuffer)
fileWriter, err := zipWriter.Create("default")
assert.NoError(t, err)
fileWriter.Write(b)
zipWriter.Close()
content := outputBuffer.Bytes()
resp := storeAndGetLfs(t, &content, nil, http.StatusOK)
checkResponseTestContentEncoding(t, &content, resp, false)
}
func TestGetLFSRangeNo(t *testing.T) {
defer tests.PrepareTestEnv(t)()
content := []byte("123456789\n")
resp := storeAndGetLfs(t, &content, nil, http.StatusOK)
assert.Equal(t, content, resp.Body.Bytes())
}
func TestGetLFSRange(t *testing.T) {
defer tests.PrepareTestEnv(t)()
content := []byte("123456789\n")
tests := []struct {
in string
out string
status int
}{
{"bytes=0-0", "1", http.StatusPartialContent},
{"bytes=0-1", "12", http.StatusPartialContent},
{"bytes=1-1", "2", http.StatusPartialContent},
{"bytes=1-3", "234", http.StatusPartialContent},
{"bytes=1-", "23456789\n", http.StatusPartialContent},
// end-range smaller than start-range is ignored
{"bytes=1-0", "23456789\n", http.StatusPartialContent},
{"bytes=0-10", "123456789\n", http.StatusPartialContent},
// end-range bigger than length-1 is ignored
{"bytes=0-11", "123456789\n", http.StatusPartialContent},
{"bytes=11-", "Requested Range Not Satisfiable", http.StatusRequestedRangeNotSatisfiable},
// incorrect header value cause whole header to be ignored
{"bytes=-", "123456789\n", http.StatusOK},
{"foobar", "123456789\n", http.StatusOK},
}
for _, tt := range tests {
t.Run(tt.in, func(t *testing.T) {
h := http.Header{
"Range": []string{tt.in},
}
resp := storeAndGetLfs(t, &content, &h, tt.status)
if tt.status == http.StatusPartialContent || tt.status == http.StatusOK {
assert.Equal(t, tt.out, resp.Body.String())
} else {
var er lfs.ErrorResponse
err := json.Unmarshal(resp.Body.Bytes(), &er)
assert.NoError(t, err)
assert.Equal(t, tt.out, er.Message)
}
})
}
}