forgejo/services/packages/packages.go
KN4CK3R 1d332342db
Add Package Registry (#16510)
* Added package store settings.

* Added models.

* Added generic package registry.

* Added tests.

* Added NuGet package registry.

* Moved service index to api file.

* Added NPM package registry.

* Added Maven package registry.

* Added PyPI package registry.

* Summary is deprecated.

* Changed npm name.

* Sanitize project url.

* Allow only scoped packages.

* Added user interface.

* Changed method name.

* Added missing migration file.

* Set page info.

* Added documentation.

* Added documentation links.

* Fixed wrong error message.

* Lint template files.

* Fixed merge errors.

* Fixed unit test storage path.

* Switch to json module.

* Added suggestions.

* Added package webhook.

* Add package api.

* Fixed swagger file.

* Fixed enum and comments.

* Fixed NuGet pagination.

* Print test names.

* Added api tests.

* Fixed access level.

* Fix User unmarshal.

* Added RubyGems package registry.

* Fix lint.

* Implemented io.Writer.

* Added support for sha256/sha512 checksum files.

* Improved maven-metadata.xml support.

* Added support for symbol package uploads.

* Added tests.

* Added overview docs.

* Added npm dependencies and keywords.

* Added no-packages information.

* Display file size.

* Display asset count.

* Fixed filter alignment.

* Added package icons.

* Formatted instructions.

* Allow anonymous package downloads.

* Fixed comments.

* Fixed postgres test.

* Moved file.

* Moved models to models/packages.

* Use correct error response format per client.

* Use simpler search form.

* Fixed IsProd.

* Restructured data model.

* Prevent empty filename.

* Fix swagger.

* Implemented user/org registry.

* Implemented UI.

* Use GetUserByIDCtx.

* Use table for dependencies.

* make svg

* Added support for unscoped npm packages.

* Add support for npm dist tags.

* Added tests for npm tags.

* Unlink packages if repository gets deleted.

* Prevent user/org delete if a packages exist.

* Use package unlink in repository service.

* Added support for composer packages.

* Restructured package docs.

* Added missing tests.

* Fixed generic content page.

* Fixed docs.

* Fixed swagger.

* Added missing type.

* Fixed ambiguous column.

* Organize content store by sha256 hash.

* Added admin package management.

* Added support for sorting.

* Add support for multiple identical versions/files.

* Added missing repository unlink.

* Added file properties.

* make fmt

* lint

* Added Conan package registry.

* Updated docs.

* Unify package names.

* Added swagger enum.

* Use longer TEXT column type.

* Removed version composite key.

* Merged package and container registry.

* Removed index.

* Use dedicated package router.

* Moved files to new location.

* Updated docs.

* Fixed JOIN order.

* Fixed GROUP BY statement.

* Fixed GROUP BY #2.

* Added symbol server support.

* Added more tests.

* Set NOT NULL.

* Added setting to disable package registries.

* Moved auth into service.

* refactor

* Use ctx everywhere.

* Added package cleanup task.

* Changed packages path.

* Added container registry.

* Refactoring

* Updated comparison.

* Fix swagger.

* Fixed table order.

* Use token auth for npm routes.

* Enabled ReverseProxy auth.

* Added packages link for orgs.

* Fixed anonymous org access.

* Enable copy button for setup instructions.

* Merge error

* Added suggestions.

* Fixed merge.

* Handle "generic".

* Added link for TODO.

* Added suggestions.

* Changed temporary buffer filename.

* Added suggestions.

* Apply suggestions from code review

Co-authored-by: Thomas Boerger <thomas@webhippie.de>

* Update docs/content/doc/packages/nuget.en-us.md

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Thomas Boerger <thomas@webhippie.de>
2022-03-30 16:42:47 +08:00

459 lines
14 KiB
Go

// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package packages
import (
"context"
"fmt"
"io"
"strings"
"time"
"code.gitea.io/gitea/models/db"
packages_model "code.gitea.io/gitea/models/packages"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification"
packages_module "code.gitea.io/gitea/modules/packages"
container_service "code.gitea.io/gitea/services/packages/container"
)
// PackageInfo describes a package
type PackageInfo struct {
Owner *user_model.User
PackageType packages_model.Type
Name string
Version string
}
// PackageCreationInfo describes a package to create
type PackageCreationInfo struct {
PackageInfo
SemverCompatible bool
Creator *user_model.User
Metadata interface{}
Properties map[string]string
}
// PackageFileInfo describes a package file
type PackageFileInfo struct {
Filename string
CompositeKey string
}
// PackageFileCreationInfo describes a package file to create
type PackageFileCreationInfo struct {
PackageFileInfo
Data packages_module.HashedSizeReader
IsLead bool
Properties map[string]string
OverwriteExisting bool
}
// CreatePackageAndAddFile creates a package with a file. If the same package exists already, ErrDuplicatePackageVersion is returned
func CreatePackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
return createPackageAndAddFile(pvci, pfci, false)
}
// CreatePackageOrAddFileToExisting creates a package with a file or adds the file if the package exists already
func CreatePackageOrAddFileToExisting(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
return createPackageAndAddFile(pvci, pfci, true)
}
func createPackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo, allowDuplicate bool) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
ctx, committer, err := db.TxContext()
if err != nil {
return nil, nil, err
}
defer committer.Close()
pv, created, err := createPackageAndVersion(ctx, pvci, allowDuplicate)
if err != nil {
return nil, nil, err
}
pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci)
removeBlob := false
defer func() {
if blobCreated && removeBlob {
contentStore := packages_module.NewContentStore()
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
log.Error("Error deleting package blob from content store: %v", err)
}
}
}()
if err != nil {
removeBlob = true
return nil, nil, err
}
if err := committer.Commit(); err != nil {
removeBlob = true
return nil, nil, err
}
if created {
pd, err := packages_model.GetPackageDescriptor(ctx, pv)
if err != nil {
return nil, nil, err
}
notification.NotifyPackageCreate(pvci.Creator, pd)
}
return pv, pf, nil
}
func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, allowDuplicate bool) (*packages_model.PackageVersion, bool, error) {
log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %v", pvci.Creator.ID, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version, pvci.Properties, allowDuplicate)
p := &packages_model.Package{
OwnerID: pvci.Owner.ID,
Type: pvci.PackageType,
Name: pvci.Name,
LowerName: strings.ToLower(pvci.Name),
SemverCompatible: pvci.SemverCompatible,
}
var err error
if p, err = packages_model.TryInsertPackage(ctx, p); err != nil {
if err != packages_model.ErrDuplicatePackage {
log.Error("Error inserting package: %v", err)
return nil, false, err
}
}
metadataJSON, err := json.Marshal(pvci.Metadata)
if err != nil {
return nil, false, err
}
created := true
pv := &packages_model.PackageVersion{
PackageID: p.ID,
CreatorID: pvci.Creator.ID,
Version: pvci.Version,
LowerVersion: strings.ToLower(pvci.Version),
MetadataJSON: string(metadataJSON),
}
if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
created = false
}
if err != packages_model.ErrDuplicatePackageVersion || !allowDuplicate {
log.Error("Error inserting package: %v", err)
return nil, false, err
}
}
if created {
for name, value := range pvci.Properties {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil {
log.Error("Error setting package version property: %v", err)
return nil, false, err
}
}
}
return pv, created, nil
}
// AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned
func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
ctx, committer, err := db.TxContext()
if err != nil {
return nil, nil, err
}
defer committer.Close()
pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
if err != nil {
return nil, nil, err
}
pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci)
removeBlob := false
defer func() {
if removeBlob {
contentStore := packages_module.NewContentStore()
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
log.Error("Error deleting package blob from content store: %v", err)
}
}
}()
if err != nil {
removeBlob = blobCreated
return nil, nil, err
}
if err := committer.Commit(); err != nil {
removeBlob = blobCreated
return nil, nil, err
}
return pv, pf, nil
}
// NewPackageBlob creates a package blob instance
func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.PackageBlob {
hashMD5, hashSHA1, hashSHA256, hashSHA512 := hsr.Sums()
return &packages_model.PackageBlob{
Size: hsr.Size(),
HashMD5: fmt.Sprintf("%x", hashMD5),
HashSHA1: fmt.Sprintf("%x", hashSHA1),
HashSHA256: fmt.Sprintf("%x", hashSHA256),
HashSHA512: fmt.Sprintf("%x", hashSHA512),
}
}
func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data))
if err != nil {
log.Error("Error inserting package blob: %v", err)
return nil, nil, false, err
}
if !exists {
contentStore := packages_module.NewContentStore()
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), pfci.Data, pfci.Data.Size()); err != nil {
log.Error("Error saving package blob in content store: %v", err)
return nil, nil, false, err
}
}
if pfci.OverwriteExisting {
pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, pfci.Filename, pfci.CompositeKey)
if err != nil && err != packages_model.ErrPackageFileNotExist {
return nil, pb, !exists, err
}
if pf != nil {
// Short circuit if blob is the same
if pf.BlobID == pb.ID {
return pf, pb, !exists, nil
}
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
return nil, pb, !exists, err
}
if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
return nil, pb, !exists, err
}
}
}
pf := &packages_model.PackageFile{
VersionID: pv.ID,
BlobID: pb.ID,
Name: pfci.Filename,
LowerName: strings.ToLower(pfci.Filename),
CompositeKey: pfci.CompositeKey,
IsLead: pfci.IsLead,
}
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
if err != packages_model.ErrDuplicatePackageFile {
log.Error("Error inserting package file: %v", err)
}
return nil, pb, !exists, err
}
for name, value := range pfci.Properties {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil {
log.Error("Error setting package file property: %v", err)
return pf, pb, !exists, err
}
}
return pf, pb, !exists, nil
}
// RemovePackageVersionByNameAndVersion deletes a package version and all associated files
func RemovePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error {
pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
if err != nil {
return err
}
return RemovePackageVersion(doer, pv)
}
// RemovePackageVersion deletes the package version and all associated files
func RemovePackageVersion(doer *user_model.User, pv *packages_model.PackageVersion) error {
ctx, committer, err := db.TxContext()
if err != nil {
return err
}
defer committer.Close()
pd, err := packages_model.GetPackageDescriptor(ctx, pv)
if err != nil {
return err
}
log.Trace("Deleting package: %v", pv.ID)
if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
return err
}
if err := committer.Commit(); err != nil {
return err
}
notification.NotifyPackageDelete(doer, pd)
return nil
}
// DeletePackageVersionAndReferences deletes the package version and its properties and files
func DeletePackageVersionAndReferences(ctx context.Context, pv *packages_model.PackageVersion) error {
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil {
return err
}
pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
if err != nil {
return err
}
for _, pf := range pfs {
if err := DeletePackageFile(ctx, pf); err != nil {
return err
}
}
return packages_model.DeleteVersionByID(ctx, pv.ID)
}
// DeletePackageFile deletes the package file and its properties
func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) error {
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
return err
}
return packages_model.DeleteFileByID(ctx, pf.ID)
}
// Cleanup removes old unreferenced package blobs
func Cleanup(unused context.Context, olderThan time.Duration) error {
ctx, committer, err := db.TxContext()
if err != nil {
return err
}
defer committer.Close()
if err := container_service.Cleanup(ctx, olderThan); err != nil {
log.Error("hier")
return err
}
if err := packages_model.DeletePackagesIfUnreferenced(ctx); err != nil {
log.Error("hier2")
return err
}
pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan)
if err != nil {
log.Error("hier3")
return err
}
for _, pb := range pbs {
if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil {
log.Error("hier4")
return err
}
}
if err := committer.Commit(); err != nil {
return err
}
contentStore := packages_module.NewContentStore()
for _, pb := range pbs {
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
log.Error("Error deleting package blob [%v]: %v", pb.ID, err)
}
}
return nil
}
// GetFileStreamByPackageNameAndVersion returns the content of the specific package file
func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) {
log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey)
pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
if err != nil {
if err == packages_model.ErrPackageNotExist {
return nil, nil, err
}
log.Error("Error getting package: %v", err)
return nil, nil, err
}
return GetFileStreamByPackageVersion(ctx, pv, pfi)
}
// GetFileStreamByPackageVersionAndFileID returns the content of the specific package file
func GetFileStreamByPackageVersionAndFileID(ctx context.Context, owner *user_model.User, versionID, fileID int64) (io.ReadCloser, *packages_model.PackageFile, error) {
log.Trace("Getting package file stream: %v, %v, %v", owner.ID, versionID, fileID)
pv, err := packages_model.GetVersionByID(ctx, versionID)
if err != nil {
if err == packages_model.ErrPackageVersionNotExist {
return nil, nil, packages_model.ErrPackageNotExist
}
log.Error("Error getting package version: %v", err)
return nil, nil, err
}
p, err := packages_model.GetPackageByID(ctx, pv.PackageID)
if err != nil {
log.Error("Error getting package: %v", err)
return nil, nil, err
}
if p.OwnerID != owner.ID {
return nil, nil, packages_model.ErrPackageNotExist
}
pf, err := packages_model.GetFileForVersionByID(ctx, versionID, fileID)
if err != nil {
log.Error("Error getting file: %v", err)
return nil, nil, err
}
return GetPackageFileStream(ctx, pv, pf)
}
// GetFileStreamByPackageVersion returns the content of the specific package file
func GetFileStreamByPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) {
pf, err := packages_model.GetFileForVersionByName(db.DefaultContext, pv.ID, pfi.Filename, pfi.CompositeKey)
if err != nil {
return nil, nil, err
}
return GetPackageFileStream(ctx, pv, pf)
}
// GetPackageFileStream returns the content of the specific package file
func GetPackageFileStream(ctx context.Context, pv *packages_model.PackageVersion, pf *packages_model.PackageFile) (io.ReadCloser, *packages_model.PackageFile, error) {
pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
if err != nil {
return nil, nil, err
}
s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256))
if err == nil {
if pf.IsLead {
if err := packages_model.IncrementDownloadCounter(ctx, pv.ID); err != nil {
log.Error("Error incrementing download counter: %v", err)
}
}
}
return s, pf, err
}