first-commit

services/packages/container/blob_uploader.go

@@ -0,0 +1,142 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package container
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/tempdir"
+)
+
+var (
+	// errWriteAfterRead occurs if Write is called after a read operation
+	errWriteAfterRead = errors.New("write is unsupported after a read operation")
+	// errOffsetMissmatch occurs if the file offset is different than the model
+	errOffsetMissmatch = errors.New("offset mismatch between file and model")
+)
+
+// BlobUploader handles chunked blob uploads
+type BlobUploader struct {
+	*packages_model.PackageBlobUpload
+	*packages_module.MultiHasher
+	file    *os.File
+	reading bool
+}
+
+func uploadPathTempDir() *tempdir.TempDir {
+	return setting.AppDataTempDir("package-upload")
+}
+
+func buildFilePath(uploadPath *tempdir.TempDir, id string) string {
+	return uploadPath.JoinPath(id)
+}
+
+// NewBlobUploader creates a new blob uploader for the given id
+func NewBlobUploader(ctx context.Context, id string) (*BlobUploader, error) {
+	model, err := packages_model.GetBlobUploadByID(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	hash := packages_module.NewMultiHasher()
+	if len(model.HashStateBytes) != 0 {
+		if err := hash.UnmarshalBinary(model.HashStateBytes); err != nil {
+			return nil, err
+		}
+	}
+
+	uploadPath := uploadPathTempDir()
+	_, err = uploadPath.MkdirAllSub("")
+	if err != nil {
+		return nil, err
+	}
+	f, err := os.OpenFile(buildFilePath(uploadPath, model.ID), os.O_RDWR|os.O_CREATE, 0o666)
+	if err != nil {
+		return nil, err
+	}
+
+	return &BlobUploader{
+		model,
+		hash,
+		f,
+		false,
+	}, nil
+}
+
+// Close implements io.Closer
+func (u *BlobUploader) Close() error {
+	return u.file.Close()
+}
+
+// Append appends a chunk of data and updates the model
+func (u *BlobUploader) Append(ctx context.Context, r io.Reader) error {
+	if u.reading {
+		return errWriteAfterRead
+	}
+
+	offset, err := u.file.Seek(0, io.SeekEnd)
+	if err != nil {
+		return err
+	}
+	if offset != u.BytesReceived {
+		return errOffsetMissmatch
+	}
+
+	n, err := io.Copy(io.MultiWriter(u.file, u.MultiHasher), r)
+	if err != nil {
+		return err
+	}
+
+	// fast path if nothing was written
+	if n == 0 {
+		return nil
+	}
+
+	u.BytesReceived += n
+
+	u.HashStateBytes, err = u.MultiHasher.MarshalBinary()
+	if err != nil {
+		return err
+	}
+
+	return packages_model.UpdateBlobUpload(ctx, u.PackageBlobUpload)
+}
+
+func (u *BlobUploader) Size() int64 {
+	return u.BytesReceived
+}
+
+// Read implements io.Reader
+func (u *BlobUploader) Read(p []byte) (int, error) {
+	if !u.reading {
+		_, err := u.file.Seek(0, io.SeekStart)
+		if err != nil {
+			return 0, err
+		}
+
+		u.reading = true
+	}
+
+	return u.file.Read(p)
+}
+
+// RemoveBlobUploadByID Remove deletes the data and the model of a blob upload
+func RemoveBlobUploadByID(ctx context.Context, id string) error {
+	if err := packages_model.DeleteBlobUploadByID(ctx, id); err != nil {
+		return err
+	}
+
+	err := os.Remove(buildFilePath(uploadPathTempDir(), id))
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	return nil
+}

services/packages/container/cleanup.go

@@ -0,0 +1,108 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package container
+
+import (
+	"context"
+	"time"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	container_model "code.gitea.io/gitea/models/packages/container"
+	"code.gitea.io/gitea/modules/optional"
+	container_module "code.gitea.io/gitea/modules/packages/container"
+	packages_service "code.gitea.io/gitea/services/packages"
+
+	"github.com/opencontainers/go-digest"
+)
+
+// Cleanup removes expired container data
+func Cleanup(ctx context.Context, olderThan time.Duration) error {
+	if err := cleanupExpiredBlobUploads(ctx, olderThan); err != nil {
+		return err
+	}
+	return cleanupExpiredUploadedBlobs(ctx, olderThan)
+}
+
+// cleanupExpiredBlobUploads removes expired blob uploads
+func cleanupExpiredBlobUploads(ctx context.Context, olderThan time.Duration) error {
+	pbus, err := packages_model.FindExpiredBlobUploads(ctx, olderThan)
+	if err != nil {
+		return err
+	}
+
+	for _, pbu := range pbus {
+		if err := RemoveBlobUploadByID(ctx, pbu.ID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// cleanupExpiredUploadedBlobs removes expired uploaded blobs not referenced by a manifest
+func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) error {
+	pfs, err := container_model.SearchExpiredUploadedBlobs(ctx, olderThan)
+	if err != nil {
+		return err
+	}
+
+	for _, pf := range pfs {
+		if err := packages_service.DeletePackageFile(ctx, pf); err != nil {
+			return err
+		}
+	}
+
+	pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
+		Type: packages_model.TypeContainer,
+		Version: packages_model.SearchValue{
+			ExactMatch: true,
+			Value:      container_module.UploadVersion,
+		},
+		IsInternal: optional.Some(true),
+		HasFiles:   optional.Some(false),
+	})
+	if err != nil {
+		return err
+	}
+
+	for _, pv := range pvs {
+		if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil {
+			return err
+		}
+
+		if err := packages_model.DeleteVersionByID(ctx, pv.ID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func ShouldBeSkipped(ctx context.Context, pcr *packages_model.PackageCleanupRule, p *packages_model.Package, pv *packages_model.PackageVersion) (bool, error) {
+	// Always skip the "latest" tag
+	if pv.LowerVersion == "latest" {
+		return true, nil
+	}
+
+	// Check if the version is a digest (or untagged)
+	if digest.Digest(pv.LowerVersion).Validate() == nil {
+		// Check if there is another manifest referencing this version
+		has, err := packages_model.ExistVersion(ctx, &packages_model.PackageSearchOptions{
+			PackageID: p.ID,
+			Properties: map[string]string{
+				container_module.PropertyManifestReference: pv.LowerVersion,
+			},
+		})
+		if err != nil {
+			return false, err
+		}
+
+		// Skip it if the version is referenced
+		if has {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}

services/packages/container/common.go

@@ -0,0 +1,64 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package container
+
+import (
+	"context"
+	"io"
+	"strings"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	container_service "code.gitea.io/gitea/models/packages/container"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/packages"
+	container_module "code.gitea.io/gitea/modules/packages/container"
+
+	"github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// UpdateRepositoryNames updates the repository name property for all packages of the specific owner
+func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error {
+	ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer)
+	if err != nil {
+		return err
+	}
+
+	newOwnerName = strings.ToLower(newOwnerName)
+
+	for _, p := range ps {
+		if err := packages_model.DeletePropertiesByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
+			return err
+		}
+
+		if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func ParseManifestMetadata(ctx context.Context, rd io.Reader, ownerID int64, imageName string) (*v1.Manifest, *packages_model.PackageFileDescriptor, *container_module.Metadata, error) {
+	var manifest v1.Manifest
+	if err := json.NewDecoder(rd).Decode(&manifest); err != nil {
+		return nil, nil, nil, err
+	}
+	configDescriptor, err := container_service.GetContainerBlob(ctx, &container_service.BlobSearchOptions{
+		OwnerID: ownerID,
+		Image:   imageName,
+		Digest:  manifest.Config.Digest.String(),
+	})
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	configReader, err := packages.NewContentStore().OpenBlob(packages.BlobHash256Key(configDescriptor.Blob.HashSHA256))
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	defer configReader.Close()
+	metadata, err := container_module.ParseImageConfig(manifest.Config.MediaType, configReader)
+	return &manifest, configDescriptor, metadata, err
+}