gitea/services/packages/cleanup/cleanup.go

// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package container

import (
	"context"
	"fmt"
	"time"

	"code.gitea.io/gitea/models/db"
	packages_model "code.gitea.io/gitea/models/packages"
	user_model "code.gitea.io/gitea/models/user"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/optional"
	packages_module "code.gitea.io/gitea/modules/packages"
	packages_service "code.gitea.io/gitea/services/packages"
	alpine_service "code.gitea.io/gitea/services/packages/alpine"
	arch_service "code.gitea.io/gitea/services/packages/arch"
	cargo_service "code.gitea.io/gitea/services/packages/cargo"
	container_service "code.gitea.io/gitea/services/packages/container"
	debian_service "code.gitea.io/gitea/services/packages/debian"
	rpm_service "code.gitea.io/gitea/services/packages/rpm"
)

// CleanupTask executes cleanup rules and cleanup expired package data
func CleanupTask(ctx context.Context, olderThan time.Duration) error {
	if err := ExecuteCleanupRules(ctx); err != nil {
		return err
	}

	return CleanupExpiredData(ctx, olderThan)
}

func executeCleanupOneRulePackage(ctx context.Context, pcr *packages_model.PackageCleanupRule, p *packages_model.Package) (versionDeleted bool, err error) {
	olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
	pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
		PackageID:  p.ID,
		IsInternal: optional.Some(false),
		Sort:       packages_model.SortCreatedDesc,
	})
	if err != nil {
		return false, fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
	}
	if pcr.KeepCount > 0 {
		if pcr.KeepCount < len(pvs) {
			pvs = pvs[pcr.KeepCount:]
		} else {
			pvs = nil
		}
	}
	for _, pv := range pvs {
		if pcr.Type == packages_model.TypeContainer {
			if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
				return false, fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
			} else if skip {
				log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
				continue
			}
		}
		toMatch := pv.LowerVersion
		if pcr.MatchFullName {
			toMatch = p.LowerName + "/" + pv.LowerVersion
		}
		if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
			log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
			continue
		}
		if pv.CreatedUnix.AsLocalTime().After(olderThan) {
			log.Debug("Rule[%d]: keep '%s/%s' (remove days) %v", pcr.ID, p.Name, pv.Version, pv.CreatedUnix.FormatDate())
			continue
		}
		if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
			log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
			continue
		}
		log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
		if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
			log.Error("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %v", pcr.ID, err)
			continue
		}
		versionDeleted = true
	}
	return versionDeleted, nil
}

func executeCleanupOneRule(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
	if err := pcr.CompiledPattern(); err != nil {
		return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
	}

	packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
	if err != nil {
		return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
	}

	anyVersionDeleted := false
	for _, p := range packages {
		versionDeleted := false
		err = db.WithTx(ctx, func(ctx context.Context) (err error) {
			versionDeleted, err = executeCleanupOneRulePackage(ctx, pcr, p)
			return err
		})
		if err != nil {
			log.Error("CleanupRule [%d]: executeCleanupOneRulePackage(%d) failed: %v", pcr.ID, p.ID, err)
			continue
		}
		anyVersionDeleted = anyVersionDeleted || versionDeleted
		if versionDeleted {
			if pcr.Type == packages_model.TypeCargo {
				owner, err := user_model.GetUserByID(ctx, pcr.OwnerID)
				if err != nil {
					return fmt.Errorf("GetUserByID failed: %w", err)
				}
				if err := cargo_service.UpdatePackageIndexIfExists(ctx, owner, owner, p.ID); err != nil {
					return fmt.Errorf("CleanupRule [%d]: cargo.UpdatePackageIndexIfExists failed: %w", pcr.ID, err)
				}
			}
		}
	}

	if anyVersionDeleted {
		switch pcr.Type {
		case packages_model.TypeDebian:
			if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
				return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
			}
		case packages_model.TypeAlpine:
			if err := alpine_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
				return fmt.Errorf("CleanupRule [%d]: alpine.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
			}
		case packages_model.TypeRpm:
			if err := rpm_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
				return fmt.Errorf("CleanupRule [%d]: rpm.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
			}
		case packages_model.TypeArch:
			release, err := arch_service.AquireRegistryLock(ctx, pcr.OwnerID)
			if err != nil {
				return err
			}
			defer release()

			if err := arch_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
				return fmt.Errorf("CleanupRule [%d]: arch.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
			}
		}
	}
	return nil
}

func ExecuteCleanupRules(ctx context.Context) error {
	return packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
		select {
		case <-ctx.Done():
			return db.ErrCancelledf("While processing package cleanup rules")
		default:
		}

		err := executeCleanupOneRule(ctx, pcr)
		if err != nil {
			log.Error("CleanupRule [%d]: executeCleanupOneRule failed: %v", pcr.ID, err)
		}
		return nil
	})
}

func CleanupExpiredData(ctx context.Context, olderThan time.Duration) error {
	pbs := make([]*packages_model.PackageBlob, 0, 100)
	if err := db.WithTx(ctx, func(ctx context.Context) error {
		if err := container_service.Cleanup(ctx, olderThan); err != nil {
			return err
		}

		ps, err := packages_model.FindUnreferencedPackages(ctx)
		if err != nil {
			return err
		}
		for _, p := range ps {
			if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, p.ID); err != nil {
				return err
			}
			if err := packages_model.DeletePackageByID(ctx, p.ID); err != nil {
				return err
			}
		}

		pbs, err = packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan)
		if err != nil {
			return err
		}

		for _, pb := range pbs {
			if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil {
				return err
			}
		}
		return nil
	}); err != nil {
		return err
	}

	contentStore := packages_module.NewContentStore()
	for _, pb := range pbs {
		if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
			log.Error("Error deleting package blob [%v]: %v", pb.ID, err)
		}
	}

	return nil
}