first-commit
This commit is contained in:
130
services/cron/cron.go
Normal file
130
services/cron/cron.go
Normal file
@@ -0,0 +1,130 @@
|
||||
// Copyright 2014 The Gogs Authors. All rights reserved.
|
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cron
|
||||
|
||||
import (
|
||||
"context"
|
||||
"runtime/pprof"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
"code.gitea.io/gitea/modules/process"
|
||||
"code.gitea.io/gitea/modules/sync"
|
||||
"code.gitea.io/gitea/modules/translation"
|
||||
|
||||
"github.com/go-co-op/gocron"
|
||||
)
|
||||
|
||||
var scheduler = gocron.NewScheduler(time.Local)
|
||||
|
||||
// Prevent duplicate running tasks.
|
||||
var taskStatusTable = sync.NewStatusTable()
|
||||
|
||||
// NewContext begins cron tasks
|
||||
// Each cron task is run within the shutdown context as a running server
|
||||
// AtShutdown the cron server is stopped
|
||||
func NewContext(original context.Context) {
|
||||
defer pprof.SetGoroutineLabels(original)
|
||||
_, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().ShutdownContext(), "Service: Cron", process.SystemProcessType, true)
|
||||
initBasicTasks()
|
||||
initExtendedTasks()
|
||||
initActionsTasks()
|
||||
|
||||
lock.Lock()
|
||||
for _, task := range tasks {
|
||||
if task.IsEnabled() && task.DoRunAtStart() {
|
||||
go task.Run()
|
||||
}
|
||||
}
|
||||
|
||||
scheduler.StartAsync()
|
||||
started = true
|
||||
lock.Unlock()
|
||||
graceful.GetManager().RunAtShutdown(context.Background(), func() {
|
||||
scheduler.Stop()
|
||||
lock.Lock()
|
||||
started = false
|
||||
lock.Unlock()
|
||||
finished()
|
||||
})
|
||||
}
|
||||
|
||||
// TaskTableRow represents a task row in the tasks table
|
||||
type TaskTableRow struct {
|
||||
Name string
|
||||
Spec string
|
||||
Next time.Time
|
||||
Prev time.Time
|
||||
Status string
|
||||
LastMessage string
|
||||
LastDoer string
|
||||
ExecTimes int64
|
||||
task *Task
|
||||
}
|
||||
|
||||
func (t *TaskTableRow) FormatLastMessage(locale translation.Locale) string {
|
||||
if t.Status == "finished" {
|
||||
return t.task.GetConfig().FormatMessage(locale, t.Name, t.Status, t.LastDoer)
|
||||
}
|
||||
|
||||
return t.task.GetConfig().FormatMessage(locale, t.Name, t.Status, t.LastDoer, t.LastMessage)
|
||||
}
|
||||
|
||||
// TaskTable represents a table of tasks
|
||||
type TaskTable []*TaskTableRow
|
||||
|
||||
// ListTasks returns all running cron tasks.
|
||||
func ListTasks() TaskTable {
|
||||
jobs := scheduler.Jobs()
|
||||
jobMap := map[string]*gocron.Job{}
|
||||
for _, job := range jobs {
|
||||
// the first tag is the task name
|
||||
tags := job.Tags()
|
||||
if len(tags) == 0 { // should never happen
|
||||
continue
|
||||
}
|
||||
jobMap[job.Tags()[0]] = job
|
||||
}
|
||||
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
tTable := make([]*TaskTableRow, 0, len(tasks))
|
||||
for _, task := range tasks {
|
||||
spec := "-"
|
||||
var (
|
||||
next time.Time
|
||||
prev time.Time
|
||||
)
|
||||
if e, ok := jobMap[task.Name]; ok {
|
||||
tags := e.Tags()
|
||||
if len(tags) > 1 {
|
||||
spec = tags[1] // the second tag is the task spec
|
||||
}
|
||||
next = e.NextRun()
|
||||
prev = e.PreviousRun()
|
||||
}
|
||||
|
||||
task.lock.Lock()
|
||||
// If the manual run is after the cron run, use that instead.
|
||||
if prev.Before(task.LastRun) {
|
||||
prev = task.LastRun
|
||||
}
|
||||
tTable = append(tTable, &TaskTableRow{
|
||||
Name: task.Name,
|
||||
Spec: spec,
|
||||
Next: next,
|
||||
Prev: prev,
|
||||
ExecTimes: task.ExecTimes,
|
||||
LastMessage: task.LastMessage,
|
||||
Status: task.Status,
|
||||
LastDoer: task.LastDoer,
|
||||
task: task,
|
||||
})
|
||||
task.lock.Unlock()
|
||||
}
|
||||
|
||||
return tTable
|
||||
}
|
86
services/cron/setting.go
Normal file
86
services/cron/setting.go
Normal file
@@ -0,0 +1,86 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cron
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/translation"
|
||||
)
|
||||
|
||||
// Config represents a basic configuration interface that cron task
|
||||
type Config interface {
|
||||
IsEnabled() bool
|
||||
DoRunAtStart() bool
|
||||
GetSchedule() string
|
||||
FormatMessage(locale translation.Locale, name, status, doer string, args ...any) string
|
||||
DoNoticeOnSuccess() bool
|
||||
}
|
||||
|
||||
// BaseConfig represents the basic config for a Cron task
|
||||
type BaseConfig struct {
|
||||
Enabled bool
|
||||
RunAtStart bool
|
||||
Schedule string
|
||||
NoticeOnSuccess bool
|
||||
}
|
||||
|
||||
// OlderThanConfig represents a cron task with OlderThan setting
|
||||
type OlderThanConfig struct {
|
||||
BaseConfig
|
||||
OlderThan time.Duration
|
||||
}
|
||||
|
||||
// UpdateExistingConfig represents a cron task with UpdateExisting setting
|
||||
type UpdateExistingConfig struct {
|
||||
BaseConfig
|
||||
UpdateExisting bool
|
||||
}
|
||||
|
||||
// CleanupHookTaskConfig represents a cron task with settings to cleanup hook_task
|
||||
type CleanupHookTaskConfig struct {
|
||||
BaseConfig
|
||||
CleanupType string
|
||||
OlderThan time.Duration
|
||||
NumberToKeep int
|
||||
}
|
||||
|
||||
// GetSchedule returns the schedule for the base config
|
||||
func (b *BaseConfig) GetSchedule() string {
|
||||
return b.Schedule
|
||||
}
|
||||
|
||||
// IsEnabled returns the enabled status for the config
|
||||
func (b *BaseConfig) IsEnabled() bool {
|
||||
return b.Enabled
|
||||
}
|
||||
|
||||
// DoRunAtStart returns whether the task should be run at the start
|
||||
func (b *BaseConfig) DoRunAtStart() bool {
|
||||
return b.RunAtStart
|
||||
}
|
||||
|
||||
// DoNoticeOnSuccess returns whether a success notice should be posted
|
||||
func (b *BaseConfig) DoNoticeOnSuccess() bool {
|
||||
return b.NoticeOnSuccess
|
||||
}
|
||||
|
||||
// FormatMessage returns a message for the task
|
||||
// Please note the `status` string will be concatenated with `admin.dashboard.cron.` and `admin.dashboard.task.` to provide locale messages. Similarly `name` will be composed with `admin.dashboard.` to provide the locale name for the task.
|
||||
func (b *BaseConfig) FormatMessage(locale translation.Locale, name, status, doer string, args ...any) string {
|
||||
realArgs := make([]any, 0, len(args)+2)
|
||||
realArgs = append(realArgs, locale.TrString("admin.dashboard."+name))
|
||||
if doer == "" {
|
||||
realArgs = append(realArgs, "(Cron)")
|
||||
} else {
|
||||
realArgs = append(realArgs, doer)
|
||||
}
|
||||
if len(args) > 0 {
|
||||
realArgs = append(realArgs, args...)
|
||||
}
|
||||
if doer == "" {
|
||||
return locale.TrString("admin.dashboard.cron."+status, realArgs...)
|
||||
}
|
||||
return locale.TrString("admin.dashboard.task."+status, realArgs...)
|
||||
}
|
230
services/cron/tasks.go
Normal file
230
services/cron/tasks.go
Normal file
@@ -0,0 +1,230 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cron
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
system_model "code.gitea.io/gitea/models/system"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/process"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/translation"
|
||||
)
|
||||
|
||||
var (
|
||||
lock = sync.Mutex{}
|
||||
started = false
|
||||
tasks = []*Task{}
|
||||
tasksMap = map[string]*Task{}
|
||||
)
|
||||
|
||||
// Task represents a Cron task
|
||||
type Task struct {
|
||||
lock sync.Mutex
|
||||
Name string
|
||||
config Config
|
||||
fun func(context.Context, *user_model.User, Config) error
|
||||
Status string
|
||||
LastMessage string
|
||||
LastDoer string
|
||||
ExecTimes int64
|
||||
// This stores the time of the last manual run of this task.
|
||||
LastRun time.Time
|
||||
}
|
||||
|
||||
// DoRunAtStart returns if this task should run at the start
|
||||
func (t *Task) DoRunAtStart() bool {
|
||||
return t.config.DoRunAtStart()
|
||||
}
|
||||
|
||||
// IsEnabled returns if this task is enabled as cron task
|
||||
func (t *Task) IsEnabled() bool {
|
||||
return t.config.IsEnabled()
|
||||
}
|
||||
|
||||
// GetConfig will return a copy of the task's config
|
||||
func (t *Task) GetConfig() Config {
|
||||
if reflect.TypeOf(t.config).Kind() == reflect.Ptr {
|
||||
// Pointer:
|
||||
return reflect.New(reflect.ValueOf(t.config).Elem().Type()).Interface().(Config)
|
||||
}
|
||||
// Not pointer:
|
||||
return reflect.New(reflect.TypeOf(t.config)).Elem().Interface().(Config)
|
||||
}
|
||||
|
||||
// Run will run the task incrementing the cron counter with no user defined
|
||||
func (t *Task) Run() {
|
||||
t.RunWithUser(&user_model.User{
|
||||
ID: -1,
|
||||
Name: "(Cron)",
|
||||
LowerName: "(cron)",
|
||||
}, t.config)
|
||||
}
|
||||
|
||||
// RunWithUser will run the task incrementing the cron counter at the time with User
|
||||
func (t *Task) RunWithUser(doer *user_model.User, config Config) {
|
||||
if !taskStatusTable.StartIfNotRunning(t.Name) {
|
||||
return
|
||||
}
|
||||
t.lock.Lock()
|
||||
if config == nil {
|
||||
config = t.config
|
||||
}
|
||||
t.ExecTimes++
|
||||
t.lock.Unlock()
|
||||
defer func() {
|
||||
taskStatusTable.Stop(t.Name)
|
||||
}()
|
||||
graceful.GetManager().RunWithShutdownContext(func(baseCtx context.Context) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
// Recover a panic within the execution of the task.
|
||||
combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2))
|
||||
log.Error("PANIC whilst running task: %s Value: %v", t.Name, combinedErr)
|
||||
}
|
||||
}()
|
||||
// Store the time of this run, before the function is executed, so it
|
||||
// matches the behavior of what the cron library does.
|
||||
t.lock.Lock()
|
||||
t.LastRun = time.Now()
|
||||
t.lock.Unlock()
|
||||
|
||||
pm := process.GetManager()
|
||||
doerName := ""
|
||||
if doer != nil && doer.ID != -1 {
|
||||
doerName = doer.Name
|
||||
}
|
||||
|
||||
ctx, _, finished := pm.AddContext(baseCtx, config.FormatMessage(translation.NewLocale("en-US"), t.Name, "process", doerName))
|
||||
defer finished()
|
||||
|
||||
if err := t.fun(ctx, doer, config); err != nil {
|
||||
var message string
|
||||
var status string
|
||||
if db.IsErrCancelled(err) {
|
||||
status = "cancelled"
|
||||
message = err.(db.ErrCancelled).Message
|
||||
} else {
|
||||
status = "error"
|
||||
message = err.Error()
|
||||
}
|
||||
|
||||
t.lock.Lock()
|
||||
t.LastMessage = message
|
||||
t.Status = status
|
||||
t.LastDoer = doerName
|
||||
t.lock.Unlock()
|
||||
|
||||
if err := system_model.CreateNotice(ctx, system_model.NoticeTask, config.FormatMessage(translation.NewLocale("en-US"), t.Name, "cancelled", doerName, message)); err != nil {
|
||||
log.Error("CreateNotice: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
t.lock.Lock()
|
||||
t.Status = "finished"
|
||||
t.LastMessage = ""
|
||||
t.LastDoer = doerName
|
||||
t.lock.Unlock()
|
||||
|
||||
if config.DoNoticeOnSuccess() {
|
||||
if err := system_model.CreateNotice(ctx, system_model.NoticeTask, config.FormatMessage(translation.NewLocale("en-US"), t.Name, "finished", doerName)); err != nil {
|
||||
log.Error("CreateNotice: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// GetTask gets the named task
|
||||
func GetTask(name string) *Task {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
log.Info("Getting %s in %v", name, tasksMap[name])
|
||||
|
||||
return tasksMap[name]
|
||||
}
|
||||
|
||||
// RegisterTask allows a task to be registered with the cron service
|
||||
func RegisterTask(name string, config Config, fun func(context.Context, *user_model.User, Config) error) error {
|
||||
log.Debug("Registering task: %s", name)
|
||||
|
||||
i18nKey := "admin.dashboard." + name
|
||||
if value := translation.NewLocale("en-US").TrString(i18nKey); value == i18nKey {
|
||||
return fmt.Errorf("translation is missing for task %q, please add translation for %q", name, i18nKey)
|
||||
}
|
||||
|
||||
_, err := setting.GetCronSettings(name, config)
|
||||
if err != nil {
|
||||
log.Error("Unable to register cron task with name: %s Error: %v", name, err)
|
||||
return err
|
||||
}
|
||||
|
||||
task := &Task{
|
||||
Name: name,
|
||||
config: config,
|
||||
fun: fun,
|
||||
}
|
||||
lock.Lock()
|
||||
locked := true
|
||||
defer func() {
|
||||
if locked {
|
||||
lock.Unlock()
|
||||
}
|
||||
}()
|
||||
if _, has := tasksMap[task.Name]; has {
|
||||
log.Error("A task with this name: %s has already been registered", name)
|
||||
return fmt.Errorf("duplicate task with name: %s", task.Name)
|
||||
}
|
||||
|
||||
if config.IsEnabled() {
|
||||
// We cannot use the entry return as there is no way to lock it
|
||||
if err := addTaskToScheduler(task); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
tasks = append(tasks, task)
|
||||
tasksMap[task.Name] = task
|
||||
if started && config.IsEnabled() && config.DoRunAtStart() {
|
||||
lock.Unlock()
|
||||
locked = false
|
||||
task.Run()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterTaskFatal will register a task but if there is an error log.Fatal
|
||||
func RegisterTaskFatal(name string, config Config, fun func(context.Context, *user_model.User, Config) error) {
|
||||
if err := RegisterTask(name, config, fun); err != nil {
|
||||
log.Fatal("Unable to register cron task %s Error: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func addTaskToScheduler(task *Task) error {
|
||||
tags := []string{task.Name, task.config.GetSchedule()} // name and schedule can't be get from job, so we add them as tag
|
||||
if scheduleHasSeconds(task.config.GetSchedule()) {
|
||||
scheduler = scheduler.CronWithSeconds(task.config.GetSchedule())
|
||||
} else {
|
||||
scheduler = scheduler.Cron(task.config.GetSchedule())
|
||||
}
|
||||
if _, err := scheduler.Tag(tags...).Do(task.Run); err != nil {
|
||||
log.Error("Unable to register cron task with name: %s Error: %v", task.Name, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scheduleHasSeconds(schedule string) bool {
|
||||
return len(strings.Fields(schedule)) >= 6
|
||||
}
|
76
services/cron/tasks_actions.go
Normal file
76
services/cron/tasks_actions.go
Normal file
@@ -0,0 +1,76 @@
|
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cron
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
actions_service "code.gitea.io/gitea/services/actions"
|
||||
)
|
||||
|
||||
func initActionsTasks() {
|
||||
if !setting.Actions.Enabled {
|
||||
return
|
||||
}
|
||||
registerStopZombieTasks()
|
||||
registerStopEndlessTasks()
|
||||
registerCancelAbandonedJobs()
|
||||
registerScheduleTasks()
|
||||
registerActionsCleanup()
|
||||
}
|
||||
|
||||
func registerStopZombieTasks() {
|
||||
RegisterTaskFatal("stop_zombie_tasks", &BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: true,
|
||||
Schedule: "@every 5m",
|
||||
}, func(ctx context.Context, _ *user_model.User, cfg Config) error {
|
||||
return actions_service.StopZombieTasks(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerStopEndlessTasks() {
|
||||
RegisterTaskFatal("stop_endless_tasks", &BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: true,
|
||||
Schedule: "@every 30m",
|
||||
}, func(ctx context.Context, _ *user_model.User, cfg Config) error {
|
||||
return actions_service.StopEndlessTasks(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerCancelAbandonedJobs() {
|
||||
RegisterTaskFatal("cancel_abandoned_jobs", &BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: true,
|
||||
Schedule: "@every 6h",
|
||||
}, func(ctx context.Context, _ *user_model.User, cfg Config) error {
|
||||
return actions_service.CancelAbandonedJobs(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// registerScheduleTasks registers a scheduled task that runs every minute to start any due schedule tasks.
|
||||
func registerScheduleTasks() {
|
||||
// Register the task with a unique name, enabled status, and schedule for every minute.
|
||||
RegisterTaskFatal("start_schedule_tasks", &BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 1m",
|
||||
}, func(ctx context.Context, _ *user_model.User, cfg Config) error {
|
||||
// Call the function to start schedule tasks and pass the context.
|
||||
return actions_service.StartScheduleTasks(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerActionsCleanup() {
|
||||
RegisterTaskFatal("cleanup_actions", &BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: false,
|
||||
Schedule: "@midnight",
|
||||
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
||||
return actions_service.Cleanup(ctx)
|
||||
})
|
||||
}
|
186
services/cron/tasks_basic.go
Normal file
186
services/cron/tasks_basic.go
Normal file
@@ -0,0 +1,186 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cron
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
git_model "code.gitea.io/gitea/models/git"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/models/webhook"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/services/auth"
|
||||
"code.gitea.io/gitea/services/migrations"
|
||||
mirror_service "code.gitea.io/gitea/services/mirror"
|
||||
packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup"
|
||||
repo_service "code.gitea.io/gitea/services/repository"
|
||||
archiver_service "code.gitea.io/gitea/services/repository/archiver"
|
||||
)
|
||||
|
||||
func registerUpdateMirrorTask() {
|
||||
type UpdateMirrorTaskConfig struct {
|
||||
BaseConfig
|
||||
PullLimit int
|
||||
PushLimit int
|
||||
}
|
||||
|
||||
RegisterTaskFatal("update_mirrors", &UpdateMirrorTaskConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 10m",
|
||||
},
|
||||
PullLimit: 50,
|
||||
PushLimit: 50,
|
||||
}, func(ctx context.Context, _ *user_model.User, cfg Config) error {
|
||||
umtc := cfg.(*UpdateMirrorTaskConfig)
|
||||
return mirror_service.Update(ctx, umtc.PullLimit, umtc.PushLimit)
|
||||
})
|
||||
}
|
||||
|
||||
func registerRepoHealthCheck() {
|
||||
type RepoHealthCheckConfig struct {
|
||||
BaseConfig
|
||||
Timeout time.Duration
|
||||
Args []string `delim:" "`
|
||||
}
|
||||
RegisterTaskFatal("repo_health_check", &RepoHealthCheckConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: false,
|
||||
Schedule: "@midnight",
|
||||
},
|
||||
Timeout: time.Duration(setting.Git.Timeout.Default) * time.Second,
|
||||
Args: []string{},
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
rhcConfig := config.(*RepoHealthCheckConfig)
|
||||
// the git args are set by config, they can be safe to be trusted
|
||||
return repo_service.GitFsckRepos(ctx, rhcConfig.Timeout, git.ToTrustedCmdArgs(rhcConfig.Args))
|
||||
})
|
||||
}
|
||||
|
||||
func registerCheckRepoStats() {
|
||||
RegisterTaskFatal("check_repo_stats", &BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: true,
|
||||
Schedule: "@midnight",
|
||||
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
||||
return models.CheckRepoStats(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerArchiveCleanup() {
|
||||
RegisterTaskFatal("archive_cleanup", &OlderThanConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: true,
|
||||
Schedule: "@midnight",
|
||||
},
|
||||
OlderThan: 24 * time.Hour,
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
acConfig := config.(*OlderThanConfig)
|
||||
return archiver_service.DeleteOldRepositoryArchives(ctx, acConfig.OlderThan)
|
||||
})
|
||||
}
|
||||
|
||||
func registerSyncExternalUsers() {
|
||||
RegisterTaskFatal("sync_external_users", &UpdateExistingConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: false,
|
||||
Schedule: "@midnight",
|
||||
},
|
||||
UpdateExisting: true,
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
realConfig := config.(*UpdateExistingConfig)
|
||||
return auth.SyncExternalUsers(ctx, realConfig.UpdateExisting)
|
||||
})
|
||||
}
|
||||
|
||||
func registerDeletedBranchesCleanup() {
|
||||
RegisterTaskFatal("deleted_branches_cleanup", &OlderThanConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: true,
|
||||
Schedule: "@midnight",
|
||||
},
|
||||
OlderThan: 24 * time.Hour,
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
realConfig := config.(*OlderThanConfig)
|
||||
git_model.RemoveOldDeletedBranches(ctx, realConfig.OlderThan)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func registerUpdateMigrationPosterID() {
|
||||
RegisterTaskFatal("update_migration_poster_id", &BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: true,
|
||||
Schedule: "@midnight",
|
||||
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
||||
return migrations.UpdateMigrationPosterID(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerCleanupHookTaskTable() {
|
||||
RegisterTaskFatal("cleanup_hook_task_table", &CleanupHookTaskConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: false,
|
||||
Schedule: "@midnight",
|
||||
},
|
||||
CleanupType: "OlderThan",
|
||||
OlderThan: 168 * time.Hour,
|
||||
NumberToKeep: 10,
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
realConfig := config.(*CleanupHookTaskConfig)
|
||||
return webhook.CleanupHookTaskTable(ctx, webhook.ToHookTaskCleanupType(realConfig.CleanupType), realConfig.OlderThan, realConfig.NumberToKeep)
|
||||
})
|
||||
}
|
||||
|
||||
func registerCleanupPackages() {
|
||||
RegisterTaskFatal("cleanup_packages", &OlderThanConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: true,
|
||||
Schedule: "@midnight",
|
||||
},
|
||||
OlderThan: 24 * time.Hour,
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
realConfig := config.(*OlderThanConfig)
|
||||
return packages_cleanup_service.CleanupTask(ctx, realConfig.OlderThan)
|
||||
})
|
||||
}
|
||||
|
||||
func registerSyncRepoLicenses() {
|
||||
RegisterTaskFatal("sync_repo_licenses", &BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@annually",
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
return repo_service.SyncRepoLicenses(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func initBasicTasks() {
|
||||
if setting.Mirror.Enabled {
|
||||
registerUpdateMirrorTask()
|
||||
}
|
||||
registerRepoHealthCheck()
|
||||
registerCheckRepoStats()
|
||||
registerArchiveCleanup()
|
||||
registerSyncExternalUsers()
|
||||
registerDeletedBranchesCleanup()
|
||||
if !setting.Repository.DisableMigrations {
|
||||
registerUpdateMigrationPosterID()
|
||||
}
|
||||
registerCleanupHookTaskTable()
|
||||
if setting.Packages.Enabled {
|
||||
registerCleanupPackages()
|
||||
}
|
||||
registerSyncRepoLicenses()
|
||||
}
|
241
services/cron/tasks_extended.go
Normal file
241
services/cron/tasks_extended.go
Normal file
@@ -0,0 +1,241 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cron
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
activities_model "code.gitea.io/gitea/models/activities"
|
||||
"code.gitea.io/gitea/models/system"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/updatechecker"
|
||||
asymkey_service "code.gitea.io/gitea/services/asymkey"
|
||||
repo_service "code.gitea.io/gitea/services/repository"
|
||||
archiver_service "code.gitea.io/gitea/services/repository/archiver"
|
||||
user_service "code.gitea.io/gitea/services/user"
|
||||
)
|
||||
|
||||
func registerDeleteInactiveUsers() {
|
||||
RegisterTaskFatal("delete_inactive_accounts", &OlderThanConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@annually",
|
||||
},
|
||||
OlderThan: time.Minute * time.Duration(setting.Service.ActiveCodeLives),
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
olderThanConfig := config.(*OlderThanConfig)
|
||||
return user_service.DeleteInactiveUsers(ctx, olderThanConfig.OlderThan)
|
||||
})
|
||||
}
|
||||
|
||||
func registerDeleteRepositoryArchives() {
|
||||
RegisterTaskFatal("delete_repo_archives", &BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@annually",
|
||||
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
||||
return archiver_service.DeleteRepositoryArchives(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerGarbageCollectRepositories() {
|
||||
type RepoHealthCheckConfig struct {
|
||||
BaseConfig
|
||||
Timeout time.Duration
|
||||
Args []string `delim:" "`
|
||||
}
|
||||
RegisterTaskFatal("git_gc_repos", &RepoHealthCheckConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 72h",
|
||||
},
|
||||
Timeout: time.Duration(setting.Git.Timeout.GC) * time.Second,
|
||||
Args: setting.Git.GCArgs,
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
rhcConfig := config.(*RepoHealthCheckConfig)
|
||||
// the git args are set by config, they can be safe to be trusted
|
||||
return repo_service.GitGcRepos(ctx, rhcConfig.Timeout, git.ToTrustedCmdArgs(rhcConfig.Args))
|
||||
})
|
||||
}
|
||||
|
||||
func registerRewriteAllPublicKeys() {
|
||||
RegisterTaskFatal("resync_all_sshkeys", &BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 72h",
|
||||
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
||||
return asymkey_service.RewriteAllPublicKeys(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerRewriteAllPrincipalKeys() {
|
||||
RegisterTaskFatal("resync_all_sshprincipals", &BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 72h",
|
||||
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
||||
return asymkey_service.RewriteAllPrincipalKeys(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerRepositoryUpdateHook() {
|
||||
RegisterTaskFatal("resync_all_hooks", &BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 72h",
|
||||
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
||||
return repo_service.SyncRepositoryHooks(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerReinitMissingRepositories() {
|
||||
RegisterTaskFatal("reinit_missing_repos", &BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 72h",
|
||||
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
||||
return repo_service.ReinitMissingRepositories(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerDeleteMissingRepositories() {
|
||||
RegisterTaskFatal("delete_missing_repos", &BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 72h",
|
||||
}, func(ctx context.Context, user *user_model.User, _ Config) error {
|
||||
return repo_service.DeleteMissingRepositories(ctx, user)
|
||||
})
|
||||
}
|
||||
|
||||
func registerRemoveRandomAvatars() {
|
||||
RegisterTaskFatal("delete_generated_repository_avatars", &BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 72h",
|
||||
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
||||
return repo_service.RemoveRandomAvatars(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerDeleteOldActions() {
|
||||
RegisterTaskFatal("delete_old_actions", &OlderThanConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 168h",
|
||||
},
|
||||
OlderThan: 365 * 24 * time.Hour,
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
olderThanConfig := config.(*OlderThanConfig)
|
||||
return activities_model.DeleteOldActions(ctx, olderThanConfig.OlderThan)
|
||||
})
|
||||
}
|
||||
|
||||
func registerUpdateGiteaChecker() {
|
||||
type UpdateCheckerConfig struct {
|
||||
BaseConfig
|
||||
HTTPEndpoint string
|
||||
}
|
||||
RegisterTaskFatal("update_checker", &UpdateCheckerConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 168h",
|
||||
},
|
||||
HTTPEndpoint: "https://dl.gitea.com/gitea/version.json",
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
updateCheckerConfig := config.(*UpdateCheckerConfig)
|
||||
return updatechecker.GiteaUpdateChecker(updateCheckerConfig.HTTPEndpoint)
|
||||
})
|
||||
}
|
||||
|
||||
func registerDeleteOldSystemNotices() {
|
||||
RegisterTaskFatal("delete_old_system_notices", &OlderThanConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 168h",
|
||||
},
|
||||
OlderThan: 365 * 24 * time.Hour,
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
olderThanConfig := config.(*OlderThanConfig)
|
||||
return system.DeleteOldSystemNotices(ctx, olderThanConfig.OlderThan)
|
||||
})
|
||||
}
|
||||
|
||||
func registerGCLFS() {
|
||||
if !setting.LFS.StartServer {
|
||||
return
|
||||
}
|
||||
type GCLFSConfig struct {
|
||||
OlderThanConfig
|
||||
LastUpdatedMoreThanAgo time.Duration
|
||||
NumberToCheckPerRepo int64
|
||||
ProportionToCheckPerRepo float64
|
||||
}
|
||||
|
||||
RegisterTaskFatal("gc_lfs", &GCLFSConfig{
|
||||
OlderThanConfig: OlderThanConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 24h",
|
||||
},
|
||||
// Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload
|
||||
// and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby
|
||||
// an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid
|
||||
// changes in new branches that might lead to lfs objects becoming temporarily unassociated with git
|
||||
// objects.
|
||||
//
|
||||
// It is likely that a week is potentially excessive but it should definitely be enough that any
|
||||
// unassociated LFS object is genuinely unassociated.
|
||||
OlderThan: 24 * time.Hour * 7,
|
||||
},
|
||||
// Only GC things that haven't been looked at in the past 3 days
|
||||
LastUpdatedMoreThanAgo: 24 * time.Hour * 3,
|
||||
NumberToCheckPerRepo: 100,
|
||||
ProportionToCheckPerRepo: 0.6,
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
gcLFSConfig := config.(*GCLFSConfig)
|
||||
return repo_service.GarbageCollectLFSMetaObjects(ctx, repo_service.GarbageCollectLFSMetaObjectsOptions{
|
||||
AutoFix: true,
|
||||
OlderThan: time.Now().Add(-gcLFSConfig.OlderThan),
|
||||
UpdatedLessRecentlyThan: time.Now().Add(-gcLFSConfig.LastUpdatedMoreThanAgo),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func registerRebuildIssueIndexer() {
|
||||
RegisterTaskFatal("rebuild_issue_indexer", &BaseConfig{
|
||||
Enabled: false,
|
||||
RunAtStart: false,
|
||||
Schedule: "@annually",
|
||||
}, func(ctx context.Context, _ *user_model.User, config Config) error {
|
||||
return issue_indexer.PopulateIssueIndexer(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func initExtendedTasks() {
|
||||
registerDeleteInactiveUsers()
|
||||
registerDeleteRepositoryArchives()
|
||||
registerGarbageCollectRepositories()
|
||||
registerRewriteAllPublicKeys()
|
||||
registerRewriteAllPrincipalKeys()
|
||||
registerRepositoryUpdateHook()
|
||||
registerReinitMissingRepositories()
|
||||
registerDeleteMissingRepositories()
|
||||
registerRemoveRandomAvatars()
|
||||
registerDeleteOldActions()
|
||||
registerUpdateGiteaChecker()
|
||||
registerDeleteOldSystemNotices()
|
||||
registerGCLFS()
|
||||
registerRebuildIssueIndexer()
|
||||
}
|
67
services/cron/tasks_test.go
Normal file
67
services/cron/tasks_test.go
Normal file
@@ -0,0 +1,67 @@
|
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cron
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestAddTaskToScheduler(t *testing.T) {
|
||||
assert.Empty(t, scheduler.Jobs())
|
||||
defer scheduler.Clear()
|
||||
|
||||
// no seconds
|
||||
err := addTaskToScheduler(&Task{
|
||||
Name: "task 1",
|
||||
config: &BaseConfig{
|
||||
Schedule: "5 4 * * *",
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
jobs := scheduler.Jobs()
|
||||
assert.Len(t, jobs, 1)
|
||||
assert.Equal(t, "task 1", jobs[0].Tags()[0])
|
||||
assert.Equal(t, "5 4 * * *", jobs[0].Tags()[1])
|
||||
|
||||
// with seconds
|
||||
err = addTaskToScheduler(&Task{
|
||||
Name: "task 2",
|
||||
config: &BaseConfig{
|
||||
Schedule: "30 5 4 * * *",
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
jobs = scheduler.Jobs() // the item order is not guaranteed, so we need to sort it before "assert"
|
||||
sort.Slice(jobs, func(i, j int) bool {
|
||||
return jobs[i].Tags()[0] < jobs[j].Tags()[0]
|
||||
})
|
||||
assert.Len(t, jobs, 2)
|
||||
assert.Equal(t, "task 2", jobs[1].Tags()[0])
|
||||
assert.Equal(t, "30 5 4 * * *", jobs[1].Tags()[1])
|
||||
}
|
||||
|
||||
func TestScheduleHasSeconds(t *testing.T) {
|
||||
tests := []struct {
|
||||
schedule string
|
||||
hasSecond bool
|
||||
}{
|
||||
{"* * * * * *", true},
|
||||
{"* * * * *", false},
|
||||
{"5 4 * * *", false},
|
||||
{"5 4 * * *", false},
|
||||
{"5,8 4 * * *", false},
|
||||
{"* * * * * *", true},
|
||||
{"5,8 4 * * *", false},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
t.Run(strconv.Itoa(i), func(t *testing.T) {
|
||||
assert.Equal(t, test.hasSecond, scheduleHasSeconds(test.schedule))
|
||||
})
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user