mirror of
https://github.com/go-gitea/gitea
synced 2024-09-11 08:56:59 +02:00
Add PULL_LIMIT
and PUSH_LIMIT
to cron.update_mirror task (#17568)
This commit is contained in:
parent
9450410ff7
commit
188fd2dd1a
@ -1719,6 +1719,12 @@ PATH =
|
||||
;RUN_AT_START = false
|
||||
;; Notice if not success
|
||||
;NO_SUCCESS_NOTICE = true
|
||||
;; Limit the number of mirrors added to the queue to this number
|
||||
;; (negative values mean no limit, 0 will result in no result in no mirrors being queued effectively disabling pull mirror updating.)
|
||||
;PULL_LIMIT=50
|
||||
;; Limit the number of mirrors added to the queue to this number
|
||||
;; (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating)
|
||||
;PUSH_LIMIT=50
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
@ -828,6 +828,8 @@ NB: You must have `DISABLE_ROUTER_LOG` set to `false` for this option to take ef
|
||||
|
||||
- `SCHEDULE`: **@every 10m**: Cron syntax for scheduling update mirrors, e.g. `@every 3h`.
|
||||
- `NO_SUCCESS_NOTICE`: **true**: The cron task for update mirrors success report is not very useful - as it just means that the mirrors have been queued. Therefore this is turned off by default.
|
||||
- `PULL_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling pull mirror updating).
|
||||
- `PUSH_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating).
|
||||
|
||||
#### Cron - Repository Health Check (`cron.repo_health_check`)
|
||||
|
||||
|
@ -119,6 +119,7 @@ func MirrorsIterate(f func(idx int, bean interface{}) error) error {
|
||||
return db.GetEngine(db.DefaultContext).
|
||||
Where("next_update_unix<=?", time.Now().Unix()).
|
||||
And("next_update_unix!=0").
|
||||
OrderBy("updated_unix ASC").
|
||||
Iterate(new(Mirror), f)
|
||||
}
|
||||
|
||||
|
@ -107,5 +107,6 @@ func PushMirrorsIterate(f func(idx int, bean interface{}) error) error {
|
||||
return db.GetEngine(db.DefaultContext).
|
||||
Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
|
||||
And("`interval` != 0").
|
||||
OrderBy("last_update ASC").
|
||||
Iterate(new(PushMirror), f)
|
||||
}
|
||||
|
@ -18,13 +18,24 @@ import (
|
||||
)
|
||||
|
||||
func registerUpdateMirrorTask() {
|
||||
RegisterTaskFatal("update_mirrors", &BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 10m",
|
||||
NoSuccessNotice: true,
|
||||
}, func(ctx context.Context, _ *models.User, _ Config) error {
|
||||
return mirror_service.Update(ctx)
|
||||
type UpdateMirrorTaskConfig struct {
|
||||
BaseConfig
|
||||
PullLimit int
|
||||
PushLimit int
|
||||
}
|
||||
|
||||
RegisterTaskFatal("update_mirrors", &UpdateMirrorTaskConfig{
|
||||
BaseConfig: BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: false,
|
||||
Schedule: "@every 10m",
|
||||
NoSuccessNotice: true,
|
||||
},
|
||||
PullLimit: 50,
|
||||
PushLimit: 50,
|
||||
}, func(ctx context.Context, _ *models.User, cfg Config) error {
|
||||
umtc := cfg.(*UpdateMirrorTaskConfig)
|
||||
return mirror_service.Update(ctx, umtc.PullLimit, umtc.PushLimit)
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -45,15 +45,19 @@ func doMirrorSync(ctx context.Context, req *SyncRequest) {
|
||||
}
|
||||
}
|
||||
|
||||
var errLimit = fmt.Errorf("reached limit")
|
||||
|
||||
// Update checks and updates mirror repositories.
|
||||
func Update(ctx context.Context) error {
|
||||
func Update(ctx context.Context, pullLimit, pushLimit int) error {
|
||||
if !setting.Mirror.Enabled {
|
||||
log.Warn("Mirror feature disabled, but cron job enabled: skip update")
|
||||
return nil
|
||||
}
|
||||
log.Trace("Doing: Update")
|
||||
|
||||
handler := func(idx int, bean interface{}) error {
|
||||
requested := 0
|
||||
|
||||
handler := func(idx int, bean interface{}, limit int) error {
|
||||
var item SyncRequest
|
||||
if m, ok := bean.(*models.Mirror); ok {
|
||||
if m.Repo == nil {
|
||||
@ -78,21 +82,49 @@ func Update(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check we've not been cancelled
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("Aborted")
|
||||
return fmt.Errorf("aborted")
|
||||
default:
|
||||
return mirrorQueue.Push(&item)
|
||||
}
|
||||
|
||||
// Check if this request is already in the queue
|
||||
has, err := mirrorQueue.Has(&item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if has {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Push to the Queue
|
||||
if err := mirrorQueue.Push(&item); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
requested++
|
||||
if limit > 0 && requested > limit {
|
||||
return errLimit
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := models.MirrorsIterate(handler); err != nil {
|
||||
log.Error("MirrorsIterate: %v", err)
|
||||
return err
|
||||
if pullLimit != 0 {
|
||||
if err := models.MirrorsIterate(func(idx int, bean interface{}) error {
|
||||
return handler(idx, bean, pullLimit)
|
||||
}); err != nil && err != errLimit {
|
||||
log.Error("MirrorsIterate: %v", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := models.PushMirrorsIterate(handler); err != nil {
|
||||
log.Error("PushMirrorsIterate: %v", err)
|
||||
return err
|
||||
if pushLimit != 0 {
|
||||
if err := models.PushMirrorsIterate(func(idx int, bean interface{}) error {
|
||||
return handler(idx, bean, pushLimit)
|
||||
}); err != nil && err != errLimit {
|
||||
log.Error("PushMirrorsIterate: %v", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
log.Trace("Finished: Update")
|
||||
return nil
|
||||
|
Loading…
Reference in New Issue
Block a user