mirror of
https://github.com/go-gitea/gitea
synced 2024-09-11 00:47:00 +02:00
Calculate MAX_WORKERS default value by CPU number (#26177)
To avoid consuming user's 100% CPU, limit the default value of MAX_WORKERS Fix #26063 (the CPU 100% problem mentioned in it)
This commit is contained in:
parent
73fb1ecdcf
commit
8baa42c8d7
@ -1420,8 +1420,8 @@ LEVEL = Info
|
|||||||
;; Provides the suffix of the default redis/disk unique queue set name - specific queues can be overridden within in their [queue.name] sections.
|
;; Provides the suffix of the default redis/disk unique queue set name - specific queues can be overridden within in their [queue.name] sections.
|
||||||
;SET_NAME = "_unique"
|
;SET_NAME = "_unique"
|
||||||
;;
|
;;
|
||||||
;; Dynamically scale the worker pool to at this many workers
|
;; Maximum number of worker go-routines for the queue. Default value is "CpuNum/2" clipped to between 1 and 10.
|
||||||
;MAX_WORKERS = 10
|
;MAX_WORKERS = ; (dynamic)
|
||||||
|
|
||||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
@ -488,7 +488,7 @@ Configuration at `[queue]` will set defaults for queues with overrides for indiv
|
|||||||
- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. For `redis-cluster` use `redis+cluster://127.0.0.1:6379/0`. Options can be set using query params. Similarly, LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR`
|
- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. For `redis-cluster` use `redis+cluster://127.0.0.1:6379/0`. Options can be set using query params. Similarly, LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR`
|
||||||
- `QUEUE_NAME`: **_queue**: The suffix for default redis and disk queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overridden in the specific `queue.name` section.
|
- `QUEUE_NAME`: **_queue**: The suffix for default redis and disk queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overridden in the specific `queue.name` section.
|
||||||
- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section.
|
- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section.
|
||||||
- `MAX_WORKERS`: **10**: Maximum number of worker go-routines for the queue.
|
- `MAX_WORKERS`: **(dynamic)**: Maximum number of worker go-routines for the queue. Default value is "CpuNum/2" clipped to between 1 and 10.
|
||||||
|
|
||||||
Gitea creates the following non-unique queues:
|
Gitea creates the following non-unique queues:
|
||||||
|
|
||||||
|
@ -51,7 +51,7 @@ CONN_STR = redis://
|
|||||||
assert.Equal(t, "", q.baseConfig.ConnStr)
|
assert.Equal(t, "", q.baseConfig.ConnStr)
|
||||||
assert.Equal(t, "default_queue", q.baseConfig.QueueFullName)
|
assert.Equal(t, "default_queue", q.baseConfig.QueueFullName)
|
||||||
assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName)
|
assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName)
|
||||||
assert.Equal(t, 10, q.GetWorkerMaxNumber())
|
assert.NotZero(t, q.GetWorkerMaxNumber())
|
||||||
assert.Equal(t, 0, q.GetWorkerNumber())
|
assert.Equal(t, 0, q.GetWorkerNumber())
|
||||||
assert.Equal(t, 0, q.GetWorkerActiveNumber())
|
assert.Equal(t, 0, q.GetWorkerActiveNumber())
|
||||||
assert.Equal(t, 0, q.GetQueueItemNumber())
|
assert.Equal(t, 0, q.GetQueueItemNumber())
|
||||||
@ -75,7 +75,7 @@ BATCH_LENGTH = 22
|
|||||||
CONN_STR =
|
CONN_STR =
|
||||||
QUEUE_NAME = _q2
|
QUEUE_NAME = _q2
|
||||||
SET_NAME = _u2
|
SET_NAME = _u2
|
||||||
MAX_WORKERS = 2
|
MAX_WORKERS = 123
|
||||||
`)
|
`)
|
||||||
|
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
@ -89,7 +89,7 @@ MAX_WORKERS = 2
|
|||||||
assert.Equal(t, "addrs=127.0.0.1:6379 db=0", q1.baseConfig.ConnStr)
|
assert.Equal(t, "addrs=127.0.0.1:6379 db=0", q1.baseConfig.ConnStr)
|
||||||
assert.Equal(t, "no-such_queue1", q1.baseConfig.QueueFullName)
|
assert.Equal(t, "no-such_queue1", q1.baseConfig.QueueFullName)
|
||||||
assert.Equal(t, "no-such_queue1_unique", q1.baseConfig.SetFullName)
|
assert.Equal(t, "no-such_queue1_unique", q1.baseConfig.SetFullName)
|
||||||
assert.Equal(t, 10, q1.GetWorkerMaxNumber())
|
assert.NotZero(t, q1.GetWorkerMaxNumber())
|
||||||
assert.Equal(t, 0, q1.GetWorkerNumber())
|
assert.Equal(t, 0, q1.GetWorkerNumber())
|
||||||
assert.Equal(t, 0, q1.GetWorkerActiveNumber())
|
assert.Equal(t, 0, q1.GetWorkerActiveNumber())
|
||||||
assert.Equal(t, 0, q1.GetQueueItemNumber())
|
assert.Equal(t, 0, q1.GetQueueItemNumber())
|
||||||
@ -105,7 +105,7 @@ MAX_WORKERS = 2
|
|||||||
assert.Equal(t, "", q2.baseConfig.ConnStr)
|
assert.Equal(t, "", q2.baseConfig.ConnStr)
|
||||||
assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName)
|
assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName)
|
||||||
assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName)
|
assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName)
|
||||||
assert.Equal(t, 2, q2.GetWorkerMaxNumber())
|
assert.Equal(t, 123, q2.GetWorkerMaxNumber())
|
||||||
assert.Equal(t, 0, q2.GetWorkerNumber())
|
assert.Equal(t, 0, q2.GetWorkerNumber())
|
||||||
assert.Equal(t, 0, q2.GetWorkerActiveNumber())
|
assert.Equal(t, 0, q2.GetWorkerActiveNumber())
|
||||||
assert.Equal(t, 0, q2.GetQueueItemNumber())
|
assert.Equal(t, 0, q2.GetQueueItemNumber())
|
||||||
|
@ -5,6 +5,7 @@ package setting
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/json"
|
"code.gitea.io/gitea/modules/json"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
@ -25,7 +26,8 @@ type QueueSettings struct {
|
|||||||
MaxWorkers int
|
MaxWorkers int
|
||||||
}
|
}
|
||||||
|
|
||||||
var queueSettingsDefault = QueueSettings{
|
func GetQueueSettings(rootCfg ConfigProvider, name string) (QueueSettings, error) {
|
||||||
|
queueSettingsDefault := QueueSettings{
|
||||||
Type: "level", // dummy, channel, level, redis
|
Type: "level", // dummy, channel, level, redis
|
||||||
Datadir: "queues/common", // relative to AppDataPath
|
Datadir: "queues/common", // relative to AppDataPath
|
||||||
Length: 100, // queue length before a channel queue will block
|
Length: 100, // queue length before a channel queue will block
|
||||||
@ -33,10 +35,15 @@ var queueSettingsDefault = QueueSettings{
|
|||||||
QueueName: "_queue",
|
QueueName: "_queue",
|
||||||
SetName: "_unique",
|
SetName: "_unique",
|
||||||
BatchLength: 20,
|
BatchLength: 20,
|
||||||
MaxWorkers: 10,
|
MaxWorkers: runtime.NumCPU() / 2,
|
||||||
}
|
}
|
||||||
|
if queueSettingsDefault.MaxWorkers < 1 {
|
||||||
|
queueSettingsDefault.MaxWorkers = 1
|
||||||
|
}
|
||||||
|
if queueSettingsDefault.MaxWorkers > 10 {
|
||||||
|
queueSettingsDefault.MaxWorkers = 10
|
||||||
|
}
|
||||||
|
|
||||||
func GetQueueSettings(rootCfg ConfigProvider, name string) (QueueSettings, error) {
|
|
||||||
// deep copy default settings
|
// deep copy default settings
|
||||||
cfg := QueueSettings{}
|
cfg := QueueSettings{}
|
||||||
if cfgBs, err := json.Marshal(queueSettingsDefault); err != nil {
|
if cfgBs, err := json.Marshal(queueSettingsDefault); err != nil {
|
||||||
|
Loading…
Reference in New Issue
Block a user