Make db connect more robust (#5738)

custom/conf/app.ini.sample

@@ -242,6 +242,11 @@ SQLITE_TIMEOUT = 500
 ITERATE_BUFFER_SIZE = 50
 ; Show the database generated SQL
 LOG_SQL = true
+; Maximum number of DB Connect retries
+DB_RETRIES = 10
+; Backoff time per DB retry (time.Duration)
+DB_RETRY_BACKOFF = 3s
+
 
 [indexer]
 ISSUE_INDEXER_PATH = indexers/issues.bleve

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -145,6 +145,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `SSL_MODE`: **disable**: For PostgreSQL and MySQL only.
 - `PATH`: **data/gitea.db**: For SQLite3 only, the database file path.
 - `LOG_SQL`: **true**: Log the executed SQL.
+- `DB_RETRIES`: **10**: How many ORM init / DB connect attempts allowed.
+- `DB_RETRY_BACKOFF`: **3s*: time.Duration to wait before trying another ORM init / DB connect attempt, if failure occured.
 
 ## Indexer (`indexer`)
 

modules/setting/setting.go

@@ -169,12 +169,14 @@ var (
 	DisableGitHooks       bool
 
 	// Database settings
-	UseSQLite3    bool
+	UseSQLite3       bool
-	UseMySQL      bool
+	UseMySQL         bool
-	UseMSSQL      bool
+	UseMSSQL         bool
-	UsePostgreSQL bool
+	UsePostgreSQL    bool
-	UseTiDB       bool
+	UseTiDB          bool
-	LogSQL        bool
+	LogSQL           bool
+	DBConnectRetries int
+	DBConnectBackoff time.Duration
 
 	// Indexer settings
 	Indexer struct {
@@ -986,6 +988,8 @@ func NewContext() {
 	}
 	IterateBufferSize = Cfg.Section("database").Key("ITERATE_BUFFER_SIZE").MustInt(50)
 	LogSQL = Cfg.Section("database").Key("LOG_SQL").MustBool(true)
+	DBConnectRetries = Cfg.Section("database").Key("DB_RETRIES").MustInt(10)
+	DBConnectBackoff = Cfg.Section("database").Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second)
 
 	sec = Cfg.Section("attachment")
 	AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))

routers/init.go

@@ -7,6 +7,7 @@ package routers
 import (
 	"path"
 	"strings"
+	"time"
 
 	"code.gitea.io/git"
 	"code.gitea.io/gitea/models"
@@ -42,6 +43,24 @@ func NewServices() {
 	cache.NewContext()
 }
 
+// In case of problems connecting to DB, retry connection. Eg, PGSQL in Docker Container on Synology
+func initDBEngine() (err error) {
+	log.Info("Beginning ORM engine initialization.")
+	for i := 0; i < setting.DBConnectRetries; i++ {
+		log.Info("ORM engine initialization attempt #%d/%d...", i+1, setting.DBConnectRetries)
+		if err := models.NewEngine(migrations.Migrate); err == nil {
+			break
+		} else if i == setting.DBConnectRetries-1 {
+			return err
+		}
+		log.Debug("ORM engine initialization attempt #%d/%d failed. Error: %v", i+1, setting.DBConnectRetries, err)
+		log.Info("Backing off for %d seconds", int64(setting.DBConnectBackoff/time.Second))
+		time.Sleep(setting.DBConnectBackoff)
+	}
+	models.HasEngine = true
+	return nil
+}
+
 // GlobalInit is for global configuration reload-able.
 func GlobalInit() {
 	setting.NewContext()
@@ -56,11 +75,12 @@ func GlobalInit() {
 	if setting.InstallLock {
 		highlight.NewContext()
 		markup.Init()
-
+		if err := initDBEngine(); err == nil {
-		if err := models.NewEngine(migrations.Migrate); err != nil {
+			log.Info("ORM engine initialization successful!")
-			log.Fatal(4, "Failed to initialize ORM engine: %v", err)
+		} else {
+			log.Fatal(4, "ORM engine initialization failed: %v", err)
 		}
-		models.HasEngine = true
+
 		if err := models.InitOAuth2(); err != nil {
 			log.Fatal(4, "Failed to initialize OAuth2 support: %v", err)
 		}