From 909f2be99d0c699f506f463bb6238378f12c0dc1 Mon Sep 17 00:00:00 2001 From: zeripath Date: Fri, 19 Mar 2021 03:23:58 +0000 Subject: [PATCH] Fix postgres ID sequences broken by recreate-table (#15015) (#15029) Backport #15015 Unfortunately there is a subtle problem with recreatetable on postgres which leads to the sequences not being renamed and not being left at 0. Fix #14725 Signed-off-by: Andrew Thornton --- cmd/doctor.go | 17 +++++++++ models/consistency.go | 61 +++++++++++++++++++++++++++++++ models/migrations/migrations.go | 63 ++++++++++++++++++++++++++++++++- 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index 2ca2bb5e70b..5ba04511109 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -670,6 +670,23 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { } } + if setting.Database.UsePostgreSQL { + count, err = models.CountBadSequences() + if err != nil { + return nil, err + } + if count > 0 { + if ctx.Bool("fix") { + err := models.FixBadSequences() + if err != nil { + return nil, err + } + results = append(results, fmt.Sprintf("%d sequences updated", count)) + } else { + results = append(results, fmt.Sprintf("%d sequences with incorrect values", count)) + } + } + } //ToDo: function to recalc all counters return results, nil diff --git a/models/consistency.go b/models/consistency.go index fbb99ca80c8..0c62d4dc241 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -5,10 +5,13 @@ package models import ( + "fmt" "reflect" + "regexp" "strings" "testing" + "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" "xorm.io/builder" ) @@ -295,3 +298,61 @@ func FixNullArchivedRepository() (int64, error) { IsArchived: false, }) } + +// CountBadSequences looks for broken sequences from recreate-table mistakes +func CountBadSequences() (int64, error) { + if !setting.Database.UsePostgreSQL { + return 0, nil + } + + sess := x.NewSession() + defer sess.Close() + + var sequences []string + schema := sess.Engine().Dialect().URI().Schema + + sess.Engine().SetSchema("") + if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil { + return 0, err + } + sess.Engine().SetSchema(schema) + + return int64(len(sequences)), nil +} + +// FixBadSequences fixes for broken sequences from recreate-table mistakes +func FixBadSequences() error { + if !setting.Database.UsePostgreSQL { + return nil + } + + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + + var sequences []string + schema := sess.Engine().Dialect().URI().Schema + + sess.Engine().SetSchema("") + if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil { + return err + } + sess.Engine().SetSchema(schema) + + sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`) + + for _, sequence := range sequences { + tableName := sequenceRegexp.FindStringSubmatch(sequence)[1] + newSequenceName := tableName + "_id_seq" + if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil { + return err + } + if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil { + return err + } + } + + return sess.Commit() +} diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 5cb85cc18cf..d16f8c7ec98 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -516,6 +516,31 @@ func recreateTable(sess *xorm.Session, bean interface{}) error { return err } case setting.Database.UsePostgreSQL: + var originalSequences []string + type sequenceData struct { + LastValue int `xorm:"'last_value'"` + IsCalled bool `xorm:"'is_called'"` + } + sequenceMap := map[string]sequenceData{} + + schema := sess.Engine().Dialect().URI().Schema + sess.Engine().SetSchema("") + if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&originalSequences); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + sess.Engine().SetSchema(schema) + + for _, sequence := range originalSequences { + sequenceData := sequenceData{} + if _, err := sess.Table(sequence).Cols("last_value", "is_called").Get(&sequenceData); err != nil { + log.Error("Unable to get last_value and is_called from %s. Error: %v", sequence, err) + return err + } + sequenceMap[sequence] = sequenceData + + } + // CASCADE causes postgres to drop all the constraints on the old table if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil { log.Error("Unable to drop old table %s. Error: %v", tableName, err) @@ -529,7 +554,6 @@ func recreateTable(sess *xorm.Session, bean interface{}) error { } var indices []string - schema := sess.Engine().Dialect().URI().Schema sess.Engine().SetSchema("") if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil { log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) @@ -545,6 +569,43 @@ func recreateTable(sess *xorm.Session, bean interface{}) error { } } + var sequences []string + sess.Engine().SetSchema("") + if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&sequences); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + sess.Engine().SetSchema(schema) + + for _, sequence := range sequences { + newSequenceName := strings.Replace(sequence, "tmp_recreate__", "", 1) + if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil { + log.Error("Unable to rename %s sequence to %s. Error: %v", sequence, newSequenceName, err) + return err + } + val, ok := sequenceMap[newSequenceName] + if newSequenceName == tableName+"_id_seq" { + if ok && val.LastValue != 0 { + if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil { + log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err) + return err + } + } else { + // We're going to try to guess this + if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil { + log.Error("Unable to reset %s. Error: %v", newSequenceName, err) + return err + } + } + } else if ok { + if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil { + log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err) + return err + } + } + + } + case setting.Database.UseMSSQL: // MSSQL will drop all the constraints on the old table if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {