Create doctor command to fix repo_units broken by dumps from 1.14.3-1.14.6 (#17136)

There was a serious issue with the `gitea dump` command in 1.14.3-1.14.6 which led to corruption of the `config` field of the `repo_unit` table. 

This PR adds a doctor command to attempt to fix the broken repo_units. Users affected by #16961 should run:

```
gitea doctor --fix --run fix-broken-repo-units
```

Fix #16961

Signed-off-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
zeripath 2021-09-27 16:55:12 +01:00 committed by GitHub
parent 4e0cca3f7d
commit b5856c4437
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 596 additions and 1 deletions

View File

@ -51,7 +51,7 @@ func JSONUnmarshalHandleDoubleEncode(bs []byte, v interface{}) error {
rs = append(rs, temp...) rs = append(rs, temp...)
} }
if ok { if ok {
if rs[0] == 0xff && rs[1] == 0xfe { if len(rs) > 1 && rs[0] == 0xff && rs[1] == 0xfe {
rs = rs[2:] rs = rs[2:]
} }
err = json.Unmarshal(rs, v) err = json.Unmarshal(rs, v)

View File

@ -220,3 +220,9 @@ func getUnitsByRepoID(e db.Engine, repoID int64) (units []*RepoUnit, err error)
return units, nil return units, nil
} }
// UpdateRepoUnit updates the provided repo unit
func UpdateRepoUnit(unit *RepoUnit) error {
_, err := db.GetEngine(db.DefaultContext).ID(unit.ID).Update(unit)
return err
}

318
modules/doctor/fix16961.go Normal file
View File

@ -0,0 +1,318 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package doctor
import (
"bytes"
"fmt"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil"
"xorm.io/builder"
)
// #16831 revealed that the dump command that was broken in 1.14.3-1.14.6 and 1.15.0 (#15885).
// This led to repo_unit and login_source cfg not being converted to JSON in the dump
// Unfortunately although it was hoped that there were only a few users affected it
// appears that many users are affected.
// We therefore need to provide a doctor command to fix this repeated issue #16961
func parseBool16961(bs []byte) (bool, error) {
if bytes.EqualFold(bs, []byte("%!s(bool=false)")) {
return false, nil
}
if bytes.EqualFold(bs, []byte("%!s(bool=true)")) {
return true, nil
}
return false, fmt.Errorf("unexpected bool format: %s", string(bs))
}
func fixUnitConfig16961(bs []byte, cfg *models.UnitConfig) (fixed bool, err error) {
err = models.JSONUnmarshalHandleDoubleEncode(bs, &cfg)
if err == nil {
return
}
// Handle #16961
if string(bs) != "&{}" && len(bs) != 0 {
return
}
return true, nil
}
func fixExternalWikiConfig16961(bs []byte, cfg *models.ExternalWikiConfig) (fixed bool, err error) {
err = models.JSONUnmarshalHandleDoubleEncode(bs, &cfg)
if err == nil {
return
}
if len(bs) < 3 {
return
}
if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
return
}
cfg.ExternalWikiURL = string(bs[2 : len(bs)-1])
return true, nil
}
func fixExternalTrackerConfig16961(bs []byte, cfg *models.ExternalTrackerConfig) (fixed bool, err error) {
err = models.JSONUnmarshalHandleDoubleEncode(bs, &cfg)
if err == nil {
return
}
// Handle #16961
if len(bs) < 3 {
return
}
if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
return
}
parts := bytes.Split(bs[2:len(bs)-1], []byte{' '})
if len(parts) != 3 {
return
}
cfg.ExternalTrackerURL = string(bytes.Join(parts[:len(parts)-2], []byte{' '}))
cfg.ExternalTrackerFormat = string(parts[len(parts)-2])
cfg.ExternalTrackerStyle = string(parts[len(parts)-1])
return true, nil
}
func fixPullRequestsConfig16961(bs []byte, cfg *models.PullRequestsConfig) (fixed bool, err error) {
err = models.JSONUnmarshalHandleDoubleEncode(bs, &cfg)
if err == nil {
return
}
// Handle #16961
if len(bs) < 3 {
return
}
if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
return
}
// PullRequestsConfig was the following in 1.14
// type PullRequestsConfig struct {
// IgnoreWhitespaceConflicts bool
// AllowMerge bool
// AllowRebase bool
// AllowRebaseMerge bool
// AllowSquash bool
// AllowManualMerge bool
// AutodetectManualMerge bool
// }
//
// 1.15 added in addition:
// DefaultDeleteBranchAfterMerge bool
// DefaultMergeStyle MergeStyle
parts := bytes.Split(bs[2:len(bs)-1], []byte{' '})
if len(parts) < 7 {
return
}
var parseErr error
cfg.IgnoreWhitespaceConflicts, parseErr = parseBool16961(parts[0])
if parseErr != nil {
return
}
cfg.AllowMerge, parseErr = parseBool16961(parts[1])
if parseErr != nil {
return
}
cfg.AllowRebase, parseErr = parseBool16961(parts[2])
if parseErr != nil {
return
}
cfg.AllowRebaseMerge, parseErr = parseBool16961(parts[3])
if parseErr != nil {
return
}
cfg.AllowSquash, parseErr = parseBool16961(parts[4])
if parseErr != nil {
return
}
cfg.AllowManualMerge, parseErr = parseBool16961(parts[5])
if parseErr != nil {
return
}
cfg.AutodetectManualMerge, parseErr = parseBool16961(parts[6])
if parseErr != nil {
return
}
// 1.14 unit
if len(parts) == 7 {
return true, nil
}
if len(parts) < 9 {
return
}
cfg.DefaultDeleteBranchAfterMerge, parseErr = parseBool16961(parts[7])
if parseErr != nil {
return
}
cfg.DefaultMergeStyle = models.MergeStyle(string(bytes.Join(parts[8:], []byte{' '})))
return true, nil
}
func fixIssuesConfig16961(bs []byte, cfg *models.IssuesConfig) (fixed bool, err error) {
err = models.JSONUnmarshalHandleDoubleEncode(bs, &cfg)
if err == nil {
return
}
// Handle #16961
if len(bs) < 3 {
return
}
if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
return
}
parts := bytes.Split(bs[2:len(bs)-1], []byte{' '})
if len(parts) != 3 {
return
}
var parseErr error
cfg.EnableTimetracker, parseErr = parseBool16961(parts[0])
if parseErr != nil {
return
}
cfg.AllowOnlyContributorsToTrackTime, parseErr = parseBool16961(parts[1])
if parseErr != nil {
return
}
cfg.EnableDependencies, parseErr = parseBool16961(parts[2])
if parseErr != nil {
return
}
return true, nil
}
func fixBrokenRepoUnit16961(repoUnit *models.RepoUnit, bs []byte) (fixed bool, err error) {
// Shortcut empty or null values
if len(bs) == 0 {
return false, nil
}
switch models.UnitType(repoUnit.Type) {
case models.UnitTypeCode, models.UnitTypeReleases, models.UnitTypeWiki, models.UnitTypeProjects:
cfg := &models.UnitConfig{}
repoUnit.Config = cfg
if fixed, err := fixUnitConfig16961(bs, cfg); !fixed {
return false, err
}
case models.UnitTypeExternalWiki:
cfg := &models.ExternalWikiConfig{}
repoUnit.Config = cfg
if fixed, err := fixExternalWikiConfig16961(bs, cfg); !fixed {
return false, err
}
case models.UnitTypeExternalTracker:
cfg := &models.ExternalTrackerConfig{}
repoUnit.Config = cfg
if fixed, err := fixExternalTrackerConfig16961(bs, cfg); !fixed {
return false, err
}
case models.UnitTypePullRequests:
cfg := &models.PullRequestsConfig{}
repoUnit.Config = cfg
if fixed, err := fixPullRequestsConfig16961(bs, cfg); !fixed {
return false, err
}
case models.UnitTypeIssues:
cfg := &models.IssuesConfig{}
repoUnit.Config = cfg
if fixed, err := fixIssuesConfig16961(bs, cfg); !fixed {
return false, err
}
default:
panic(fmt.Sprintf("unrecognized repo unit type: %v", repoUnit.Type))
}
return true, nil
}
func fixBrokenRepoUnits16961(logger log.Logger, autofix bool) error {
// RepoUnit describes all units of a repository
type RepoUnit struct {
ID int64
RepoID int64
Type models.UnitType
Config []byte
CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
}
count := 0
err := db.Iterate(
db.DefaultContext,
new(RepoUnit),
builder.Gt{
"id": 0,
},
func(idx int, bean interface{}) error {
unit := bean.(*RepoUnit)
bs := unit.Config
repoUnit := &models.RepoUnit{
ID: unit.ID,
RepoID: unit.RepoID,
Type: unit.Type,
CreatedUnix: unit.CreatedUnix,
}
if fixed, err := fixBrokenRepoUnit16961(repoUnit, bs); !fixed {
return err
}
count++
if !autofix {
return nil
}
return models.UpdateRepoUnit(repoUnit)
},
)
if err != nil {
logger.Critical("Unable to iterate acrosss repounits to fix the broken units: Error %v", err)
return err
}
if !autofix {
logger.Warn("Found %d broken repo_units", count)
return nil
}
logger.Info("Fixed %d broken repo_units", count)
return nil
}
func init() {
Register(&Check{
Title: "Check for incorrectly dumped repo_units (See #16961)",
Name: "fix-broken-repo-units",
IsDefault: false,
Run: fixBrokenRepoUnits16961,
Priority: 7,
})
}

View File

@ -0,0 +1,271 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package doctor
import (
"testing"
"code.gitea.io/gitea/models"
"github.com/stretchr/testify/assert"
)
func Test_fixUnitConfig_16961(t *testing.T) {
tests := []struct {
name string
bs string
wantFixed bool
wantErr bool
}{
{
name: "empty",
bs: "",
wantFixed: true,
wantErr: false,
},
{
name: "normal: {}",
bs: "{}",
wantFixed: false,
wantErr: false,
},
{
name: "broken but fixable: &{}",
bs: "&{}",
wantFixed: true,
wantErr: false,
},
{
name: "broken but unfixable: &{asdasd}",
bs: "&{asdasd}",
wantFixed: false,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotFixed, err := fixUnitConfig16961([]byte(tt.bs), &models.UnitConfig{})
if (err != nil) != tt.wantErr {
t.Errorf("fixUnitConfig_16961() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotFixed != tt.wantFixed {
t.Errorf("fixUnitConfig_16961() = %v, want %v", gotFixed, tt.wantFixed)
}
})
}
}
func Test_fixExternalWikiConfig_16961(t *testing.T) {
tests := []struct {
name string
bs string
expected string
wantFixed bool
wantErr bool
}{
{
name: "normal: {\"ExternalWikiURL\":\"http://someurl\"}",
bs: "{\"ExternalWikiURL\":\"http://someurl\"}",
expected: "http://someurl",
wantFixed: false,
wantErr: false,
},
{
name: "broken: &{http://someurl}",
bs: "&{http://someurl}",
expected: "http://someurl",
wantFixed: true,
wantErr: false,
},
{
name: "broken but unfixable: http://someurl",
bs: "http://someurl",
wantFixed: false,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &models.ExternalWikiConfig{}
gotFixed, err := fixExternalWikiConfig16961([]byte(tt.bs), cfg)
if (err != nil) != tt.wantErr {
t.Errorf("fixExternalWikiConfig_16961() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotFixed != tt.wantFixed {
t.Errorf("fixExternalWikiConfig_16961() = %v, want %v", gotFixed, tt.wantFixed)
}
if cfg.ExternalWikiURL != tt.expected {
t.Errorf("fixExternalWikiConfig_16961().ExternalWikiURL = %v, want %v", cfg.ExternalWikiURL, tt.expected)
}
})
}
}
func Test_fixExternalTrackerConfig_16961(t *testing.T) {
tests := []struct {
name string
bs string
expected models.ExternalTrackerConfig
wantFixed bool
wantErr bool
}{
{
name: "normal",
bs: `{"ExternalTrackerURL":"a","ExternalTrackerFormat":"b","ExternalTrackerStyle":"c"}`,
expected: models.ExternalTrackerConfig{
ExternalTrackerURL: "a",
ExternalTrackerFormat: "b",
ExternalTrackerStyle: "c",
},
wantFixed: false,
wantErr: false,
},
{
name: "broken",
bs: "&{a b c}",
expected: models.ExternalTrackerConfig{
ExternalTrackerURL: "a",
ExternalTrackerFormat: "b",
ExternalTrackerStyle: "c",
},
wantFixed: true,
wantErr: false,
},
{
name: "broken - too many fields",
bs: "&{a b c d}",
wantFixed: false,
wantErr: true,
},
{
name: "broken - wrong format",
bs: "a b c d}",
wantFixed: false,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &models.ExternalTrackerConfig{}
gotFixed, err := fixExternalTrackerConfig16961([]byte(tt.bs), cfg)
if (err != nil) != tt.wantErr {
t.Errorf("fixExternalTrackerConfig_16961() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotFixed != tt.wantFixed {
t.Errorf("fixExternalTrackerConfig_16961() = %v, want %v", gotFixed, tt.wantFixed)
}
if cfg.ExternalTrackerFormat != tt.expected.ExternalTrackerFormat {
t.Errorf("fixExternalTrackerConfig_16961().ExternalTrackerFormat = %v, want %v", tt.expected.ExternalTrackerFormat, cfg.ExternalTrackerFormat)
}
if cfg.ExternalTrackerStyle != tt.expected.ExternalTrackerStyle {
t.Errorf("fixExternalTrackerConfig_16961().ExternalTrackerStyle = %v, want %v", tt.expected.ExternalTrackerStyle, cfg.ExternalTrackerStyle)
}
if cfg.ExternalTrackerURL != tt.expected.ExternalTrackerURL {
t.Errorf("fixExternalTrackerConfig_16961().ExternalTrackerURL = %v, want %v", tt.expected.ExternalTrackerURL, cfg.ExternalTrackerURL)
}
})
}
}
func Test_fixPullRequestsConfig_16961(t *testing.T) {
tests := []struct {
name string
bs string
expected models.PullRequestsConfig
wantFixed bool
wantErr bool
}{
{
name: "normal",
bs: `{"IgnoreWhitespaceConflicts":false,"AllowMerge":false,"AllowRebase":false,"AllowRebaseMerge":false,"AllowSquash":false,"AllowManualMerge":false,"AutodetectManualMerge":false,"DefaultDeleteBranchAfterMerge":false,"DefaultMergeStyle":""}`,
},
{
name: "broken - 1.14",
bs: `&{%!s(bool=false) %!s(bool=true) %!s(bool=true) %!s(bool=true) %!s(bool=true) %!s(bool=false) %!s(bool=false)}`,
expected: models.PullRequestsConfig{
IgnoreWhitespaceConflicts: false,
AllowMerge: true,
AllowRebase: true,
AllowRebaseMerge: true,
AllowSquash: true,
AllowManualMerge: false,
AutodetectManualMerge: false,
},
wantFixed: true,
},
{
name: "broken - 1.15",
bs: `&{%!s(bool=false) %!s(bool=true) %!s(bool=true) %!s(bool=true) %!s(bool=true) %!s(bool=false) %!s(bool=false) %!s(bool=false) merge}`,
expected: models.PullRequestsConfig{
AllowMerge: true,
AllowRebase: true,
AllowRebaseMerge: true,
AllowSquash: true,
DefaultMergeStyle: models.MergeStyleMerge,
},
wantFixed: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &models.PullRequestsConfig{}
gotFixed, err := fixPullRequestsConfig16961([]byte(tt.bs), cfg)
if (err != nil) != tt.wantErr {
t.Errorf("fixPullRequestsConfig_16961() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotFixed != tt.wantFixed {
t.Errorf("fixPullRequestsConfig_16961() = %v, want %v", gotFixed, tt.wantFixed)
}
assert.EqualValues(t, &tt.expected, cfg)
})
}
}
func Test_fixIssuesConfig_16961(t *testing.T) {
tests := []struct {
name string
bs string
expected models.IssuesConfig
wantFixed bool
wantErr bool
}{
{
name: "normal",
bs: `{"EnableTimetracker":true,"AllowOnlyContributorsToTrackTime":true,"EnableDependencies":true}`,
expected: models.IssuesConfig{
EnableTimetracker: true,
AllowOnlyContributorsToTrackTime: true,
EnableDependencies: true,
},
},
{
name: "broken",
bs: `&{%!s(bool=true) %!s(bool=true) %!s(bool=true)}`,
expected: models.IssuesConfig{
EnableTimetracker: true,
AllowOnlyContributorsToTrackTime: true,
EnableDependencies: true,
},
wantFixed: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &models.IssuesConfig{}
gotFixed, err := fixIssuesConfig16961([]byte(tt.bs), cfg)
if (err != nil) != tt.wantErr {
t.Errorf("fixIssuesConfig_16961() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotFixed != tt.wantFixed {
t.Errorf("fixIssuesConfig_16961() = %v, want %v", gotFixed, tt.wantFixed)
}
assert.EqualValues(t, &tt.expected, cfg)
})
}
}