mirror of
https://github.com/go-gitea/gitea
synced 2024-08-19 13:50:42 +02:00
Storage configuration support [storage]
(#13314)
* Fix minio bug * Add tests for storage configuration * Change the Seek flag to keep compitable minio? * Fix test when first-byte-pos of all ranges is greater than the resource length Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
parent
e7750e0f6a
commit
e4e85a3e51
@ -11,7 +11,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/http/httptest"
|
||||
@ -27,8 +26,10 @@ import (
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/base"
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/queue"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/storage"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
"code.gitea.io/gitea/routers"
|
||||
"code.gitea.io/gitea/routers/routes"
|
||||
@ -59,6 +60,8 @@ func NewNilResponseRecorder() *NilResponseRecorder {
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
defer log.Close()
|
||||
|
||||
managerCtx, cancel := context.WithCancel(context.Background())
|
||||
graceful.InitManager(managerCtx)
|
||||
defer cancel()
|
||||
@ -142,6 +145,10 @@ func initIntegrationTest() {
|
||||
util.RemoveAll(models.LocalCopyPath())
|
||||
setting.CheckLFSVersion()
|
||||
setting.InitDBConfig()
|
||||
if err := storage.Init(); err != nil {
|
||||
fmt.Printf("Init storage failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
switch {
|
||||
case setting.Database.UseMySQL:
|
||||
@ -149,27 +156,27 @@ func initIntegrationTest() {
|
||||
setting.Database.User, setting.Database.Passwd, setting.Database.Host))
|
||||
defer db.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("sql.Open: %v", err)
|
||||
log.Fatal("sql.Open: %v", err)
|
||||
}
|
||||
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
|
||||
log.Fatalf("db.Exec: %v", err)
|
||||
log.Fatal("db.Exec: %v", err)
|
||||
}
|
||||
case setting.Database.UsePostgreSQL:
|
||||
db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
|
||||
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
|
||||
defer db.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("sql.Open: %v", err)
|
||||
log.Fatal("sql.Open: %v", err)
|
||||
}
|
||||
dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
|
||||
if err != nil {
|
||||
log.Fatalf("db.Query: %v", err)
|
||||
log.Fatal("db.Query: %v", err)
|
||||
}
|
||||
defer dbrows.Close()
|
||||
|
||||
if !dbrows.Next() {
|
||||
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
|
||||
log.Fatalf("db.Exec: CREATE DATABASE: %v", err)
|
||||
log.Fatal("db.Exec: CREATE DATABASE: %v", err)
|
||||
}
|
||||
}
|
||||
// Check if we need to setup a specific schema
|
||||
@ -183,18 +190,18 @@ func initIntegrationTest() {
|
||||
// This is a different db object; requires a different Close()
|
||||
defer db.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("sql.Open: %v", err)
|
||||
log.Fatal("sql.Open: %v", err)
|
||||
}
|
||||
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
|
||||
if err != nil {
|
||||
log.Fatalf("db.Query: %v", err)
|
||||
log.Fatal("db.Query: %v", err)
|
||||
}
|
||||
defer schrows.Close()
|
||||
|
||||
if !schrows.Next() {
|
||||
// Create and setup a DB schema
|
||||
if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
|
||||
log.Fatalf("db.Exec: CREATE SCHEMA: %v", err)
|
||||
log.Fatal("db.Exec: CREATE SCHEMA: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -203,10 +210,10 @@ func initIntegrationTest() {
|
||||
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
|
||||
host, port, "master", setting.Database.User, setting.Database.Passwd))
|
||||
if err != nil {
|
||||
log.Fatalf("sql.Open: %v", err)
|
||||
log.Fatal("sql.Open: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil {
|
||||
log.Fatalf("db.Exec: %v", err)
|
||||
log.Fatal("db.Exec: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
@ -78,6 +78,7 @@ func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, exp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resp := session.MakeRequest(t, req, expectedStatus)
|
||||
|
||||
return resp
|
||||
@ -210,7 +211,7 @@ func TestGetLFSRange(t *testing.T) {
|
||||
{"bytes=0-10", "123456789\n", http.StatusPartialContent},
|
||||
// end-range bigger than length-1 is ignored
|
||||
{"bytes=0-11", "123456789\n", http.StatusPartialContent},
|
||||
{"bytes=11-", "", http.StatusPartialContent},
|
||||
{"bytes=11-", "Requested Range Not Satisfiable", http.StatusRequestedRangeNotSatisfiable},
|
||||
// incorrect header value cause whole header to be ignored
|
||||
{"bytes=-", "123456789\n", http.StatusOK},
|
||||
{"foobar", "123456789\n", http.StatusOK},
|
||||
|
@ -45,19 +45,21 @@ START_SSH_SERVER = true
|
||||
OFFLINE_MODE = false
|
||||
|
||||
LFS_START_SERVER = true
|
||||
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
|
||||
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
|
||||
LFS_STORE_TYPE = minio
|
||||
LFS_SERVE_DIRECT = false
|
||||
LFS_MINIO_ENDPOINT = minio:9000
|
||||
LFS_MINIO_ACCESS_KEY_ID = 123456
|
||||
LFS_MINIO_SECRET_ACCESS_KEY = 12345678
|
||||
LFS_MINIO_BUCKET = gitea
|
||||
LFS_MINIO_LOCATION = us-east-1
|
||||
LFS_MINIO_BASE_PATH = lfs/
|
||||
LFS_MINIO_USE_SSL = false
|
||||
|
||||
[lfs]
|
||||
MINIO_BASE_PATH = lfs/
|
||||
|
||||
[attachment]
|
||||
MINIO_BASE_PATH = attachments/
|
||||
|
||||
[avatars]
|
||||
MINIO_BASE_PATH = avatars/
|
||||
|
||||
[repo-avatars]
|
||||
MINIO_BASE_PATH = repo-avatars/
|
||||
|
||||
[storage]
|
||||
STORAGE_TYPE = minio
|
||||
SERVE_DIRECT = false
|
||||
MINIO_ENDPOINT = minio:9000
|
||||
@ -65,7 +67,6 @@ MINIO_ACCESS_KEY_ID = 123456
|
||||
MINIO_SECRET_ACCESS_KEY = 12345678
|
||||
MINIO_BUCKET = gitea
|
||||
MINIO_LOCATION = us-east-1
|
||||
MINIO_BASE_PATH = attachments/
|
||||
MINIO_USE_SSL = false
|
||||
|
||||
[mailer]
|
||||
@ -88,9 +89,6 @@ ENABLE_NOTIFY_MAIL = true
|
||||
DISABLE_GRAVATAR = false
|
||||
ENABLE_FEDERATED_AVATAR = false
|
||||
|
||||
AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/avatars
|
||||
REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/repo-avatars
|
||||
|
||||
[session]
|
||||
PROVIDER = file
|
||||
PROVIDER_CONFIG = integrations/gitea-integration-mysql/data/sessions
|
||||
|
@ -8,6 +8,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
@ -21,6 +22,21 @@ var (
|
||||
errSizeMismatch = errors.New("Content size does not match")
|
||||
)
|
||||
|
||||
// ErrRangeNotSatisfiable represents an error which request range is not satisfiable.
|
||||
type ErrRangeNotSatisfiable struct {
|
||||
FromByte int64
|
||||
}
|
||||
|
||||
func (err ErrRangeNotSatisfiable) Error() string {
|
||||
return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
|
||||
}
|
||||
|
||||
// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable
|
||||
func IsErrRangeNotSatisfiable(err error) bool {
|
||||
_, ok := err.(ErrRangeNotSatisfiable)
|
||||
return ok
|
||||
}
|
||||
|
||||
// ContentStore provides a simple file system based storage.
|
||||
type ContentStore struct {
|
||||
storage.ObjectStorage
|
||||
@ -35,7 +51,12 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
|
||||
return nil, err
|
||||
}
|
||||
if fromByte > 0 {
|
||||
_, err = f.Seek(fromByte, os.SEEK_CUR)
|
||||
if fromByte >= meta.Size {
|
||||
return nil, ErrRangeNotSatisfiable{
|
||||
FromByte: fromByte,
|
||||
}
|
||||
}
|
||||
_, err = f.Seek(fromByte, io.SeekStart)
|
||||
if err != nil {
|
||||
log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
|
||||
}
|
||||
|
@ -191,8 +191,12 @@ func getContentHandler(ctx *context.Context) {
|
||||
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||
content, err := contentStore.Get(meta, fromByte)
|
||||
if err != nil {
|
||||
// Errors are logged in contentStore.Get
|
||||
writeStatus(ctx, 404)
|
||||
if IsErrRangeNotSatisfiable(err) {
|
||||
writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable)
|
||||
} else {
|
||||
// Errors are logged in contentStore.Get
|
||||
writeStatus(ctx, 404)
|
||||
}
|
||||
return
|
||||
}
|
||||
defer content.Close()
|
||||
|
@ -32,14 +32,12 @@ func (s *Storage) MapTo(v interface{}) error {
|
||||
}
|
||||
|
||||
func getStorage(name, typ string, overrides ...*ini.Section) Storage {
|
||||
sectionName := "storage"
|
||||
if len(name) > 0 {
|
||||
sectionName = sectionName + "." + typ
|
||||
}
|
||||
const sectionName = "storage"
|
||||
sec := Cfg.Section(sectionName)
|
||||
|
||||
if len(overrides) == 0 {
|
||||
overrides = []*ini.Section{
|
||||
Cfg.Section(sectionName + "." + typ),
|
||||
Cfg.Section(sectionName + "." + name),
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user