From 7a5465fc56f79f5fc3c32547c89a80b7ebb24c8f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 8 Sep 2020 23:45:10 +0800 Subject: [PATCH] LFS support to be stored on minio (#12518) * LFS support to be stored on minio * Fix test * Fix lint * Fix lint * Fix check * Fix test * Update documents and add migration for LFS * Fix some bugs --- cmd/migrate_storage.go | 76 ++++++----- .../doc/advanced/config-cheat-sheet.en-us.md | 13 +- .../doc/advanced/config-cheat-sheet.zh-cn.md | 10 ++ integrations/lfs_getobject_test.go | 7 +- integrations/mysql.ini.tmpl | 20 ++- models/lfs.go | 32 +++++ models/unit_tests.go | 1 + modules/lfs/content_store.go | 82 ++++-------- modules/lfs/pointers.go | 8 +- modules/lfs/server.go | 41 ++++-- modules/repofiles/update.go | 9 +- modules/repofiles/upload.go | 9 +- modules/setting/lfs.go | 122 ++++++++++++++++++ modules/setting/setting.go | 86 +----------- modules/storage/local.go | 7 +- modules/storage/minio.go | 38 +++++- modules/storage/storage.go | 56 +++++++- routers/repo/lfs.go | 9 +- 18 files changed, 423 insertions(+), 203 deletions(-) create mode 100644 modules/setting/lfs.go diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 3a26f0b3f5b..b8e45c954d6 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -83,6 +83,13 @@ func migrateAttachments(dstStorage storage.ObjectStorage) error { }) } +func migrateLFS(dstStorage storage.ObjectStorage) error { + return models.IterateLFS(func(mo *models.LFSMetaObject) error { + _, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath()) + return err + }) +} + func runMigrateStorage(ctx *cli.Context) error { if err := initDB(); err != nil { return err @@ -103,45 +110,50 @@ func runMigrateStorage(ctx *cli.Context) error { return err } + var dstStorage storage.ObjectStorage + var err error + switch ctx.String("store") { + case "local": + p := ctx.String("path") + if p == "" { + log.Fatal("Path must be given when store is loal") + return nil + } + dstStorage, err = storage.NewLocalStorage(p) + case "minio": + dstStorage, err = storage.NewMinioStorage( + context.Background(), + ctx.String("minio-endpoint"), + ctx.String("minio-access-key-id"), + ctx.String("minio-secret-access-key"), + ctx.String("minio-bucket"), + ctx.String("minio-location"), + ctx.String("minio-base-path"), + ctx.Bool("minio-use-ssl"), + ) + default: + return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store")) + } + + if err != nil { + return err + } + tp := ctx.String("type") switch tp { case "attachments": - var dstStorage storage.ObjectStorage - var err error - switch ctx.String("store") { - case "local": - p := ctx.String("path") - if p == "" { - log.Fatal("Path must be given when store is loal") - return nil - } - dstStorage, err = storage.NewLocalStorage(p) - case "minio": - dstStorage, err = storage.NewMinioStorage( - context.Background(), - ctx.String("minio-endpoint"), - ctx.String("minio-access-key-id"), - ctx.String("minio-secret-access-key"), - ctx.String("minio-bucket"), - ctx.String("minio-location"), - ctx.String("minio-base-path"), - ctx.Bool("minio-use-ssl"), - ) - default: - return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store")) - } - - if err != nil { - return err - } if err := migrateAttachments(dstStorage); err != nil { return err } - - log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.") - - return nil + case "lfs": + if err := migrateLFS(dstStorage); err != nil { + return err + } + default: + return fmt.Errorf("Unsupported storage: %s", ctx.String("type")) } + log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.") + return nil } diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index f86415c2888..86f60e08271 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -206,12 +206,23 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. - `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars. - `ENABLE_GZIP`: **false**: Enables application-level GZIP support. - `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\]. + - `LFS_START_SERVER`: **false**: Enables git-lfs support. -- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files. +- `LFS_STORE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service. +- `LFS_SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing. +- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files, only available when `LFS_STORE_TYPE` is `local`. +- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `LFS_STORE_TYPE` is `minio` +- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `LFS_STORE_TYPE` is `minio` +- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `LFS_STORE_TYPE is` `minio` +- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `LFS_STORE_TYPE` is `minio` +- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `LFS_STORE_TYPE` is `minio` +- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `LFS_STORE_TYPE` is `minio` +- `LFS_MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `LFS_STORE_TYPE` is `minio` - `LFS_JWT_SECRET`: **\**: LFS authentication secret, change this a unique string. - `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail. - `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit). - `LFS_LOCK_PAGING_NUM`: **50**: Maximum number of LFS Locks returned per page. + - `REDIRECT_OTHER_PORT`: **false**: If true and `PROTOCOL` is https, allows redirecting http requests on `PORT_TO_REDIRECT` to the https port Gitea listens on. - `PORT_TO_REDIRECT`: **80**: Port for the http redirection service to listen on. Used when `REDIRECT_OTHER_PORT` is true. - `ENABLE_LETSENCRYPT`: **false**: If enabled you must set `DOMAIN` to valid internet facing domain (ensure DNS is set and port 80 is accessible by letsencrypt validation server). diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index d9a851b5084..ac6c94dff2f 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -69,8 +69,18 @@ menu: - `STATIC_CACHE_TIME`: **6h**: 静态资源文件，包括 `custom/`, `public/` 和所有上传的头像的浏览器缓存时间。 - `ENABLE_GZIP`: 启用应用级别的 GZIP 压缩。 - `LANDING_PAGE`: 未登录用户的默认页面，可选 `home` 或 `explore`。 + - `LFS_START_SERVER`: 是否启用 git-lfs 支持. 可以为 `true` 或 `false`， 默认是 `false`。 +- `LFS_STORE_TYPE`: **local**: LFS 的存储类型，`local` 将存储到磁盘，`minio` 将存储到 s3 兼容的对象服务。 +- `LFS_SERVE_DIRECT`: **false**: 允许直接重定向到存储系统。当前，仅 Minio/S3 是支持的。 - `LFS_CONTENT_PATH`: 存放 lfs 命令上传的文件的地方，默认是 `data/lfs`。 +- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio 地址，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。 +- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。 +- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。 +- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。 +- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location ，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。 +- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path ，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。 +- `LFS_MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。 - `LFS_JWT_SECRET`: LFS 认证密钥，改成自己的。 ## Database (`database`) diff --git a/integrations/lfs_getobject_test.go b/integrations/lfs_getobject_test.go index 45e94406a59..431c7ed9e8d 100644 --- a/integrations/lfs_getobject_test.go +++ b/integrations/lfs_getobject_test.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "gitea.com/macaron/gzip" gzipp "github.com/klauspost/compress/gzip" @@ -49,8 +50,10 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string lfsID++ lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) assert.NoError(t, err) - contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} - if !contentStore.Exists(lfsMetaObject) { + contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + exist, err := contentStore.Exists(lfsMetaObject) + assert.NoError(t, err) + if !exist { err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content)) assert.NoError(t, err) } diff --git a/integrations/mysql.ini.tmpl b/integrations/mysql.ini.tmpl index 8e3d2b3f14e..5211a4693af 100644 --- a/integrations/mysql.ini.tmpl +++ b/integrations/mysql.ini.tmpl @@ -36,13 +36,23 @@ ROOT_URL = http://localhost:3001/ DISABLE_SSH = false SSH_LISTEN_HOST = localhost SSH_PORT = 2201 -START_SSH_SERVER = true -LFS_START_SERVER = true -LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql -OFFLINE_MODE = false -LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w APP_DATA_PATH = integrations/gitea-integration-mysql/data BUILTIN_SSH_SERVER_USER = git +START_SSH_SERVER = true +OFFLINE_MODE = false + +LFS_START_SERVER = true +LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql +LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w +LFS_STORE_TYPE = minio +LFS_SERVE_DIRECT = false +LFS_MINIO_ENDPOINT = minio:9000 +LFS_MINIO_ACCESS_KEY_ID = 123456 +LFS_MINIO_SECRET_ACCESS_KEY = 12345678 +LFS_MINIO_BUCKET = gitea +LFS_MINIO_LOCATION = us-east-1 +LFS_MINIO_BASE_PATH = lfs/ +LFS_MINIO_USE_SSL = false [attachment] STORE_TYPE = minio diff --git a/models/lfs.go b/models/lfs.go index 7a04f799c0f..274b32a7367 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "path" "code.gitea.io/gitea/modules/timeutil" @@ -26,6 +27,15 @@ type LFSMetaObject struct { CreatedUnix timeutil.TimeStamp `xorm:"created"` } +// RelativePath returns the relative path of the lfs object +func (m *LFSMetaObject) RelativePath() string { + if len(m.Oid) < 5 { + return m.Oid + } + + return path.Join(m.Oid[0:2], m.Oid[2:4], m.Oid[4:]) +} + // Pointer returns the string representation of an LFS pointer file func (m *LFSMetaObject) Pointer() string { return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size) @@ -202,3 +212,25 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error { return sess.Commit() } + +// IterateLFS iterates lfs object +func IterateLFS(f func(mo *LFSMetaObject) error) error { + var start int + const batchSize = 100 + for { + var mos = make([]*LFSMetaObject, 0, batchSize) + if err := x.Limit(batchSize, start).Find(&mos); err != nil { + return err + } + if len(mos) == 0 { + return nil + } + start += len(mos) + + for _, mo := range mos { + if err := f(mo); err != nil { + return err + } + } + } +} diff --git a/models/unit_tests.go b/models/unit_tests.go index 11b7708ffd8..e977f706f84 100644 --- a/models/unit_tests.go +++ b/models/unit_tests.go @@ -69,6 +69,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string) { } setting.Attachment.Path = filepath.Join(setting.AppDataPath, "attachments") + setting.LFS.ContentPath = filepath.Join(setting.AppDataPath, "lfs") if err = storage.Init(); err != nil { fatalTestError("storage.Init: %v\n", err) } diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index b0fa77e2550..cf0a05d644c 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -10,11 +10,10 @@ import ( "errors" "io" "os" - "path/filepath" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/storage" ) var ( @@ -24,17 +23,15 @@ var ( // ContentStore provides a simple file system based storage. type ContentStore struct { - BasePath string + storage.ObjectStorage } // Get takes a Meta object and retrieves the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) { - path := filepath.Join(s.BasePath, transformKey(meta.Oid)) - - f, err := os.Open(path) + f, err := s.Open(meta.RelativePath()) if err != nil { - log.Error("Whilst trying to read LFS OID[%s]: Unable to open %s Error: %v", meta.Oid, path, err) + log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err) return nil, err } if fromByte > 0 { @@ -48,82 +45,55 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC // Put takes a Meta object and an io.Reader and writes the content to the store. func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { - path := filepath.Join(s.BasePath, transformKey(meta.Oid)) - tmpPath := path + ".tmp" - - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0750); err != nil { - log.Error("Whilst putting LFS OID[%s]: Unable to create the LFS directory: %s Error: %v", meta.Oid, dir, err) - return err - } - - file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640) - if err != nil { - log.Error("Whilst putting LFS OID[%s]: Unable to open temporary file for writing: %s Error: %v", tmpPath, err) - return err - } - defer func() { - if err := util.Remove(tmpPath); err != nil { - log.Warn("Unable to remove temporary path: %s: Error: %v", tmpPath, err) - } - }() - hash := sha256.New() - hw := io.MultiWriter(hash, file) - - written, err := io.Copy(hw, r) + rd := io.TeeReader(r, hash) + p := meta.RelativePath() + written, err := s.Save(p, rd) if err != nil { - log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, tmpPath, err) - file.Close() + log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err) return err } - file.Close() if written != meta.Size { + if err := s.Delete(p); err != nil { + log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err) + } return errSizeMismatch } shaStr := hex.EncodeToString(hash.Sum(nil)) if shaStr != meta.Oid { + if err := s.Delete(p); err != nil { + log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err) + } return errHashMismatch } - if err := os.Rename(tmpPath, path); err != nil { - log.Error("Whilst putting LFS OID[%s]: Unable to move tmp file to final destination: %s Error: %v", meta.Oid, path, err) - return err - } - return nil } // Exists returns true if the object exists in the content store. -func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool { - path := filepath.Join(s.BasePath, transformKey(meta.Oid)) - if _, err := os.Stat(path); os.IsNotExist(err) { - return false +func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) { + _, err := s.ObjectStorage.Stat(meta.RelativePath()) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err } - return true + return true, nil } // Verify returns true if the object exists in the content store and size is correct. func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) { - path := filepath.Join(s.BasePath, transformKey(meta.Oid)) - - fi, err := os.Stat(path) - if os.IsNotExist(err) || err == nil && fi.Size() != meta.Size { + p := meta.RelativePath() + fi, err := s.ObjectStorage.Stat(p) + if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) { return false, nil } else if err != nil { - log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", path, meta.Oid, err) + log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err) return false, err } return true, nil } - -func transformKey(key string) string { - if len(key) < 5 { - return key - } - - return filepath.Join(key[0:2], key[2:4], key[4:]) -} diff --git a/modules/lfs/pointers.go b/modules/lfs/pointers.go index bc27ee37a7f..c6fbf090e51 100644 --- a/modules/lfs/pointers.go +++ b/modules/lfs/pointers.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" ) // ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file @@ -53,9 +54,10 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject { return nil } - contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + contentStore := &ContentStore{ObjectStorage: storage.LFS} meta := &models.LFSMetaObject{Oid: oid, Size: size} - if !contentStore.Exists(meta) { + exist, err := contentStore.Exists(meta) + if err != nil || !exist { return nil } @@ -64,6 +66,6 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject { // ReadMetaObject will read a models.LFSMetaObject and return a reader func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) { - contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + contentStore := &ContentStore{ObjectStorage: storage.LFS} return contentStore.Get(meta, 0) } diff --git a/modules/lfs/server.go b/modules/lfs/server.go index f227ebe2eb1..2801f8410cc 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "gitea.com/macaron/macaron" "github.com/dgrijalva/jwt-go" @@ -187,7 +188,7 @@ func getContentHandler(ctx *context.Context) { } } - contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + contentStore := &ContentStore{ObjectStorage: storage.LFS} content, err := contentStore.Get(meta, fromByte) if err != nil { // Errors are logged in contentStore.Get @@ -288,8 +289,14 @@ func PostHandler(ctx *context.Context) { ctx.Resp.Header().Set("Content-Type", metaMediaType) sentStatus := 202 - contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} - if meta.Existing && contentStore.Exists(meta) { + contentStore := &ContentStore{ObjectStorage: storage.LFS} + exist, err := contentStore.Exists(meta) + if err != nil { + log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err) + writeStatus(ctx, 500) + return + } + if meta.Existing && exist { sentStatus = 200 } ctx.Resp.WriteHeader(sentStatus) @@ -343,12 +350,20 @@ func BatchHandler(ctx *context.Context) { return } - contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + contentStore := &ContentStore{ObjectStorage: storage.LFS} meta, err := repository.GetLFSMetaObjectByOid(object.Oid) - if err == nil && contentStore.Exists(meta) { // Object is found and exists - responseObjects = append(responseObjects, Represent(object, meta, true, false)) - continue + if err == nil { // Object is found and exists + exist, err := contentStore.Exists(meta) + if err != nil { + log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err) + writeStatus(ctx, 500) + return + } + if exist { + responseObjects = append(responseObjects, Represent(object, meta, true, false)) + continue + } } if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize { @@ -360,7 +375,13 @@ func BatchHandler(ctx *context.Context) { // Object is not found meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID}) if err == nil { - responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !contentStore.Exists(meta))) + exist, err := contentStore.Exists(meta) + if err != nil { + log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err) + writeStatus(ctx, 500) + return + } + responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !exist)) } else { log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err) } @@ -387,7 +408,7 @@ func PutHandler(ctx *context.Context) { return } - contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + contentStore := &ContentStore{ObjectStorage: storage.LFS} bodyReader := ctx.Req.Body().ReadCloser() defer bodyReader.Close() if err := contentStore.Put(meta, bodyReader); err != nil { @@ -429,7 +450,7 @@ func VerifyHandler(ctx *context.Context) { return } - contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + contentStore := &ContentStore{ObjectStorage: storage.LFS} ok, err := contentStore.Verify(meta) if err != nil { // Error will be logged in Verify diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index d65f61c8409..84a3bcb64ea 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" pull_service "code.gitea.io/gitea/services/pull" @@ -433,8 +434,12 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up if err != nil { return nil, err } - contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} - if !contentStore.Exists(lfsMetaObject) { + contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + exist, err := contentStore.Exists(lfsMetaObject) + if err != nil { + return nil, err + } + if !exist { if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil { if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err) diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index eb1379560df..e3ec48ec0f9 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" ) // UploadRepoFileOptions contains the uploaded repository file options @@ -163,12 +164,16 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep // OK now we can insert the data into the store - there's no way to clean up the store // once it's in there, it's in there. - contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} + contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} for _, uploadInfo := range infos { if uploadInfo.lfsMetaObject == nil { continue } - if !contentStore.Exists(uploadInfo.lfsMetaObject) { + exist, err := contentStore.Exists(uploadInfo.lfsMetaObject) + if err != nil { + return cleanUpAfterFailure(&infos, t, err) + } + if !exist { file, err := os.Open(uploadInfo.upload.LocalPath()) if err != nil { return cleanUpAfterFailure(&infos, t, err) diff --git a/modules/setting/lfs.go b/modules/setting/lfs.go new file mode 100644 index 00000000000..a740a6d6292 --- /dev/null +++ b/modules/setting/lfs.go @@ -0,0 +1,122 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "encoding/base64" + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/modules/generate" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + + "github.com/unknwon/com" + ini "gopkg.in/ini.v1" +) + +// LFS represents the configuration for Git LFS +var LFS = struct { + StartServer bool `ini:"LFS_START_SERVER"` + ContentPath string `ini:"LFS_CONTENT_PATH"` + JWTSecretBase64 string `ini:"LFS_JWT_SECRET"` + JWTSecretBytes []byte `ini:"-"` + HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"` + MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"` + LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"` + + StoreType string + ServeDirect bool + Minio struct { + Endpoint string + AccessKeyID string + SecretAccessKey string + UseSSL bool + Bucket string + Location string + BasePath string + } +}{ + StoreType: "local", +} + +func newLFSService() { + sec := Cfg.Section("server") + if err := sec.MapTo(&LFS); err != nil { + log.Fatal("Failed to map LFS settings: %v", err) + } + + LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs")) + if !filepath.IsAbs(LFS.ContentPath) { + LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath) + } + if LFS.LocksPagingNum == 0 { + LFS.LocksPagingNum = 50 + } + + LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(20 * time.Minute) + + if LFS.StartServer { + LFS.JWTSecretBytes = make([]byte, 32) + n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) + + if err != nil || n != 32 { + LFS.JWTSecretBase64, err = generate.NewJwtSecret() + if err != nil { + log.Fatal("Error generating JWT Secret for custom config: %v", err) + return + } + + // Save secret + cfg := ini.Empty() + if com.IsFile(CustomConf) { + // Keeps custom settings if there is already something. + if err := cfg.Append(CustomConf); err != nil { + log.Error("Failed to load custom conf '%s': %v", CustomConf, err) + } + } + + cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) + + if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil { + log.Fatal("Failed to create '%s': %v", CustomConf, err) + } + if err := cfg.SaveTo(CustomConf); err != nil { + log.Fatal("Error saving generated JWT Secret to custom config: %v", err) + return + } + } + } +} + +func ensureLFSDirectory() { + if LFS.StartServer { + if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil { + log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err) + } + } +} + +// CheckLFSVersion will check lfs version, if not satisfied, then disable it. +func CheckLFSVersion() { + if LFS.StartServer { + //Disable LFS client hooks if installed for the current OS user + //Needs at least git v2.1.2 + + err := git.LoadGitVersion() + if err != nil { + log.Fatal("Error retrieving git version: %v", err) + } + + if git.CheckGitVersionConstraint(">= 2.1.2") != nil { + LFS.StartServer = false + log.Error("LFS server support needs at least Git v2.1.2") + } else { + git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=", + "-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=") + } + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 8449851efbd..c93cc928045 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -23,7 +23,6 @@ import ( "time" "code.gitea.io/gitea/modules/generate" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/user" @@ -133,16 +132,6 @@ var ( MinimumKeySizes: map[string]int{"ed25519": 256, "ecdsa": 256, "rsa": 2048, "dsa": 1024}, } - LFS struct { - StartServer bool `ini:"LFS_START_SERVER"` - ContentPath string `ini:"LFS_CONTENT_PATH"` - JWTSecretBase64 string `ini:"LFS_JWT_SECRET"` - JWTSecretBytes []byte `ini:"-"` - HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"` - MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"` - LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"` - } - // Security settings InstallLock bool SecretKey string @@ -472,27 +461,6 @@ func createPIDFile(pidPath string) { } } -// CheckLFSVersion will check lfs version, if not satisfied, then disable it. -func CheckLFSVersion() { - if LFS.StartServer { - //Disable LFS client hooks if installed for the current OS user - //Needs at least git v2.1.2 - - err := git.LoadGitVersion() - if err != nil { - log.Fatal("Error retrieving git version: %v", err) - } - - if git.CheckGitVersionConstraint(">= 2.1.2") != nil { - LFS.StartServer = false - log.Error("LFS server support needs at least Git v2.1.2") - } else { - git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=", - "-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=") - } - } -} - // SetCustomPathAndConf will set CustomPath and CustomConf with reference to the // GITEA_CUSTOM environment variable and with provided overrides before stepping // back to the default @@ -722,51 +690,7 @@ func NewContext() { SSH.CreateAuthorizedKeysFile = sec.Key("SSH_CREATE_AUTHORIZED_KEYS_FILE").MustBool(true) SSH.ExposeAnonymous = sec.Key("SSH_EXPOSE_ANONYMOUS").MustBool(false) - sec = Cfg.Section("server") - if err = sec.MapTo(&LFS); err != nil { - log.Fatal("Failed to map LFS settings: %v", err) - } - LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs")) - if !filepath.IsAbs(LFS.ContentPath) { - LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath) - } - if LFS.LocksPagingNum == 0 { - LFS.LocksPagingNum = 50 - } - - LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(20 * time.Minute) - - if LFS.StartServer { - LFS.JWTSecretBytes = make([]byte, 32) - n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) - - if err != nil || n != 32 { - LFS.JWTSecretBase64, err = generate.NewJwtSecret() - if err != nil { - log.Fatal("Error generating JWT Secret for custom config: %v", err) - return - } - - // Save secret - cfg := ini.Empty() - if com.IsFile(CustomConf) { - // Keeps custom settings if there is already something. - if err := cfg.Append(CustomConf); err != nil { - log.Error("Failed to load custom conf '%s': %v", CustomConf, err) - } - } - - cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) - - if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil { - log.Fatal("Failed to create '%s': %v", CustomConf, err) - } - if err := cfg.SaveTo(CustomConf); err != nil { - log.Fatal("Error saving generated JWT Secret to custom config: %v", err) - return - } - } - } + newLFSService() if err = Cfg.Section("oauth2").MapTo(&OAuth2); err != nil { log.Fatal("Failed to OAuth2 settings: %v", err) @@ -1086,14 +1010,6 @@ func loadOrGenerateInternalToken(sec *ini.Section) string { return token } -func ensureLFSDirectory() { - if LFS.StartServer { - if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil { - log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err) - } - } -} - // NewServices initializes the services func NewServices() { InitDBConfig() diff --git a/modules/storage/local.go b/modules/storage/local.go index 4c830211d96..a937a831f15 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -34,7 +34,7 @@ func NewLocalStorage(bucket string) (*LocalStorage, error) { } // Open a file -func (l *LocalStorage) Open(path string) (io.ReadCloser, error) { +func (l *LocalStorage) Open(path string) (Object, error) { return os.Open(filepath.Join(l.dir, path)) } @@ -58,6 +58,11 @@ func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) { return io.Copy(f, r) } +// Stat returns the info of the file +func (l *LocalStorage) Stat(path string) (ObjectInfo, error) { + return os.Stat(filepath.Join(l.dir, path)) +} + // Delete delete a file func (l *LocalStorage) Delete(path string) error { p := filepath.Join(l.dir, path) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 77d24e6b732..30751755f3e 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -8,6 +8,7 @@ import ( "context" "io" "net/url" + "os" "path" "strings" "time" @@ -62,7 +63,7 @@ func (m *MinioStorage) buildMinioPath(p string) string { } // Open open a file -func (m *MinioStorage) Open(path string) (io.ReadCloser, error) { +func (m *MinioStorage) Open(path string) (Object, error) { var opts = minio.GetObjectOptions{} object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts) if err != nil { @@ -87,6 +88,41 @@ func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) { return uploadInfo.Size, nil } +type minioFileInfo struct { + minio.ObjectInfo +} + +func (m minioFileInfo) Name() string { + return m.ObjectInfo.Key +} + +func (m minioFileInfo) Size() int64 { + return m.ObjectInfo.Size +} + +func (m minioFileInfo) ModTime() time.Time { + return m.LastModified +} + +// Stat returns the stat information of the object +func (m *MinioStorage) Stat(path string) (ObjectInfo, error) { + info, err := m.client.StatObject( + m.ctx, + m.bucket, + m.buildMinioPath(path), + minio.StatObjectOptions{}, + ) + if err != nil { + if errResp, ok := err.(minio.ErrorResponse); ok { + if errResp.Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + } + return nil, err + } + return &minioFileInfo{info}, nil +} + // Delete delete a file func (m *MinioStorage) Delete(path string) error { return m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{}) diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 8528ebc5cb3..e355d2459f5 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "net/url" + "time" "code.gitea.io/gitea/modules/setting" ) @@ -19,10 +20,24 @@ var ( ErrURLNotSupported = errors.New("url method not supported") ) +// Object represents the object on the storage +type Object interface { + io.ReadCloser + io.Seeker +} + +// ObjectInfo represents the object info on the storage +type ObjectInfo interface { + Name() string + Size() int64 + ModTime() time.Time +} + // ObjectStorage represents an object storage to handle a bucket and files type ObjectStorage interface { + Open(path string) (Object, error) Save(path string, r io.Reader) (int64, error) - Open(path string) (io.ReadCloser, error) + Stat(path string) (ObjectInfo, error) Delete(path string) error URL(path, name string) (*url.URL, error) } @@ -41,10 +56,21 @@ func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, sr var ( // Attachments represents attachments storage Attachments ObjectStorage + + // LFS represents lfs storage + LFS ObjectStorage ) // Init init the stoarge func Init() error { + if err := initAttachments(); err != nil { + return err + } + + return initLFS() +} + +func initAttachments() error { var err error switch setting.Attachment.StoreType { case "local": @@ -71,3 +97,31 @@ func Init() error { return nil } + +func initLFS() error { + var err error + switch setting.LFS.StoreType { + case "local": + LFS, err = NewLocalStorage(setting.LFS.ContentPath) + case "minio": + minio := setting.LFS.Minio + LFS, err = NewMinioStorage( + context.Background(), + minio.Endpoint, + minio.AccessKeyID, + minio.SecretAccessKey, + minio.Bucket, + minio.Location, + minio.BasePath, + minio.UseSSL, + ) + default: + return fmt.Errorf("Unsupported LFS store type: %s", setting.LFS.StoreType) + } + + if err != nil { + return err + } + + return nil +} diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 4caf01adfdc..481a3e57ad9 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -28,6 +28,7 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" gogit "github.com/go-git/go-git/v5" @@ -619,7 +620,7 @@ type pointerResult struct { func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { defer wg.Done() defer catFileBatchReader.Close() - contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath} + contentStore := lfs.ContentStore{ObjectStorage: storage.LFS} bufferedReader := bufio.NewReader(catFileBatchReader) buf := make([]byte, 1025) @@ -673,7 +674,11 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg result.InRepo = true } - result.Exists = contentStore.Exists(pointer) + result.Exists, err = contentStore.Exists(pointer) + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } if result.Exists { if !result.InRepo {