mirror of
https://github.com/go-gitea/gitea
synced 2024-11-05 14:27:02 +01:00
fb7b743bd0
This PR implemented object storages(LFS/Packages/Attachments and etc.) for Azure Blob Storage. It depends on azure official golang SDK and can support both the azure blob storage cloud service and azurite mock server. Replace #25458 Fix #22527 - [x] CI Tests - [x] integration test, MSSQL integration tests will now based on azureblob - [x] unit test - [x] CLI Migrate Storage - [x] Documentation for configuration added ------ TODO (other PRs): - [ ] Improve performance of `blob download`. --------- Co-authored-by: yp05327 <576951401@qq.com>
100 lines
2.1 KiB
Go
100 lines
2.1 KiB
Go
// Copyright 2021 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package util
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"io"
|
|
)
|
|
|
|
// ReadAtMost reads at most len(buf) bytes from r into buf.
|
|
// It returns the number of bytes copied. n is only less than len(buf) if r provides fewer bytes.
|
|
// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
|
|
func ReadAtMost(r io.Reader, buf []byte) (n int, err error) {
|
|
n, err = io.ReadFull(r, buf)
|
|
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
|
err = nil
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
// ReadWithLimit reads at most "limit" bytes from r into buf.
|
|
// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
|
|
func ReadWithLimit(r io.Reader, n int) (buf []byte, err error) {
|
|
return readWithLimit(r, 1024, n)
|
|
}
|
|
|
|
func readWithLimit(r io.Reader, batch, limit int) ([]byte, error) {
|
|
if limit <= batch {
|
|
buf := make([]byte, limit)
|
|
n, err := ReadAtMost(r, buf)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return buf[:n], nil
|
|
}
|
|
res := bytes.NewBuffer(make([]byte, 0, batch))
|
|
bufFix := make([]byte, batch)
|
|
eof := false
|
|
for res.Len() < limit && !eof {
|
|
bufTmp := bufFix
|
|
if res.Len()+batch > limit {
|
|
bufTmp = bufFix[:limit-res.Len()]
|
|
}
|
|
n, err := io.ReadFull(r, bufTmp)
|
|
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
|
eof = true
|
|
} else if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err = res.Write(bufTmp[:n]); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return res.Bytes(), nil
|
|
}
|
|
|
|
// ErrNotEmpty is an error reported when there is a non-empty reader
|
|
var ErrNotEmpty = errors.New("not-empty")
|
|
|
|
// IsEmptyReader reads a reader and ensures it is empty
|
|
func IsEmptyReader(r io.Reader) (err error) {
|
|
var buf [1]byte
|
|
|
|
for {
|
|
n, err := r.Read(buf[:])
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if n > 0 {
|
|
return ErrNotEmpty
|
|
}
|
|
}
|
|
}
|
|
|
|
type CountingReader struct {
|
|
io.Reader
|
|
n int
|
|
}
|
|
|
|
var _ io.Reader = &CountingReader{}
|
|
|
|
func (w *CountingReader) Count() int {
|
|
return w.n
|
|
}
|
|
|
|
func (w *CountingReader) Read(p []byte) (int, error) {
|
|
n, err := w.Reader.Read(p)
|
|
w.n += n
|
|
return n, err
|
|
}
|
|
|
|
func NewCountingReader(rd io.Reader) *CountingReader {
|
|
return &CountingReader{Reader: rd}
|
|
}
|