support Uploader.MaxConcurrency=1 and updated tests
This commit is contained in:
parent
44289a93a2
commit
77c05dbd2a
|
|
@ -3,6 +3,8 @@
|
||||||
- ⚠️ Excluded the `lost+found` directory from the backups ([#7208](https://github.com/pocketbase/pocketbase/pull/7208); thanks @lbndev).
|
- ⚠️ Excluded the `lost+found` directory from the backups ([#7208](https://github.com/pocketbase/pocketbase/pull/7208); thanks @lbndev).
|
||||||
_If for some reason you want to keep it, you can restore it by editing the `e.Exclude` list of the `OnBackupCreate` and `OnBackupRestore` hooks._
|
_If for some reason you want to keep it, you can restore it by editing the `e.Exclude` list of the `OnBackupCreate` and `OnBackupRestore` hooks._
|
||||||
|
|
||||||
|
- Minor tests improvements (disabled initial superuser creation for the test app to avoid cluttering the std output, added more tests for the s3.Uploader.MaxConcurrency, etc.).
|
||||||
|
|
||||||
- Updated Go dependencies.
|
- Updated Go dependencies.
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -334,10 +334,10 @@ func (u *Uploader) multipartUpload(ctx context.Context, initPart []byte, optReqF
|
||||||
var g errgroup.Group
|
var g errgroup.Group
|
||||||
g.SetLimit(u.MaxConcurrency)
|
g.SetLimit(u.MaxConcurrency)
|
||||||
|
|
||||||
totalParallel := u.MaxConcurrency
|
totalWorkers := u.MaxConcurrency
|
||||||
|
|
||||||
if len(initPart) != 0 {
|
if len(initPart) != 0 {
|
||||||
totalParallel--
|
totalWorkers--
|
||||||
initPartNumber := u.lastPartNumber
|
initPartNumber := u.lastPartNumber
|
||||||
g.Go(func() error {
|
g.Go(func() error {
|
||||||
mp, err := u.uploadPart(ctx, initPartNumber, initPart, optReqFuncs...)
|
mp, err := u.uploadPart(ctx, initPartNumber, initPart, optReqFuncs...)
|
||||||
|
|
@ -353,7 +353,9 @@ func (u *Uploader) multipartUpload(ctx context.Context, initPart []byte, optReqF
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := 0; i < totalParallel; i++ {
|
totalWorkers = max(totalWorkers, 1)
|
||||||
|
|
||||||
|
for i := 0; i < totalWorkers; i++ {
|
||||||
g.Go(func() error {
|
g.Go(func() error {
|
||||||
for {
|
for {
|
||||||
part, num, err := u.nextPart()
|
part, num, err := u.nextPart()
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
|
@ -124,6 +125,10 @@ func TestUploaderSingleUpload(t *testing.T) {
|
||||||
func TestUploaderMultipartUploadSuccess(t *testing.T) {
|
func TestUploaderMultipartUploadSuccess(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
maxConcurrencies := []int{-1, 0, 1, 10}
|
||||||
|
|
||||||
|
for _, mc := range maxConcurrencies {
|
||||||
|
t.Run("MaxConcurrency_"+strconv.Itoa(mc), func(t *testing.T) {
|
||||||
httpClient := tests.NewClient(
|
httpClient := tests.NewClient(
|
||||||
&tests.RequestStub{
|
&tests.RequestStub{
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
|
|
@ -235,6 +240,7 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
|
||||||
Payload: strings.NewReader("abcdefg"),
|
Payload: strings.NewReader("abcdefg"),
|
||||||
Metadata: map[string]string{"a": "123", "b": "456"},
|
Metadata: map[string]string{"a": "123", "b": "456"},
|
||||||
MinPartSize: 3,
|
MinPartSize: 3,
|
||||||
|
MaxConcurrency: mc,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := uploader.Upload(context.Background(), func(r *http.Request) {
|
err := uploader.Upload(context.Background(), func(r *http.Request) {
|
||||||
|
|
@ -248,6 +254,8 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUploaderMultipartUploadPartFailure(t *testing.T) {
|
func TestUploaderMultipartUploadPartFailure(t *testing.T) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue