diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c05c6d3..871008b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ - ⚠️ Excluded the `lost+found` directory from the backups ([#7208](https://github.com/pocketbase/pocketbase/pull/7208); thanks @lbndev). _If for some reason you want to keep it, you can restore it by editing the `e.Exclude` list of the `OnBackupCreate` and `OnBackupRestore` hooks._ +- Minor tests improvements (disabled initial superuser creation for the test app to avoid cluttering the std output, added more tests for the s3.Uploader.MaxConcurrency, etc.). + - Updated Go dependencies. diff --git a/tools/filesystem/internal/s3blob/s3/uploader.go b/tools/filesystem/internal/s3blob/s3/uploader.go index afc54285..1c12ba3a 100644 --- a/tools/filesystem/internal/s3blob/s3/uploader.go +++ b/tools/filesystem/internal/s3blob/s3/uploader.go @@ -334,10 +334,10 @@ func (u *Uploader) multipartUpload(ctx context.Context, initPart []byte, optReqF var g errgroup.Group g.SetLimit(u.MaxConcurrency) - totalParallel := u.MaxConcurrency + totalWorkers := u.MaxConcurrency if len(initPart) != 0 { - totalParallel-- + totalWorkers-- initPartNumber := u.lastPartNumber g.Go(func() error { mp, err := u.uploadPart(ctx, initPartNumber, initPart, optReqFuncs...) @@ -353,7 +353,9 @@ func (u *Uploader) multipartUpload(ctx context.Context, initPart []byte, optReqF }) } - for i := 0; i < totalParallel; i++ { + totalWorkers = max(totalWorkers, 1) + + for i := 0; i < totalWorkers; i++ { g.Go(func() error { for { part, num, err := u.nextPart() diff --git a/tools/filesystem/internal/s3blob/s3/uploader_test.go b/tools/filesystem/internal/s3blob/s3/uploader_test.go index 92314855..16652022 100644 --- a/tools/filesystem/internal/s3blob/s3/uploader_test.go +++ b/tools/filesystem/internal/s3blob/s3/uploader_test.go @@ -4,6 +4,7 @@ import ( "context" "io" "net/http" + "strconv" "strings" "testing" @@ -124,20 +125,24 @@ func TestUploaderSingleUpload(t *testing.T) { func TestUploaderMultipartUploadSuccess(t *testing.T) { t.Parallel() - httpClient := tests.NewClient( - &tests.RequestStub{ - Method: http.MethodPost, - URL: "http://test_bucket.example.com/test_key?uploads", - Match: func(req *http.Request) bool { - return tests.ExpectHeaders(req.Header, map[string]string{ - "x-amz-meta-a": "123", - "x-amz-meta-b": "456", - "test_header": "test", - "Authorization": "^.+Credential=123/.+$", - }) - }, - Response: &http.Response{ - Body: io.NopCloser(strings.NewReader(` + maxConcurrencies := []int{-1, 0, 1, 10} + + for _, mc := range maxConcurrencies { + t.Run("MaxConcurrency_"+strconv.Itoa(mc), func(t *testing.T) { + httpClient := tests.NewClient( + &tests.RequestStub{ + Method: http.MethodPost, + URL: "http://test_bucket.example.com/test_key?uploads", + Match: func(req *http.Request) bool { + return tests.ExpectHeaders(req.Header, map[string]string{ + "x-amz-meta-a": "123", + "x-amz-meta-b": "456", + "test_header": "test", + "Authorization": "^.+Credential=123/.+$", + }) + }, + Response: &http.Response{ + Body: io.NopCloser(strings.NewReader(` test_bucket @@ -145,108 +150,111 @@ func TestUploaderMultipartUploadSuccess(t *testing.T) { test_id `)), - }, - }, - &tests.RequestStub{ - Method: http.MethodPut, - URL: "http://test_bucket.example.com/test_key?partNumber=1&uploadId=test_id", - Match: func(req *http.Request) bool { - body, err := io.ReadAll(req.Body) - if err != nil { - return false - } + }, + }, + &tests.RequestStub{ + Method: http.MethodPut, + URL: "http://test_bucket.example.com/test_key?partNumber=1&uploadId=test_id", + Match: func(req *http.Request) bool { + body, err := io.ReadAll(req.Body) + if err != nil { + return false + } - return string(body) == "abc" && tests.ExpectHeaders(req.Header, map[string]string{ - "Content-Length": "3", - "test_header": "test", - "Authorization": "^.+Credential=123/.+$", - }) - }, - Response: &http.Response{ - Header: http.Header{"Etag": []string{"etag1"}}, - }, - }, - &tests.RequestStub{ - Method: http.MethodPut, - URL: "http://test_bucket.example.com/test_key?partNumber=2&uploadId=test_id", - Match: func(req *http.Request) bool { - body, err := io.ReadAll(req.Body) - if err != nil { - return false - } + return string(body) == "abc" && tests.ExpectHeaders(req.Header, map[string]string{ + "Content-Length": "3", + "test_header": "test", + "Authorization": "^.+Credential=123/.+$", + }) + }, + Response: &http.Response{ + Header: http.Header{"Etag": []string{"etag1"}}, + }, + }, + &tests.RequestStub{ + Method: http.MethodPut, + URL: "http://test_bucket.example.com/test_key?partNumber=2&uploadId=test_id", + Match: func(req *http.Request) bool { + body, err := io.ReadAll(req.Body) + if err != nil { + return false + } - return string(body) == "def" && tests.ExpectHeaders(req.Header, map[string]string{ - "Content-Length": "3", - "test_header": "test", - "Authorization": "^.+Credential=123/.+$", - }) - }, - Response: &http.Response{ - Header: http.Header{"Etag": []string{"etag2"}}, - }, - }, - &tests.RequestStub{ - Method: http.MethodPut, - URL: "http://test_bucket.example.com/test_key?partNumber=3&uploadId=test_id", - Match: func(req *http.Request) bool { - body, err := io.ReadAll(req.Body) - if err != nil { - return false - } - return string(body) == "g" && tests.ExpectHeaders(req.Header, map[string]string{ - "Content-Length": "1", - "test_header": "test", - "Authorization": "^.+Credential=123/.+$", - }) - }, - Response: &http.Response{ - Header: http.Header{"Etag": []string{"etag3"}}, - }, - }, - &tests.RequestStub{ - Method: http.MethodPost, - URL: "http://test_bucket.example.com/test_key?uploadId=test_id", - Match: func(req *http.Request) bool { - body, err := io.ReadAll(req.Body) - if err != nil { - return false - } + return string(body) == "def" && tests.ExpectHeaders(req.Header, map[string]string{ + "Content-Length": "3", + "test_header": "test", + "Authorization": "^.+Credential=123/.+$", + }) + }, + Response: &http.Response{ + Header: http.Header{"Etag": []string{"etag2"}}, + }, + }, + &tests.RequestStub{ + Method: http.MethodPut, + URL: "http://test_bucket.example.com/test_key?partNumber=3&uploadId=test_id", + Match: func(req *http.Request) bool { + body, err := io.ReadAll(req.Body) + if err != nil { + return false + } + return string(body) == "g" && tests.ExpectHeaders(req.Header, map[string]string{ + "Content-Length": "1", + "test_header": "test", + "Authorization": "^.+Credential=123/.+$", + }) + }, + Response: &http.Response{ + Header: http.Header{"Etag": []string{"etag3"}}, + }, + }, + &tests.RequestStub{ + Method: http.MethodPost, + URL: "http://test_bucket.example.com/test_key?uploadId=test_id", + Match: func(req *http.Request) bool { + body, err := io.ReadAll(req.Body) + if err != nil { + return false + } - expected := `etag11etag22etag33` + expected := `etag11etag22etag33` - return strings.Contains(string(body), expected) && tests.ExpectHeaders(req.Header, map[string]string{ - "test_header": "test", - "Authorization": "^.+Credential=123/.+$", - }) - }, - }, - ) + return strings.Contains(string(body), expected) && tests.ExpectHeaders(req.Header, map[string]string{ + "test_header": "test", + "Authorization": "^.+Credential=123/.+$", + }) + }, + }, + ) - uploader := &s3.Uploader{ - S3: &s3.S3{ - Client: httpClient, - Region: "test_region", - Bucket: "test_bucket", - Endpoint: "http://example.com", - AccessKey: "123", - SecretKey: "abc", - }, - Key: "test_key", - Payload: strings.NewReader("abcdefg"), - Metadata: map[string]string{"a": "123", "b": "456"}, - MinPartSize: 3, - } + uploader := &s3.Uploader{ + S3: &s3.S3{ + Client: httpClient, + Region: "test_region", + Bucket: "test_bucket", + Endpoint: "http://example.com", + AccessKey: "123", + SecretKey: "abc", + }, + Key: "test_key", + Payload: strings.NewReader("abcdefg"), + Metadata: map[string]string{"a": "123", "b": "456"}, + MinPartSize: 3, + MaxConcurrency: mc, + } - err := uploader.Upload(context.Background(), func(r *http.Request) { - r.Header.Set("test_header", "test") - }) - if err != nil { - t.Fatal(err) - } + err := uploader.Upload(context.Background(), func(r *http.Request) { + r.Header.Set("test_header", "test") + }) + if err != nil { + t.Fatal(err) + } - err = httpClient.AssertNoRemaining() - if err != nil { - t.Fatal(err) + err = httpClient.AssertNoRemaining() + if err != nil { + t.Fatal(err) + } + }) } }