tomas/pocketbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge v0.23.0-rc changes

Browse Source
This commit is contained in:
Gani Georgiev
2024-09-29 19:23:19 +03:00
parent ad92992324
commit 844f18cac3
753 changed files with 85141 additions and 63396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/filesystem/file.go
+21 -9
View File
@@ -27,10 +27,20 @@ type FileReader interface {
//
// The file could be from a local path, multipart/form-data header, etc.
type File struct {
Reader FileReader
Name string
OriginalName string
Size int64
Reader FileReader `form:"-" json:"-" xml:"-"`
Name string `form:"name" json:"name" xml:"name"`
OriginalName string `form:"originalName" json:"originalName" xml:"originalName"`
Size int64 `form:"size" json:"size" xml:"size"`
}
// AsMap implements [core.mapExtractor] and returns a value suitable
// to be used in an API rule expression.
func (f *File) AsMap() map[string]any {
return map[string]any{
"name": f.Name,
"originalName": f.OriginalName,
"size": f.Size,
}
}
// NewFileFromPath creates a new File instance from the provided local file path.
@@ -79,7 +89,7 @@ func NewFileFromMultipart(mh *multipart.FileHeader) (*File, error) {
return f, nil
}
// NewFileFromUrl creates a new File from the provided url by
// NewFileFromURL creates a new File from the provided url by
// downloading the resource and load it as BytesReader.
//
// Example
@@ -87,8 +97,8 @@ func NewFileFromMultipart(mh *multipart.FileHeader) (*File, error) {
// ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
// defer cancel()
//
// file, err := filesystem.NewFileFromUrl(ctx, "https://example.com/image.png")
func NewFileFromUrl(ctx context.Context, url string) (*File, error) {
// file, err := filesystem.NewFileFromURL(ctx, "https://example.com/image.png")
func NewFileFromURL(ctx context.Context, url string) (*File, error) {
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, err
@@ -168,6 +178,8 @@ func (r *bytesReadSeekCloser) Close() error {
var extInvalidCharsRegex = regexp.MustCompile(`[^\w\.\*\-\+\=\#]+`)
const randomAlphabet = "abcdefghijklmnopqrstuvwxyz0123456789"
func normalizeName(fr FileReader, name string) string {
// extension
// ---
@@ -187,7 +199,7 @@ func normalizeName(fr FileReader, name string) string {
cleanName := inflector.Snakecase(strings.TrimSuffix(name, originalExt))
if length := len(cleanName); length < 3 {
// the name is too short so we concatenate an additional random part
cleanName += security.RandomString(10)
cleanName += security.RandomStringWithAlphabet(10, randomAlphabet)
} else if length > 100 {
// keep only the first 100 characters (it is multibyte safe after Snakecase)
cleanName = cleanName[:100]
@@ -196,7 +208,7 @@ func normalizeName(fr FileReader, name string) string {
return fmt.Sprintf(
"%s_%s%s",
cleanName,
security.RandomString(10), // ensure that there is always a random part
security.RandomStringWithAlphabet(10, randomAlphabet), // ensure that there is always a random part
cleanExt,
)
}
tools/filesystem/file_test.go
+30 -6
View File
@@ -12,11 +12,35 @@ import (
"strings"
"testing"
"github.com/labstack/echo/v5"
"github.com/pocketbase/pocketbase/tests"
"github.com/pocketbase/pocketbase/tools/filesystem"
)
func TestFileAsMap(t *testing.T) {
file, err := filesystem.NewFileFromBytes([]byte("test"), "test123.txt")
if err != nil {
t.Fatal(err)
}
result := file.AsMap()
if len(result) != 3 {
t.Fatalf("Expected map with %d keys, got\n%v", 3, result)
}
if result["size"] != int64(4) {
t.Fatalf("Expected size %d, got %#v", 4, result["size"])
}
if str, ok := result["name"].(string); !ok || !strings.HasPrefix(str, "test123") {
t.Fatalf("Expected name to have prefix %q, got %#v", "test123", result["name"])
}
if result["originalName"] != "test123.txt" {
t.Fatalf("Expected originalName %q, got %#v", "test123.txt", result["originalName"])
}
}
func TestNewFileFromPath(t *testing.T) {
testDir := createTestDir(t)
defer os.RemoveAll(testDir)
@@ -83,7 +107,7 @@ func TestNewFileFromMultipart(t *testing.T) {
}
req := httptest.NewRequest("", "/", formData)
req.Header.Set(echo.HeaderContentType, mp.FormDataContentType())
req.Header.Set("Content-Type", mp.FormDataContentType())
req.ParseMultipartForm(32 << 20)
_, mh, err := req.FormFile("test")
@@ -115,7 +139,7 @@ func TestNewFileFromMultipart(t *testing.T) {
}
}
func TestNewFileFromUrlTimeout(t *testing.T) {
func TestNewFileFromURLTimeout(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/error" {
w.WriteHeader(http.StatusInternalServerError)
@@ -129,7 +153,7 @@ func TestNewFileFromUrlTimeout(t *testing.T) {
{
ctx, cancel := context.WithCancel(context.Background())
cancel()
f, err := filesystem.NewFileFromUrl(ctx, srv.URL+"/cancel")
f, err := filesystem.NewFileFromURL(ctx, srv.URL+"/cancel")
if err == nil {
t.Fatal("[ctx_cancel] Expected error, got nil")
}
@@ -140,7 +164,7 @@ func TestNewFileFromUrlTimeout(t *testing.T) {
// error response
{
f, err := filesystem.NewFileFromUrl(context.Background(), srv.URL+"/error")
f, err := filesystem.NewFileFromURL(context.Background(), srv.URL+"/error")
if err == nil {
t.Fatal("[error_status] Expected error, got nil")
}
@@ -154,7 +178,7 @@ func TestNewFileFromUrlTimeout(t *testing.T) {
originalName := "image_! noext"
normalizedNamePattern := regexp.QuoteMeta("image_noext_") + `\w{10}` + regexp.QuoteMeta(".txt")
f, err := filesystem.NewFileFromUrl(context.Background(), srv.URL+"/"+originalName)
f, err := filesystem.NewFileFromURL(context.Background(), srv.URL+"/"+originalName)
if err != nil {
t.Fatalf("[valid] Unexpected error %v", err)
}
tools/filesystem/filesystem.go
+84 -25
View File
@@ -20,13 +20,17 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/disintegration/imaging"
"github.com/gabriel-vasile/mimetype"
"github.com/pocketbase/pocketbase/tools/filesystem/internal/s3lite"
"github.com/pocketbase/pocketbase/tools/list"
"gocloud.dev/blob"
"gocloud.dev/blob/fileblob"
"gocloud.dev/gcerrors"
)
var gcpIgnoreHeaders = []string{"Accept-Encoding"}
var ErrNotFound = errors.New("blob not found")
type System struct {
ctx context.Context
bucket *blob.Bucket
@@ -47,25 +51,23 @@ func NewS3(
cred := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
cfg, err := config.LoadDefaultConfig(ctx,
cfg, err := config.LoadDefaultConfig(
ctx,
config.WithCredentialsProvider(cred),
config.WithRegion(region),
config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
// ensure that the endpoint has url scheme for
// backward compatibility with v1 of the aws sdk
prefixedEndpoint := endpoint
if !strings.Contains(endpoint, "://") {
prefixedEndpoint = "https://" + endpoint
}
return aws.Endpoint{URL: prefixedEndpoint, SigningRegion: region}, nil
})),
)
if err != nil {
return nil, err
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
// ensure that the endpoint has url scheme for
// backward compatibility with v1 of the aws sdk
if !strings.Contains(endpoint, "://") {
endpoint = "https://" + endpoint
}
o.BaseEndpoint = aws.String(endpoint)
o.UsePathStyle = s3ForcePathStyle
// Google Cloud Storage alters the Accept-Encoding header,
@@ -76,7 +78,7 @@ func NewS3(
}
})
bucket, err := OpenBucketV2(ctx, client, bucketName, nil)
bucket, err := s3lite.OpenBucketV2(ctx, client, bucketName, nil)
if err != nil {
return nil, err
}
@@ -116,32 +118,59 @@ func (s *System) Close() error {
}
// Exists checks if file with fileKey path exists or not.
//
// If the file doesn't exist returns false and ErrNotFound.
func (s *System) Exists(fileKey string) (bool, error) {
return s.bucket.Exists(s.ctx, fileKey)
exists, err := s.bucket.Exists(s.ctx, fileKey)
if gcerrors.Code(err) == gcerrors.NotFound {
err = ErrNotFound
}
return exists, err
}
// Attributes returns the attributes for the file with fileKey path.
//
// If the file doesn't exist it returns ErrNotFound.
func (s *System) Attributes(fileKey string) (*blob.Attributes, error) {
return s.bucket.Attributes(s.ctx, fileKey)
attrs, err := s.bucket.Attributes(s.ctx, fileKey)
if gcerrors.Code(err) == gcerrors.NotFound {
err = ErrNotFound
}
return attrs, err
}
// GetFile returns a file content reader for the given fileKey.
//
// NB! Make sure to call `Close()` after you are done working with it.
// NB! Make sure to call Close() on the file after you are done working with it.
//
// If the file doesn't exist returns ErrNotFound.
func (s *System) GetFile(fileKey string) (*blob.Reader, error) {
br, err := s.bucket.NewReader(s.ctx, fileKey, nil)
if err != nil {
return nil, err
if gcerrors.Code(err) == gcerrors.NotFound {
err = ErrNotFound
}
return br, nil
return br, err
}
// Copy copies the file stored at srcKey to dstKey.
//
// If srcKey file doesn't exist, it returns ErrNotFound.
//
// If dstKey file already exists, it is overwritten.
func (s *System) Copy(srcKey, dstKey string) error {
return s.bucket.Copy(s.ctx, dstKey, srcKey, nil)
err := s.bucket.Copy(s.ctx, dstKey, srcKey, nil)
if gcerrors.Code(err) == gcerrors.NotFound {
err = ErrNotFound
}
return err
}
// List returns a flat list with info for all files under the specified prefix.
@@ -178,14 +207,13 @@ func (s *System) Upload(content []byte, fileKey string) error {
}
if _, err := w.Write(content); err != nil {
w.Close()
return err
return errors.Join(err, w.Close())
}
return w.Close()
}
// UploadFile uploads the provided multipart file to the fileKey location.
// UploadFile uploads the provided File to the fileKey location.
func (s *System) UploadFile(file *File, fileKey string) error {
f, err := file.Reader.Open()
if err != nil {
@@ -270,8 +298,16 @@ func (s *System) UploadMultipart(fh *multipart.FileHeader, fileKey string) error
}
// Delete deletes stored file at fileKey location.
//
// If the file doesn't exist returns ErrNotFound.
func (s *System) Delete(fileKey string) error {
return s.bucket.Delete(s.ctx, fileKey)
err := s.bucket.Delete(s.ctx, fileKey)
if gcerrors.Code(err) == gcerrors.NotFound {
return ErrNotFound
}
return err
}
// DeletePrefix deletes everything starting with the specified prefix.
@@ -345,6 +381,26 @@ func (s *System) DeletePrefix(prefix string) []error {
return failed
}
// Checks if the provided dir prefix doesn't have any files.
//
// A trailing slash will be appended to a non-empty dir string argument
// to ensure that the checked prefix is a "directory".
//
// Returns "false" in case the has at least one file, otherwise - "true".
func (s *System) IsEmptyDir(dir string) bool {
if dir != "" && !strings.HasSuffix(dir, "/") {
dir += "/"
}
iter := s.bucket.List(&blob.ListOptions{
Prefix: dir,
})
_, err := iter.Next(s.ctx)
return err == io.EOF
}
var inlineServeContentTypes = []string{
// image
"image/png", "image/jpg", "image/jpeg", "image/gif", "image/webp", "image/x-icon", "image/bmp",
@@ -371,8 +427,11 @@ const forceAttachmentParam = "download"
//
// If the `download` query parameter is used the file will be always served for
// download no matter of its type (aka. with "Content-Disposition: attachment").
//
// Internally this method uses [http.ServeContent] so Range requests,
// If-Match, If-Unmodified-Since, etc. headers are handled transparently.
func (s *System) Serve(res http.ResponseWriter, req *http.Request, fileKey string, name string) error {
br, readErr := s.bucket.NewReader(s.ctx, fileKey, nil)
br, readErr := s.GetFile(fileKey)
if readErr != nil {
return readErr
}
@@ -444,7 +503,7 @@ func (s *System) CreateThumb(originalKey string, thumbKey, thumbSize string) err
}
// fetch the original
r, readErr := s.bucket.NewReader(s.ctx, originalKey, nil)
r, readErr := s.GetFile(originalKey)
if readErr != nil {
return readErr
}
tools/filesystem/filesystem_test.go
+182 -111
View File
@@ -2,6 +2,7 @@ package filesystem_test
import (
"bytes"
"errors"
"image"
"image/png"
"mime/multipart"
@@ -19,11 +20,11 @@ func TestFileSystemExists(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
scenarios := []struct {
file string
@@ -35,12 +36,18 @@ func TestFileSystemExists(t *testing.T) {
{"image.png", true},
}
for i, scenario := range scenarios {
exists, _ := fs.Exists(scenario.file)
for _, s := range scenarios {
t.Run(s.file, func(t *testing.T) {
exists, err := fsys.Exists(s.file)
if exists != scenario.exists {
t.Errorf("(%d) Expected %v, got %v", i, scenario.exists, exists)
}
if err != nil {
t.Fatal(err)
}
if exists != s.exists {
t.Fatalf("Expected exists %v, got %v", s.exists, exists)
}
})
}
}
@@ -48,11 +55,11 @@ func TestFileSystemAttributes(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
scenarios := []struct {
file string
@@ -65,20 +72,24 @@ func TestFileSystemAttributes(t *testing.T) {
{"image.png", false, "image/png"},
}
for i, scenario := range scenarios {
attr, err := fs.Attributes(scenario.file)
for _, s := range scenarios {
t.Run(s.file, func(t *testing.T) {
attr, err := fsys.Attributes(s.file)
if err == nil && scenario.expectError {
t.Errorf("(%d) Expected error, got nil", i)
}
hasErr := err != nil
if err != nil && !scenario.expectError {
t.Errorf("(%d) Expected nil, got error, %v", i, err)
}
if hasErr != s.expectError {
t.Fatalf("Expected hasErr %v, got %v", s.expectError, hasErr)
}
if err == nil && attr.ContentType != scenario.expectContentType {
t.Errorf("(%d) Expected attr.ContentType to be %q, got %q", i, scenario.expectContentType, attr.ContentType)
}
if hasErr && !errors.Is(err, filesystem.ErrNotFound) {
t.Fatalf("Expected ErrNotFound err, got %q", err)
}
if !hasErr && attr.ContentType != s.expectContentType {
t.Fatalf("Expected attr.ContentType to be %q, got %q", s.expectContentType, attr.ContentType)
}
})
}
}
@@ -86,17 +97,17 @@ func TestFileSystemDelete(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
if err := fs.Delete("missing.txt"); err == nil {
t.Fatal("Expected error, got nil")
if err := fsys.Delete("missing.txt"); err == nil || !errors.Is(err, filesystem.ErrNotFound) {
t.Fatalf("Expected ErrNotFound error, got %v", err)
}
if err := fs.Delete("image.png"); err != nil {
if err := fsys.Delete("image.png"); err != nil {
t.Fatalf("Expected nil, got error %v", err)
}
}
@@ -105,29 +116,29 @@ func TestFileSystemDeletePrefixWithoutTrailingSlash(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
if errs := fs.DeletePrefix(""); len(errs) == 0 {
if errs := fsys.DeletePrefix(""); len(errs) == 0 {
t.Fatal("Expected error, got nil", errs)
}
if errs := fs.DeletePrefix("missing"); len(errs) != 0 {
if errs := fsys.DeletePrefix("missing"); len(errs) != 0 {
t.Fatalf("Not existing prefix shouldn't error, got %v", errs)
}
if errs := fs.DeletePrefix("test"); len(errs) != 0 {
if errs := fsys.DeletePrefix("test"); len(errs) != 0 {
t.Fatalf("Expected nil, got errors %v", errs)
}
// ensure that the test/* files are deleted
if exists, _ := fs.Exists("test/sub1.txt"); exists {
if exists, _ := fsys.Exists("test/sub1.txt"); exists {
t.Fatalf("Expected test/sub1.txt to be deleted")
}
if exists, _ := fs.Exists("test/sub2.txt"); exists {
if exists, _ := fsys.Exists("test/sub2.txt"); exists {
t.Fatalf("Expected test/sub2.txt to be deleted")
}
@@ -141,25 +152,25 @@ func TestFileSystemDeletePrefixWithTrailingSlash(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
if errs := fs.DeletePrefix("missing/"); len(errs) != 0 {
if errs := fsys.DeletePrefix("missing/"); len(errs) != 0 {
t.Fatalf("Not existing prefix shouldn't error, got %v", errs)
}
if errs := fs.DeletePrefix("test/"); len(errs) != 0 {
if errs := fsys.DeletePrefix("test/"); len(errs) != 0 {
t.Fatalf("Expected nil, got errors %v", errs)
}
// ensure that the test/* files are deleted
if exists, _ := fs.Exists("test/sub1.txt"); exists {
if exists, _ := fsys.Exists("test/sub1.txt"); exists {
t.Fatalf("Expected test/sub1.txt to be deleted")
}
if exists, _ := fs.Exists("test/sub2.txt"); exists {
if exists, _ := fsys.Exists("test/sub2.txt"); exists {
t.Fatalf("Expected test/sub2.txt to be deleted")
}
@@ -169,6 +180,41 @@ func TestFileSystemDeletePrefixWithTrailingSlash(t *testing.T) {
}
}
func TestFileSystemIsEmptyDir(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fsys.Close()
scenarios := []struct {
dir string
expected bool
}{
{"", false}, // special case that shouldn't be suffixed with delimiter to search for any files within the bucket
{"/", true},
{"missing", true},
{"missing/", true},
{"test", false},
{"test/", false},
{"empty", true},
{"empty/", true},
}
for _, s := range scenarios {
t.Run(s.dir, func(t *testing.T) {
result := fsys.IsEmptyDir(s.dir)
if result != s.expected {
t.Fatalf("Expected %v, got %v", s.expected, result)
}
})
}
}
func TestFileSystemUploadMultipart(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
@@ -193,24 +239,24 @@ func TestFileSystemUploadMultipart(t *testing.T) {
defer file.Close()
// ---
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
fileKey := "newdir/newkey.txt"
uploadErr := fs.UploadMultipart(fh, fileKey)
uploadErr := fsys.UploadMultipart(fh, fileKey)
if uploadErr != nil {
t.Fatal(uploadErr)
}
if exists, _ := fs.Exists(fileKey); !exists {
if exists, _ := fsys.Exists(fileKey); !exists {
t.Fatalf("Expected %q to exist", fileKey)
}
attrs, err := fs.Attributes(fileKey)
attrs, err := fsys.Attributes(fileKey)
if err != nil {
t.Fatalf("Failed to fetch file attributes: %v", err)
}
@@ -223,11 +269,11 @@ func TestFileSystemUploadFile(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
fileKey := "newdir/newkey.txt"
@@ -238,16 +284,16 @@ func TestFileSystemUploadFile(t *testing.T) {
file.OriginalName = "test.txt"
uploadErr := fs.UploadFile(file, fileKey)
uploadErr := fsys.UploadFile(file, fileKey)
if uploadErr != nil {
t.Fatal(uploadErr)
}
if exists, _ := fs.Exists(fileKey); !exists {
if exists, _ := fsys.Exists(fileKey); !exists {
t.Fatalf("Expected %q to exist", fileKey)
}
attrs, err := fs.Attributes(fileKey)
attrs, err := fsys.Attributes(fileKey)
if err != nil {
t.Fatalf("Failed to fetch file attributes: %v", err)
}
@@ -260,20 +306,20 @@ func TestFileSystemUpload(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
fileKey := "newdir/newkey.txt"
uploadErr := fs.Upload([]byte("demo"), fileKey)
uploadErr := fsys.Upload([]byte("demo"), fileKey)
if uploadErr != nil {
t.Fatal(uploadErr)
}
if exists, _ := fs.Exists(fileKey); !exists {
if exists, _ := fsys.Exists(fileKey); !exists {
t.Fatalf("Expected %s to exist", fileKey)
}
}
@@ -282,11 +328,11 @@ func TestFileSystemServe(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
csp := "default-src 'none'; media-src 'self'; style-src 'unsafe-inline'; sandbox"
cacheControl := "max-age=2592000, stale-while-revalidate=86400"
@@ -409,39 +455,41 @@ func TestFileSystemServe(t *testing.T) {
}
for _, s := range scenarios {
res := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/", nil)
t.Run(s.path, func(t *testing.T) {
res := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/", nil)
query := req.URL.Query()
for k, v := range s.query {
query.Set(k, v)
}
req.URL.RawQuery = query.Encode()
for k, v := range s.headers {
res.Header().Set(k, v)
}
err := fs.Serve(res, req, s.path, s.name)
hasErr := err != nil
if hasErr != s.expectError {
t.Errorf("(%s) Expected hasError %v, got %v (%v)", s.path, s.expectError, hasErr, err)
continue
}
if s.expectError {
continue
}
result := res.Result()
for hName, hValue := range s.expectHeaders {
v := result.Header.Get(hName)
if v != hValue {
t.Errorf("(%s) Expected value %q for header %q, got %q", s.path, hValue, hName, v)
query := req.URL.Query()
for k, v := range s.query {
query.Set(k, v)
}
}
req.URL.RawQuery = query.Encode()
for k, v := range s.headers {
res.Header().Set(k, v)
}
err := fsys.Serve(res, req, s.path, s.name)
hasErr := err != nil
if hasErr != s.expectError {
t.Fatalf("Expected hasError %v, got %v (%v)", s.expectError, hasErr, err)
}
if s.expectError {
return
}
result := res.Result()
defer result.Body.Close()
for hName, hValue := range s.expectHeaders {
v := result.Header.Get(hName)
if v != hValue {
t.Errorf("Expected value %q for header %q, got %q", hValue, hName, v)
}
}
})
}
}
@@ -449,20 +497,38 @@ func TestFileSystemGetFile(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
f, err := fs.GetFile("image.png")
if err != nil {
t.Fatal(err)
scenarios := []struct {
file string
expectError bool
}{
{"missing.png", true},
{"image.png", false},
}
defer f.Close()
if f == nil {
t.Fatal("File is supposed to be found")
for _, s := range scenarios {
t.Run(s.file, func(t *testing.T) {
f, err := fsys.GetFile(s.file)
hasErr := err != nil
if !hasErr {
defer f.Close()
}
if hasErr != s.expectError {
t.Fatalf("Expected hasErr %v, got %v", s.expectError, hasErr)
}
if hasErr && !errors.Is(err, filesystem.ErrNotFound) {
t.Fatalf("Expected ErrNotFound error, got %v", err)
}
})
}
}
@@ -470,25 +536,26 @@ func TestFileSystemCopy(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
src := "image.png"
dst := "image.png_copy"
// copy missing file
if err := fs.Copy(dst, src); err == nil {
if err := fsys.Copy(dst, src); err == nil {
t.Fatalf("Expected to fail copying %q to %q, got nil", dst, src)
}
// copy existing file
if err := fs.Copy(src, dst); err != nil {
if err := fsys.Copy(src, dst); err != nil {
t.Fatalf("Failed to copy %q to %q: %v", src, dst, err)
}
f, err := fs.GetFile(dst)
f, err := fsys.GetFile(dst)
//nolint
defer f.Close()
if err != nil {
t.Fatalf("Missing copied file %q: %v", dst, err)
@@ -502,11 +569,11 @@ func TestFileSystemList(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
scenarios := []struct {
prefix string
@@ -537,7 +604,7 @@ func TestFileSystemList(t *testing.T) {
}
for _, s := range scenarios {
objs, err := fs.List(s.prefix)
objs, err := fsys.List(s.prefix)
if err != nil {
t.Fatalf("[%s] %v", s.prefix, err)
}
@@ -563,17 +630,17 @@ func TestFileSystemServeSingleRange(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
res := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/", nil)
req.Header.Add("Range", "bytes=0-20")
if err := fs.Serve(res, req, "image.png", "image.png"); err != nil {
if err := fsys.Serve(res, req, "image.png", "image.png"); err != nil {
t.Fatal(err)
}
@@ -597,17 +664,17 @@ func TestFileSystemServeMultiRange(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
res := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/", nil)
req.Header.Add("Range", "bytes=0-20, 25-30")
if err := fs.Serve(res, req, "image.png", "image.png"); err != nil {
if err := fsys.Serve(res, req, "image.png", "image.png"); err != nil {
t.Fatal(err)
}
@@ -626,11 +693,11 @@ func TestFileSystemCreateThumb(t *testing.T) {
dir := createTestDir(t)
defer os.RemoveAll(dir)
fs, err := filesystem.NewLocal(dir)
fsys, err := filesystem.NewLocal(dir)
if err != nil {
t.Fatal(err)
}
defer fs.Close()
defer fsys.Close()
scenarios := []struct {
file string
@@ -651,7 +718,7 @@ func TestFileSystemCreateThumb(t *testing.T) {
}
for i, scenario := range scenarios {
err := fs.CreateThumb(scenario.file, scenario.thumb, "100x100")
err := fsys.CreateThumb(scenario.file, scenario.thumb, "100x100")
hasErr := err != nil
if hasErr != scenario.expectError {
@@ -663,7 +730,7 @@ func TestFileSystemCreateThumb(t *testing.T) {
continue
}
if exists, _ := fs.Exists(scenario.thumb); !exists {
if exists, _ := fsys.Exists(scenario.thumb); !exists {
t.Errorf("(%d) Couldn't find %q thumb", i, scenario.thumb)
}
}
@@ -677,6 +744,10 @@ func createTestDir(t *testing.T) string {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Join(dir, "empty"), os.ModePerm); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Join(dir, "test"), os.ModePerm); err != nil {
t.Fatal(err)
}
tools/filesystem/s3_trimmed.go → tools/filesystem/internal/s3lite/s3lite.go
+1 -168
View File
@@ -66,7 +66,7 @@
// (V1) *s3.PutObjectInput; (V2) *s3v2.PutObjectInput, when Options.Method == http.MethodPut, or
// (V1) *s3.DeleteObjectInput; (V2) [not supported] when Options.Method == http.MethodDelete
package filesystem
package s3lite
import (
"context"
@@ -82,7 +82,6 @@ import (
"strings"
awsv2 "github.com/aws/aws-sdk-go-v2/aws"
awsv2cfg "github.com/aws/aws-sdk-go-v2/config"
s3managerv2 "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
s3v2 "github.com/aws/aws-sdk-go-v2/service/s3"
typesv2 "github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -244,116 +243,8 @@ func URLUnescape(s string) string {
// -------------------------------------------------------------------
// UseV2 returns true iff the URL parameters indicate that the provider
// should use the AWS SDK v2.
//
// "awssdk=v1" will force V1.
// "awssdk=v2" will force V2.
// No "awssdk" parameter (or any other value) will return the default, currently V1.
// Note that the default may change in the future.
func UseV2(q url.Values) bool {
if values, ok := q["awssdk"]; ok {
if values[0] == "v2" || values[0] == "V2" {
return true
}
}
return false
}
// NewDefaultV2Config returns a aws.Config for AWS SDK v2, using the default options.
func NewDefaultV2Config(ctx context.Context) (awsv2.Config, error) {
return awsv2cfg.LoadDefaultConfig(ctx)
}
// V2ConfigFromURLParams returns an aws.Config for AWS SDK v2 initialized based on the URL
// parameters in q. It is intended to be used by URLOpeners for AWS services if
// UseV2 returns true.
//
// https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/aws#Config
//
// It returns an error if q contains any unknown query parameters; callers
// should remove any query parameters they know about from q before calling
// V2ConfigFromURLParams.
//
// The following query options are supported:
// - region: The AWS region for requests; sets WithRegion.
// - profile: The shared config profile to use; sets SharedConfigProfile.
// - endpoint: The AWS service endpoint to send HTTP request.
func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, error) {
var opts []func(*awsv2cfg.LoadOptions) error
for param, values := range q {
value := values[0]
switch param {
case "region":
opts = append(opts, awsv2cfg.WithRegion(value))
case "endpoint":
customResolver := awsv2.EndpointResolverWithOptionsFunc(
func(service, region string, options ...interface{}) (awsv2.Endpoint, error) {
return awsv2.Endpoint{
PartitionID: "aws",
URL: value,
SigningRegion: region,
}, nil
})
opts = append(opts, awsv2cfg.WithEndpointResolverWithOptions(customResolver))
case "profile":
opts = append(opts, awsv2cfg.WithSharedConfigProfile(value))
case "awssdk":
// ignore, should be handled before this
default:
return awsv2.Config{}, fmt.Errorf("unknown query parameter %q", param)
}
}
return awsv2cfg.LoadDefaultConfig(ctx, opts...)
}
// -------------------------------------------------------------------
const defaultPageSize = 1000
func init() {
blob.DefaultURLMux().RegisterBucket(Scheme, new(urlSessionOpener))
}
type urlSessionOpener struct{}
func (o *urlSessionOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket, error) {
opener := &URLOpener{UseV2: true}
return opener.OpenBucketURL(ctx, u)
}
// Scheme is the URL scheme s3blob registers its URLOpener under on
// blob.DefaultMux.
const Scheme = "s3"
// URLOpener opens S3 URLs like "s3://mybucket".
//
// The URL host is used as the bucket name.
//
// Use "awssdk=v1" to force using AWS SDK v1, "awssdk=v2" to force using AWS SDK v2,
// or anything else to accept the default.
//
// For V1, see gocloud.dev/aws/ConfigFromURLParams for supported query parameters
// for overriding the aws.Session from the URL.
// For V2, see gocloud.dev/aws/V2ConfigFromURLParams.
type URLOpener struct {
// UseV2 indicates whether the AWS SDK V2 should be used.
UseV2 bool
// Options specifies the options to pass to OpenBucket.
Options Options
}
// OpenBucketURL opens a blob.Bucket based on u.
func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket, error) {
cfg, err := V2ConfigFromURLParams(ctx, u.Query())
if err != nil {
return nil, fmt.Errorf("open bucket %v: %v", u, err)
}
clientV2 := s3v2.NewFromConfig(cfg)
return OpenBucketV2(ctx, clientV2, u.Host, &o.Options)
}
// Options sets options for constructing a *blob.Bucket backed by fileblob.
type Options struct {
// UseLegacyList forces the use of ListObjects instead of ListObjectsV2.
@@ -676,64 +567,6 @@ func (b *bucket) listObjectsV2(ctx context.Context, in *s3v2.ListObjectsV2Input,
}, nil
}
// func (b *bucket) listObjects(ctx context.Context, in *s3.ListObjectsV2Input, opts *driver.ListOptions) (*s3.ListObjectsV2Output, error) {
// if !b.useLegacyList {
// if opts.BeforeList != nil {
// asFunc := func(i interface{}) bool {
// if p, ok := i.(**s3.ListObjectsV2Input); ok {
// *p = in
// return true
// }
// return false
// }
// if err := opts.BeforeList(asFunc); err != nil {
// return nil, err
// }
// }
// return b.client.ListObjectsV2WithContext(ctx, in)
// }
// // Use the legacy ListObjects request.
// legacyIn := &s3.ListObjectsInput{
// Bucket: in.Bucket,
// Delimiter: in.Delimiter,
// EncodingType: in.EncodingType,
// Marker: in.ContinuationToken,
// MaxKeys: in.MaxKeys,
// Prefix: in.Prefix,
// RequestPayer: in.RequestPayer,
// }
// if opts.BeforeList != nil {
// asFunc := func(i interface{}) bool {
// p, ok := i.(**s3.ListObjectsInput)
// if !ok {
// return false
// }
// *p = legacyIn
// return true
// }
// if err := opts.BeforeList(asFunc); err != nil {
// return nil, err
// }
// }
// legacyResp, err := b.client.ListObjectsWithContext(ctx, legacyIn)
// if err != nil {
// return nil, err
// }
// var nextContinuationToken *string
// if legacyResp.NextMarker != nil {
// nextContinuationToken = legacyResp.NextMarker
// } else if awsv2.ToBool(legacyResp.IsTruncated) {
// nextContinuationToken = awsv2.String(awsv2.ToString(legacyResp.Contents[len(legacyResp.Contents)-1].Key))
// }
// return &s3.ListObjectsV2Output{
// CommonPrefixes: legacyResp.CommonPrefixes,
// Contents: legacyResp.Contents,
// NextContinuationToken: nextContinuationToken,
// }, nil
// }
// As implements driver.As.
func (b *bucket) As(i interface{}) bool {
p, ok := i.(**s3v2.Client)