tomas/pocketbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

abstract rest.UploadedFile to allow loading local files

Browse Source
This commit is contained in:
Gani Georgiev
2022-12-10 16:47:45 +02:00
parent aa6eaa7319
commit 37bac5cc50
12 changed files with 322 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/filesystem/file.go
+135
View File
@@ -0,0 +1,135 @@
package filesystem
import (
"fmt"
"io"
"mime/multipart"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/gabriel-vasile/mimetype"
"github.com/pocketbase/pocketbase/tools/inflector"
"github.com/pocketbase/pocketbase/tools/security"
)
// FileReader defines an interface for a file resource reader.
type FileReader interface {
Open() (io.ReadSeekCloser, error)
}
// File defines a single file [io.ReadSeekCloser] resource.
//
// The file could be from a local path, multipipart/formdata header, etc.
type File struct {
Name string
OriginalName string
Size int64
Reader FileReader
}
// NewFileFromPath creates a new File instance from the provided local file path.
func NewFileFromPath(path string) (*File, error) {
f := &File{}
info, err := os.Stat(path)
if err != nil {
return nil, err
}
f.Reader = &PathReader{Path: path}
f.Size = info.Size()
f.OriginalName = info.Name()
f.Name = normalizeName(f.Reader, f.OriginalName)
return f, nil
}
// NewFileFromMultipart creates a new File instace from the provided multipart header.
func NewFileFromMultipart(mh *multipart.FileHeader) (*File, error) {
f := &File{}
f.Reader = &MultipartReader{Header: mh}
f.Size = mh.Size
f.OriginalName = mh.Filename
f.Name = normalizeName(f.Reader, f.OriginalName)
return f, nil
}
// -------------------------------------------------------------------
var _ FileReader = (*MultipartReader)(nil)
// MultipartReader defines a [multipart.FileHeader] reader.
type MultipartReader struct {
Header *multipart.FileHeader
}
// Open implements the [filesystem.FileReader] interface.
func (r *MultipartReader) Open() (io.ReadSeekCloser, error) {
return r.Header.Open()
}
// -------------------------------------------------------------------
var _ FileReader = (*PathReader)(nil)
type PathReader struct {
Path string
}
// Open implements the [filesystem.FileReader] interface.
func (r *PathReader) Open() (io.ReadSeekCloser, error) {
return os.Open(r.Path)
}
// -------------------------------------------------------------------
var extInvalidCharsRegex = regexp.MustCompile(`[^\w\.\*\-\+\=\#]+`)
func normalizeName(fr FileReader, name string) string {
// extension
// ---
originalExt := filepath.Ext(name)
cleanExt := extInvalidCharsRegex.ReplaceAllString(originalExt, "")
if cleanExt == "" {
// try to detect the extension from the file content
cleanExt, _ = detectExtension(fr)
}
// name
// ---
cleanName := inflector.Snakecase(strings.TrimSuffix(name, originalExt))
if length := len(cleanName); length < 3 {
// the name is too short so we concatenate an additional random part
cleanName += security.RandomString(10)
} else if length > 100 {
// keep only the first 100 characters (it is multibyte safe after Snakecase)
cleanName = cleanName[:100]
}
return fmt.Sprintf(
"%s_%s%s",
cleanName,
security.RandomString(10), // ensure that there is always a random part
cleanExt,
)
}
func detectExtension(fr FileReader) (string, error) {
// try to detect the extension from the mime type
r, err := fr.Open()
if err != nil {
return "", err
}
defer r.Close()
mt, _ := mimetype.DetectReader(r)
if err != nil {
return "", err
}
return mt.Extension(), nil
}
tools/filesystem/file_test.go
+79
View File
@@ -0,0 +1,79 @@
package filesystem_test
import (
"net/http/httptest"
"os"
"path/filepath"
"regexp"
"testing"
"github.com/labstack/echo/v5"
"github.com/pocketbase/pocketbase/tests"
"github.com/pocketbase/pocketbase/tools/filesystem"
)
func TestNewFileFromFromPath(t *testing.T) {
testDir := createTestDir(t)
defer os.RemoveAll(testDir)
// missing file
_, err := filesystem.NewFileFromPath("missing")
if err == nil {
t.Fatal("Expected error, got nil")
}
// existing file
originalName := "image_! noext"
normalizedNamePattern := regexp.QuoteMeta("image_noext_") + `\w{10}` + regexp.QuoteMeta(".png")
f, err := filesystem.NewFileFromPath(filepath.Join(testDir, originalName))
if err != nil {
t.Fatalf("Expected nil error, got %v", err)
}
if f.OriginalName != originalName {
t.Fatalf("Expected originalName %q, got %q", originalName, f.OriginalName)
}
if match, _ := regexp.Match(normalizedNamePattern, []byte(f.Name)); !match {
t.Fatalf("Expected Name to match %v, got %q (%v)", normalizedNamePattern, f.Name, err)
}
if f.Size != 73 {
t.Fatalf("Expected Size %v, got %v", 73, f.Size)
}
if _, ok := f.Reader.(*filesystem.PathReader); !ok {
t.Fatalf("Expected Reader to be PathReader, got %v", f.Reader)
}
}
func TestNewFileFromMultipart(t *testing.T) {
formData, mp, err := tests.MockMultipartData(nil, "test")
req := httptest.NewRequest("", "/", formData)
req.Header.Set(echo.HeaderContentType, mp.FormDataContentType())
req.ParseMultipartForm(32 << 20)
_, mh, err := req.FormFile("test")
if err != nil {
t.Fatal(err)
}
f, err := filesystem.NewFileFromMultipart(mh)
if err != nil {
t.Fatal(err)
}
originalNamePattern := regexp.QuoteMeta("tmpfile-") + `\w+` + regexp.QuoteMeta(".txt")
if match, _ := regexp.Match(originalNamePattern, []byte(f.OriginalName)); !match {
t.Fatalf("Expected OriginalName to match %v, got %q (%v)", originalNamePattern, f.OriginalName, err)
}
normalizedNamePattern := regexp.QuoteMeta("tmpfile_") + `\w+\_\w{10}` + regexp.QuoteMeta(".txt")
if match, _ := regexp.Match(normalizedNamePattern, []byte(f.Name)); !match {
t.Fatalf("Expected Name to match %v, got %q (%v)", normalizedNamePattern, f.Name, err)
}
if f.Size != 4 {
t.Fatalf("Expected Size %v, got %v", 4, f.Size)
}
if _, ok := f.Reader.(*filesystem.MultipartReader); !ok {
t.Fatalf("Expected Reader to be MultipartReader, got %v", f.Reader)
}
}
tools/filesystem/filesystem.go
+43 -1
View File
@@ -117,7 +117,49 @@ func (s *System) Upload(content []byte, fileKey string) error {
return w.Close()
}
// UploadMultipart upload the provided multipart file to the fileKey location.
// UploadFile uploads the provided multipart file to the fileKey location.
func (s *System) UploadFile(file *File, fileKey string) error {
f, err := file.Reader.Open()
if err != nil {
return err
}
defer f.Close()
mt, err := mimetype.DetectReader(f)
if err != nil {
return err
}
// rewind
f.Seek(0, io.SeekStart)
originalName := file.OriginalName
if len(originalName) > 255 {
// keep only the first 255 chars as a very rudimentary measure
// to prevent the metadata to grow too big in size
originalName = originalName[:255]
}
opts := &blob.WriterOptions{
ContentType: mt.String(),
Metadata: map[string]string{
"original_filename": originalName,
},
}
w, err := s.bucket.NewWriter(s.ctx, fileKey, opts)
if err != nil {
return err
}
if _, err := w.ReadFrom(f); err != nil {
w.Close()
return err
}
return w.Close()
}
// UploadMultipart uploads the provided multipart file to the fileKey location.
func (s *System) UploadMultipart(fh *multipart.FileHeader, fileKey string) error {
f, err := fh.Open()
if err != nil {
tools/filesystem/filesystem_test.go
+8 -2
View File
@@ -448,8 +448,7 @@ func createTestDir(t *testing.T) string {
if err != nil {
t.Fatal(err)
}
// tiny 1x1 png
imgRect := image.Rect(0, 0, 1, 1)
imgRect := image.Rect(0, 0, 1, 1) // tiny 1x1 png
png.Encode(file3, imgRect)
file3.Close()
err2 := os.WriteFile(filepath.Join(dir, "image.png.attrs"), []byte(`{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null}`), 0644)
@@ -469,5 +468,12 @@ func createTestDir(t *testing.T) string {
}
file5.Close()
file6, err := os.OpenFile(filepath.Join(dir, "image_! noext"), os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
t.Fatal(err)
}
png.Encode(file6, image.Rect(0, 0, 1, 1)) // tiny 1x1 png
file6.Close()
return dir
}
tools/rest/uploaded_file.go
+7 -68
View File
@@ -1,43 +1,18 @@
package rest
import (
"fmt"
"mime/multipart"
"net/http"
"path/filepath"
"regexp"
"strings"
"github.com/gabriel-vasile/mimetype"
"github.com/pocketbase/pocketbase/tools/inflector"
"github.com/pocketbase/pocketbase/tools/security"
"github.com/pocketbase/pocketbase/tools/filesystem"
)
// DefaultMaxMemory defines the default max memory bytes that
// will be used when parsing a form request body.
const DefaultMaxMemory = 32 << 20 // 32mb
var extensionInvalidCharsRegex = regexp.MustCompile(`[^\w\.\*\-\+\=\#]+`)
// UploadedFile defines a single multipart uploaded file instance.
type UploadedFile struct {
name string
header *multipart.FileHeader
}
// Name returns an assigned unique name to the uploaded file.
func (f *UploadedFile) Name() string {
return f.name
}
// Header returns the file header that comes with the multipart request.
func (f *UploadedFile) Header() *multipart.FileHeader {
return f.header
}
// FindUploadedFiles extracts all form files of `key` from a http request
// and returns a slice with `UploadedFile` instances (if any).
func FindUploadedFiles(r *http.Request, key string) ([]*UploadedFile, error) {
// FindUploadedFiles extracts all form files of "key" from a http request
// and returns a slice with filesystem.File instances (if any).
func FindUploadedFiles(r *http.Request, key string) ([]*filesystem.File, error) {
if r.MultipartForm == nil {
err := r.ParseMultipartForm(DefaultMaxMemory)
if err != nil {
@@ -49,51 +24,15 @@ func FindUploadedFiles(r *http.Request, key string) ([]*UploadedFile, error) {
return nil, http.ErrMissingFile
}
result := make([]*UploadedFile, 0, len(r.MultipartForm.File[key]))
result := make([]*filesystem.File, 0, len(r.MultipartForm.File[key]))
for _, fh := range r.MultipartForm.File[key] {
file, err := fh.Open()
file, err := filesystem.NewFileFromMultipart(fh)
if err != nil {
return nil, err
}
defer file.Close()
// extension
// ---
originalExt := filepath.Ext(fh.Filename)
sanitizedExt := extensionInvalidCharsRegex.ReplaceAllString(originalExt, "")
if sanitizedExt == "" {
// try to detect the extension from the mime type
mt, err := mimetype.DetectReader(file)
if err != nil {
return nil, err
}
sanitizedExt = mt.Extension()
}
// name
// ---
originalName := strings.TrimSuffix(fh.Filename, originalExt)
sanitizedName := inflector.Snakecase(originalName)
if length := len(sanitizedName); length < 3 {
// the name is too short so we concatenate an additional random part
sanitizedName += security.RandomString(10)
} else if length > 100 {
// keep only the first 100 characters (it is multibyte safe after Snakecase)
sanitizedName = sanitizedName[:100]
}
uploadedFilename := fmt.Sprintf(
"%s_%s%s",
sanitizedName,
security.RandomString(10), // ensure that there is always a random part
sanitizedExt,
)
result = append(result, &UploadedFile{
name: uploadedFilename,
header: fh,
})
result = append(result, file)
}
return result, nil
tools/rest/uploaded_file_test.go
+4 -4
View File
@@ -47,16 +47,16 @@ func TestFindUploadedFiles(t *testing.T) {
t.Errorf("[%d] Expected 1 file, got %d", i, len(result))
}
if result[0].Header().Size != 4 {
t.Errorf("[%d] Expected the file size to be 4 bytes, got %d", i, result[0].Header().Size)
if result[0].Size != 4 {
t.Errorf("[%d] Expected the file size to be 4 bytes, got %d", i, result[0].Size)
}
pattern, err := regexp.Compile(s.expectedPattern)
if err != nil {
t.Errorf("[%d] Invalid filename pattern %q: %v", i, s.expectedPattern, err)
}
if !pattern.MatchString(result[0].Name()) {
t.Fatalf("Expected filename to match %s, got filename %s", s.expectedPattern, result[0].Name())
if !pattern.MatchString(result[0].Name) {
t.Fatalf("Expected filename to match %s, got filename %s", s.expectedPattern, result[0].Name)
}
}
}
tools/types/datetime.go
+9 -5
View File
@@ -81,12 +81,16 @@ func (d *DateTime) Scan(value any) error {
case int:
d.t = cast.ToTime(v)
case string:
t, err := time.Parse(DefaultDateLayout, v)
if err != nil {
// check for other common date layouts
t = cast.ToTime(v)
if v == "" {
d.t = time.Time{}
} else {
t, err := time.Parse(DefaultDateLayout, v)
if err != nil {
// check for other common date layouts
t = cast.ToTime(v)
}
d.t = t
}
d.t = t
default:
str := cast.ToString(v)
if str == "" {