- commit
- 15dc5a8
- parent
- 1e436da
- author
- Eric Bower
- date
- 2024-02-18 14:40:20 +0000 UTC
refactor: use pobj (#76)
+1,
-1
1@@ -24,7 +24,7 @@ jobs:
2 - name: Set up Go
3 uses: actions/setup-go@v3
4 with:
5- go-version: 1.21
6+ go-version: 1.22
7 - name: Checkout repo
8 uses: actions/checkout@v3
9 - name: Lint the codebase
+1,
-1
1@@ -13,7 +13,7 @@ jobs:
2 fetch-depth: 0
3 - uses: actions/setup-go@v4
4 with:
5- go-version: '1.20'
6+ go-version: 1.22
7 - name: install pgit
8 run: |
9 go install github.com/picosh/pgit@latest
+1,
-1
1@@ -1,4 +1,4 @@
2-FROM --platform=$BUILDPLATFORM golang:1.21 as builder-deps
3+FROM --platform=$BUILDPLATFORM golang:1.22 as builder-deps
4 LABEL maintainer="Pico Maintainers <hello@pico.sh>"
5
6 WORKDIR /app
1@@ -43,7 +43,7 @@ func main() {
2 picoCfg.MinioPass = os.Getenv("MINIO_ROOT_PASSWORD")
3 picoDb := postgres.NewDB(picoCfg.DbURL, picoCfg.Logger)
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 var err error
8 st, err = storage.NewStorageMinio(picoCfg.MinioURL, picoCfg.MinioUser, picoCfg.MinioPass)
9 bail(err)
10@@ -66,7 +66,7 @@ func main() {
11
12 bucket, err := st.GetBucket(bucketName)
13 bail(err)
14- bucketProjects, err := st.ListFiles(bucket, "/", false)
15+ bucketProjects, err := st.ListObjects(bucket, "/", false)
16 bail(err)
17
18 userID := strings.Replace(bucketName, "static-", "", 1)
+1,
-1
1@@ -42,7 +42,7 @@ func StartApiServer() {
2 defer db.Close()
3 logger := cfg.Logger
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 var err error
8 if cfg.MinioURL == "" {
9 st, err = storage.NewStorageFS(cfg.StorageDir)
+1,
-1
1@@ -72,7 +72,7 @@ func StartSshServer() {
2 Db: dbh,
3 }
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 var err error
8 if cfg.MinioURL == "" {
9 st, err = storage.NewStorageFS(cfg.StorageDir)
+13,
-11
1@@ -17,6 +17,8 @@ import (
2 "github.com/picosh/pico/shared"
3 "github.com/picosh/pico/shared/storage"
4 "github.com/picosh/pico/wish/cms/util"
5+ "github.com/picosh/pobj"
6+ sst "github.com/picosh/pobj/storage"
7 "github.com/picosh/send/send/utils"
8 )
9
10@@ -33,8 +35,8 @@ func getProject(s ssh.Session) *db.Project {
11 return project
12 }
13
14-func getBucket(s ssh.Session) (storage.Bucket, error) {
15- bucket := s.Context().Value(ctxBucketKey{}).(storage.Bucket)
16+func getBucket(s ssh.Session) (sst.Bucket, error) {
17+ bucket := s.Context().Value(ctxBucketKey{}).(sst.Bucket)
18 if bucket.Name == "" {
19 return bucket, fmt.Errorf("bucket not set on `ssh.Context()` for connection")
20 }
21@@ -61,7 +63,7 @@ type FileData struct {
22 *utils.FileEntry
23 Text []byte
24 User *db.User
25- Bucket storage.Bucket
26+ Bucket sst.Bucket
27 StorageSize uint64
28 FeatureFlag *db.FeatureFlag
29 DeltaFileSize int64
30@@ -70,10 +72,10 @@ type FileData struct {
31 type UploadAssetHandler struct {
32 DBPool db.DB
33 Cfg *shared.ConfigSite
34- Storage storage.ObjectStorage
35+ Storage storage.StorageServe
36 }
37
38-func NewUploadAssetHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.ObjectStorage) *UploadAssetHandler {
39+func NewUploadAssetHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.StorageServe) *UploadAssetHandler {
40 return &UploadAssetHandler{
41 DBPool: dbpool,
42 Cfg: cfg,
43@@ -104,7 +106,7 @@ func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.Fil
44 }
45
46 fname := shared.GetAssetFileName(entry)
47- contents, size, modTime, err := h.Storage.GetFile(bucket, fname)
48+ contents, size, modTime, err := h.Storage.GetObject(bucket, fname)
49 if err != nil {
50 return nil, nil, err
51 }
52@@ -112,7 +114,7 @@ func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.Fil
53 fileInfo.FSize = size
54 fileInfo.FModTime = modTime
55
56- reader := shared.NewAllReaderAt(contents)
57+ reader := pobj.NewAllReaderAt(contents)
58
59 return fileInfo, reader, nil
60 }
61@@ -150,7 +152,7 @@ func (h *UploadAssetHandler) List(s ssh.Session, fpath string, isDir bool, recur
62 cleanFilename += "/"
63 }
64
65- foundList, err := h.Storage.ListFiles(bucket, cleanFilename, recursive)
66+ foundList, err := h.Storage.ListObjects(bucket, cleanFilename, recursive)
67 if err != nil {
68 return fileList, err
69 }
70@@ -271,7 +273,7 @@ func (h *UploadAssetHandler) Write(s ssh.Session, entry *utils.FileEntry) (strin
71 // calculate the filsize difference between the same file already
72 // stored and the updated file being uploaded
73 assetFilename := shared.GetAssetFileName(entry)
74- curFileSize, _ := h.Storage.GetFileSize(bucket, assetFilename)
75+ curFileSize, _ := h.Storage.GetObjectSize(bucket, assetFilename)
76 deltaFileSize := curFileSize - entry.Size
77
78 data := &FileData{
79@@ -373,7 +375,7 @@ func (h *UploadAssetHandler) writeAsset(data *FileData) error {
80 assetFilename := shared.GetAssetFileName(data.FileEntry)
81
82 if data.Size == 0 {
83- err = h.Storage.DeleteFile(data.Bucket, assetFilename)
84+ err = h.Storage.DeleteObject(data.Bucket, assetFilename)
85 if err != nil {
86 return err
87 }
88@@ -390,7 +392,7 @@ func (h *UploadAssetHandler) writeAsset(data *FileData) error {
89 assetFilename,
90 )
91
92- _, err := h.Storage.PutFile(
93+ _, err := h.Storage.PutObject(
94 data.Bucket,
95 assetFilename,
96 utils.NopReaderAtCloser(reader),
+5,
-4
1@@ -17,6 +17,7 @@ import (
2 "github.com/picosh/pico/filehandlers/util"
3 "github.com/picosh/pico/shared"
4 "github.com/picosh/pico/shared/storage"
5+ "github.com/picosh/pobj"
6 "github.com/picosh/send/send/utils"
7 )
8
9@@ -35,10 +36,10 @@ type PostMetaData struct {
10 type UploadImgHandler struct {
11 DBPool db.DB
12 Cfg *shared.ConfigSite
13- Storage storage.ObjectStorage
14+ Storage storage.StorageServe
15 }
16
17-func NewUploadImgHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.ObjectStorage) *UploadImgHandler {
18+func NewUploadImgHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.StorageServe) *UploadImgHandler {
19 return &UploadImgHandler{
20 DBPool: dbpool,
21 Cfg: cfg,
22@@ -92,12 +93,12 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileI
23 return nil, nil, err
24 }
25
26- contents, _, _, err := h.Storage.GetFile(bucket, post.Filename)
27+ contents, _, _, err := h.Storage.GetObject(bucket, post.Filename)
28 if err != nil {
29 return nil, nil, err
30 }
31
32- reader := shared.NewAllReaderAt(contents)
33+ reader := pobj.NewAllReaderAt(contents)
34
35 return fileInfo, reader, nil
36 }
+2,
-2
1@@ -56,7 +56,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
2
3 reader := bytes.NewReader([]byte(data.Text))
4
5- fname, err := h.Storage.PutFile(
6+ fname, err := h.Storage.PutObject(
7 bucket,
8 data.Filename,
9 utils.NopReaderAtCloser(reader),
10@@ -107,7 +107,7 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
11 if err != nil {
12 return err
13 }
14- err = h.Storage.DeleteFile(bucket, data.Filename)
15+ err = h.Storage.DeleteObject(bucket, data.Filename)
16 if err != nil {
17 return err
18 }
+1,
-1
1@@ -38,7 +38,7 @@ type ScpUploadHandler struct {
2 Hooks ScpFileHooks
3 }
4
5-func NewScpPostHandler(dbpool db.DB, cfg *shared.ConfigSite, hooks ScpFileHooks, st storage.ObjectStorage) *ScpUploadHandler {
6+func NewScpPostHandler(dbpool db.DB, cfg *shared.ConfigSite, hooks ScpFileHooks, st storage.StorageServe) *ScpUploadHandler {
7 return &ScpUploadHandler{
8 DBPool: dbpool,
9 Cfg: cfg,
+4,
-4
1@@ -11,8 +11,8 @@ type ctxUserKey struct{}
2 type ctxFeatureFlagKey struct{}
3
4 func GetUser(s ssh.Session) (*db.User, error) {
5- user := s.Context().Value(ctxUserKey{}).(*db.User)
6- if user == nil {
7+ user, ok := s.Context().Value(ctxUserKey{}).(*db.User)
8+ if !ok {
9 return user, fmt.Errorf("user not set on `ssh.Context()` for connection")
10 }
11 return user, nil
12@@ -23,8 +23,8 @@ func SetUser(s ssh.Session, user *db.User) {
13 }
14
15 func GetFeatureFlag(s ssh.Session) (*db.FeatureFlag, error) {
16- ff := s.Context().Value(ctxFeatureFlagKey{}).(*db.FeatureFlag)
17- if ff.Name == "" {
18+ ff, ok := s.Context().Value(ctxFeatureFlagKey{}).(*db.FeatureFlag)
19+ if !ok || ff.Name == "" {
20 return ff, fmt.Errorf("feature flag not set on `ssh.Context()` for connection")
21 }
22 return ff, nil
M
go.mod
+5,
-4
1@@ -1,6 +1,6 @@
2 module github.com/picosh/pico
3
4-go 1.21.4
5+go 1.22.0
6
7 require (
8 github.com/alecthomas/chroma v0.10.0
9@@ -15,13 +15,12 @@ require (
10 github.com/gorilla/feeds v1.1.2
11 github.com/lib/pq v1.10.9
12 github.com/microcosm-cc/bluemonday v1.0.26
13- github.com/minio/madmin-go/v3 v3.0.29
14- github.com/minio/minio-go/v7 v7.0.63
15 github.com/mmcdole/gofeed v1.2.1
16 github.com/muesli/reflow v0.3.0
17 github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577
18 github.com/patrickmn/go-cache v2.1.0+incompatible
19- github.com/picosh/send v0.0.0-20240217010313-c282075fbdf8
20+ github.com/picosh/pobj v0.0.0-20240217191723-5a89dec577a1
21+ github.com/picosh/send v0.0.0-20240217194807-77b972121e63
22 github.com/sendgrid/sendgrid-go v3.13.0+incompatible
23 github.com/yuin/goldmark v1.6.0
24 github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594
25@@ -75,7 +74,9 @@ require (
26 github.com/mattn/go-runewidth v0.0.15 // indirect
27 github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
28 github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
29+ github.com/minio/madmin-go/v3 v3.0.29 // indirect
30 github.com/minio/md5-simd v1.1.2 // indirect
31+ github.com/minio/minio-go/v7 v7.0.63 // indirect
32 github.com/minio/sha256-simd v1.0.1 // indirect
33 github.com/mmcdole/goxpp v1.1.0 // indirect
34 github.com/mmcloughlin/md4 v0.1.2 // indirect
M
go.sum
+4,
-2
1@@ -182,8 +182,10 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
2 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
3 github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
4 github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
5-github.com/picosh/send v0.0.0-20240217010313-c282075fbdf8 h1:yYsjCSE+SRMDVj7efPu4I048jevI42ZvaD0GQiRiRBI=
6-github.com/picosh/send v0.0.0-20240217010313-c282075fbdf8/go.mod h1:1JCq0NVOdTDenQ0/Kd8e4rP80lu06UHJJ+6dQxhcpew=
7+github.com/picosh/pobj v0.0.0-20240217191723-5a89dec577a1 h1:6aJBn2UsA0pWvwjV6C6/klcMzFUu6PpOfkI5DzHr6xo=
8+github.com/picosh/pobj v0.0.0-20240217191723-5a89dec577a1/go.mod h1:ILtZ0GOqkozrrGCvyJqCSUIwal2XQqzSfbKCNdS+HyU=
9+github.com/picosh/send v0.0.0-20240217194807-77b972121e63 h1:VSSbAejFzj2KBThfVnMcNXQwzHmwjPUridgi29LxihU=
10+github.com/picosh/send v0.0.0-20240217194807-77b972121e63/go.mod h1:1JCq0NVOdTDenQ0/Kd8e4rP80lu06UHJJ+6dQxhcpew=
11 github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
12 github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
13 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+1,
-1
1@@ -299,7 +299,7 @@ func StartApiServer() {
2 db := postgres.NewDB(cfg.DbURL, cfg.Logger)
3 defer db.Close()
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 var err error
8 if cfg.MinioURL == "" {
9 st, err = storage.NewStorageFS(cfg.StorageDir)
+1,
-1
1@@ -356,7 +356,7 @@ func StartApiServer() {
2 defer db.Close()
3 logger := cfg.Logger
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 var err error
8 if cfg.MinioURL == "" {
9 st, err = storage.NewStorageFS(cfg.StorageDir)
+1,
-1
1@@ -71,7 +71,7 @@ func StartSshServer() {
2 Db: dbh,
3 }
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 var err error
8 if cfg.MinioURL == "" {
9 st, err = storage.NewStorageFS(cfg.StorageDir)
+8,
-7
1@@ -20,6 +20,7 @@ import (
2 "github.com/picosh/pico/db/postgres"
3 "github.com/picosh/pico/shared"
4 "github.com/picosh/pico/shared/storage"
5+ sst "github.com/picosh/pobj/storage"
6 "github.com/picosh/send/send/utils"
7 )
8
9@@ -30,11 +31,11 @@ type AssetHandler struct {
10 ProjectDir string
11 Cfg *shared.ConfigSite
12 Dbpool db.DB
13- Storage storage.ObjectStorage
14+ Storage storage.StorageServe
15 Logger *slog.Logger
16 Cache *gocache.Cache
17 UserID string
18- Bucket storage.Bucket
19+ Bucket sst.Bucket
20 ImgProcessOpts *storage.ImgProcessOpts
21 }
22
23@@ -224,7 +225,7 @@ func calcPossibleRoutes(projectName, fp string, userRedirects []*RedirectRule) [
24
25 func (h *AssetHandler) handle(w http.ResponseWriter) {
26 var redirects []*RedirectRule
27- redirectFp, _, _, err := h.Storage.GetFile(h.Bucket, filepath.Join(h.ProjectDir, "_redirects"))
28+ redirectFp, _, _, err := h.Storage.GetObject(h.Bucket, filepath.Join(h.ProjectDir, "_redirects"))
29 if err == nil {
30 defer redirectFp.Close()
31 buf := new(strings.Builder)
32@@ -253,13 +254,13 @@ func (h *AssetHandler) handle(w http.ResponseWriter) {
33 var c io.ReadCloser
34 var err error
35 if strings.HasPrefix(mimeType, "image/") {
36- c, contentType, err = h.Storage.ServeFile(
37+ c, contentType, err = h.Storage.ServeObject(
38 h.Bucket,
39 fp.Filepath,
40 h.ImgProcessOpts,
41 )
42 } else {
43- c, _, _, err = h.Storage.GetFile(h.Bucket, fp.Filepath)
44+ c, _, _, err = h.Storage.GetObject(h.Bucket, fp.Filepath)
45 }
46 if err == nil {
47 contents = c
48@@ -337,7 +338,7 @@ func ServeAsset(fname string, opts *storage.ImgProcessOpts, fromImgs bool, w htt
49 // TODO: this could probably be cleaned up more
50 // imgs wont have a project directory
51 projectDir := ""
52- var bucket storage.Bucket
53+ var bucket sst.Bucket
54 // imgs has a different bucket directory
55 if fromImgs {
56 bucket, err = st.GetBucket(shared.GetImgsBucketName(user.ID))
57@@ -400,7 +401,7 @@ func StartApiServer() {
58 db := postgres.NewDB(cfg.DbURL, cfg.Logger)
59 defer db.Close()
60
61- var st storage.ObjectStorage
62+ var st storage.StorageServe
63 var err error
64 if cfg.MinioURL == "" {
65 st, err = storage.NewStorageFS(cfg.StorageDir)
+3,
-3
1@@ -66,7 +66,7 @@ type Cmd struct {
2 User *db.User
3 Session CmdSession
4 Log *slog.Logger
5- Store storage.ObjectStorage
6+ Store storage.StorageServe
7 Dbpool db.DB
8 Write bool
9 }
10@@ -103,7 +103,7 @@ func (c *Cmd) RmProjectAssets(projectName string) error {
11 }
12 c.output(fmt.Sprintf("removing project assets (%s)", projectName))
13
14- fileList, err := c.Store.ListFiles(bucket, projectName+"/", true)
15+ fileList, err := c.Store.ListObjects(bucket, projectName+"/", true)
16 if err != nil {
17 return err
18 }
19@@ -123,7 +123,7 @@ func (c *Cmd) RmProjectAssets(projectName string) error {
20 "filename", file.Name(),
21 )
22 if c.Write {
23- err = c.Store.DeleteFile(
24+ err = c.Store.DeleteObject(
25 bucket,
26 filepath.Join(projectName, file.Name()),
27 )
+2,
-2
1@@ -100,7 +100,7 @@ func CmsMiddleware(cfg *config.ConfigCms, urls config.ConfigURL) bm.Handler {
2
3 dbpool := postgres.NewDB(cfg.DbURL, cfg.Logger)
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 if cfg.MinioURL == "" {
8 st, err = storage.NewStorageFS(cfg.StorageDir)
9 } else {
10@@ -145,7 +145,7 @@ type model struct {
11 urls config.ConfigURL
12 publicKey string
13 dbpool db.DB
14- st storage.ObjectStorage
15+ st storage.StorageServe
16 user *db.User
17 err error
18 sshUser string
+1,
-1
1@@ -67,7 +67,7 @@ func StartSshServer() {
2 dbh := postgres.NewDB(cfg.DbURL, cfg.Logger)
3 defer dbh.Close()
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 var err error
8 if cfg.MinioURL == "" {
9 st, err = storage.NewStorageFS(cfg.StorageDir)
+1,
-1
1@@ -871,7 +871,7 @@ func StartApiServer() {
2 defer db.Close()
3 logger := cfg.Logger
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 var err error
8 if cfg.MinioURL == "" {
9 st, err = storage.NewStorageFS(cfg.StorageDir)
+1,
-1
1@@ -72,7 +72,7 @@ func StartSshServer() {
2 Db: dbh,
3 }
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 var err error
8 if cfg.MinioURL == "" {
9 st, err = storage.NewStorageFS(cfg.StorageDir)
1@@ -1,42 +0,0 @@
2-package shared
3-
4-import (
5- "errors"
6- "io"
7- "net/http"
8-
9- "github.com/minio/minio-go/v7"
10- "github.com/picosh/send/send/utils"
11-)
12-
13-type AllReaderAt struct {
14- Reader utils.ReaderAtCloser
15-}
16-
17-func NewAllReaderAt(reader utils.ReaderAtCloser) *AllReaderAt {
18- return &AllReaderAt{reader}
19-}
20-
21-func (a *AllReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
22- n, err = a.Reader.ReadAt(p, off)
23-
24- if errors.Is(err, io.EOF) {
25- return
26- }
27-
28- resp := minio.ToErrorResponse(err)
29-
30- if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
31- err = io.EOF
32- }
33-
34- return
35-}
36-
37-func (a *AllReaderAt) Read(p []byte) (int, error) {
38- return a.Reader.Read(p)
39-}
40-
41-func (a *AllReaderAt) Close() error {
42- return a.Reader.Close()
43-}
1@@ -46,7 +46,7 @@ func CreatePProfRoutes(routes []Route) []Route {
2
3 type ServeFn func(http.ResponseWriter, *http.Request)
4
5-func CreateServe(routes []Route, subdomainRoutes []Route, cfg *ConfigSite, dbpool db.DB, st storage.ObjectStorage, logger *slog.Logger, cache *cache.Cache) ServeFn {
6+func CreateServe(routes []Route, subdomainRoutes []Route, cfg *ConfigSite, dbpool db.DB, st storage.StorageServe, logger *slog.Logger, cache *cache.Cache) ServeFn {
7 return func(w http.ResponseWriter, r *http.Request) {
8 var allow []string
9 var subdomain string
10@@ -122,8 +122,8 @@ func GetDB(r *http.Request) db.DB {
11 return r.Context().Value(ctxDBKey{}).(db.DB)
12 }
13
14-func GetStorage(r *http.Request) storage.ObjectStorage {
15- return r.Context().Value(ctxStorageKey{}).(storage.ObjectStorage)
16+func GetStorage(r *http.Request) storage.StorageServe {
17+ return r.Context().Value(ctxStorageKey{}).(storage.StorageServe)
18 }
19
20 func GetField(r *http.Request, index int) string {
1@@ -3,114 +3,28 @@ package storage
2 import (
3 "fmt"
4 "io"
5- "io/fs"
6 "os"
7- "path"
8 "path/filepath"
9- "strings"
10- "time"
11
12- "github.com/picosh/send/send/utils"
13+ sst "github.com/picosh/pobj/storage"
14 )
15
16-// https://stackoverflow.com/a/32482941
17-func dirSize(path string) (int64, error) {
18- var size int64
19- err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
20- if err != nil {
21- return err
22- }
23- if !info.IsDir() {
24- size += info.Size()
25- }
26- return err
27- })
28-
29- return size, err
30-}
31-
32 type StorageFS struct {
33- Dir string
34+ *sst.StorageFS
35 }
36
37 func NewStorageFS(dir string) (*StorageFS, error) {
38- return &StorageFS{Dir: dir}, nil
39-}
40-
41-func (s *StorageFS) GetBucket(name string) (Bucket, error) {
42- dirPath := filepath.Join(s.Dir, name)
43- bucket := Bucket{
44- Name: name,
45- Path: dirPath,
46- }
47-
48- info, err := os.Stat(dirPath)
49- if os.IsNotExist(err) {
50- return bucket, fmt.Errorf("directory does not exist: %v %w", dirPath, err)
51- }
52-
53+ st, err := sst.NewStorageFS(dir)
54 if err != nil {
55- return bucket, fmt.Errorf("directory error: %v %w", dirPath, err)
56-
57+ return nil, err
58 }
59-
60- if !info.IsDir() {
61- return bucket, fmt.Errorf("directory is a file, not a directory: %#v", dirPath)
62- }
63-
64- return bucket, nil
65+ return &StorageFS{st}, nil
66 }
67
68-func (s *StorageFS) UpsertBucket(name string) (Bucket, error) {
69- bucket, err := s.GetBucket(name)
70- if err == nil {
71- return bucket, nil
72- }
73-
74- err = os.MkdirAll(bucket.Path, os.ModePerm)
75- if err != nil {
76- return bucket, err
77- }
78-
79- return bucket, nil
80-}
81-
82-func (s *StorageFS) GetBucketQuota(bucket Bucket) (uint64, error) {
83- dsize, err := dirSize(bucket.Path)
84- return uint64(dsize), err
85-}
86-
87-func (s *StorageFS) DeleteBucket(bucket Bucket) error {
88- return os.RemoveAll(bucket.Path)
89-}
90-
91-func (s *StorageFS) GetFileSize(bucket Bucket, fpath string) (int64, error) {
92- fi, err := os.Stat(filepath.Join(bucket.Path, fpath))
93- if err != nil {
94- return 0, err
95- }
96- size := fi.Size()
97- return size, nil
98-}
99-
100-func (s *StorageFS) GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser, int64, time.Time, error) {
101- dat, err := os.Open(filepath.Join(bucket.Path, fpath))
102- if err != nil {
103- return nil, 0, time.Time{}, err
104- }
105-
106- info, err := dat.Stat()
107- if err != nil {
108- return nil, 0, time.Time{}, err
109- }
110-
111- return dat, info.Size(), info.ModTime(), nil
112-}
113-
114-func (s *StorageFS) ServeFile(bucket Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, string, error) {
115+func (s *StorageFS) ServeObject(bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, string, error) {
116 if opts == nil || os.Getenv("IMGPROXY_URL") == "" {
117 contentType := GetMimeType(fpath)
118- rc, _, _, err := s.GetFile(bucket, fpath)
119+ rc, _, _, err := s.GetObject(bucket, fpath)
120 return rc, contentType, err
121 }
122
123@@ -118,105 +32,3 @@ func (s *StorageFS) ServeFile(bucket Bucket, fpath string, opts *ImgProcessOpts)
124 dataURL := fmt.Sprintf("local://%s", filePath)
125 return HandleProxy(dataURL, opts)
126 }
127-
128-func (s *StorageFS) PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error) {
129- loc := filepath.Join(bucket.Path, fpath)
130- err := os.MkdirAll(filepath.Dir(loc), os.ModePerm)
131- if err != nil {
132- return "", err
133- }
134- f, err := os.OpenFile(loc, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
135- if err != nil {
136- return "", err
137- }
138-
139- _, err = io.Copy(f, contents)
140- if err != nil {
141- return "", err
142- }
143-
144- f.Close()
145-
146- if entry.Mtime > 0 {
147- uTime := time.Unix(entry.Mtime, 0)
148- _ = os.Chtimes(loc, uTime, uTime)
149- }
150-
151- return loc, nil
152-}
153-
154-func (s *StorageFS) DeleteFile(bucket Bucket, fpath string) error {
155- loc := filepath.Join(bucket.Path, fpath)
156- err := os.Remove(loc)
157- if err != nil {
158- return err
159- }
160-
161- return nil
162-}
163-
164-func (s *StorageFS) ListBuckets() ([]string, error) {
165- return []string{}, fmt.Errorf("not implemented")
166-}
167-
168-func (s *StorageFS) ListFiles(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error) {
169- var fileList []os.FileInfo
170-
171- fpath := path.Join(bucket.Path, dir)
172-
173- info, err := os.Stat(fpath)
174- if err != nil {
175- return fileList, err
176- }
177-
178- if info.IsDir() && !strings.HasSuffix(dir, "/") {
179- fileList = append(fileList, &utils.VirtualFile{
180- FName: "",
181- FIsDir: info.IsDir(),
182- FSize: info.Size(),
183- FModTime: info.ModTime(),
184- })
185-
186- return fileList, err
187- }
188-
189- var files []fs.DirEntry
190-
191- if recursive {
192- err = filepath.WalkDir(fpath, func(s string, d fs.DirEntry, err error) error {
193- if err != nil {
194- return err
195- }
196- files = append(files, d)
197- return nil
198- })
199- if err != nil {
200- fileList = append(fileList, info)
201- return fileList, nil
202- }
203- } else {
204- files, err = os.ReadDir(fpath)
205- if err != nil {
206- fileList = append(fileList, info)
207- return fileList, nil
208- }
209- }
210-
211- for _, f := range files {
212- info, err := f.Info()
213- if err != nil {
214- return fileList, err
215- }
216-
217- i := &utils.VirtualFile{
218- FName: f.Name(),
219- FIsDir: f.IsDir(),
220- FSize: info.Size(),
221- FModTime: info.ModTime(),
222- }
223-
224- fileList = append(fileList, i)
225- }
226-
227- return fileList, err
228-}
1@@ -1,192 +1,30 @@
2 package storage
3
4 import (
5- "context"
6- "errors"
7 "fmt"
8 "io"
9- "net/url"
10 "os"
11 "path/filepath"
12- "strconv"
13- "strings"
14- "time"
15
16- "github.com/minio/madmin-go/v3"
17- "github.com/minio/minio-go/v7"
18- "github.com/minio/minio-go/v7/pkg/credentials"
19- "github.com/picosh/send/send/utils"
20+ sst "github.com/picosh/pobj/storage"
21 )
22
23 type StorageMinio struct {
24- Client *minio.Client
25- Admin *madmin.AdminClient
26+ *sst.StorageMinio
27 }
28
29 func NewStorageMinio(address, user, pass string) (*StorageMinio, error) {
30- endpoint, err := url.Parse(address)
31+ st, err := sst.NewStorageMinio(address, user, pass)
32 if err != nil {
33 return nil, err
34 }
35- ssl := endpoint.Scheme == "https"
36-
37- mClient, err := minio.New(endpoint.Host, &minio.Options{
38- Creds: credentials.NewStaticV4(user, pass, ""),
39- Secure: ssl,
40- })
41- if err != nil {
42- return nil, err
43- }
44-
45- aClient, err := madmin.New(
46- endpoint.Host,
47- user,
48- pass,
49- ssl,
50- )
51- if err != nil {
52- return nil, err
53- }
54-
55- mini := &StorageMinio{
56- Client: mClient,
57- Admin: aClient,
58- }
59- return mini, err
60-}
61-
62-func (s *StorageMinio) GetBucket(name string) (Bucket, error) {
63- bucket := Bucket{
64- Name: name,
65- }
66-
67- exists, err := s.Client.BucketExists(context.TODO(), bucket.Name)
68- if err != nil || !exists {
69- if err == nil {
70- err = errors.New("bucket does not exist")
71- }
72- return bucket, err
73- }
74-
75- return bucket, nil
76-}
77-
78-func (s *StorageMinio) UpsertBucket(name string) (Bucket, error) {
79- bucket, err := s.GetBucket(name)
80- if err == nil {
81- return bucket, nil
82- }
83-
84- err = s.Client.MakeBucket(context.TODO(), name, minio.MakeBucketOptions{})
85- if err != nil {
86- return bucket, err
87- }
88-
89- return bucket, nil
90-}
91-
92-func (s *StorageMinio) GetBucketQuota(bucket Bucket) (uint64, error) {
93- info, err := s.Admin.AccountInfo(context.TODO(), madmin.AccountOpts{})
94- if err != nil {
95- return 0, nil
96- }
97- for _, b := range info.Buckets {
98- if b.Name == bucket.Name {
99- return b.Size, nil
100- }
101- }
102-
103- return 0, fmt.Errorf("%s bucket not found in account info", bucket.Name)
104-}
105-
106-func (s *StorageMinio) ListBuckets() ([]string, error) {
107- bcks := []string{}
108- buckets, err := s.Client.ListBuckets(context.Background())
109- if err != nil {
110- return bcks, err
111- }
112- for _, bucket := range buckets {
113- bcks = append(bcks, bucket.Name)
114- }
115-
116- return bcks, nil
117-}
118-
119-func (s *StorageMinio) ListFiles(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error) {
120- var fileList []os.FileInfo
121-
122- resolved := strings.TrimPrefix(dir, "/")
123-
124- opts := minio.ListObjectsOptions{Prefix: resolved, Recursive: recursive}
125- for obj := range s.Client.ListObjects(context.Background(), bucket.Name, opts) {
126- if obj.Err != nil {
127- return fileList, obj.Err
128- }
129- isDir := false
130- if obj.Size == 0 {
131- isDir = true
132- }
133-
134- modTime := time.Time{}
135-
136- if mtime, ok := obj.UserMetadata["Mtime"]; ok {
137- mtimeUnix, err := strconv.Atoi(mtime)
138- if err == nil {
139- modTime = time.Unix(int64(mtimeUnix), 0)
140- }
141- }
142-
143- info := &utils.VirtualFile{
144- FName: strings.TrimSuffix(strings.TrimPrefix(obj.Key, resolved), "/"),
145- FIsDir: isDir,
146- FSize: obj.Size,
147- FModTime: modTime,
148- }
149- fileList = append(fileList, info)
150- }
151-
152- return fileList, nil
153-}
154-
155-func (s *StorageMinio) DeleteBucket(bucket Bucket) error {
156- return s.Client.RemoveBucket(context.TODO(), bucket.Name)
157-}
158-
159-func (s *StorageMinio) GetFileSize(bucket Bucket, fpath string) (int64, error) {
160- info, err := s.Client.StatObject(context.Background(), bucket.Name, fpath, minio.StatObjectOptions{})
161- if err != nil {
162- return 0, err
163- }
164- return info.Size, nil
165-}
166-
167-func (s *StorageMinio) GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser, int64, time.Time, error) {
168- modTime := time.Time{}
169-
170- info, err := s.Client.StatObject(context.Background(), bucket.Name, fpath, minio.StatObjectOptions{})
171- if err != nil {
172- return nil, 0, modTime, err
173- }
174-
175- obj, err := s.Client.GetObject(context.Background(), bucket.Name, fpath, minio.GetObjectOptions{})
176- if err != nil {
177- return nil, 0, modTime, err
178- }
179-
180- if mtime, ok := info.UserMetadata["Mtime"]; ok {
181- mtimeUnix, err := strconv.Atoi(mtime)
182- if err == nil {
183- modTime = time.Unix(int64(mtimeUnix), 0)
184- }
185- }
186-
187- return obj, info.Size, modTime, nil
188+ return &StorageMinio{st}, nil
189 }
190
191-func (s *StorageMinio) ServeFile(bucket Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, string, error) {
192+func (s *StorageMinio) ServeObject(bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, string, error) {
193 if opts == nil || os.Getenv("IMGPROXY_URL") == "" {
194 contentType := GetMimeType(fpath)
195- rc, _, _, err := s.GetFile(bucket, fpath)
196+ rc, _, _, err := s.GetObject(bucket, fpath)
197 return rc, contentType, err
198 }
199
200@@ -194,26 +32,3 @@ func (s *StorageMinio) ServeFile(bucket Bucket, fpath string, opts *ImgProcessOp
201 dataURL := fmt.Sprintf("s3://%s", filePath)
202 return HandleProxy(dataURL, opts)
203 }
204-
205-func (s *StorageMinio) PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error) {
206- opts := minio.PutObjectOptions{}
207-
208- if entry.Mtime > 0 {
209- opts.UserMetadata = map[string]string{
210- "Mtime": fmt.Sprint(entry.Mtime),
211- }
212- }
213-
214- info, err := s.Client.PutObject(context.TODO(), bucket.Name, fpath, contents, -1, opts)
215-
216- if err != nil {
217- return "", err
218- }
219-
220- return fmt.Sprintf("%s/%s", info.Bucket, info.Key), nil
221-}
222-
223-func (s *StorageMinio) DeleteFile(bucket Bucket, fpath string) error {
224- err := s.Client.RemoveObject(context.TODO(), bucket.Name, fpath, minio.RemoveObjectOptions{})
225- return err
226-}
1@@ -2,29 +2,11 @@ package storage
2
3 import (
4 "io"
5- "os"
6- "time"
7
8- "github.com/picosh/send/send/utils"
9+ sst "github.com/picosh/pobj/storage"
10 )
11
12-type Bucket struct {
13- Name string
14- Path string
15- Root string
16-}
17-
18-type ObjectStorage interface {
19- GetBucket(name string) (Bucket, error)
20- UpsertBucket(name string) (Bucket, error)
21- ListBuckets() ([]string, error)
22-
23- DeleteBucket(bucket Bucket) error
24- GetBucketQuota(bucket Bucket) (uint64, error)
25- GetFileSize(bucket Bucket, fpath string) (int64, error)
26- GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser, int64, time.Time, error)
27- ServeFile(bucket Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, string, error)
28- PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error)
29- DeleteFile(bucket Bucket, fpath string) error
30- ListFiles(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error)
31+type StorageServe interface {
32+ sst.ObjectStorage
33+ ServeObject(bucket sst.Bucket, fpath string, opts *ImgProcessOpts) (io.ReadCloser, string, error)
34 }
+2,
-2
1@@ -106,7 +106,7 @@ func Middleware(cfg *config.ConfigCms, urls config.ConfigURL) bm.Handler {
2
3 dbpool := postgres.NewDB(cfg.DbURL, cfg.Logger)
4
5- var st storage.ObjectStorage
6+ var st storage.StorageServe
7 if cfg.MinioURL == "" {
8 st, err = storage.NewStorageFS(cfg.StorageDir)
9 } else {
10@@ -151,7 +151,7 @@ type model struct {
11 urls config.ConfigURL
12 publicKey string
13 dbpool db.DB
14- st storage.ObjectStorage
15+ st storage.StorageServe
16 user *db.User
17 err error
18 sshUser string
+3,
-3
1@@ -50,7 +50,7 @@ type Model struct {
2 cfg *config.ConfigCms
3 urls config.ConfigURL
4 dbpool db.DB
5- st storage.ObjectStorage
6+ st storage.StorageServe
7 user *db.User
8 posts []*db.Post
9 styles common.Styles
10@@ -83,7 +83,7 @@ func (m *Model) UpdatePaging(msg tea.Msg) {
11 }
12
13 // NewModel creates a new model with defaults.
14-func NewModel(cfg *config.ConfigCms, urls config.ConfigURL, dbpool db.DB, user *db.User, stor storage.ObjectStorage, perPage int) Model {
15+func NewModel(cfg *config.ConfigCms, urls config.ConfigURL, dbpool db.DB, user *db.User, stor storage.StorageServe, perPage int) Model {
16 logger := cfg.Logger
17 st := common.DefaultStyles()
18
19@@ -339,7 +339,7 @@ func removePost(m Model) tea.Cmd {
20 return errMsg{err}
21 }
22
23- err = m.st.DeleteFile(bucket, m.posts[m.getSelectedIndex()].Filename)
24+ err = m.st.DeleteObject(bucket, m.posts[m.getSelectedIndex()].Filename)
25 if err != nil {
26 return errMsg{err}
27 }