- commit
- b144c29
- parent
- 00dc4ec
- author
- Antonio Mika
- date
- 2024-05-29 20:12:04 +0000 UTC
Added support for sshfs
12 files changed,
+245,
-96
+1,
-0
1@@ -1,6 +1,7 @@
2 DATABASE_URL=postgresql://postgres:secret@postgres:5432/pico?sslmode=disable
3 POSTGRES_PASSWORD=secret
4 CF_API_TOKEN=secret
5+PICO_SECRET=secret
6 REGISTRY_URL=registry:5000
7 PICO_SECRET=""
8 PICO_SECRET_WEBHOOK=""
+1,
-1
1@@ -1553,7 +1553,7 @@ func (me *PsqlDB) FindFeedItemsByPostID(postID string) ([]*db.FeedItem, error) {
2
3 func (me *PsqlDB) InsertProject(userID, name, projectDir string) (string, error) {
4 if !shared.IsValidSubdomain(name) {
5- return "", fmt.Errorf("(%s) is not a valid project name, must match /^[a-z0-9-]+$/", name)
6+ return "", fmt.Errorf("'%s' is not a valid project name, must match /^[a-z0-9-]+$/", name)
7 }
8
9 var id string
+0,
-4
1@@ -218,10 +218,6 @@ func (f *Fetcher) ParseURL(fp *gofeed.Parser, url string) (*gofeed.Feed, error)
2 return nil, fmt.Errorf("fetching feed resulted in an error: %s %s", resp.Status, body)
3 }
4
5- if err != nil {
6- return nil, err
7- }
8-
9 feed, err := fp.ParseString(string(body))
10
11 if err != nil {
+103,
-31
1@@ -5,9 +5,12 @@ import (
2 "encoding/binary"
3 "fmt"
4 "io"
5+ "io/fs"
6 "log/slog"
7 "os"
8+ "path"
9 "path/filepath"
10+ "slices"
11 "strings"
12 "time"
13
14@@ -277,20 +280,16 @@ func (h *UploadAssetHandler) Write(s ssh.Session, entry *utils.FileEntry) (strin
15 h.Cfg.Logger.Error("user not found in ctx", "err", err.Error())
16 return "", err
17 }
18+
19+ if entry.Mode.IsDir() && strings.Count(entry.Filepath, "/") == 1 {
20+ entry.Filepath = strings.TrimPrefix(entry.Filepath, "/")
21+ }
22+
23 logger := h.GetLogger().With(
24 "user", user.Name,
25 "file", entry.Filepath,
26 )
27
28- var origText []byte
29- if b, err := io.ReadAll(entry.Reader); err == nil {
30- origText = b
31- }
32- fileSize := binary.Size(origText)
33- // TODO: hack for now until I figure out how to get correct
34- // filesize from sftp,scp,rsync
35- entry.Size = int64(fileSize)
36-
37 bucket, err := getBucket(s)
38 if err != nil {
39 logger.Error("could not find bucket in ctx", "err", err.Error())
40@@ -325,11 +324,31 @@ func (h *UploadAssetHandler) Write(s ssh.Session, entry *utils.FileEntry) (strin
41 setProject(s, project)
42 }
43
44+ if entry.Mode.IsDir() {
45+ _, err := h.Storage.PutObject(
46+ bucket,
47+ path.Join(shared.GetAssetFileName(entry), "._pico_keep_dir"),
48+ utils.NopReaderAtCloser(bytes.NewReader([]byte{})),
49+ entry,
50+ )
51+ return "", err
52+ }
53+
54+ var origText []byte
55+ if b, err := io.ReadAll(entry.Reader); err == nil {
56+ origText = b
57+ }
58+ fileSize := binary.Size(origText)
59+ // TODO: hack for now until I figure out how to get correct
60+ // filesize from sftp,scp,rsync
61+ entry.Size = int64(fileSize)
62+
63 storageSize := getStorageSize(s)
64 featureFlag, err := futil.GetFeatureFlag(s)
65 if err != nil {
66 return "", err
67 }
68+
69 // calculate the filsize difference between the same file already
70 // stored and the updated file being uploaded
71 assetFilename := shared.GetAssetFileName(entry)
72@@ -389,6 +408,66 @@ func (h *UploadAssetHandler) Write(s ssh.Session, entry *utils.FileEntry) (strin
73 return str, nil
74 }
75
76+func (h *UploadAssetHandler) Delete(s ssh.Session, entry *utils.FileEntry) error {
77+ user, err := futil.GetUser(s)
78+ if err != nil {
79+ h.Cfg.Logger.Error("user not found in ctx", "err", err.Error())
80+ return err
81+ }
82+
83+ if entry.Mode.IsDir() && strings.Count(entry.Filepath, "/") == 1 {
84+ entry.Filepath = strings.TrimPrefix(entry.Filepath, "/")
85+ }
86+
87+ assetFilepath := shared.GetAssetFileName(entry)
88+
89+ logger := h.GetLogger().With(
90+ "user", user.Name,
91+ "file", assetFilepath,
92+ )
93+
94+ bucket, err := getBucket(s)
95+ if err != nil {
96+ logger.Error("could not find bucket in ctx", "err", err.Error())
97+ return err
98+ }
99+
100+ projectName := shared.GetProjectName(entry)
101+ logger = logger.With("project", projectName)
102+
103+ if assetFilepath == filepath.Join("/", projectName, "._pico_keep_dir") {
104+ return os.ErrPermission
105+ }
106+
107+ logger.Info("deleting file")
108+
109+ pathDir := filepath.Dir(assetFilepath)
110+ fileName := filepath.Base(assetFilepath)
111+
112+ sibs, err := h.Storage.ListObjects(bucket, pathDir+"/", false)
113+ if err != nil {
114+ return err
115+ }
116+
117+ sibs = slices.DeleteFunc(sibs, func(sib fs.FileInfo) bool {
118+ return sib.Name() == fileName
119+ })
120+
121+ if len(sibs) == 0 {
122+ _, err := h.Storage.PutObject(
123+ bucket,
124+ filepath.Join(pathDir, "._pico_keep_dir"),
125+ utils.NopReaderAtCloser(bytes.NewReader([]byte{})),
126+ entry,
127+ )
128+ if err != nil {
129+ return err
130+ }
131+ }
132+
133+ return h.Storage.DeleteObject(bucket, assetFilepath)
134+}
135+
136 func (h *UploadAssetHandler) validateAsset(data *FileData) (bool, error) {
137 storageMax := data.FeatureFlag.Data.StorageMax
138 var nextStorageSize uint64
139@@ -447,30 +526,23 @@ func (h *UploadAssetHandler) validateAsset(data *FileData) (bool, error) {
140 func (h *UploadAssetHandler) writeAsset(data *FileData) error {
141 assetFilepath := shared.GetAssetFileName(data.FileEntry)
142
143- if data.Size == 0 {
144- err := h.Storage.DeleteObject(data.Bucket, assetFilepath)
145- if err != nil {
146- return err
147- }
148- } else {
149- reader := bytes.NewReader(data.Text)
150+ reader := bytes.NewReader(data.Text)
151
152- h.Cfg.Logger.Info(
153- "uploading file to bucket",
154- "user", data.User.Name,
155- "bucket", data.Bucket.Name,
156- "filename", assetFilepath,
157- )
158+ h.Cfg.Logger.Info(
159+ "uploading file to bucket",
160+ "user", data.User.Name,
161+ "bucket", data.Bucket.Name,
162+ "filename", assetFilepath,
163+ )
164
165- _, err := h.Storage.PutObject(
166- data.Bucket,
167- assetFilepath,
168- utils.NopReaderAtCloser(reader),
169- data.FileEntry,
170- )
171- if err != nil {
172- return err
173- }
174+ _, err := h.Storage.PutObject(
175+ data.Bucket,
176+ assetFilepath,
177+ utils.NopReaderAtCloser(reader),
178+ data.FileEntry,
179+ )
180+ if err != nil {
181+ return err
182 }
183
184 return nil
+44,
-17
1@@ -47,23 +47,6 @@ func NewUploadImgHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.S
2 }
3 }
4
5-func (h *UploadImgHandler) removePost(data *PostMetaData) error {
6- // skip empty files from being added to db
7- if data.Post == nil {
8- h.Cfg.Logger.Info("file is empty, skipping record", "filename", data.Filename)
9- return nil
10- }
11-
12- h.Cfg.Logger.Info("file is empty, removing record", "filename", data.Filename, "recordId", data.Cur.ID)
13- err := h.DBPool.RemovePosts([]string{data.Cur.ID})
14- if err != nil {
15- h.Cfg.Logger.Error(err.Error(), "filename", data.Filename)
16- return fmt.Errorf("error for %s: %v", data.Filename, err)
17- }
18-
19- return nil
20-}
21-
22 func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
23 user, err := util.GetUser(s)
24 if err != nil {
25@@ -205,3 +188,47 @@ func (h *UploadImgHandler) Write(s ssh.Session, entry *utils.FileEntry) (string,
26 )
27 return str, nil
28 }
29+
30+func (h *UploadImgHandler) Delete(s ssh.Session, entry *utils.FileEntry) error {
31+ user, err := util.GetUser(s)
32+ if err != nil {
33+ return err
34+ }
35+
36+ filename := filepath.Base(entry.Filepath)
37+
38+ logger := h.Cfg.Logger.With(
39+ "user", user.Name,
40+ "filename", filename,
41+ )
42+
43+ post, err := h.DBPool.FindPostWithFilename(
44+ filename,
45+ user.ID,
46+ Space,
47+ )
48+ if err != nil {
49+ logger.Info("unable to find image, continuing", "err", err.Error())
50+ return err
51+ }
52+
53+ err = h.DBPool.RemovePosts([]string{post.ID})
54+ if err != nil {
55+ logger.Error("error removing image", err)
56+ return fmt.Errorf("error for %s: %v", filename, err)
57+ }
58+
59+ bucket, err := h.Storage.UpsertBucket(user.ID)
60+ if err != nil {
61+ return err
62+ }
63+
64+ err = h.Storage.DeleteObject(bucket, filename)
65+ if err != nil {
66+ return err
67+ }
68+
69+ logger.Info("deleting image")
70+
71+ return nil
72+}
+7,
-16
1@@ -91,27 +91,18 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
2 return err
3 }
4
5- modTime := time.Unix(data.Mtime, 0)
6+ modTime := time.Now()
7+
8+ if data.Mtime > 0 {
9+ modTime = time.Unix(data.Mtime, 0)
10+ }
11+
12 logger := h.Cfg.Logger.With(
13 "user", data.Username,
14 "filename", data.Filename,
15 )
16
17- if len(data.OrigText) == 0 {
18- err = h.removePost(data)
19- if err != nil {
20- return err
21- }
22-
23- bucket, err := h.Storage.UpsertBucket(data.User.ID)
24- if err != nil {
25- return err
26- }
27- err = h.Storage.DeleteObject(bucket, data.Filename)
28- if err != nil {
29- return err
30- }
31- } else if data.Cur == nil {
32+ if data.Cur == nil {
33 logger.Info("file not found, adding record")
34 insertPost := db.Post{
35 UserID: user.ID,
+43,
-15
1@@ -89,6 +89,10 @@ func (h *ScpUploadHandler) Write(s ssh.Session, entry *utils.FileEntry) (string,
2 "filename", filename,
3 )
4
5+ if entry.Mode.IsDir() {
6+ return "", fmt.Errorf("file entry is directory, but only files are supported: %s", filename)
7+ }
8+
9 var origText []byte
10 if b, err := io.ReadAll(entry.Reader); err == nil {
11 origText = b
12@@ -144,23 +148,14 @@ func (h *ScpUploadHandler) Write(s ssh.Session, entry *utils.FileEntry) (string,
13 return "", err
14 }
15
16- modTime := time.Unix(entry.Mtime, 0)
17+ modTime := time.Now()
18
19- // if the file is empty we remove it from our database
20- if len(origText) == 0 {
21- // skip empty files from being added to db
22- if post == nil {
23- logger.Info("file is empty, skipping record")
24- return "", nil
25- }
26+ if entry.Mtime > 0 {
27+ modTime = time.Unix(entry.Mtime, 0)
28+ }
29
30- err := h.DBPool.RemovePosts([]string{post.ID})
31- logger.Info("file is empty, removing record")
32- if err != nil {
33- logger.Error(err.Error())
34- return "", fmt.Errorf("error for %s: %v", filename, err)
35- }
36- } else if post == nil {
37+ // if the file is empty we remove it from our database
38+ if post == nil {
39 logger.Info("file not found, adding record")
40 insertPost := db.Post{
41 UserID: userID,
42@@ -264,3 +259,36 @@ func (h *ScpUploadHandler) Write(s ssh.Session, entry *utils.FileEntry) (string,
43 curl := shared.NewCreateURL(h.Cfg)
44 return h.Cfg.FullPostURL(curl, user.Name, metadata.Slug), nil
45 }
46+
47+func (h *ScpUploadHandler) Delete(s ssh.Session, entry *utils.FileEntry) error {
48+ logger := h.Cfg.Logger
49+ user, err := util.GetUser(s)
50+ if err != nil {
51+ logger.Error(err.Error())
52+ return err
53+ }
54+
55+ userID := user.ID
56+ filename := filepath.Base(entry.Filepath)
57+ logger = logger.With(
58+ "user", user.Name,
59+ "filename", filename,
60+ )
61+
62+ post, err := h.DBPool.FindPostWithFilename(filename, userID, h.Cfg.Space)
63+ if err != nil {
64+ return err
65+ }
66+
67+ if post == nil {
68+ return os.ErrNotExist
69+ }
70+
71+ err = h.DBPool.RemovePosts([]string{post.ID})
72+ logger.Info("removing record")
73+ if err != nil {
74+ logger.Error(err.Error())
75+ return fmt.Errorf("error for %s: %v", filename, err)
76+ }
77+ return nil
78+}
+21,
-1
1@@ -1,6 +1,8 @@
2 package filehandlers
3
4 import (
5+ "database/sql"
6+ "errors"
7 "fmt"
8 "log/slog"
9 "os"
10@@ -16,6 +18,7 @@ import (
11 type ReadWriteHandler interface {
12 Write(ssh.Session, *utils.FileEntry) (string, error)
13 Read(ssh.Session, *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error)
14+ Delete(ssh.Session, *utils.FileEntry) error
15 }
16
17 type FileHandlerRouter struct {
18@@ -51,6 +54,10 @@ func (r *FileHandlerRouter) findHandler(entry *utils.FileEntry) (ReadWriteHandle
19 }
20
21 func (r *FileHandlerRouter) Write(s ssh.Session, entry *utils.FileEntry) (string, error) {
22+ if entry.Mode.IsDir() {
23+ return "", os.ErrInvalid
24+ }
25+
26 handler, err := r.findHandler(entry)
27 if err != nil {
28 return "", err
29@@ -58,6 +65,14 @@ func (r *FileHandlerRouter) Write(s ssh.Session, entry *utils.FileEntry) (string
30 return handler.Write(s, entry)
31 }
32
33+func (r *FileHandlerRouter) Delete(s ssh.Session, entry *utils.FileEntry) error {
34+ handler, err := r.findHandler(entry)
35+ if err != nil {
36+ return err
37+ }
38+ return handler.Delete(s, entry)
39+}
40+
41 func (r *FileHandlerRouter) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
42 handler, err := r.findHandler(entry)
43 if err != nil {
44@@ -98,6 +113,7 @@ func BaseList(s ssh.Session, fpath string, isDir bool, recursive bool, spaces []
45 }
46 } else {
47 for _, space := range spaces {
48+
49 p, e := dbpool.FindPostWithFilename(cleanFilename, user.ID, space)
50 if e != nil {
51 err = e
52@@ -109,11 +125,15 @@ func BaseList(s ssh.Session, fpath string, isDir bool, recursive bool, spaces []
53 posts = append(posts, post)
54 }
55
56- if err != nil {
57+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
58 return nil, err
59 }
60
61 for _, post := range posts {
62+ if post == nil {
63+ continue
64+ }
65+
66 fileList = append(fileList, &utils.VirtualFile{
67 FName: post.Filename,
68 FIsDir: false,
M
go.mod
+10,
-5
1@@ -1,8 +1,13 @@
2 module github.com/picosh/pico
3
4-go 1.21.9
5+go 1.22
6+
7+// replace github.com/picosh/ptun => ../ptun
8+
9+// replace github.com/picosh/send => ../send
10+
11+// replace github.com/picosh/pobj => ../pobj
12
13-// replace github.com/picosh/ptun => /home/erock/dev/pico/ptun
14 replace git.sr.ht/~delthas/senpai => github.com/picosh/senpai v0.0.0-20240503200611-af89e73973b0
15
16 replace github.com/gdamore/tcell/v2 => github.com/delthas/tcell/v2 v2.4.1-0.20230710100648-1489e78d90fb
17@@ -26,9 +31,9 @@ require (
18 github.com/mmcdole/gofeed v1.3.0
19 github.com/muesli/reflow v0.3.0
20 github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577
21- github.com/picosh/pobj v0.0.0-20240417140600-2071618b61c5
22- github.com/picosh/ptun v0.0.0-20240417140706-811cc2b70d9a
23- github.com/picosh/send v0.0.0-20240217194807-77b972121e63
24+ github.com/picosh/pobj v0.0.0-20240529200402-7b5398cf8a9f
25+ github.com/picosh/ptun v0.0.0-20240529133708-fcf1376b935e
26+ github.com/picosh/send v0.0.0-20240529200640-3667d1ad154e
27 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
28 github.com/sendgrid/sendgrid-go v3.14.0+incompatible
29 github.com/simplesurance/go-ip-anonymizer v0.0.0-20200429124537-35a880f8e87d
M
go.sum
+6,
-6
1@@ -213,12 +213,12 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N
2 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
3 github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
4 github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
5-github.com/picosh/pobj v0.0.0-20240417140600-2071618b61c5 h1:iS9zagScak8DCVMCXX5Rvdk7OOQzWUwMPiomrbHCks8=
6-github.com/picosh/pobj v0.0.0-20240417140600-2071618b61c5/go.mod h1:ctpMgMVUAspcK6kjgpEX5WWHGhDw04l6lDd8s4cI3Uo=
7-github.com/picosh/ptun v0.0.0-20240417140706-811cc2b70d9a h1:sBqfT6KBIYllVaw4bT1MQtFvuPNINdCgol5yttbuGsg=
8-github.com/picosh/ptun v0.0.0-20240417140706-811cc2b70d9a/go.mod h1:WXCrhe0l9VL3ji0pdhvSJD6LLx99rJoAA/+PUQXf0Mo=
9-github.com/picosh/send v0.0.0-20240217194807-77b972121e63 h1:VSSbAejFzj2KBThfVnMcNXQwzHmwjPUridgi29LxihU=
10-github.com/picosh/send v0.0.0-20240217194807-77b972121e63/go.mod h1:1JCq0NVOdTDenQ0/Kd8e4rP80lu06UHJJ+6dQxhcpew=
11+github.com/picosh/pobj v0.0.0-20240529200402-7b5398cf8a9f h1:9Y0xaTqq/7JiW7iUX4jSgJkN81X2gQucVqBPVXJDux0=
12+github.com/picosh/pobj v0.0.0-20240529200402-7b5398cf8a9f/go.mod h1:CViLWaCp2KP/zJdd7miNPkJb6i0v9HOgZ2wdbQuxCrQ=
13+github.com/picosh/ptun v0.0.0-20240529133708-fcf1376b935e h1:Um9aCUg1ysiUaB0nh3400UHlFAnhd8BXBsawqePxxqQ=
14+github.com/picosh/ptun v0.0.0-20240529133708-fcf1376b935e/go.mod h1:WXCrhe0l9VL3ji0pdhvSJD6LLx99rJoAA/+PUQXf0Mo=
15+github.com/picosh/send v0.0.0-20240529200640-3667d1ad154e h1:2NMuieR/7GIjiGYNPQsh6KOJiz2WhzU5ispxQCXmOyU=
16+github.com/picosh/send v0.0.0-20240529200640-3667d1ad154e/go.mod h1:V418obz9YdzjS3/oFzyDFzmPDnLu1nvy3wkLaixiT84=
17 github.com/picosh/senpai v0.0.0-20240503200611-af89e73973b0 h1:pBRIbiCj7K6rGELijb//dYhyCo8A3fvxW5dijrJVtjs=
18 github.com/picosh/senpai v0.0.0-20240503200611-af89e73973b0/go.mod h1:QaBDtybFC5gz7EG/9c3bgzuyW7W5W2rYLFZxWNuWk3Q=
19 github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
+5,
-0
1@@ -2,6 +2,7 @@ package pico
2
3 import (
4 "bytes"
5+ "errors"
6 "fmt"
7 "io"
8 "log/slog"
9@@ -50,6 +51,10 @@ func (h *UploadHandler) getAuthorizedKeyFile(user *db.User) (*utils.VirtualFile,
10 return fileInfo, text, nil
11 }
12
13+func (h *UploadHandler) Delete(s ssh.Session, entry *utils.FileEntry) error {
14+ return errors.New("unsupported")
15+}
16+
17 func (h *UploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
18 user, err := util.GetUser(s)
19 if err != nil {
1@@ -18,6 +18,10 @@ func GetAssetBucketName(userID string) string {
2 }
3
4 func GetProjectName(entry *utils.FileEntry) string {
5+ if entry.Mode.IsDir() && strings.Count(entry.Filepath, string(os.PathSeparator)) == 0 {
6+ return entry.Filepath
7+ }
8+
9 dir := filepath.Dir(entry.Filepath)
10 list := strings.Split(dir, string(os.PathSeparator))
11 return list[1]