- commit
- db77579
- parent
- a8ab7f3
- author
- Antonio Mika
- date
- 2023-11-10 17:42:53 +0000 UTC
Merge pull request #51 from picosh/am/rsync-updates Am/rsync updates
+30,
-0
1@@ -0,0 +1,30 @@
2+{
3+ // Use IntelliSense to learn about possible attributes.
4+ // Hover to view descriptions of existing attributes.
5+ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
6+ "version": "0.2.0",
7+ "configurations": [
8+ {
9+ "name": "Debug App",
10+ "type": "go",
11+ "request": "launch",
12+ "mode": "auto",
13+ "program": "${workspaceFolder}/cmd/${input:service}/${input:type}/main.go",
14+ "envFile": "${workspaceFolder}/.env"
15+ }
16+ ],
17+ "inputs": [
18+ {
19+ "id": "service",
20+ "type": "promptString",
21+ "description": "The service to debug",
22+ "default": "pgs"
23+ },
24+ {
25+ "id": "type",
26+ "type": "promptString",
27+ "description": "The service type to debug",
28+ "default": "ssh"
29+ }
30+ ]
31+}
+4,
-2
1@@ -9,6 +9,7 @@ import (
2 "github.com/picosh/pico/imgs"
3 "github.com/picosh/pico/shared"
4 "github.com/picosh/pico/shared/storage"
5+ "github.com/picosh/pico/wish/send/utils"
6 )
7
8 func main() {
9@@ -42,7 +43,7 @@ func main() {
10 continue
11 }
12
13- reader, err := st.GetFile(bucket, post.Filename)
14+ reader, _, _, err := st.GetFile(bucket, post.Filename)
15 if err != nil {
16 cfg.Logger.Infof("file not found %s/%s", post.UserID, post.Filename)
17 continue
18@@ -67,7 +68,8 @@ func main() {
19 _, err = st.PutFile(
20 bucket,
21 fmt.Sprintf("%s.webp", shared.SanitizeFileExt(post.Filename)),
22- storage.NopReaderAtCloser(webpReader),
23+ utils.NopReaderAtCloser(webpReader),
24+ &utils.FileEntry{},
25 )
26 if err != nil {
27 cfg.Logger.Error(err)
+2,
-1
1@@ -9,11 +9,12 @@ import (
2 "strings"
3 "time"
4
5+ "slices"
6+
7 _ "github.com/lib/pq"
8 "github.com/picosh/pico/db"
9 "github.com/picosh/pico/shared"
10 "go.uber.org/zap"
11- "golang.org/x/exp/slices"
12 )
13
14 var PAGER_SIZE = 15
+2,
-1
1@@ -5,11 +5,12 @@ import (
2 "strings"
3 "time"
4
5+ "slices"
6+
7 "github.com/picosh/pico/db"
8 "github.com/picosh/pico/filehandlers"
9 "github.com/picosh/pico/imgs"
10 "github.com/picosh/pico/shared"
11- "golang.org/x/exp/slices"
12 )
13
14 type FeedHooks struct {
+5,
-2
1@@ -7,7 +7,7 @@ import (
2 "strings"
3
4 "github.com/picosh/pico/shared"
5- "github.com/picosh/pico/shared/storage"
6+ "github.com/picosh/pico/wish/send/utils"
7 )
8
9 func (h *UploadAssetHandler) validateAsset(data *FileData) (bool, error) {
10@@ -72,16 +72,19 @@ func (h *UploadAssetHandler) writeAsset(data *FileData) error {
11 }
12 } else {
13 reader := bytes.NewReader(data.Text)
14+
15 h.Cfg.Logger.Infof(
16 "(%s) uploading to (bucket: %s) (%s)",
17 data.User.Name,
18 data.Bucket.Name,
19 assetFilename,
20 )
21+
22 _, err := h.Storage.PutFile(
23 data.Bucket,
24 assetFilename,
25- storage.NopReaderAtCloser(reader),
26+ utils.NopReaderAtCloser(reader),
27+ data.FileEntry,
28 )
29 if err != nil {
30 return err
+12,
-4
1@@ -15,6 +15,7 @@ import (
2 "github.com/picosh/pico/shared/storage"
3 "github.com/picosh/pico/wish/cms/util"
4 "github.com/picosh/pico/wish/send/utils"
5+ "go.uber.org/zap"
6 )
7
8 type ctxUserKey struct{}
9@@ -73,7 +74,11 @@ func NewUploadAssetHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage
10 }
11 }
12
13-func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) {
14+func (h *UploadAssetHandler) GetLogger() *zap.SugaredLogger {
15+ return h.Cfg.Logger
16+}
17+
18+func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
19 user, err := getUser(s)
20 if err != nil {
21 return nil, nil, err
22@@ -92,17 +97,20 @@ func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.Fil
23 }
24
25 fname := shared.GetAssetFileName(entry)
26- contents, err := h.Storage.GetFile(bucket, fname)
27+ contents, size, modTime, err := h.Storage.GetFile(bucket, fname)
28 if err != nil {
29 return nil, nil, err
30 }
31
32+ fileInfo.FSize = size
33+ fileInfo.FModTime = modTime
34+
35 reader := utils.NewAllReaderAt(contents)
36
37 return fileInfo, reader, nil
38 }
39
40-func (h *UploadAssetHandler) List(s ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) {
41+func (h *UploadAssetHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) {
42 var fileList []os.FileInfo
43
44 user, err := getUser(s)
45@@ -135,7 +143,7 @@ func (h *UploadAssetHandler) List(s ssh.Session, fpath string, isDir bool) ([]os
46 cleanFilename += "/"
47 }
48
49- foundList, err := h.Storage.ListFiles(bucket, cleanFilename, false)
50+ foundList, err := h.Storage.ListFiles(bucket, cleanFilename, recursive)
51 if err != nil {
52 return fileList, err
53 }
+10,
-4
1@@ -9,6 +9,8 @@ import (
2 "path/filepath"
3 "time"
4
5+ "slices"
6+
7 "github.com/charmbracelet/ssh"
8 exifremove "github.com/neurosnap/go-exif-remove"
9 "github.com/picosh/pico/db"
10@@ -16,7 +18,7 @@ import (
11 "github.com/picosh/pico/shared/storage"
12 "github.com/picosh/pico/wish/cms/util"
13 "github.com/picosh/pico/wish/send/utils"
14- "golang.org/x/exp/slices"
15+ "go.uber.org/zap"
16 )
17
18 var maxSize = 1 * shared.GB
19@@ -72,7 +74,11 @@ func (h *UploadImgHandler) removePost(data *PostMetaData) error {
20 return nil
21 }
22
23-func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) {
24+func (h *UploadImgHandler) GetLogger() *zap.SugaredLogger {
25+ return h.Cfg.Logger
26+}
27+
28+func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
29 user, err := getUser(s)
30 if err != nil {
31 return nil, nil, err
32@@ -101,7 +107,7 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileI
33 return nil, nil, err
34 }
35
36- contents, err := h.Storage.GetFile(bucket, post.Filename)
37+ contents, _, _, err := h.Storage.GetFile(bucket, post.Filename)
38 if err != nil {
39 return nil, nil, err
40 }
41@@ -111,7 +117,7 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileI
42 return fileInfo, reader, nil
43 }
44
45-func (h *UploadImgHandler) List(s ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) {
46+func (h *UploadImgHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) {
47 var fileList []os.FileInfo
48 user, err := getUser(s)
49 if err != nil {
+5,
-3
1@@ -9,7 +9,7 @@ import (
2 "github.com/charmbracelet/ssh"
3 "github.com/picosh/pico/db"
4 "github.com/picosh/pico/shared"
5- "github.com/picosh/pico/shared/storage"
6+ "github.com/picosh/pico/wish/send/utils"
7 )
8
9 func (h *UploadImgHandler) validateImg(data *PostMetaData) (bool, error) {
10@@ -59,7 +59,8 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
11 fname, err := h.Storage.PutFile(
12 bucket,
13 data.Filename,
14- storage.NopReaderAtCloser(reader),
15+ utils.NopReaderAtCloser(reader),
16+ &utils.FileEntry{},
17 )
18 if err != nil {
19 return err
20@@ -105,7 +106,8 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
21 _, err = h.Storage.PutFile(
22 bucket,
23 finalName,
24- storage.NopReaderAtCloser(webpReader),
25+ utils.NopReaderAtCloser(webpReader),
26+ &utils.FileEntry{},
27 )
28 if err != nil {
29 return err
+8,
-3
1@@ -17,6 +17,7 @@ import (
2 "github.com/picosh/pico/shared/storage"
3 "github.com/picosh/pico/wish/cms/util"
4 "github.com/picosh/pico/wish/send/utils"
5+ "go.uber.org/zap"
6 )
7
8 type ctxUserKey struct{}
9@@ -61,7 +62,11 @@ func NewScpPostHandler(dbpool db.DB, cfg *shared.ConfigSite, hooks ScpFileHooks,
10 }
11 }
12
13-func (h *ScpUploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) {
14+func (h *ScpUploadHandler) GetLogger() *zap.SugaredLogger {
15+ return h.Cfg.Logger
16+}
17+
18+func (h *ScpUploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
19 user, err := getUser(s)
20 if err != nil {
21 return nil, nil, err
22@@ -84,12 +89,12 @@ func (h *ScpUploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileI
23 FModTime: *post.UpdatedAt,
24 }
25
26- reader := strings.NewReader(post.Text)
27+ reader := utils.NopReaderAtCloser(strings.NewReader(post.Text))
28
29 return fileInfo, reader, nil
30 }
31
32-func (h *ScpUploadHandler) List(s ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) {
33+func (h *ScpUploadHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) {
34 var fileList []os.FileInfo
35 user, err := getUser(s)
36 if err != nil {
M
go.mod
+2,
-2
1@@ -4,7 +4,7 @@ go 1.21
2
3 require (
4 github.com/alecthomas/chroma v0.10.0
5- github.com/antoniomika/go-rsync-receiver v0.0.0-20220901010427-e6494124f0c8
6+ github.com/antoniomika/go-rsync-receiver v0.0.0-20231110145728-c94949e1ab7d
7 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
8 github.com/charmbracelet/bubbles v0.16.1
9 github.com/charmbracelet/bubbletea v0.24.2
10@@ -33,7 +33,7 @@ require (
11 go.abhg.dev/goldmark/anchor v0.1.1
12 go.uber.org/zap v1.26.0
13 golang.org/x/crypto v0.15.0
14- golang.org/x/exp v0.0.0-20231006140011-7918f672742d
15+ golang.org/x/exp v0.0.0-20231108232855-2478ac86f678
16 gopkg.in/yaml.v2 v2.4.0
17 )
18
M
go.sum
+4,
-0
1@@ -11,6 +11,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI
2 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
3 github.com/antoniomika/go-rsync-receiver v0.0.0-20220901010427-e6494124f0c8 h1:gR27C6N8s5b+ciBRymi0zhUx8TylKFO755z6yrBuMiI=
4 github.com/antoniomika/go-rsync-receiver v0.0.0-20220901010427-e6494124f0c8/go.mod h1:zmqePVIo1hp+WEKxERLLGHJBDOr8/z/T4eFqXgWIw1w=
5+github.com/antoniomika/go-rsync-receiver v0.0.0-20231110145728-c94949e1ab7d h1:NyzUTxebDLLdtNu1gY5hn/amdAEnKG9DOawz82LwNTY=
6+github.com/antoniomika/go-rsync-receiver v0.0.0-20231110145728-c94949e1ab7d/go.mod h1:zmqePVIo1hp+WEKxERLLGHJBDOr8/z/T4eFqXgWIw1w=
7 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
8 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
9 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
10@@ -271,6 +273,8 @@ golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
11 golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
12 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
13 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
14+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w=
15+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
16 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
17 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
18 golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
+5,
-3
1@@ -11,14 +11,16 @@ import (
2
3 _ "net/http/pprof"
4
5+ "slices"
6+
7 "github.com/gorilla/feeds"
8 gocache "github.com/patrickmn/go-cache"
9 "github.com/picosh/pico/db"
10 "github.com/picosh/pico/db/postgres"
11 "github.com/picosh/pico/shared"
12 "github.com/picosh/pico/shared/storage"
13+ "github.com/picosh/pico/wish/send/utils"
14 "go.uber.org/zap"
15- "golang.org/x/exp/slices"
16 )
17
18 type PageData struct {
19@@ -199,7 +201,7 @@ type ImgHandler struct {
20
21 type ImgResizer struct {
22 Key string
23- contents storage.ReaderAtCloser
24+ contents utils.ReaderAtCloser
25 writer io.Writer
26 Img *shared.ImgOptimizer
27 Cache *gocache.Cache
28@@ -282,7 +284,7 @@ func imgHandler(w http.ResponseWriter, h *ImgHandler) {
29 fname = fmt.Sprintf("%s.webp", shared.SanitizeFileExt(post.Filename))
30 }
31
32- contents, err := h.Storage.GetFile(bucket, fname)
33+ contents, _, _, err := h.Storage.GetFile(bucket, fname)
34 if err != nil {
35 h.Logger.Infof(
36 "file not found %s/%s in storage (bucket: %s, name: %s)",
+2,
-1
1@@ -10,6 +10,8 @@ import (
2 "strconv"
3 "time"
4
5+ "slices"
6+
7 "github.com/gorilla/feeds"
8 gocache "github.com/patrickmn/go-cache"
9 "github.com/picosh/pico/db"
10@@ -17,7 +19,6 @@ import (
11 "github.com/picosh/pico/imgs"
12 "github.com/picosh/pico/shared"
13 "github.com/picosh/pico/shared/storage"
14- "golang.org/x/exp/slices"
15 )
16
17 type PostItemData struct {
+2,
-1
1@@ -4,11 +4,12 @@ import (
2 "fmt"
3 "strings"
4
5+ "slices"
6+
7 "github.com/picosh/pico/db"
8 "github.com/picosh/pico/filehandlers"
9 "github.com/picosh/pico/imgs"
10 "github.com/picosh/pico/shared"
11- "golang.org/x/exp/slices"
12 )
13
14 type ListHooks struct {
+3,
-3
1@@ -210,7 +210,7 @@ func assetHandler(w http.ResponseWriter, h *AssetHandler) {
2 }
3
4 var redirects []*RedirectRule
5- redirectFp, err := h.Storage.GetFile(bucket, filepath.Join(h.ProjectDir, "_redirects"))
6+ redirectFp, _, _, err := h.Storage.GetFile(bucket, filepath.Join(h.ProjectDir, "_redirects"))
7 if err == nil {
8 defer redirectFp.Close()
9 buf := new(strings.Builder)
10@@ -228,11 +228,11 @@ func assetHandler(w http.ResponseWriter, h *AssetHandler) {
11 }
12
13 routes := calcPossibleRoutes(h.ProjectDir, h.Filepath, redirects)
14- var contents storage.ReaderAtCloser
15+ var contents utils.ReaderAtCloser
16 assetFilepath := ""
17 status := 200
18 for _, fp := range routes {
19- c, err := h.Storage.GetFile(bucket, fp.Filepath)
20+ c, _, _, err := h.Storage.GetFile(bucket, fp.Filepath)
21 if err == nil {
22 contents = c
23 assetFilepath = fp.Filepath
+2,
-2
1@@ -93,7 +93,7 @@ func parseRedirectText(text string) ([]*RedirectRule, error) {
2
3 parts := reSplitWhitespace.Split(trimmed, -1)
4 if len(parts) < 2 {
5- return rules, fmt.Errorf("Missing destination path/URL")
6+ return rules, fmt.Errorf("missing destination path/URL")
7 }
8
9 from := parts[0]
10@@ -114,7 +114,7 @@ func parseRedirectText(text string) ([]*RedirectRule, error) {
11 }
12
13 if toIndex == -1 {
14- return rules, fmt.Errorf("The destination path/URL must start with '/', 'http:' or 'https:'")
15+ return rules, fmt.Errorf("the destination path/URL must start with '/', 'http:' or 'https:'")
16 }
17
18 queryParts := parts[:toIndex]
+2,
-1
1@@ -10,6 +10,8 @@ import (
2 "strconv"
3 "time"
4
5+ "slices"
6+
7 "github.com/gorilla/feeds"
8 gocache "github.com/patrickmn/go-cache"
9 "github.com/picosh/pico/db"
10@@ -17,7 +19,6 @@ import (
11 "github.com/picosh/pico/imgs"
12 "github.com/picosh/pico/shared"
13 "github.com/picosh/pico/shared/storage"
14- "golang.org/x/exp/slices"
15 )
16
17 type PageData struct {
+2,
-1
1@@ -4,11 +4,12 @@ import (
2 "fmt"
3 "strings"
4
5+ "slices"
6+
7 "github.com/picosh/pico/db"
8 "github.com/picosh/pico/filehandlers"
9 "github.com/picosh/pico/imgs"
10 "github.com/picosh/pico/shared"
11- "golang.org/x/exp/slices"
12 )
13
14 type MarkdownHooks struct {
1@@ -9,8 +9,9 @@ import (
2 "strings"
3 "time"
4
5+ "slices"
6+
7 "github.com/araddon/dateparse"
8- "golang.org/x/exp/slices"
9 )
10
11 var reIndent = regexp.MustCompile(`^[[:blank:]]+`)
1@@ -3,10 +3,12 @@ package storage
2 import (
3 "fmt"
4 "io"
5+ "io/fs"
6 "os"
7 "path"
8 "path/filepath"
9 "strings"
10+ "time"
11
12 "github.com/picosh/pico/wish/send/utils"
13 )
14@@ -82,16 +84,21 @@ func (s *StorageFS) DeleteBucket(bucket Bucket) error {
15 return os.RemoveAll(bucket.Path)
16 }
17
18-func (s *StorageFS) GetFile(bucket Bucket, fpath string) (ReaderAtCloser, error) {
19+func (s *StorageFS) GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser, int64, time.Time, error) {
20 dat, err := os.Open(filepath.Join(bucket.Path, fpath))
21 if err != nil {
22- return nil, err
23+ return nil, 0, time.Time{}, err
24 }
25
26- return dat, nil
27+ info, err := dat.Stat()
28+ if err != nil {
29+ return nil, 0, time.Time{}, err
30+ }
31+
32+ return dat, info.Size(), info.ModTime(), nil
33 }
34
35-func (s *StorageFS) PutFile(bucket Bucket, fpath string, contents ReaderAtCloser) (string, error) {
36+func (s *StorageFS) PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error) {
37 loc := filepath.Join(bucket.Path, fpath)
38 err := os.MkdirAll(filepath.Dir(loc), os.ModePerm)
39 if err != nil {
40@@ -101,13 +108,19 @@ func (s *StorageFS) PutFile(bucket Bucket, fpath string, contents ReaderAtCloser
41 if err != nil {
42 return "", err
43 }
44- defer f.Close()
45
46 _, err = io.Copy(f, contents)
47 if err != nil {
48 return "", err
49 }
50
51+ f.Close()
52+
53+ if entry.Mtime > 0 {
54+ uTime := time.Unix(entry.Mtime, 0)
55+ _ = os.Chtimes(loc, uTime, uTime)
56+ }
57+
58 return loc, nil
59 }
60
61@@ -142,10 +155,26 @@ func (s *StorageFS) ListFiles(bucket Bucket, dir string, recursive bool) ([]os.F
62 return fileList, err
63 }
64
65- files, err := os.ReadDir(fpath)
66- if err != nil {
67- fileList = append(fileList, info)
68- return fileList, nil
69+ var files []fs.DirEntry
70+
71+ if recursive {
72+ err = filepath.WalkDir(fpath, func(s string, d fs.DirEntry, err error) error {
73+ if err != nil {
74+ return err
75+ }
76+ files = append(files, d)
77+ return nil
78+ })
79+ if err != nil {
80+ fileList = append(fileList, info)
81+ return fileList, nil
82+ }
83+ } else {
84+ files, err = os.ReadDir(fpath)
85+ if err != nil {
86+ fileList = append(fileList, info)
87+ return fileList, nil
88+ }
89 }
90
91 for _, f := range files {
1@@ -6,7 +6,9 @@ import (
2 "fmt"
3 "net/url"
4 "os"
5+ "strconv"
6 "strings"
7+ "time"
8
9 "github.com/minio/madmin-go/v3"
10 "github.com/minio/minio-go/v7"
11@@ -110,11 +112,20 @@ func (s *StorageMinio) ListFiles(bucket Bucket, dir string, recursive bool) ([]o
12 isDir = true
13 }
14
15+ modTime := time.Time{}
16+
17+ if mtime, ok := obj.UserMetadata["Mtime"]; ok {
18+ mtimeUnix, err := strconv.Atoi(mtime)
19+ if err == nil {
20+ modTime = time.Unix(int64(mtimeUnix), 0)
21+ }
22+ }
23+
24 info := &utils.VirtualFile{
25 FName: strings.TrimSuffix(strings.TrimPrefix(obj.Key, resolved), "/"),
26 FIsDir: isDir,
27 FSize: obj.Size,
28- FModTime: obj.LastModified,
29+ FModTime: modTime,
30 }
31 fileList = append(fileList, info)
32 }
33@@ -126,24 +137,40 @@ func (s *StorageMinio) DeleteBucket(bucket Bucket) error {
34 return s.Client.RemoveBucket(context.TODO(), bucket.Name)
35 }
36
37-func (s *StorageMinio) GetFile(bucket Bucket, fpath string) (ReaderAtCloser, error) {
38- // we have to stat the object first to see if it exists
39- // https://github.com/minio/minio-go/issues/654
40- _, err := s.Client.StatObject(context.Background(), bucket.Name, fpath, minio.StatObjectOptions{})
41+func (s *StorageMinio) GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser, int64, time.Time, error) {
42+ modTime := time.Time{}
43+
44+ info, err := s.Client.StatObject(context.Background(), bucket.Name, fpath, minio.StatObjectOptions{})
45 if err != nil {
46- return nil, err
47+ return nil, 0, modTime, err
48 }
49
50 obj, err := s.Client.GetObject(context.Background(), bucket.Name, fpath, minio.GetObjectOptions{})
51 if err != nil {
52- return nil, err
53+ return nil, 0, modTime, err
54 }
55
56- return obj, nil
57+ if mtime, ok := info.UserMetadata["Mtime"]; ok {
58+ mtimeUnix, err := strconv.Atoi(mtime)
59+ if err == nil {
60+ modTime = time.Unix(int64(mtimeUnix), 0)
61+ }
62+ }
63+
64+ return obj, info.Size, modTime, nil
65 }
66
67-func (s *StorageMinio) PutFile(bucket Bucket, fpath string, contents ReaderAtCloser) (string, error) {
68- info, err := s.Client.PutObject(context.TODO(), bucket.Name, fpath, contents, -1, minio.PutObjectOptions{})
69+func (s *StorageMinio) PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error) {
70+ opts := minio.PutObjectOptions{}
71+
72+ if entry.Mtime > 0 {
73+ opts.UserMetadata = map[string]string{
74+ "Mtime": fmt.Sprint(entry.Mtime),
75+ }
76+ }
77+
78+ info, err := s.Client.PutObject(context.TODO(), bucket.Name, fpath, contents, -1, opts)
79+
80 if err != nil {
81 return "", err
82 }
1@@ -1,8 +1,10 @@
2 package storage
3
4 import (
5- "io"
6 "os"
7+ "time"
8+
9+ "github.com/picosh/pico/wish/send/utils"
10 )
11
12 type Bucket struct {
13@@ -10,34 +12,14 @@ type Bucket struct {
14 Path string
15 }
16
17-type ReadAndReaderAt interface {
18- io.ReaderAt
19- io.Reader
20-}
21-
22-type ReaderAtCloser interface {
23- io.ReaderAt
24- io.ReadCloser
25-}
26-
27-func NopReaderAtCloser(r ReadAndReaderAt) ReaderAtCloser {
28- return nopReaderAtCloser{r}
29-}
30-
31-type nopReaderAtCloser struct {
32- ReadAndReaderAt
33-}
34-
35-func (nopReaderAtCloser) Close() error { return nil }
36-
37 type ObjectStorage interface {
38 GetBucket(name string) (Bucket, error)
39 UpsertBucket(name string) (Bucket, error)
40
41 DeleteBucket(bucket Bucket) error
42 GetBucketQuota(bucket Bucket) (uint64, error)
43- GetFile(bucket Bucket, fpath string) (ReaderAtCloser, error)
44- PutFile(bucket Bucket, fpath string, contents ReaderAtCloser) (string, error)
45+ GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser, int64, time.Time, error)
46+ PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error)
47 DeleteFile(bucket Bucket, fpath string) error
48 ListFiles(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error)
49 }
1@@ -15,8 +15,9 @@ import (
2 "unicode"
3 "unicode/utf8"
4
5+ "slices"
6+
7 "github.com/charmbracelet/ssh"
8- "golang.org/x/exp/slices"
9 )
10
11 var fnameRe = regexp.MustCompile(`[-_]+`)
+8,
-4
1@@ -2,7 +2,6 @@ package main
2
3 import (
4 "fmt"
5- "io"
6 "log"
7 "os"
8 "strings"
9@@ -12,11 +11,16 @@ import (
10 "github.com/charmbracelet/wish"
11 "github.com/picosh/pico/wish/send"
12 "github.com/picosh/pico/wish/send/utils"
13+ "go.uber.org/zap"
14 )
15
16 type handler struct {
17 }
18
19+func (h *handler) GetLogger() *zap.SugaredLogger {
20+ return zap.NewNop().Sugar()
21+}
22+
23 func (h *handler) Write(session ssh.Session, file *utils.FileEntry) (string, error) {
24 str := fmt.Sprintf("Received file: %+v from session: %+v", file, session)
25 log.Print(str)
26@@ -29,7 +33,7 @@ func (h *handler) Validate(session ssh.Session) error {
27 return nil
28 }
29
30-func (h *handler) Read(session ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) {
31+func (h *handler) Read(session ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
32 log.Printf("Received validate from session: %+v", session)
33
34 data := strings.NewReader("lorem ipsum dolor")
35@@ -39,10 +43,10 @@ func (h *handler) Read(session ssh.Session, entry *utils.FileEntry) (os.FileInfo
36 FIsDir: false,
37 FSize: data.Size(),
38 FModTime: time.Now(),
39- }, data, nil
40+ }, utils.NopReaderAtCloser(data), nil
41 }
42
43-func (h *handler) List(session ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) {
44+func (h *handler) List(session ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) {
45 return nil, nil
46 }
47
+1,
-1
1@@ -18,7 +18,7 @@ func Middleware(writeHandler utils.CopyFromClientHandler) wish.Middleware {
2 return
3 }
4
5- fileList, err := writeHandler.List(session, "/", true)
6+ fileList, err := writeHandler.List(session, "/", true, false)
7 if err != nil {
8 utils.ErrorHandler(session, err)
9 return
+132,
-21
1@@ -1,12 +1,14 @@
2 package rsync
3
4 import (
5+ "errors"
6 "fmt"
7 "io"
8 "io/fs"
9- "log"
10 "os"
11- "path/filepath"
12+ "path"
13+ "slices"
14+ "strings"
15
16 "github.com/antoniomika/go-rsync-receiver/rsyncreceiver"
17 "github.com/antoniomika/go-rsync-receiver/rsyncsender"
18@@ -19,41 +21,117 @@ import (
19 type handler struct {
20 session ssh.Session
21 writeHandler utils.CopyFromClientHandler
22+ root string
23+ recursive bool
24+ ignoreTimes bool
25 }
26
27 func (h *handler) Skip(file *rsyncutils.ReceiverFile) bool {
28+ if file.FileMode().IsDir() {
29+ return true
30+ }
31+
32+ fI, _, err := h.writeHandler.Read(h.session, &utils.FileEntry{Filepath: path.Join("/", h.root, file.Name)})
33+ if err == nil && fI.ModTime().Equal(file.ModTime) && file.Length == fI.Size() {
34+ return true
35+ }
36+
37 return false
38 }
39
40-func (h *handler) List(path string) ([]fs.FileInfo, error) {
41- list, err := h.writeHandler.List(h.session, path, true)
42+func (h *handler) List(rPath string) ([]fs.FileInfo, error) {
43+ isDir := false
44+ if rPath == "." {
45+ rPath = "/"
46+ isDir = true
47+ }
48+
49+ list, err := h.writeHandler.List(h.session, rPath, isDir, h.recursive)
50 if err != nil {
51 return nil, err
52 }
53
54- newList := list
55- if list[0].IsDir() {
56- newList = list[1:]
57+ var dirs []string
58+
59+ var newList []fs.FileInfo
60+
61+ for _, f := range list {
62+ fname := f.Name()
63+ if strings.HasPrefix(f.Name(), "/") {
64+ fname = path.Join(rPath, f.Name())
65+ }
66+
67+ if fname == "" && !f.IsDir() {
68+ fname = path.Base(rPath)
69+ }
70+
71+ newFile := &utils.VirtualFile{
72+ FName: fname,
73+ FIsDir: f.IsDir(),
74+ FSize: f.Size(),
75+ FModTime: f.ModTime(),
76+ FSys: f.Sys(),
77+ }
78+
79+ newList = append(newList, newFile)
80+
81+ parts := strings.Split(newFile.Name(), string(os.PathSeparator))
82+ lastDir := newFile.Name()
83+ for i := 0; i < len(parts); i++ {
84+ lastDir, _ = path.Split(lastDir)
85+ if lastDir == "" {
86+ continue
87+ }
88+
89+ lastDir = lastDir[:len(lastDir)-1]
90+ dirs = append(dirs, lastDir)
91+ }
92+ }
93+
94+ for _, dir := range dirs {
95+ newList = append(newList, &utils.VirtualFile{
96+ FName: dir,
97+ FIsDir: true,
98+ })
99+ }
100+
101+ slices.Reverse(newList)
102+
103+ onlyEmpty := true
104+ for _, f := range newList {
105+ if f.Name() != "" {
106+ onlyEmpty = false
107+ }
108+ }
109+
110+ if len(newList) == 0 || onlyEmpty {
111+ return nil, errors.New("no files to send, the directory may not exist or could be empty")
112 }
113
114 return newList, nil
115 }
116
117-func (h *handler) Read(path string) (os.FileInfo, io.ReaderAt, error) {
118- return h.writeHandler.Read(h.session, &utils.FileEntry{Filepath: path})
119+func (h *handler) Read(file *rsyncutils.SenderFile) (os.FileInfo, io.ReaderAt, error) {
120+ filePath := file.WPath
121+
122+ if strings.HasSuffix(h.root, file.WPath) {
123+ filePath = h.root
124+ } else if !strings.HasPrefix(filePath, h.root) {
125+ filePath = path.Join(h.root, file.Path, file.WPath)
126+ }
127+
128+ return h.writeHandler.Read(h.session, &utils.FileEntry{Filepath: filePath})
129 }
130
131-func (h *handler) Put(fileName string, content io.Reader, fileSize int64, mTime int64, aTime int64) (int64, error) {
132- cleanName := filepath.Base(fileName)
133- fpath := "/"
134+func (h *handler) Put(file *rsyncutils.ReceiverFile) (int64, error) {
135 fileEntry := &utils.FileEntry{
136- Filepath: filepath.Join(fpath, cleanName),
137+ Filepath: path.Join("/", h.root, file.Name),
138 Mode: fs.FileMode(0600),
139- Size: fileSize,
140- Mtime: mTime,
141- Atime: aTime,
142+ Size: file.Length,
143+ Mtime: file.ModTime.Unix(),
144+ Atime: file.ModTime.Unix(),
145 }
146- fileEntry.Reader = content
147+ fileEntry.Reader = file.Buf
148
149 msg, err := h.writeHandler.Write(h.session, fileEntry)
150 if err != nil {
151@@ -79,19 +157,52 @@ func Middleware(writeHandler utils.CopyFromClientHandler) wish.Middleware {
152 fileHandler := &handler{
153 session: session,
154 writeHandler: writeHandler,
155+ root: strings.TrimPrefix(cmd[len(cmd)-1], "/"),
156 }
157
158+ cmdFlags := session.Command()
159+
160 for _, arg := range cmd {
161 if arg == "--sender" {
162- if err := rsyncsender.ClientRun(nil, session, fileHandler, cmd[len(cmd)-1], true); err != nil {
163- log.Println("error running rsync:", err)
164+ opts, parser := rsyncsender.NewGetOpt()
165+
166+ compress := parser.Bool("z", false)
167+
168+ _, _ = parser.Parse(cmdFlags[1:])
169+
170+ fileHandler.recursive = opts.Recurse
171+ fileHandler.ignoreTimes = opts.IgnoreTimes
172+
173+ if *compress {
174+ _, _ = session.Stderr().Write([]byte("compression is currently unsupported\r\n"))
175+ return
176+ }
177+
178+ if opts.PreserveUid {
179+ _, _ = session.Stderr().Write([]byte("uid preservation will not work as we don't retain user information\r\n"))
180+ return
181+ }
182+
183+ if opts.PreserveGid {
184+ _, _ = session.Stderr().Write([]byte("gid preservation will not work as we don't retain user information\r\n"))
185+ return
186+ }
187+
188+ if err := rsyncsender.ClientRun(opts, session, fileHandler, fileHandler.root, true); err != nil {
189+ writeHandler.GetLogger().Error("error running rsync sender:", err)
190 }
191 return
192 }
193 }
194
195- if _, err := rsyncreceiver.ClientRun(nil, session, fileHandler, true); err != nil {
196- log.Println("error running rsync:", err)
197+ opts, parser := rsyncreceiver.NewGetOpt()
198+ _, _ = parser.Parse(cmdFlags[1:])
199+
200+ fileHandler.recursive = opts.Recurse
201+ fileHandler.ignoreTimes = opts.IgnoreTimes
202+
203+ if _, err := rsyncreceiver.ClientRun(opts, session, fileHandler, true); err != nil {
204+ writeHandler.GetLogger().Error("error running rsync receiver:", err)
205 }
206 }
207 }
+3,
-2
1@@ -6,10 +6,11 @@ import (
2 "io"
3 "os"
4
5+ "slices"
6+
7 "github.com/charmbracelet/ssh"
8 "github.com/picosh/pico/wish/send/utils"
9 "github.com/pkg/sftp"
10- "golang.org/x/exp/slices"
11 )
12
13 type listerat []os.FileInfo
14@@ -51,7 +52,7 @@ func (f *handler) Filelist(r *sftp.Request) (sftp.ListerAt, error) {
15 case "List", "Stat":
16 list := r.Method == "List"
17
18- listData, err := f.writeHandler.List(f.session, r.Filepath, list)
19+ listData, err := f.writeHandler.List(f.session, r.Filepath, list, false)
20 if err != nil {
21 return nil, err
22 }
+1,
-2
1@@ -3,7 +3,6 @@ package sftp
2 import (
3 "errors"
4 "io"
5- "log"
6
7 "github.com/charmbracelet/ssh"
8 "github.com/picosh/pico/wish/send/utils"
9@@ -45,7 +44,7 @@ func SubsystemHandler(writeHandler utils.CopyFromClientHandler) ssh.SubsystemHan
10
11 err = requestServer.Serve()
12 if err != nil && !errors.Is(err, io.EOF) {
13- log.Println("Error serving sftp subsystem:", err)
14+ writeHandler.GetLogger().Error("Error serving sftp subsystem:", err)
15 }
16 }
17 }
+0,
-33
1@@ -1,33 +0,0 @@
2-package utils
3-
4-import (
5- "errors"
6- "io"
7- "net/http"
8-
9- "github.com/minio/minio-go/v7"
10-)
11-
12-type AllReaderAt struct {
13- Reader io.ReaderAt
14-}
15-
16-func NewAllReaderAt(reader io.ReaderAt) *AllReaderAt {
17- return &AllReaderAt{reader}
18-}
19-
20-func (a *AllReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
21- n, err = a.Reader.ReadAt(p, off)
22-
23- if errors.Is(err, io.EOF) {
24- return
25- }
26-
27- resp := minio.ToErrorResponse(err)
28-
29- if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
30- err = io.EOF
31- }
32-
33- return
34-}
+61,
-0
1@@ -0,0 +1,61 @@
2+package utils
3+
4+import (
5+ "errors"
6+ "io"
7+ "net/http"
8+
9+ "github.com/minio/minio-go/v7"
10+)
11+
12+type ReadAndReaderAt interface {
13+ io.ReaderAt
14+ io.Reader
15+}
16+
17+type ReaderAtCloser interface {
18+ io.ReaderAt
19+ io.ReadCloser
20+}
21+
22+func NopReaderAtCloser(r ReadAndReaderAt) ReaderAtCloser {
23+ return nopReaderAtCloser{r}
24+}
25+
26+type nopReaderAtCloser struct {
27+ ReadAndReaderAt
28+}
29+
30+func (nopReaderAtCloser) Close() error { return nil }
31+
32+type AllReaderAt struct {
33+ Reader ReaderAtCloser
34+}
35+
36+func NewAllReaderAt(reader ReaderAtCloser) *AllReaderAt {
37+ return &AllReaderAt{reader}
38+}
39+
40+func (a *AllReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
41+ n, err = a.Reader.ReadAt(p, off)
42+
43+ if errors.Is(err, io.EOF) {
44+ return
45+ }
46+
47+ resp := minio.ToErrorResponse(err)
48+
49+ if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
50+ err = io.EOF
51+ }
52+
53+ return
54+}
55+
56+func (a *AllReaderAt) Read(p []byte) (int, error) {
57+ return a.Reader.Read(p)
58+}
59+
60+func (a *AllReaderAt) Close() error {
61+ return a.Reader.Close()
62+}
+4,
-2
1@@ -10,6 +10,7 @@ import (
2 "strconv"
3
4 "github.com/charmbracelet/ssh"
5+ "go.uber.org/zap"
6 )
7
8 // NULL is an array with a single NULL byte.
9@@ -57,8 +58,9 @@ func octalPerms(info fs.FileMode) string {
10 type CopyFromClientHandler interface {
11 // Write should write the given file.
12 Write(ssh.Session, *FileEntry) (string, error)
13- Read(ssh.Session, *FileEntry) (os.FileInfo, io.ReaderAt, error)
14- List(ssh.Session, string, bool) ([]os.FileInfo, error)
15+ Read(ssh.Session, *FileEntry) (os.FileInfo, ReaderAtCloser, error)
16+ List(ssh ssh.Session, path string, isDir bool, recursive bool) ([]os.FileInfo, error)
17+ GetLogger() *zap.SugaredLogger
18 Validate(ssh.Session) error
19 }
20