- commit
- 11d6ebe
- parent
- f3f69ff
- author
- Eric Bower
- date
- 2022-09-05 14:55:39 +0000 UTC
feat(imgs): we save webp images next to main img
5 files changed,
+188,
-53
+76,
-0
1@@ -0,0 +1,76 @@
2+package main
3+
4+import (
5+ "bytes"
6+ "fmt"
7+
8+ "git.sr.ht/~erock/pico/db"
9+ "git.sr.ht/~erock/pico/db/postgres"
10+ "git.sr.ht/~erock/pico/imgs"
11+ "git.sr.ht/~erock/pico/imgs/storage"
12+ "git.sr.ht/~erock/pico/shared"
13+)
14+
15+func main() {
16+ cfg := imgs.NewConfigSite()
17+ dbp := postgres.NewDB(&cfg.ConfigCms)
18+
19+ cfg.Logger.Info("fetching all img posts")
20+ posts, err := dbp.FindAllPosts(&db.Pager{Num: 1000, Page: 0}, "imgs")
21+ if err != nil {
22+ panic(err)
23+ }
24+
25+ var st storage.ObjectStorage
26+ if cfg.MinioURL == "" {
27+ st, err = storage.NewStorageFS(cfg.StorageDir)
28+ } else {
29+ st, err = storage.NewStorageMinio(cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
30+ }
31+
32+ if err != nil {
33+ panic(err)
34+ }
35+
36+ total := len(posts.Data)
37+ cfg.Logger.Infof("%d posts", total)
38+ for i, post := range posts.Data {
39+ cfg.Logger.Infof("%d%% %s %s", ((i+1)/total)*100, post.Filename, post.MimeType)
40+ bucket, err := st.GetBucket(post.UserID)
41+ if err != nil {
42+ cfg.Logger.Infof("bucket not found %s", post.UserID)
43+ continue
44+ }
45+
46+ reader, err := st.GetFile(bucket, post.Filename)
47+ if err != nil {
48+ cfg.Logger.Infof("file not found %s/%s", post.UserID, post.Filename)
49+ continue
50+ }
51+ defer reader.Close()
52+
53+ opt := shared.NewImgOptimizer(cfg.Logger, "")
54+ contents := &bytes.Buffer{}
55+ img, err := opt.GetImage(reader, post.MimeType)
56+ if err != nil {
57+ cfg.Logger.Error(err)
58+ continue
59+ }
60+
61+ err = opt.EncodeWebp(contents, img)
62+ if err != nil {
63+ cfg.Logger.Error(err)
64+ continue
65+ }
66+
67+ webpReader := bytes.NewReader(contents.Bytes())
68+ _, err = st.PutFile(
69+ bucket,
70+ fmt.Sprintf("%s.webp", shared.SanitizeFileExt(post.Filename)),
71+ storage.NopReaderAtCloser(webpReader),
72+ )
73+ if err != nil {
74+ cfg.Logger.Error(err)
75+ }
76+ }
77+}
+4,
-0
1@@ -98,6 +98,7 @@ var (
2 publish_at,
3 app_users.name as username,
4 posts.updated_at,
5+ posts.mime_type,
6 0 AS "score"
7 FROM posts
8 LEFT OUTER JOIN app_users ON app_users.id = posts.user_id
9@@ -171,6 +172,7 @@ const (
10 publish_at,
11 app_users.name as username,
12 posts.updated_at,
13+ posts.mime_type,
14 0 AS "score"
15 FROM posts
16 LEFT OUTER JOIN app_users ON app_users.id = posts.user_id
17@@ -189,6 +191,7 @@ const (
18 publish_at,
19 app_users.name as username,
20 posts.updated_at,
21+ posts.mime_type,
22 (
23 LOG(2.0, COALESCE(NULLIF(posts.views, 0), 1)) / (
24 EXTRACT(
25@@ -610,6 +613,7 @@ func (me *PsqlDB) postPager(rs *sql.Rows, pageNum int, space string, tag string)
26 &post.PublishAt,
27 &post.Username,
28 &post.UpdatedAt,
29+ &post.MimeType,
30 &post.Score,
31 )
32 if err != nil {
+37,
-1
1@@ -42,7 +42,38 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
2 if err != nil {
3 return err
4 }
5- fname, err := h.Storage.PutFile(bucket, data.Filename, storage.NopReaderAtCloser(bytes.NewReader([]byte(data.Text))))
6+
7+ reader := bytes.NewReader([]byte(data.Text))
8+ tee := bytes.NewReader([]byte(data.Text))
9+ // var buf bytes.Buffer
10+ // tee := io.TeeReader(reader, &buf)
11+
12+ fname, err := h.Storage.PutFile(
13+ bucket,
14+ data.Filename,
15+ storage.NopReaderAtCloser(reader),
16+ )
17+ if err != nil {
18+ return err
19+ }
20+
21+ opt := shared.NewImgOptimizer(h.Cfg.Logger, "")
22+ // opt.Quality = 100
23+ contents := &bytes.Buffer{}
24+ img, err := opt.GetImage(tee, data.MimeType)
25+ if err != nil {
26+ return err
27+ }
28+ err = opt.EncodeWebp(contents, img)
29+ if err != nil {
30+ return err
31+ }
32+ webpReader := bytes.NewReader(contents.Bytes())
33+ _, err = h.Storage.PutFile(
34+ bucket,
35+ fmt.Sprintf("%s.webp", shared.SanitizeFileExt(data.Filename)),
36+ storage.NopReaderAtCloser(webpReader),
37+ )
38 if err != nil {
39 return err
40 }
41@@ -82,6 +113,11 @@ func (h *UploadImgHandler) writeImg(data *PostMetaData) error {
42 if err != nil {
43 return err
44 }
45+ webp := fmt.Sprintf("%s.webp", shared.SanitizeFileExt(data.Filename))
46+ err = h.Storage.DeleteFile(bucket, webp)
47+ if err != nil {
48+ return err
49+ }
50 } else if data.Cur == nil {
51 h.Cfg.Logger.Infof("(%s) not found, adding record", data.Filename)
52 insertPost := db.Post{
+29,
-10
1@@ -4,9 +4,9 @@ import (
2 "bytes"
3 "fmt"
4 "html/template"
5+ "io"
6 "net/http"
7 "net/url"
8- "strings"
9 "time"
10
11 _ "net/http/pprof"
12@@ -189,6 +189,7 @@ type ImgHandler struct {
13 Storage storage.ObjectStorage
14 Logger *zap.SugaredLogger
15 Img *shared.ImgOptimizer
16+ Optimized bool
17 }
18
19 func imgHandler(w http.ResponseWriter, h *ImgHandler) {
20@@ -218,7 +219,12 @@ func imgHandler(w http.ResponseWriter, h *ImgHandler) {
21 return
22 }
23
24- contents, err := h.Storage.GetFile(bucket, post.Filename)
25+ fname := post.Filename
26+ if h.Optimized {
27+ fname = fmt.Sprintf("%s.webp", shared.SanitizeFileExt(post.Filename))
28+ }
29+
30+ contents, err := h.Storage.GetFile(bucket, fname)
31 if err != nil {
32 h.Logger.Infof("file not found %s/%s", h.Username, post.Filename)
33 http.Error(w, err.Error(), http.StatusInternalServerError)
34@@ -226,15 +232,25 @@ func imgHandler(w http.ResponseWriter, h *ImgHandler) {
35 }
36 defer contents.Close()
37
38- if h.Img.Optimized {
39+ if h.Optimized {
40 w.Header().Add("Content-Type", "image/webp")
41+ if h.Img.Width != 0 || h.Img.Height != 0 {
42+ err := h.Img.Process(w, contents)
43+ if err != nil {
44+ h.Logger.Error(err)
45+ }
46+ } else {
47+ _, err := io.Copy(w, contents)
48+ if err != nil {
49+ h.Logger.Error(err)
50+ }
51+ }
52 } else {
53 w.Header().Add("Content-Type", post.MimeType)
54- }
55-
56- err = h.Img.Process(contents, w, strings.TrimSpace(post.MimeType))
57- if err != nil {
58- h.Logger.Error(err)
59+ _, err := io.Copy(w, contents)
60+ if err != nil {
61+ h.Logger.Error(err)
62+ }
63 }
64 }
65
66@@ -266,7 +282,8 @@ func imgRequestOriginal(w http.ResponseWriter, r *http.Request) {
67 Dbpool: dbpool,
68 Storage: st,
69 Logger: logger,
70- Img: shared.NewImgOptimizer(logger, false, ""),
71+ Img: shared.NewImgOptimizer(logger, ""),
72+ Optimized: false,
73 })
74 }
75
76@@ -301,7 +318,8 @@ func imgRequest(w http.ResponseWriter, r *http.Request) {
77 Dbpool: dbpool,
78 Storage: st,
79 Logger: logger,
80- Img: shared.NewImgOptimizer(logger, true, dimes),
81+ Img: shared.NewImgOptimizer(logger, dimes),
82+ Optimized: true,
83 })
84 }
85
86@@ -650,6 +668,7 @@ func createMainRoutes(staticRoutes []shared.Route) []shared.Route {
87 shared.NewRoute("GET", "/([^/]+)/o/([^/]+)", imgRequestOriginal),
88 shared.NewRoute("GET", "/([^/]+)/p/([^/]+)", postHandler),
89 shared.NewRoute("GET", "/([^/]+)/([^/]+)", imgRequest),
90+ shared.NewRoute("GET", "/([^/]+)/([^/]+)/([a-z0-9]+)", imgRequest),
91 )
92
93 return routes
1@@ -11,6 +11,7 @@ import (
2 "strings"
3
4 "github.com/disintegration/imaging"
5+ "github.com/kolesa-team/go-webp/decoder"
6 "github.com/kolesa-team/go-webp/encoder"
7 "github.com/kolesa-team/go-webp/webp"
8 "go.uber.org/zap"
9@@ -26,27 +27,11 @@ type ImgOptimizer struct {
10 // Specify the compression factor for RGB channels between 0 and 100. The default is 75.
11 // A small factor produces a smaller file with lower quality.
12 // Best quality is achieved by using a value of 100.
13- Quality float32
14- Optimized bool
15+ Quality float32
16 *Ratio
17 DeviceType deviceType
18 }
19
20-func (h *ImgOptimizer) GetImage(r io.Reader, mimeType string) (image.Image, error) {
21- switch mimeType {
22- case "image/png":
23- return png.Decode(r)
24- case "image/jpeg":
25- return jpeg.Decode(r)
26- case "image/jpg":
27- return jpeg.Decode(r)
28- case "image/gif":
29- return gif.Decode(r)
30- }
31-
32- return nil, fmt.Errorf("(%s) not supported optimization", mimeType)
33-}
34-
35 type Ratio struct {
36 Width int
37 Height int
38@@ -75,6 +60,7 @@ func GetRatio(dimes string) (*Ratio, error) {
39 return &Ratio{Width: width, Height: 0}, nil
40 }
41
42+ // dimes = 250x250
43 res := strings.Split(dimes, "x")
44 if len(res) != 2 {
45 return nil, fmt.Errorf("(%s) must be in format (x200, 200x, or 200x200)", dimes)
46@@ -96,35 +82,41 @@ func GetRatio(dimes string) (*Ratio, error) {
47 return ratio, nil
48 }
49
50-type SubImager interface {
51- SubImage(r image.Rectangle) image.Image
52-}
53+func (h *ImgOptimizer) GetImage(r io.Reader, mimeType string) (image.Image, error) {
54+ switch mimeType {
55+ case "image/png":
56+ return png.Decode(r)
57+ case "image/jpeg":
58+ return jpeg.Decode(r)
59+ case "image/jpg":
60+ return jpeg.Decode(r)
61+ case "image/gif":
62+ return gif.Decode(r)
63+ }
64
65-func (h *ImgOptimizer) Resize(img image.Image) *image.NRGBA {
66- return imaging.Resize(
67- img,
68- h.Width,
69- h.Height,
70- imaging.MitchellNetravali,
71- )
72+ return nil, fmt.Errorf("(%s) not supported for optimization", mimeType)
73 }
74
75-func (h *ImgOptimizer) Process(contents io.Reader, writer io.Writer, mimeType string) error {
76- if !h.Optimized {
77- _, err := io.Copy(writer, contents)
78- return err
79- }
80-
81- img, err := h.GetImage(contents, mimeType)
82+func (h *ImgOptimizer) DecodeWebp(r io.Reader) (image.Image, error) {
83+ opt := decoder.Options{}
84+ img, err := webp.Decode(r, &opt)
85 if err != nil {
86- return err
87+ return nil, err
88 }
89
90- nextImg := img
91- if h.Height > 0 || h.Width > 0 {
92- nextImg = h.Resize(img)
93+ if h.Width != 0 || h.Height != 0 {
94+ return imaging.Resize(
95+ img,
96+ h.Width,
97+ h.Height,
98+ imaging.MitchellNetravali,
99+ ), nil
100 }
101
102+ return img, nil
103+}
104+
105+func (h *ImgOptimizer) EncodeWebp(writer io.Writer, img image.Image) error {
106 options, err := encoder.NewLossyEncoderOptions(
107 encoder.PresetDefault,
108 h.Quality,
109@@ -133,14 +125,22 @@ func (h *ImgOptimizer) Process(contents io.Reader, writer io.Writer, mimeType st
110 return err
111 }
112
113- return webp.Encode(writer, nextImg, options)
114+ return webp.Encode(writer, img, options)
115+}
116+
117+func (h *ImgOptimizer) Process(writer io.Writer, contents io.Reader) error {
118+ img, err := h.DecodeWebp(contents)
119+ if err != nil {
120+ return err
121+ }
122+
123+ return h.EncodeWebp(writer, img)
124 }
125
126-func NewImgOptimizer(logger *zap.SugaredLogger, optimized bool, dimes string) *ImgOptimizer {
127+func NewImgOptimizer(logger *zap.SugaredLogger, dimes string) *ImgOptimizer {
128 opt := &ImgOptimizer{
129- Optimized: optimized,
130 DeviceType: desktopDevice,
131- Quality: 75,
132+ Quality: 80,
133 Ratio: &Ratio{Width: 0, Height: 0},
134 }
135