- commit
- 55ef1d6
- parent
- 5fbb376
- author
- Eric Bower
- date
- 2023-03-16 02:54:54 +0000 UTC
chore(imgs): add in-memory caching for resizing images This caching could also be used for the other services as well now.
+6,
-1
1@@ -3,7 +3,9 @@ package feeds
2 import (
3 "fmt"
4 "net/http"
5+ "time"
6
7+ gocache "github.com/patrickmn/go-cache"
8 "github.com/picosh/pico/db/postgres"
9 "github.com/picosh/pico/imgs/storage"
10 "github.com/picosh/pico/shared"
11@@ -51,6 +53,9 @@ func StartApiServer() {
12 } else {
13 st, err = storage.NewStorageMinio(cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
14 }
15+
16+ cache := gocache.New(2*time.Minute, 5*time.Minute)
17+
18 if err != nil {
19 logger.Fatal(err)
20 }
21@@ -67,7 +72,7 @@ func StartApiServer() {
22
23 mainRoutes := createMainRoutes(staticRoutes)
24
25- handler := shared.CreateServe(mainRoutes, []shared.Route{}, cfg, db, st, logger)
26+ handler := shared.CreateServe(mainRoutes, []shared.Route{}, cfg, db, st, logger, cache)
27 router := http.HandlerFunc(handler)
28
29 portStr := fmt.Sprintf(":%s", cfg.Port)
M
go.mod
+1,
-0
1@@ -22,6 +22,7 @@ require (
2 github.com/mmcdole/gofeed v1.1.3
3 github.com/muesli/reflow v0.3.0
4 github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577
5+ github.com/patrickmn/go-cache v2.1.0+incompatible
6 github.com/pkg/sftp v1.13.5
7 github.com/sendgrid/sendgrid-go v3.12.0+incompatible
8 github.com/yuin/goldmark v1.5.3
M
go.sum
+2,
-0
1@@ -189,6 +189,8 @@ github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577 h1:hVmVNt
2 github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577/go.mod h1:G3Cu1AW+dmRLDFpOi8eUAfc3cGoRHUjTkGjeRcndgl4=
3 github.com/neurosnap/go-jpeg-image-structure v0.0.0-20221010133817-70b1c1ff679e h1:76Dng5ms0fR+26doKZAvNqhi2UPfnLxGfPIDEr+BBlM=
4 github.com/neurosnap/go-jpeg-image-structure v0.0.0-20221010133817-70b1c1ff679e/go.mod h1:nZBDA7+RD63GDJwjZmxhxac65MJqiCIHUUUvdYOsFkk=
5+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
6+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
7 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
8 github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
9 github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
+75,
-6
1@@ -12,6 +12,7 @@ import (
2 _ "net/http/pprof"
3
4 "github.com/gorilla/feeds"
5+ gocache "github.com/patrickmn/go-cache"
6 "github.com/picosh/pico/db"
7 "github.com/picosh/pico/db/postgres"
8 "github.com/picosh/pico/imgs/storage"
9@@ -188,6 +189,7 @@ type ImgHandler struct {
10 Dbpool db.DB
11 Storage storage.ObjectStorage
12 Logger *zap.SugaredLogger
13+ Cache *gocache.Cache
14 Img *shared.ImgOptimizer
15 // We should try to use the optimized image if it's available
16 // not all images are optimized so this flag isn't enough
17@@ -195,6 +197,54 @@ type ImgHandler struct {
18 UseOptimized bool
19 }
20
21+type ImgResizer struct {
22+ Key string
23+ contents storage.ReaderAtCloser
24+ writer io.Writer
25+ Img *shared.ImgOptimizer
26+ Cache *gocache.Cache
27+}
28+
29+func (r *ImgResizer) Resize() error {
30+ cached, found := r.Cache.Get(r.Key)
31+ if found {
32+ reader := bytes.NewReader(cached.([]byte))
33+ _, err := io.Copy(r.writer, reader)
34+ if err != nil {
35+ return err
36+ }
37+ return nil
38+ }
39+
40+ // when resizing an image we don't want to mess with quality
41+ // since that was already applied when converting to webp
42+ r.Img.Quality = 100
43+ r.Img.Lossless = false
44+ img, err := r.Img.DecodeWebp(r.contents)
45+ if err != nil {
46+ return err
47+ }
48+
49+ buf := new(bytes.Buffer)
50+ err = r.Img.EncodeWebp(buf, img)
51+ if err != nil {
52+ return err
53+ }
54+
55+ r.Cache.Set(
56+ r.Key,
57+ buf.Bytes(),
58+ gocache.DefaultExpiration,
59+ )
60+
61+ err = r.Img.EncodeWebp(r.writer, img)
62+ if err != nil {
63+ return err
64+ }
65+
66+ return nil
67+}
68+
69 func imgHandler(w http.ResponseWriter, h *ImgHandler) {
70 user, err := h.Dbpool.FindUserForName(h.Username)
71 if err != nil {
72@@ -250,11 +300,21 @@ func imgHandler(w http.ResponseWriter, h *ImgHandler) {
73 resizeImg := h.Img.Width != 0 || h.Img.Height != 0
74
75 if h.UseOptimized && resizeImg && isWebOptimized {
76- // when resizing an image we don't want to mess with quality
77- // since that was already applied when converting to webp
78- h.Img.Quality = 100
79- h.Img.Lossless = false
80- err = h.Img.Process(w, contents)
81+ cacheKey := fmt.Sprintf(
82+ "%s/%s (%d:%d)",
83+ bucket.Name,
84+ fname,
85+ h.Img.Width,
86+ h.Img.Height,
87+ )
88+ resizer := ImgResizer{
89+ Img: h.Img,
90+ Cache: h.Cache,
91+ Key: cacheKey,
92+ contents: contents,
93+ writer: w,
94+ }
95+ err = resizer.Resize()
96 } else {
97 _, err = io.Copy(w, contents)
98 }
99@@ -283,6 +343,7 @@ func imgRequestOriginal(w http.ResponseWriter, r *http.Request) {
100 dbpool := shared.GetDB(r)
101 st := shared.GetStorage(r)
102 logger := shared.GetLogger(r)
103+ cache := shared.GetCache(r)
104
105 imgHandler(w, &ImgHandler{
106 Username: username,
107@@ -292,6 +353,7 @@ func imgRequestOriginal(w http.ResponseWriter, r *http.Request) {
108 Dbpool: dbpool,
109 Storage: st,
110 Logger: logger,
111+ Cache: cache,
112 Img: shared.NewImgOptimizer(logger, ""),
113 UseOptimized: false,
114 })
115@@ -319,6 +381,7 @@ func imgRequest(w http.ResponseWriter, r *http.Request) {
116 dbpool := shared.GetDB(r)
117 st := shared.GetStorage(r)
118 logger := shared.GetLogger(r)
119+ cache := shared.GetCache(r)
120
121 imgHandler(w, &ImgHandler{
122 Username: username,
123@@ -328,6 +391,7 @@ func imgRequest(w http.ResponseWriter, r *http.Request) {
124 Dbpool: dbpool,
125 Storage: st,
126 Logger: logger,
127+ Cache: cache,
128 Img: shared.NewImgOptimizer(logger, dimes),
129 UseOptimized: true,
130 })
131@@ -724,6 +788,11 @@ func StartApiServer() {
132 st, err = storage.NewStorageMinio(cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
133 }
134
135+ // cache resizing images since they are CPU-bound
136+ // we want to clear the cache since we are storing images
137+ // as []byte in-memory
138+ cache := gocache.New(2*time.Minute, 5*time.Minute)
139+
140 if err != nil {
141 logger.Fatal(err)
142 }
143@@ -737,7 +806,7 @@ func StartApiServer() {
144 mainRoutes := createMainRoutes(staticRoutes)
145 subdomainRoutes := createSubdomainRoutes(staticRoutes)
146
147- handler := shared.CreateServe(mainRoutes, subdomainRoutes, cfg, db, st, logger)
148+ handler := shared.CreateServe(mainRoutes, subdomainRoutes, cfg, db, st, logger, cache)
149 router := http.HandlerFunc(handler)
150
151 portStr := fmt.Sprintf(":%s", cfg.Port)
+5,
-1
1@@ -11,6 +11,7 @@ import (
2 "time"
3
4 "github.com/gorilla/feeds"
5+ gocache "github.com/patrickmn/go-cache"
6 "github.com/picosh/pico/db"
7 "github.com/picosh/pico/db/postgres"
8 "github.com/picosh/pico/imgs"
9@@ -757,6 +758,9 @@ func StartApiServer() {
10 } else {
11 st, err = storage.NewStorageMinio(cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
12 }
13+
14+ cache := gocache.New(2*time.Minute, 5*time.Minute)
15+
16 if err != nil {
17 logger.Fatal(err)
18 }
19@@ -770,7 +774,7 @@ func StartApiServer() {
20 mainRoutes := createMainRoutes(staticRoutes)
21 subdomainRoutes := createSubdomainRoutes(staticRoutes)
22
23- handler := shared.CreateServe(mainRoutes, subdomainRoutes, cfg, db, st, logger)
24+ handler := shared.CreateServe(mainRoutes, subdomainRoutes, cfg, db, st, logger, cache)
25 router := http.HandlerFunc(handler)
26
27 portStr := fmt.Sprintf(":%s", cfg.Port)
+4,
-1
1@@ -8,6 +8,7 @@ import (
2 "os"
3 "time"
4
5+ gocache "github.com/patrickmn/go-cache"
6 "github.com/picosh/pico/db"
7 "github.com/picosh/pico/db/postgres"
8 "github.com/picosh/pico/imgs/storage"
9@@ -394,6 +395,8 @@ func StartApiServer() {
10 st, err = storage.NewStorageMinio(cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
11 }
12
13+ cache := gocache.New(2*time.Minute, 5*time.Minute)
14+
15 if err != nil {
16 logger.Fatal(err)
17 }
18@@ -409,7 +412,7 @@ func StartApiServer() {
19 mainRoutes := createMainRoutes(staticRoutes)
20 subdomainRoutes := createSubdomainRoutes(staticRoutes)
21
22- handler := shared.CreateServe(mainRoutes, subdomainRoutes, cfg, db, st, logger)
23+ handler := shared.CreateServe(mainRoutes, subdomainRoutes, cfg, db, st, logger, cache)
24 router := http.HandlerFunc(handler)
25
26 portStr := fmt.Sprintf(":%s", cfg.Port)
+4,
-1
1@@ -11,6 +11,7 @@ import (
2 "time"
3
4 "github.com/gorilla/feeds"
5+ gocache "github.com/patrickmn/go-cache"
6 "github.com/picosh/pico/db"
7 "github.com/picosh/pico/db/postgres"
8 "github.com/picosh/pico/imgs"
9@@ -895,6 +896,8 @@ func StartApiServer() {
10 st, err = storage.NewStorageMinio(cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
11 }
12
13+ cache := gocache.New(2*time.Minute, 5*time.Minute)
14+
15 if err != nil {
16 logger.Fatal(err)
17 }
18@@ -908,7 +911,7 @@ func StartApiServer() {
19 mainRoutes := createMainRoutes(staticRoutes)
20 subdomainRoutes := createSubdomainRoutes(staticRoutes)
21
22- handler := shared.CreateServe(mainRoutes, subdomainRoutes, cfg, db, st, logger)
23+ handler := shared.CreateServe(mainRoutes, subdomainRoutes, cfg, db, st, logger, cache)
24 router := http.HandlerFunc(handler)
25
26 portStr := fmt.Sprintf(":%s", cfg.Port)
1@@ -9,6 +9,7 @@ import (
2 "regexp"
3 "strings"
4
5+ "github.com/patrickmn/go-cache"
6 "github.com/picosh/pico/db"
7 "github.com/picosh/pico/imgs/storage"
8 "go.uber.org/zap"
9@@ -45,7 +46,7 @@ func CreatePProfRoutes(routes []Route) []Route {
10
11 type ServeFn func(http.ResponseWriter, *http.Request)
12
13-func CreateServe(routes []Route, subdomainRoutes []Route, cfg *ConfigSite, dbpool db.DB, st storage.ObjectStorage, logger *zap.SugaredLogger) ServeFn {
14+func CreateServe(routes []Route, subdomainRoutes []Route, cfg *ConfigSite, dbpool db.DB, st storage.ObjectStorage, logger *zap.SugaredLogger, cache *cache.Cache) ServeFn {
15 return func(w http.ResponseWriter, r *http.Request) {
16 var allow []string
17 var subdomain string
18@@ -82,7 +83,8 @@ func CreateServe(routes []Route, subdomainRoutes []Route, cfg *ConfigSite, dbpoo
19 dbCtx := context.WithValue(subdomainCtx, ctxDBKey{}, dbpool)
20 storageCtx := context.WithValue(dbCtx, ctxStorageKey{}, st)
21 cfgCtx := context.WithValue(storageCtx, ctxCfg{}, cfg)
22- ctx := context.WithValue(cfgCtx, ctxKey{}, matches[1:])
23+ cacheCtx := context.WithValue(cfgCtx, ctxCacheKey{}, cache)
24+ ctx := context.WithValue(cacheCtx, ctxKey{}, matches[1:])
25 route.handler(w, r.WithContext(ctx))
26 return
27 }
28@@ -100,6 +102,7 @@ type ctxDBKey struct{}
29 type ctxStorageKey struct{}
30 type ctxKey struct{}
31 type ctxLoggerKey struct{}
32+type ctxCacheKey struct{}
33 type ctxSubdomainKey struct{}
34 type ctxCfg struct{}
35
36@@ -111,6 +114,10 @@ func GetLogger(r *http.Request) *zap.SugaredLogger {
37 return r.Context().Value(ctxLoggerKey{}).(*zap.SugaredLogger)
38 }
39
40+func GetCache(r *http.Request) *cache.Cache {
41+ return r.Context().Value(ctxCacheKey{}).(*cache.Cache)
42+}
43+
44 func GetDB(r *http.Request) db.DB {
45 return r.Context().Value(ctxDBKey{}).(db.DB)
46 }