- commit
- f1e1fbb
- parent
- 3f6a880
- author
- Eric Bower
- date
- 2023-11-13 19:29:07 +0000 UTC
refactor(imgs): use imgproxy instead of libwebp (#56) --------- Co-authored-by: Antonio Mika <antoniomika@gmail.com>
+11,
-0
1@@ -13,6 +13,17 @@ MINIO_PROMETHEUS_AUTH_TYPE=public
2 MINIO_PROMETHEUS_URL=
3 MINIO_PROMETHEUS_JOB_ID=minio
4
5+USE_IMGPROXY=1
6+IMGPROXY_URL=http://imgproxy:8080
7+IMGPROXY_ALLOWED_SOURCES=s3://,local://
8+IMGPROXY_LOCAL_FILESYSTEM_ROOT=/storage
9+IMGPROXY_USE_S3=true
10+IMGPROXY_S3_ENDPOINT=$MINIO_URL
11+IMGPROXY_KEY=6465616462656566 # deadbeef
12+IMGPROXY_SALT=6465616462656566 # deadbeef
13+AWS_ACCESS_KEY_ID=$MINIO_ROOT_USER
14+AWS_SECRET_ACCESS_KEY=$MINIO_ROOT_PASSWORD
15+
16 LISTS_CADDYFILE=./caddy/Caddyfile
17 LISTS_V4=
18 LISTS_V6=
+0,
-3
1@@ -21,9 +21,6 @@ jobs:
2 test:
3 runs-on: ubuntu-22.04
4 steps:
5- - name: Install package
6- run: |
7- sudo apt-get -y install libwebp-dev
8 - name: Set up Go
9 uses: actions/setup-go@v3
10 with:
+0,
-3
1@@ -8,9 +8,6 @@ jobs:
2 runs-on: ubuntu-latest
3 steps:
4 - uses: actions/checkout@v3
5- - name: Install package
6- run: |
7- sudo apt-get -y install libwebp-dev
8 - name: Set outputs
9 id: vars
10 run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+5,
-10
1@@ -3,12 +3,8 @@ LABEL maintainer="Pico Maintainers <hello@pico.sh>"
2
3 WORKDIR /app
4
5-RUN dpkg --add-architecture arm64 && dpkg --add-architecture amd64
6 RUN apt-get update
7-RUN apt-get install -y git ca-certificates \
8- libwebp-dev:amd64 libwebp-dev:arm64 \
9- crossbuild-essential-amd64 crossbuild-essential-arm64 \
10- libc-dev:amd64 libc-dev:arm64
11+RUN apt-get install -y git ca-certificates
12
13 COPY go.* ./
14
15@@ -22,17 +18,16 @@ ARG APP=lists
16 ARG TARGETOS
17 ARG TARGETARCH
18
19-ENV CGO_ENABLED=1
20-ENV LDFLAGS="-s -w -linkmode external -extldflags '-static -lm -pthread'"
21-ENV CC=/app/scripts/gccwrap.sh
22+ENV CGO_ENABLED=0
23+ENV LDFLAGS="-s -w"
24
25 ENV GOOS=${TARGETOS} GOARCH=${TARGETARCH}
26
27-RUN go build -ldflags "$LDFLAGS" -tags "netgo osusergo" -o /go/bin/${APP}-web ./cmd/${APP}/web
28+RUN go build -ldflags "$LDFLAGS" -o /go/bin/${APP}-web ./cmd/${APP}/web
29
30 FROM builder-web as builder-ssh
31
32-RUN go build -ldflags "$LDFLAGS" -tags "netgo osusergo" -o /go/bin/${APP}-ssh ./cmd/${APP}/ssh
33+RUN go build -ldflags "$LDFLAGS" -o /go/bin/${APP}-ssh ./cmd/${APP}/ssh
34
35 FROM scratch as release-web
36
M
Makefile
+1,
-1
1@@ -22,7 +22,7 @@ css:
2 .PHONY: css
3
4 lint:
5- $(DOCKER_CMD) run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:latest bash -c 'apt-get update > /dev/null 2>&1 && apt-get install -y libwebp-dev > /dev/null 2>&1; golangci-lint run -E goimports -E godot --timeout 10m'
6+ $(DOCKER_CMD) run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:latest run -E goimports -E godot --timeout 10m
7 .PHONY: lint
8
9 lint-dev:
+0,
-3
1@@ -21,9 +21,6 @@ This repo hosts the following pico services:
2
3 - `golang` >= 1.21.0
4 - `direnv` to load environment vars
5-- `webp` package dependency
6- - on mac can be installed with `brew install webp`
7- - on ubuntu can be installed with `sudo apt install libwebp-dev`
8
9 ```bash
10 cp ./.env.example .env
+0,
-78
1@@ -1,78 +0,0 @@
2-package main
3-
4-import (
5- "bytes"
6- "fmt"
7-
8- "github.com/picosh/pico/db"
9- "github.com/picosh/pico/db/postgres"
10- "github.com/picosh/pico/imgs"
11- "github.com/picosh/pico/shared"
12- "github.com/picosh/pico/shared/storage"
13- "github.com/picosh/pico/wish/send/utils"
14-)
15-
16-func main() {
17- cfg := imgs.NewConfigSite()
18- dbp := postgres.NewDB(cfg.DbURL, cfg.Logger)
19-
20- cfg.Logger.Info("fetching all img posts")
21- posts, err := dbp.FindAllPosts(&db.Pager{Num: 1000, Page: 0}, "imgs")
22- if err != nil {
23- panic(err)
24- }
25-
26- var st storage.ObjectStorage
27- if cfg.MinioURL == "" {
28- st, err = storage.NewStorageFS(cfg.StorageDir)
29- } else {
30- st, err = storage.NewStorageMinio(cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
31- }
32-
33- if err != nil {
34- panic(err)
35- }
36-
37- total := len(posts.Data)
38- cfg.Logger.Infof("%d posts", total)
39- for i, post := range posts.Data {
40- cfg.Logger.Infof("%d%% %s %s", ((i+1)/total)*100, post.Filename, post.MimeType)
41- bucket, err := st.GetBucket(post.UserID)
42- if err != nil {
43- cfg.Logger.Infof("bucket not found %s", post.UserID)
44- continue
45- }
46-
47- reader, _, _, err := st.GetFile(bucket, post.Filename)
48- if err != nil {
49- cfg.Logger.Infof("file not found %s/%s", post.UserID, post.Filename)
50- continue
51- }
52- defer reader.Close()
53-
54- opt := imgs.NewImgOptimizer(cfg.Logger, "")
55- contents := &bytes.Buffer{}
56- img, err := imgs.GetImageForOptimization(reader, post.MimeType)
57- if err != nil {
58- cfg.Logger.Error(err)
59- continue
60- }
61-
62- err = opt.EncodeWebp(contents, img)
63- if err != nil {
64- cfg.Logger.Error(err)
65- continue
66- }
67-
68- webpReader := bytes.NewReader(contents.Bytes())
69- _, err = st.PutFile(
70- bucket,
71- fmt.Sprintf("%s.webp", shared.SanitizeFileExt(post.Filename)),
72- utils.NopReaderAtCloser(webpReader),
73- &utils.FileEntry{},
74- )
75- if err != nil {
76- cfg.Logger.Error(err)
77- }
78- }
79-}
+12,
-5
1@@ -11,6 +11,13 @@ services:
2 ports:
3 - "9000:9000"
4 - "9001:9001"
5+ imgproxy:
6+ env_file:
7+ - .env.example
8+ volumes:
9+ - ./data/storage/data:/storage
10+ ports:
11+ - "8080:8080"
12 lists-web:
13 build:
14 args:
15@@ -79,7 +86,7 @@ services:
16 env_file:
17 - .env.example
18 volumes:
19- - ./data/imgs-storage/data:/app/.storage
20+ - ./data/storage/data:/app/.storage
21 ports:
22 - "3003:3000"
23 imgs-ssh:
24@@ -90,7 +97,7 @@ services:
25 env_file:
26 - .env.example
27 volumes:
28- - ./data/imgs-storage/data:/app/.storage
29+ - ./data/storage/data:/app/.storage
30 - ./data/imgs-ssh/data:/app/ssh_data
31 ports:
32 - "2223:2222"
33@@ -102,7 +109,7 @@ services:
34 env_file:
35 - .env.example
36 volumes:
37- - ./data/pgs-storage/data:/app/.storage
38+ - ./data/storage/data:/app/.storage
39 ports:
40 - "3004:3000"
41 pgs-ssh:
42@@ -113,7 +120,7 @@ services:
43 env_file:
44 - .env.example
45 volumes:
46- - ./data/pgs-storage/data:/app/.storage
47+ - ./data/storage/data:/app/.storage
48 - ./data/pgs-ssh/data:/app/ssh_data
49 ports:
50 - "2224:2222"
51@@ -156,4 +163,4 @@ services:
52 - ./data/certs:/certs
53 ports:
54 - "6697:6697"
55- - "8080:8080"
56+ - "8081:8080"
+10,
-7
1@@ -31,9 +31,12 @@ services:
2 - .env.prod
3 volumes:
4 - ./data/minio-data:/data
5- ports:
6- - "9000:9000"
7- - "9001:9001"
8+ imgproxy:
9+ env_file:
10+ - .env.prod
11+ volumes:
12+ - ./data/imgs-storage/data:/storage/imgs
13+ - ./data/pgs-storage/data:/storage/pgs
14 lists-caddy:
15 image: ghcr.io/picosh/pico/caddy:latest
16 restart: always
17@@ -191,7 +194,7 @@ services:
18 env_file:
19 - .env.prod
20 volumes:
21- - ./data/imgs-storage/data:/app/.storage
22+ - ./data/storage/data:/app/.storage
23 imgs-ssh:
24 networks:
25 imgs:
26@@ -200,7 +203,7 @@ services:
27 env_file:
28 - .env.prod
29 volumes:
30- - ./data/imgs-storage/data:/app/.storage
31+ - ./data/storage/data:/app/.storage
32 - ./data/imgs-ssh/data:/app/ssh_data
33 ports:
34 - "${IMGS_SSH_V4:-22}:2222"
35@@ -236,7 +239,7 @@ services:
36 env_file:
37 - .env.prod
38 volumes:
39- - ./data/pgs-storage/data:/app/.storage
40+ - ./data/storage/data:/app/.storage
41 pgs-ssh:
42 networks:
43 pgs:
44@@ -245,7 +248,7 @@ services:
45 env_file:
46 - .env.prod
47 volumes:
48- - ./data/pgs-storage/data:/app/.storage
49+ - ./data/storage/data:/app/.storage
50 - ./data/pgs-ssh/data:/app/ssh_data
51 ports:
52 - "${PGS_SSH_V4:-22}:2222"
+6,
-0
1@@ -13,6 +13,12 @@ services:
2 profiles:
3 - db
4 - all
5+ imgproxy:
6+ image: darthsim/imgproxy:latest
7+ restart: always
8+ profiles:
9+ - db
10+ - all
11 lists-web:
12 image: ghcr.io/picosh/pico/lists-web:latest
13 restart: always
+2,
-0
1@@ -22,6 +22,7 @@ func NewConfigSite() *shared.ConfigSite {
2 minioPass := shared.GetEnv("MINIO_ROOT_PASSWORD", "")
3 dbURL := shared.GetEnv("DATABASE_URL", "")
4 sendgridKey := shared.GetEnv("SENDGRID_API_KEY", "")
5+ useImgProxy := shared.GetEnv("USE_IMGPROXY", "1")
6
7 intro := "To get started, enter a username and email.\n"
8 intro += "Then upload a file containing a list of rss feeds (e.g. ~/feeds.txt)\n"
9@@ -32,6 +33,7 @@ func NewConfigSite() *shared.ConfigSite {
10 Debug: debug == "1",
11 SubdomainsEnabled: subdomains == "1",
12 CustomdomainsEnabled: customdomains == "1",
13+ UseImgProxy: useImgProxy == "1",
14 SendgridKey: sendgridKey,
15 ConfigCms: config.ConfigCms{
16 Domain: domain,
+0,
-57
1@@ -2,14 +2,12 @@ package uploadimgs
2
3 import (
4 "bytes"
5- "errors"
6 "fmt"
7 "strings"
8 "time"
9
10 "github.com/charmbracelet/ssh"
11 "github.com/picosh/pico/db"
12- "github.com/picosh/pico/imgs"
13 "github.com/picosh/pico/shared"
14 "github.com/picosh/pico/wish/send/utils"
15 )
16@@ -54,10 +52,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
17 }
18
19 reader := bytes.NewReader([]byte(data.Text))
20- tee := bytes.NewReader([]byte(data.Text))
21
22- // we want to keep the original file so people can use that
23- // if our webp optimizer doesn't work properly
24 fname, err := h.Storage.PutFile(
25 bucket,
26 data.Filename,
27@@ -68,53 +63,6 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
28 return err
29 }
30
31- opt := imgs.NewImgOptimizer(h.Cfg.Logger, "")
32- // for small images we want to preserve quality
33- // since it can have a dramatic effect
34- if data.FileSize < 3*shared.MB {
35- opt.Quality = 100
36- opt.Lossless = true
37- } else {
38- opt.Quality = 80
39- opt.Lossless = false
40- }
41-
42- var webpReader *bytes.Reader
43- contents := &bytes.Buffer{}
44-
45- img, err := imgs.GetImageForOptimization(tee, data.MimeType)
46- finalName := shared.SanitizeFileExt(data.Filename)
47- if errors.Is(err, imgs.ErrAlreadyWebPError) {
48- h.Cfg.Logger.Infof("(%s) is already webp, skipping encoding", data.Filename)
49- finalName = fmt.Sprintf("%s.webp", finalName)
50- webpReader = tee
51- } else if err != nil {
52- h.Cfg.Logger.Infof("(%s) is a file format (%s) that we cannot convert to webp, skipping encoding", data.Filename, data.MimeType)
53- webpReader = tee
54- } else {
55- err = opt.EncodeWebp(contents, img)
56- if err != nil {
57- return err
58- }
59-
60- finalName = fmt.Sprintf("%s.webp", finalName)
61- webpReader = bytes.NewReader(contents.Bytes())
62- }
63-
64- if webpReader == nil {
65- return fmt.Errorf("contents of webp file is nil")
66- }
67-
68- _, err = h.Storage.PutFile(
69- bucket,
70- finalName,
71- utils.NopReaderAtCloser(webpReader),
72- &utils.FileEntry{},
73- )
74- if err != nil {
75- return err
76- }
77-
78 data.Data = db.PostData{
79 ImgPath: fname,
80 }
81@@ -156,11 +104,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error {
82 if err != nil {
83 return err
84 }
85- webp := fmt.Sprintf("%s.webp", shared.SanitizeFileExt(data.Filename))
86- err = h.Storage.DeleteFile(bucket, webp)
87- if err != nil {
88- return err
89- }
90 } else if data.Cur == nil {
91 h.Cfg.Logger.Infof("(%s) not found, adding record", data.Filename)
92 insertPost := db.Post{
M
go.mod
+0,
-3
1@@ -12,10 +12,8 @@ require (
2 github.com/charmbracelet/promwish v0.7.0
3 github.com/charmbracelet/ssh v0.0.0-20230822194956-1a051f898e09
4 github.com/charmbracelet/wish v1.2.0
5- github.com/disintegration/imaging v1.6.2
6 github.com/google/go-cmp v0.6.0
7 github.com/gorilla/feeds v1.1.2
8- github.com/kolesa-team/go-webp v1.0.4
9 github.com/lib/pq v1.10.9
10 github.com/matryer/is v1.4.1
11 github.com/microcosm-cc/bluemonday v1.0.26
12@@ -110,7 +108,6 @@ require (
13 github.com/yusufpapurcu/wmi v1.2.3 // indirect
14 go.uber.org/multierr v1.11.0 // indirect
15 golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect
16- golang.org/x/image v0.14.0 // indirect
17 golang.org/x/net v0.18.0 // indirect
18 golang.org/x/sync v0.5.0 // indirect
19 golang.org/x/sys v0.14.0 // indirect
M
go.sum
+0,
-8
1@@ -44,8 +44,6 @@ github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:Yyn
2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
4 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5-github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
6-github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
7 github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
8 github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
9 github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
10@@ -120,8 +118,6 @@ github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs
11 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
12 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
13 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
14-github.com/kolesa-team/go-webp v1.0.4 h1:wQvU4PLG/X7RS0vAeyhiivhLRoxfLVRlDq4I3frdxIQ=
15-github.com/kolesa-team/go-webp v1.0.4/go.mod h1:oMvdivD6K+Q5qIIkVC2w4k2ZUnI1H+MyP7inwgWq9aA=
16 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
17 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
18 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
19@@ -271,10 +267,6 @@ golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
20 golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
21 golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w=
22 golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
23-golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
24-golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
25-golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
26-golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
27 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
28 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
29 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+25,
-106
1@@ -19,7 +19,6 @@ import (
2 "github.com/picosh/pico/db/postgres"
3 "github.com/picosh/pico/shared"
4 "github.com/picosh/pico/shared/storage"
5- "github.com/picosh/pico/wish/send/utils"
6 "go.uber.org/zap"
7 )
8
9@@ -192,59 +191,8 @@ type ImgHandler struct {
10 Storage storage.ObjectStorage
11 Logger *zap.SugaredLogger
12 Cache *gocache.Cache
13- Img *ImgOptimizer
14- // We should try to use the optimized image if it's available
15- // not all images are optimized so this flag isn't enough
16- // because we also need to check the mime type
17- UseOptimized bool
18-}
19-
20-type ImgResizer struct {
21- Key string
22- contents utils.ReaderAtCloser
23- writer io.Writer
24- Img *ImgOptimizer
25- Cache *gocache.Cache
26-}
27-
28-func (r *ImgResizer) Resize() error {
29- cached, found := r.Cache.Get(r.Key)
30- if found {
31- reader := bytes.NewReader(cached.([]byte))
32- _, err := io.Copy(r.writer, reader)
33- if err != nil {
34- return err
35- }
36- return nil
37- }
38-
39- // when resizing an image we don't want to mess with quality
40- // since that was already applied when converting to webp
41- r.Img.Quality = 100
42- r.Img.Lossless = false
43- img, err := r.Img.DecodeWebp(r.contents)
44- if err != nil {
45- return err
46- }
47-
48- buf := new(bytes.Buffer)
49- err = r.Img.EncodeWebp(buf, img)
50- if err != nil {
51- return err
52- }
53-
54- r.Cache.Set(
55- r.Key,
56- buf.Bytes(),
57- gocache.DefaultExpiration,
58- )
59-
60- err = r.Img.EncodeWebp(r.writer, img)
61- if err != nil {
62- return err
63- }
64-
65- return nil
66+ Ratio *storage.Ratio
67+ Original bool
68 }
69
70 func imgHandler(w http.ResponseWriter, h *ImgHandler) {
71@@ -275,23 +223,14 @@ func imgHandler(w http.ResponseWriter, h *ImgHandler) {
72 return
73 }
74
75- contentType := post.MimeType
76- fname := post.Filename
77- isWebOptimized := IsWebOptimized(contentType)
78-
79- if h.UseOptimized && isWebOptimized {
80- contentType = "image/webp"
81- fname = fmt.Sprintf("%s.webp", shared.SanitizeFileExt(post.Filename))
82- }
83-
84- contents, _, _, err := h.Storage.GetFile(bucket, fname)
85+ contents, contentType, err := h.Storage.ServeFile(bucket, post.Filename, h.Ratio, h.Original, h.Cfg.UseImgProxy)
86 if err != nil {
87 h.Logger.Infof(
88 "file not found %s/%s in storage (bucket: %s, name: %s)",
89 h.Username,
90 post.Filename,
91 bucket.Name,
92- fname,
93+ post.Filename,
94 )
95 http.Error(w, err.Error(), http.StatusInternalServerError)
96 return
97@@ -300,27 +239,7 @@ func imgHandler(w http.ResponseWriter, h *ImgHandler) {
98
99 w.Header().Add("Content-Type", contentType)
100
101- resizeImg := h.Img.Width != 0 || h.Img.Height != 0
102-
103- if h.UseOptimized && resizeImg && isWebOptimized {
104- cacheKey := fmt.Sprintf(
105- "%s/%s (%d:%d)",
106- bucket.Name,
107- fname,
108- h.Img.Width,
109- h.Img.Height,
110- )
111- resizer := ImgResizer{
112- Img: h.Img,
113- Cache: h.Cache,
114- Key: cacheKey,
115- contents: contents,
116- writer: w,
117- }
118- err = resizer.Resize()
119- } else {
120- _, err = io.Copy(w, contents)
121- }
122+ _, err = io.Copy(w, contents)
123
124 if err != nil {
125 h.Logger.Error(err)
126@@ -349,16 +268,15 @@ func imgRequestOriginal(w http.ResponseWriter, r *http.Request) {
127 cache := shared.GetCache(r)
128
129 imgHandler(w, &ImgHandler{
130- Username: username,
131- Subdomain: subdomain,
132- Slug: slug,
133- Cfg: cfg,
134- Dbpool: dbpool,
135- Storage: st,
136- Logger: logger,
137- Cache: cache,
138- Img: NewImgOptimizer(logger, ""),
139- UseOptimized: false,
140+ Username: username,
141+ Subdomain: subdomain,
142+ Slug: slug,
143+ Cfg: cfg,
144+ Dbpool: dbpool,
145+ Storage: st,
146+ Logger: logger,
147+ Cache: cache,
148+ Original: true,
149 })
150 }
151
152@@ -377,6 +295,8 @@ func imgRequest(w http.ResponseWriter, r *http.Request) {
153 dimes, _ = url.PathUnescape(shared.GetField(r, 1))
154 }
155
156+ ratio, _ := storage.GetRatio(dimes)
157+
158 // users might add the file extension when requesting an image
159 // but we want to remove that
160 slug = shared.SanitizeFileExt(slug)
161@@ -387,16 +307,15 @@ func imgRequest(w http.ResponseWriter, r *http.Request) {
162 cache := shared.GetCache(r)
163
164 imgHandler(w, &ImgHandler{
165- Username: username,
166- Subdomain: subdomain,
167- Slug: slug,
168- Cfg: cfg,
169- Dbpool: dbpool,
170- Storage: st,
171- Logger: logger,
172- Cache: cache,
173- Img: NewImgOptimizer(logger, dimes),
174- UseOptimized: true,
175+ Username: username,
176+ Subdomain: subdomain,
177+ Slug: slug,
178+ Cfg: cfg,
179+ Dbpool: dbpool,
180+ Storage: st,
181+ Logger: logger,
182+ Cache: cache,
183+ Ratio: ratio,
184 })
185 }
186
+2,
-0
1@@ -40,6 +40,7 @@ func NewConfigSite() *shared.ConfigSite {
2 minioUser := shared.GetEnv("MINIO_ROOT_USER", "")
3 minioPass := shared.GetEnv("MINIO_ROOT_PASSWORD", "")
4 dbURL := shared.GetEnv("DATABASE_URL", "")
5+ useImgProxy := shared.GetEnv("USE_IMGPROXY", "1")
6
7 intro := "To get started, enter a username.\n"
8 intro += "Then create a folder locally (e.g. ~/imgs).\n"
9@@ -50,6 +51,7 @@ func NewConfigSite() *shared.ConfigSite {
10 Debug: debug == "1",
11 SubdomainsEnabled: subdomains == "1",
12 CustomdomainsEnabled: customdomains == "1",
13+ UseImgProxy: useImgProxy == "1",
14 ConfigCms: config.ConfigCms{
15 Domain: domain,
16 Email: email,
+0,
-182
1@@ -1,182 +0,0 @@
2-package imgs
3-
4-import (
5- "errors"
6- "fmt"
7- "image"
8- gif "image/gif"
9- jpeg "image/jpeg"
10- png "image/png"
11- "io"
12- "strconv"
13- "strings"
14-
15- "github.com/disintegration/imaging"
16- "github.com/kolesa-team/go-webp/decoder"
17- "github.com/kolesa-team/go-webp/encoder"
18- "github.com/kolesa-team/go-webp/webp"
19- "go.uber.org/zap"
20-)
21-
22-type deviceType int
23-
24-const (
25- desktopDevice deviceType = iota
26-)
27-
28-type ImgOptimizer struct {
29- // Specify the compression factor for RGB channels between 0 and 100. The default is 75.
30- // A small factor produces a smaller file with lower quality.
31- // Best quality is achieved by using a value of 100.
32- Quality float32
33- *Ratio
34- DeviceType deviceType
35- Lossless bool
36-}
37-
38-type Ratio struct {
39- Width int
40- Height int
41-}
42-
43-var ErrAlreadyWebPError = errors.New("image is already webp")
44-var ErrIsSvgError = errors.New("image is an svg")
45-
46-func GetImageForOptimization(r io.Reader, mimeType string) (image.Image, error) {
47- switch mimeType {
48- case "image/png":
49- return png.Decode(r)
50- case "image/jpeg":
51- return jpeg.Decode(r)
52- case "image/jpg":
53- return jpeg.Decode(r)
54- case "image/gif":
55- return gif.Decode(r)
56- case "image/webp":
57- return nil, ErrAlreadyWebPError
58- }
59-
60- return nil, fmt.Errorf("(%s) not supported for optimization", mimeType)
61-}
62-
63-func IsWebOptimized(mimeType string) bool {
64- switch mimeType {
65- case "image/png", "image/jpeg", "image/jpg", "image/gif", "image/webp":
66- return true
67- }
68-
69- return false
70-}
71-
72-func GetRatio(dimes string) (*Ratio, error) {
73- if dimes == "" {
74- return nil, nil
75- }
76-
77- // dimes = x250 -- width is auto scaled and height is 250
78- if strings.HasPrefix(dimes, "x") {
79- height, err := strconv.Atoi(dimes[1:])
80- if err != nil {
81- return nil, err
82- }
83- return &Ratio{Width: 0, Height: height}, nil
84- }
85-
86- // dimes = 250x -- width is 250 and height is auto scaled
87- if strings.HasSuffix(dimes, "x") {
88- width, err := strconv.Atoi(dimes[:len(dimes)-1])
89- if err != nil {
90- return nil, err
91- }
92- return &Ratio{Width: width, Height: 0}, nil
93- }
94-
95- // dimes = 250x250
96- res := strings.Split(dimes, "x")
97- if len(res) != 2 {
98- return nil, fmt.Errorf("(%s) must be in format (x200, 200x, or 200x200)", dimes)
99- }
100-
101- ratio := &Ratio{}
102- width, err := strconv.Atoi(res[0])
103- if err != nil {
104- return nil, err
105- }
106- ratio.Width = width
107-
108- height, err := strconv.Atoi(res[1])
109- if err != nil {
110- return nil, err
111- }
112- ratio.Height = height
113-
114- return ratio, nil
115-}
116-
117-func (h *ImgOptimizer) DecodeWebp(r io.Reader) (image.Image, error) {
118- opt := decoder.Options{}
119- img, err := webp.Decode(r, &opt)
120- if err != nil {
121- return nil, err
122- }
123-
124- if h.Width != 0 || h.Height != 0 {
125- return imaging.Resize(
126- img,
127- h.Width,
128- h.Height,
129- imaging.CatmullRom,
130- ), nil
131- }
132-
133- return img, nil
134-}
135-
136-func (h *ImgOptimizer) EncodeWebp(writer io.Writer, img image.Image) error {
137- var options *encoder.Options
138- var err error
139- if h.Lossless {
140- options, err = encoder.NewLosslessEncoderOptions(
141- encoder.PresetDefault,
142- 1, // only value that I could get to work
143- )
144- } else {
145- options, err = encoder.NewLossyEncoderOptions(
146- encoder.PresetDefault,
147- h.Quality,
148- )
149- }
150- if err != nil {
151- return err
152- }
153-
154- return webp.Encode(writer, img, options)
155-}
156-
157-func (h *ImgOptimizer) Process(writer io.Writer, contents io.Reader) error {
158- img, err := h.DecodeWebp(contents)
159- if err != nil {
160- return err
161- }
162-
163- return h.EncodeWebp(writer, img)
164-}
165-
166-func NewImgOptimizer(logger *zap.SugaredLogger, dimes string) *ImgOptimizer {
167- opt := &ImgOptimizer{
168- DeviceType: desktopDevice,
169- Quality: 80,
170- Ratio: &Ratio{Width: 0, Height: 0},
171- }
172-
173- ratio, err := GetRatio(dimes)
174- if ratio != nil {
175- opt.Ratio = ratio
176- }
177-
178- if err != nil {
179- logger.Error(err)
180- }
181-
182- return opt
183-}
+2,
-0
1@@ -21,6 +21,7 @@ func NewConfigSite() *shared.ConfigSite {
2 minioUser := shared.GetEnv("MINIO_ROOT_USER", "")
3 minioPass := shared.GetEnv("MINIO_ROOT_PASSWORD", "")
4 dbURL := shared.GetEnv("DATABASE_URL", "")
5+ useImgProxy := shared.GetEnv("USE_IMGPROXY", "1")
6
7 intro := "To get started, enter a username.\n"
8 intro += "Then create a folder locally (e.g. ~/blog).\n"
9@@ -32,6 +33,7 @@ func NewConfigSite() *shared.ConfigSite {
10 Debug: debug == "1",
11 SubdomainsEnabled: subdomains == "1",
12 CustomdomainsEnabled: customdomains == "1",
13+ UseImgProxy: useImgProxy == "1",
14 ConfigCms: config.ConfigCms{
15 Domain: domain,
16 Email: email,
+2,
-0
1@@ -21,6 +21,7 @@ func NewConfigSite() *shared.ConfigSite {
2 minioURL := shared.GetEnv("MINIO_URL", "")
3 minioUser := shared.GetEnv("MINIO_ROOT_USER", "")
4 minioPass := shared.GetEnv("MINIO_ROOT_PASSWORD", "")
5+ useImgProxy := shared.GetEnv("USE_IMGPROXY", "1")
6
7 intro := "To get started, enter a username.\n"
8 intro += "Then all you need to do is send your pastes to us:\n\n"
9@@ -30,6 +31,7 @@ func NewConfigSite() *shared.ConfigSite {
10 Debug: debug == "1",
11 SubdomainsEnabled: subdomains == "1",
12 CustomdomainsEnabled: customdomains == "1",
13+ UseImgProxy: useImgProxy == "1",
14 ConfigCms: config.ConfigCms{
15 Domain: domain,
16 Port: port,
+1,
-1
1@@ -254,7 +254,7 @@ func assetHandler(w http.ResponseWriter, h *AssetHandler) {
2 }
3 defer contents.Close()
4
5- contentType := shared.GetMimeType(assetFilepath)
6+ contentType := storage.GetMimeType(assetFilepath)
7 w.Header().Add("Content-Type", contentType)
8 w.WriteHeader(status)
9 _, err = io.Copy(w, contents)
+2,
-0
1@@ -24,6 +24,7 @@ func NewConfigSite() *shared.ConfigSite {
2 minioUser := shared.GetEnv("MINIO_ROOT_USER", "")
3 minioPass := shared.GetEnv("MINIO_ROOT_PASSWORD", "")
4 dbURL := shared.GetEnv("DATABASE_URL", "")
5+ useImgProxy := shared.GetEnv("USE_IMGPROXY", "1")
6
7 intro := "To get started, enter a username.\n"
8 intro += "Then create a folder locally (e.g. ~/sites).\n"
9@@ -34,6 +35,7 @@ func NewConfigSite() *shared.ConfigSite {
10 Debug: debug == "1",
11 SubdomainsEnabled: subdomains == "1",
12 CustomdomainsEnabled: customdomains == "1",
13+ UseImgProxy: useImgProxy == "1",
14 ConfigCms: config.ConfigCms{
15 Domain: domain,
16 Email: email,
+2,
-0
1@@ -21,6 +21,7 @@ func NewConfigSite() *shared.ConfigSite {
2 minioUser := shared.GetEnv("MINIO_ROOT_USER", "")
3 minioPass := shared.GetEnv("MINIO_ROOT_PASSWORD", "")
4 dbURL := shared.GetEnv("DATABASE_URL", "")
5+ useImgProxy := shared.GetEnv("USE_IMGPROXY", "1")
6
7 intro := "To get started, enter a username.\n"
8 intro += "Then create a folder locally (e.g. ~/blog).\n"
9@@ -32,6 +33,7 @@ func NewConfigSite() *shared.ConfigSite {
10 Debug: debug == "1",
11 SubdomainsEnabled: subdomains == "1",
12 CustomdomainsEnabled: customdomains == "1",
13+ UseImgProxy: useImgProxy == "1",
14 ConfigCms: config.ConfigCms{
15 Domain: domain,
16 Email: email,
1@@ -31,6 +31,7 @@ type ConfigSite struct {
2 SubdomainsEnabled bool
3 CustomdomainsEnabled bool
4 SendgridKey string
5+ UseImgProxy bool
6 }
7
8 type CreateURL struct {
1@@ -98,6 +98,19 @@ func (s *StorageFS) GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser,
2 return dat, info.Size(), info.ModTime(), nil
3 }
4
5+func (s *StorageFS) ServeFile(bucket Bucket, fpath string, ratio *Ratio, original bool, useProxy bool) (io.ReadCloser, string, error) {
6+ if !useProxy || original || os.Getenv("IMGPROXY_URL") == "" {
7+ contentType := GetMimeType(fpath)
8+ rc, _, _, err := s.GetFile(bucket, fpath)
9+ return rc, contentType, err
10+ }
11+
12+ filePath := filepath.Join(bucket.Path, fpath)
13+ dataURL := fmt.Sprintf("local://%s", filePath)
14+
15+ return HandleProxy(dataURL, ratio, original, useProxy)
16+}
17+
18 func (s *StorageFS) PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error) {
19 loc := filepath.Join(bucket.Path, fpath)
20 err := os.MkdirAll(filepath.Dir(loc), os.ModePerm)
1@@ -4,8 +4,10 @@ import (
2 "context"
3 "errors"
4 "fmt"
5+ "io"
6 "net/url"
7 "os"
8+ "path/filepath"
9 "strconv"
10 "strings"
11 "time"
12@@ -173,6 +175,19 @@ func (s *StorageMinio) GetFile(bucket Bucket, fpath string) (utils.ReaderAtClose
13 return obj, info.Size, modTime, nil
14 }
15
16+func (s *StorageMinio) ServeFile(bucket Bucket, fpath string, ratio *Ratio, original bool, useProxy bool) (io.ReadCloser, string, error) {
17+ if !useProxy || original || os.Getenv("IMGPROXY_URL") == "" {
18+ contentType := GetMimeType(fpath)
19+ rc, _, _, err := s.GetFile(bucket, fpath)
20+ return rc, contentType, err
21+ }
22+
23+ filePath := filepath.Join(bucket.Name, fpath)
24+ dataURL := fmt.Sprintf("s3://%s", filePath)
25+
26+ return HandleProxy(dataURL, ratio, original, useProxy)
27+}
28+
29 func (s *StorageMinio) PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error) {
30 opts := minio.PutObjectOptions{}
31
1@@ -0,0 +1,105 @@
2+package storage
3+
4+import (
5+ "crypto/hmac"
6+ "crypto/sha256"
7+ "encoding/base64"
8+ "encoding/hex"
9+ "fmt"
10+ "io"
11+ "net/http"
12+ "os"
13+ "path/filepath"
14+)
15+
16+func GetMimeType(fpath string) string {
17+ ext := filepath.Ext(fpath)
18+ if ext == ".svg" {
19+ return "image/svg+xml"
20+ } else if ext == ".css" {
21+ return "text/css"
22+ } else if ext == ".js" {
23+ return "text/javascript"
24+ } else if ext == ".ico" {
25+ return "image/x-icon"
26+ } else if ext == ".pdf" {
27+ return "application/pdf"
28+ } else if ext == ".html" || ext == ".htm" {
29+ return "text/html"
30+ } else if ext == ".jpg" || ext == ".jpeg" {
31+ return "image/jpeg"
32+ } else if ext == ".png" {
33+ return "image/png"
34+ } else if ext == ".gif" {
35+ return "image/gif"
36+ } else if ext == ".webp" {
37+ return "image/webp"
38+ } else if ext == ".otf" {
39+ return "font/otf"
40+ } else if ext == ".woff" {
41+ return "font/woff"
42+ } else if ext == ".woff2" {
43+ return "font/woff2"
44+ } else if ext == ".ttf" {
45+ return "font/ttf"
46+ } else if ext == ".md" {
47+ return "text/markdown; charset=UTF-8"
48+ } else if ext == ".json" || ext == ".map" {
49+ return "application/json"
50+ } else if ext == ".rss" {
51+ return "application/rss+xml"
52+ } else if ext == ".atom" {
53+ return "application/atom+xml"
54+ } else if ext == ".webmanifest" {
55+ return "application/manifest+json"
56+ }
57+
58+ return "text/plain"
59+}
60+
61+func HandleProxy(dataURL string, ratio *Ratio, original bool, useProxy bool) (io.ReadCloser, string, error) {
62+ imgProxyURL := os.Getenv("IMGPROXY_URL")
63+ imgProxySalt := os.Getenv("IMGPROXY_SALT")
64+ imgProxyKey := os.Getenv("IMGPROXY_KEY")
65+
66+ signature := "_"
67+ processOpts := "q:80"
68+
69+ if ratio != nil {
70+ processOpts += fmt.Sprintf("/s:%d:%d", ratio.Width, ratio.Height)
71+ }
72+
73+ fileType := ".webp"
74+ if original {
75+ fileType = ""
76+ processOpts = "raw:1"
77+ }
78+
79+ processPath := fmt.Sprintf("/%s/%s%s", processOpts, base64.StdEncoding.EncodeToString([]byte(dataURL)), fileType)
80+
81+ if imgProxySalt != "" && imgProxyKey != "" {
82+ keyBin, err := hex.DecodeString(imgProxyKey)
83+ if err != nil {
84+ return nil, "", err
85+ }
86+
87+ saltBin, err := hex.DecodeString(imgProxySalt)
88+ if err != nil {
89+ return nil, "", err
90+ }
91+
92+ mac := hmac.New(sha256.New, keyBin)
93+ mac.Write(saltBin)
94+ mac.Write([]byte(processPath))
95+ signature = base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
96+ }
97+
98+ proxyAddress := fmt.Sprintf("%s/%s%s", imgProxyURL, signature, processPath)
99+
100+ res, err := http.Get(proxyAddress)
101+ if err != nil {
102+ return nil, "", err
103+ }
104+
105+ return res.Body, res.Header.Get("Content-Type"), nil
106+}
1@@ -0,0 +1,57 @@
2+package storage
3+
4+import (
5+ "fmt"
6+ "strconv"
7+ "strings"
8+)
9+
10+type Ratio struct {
11+ Width int
12+ Height int
13+}
14+
15+func GetRatio(dimes string) (*Ratio, error) {
16+ if dimes == "" {
17+ return nil, nil
18+ }
19+
20+ // dimes = x250 -- width is auto scaled and height is 250
21+ if strings.HasPrefix(dimes, "x") {
22+ height, err := strconv.Atoi(dimes[1:])
23+ if err != nil {
24+ return nil, err
25+ }
26+ return &Ratio{Width: 0, Height: height}, nil
27+ }
28+
29+ // dimes = 250x -- width is 250 and height is auto scaled
30+ if strings.HasSuffix(dimes, "x") {
31+ width, err := strconv.Atoi(dimes[:len(dimes)-1])
32+ if err != nil {
33+ return nil, err
34+ }
35+ return &Ratio{Width: width, Height: 0}, nil
36+ }
37+
38+ // dimes = 250x250
39+ res := strings.Split(dimes, "x")
40+ if len(res) != 2 {
41+ return nil, fmt.Errorf("(%s) must be in format (x200, 200x, or 200x200)", dimes)
42+ }
43+
44+ ratio := &Ratio{}
45+ width, err := strconv.Atoi(res[0])
46+ if err != nil {
47+ return nil, err
48+ }
49+ ratio.Width = width
50+
51+ height, err := strconv.Atoi(res[1])
52+ if err != nil {
53+ return nil, err
54+ }
55+ ratio.Height = height
56+
57+ return ratio, nil
58+}
1@@ -1,6 +1,7 @@
2 package storage
3
4 import (
5+ "io"
6 "os"
7 "time"
8
9@@ -10,6 +11,7 @@ import (
10 type Bucket struct {
11 Name string
12 Path string
13+ Root string
14 }
15
16 type ObjectStorage interface {
17@@ -20,6 +22,7 @@ type ObjectStorage interface {
18 DeleteBucket(bucket Bucket) error
19 GetBucketQuota(bucket Bucket) (uint64, error)
20 GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser, int64, time.Time, error)
21+ ServeFile(bucket Bucket, fpath string, ratio *Ratio, original bool, useProxy bool) (io.ReadCloser, string, error)
22 PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error)
23 DeleteFile(bucket Bucket, fpath string) error
24 ListFiles(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error)
1@@ -140,51 +140,6 @@ func Shasum(data []byte) string {
2 return hex.EncodeToString(bs)
3 }
4
5-func GetMimeType(fpath string) string {
6- ext := filepath.Ext(fpath)
7- if ext == ".svg" {
8- return "image/svg+xml"
9- } else if ext == ".css" {
10- return "text/css"
11- } else if ext == ".js" {
12- return "text/javascript"
13- } else if ext == ".ico" {
14- return "image/x-icon"
15- } else if ext == ".pdf" {
16- return "application/pdf"
17- } else if ext == ".html" || ext == ".htm" {
18- return "text/html"
19- } else if ext == ".jpg" || ext == ".jpeg" {
20- return "image/jpeg"
21- } else if ext == ".png" {
22- return "image/png"
23- } else if ext == ".gif" {
24- return "image/gif"
25- } else if ext == ".webp" {
26- return "image/webp"
27- } else if ext == ".otf" {
28- return "font/otf"
29- } else if ext == ".woff" {
30- return "font/woff"
31- } else if ext == ".woff2" {
32- return "font/woff2"
33- } else if ext == ".ttf" {
34- return "font/ttf"
35- } else if ext == ".md" {
36- return "text/markdown; charset=UTF-8"
37- } else if ext == ".json" || ext == ".map" {
38- return "application/json"
39- } else if ext == ".rss" {
40- return "application/rss+xml"
41- } else if ext == ".atom" {
42- return "application/atom+xml"
43- } else if ext == ".webmanifest" {
44- return "application/manifest+json"
45- }
46-
47- return "text/plain"
48-}
49-
50 func BytesToGB(size int) float32 {
51 return (((float32(size) / 1024) / 1024) / 1024)
52 }