Eric Bower
·
18 Dec 24
web.go
1package pgs
2
3import (
4 "bufio"
5 "context"
6 "fmt"
7 "log/slog"
8 "net/http"
9 "net/url"
10 "os"
11 "regexp"
12 "strings"
13 "time"
14
15 _ "net/http/pprof"
16
17 "github.com/darkweak/souin/configurationtypes"
18 "github.com/darkweak/souin/pkg/middleware"
19 "github.com/darkweak/souin/plugins/souin/storages"
20 "github.com/darkweak/storages/core"
21 "github.com/gorilla/feeds"
22 "github.com/picosh/pico/db"
23 "github.com/picosh/pico/db/postgres"
24 "github.com/picosh/pico/shared"
25 "github.com/picosh/pico/shared/storage"
26 sst "github.com/picosh/pobj/storage"
27 "google.golang.org/protobuf/proto"
28)
29
30type CachedHttp struct {
31 handler *middleware.SouinBaseHandler
32 routes *WebRouter
33}
34
35func (c *CachedHttp) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
36 err := c.handler.ServeHTTP(writer, req, func(w http.ResponseWriter, r *http.Request) error {
37 c.routes.ServeHTTP(w, r)
38 return nil
39 })
40 if err != nil {
41 c.routes.Logger.Error("serve http", "err", err)
42 }
43}
44
45func StartApiServer() {
46 ctx := context.Background()
47 cfg := NewConfigSite()
48 logger := cfg.Logger
49
50 dbpool := postgres.NewDB(cfg.DbURL, cfg.Logger)
51 defer dbpool.Close()
52
53 var st storage.StorageServe
54 var err error
55 if cfg.MinioURL == "" {
56 st, err = storage.NewStorageFS(cfg.StorageDir)
57 } else {
58 st, err = storage.NewStorageMinio(cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
59 }
60
61 if err != nil {
62 logger.Error("could not connect to object storage", "err", err.Error())
63 return
64 }
65 ttl := configurationtypes.Duration{Duration: cfg.CacheTTL}
66 stale := configurationtypes.Duration{Duration: cfg.CacheTTL * 2}
67 c := &middleware.BaseConfiguration{
68 API: configurationtypes.API{
69 Prometheus: configurationtypes.APIEndpoint{
70 Enable: true,
71 },
72 },
73 DefaultCache: &configurationtypes.DefaultCache{
74 TTL: ttl,
75 Stale: stale,
76 Otter: configurationtypes.CacheProvider{
77 Uuid: fmt.Sprintf("OTTER-%s", stale),
78 Configuration: map[string]interface{}{},
79 },
80 Regex: configurationtypes.Regex{
81 Exclude: "/check",
82 },
83 MaxBodyBytes: uint64(cfg.MaxAssetSize),
84 DefaultCacheControl: cfg.CacheControl,
85 },
86 LogLevel: "debug",
87 }
88 c.SetLogger(&CompatLogger{logger})
89 storages.InitFromConfiguration(c)
90 httpCache := middleware.NewHTTPCacheHandler(c)
91 routes := NewWebRouter(cfg, logger, dbpool, st)
92 cacher := &CachedHttp{
93 handler: httpCache,
94 routes: routes,
95 }
96
97 go routes.cacheMgmt(ctx, httpCache)
98
99 portStr := fmt.Sprintf(":%s", cfg.Port)
100 logger.Info(
101 "starting server on port",
102 "port", cfg.Port,
103 "domain", cfg.Domain,
104 )
105 err = http.ListenAndServe(portStr, cacher)
106 logger.Error(
107 "listen and serve",
108 "err", err.Error(),
109 )
110}
111
112type HasPerm = func(proj *db.Project) bool
113
114type WebRouter struct {
115 Cfg *shared.ConfigSite
116 Logger *slog.Logger
117 Dbpool db.DB
118 Storage storage.StorageServe
119 RootRouter *http.ServeMux
120 UserRouter *http.ServeMux
121}
122
123func NewWebRouter(cfg *shared.ConfigSite, logger *slog.Logger, dbpool db.DB, st storage.StorageServe) *WebRouter {
124 router := &WebRouter{
125 Cfg: cfg,
126 Logger: logger,
127 Dbpool: dbpool,
128 Storage: st,
129 }
130 router.initRouters()
131 return router
132}
133
134func (web *WebRouter) initRouters() {
135 // ensure legacy router is disabled
136 // GODEBUG=httpmuxgo121=0
137
138 // root domain
139 rootRouter := http.NewServeMux()
140 rootRouter.HandleFunc("GET /check", web.checkHandler)
141 rootRouter.Handle("GET /main.css", web.serveFile("main.css", "text/css"))
142 rootRouter.Handle("GET /favicon-16x16.png", web.serveFile("favicon-16x16.png", "image/png"))
143 rootRouter.Handle("GET /apple-touch-icon.png", web.serveFile("apple-touch-icon.png", "image/png"))
144 rootRouter.Handle("GET /favicon.ico", web.serveFile("favicon.ico", "image/x-icon"))
145 rootRouter.Handle("GET /robots.txt", web.serveFile("robots.txt", "text/plain"))
146
147 rootRouter.Handle("GET /rss/updated", web.createRssHandler("updated_at"))
148 rootRouter.Handle("GET /rss", web.createRssHandler("created_at"))
149 rootRouter.Handle("GET /{$}", web.createPageHandler("html/marketing.page.tmpl"))
150 web.RootRouter = rootRouter
151
152 // subdomain or custom domains
153 userRouter := http.NewServeMux()
154 userRouter.HandleFunc("GET /{fname...}", web.AssetRequest)
155 userRouter.HandleFunc("GET /{$}", web.AssetRequest)
156 web.UserRouter = userRouter
157}
158
159func (web *WebRouter) serveFile(file string, contentType string) http.HandlerFunc {
160 return func(w http.ResponseWriter, r *http.Request) {
161 logger := web.Logger
162 cfg := web.Cfg
163
164 contents, err := os.ReadFile(cfg.StaticPath(fmt.Sprintf("public/%s", file)))
165 if err != nil {
166 logger.Error(
167 "could not read file",
168 "fname", file,
169 "err", err.Error(),
170 )
171 http.Error(w, "file not found", 404)
172 }
173
174 w.Header().Add("Content-Type", contentType)
175
176 _, err = w.Write(contents)
177 if err != nil {
178 logger.Error(
179 "could not write http response",
180 "file", file,
181 "err", err.Error(),
182 )
183 }
184 }
185}
186
187func (web *WebRouter) createPageHandler(fname string) http.HandlerFunc {
188 return func(w http.ResponseWriter, r *http.Request) {
189 logger := web.Logger
190 cfg := web.Cfg
191 ts, err := shared.RenderTemplate(cfg, []string{cfg.StaticPath(fname)})
192
193 if err != nil {
194 logger.Error(
195 "could not render template",
196 "fname", fname,
197 "err", err.Error(),
198 )
199 http.Error(w, err.Error(), http.StatusInternalServerError)
200 return
201 }
202
203 data := shared.PageData{
204 Site: *cfg.GetSiteData(),
205 }
206 err = ts.Execute(w, data)
207 if err != nil {
208 logger.Error(
209 "could not execute template",
210 "fname", fname,
211 "err", err.Error(),
212 )
213 http.Error(w, err.Error(), http.StatusInternalServerError)
214 }
215 }
216}
217
218func (web *WebRouter) checkHandler(w http.ResponseWriter, r *http.Request) {
219 dbpool := web.Dbpool
220 cfg := web.Cfg
221 logger := web.Logger
222
223 if cfg.IsCustomdomains() {
224 hostDomain := r.URL.Query().Get("domain")
225 appDomain := strings.Split(cfg.Domain, ":")[0]
226
227 if !strings.Contains(hostDomain, appDomain) {
228 subdomain := shared.GetCustomDomain(hostDomain, cfg.Space)
229 props, err := shared.GetProjectFromSubdomain(subdomain)
230 if err != nil {
231 logger.Error(
232 "could not get project from subdomain",
233 "subdomain", subdomain,
234 "err", err.Error(),
235 )
236 w.WriteHeader(http.StatusNotFound)
237 return
238 }
239
240 u, err := dbpool.FindUserForName(props.Username)
241 if err != nil {
242 logger.Error("could not find user", "err", err.Error())
243 w.WriteHeader(http.StatusNotFound)
244 return
245 }
246
247 logger = logger.With(
248 "user", u.Name,
249 "project", props.ProjectName,
250 )
251 p, err := dbpool.FindProjectByName(u.ID, props.ProjectName)
252 if err != nil {
253 logger.Error(
254 "could not find project for user",
255 "user", u.Name,
256 "project", props.ProjectName,
257 "err", err.Error(),
258 )
259 w.WriteHeader(http.StatusNotFound)
260 return
261 }
262
263 if u != nil && p != nil {
264 w.WriteHeader(http.StatusOK)
265 return
266 }
267 }
268 }
269
270 w.WriteHeader(http.StatusNotFound)
271}
272
273func (web *WebRouter) cacheMgmt(ctx context.Context, httpCache *middleware.SouinBaseHandler) {
274 storer := httpCache.Storers[0]
275 drain := createSubCacheDrain(ctx, web.Logger)
276
277 for {
278 scanner := bufio.NewScanner(drain)
279 for scanner.Scan() {
280 surrogateKey := strings.TrimSpace(scanner.Text())
281 web.Logger.Info("received cache-drain item", "surrogateKey", surrogateKey)
282
283 if surrogateKey == "*" {
284 storer.DeleteMany(".+")
285 err := httpCache.SurrogateKeyStorer.Destruct()
286 if err != nil {
287 web.Logger.Error("could not clear cache and surrogate key store", "err", err)
288 } else {
289 web.Logger.Info("successfully cleared cache and surrogate keys store")
290 }
291 continue
292 }
293
294 var header http.Header = map[string][]string{}
295 header.Add("Surrogate-Key", surrogateKey)
296
297 ck, _ := httpCache.SurrogateKeyStorer.Purge(header)
298 for _, key := range ck {
299 key, _ = strings.CutPrefix(key, core.MappingKeyPrefix)
300 if b := storer.Get(core.MappingKeyPrefix + key); len(b) > 0 {
301 var mapping core.StorageMapper
302 if e := proto.Unmarshal(b, &mapping); e == nil {
303 for k := range mapping.GetMapping() {
304 qkey, _ := url.QueryUnescape(k)
305 web.Logger.Info(
306 "deleting key from surrogate cache",
307 "surrogateKey", surrogateKey,
308 "key", qkey,
309 )
310 storer.Delete(qkey)
311 }
312 }
313 }
314
315 qkey, _ := url.QueryUnescape(key)
316 web.Logger.Info(
317 "deleting from cache",
318 "surrogateKey", surrogateKey,
319 "key", core.MappingKeyPrefix+qkey,
320 )
321 storer.Delete(core.MappingKeyPrefix + qkey)
322 }
323 }
324 }
325}
326
327func (web *WebRouter) createRssHandler(by string) http.HandlerFunc {
328 return func(w http.ResponseWriter, r *http.Request) {
329 dbpool := web.Dbpool
330 logger := web.Logger
331 cfg := web.Cfg
332
333 pager, err := dbpool.FindAllProjects(&db.Pager{Num: 100, Page: 0}, by)
334 if err != nil {
335 logger.Error("could not find projects", "err", err.Error())
336 http.Error(w, err.Error(), http.StatusInternalServerError)
337 return
338 }
339
340 feed := &feeds.Feed{
341 Title: fmt.Sprintf("%s discovery feed %s", cfg.Domain, by),
342 Link: &feeds.Link{Href: cfg.ReadURL()},
343 Description: fmt.Sprintf("%s projects %s", cfg.Domain, by),
344 Author: &feeds.Author{Name: cfg.Domain},
345 Created: time.Now(),
346 }
347
348 var feedItems []*feeds.Item
349 for _, project := range pager.Data {
350 realUrl := strings.TrimSuffix(
351 cfg.AssetURL(project.Username, project.Name, ""),
352 "/",
353 )
354 uat := project.UpdatedAt.Unix()
355 id := realUrl
356 title := fmt.Sprintf("%s-%s", project.Username, project.Name)
357 if by == "updated_at" {
358 id = fmt.Sprintf("%s:%d", realUrl, uat)
359 title = fmt.Sprintf("%s - %d", title, uat)
360 }
361
362 item := &feeds.Item{
363 Id: id,
364 Title: title,
365 Link: &feeds.Link{Href: realUrl},
366 Content: fmt.Sprintf(`<a href="%s">%s</a>`, realUrl, realUrl),
367 Created: *project.CreatedAt,
368 Updated: *project.CreatedAt,
369 Description: "",
370 Author: &feeds.Author{Name: project.Username},
371 }
372
373 feedItems = append(feedItems, item)
374 }
375 feed.Items = feedItems
376
377 rss, err := feed.ToAtom()
378 if err != nil {
379 logger.Error("could not convert feed to atom", "err", err.Error())
380 http.Error(w, "Could not generate atom rss feed", http.StatusInternalServerError)
381 }
382
383 w.Header().Add("Content-Type", "application/atom+xml")
384 _, err = w.Write([]byte(rss))
385 if err != nil {
386 logger.Error("http write failed", "err", err.Error())
387 }
388 }
389}
390
391func (web *WebRouter) Perm(proj *db.Project) bool {
392 return proj.Acl.Type == "public"
393}
394
395var imgRegex = regexp.MustCompile("(.+.(?:jpg|jpeg|png|gif|webp|svg))(/.+)")
396
397func (web *WebRouter) AssetRequest(w http.ResponseWriter, r *http.Request) {
398 fname := r.PathValue("fname")
399 if imgRegex.MatchString(fname) {
400 web.ImageRequest(w, r)
401 return
402 }
403 web.ServeAsset(fname, nil, false, web.Perm, w, r)
404}
405
406func (web *WebRouter) ImageRequest(w http.ResponseWriter, r *http.Request) {
407 rawname := r.PathValue("fname")
408 matches := imgRegex.FindStringSubmatch(rawname)
409 fname := rawname
410 imgOpts := ""
411 if len(matches) >= 2 {
412 fname = matches[1]
413 }
414 if len(matches) >= 3 {
415 imgOpts = matches[2]
416 }
417
418 opts, err := storage.UriToImgProcessOpts(imgOpts)
419 if err != nil {
420 errMsg := fmt.Sprintf("error processing img options: %s", err.Error())
421 web.Logger.Error("error processing img options", "err", errMsg)
422 http.Error(w, errMsg, http.StatusUnprocessableEntity)
423 return
424 }
425
426 web.ServeAsset(fname, opts, false, web.Perm, w, r)
427}
428
429func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, fromImgs bool, hasPerm HasPerm, w http.ResponseWriter, r *http.Request) {
430 subdomain := shared.GetSubdomain(r)
431
432 logger := web.Logger.With(
433 "subdomain", subdomain,
434 "filename", fname,
435 "url", fmt.Sprintf("%s%s", r.Host, r.URL.Path),
436 "host", r.Host,
437 )
438
439 props, err := shared.GetProjectFromSubdomain(subdomain)
440 if err != nil {
441 logger.Info(
442 "could not determine project from subdomain",
443 "err", err,
444 )
445 http.Error(w, err.Error(), http.StatusNotFound)
446 return
447 }
448
449 logger = logger.With(
450 "project", props.ProjectName,
451 "user", props.Username,
452 )
453
454 user, err := web.Dbpool.FindUserForName(props.Username)
455 if err != nil {
456 logger.Info("user not found")
457 http.Error(w, "user not found", http.StatusNotFound)
458 return
459 }
460
461 logger = logger.With(
462 "userId", user.ID,
463 )
464
465 projectID := ""
466 // TODO: this could probably be cleaned up more
467 // imgs wont have a project directory
468 projectDir := ""
469 var bucket sst.Bucket
470 // imgs has a different bucket directory
471 if fromImgs {
472 bucket, err = web.Storage.GetBucket(shared.GetImgsBucketName(user.ID))
473 } else {
474 bucket, err = web.Storage.GetBucket(shared.GetAssetBucketName(user.ID))
475 project, err := web.Dbpool.FindProjectByName(user.ID, props.ProjectName)
476 if err != nil {
477 logger.Info("project not found")
478 http.Error(w, "project not found", http.StatusNotFound)
479 return
480 }
481
482 logger = logger.With(
483 "projectId", project.ID,
484 "project", project.Name,
485 )
486
487 if project.Blocked != "" {
488 logger.Error("project has been blocked")
489 http.Error(w, project.Blocked, http.StatusForbidden)
490 return
491 }
492
493 projectID = project.ID
494 projectDir = project.ProjectDir
495 if !hasPerm(project) {
496 http.Error(w, "You do not have access to this site", http.StatusUnauthorized)
497 return
498 }
499 }
500
501 if err != nil {
502 logger.Info("bucket not found")
503 http.Error(w, "bucket not found", http.StatusNotFound)
504 return
505 }
506
507 hasPicoPlus := web.Dbpool.HasFeatureForUser(user.ID, "plus")
508
509 asset := &ApiAssetHandler{
510 WebRouter: web,
511 Logger: logger,
512
513 Username: props.Username,
514 UserID: user.ID,
515 Subdomain: subdomain,
516 ProjectDir: projectDir,
517 Filepath: fname,
518 Bucket: bucket,
519 ImgProcessOpts: opts,
520 ProjectID: projectID,
521 HasPicoPlus: hasPicoPlus,
522 }
523
524 asset.ServeHTTP(w, r)
525}
526
527func (web *WebRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
528 subdomain := shared.GetSubdomainFromRequest(r, web.Cfg.Domain, web.Cfg.Space)
529 if web.RootRouter == nil || web.UserRouter == nil {
530 web.Logger.Error("routers not initialized")
531 http.Error(w, "routers not initialized", http.StatusInternalServerError)
532 return
533 }
534
535 var router *http.ServeMux
536 if subdomain == "" {
537 router = web.RootRouter
538 } else {
539 router = web.UserRouter
540 }
541
542 // enable cors
543 // TODO: I don't think we want this for pgs as a default
544 // users can enable cors headers using `_headers` file
545 /* if r.Method == "OPTIONS" {
546 shared.CorsHeaders(w.Header())
547 w.WriteHeader(http.StatusOK)
548 return
549 }
550 shared.CorsHeaders(w.Header()) */
551
552 ctx := r.Context()
553 ctx = context.WithValue(ctx, shared.CtxSubdomainKey{}, subdomain)
554 router.ServeHTTP(w, r.WithContext(ctx))
555}
556
557type CompatLogger struct {
558 logger *slog.Logger
559}
560
561func (cl *CompatLogger) marshall(int ...interface{}) string {
562 res := ""
563 for _, val := range int {
564 switch r := val.(type) {
565 case string:
566 res += " " + r
567 }
568 }
569 return res
570}
571func (cl *CompatLogger) DPanic(int ...interface{}) {
572 cl.logger.Error("panic", "output", cl.marshall(int))
573}
574func (cl *CompatLogger) DPanicf(st string, int ...interface{}) {
575 cl.logger.Error(fmt.Sprintf(st, int...))
576}
577func (cl *CompatLogger) Debug(int ...interface{}) {
578 cl.logger.Debug("debug", "output", cl.marshall(int))
579}
580func (cl *CompatLogger) Debugf(st string, int ...interface{}) {
581 cl.logger.Debug(fmt.Sprintf(st, int...))
582}
583func (cl *CompatLogger) Error(int ...interface{}) {
584 cl.logger.Error("error", "output", cl.marshall(int))
585}
586func (cl *CompatLogger) Errorf(st string, int ...interface{}) {
587 cl.logger.Error(fmt.Sprintf(st, int...))
588}
589func (cl *CompatLogger) Fatal(int ...interface{}) {
590 cl.logger.Error("fatal", "outpu", cl.marshall(int))
591}
592func (cl *CompatLogger) Fatalf(st string, int ...interface{}) {
593 cl.logger.Error(fmt.Sprintf(st, int...))
594}
595func (cl *CompatLogger) Info(int ...interface{}) {
596 cl.logger.Info("info", "output", cl.marshall(int))
597}
598func (cl *CompatLogger) Infof(st string, int ...interface{}) {
599 cl.logger.Info(fmt.Sprintf(st, int...))
600}
601func (cl *CompatLogger) Panic(int ...interface{}) {
602 cl.logger.Error("panic", "output", cl.marshall(int))
603}
604func (cl *CompatLogger) Panicf(st string, int ...interface{}) {
605 cl.logger.Error(fmt.Sprintf(st, int...))
606}
607func (cl *CompatLogger) Warn(int ...interface{}) {
608 cl.logger.Warn("warn", "output", cl.marshall(int))
609}
610func (cl *CompatLogger) Warnf(st string, int ...interface{}) {
611 cl.logger.Warn(fmt.Sprintf(st, int...))
612}