- commit
- 94ad627
- parent
- e5109c2
- author
- Eric Bower
- date
- 2023-11-12 02:30:42 +0000 UTC
chore(pgs): rm objects from store when project no longer exists
6 files changed,
+254,
-65
+136,
-0
1@@ -0,0 +1,136 @@
2+package main
3+
4+import (
5+ "log"
6+ "os"
7+ "strings"
8+
9+ "github.com/picosh/pico/db"
10+ "github.com/picosh/pico/db/postgres"
11+ "github.com/picosh/pico/pgs"
12+ "github.com/picosh/pico/shared/storage"
13+ "github.com/picosh/pico/wish/cms/config"
14+ "go.uber.org/zap"
15+)
16+
17+func createLogger() *zap.SugaredLogger {
18+ logger, err := zap.NewProduction()
19+ if err != nil {
20+ log.Fatal(err)
21+ }
22+
23+ return logger.Sugar()
24+}
25+
26+func bail(err error) {
27+ if err != nil {
28+ panic(err)
29+ }
30+}
31+
32+type RmProject struct {
33+ user *db.User
34+ name string
35+}
36+
37+// this script will find any objects stored within Store that does not
38+// have a corresponding project inside our database
39+func main() {
40+ // to actually commit changes, set to true
41+ write := false
42+ logger := createLogger()
43+
44+ picoCfg := config.NewConfigCms()
45+ picoCfg.Logger = logger
46+ picoCfg.DbURL = os.Getenv("DATABASE_URL")
47+ picoCfg.MinioURL = os.Getenv("MINIO_URL")
48+ picoCfg.MinioUser = os.Getenv("MINIO_ROOT_USER")
49+ picoCfg.MinioPass = os.Getenv("MINIO_ROOT_PASSWORD")
50+ picoDb := postgres.NewDB(picoCfg.DbURL, picoCfg.Logger)
51+
52+ var st storage.ObjectStorage
53+ var err error
54+ logger.Info(picoCfg)
55+ st, err = storage.NewStorageMinio(picoCfg.MinioURL, picoCfg.MinioUser, picoCfg.MinioPass)
56+ bail(err)
57+
58+ logger.Info("fetching all users")
59+ users, err := picoDb.FindUsers()
60+ bail(err)
61+
62+ logger.Info("fetching all buckets")
63+ buckets, err := st.ListBuckets()
64+ bail(err)
65+
66+ rmProjects := []RmProject{}
67+
68+ for _, bucketName := range buckets {
69+ // only care about pgs
70+ if !strings.HasPrefix(bucketName, "static-") {
71+ continue
72+ }
73+
74+ bucket, err := st.GetBucket(bucketName)
75+ bail(err)
76+ bucketProjects, err := st.ListFiles(bucket, "/", false)
77+ bail(err)
78+
79+ userID := strings.Replace(bucketName, "static-", "", 1)
80+ user := &db.User{
81+ ID: userID,
82+ Name: userID,
83+ }
84+ for _, u := range users {
85+ if u.ID == userID {
86+ user = u
87+ break
88+ }
89+ }
90+ projects, err := picoDb.FindProjectsByUser(userID)
91+ bail(err)
92+ for _, bucketProject := range bucketProjects {
93+ found := false
94+ for _, project := range projects {
95+ // ignore links
96+ if project.Name != project.ProjectDir {
97+ continue
98+ }
99+ if project.Name == bucketProject.Name() {
100+ found = true
101+ }
102+ }
103+ if !found {
104+ logger.Infof("marking (bucket: %s) (%s) for removal", bucketName, bucketProject.Name())
105+ rmProjects = append(rmProjects, RmProject{
106+ name: bucketProject.Name(),
107+ user: user,
108+ })
109+ }
110+ }
111+ }
112+
113+ session := &pgs.CmdSessionLogger{
114+ Log: logger,
115+ }
116+
117+ for _, project := range rmProjects {
118+ opts := &pgs.Cmd{
119+ Session: session,
120+ User: project.user,
121+ Store: st,
122+ Log: logger,
123+ Dbpool: picoDb,
124+ Write: write,
125+ }
126+ err := opts.RmProjectAssets(project.name)
127+ bail(err)
128+ }
129+
130+ logger.Infof("(%d) Store projects marked for deletion", len(rmProjects))
131+ for _, project := range rmProjects {
132+ logger.Infof("(user: %s) (project: %s)", project.user.Name, project.name)
133+ }
134+ if !write {
135+ logger.Info("WARNING: changes not committed, please go into binary and change `write` var")
136+ }
137+}
+92,
-57
1@@ -3,13 +3,13 @@ package pgs
2 import (
3 "errors"
4 "fmt"
5+ "io"
6+ "os"
7 "path/filepath"
8
9- "github.com/charmbracelet/ssh"
10 "github.com/picosh/pico/db"
11 "github.com/picosh/pico/shared"
12 "github.com/picosh/pico/shared/storage"
13- "github.com/picosh/pico/wish/send/utils"
14 "go.uber.org/zap"
15 )
16
17@@ -33,42 +33,77 @@ func getHelpText(userName, projectName string) string {
18 return helpStr
19 }
20
21+type CmdSessionLogger struct {
22+ Log *zap.SugaredLogger
23+}
24+
25+func (c *CmdSessionLogger) Write(out []byte) (int, error) {
26+ c.Log.Info(string(out))
27+ return 0, nil
28+}
29+
30+func (c *CmdSessionLogger) Exit(code int) error {
31+ os.Exit(code)
32+ return fmt.Errorf("panic %d", code)
33+}
34+
35+func (c *CmdSessionLogger) Close() error {
36+ return fmt.Errorf("closing")
37+}
38+
39+func (c *CmdSessionLogger) Stderr() io.ReadWriter {
40+ return nil
41+}
42+
43+type CmdSession interface {
44+ Write([]byte) (int, error)
45+ Exit(code int) error
46+ Close() error
47+ Stderr() io.ReadWriter
48+}
49+
50 type Cmd struct {
51- user *db.User
52- session ssh.Session
53- log *zap.SugaredLogger
54- store storage.ObjectStorage
55- dbpool db.DB
56- write bool
57+ User *db.User
58+ Session CmdSession
59+ Log *zap.SugaredLogger
60+ Store storage.ObjectStorage
61+ Dbpool db.DB
62+ Write bool
63 }
64
65 func (c *Cmd) output(out string) {
66- _, _ = c.session.Write([]byte(out + "\r\n"))
67+ _, _ = c.Session.Write([]byte(out + "\r\n"))
68+}
69+
70+func (c *Cmd) error(err error) {
71+ _, _ = fmt.Fprint(c.Session.Stderr(), err, "\r\n")
72+ _ = c.Session.Exit(1)
73+ _ = c.Session.Close()
74 }
75
76 func (c *Cmd) bail(err error) {
77 if err == nil {
78 return
79 }
80- c.log.Error(err)
81- utils.ErrorHandler(c.session, err)
82+ c.Log.Error(err)
83+ c.error(err)
84 }
85
86 func (c *Cmd) notice() {
87- if !c.write {
88+ if !c.Write {
89 c.output("\nNOTICE: changes not commited, use `--write` to save operation")
90 }
91 }
92
93-func (c *Cmd) rmProjectAssets(projectName string) error {
94- bucketName := shared.GetAssetBucketName(c.user.ID)
95- bucket, err := c.store.GetBucket(bucketName)
96+func (c *Cmd) RmProjectAssets(projectName string) error {
97+ bucketName := shared.GetAssetBucketName(c.User.ID)
98+ bucket, err := c.Store.GetBucket(bucketName)
99 if err != nil {
100 return err
101 }
102 c.output(fmt.Sprintf("removing project assets (%s)", projectName))
103
104- fileList, err := c.store.ListFiles(bucket, projectName+"/", true)
105+ fileList, err := c.Store.ListFiles(bucket, projectName+"/", true)
106 if err != nil {
107 return err
108 }
109@@ -81,14 +116,14 @@ func (c *Cmd) rmProjectAssets(projectName string) error {
110
111 for _, file := range fileList {
112 intent := fmt.Sprintf("deleted (%s)", file.Name())
113- c.log.Infof(
114+ c.Log.Infof(
115 "(%s) attempting to delete (bucket: %s) (%s)",
116- c.user.Name,
117+ c.User.Name,
118 bucket.Name,
119 file.Name(),
120 )
121- if c.write {
122- err = c.store.DeleteFile(
123+ if c.Write {
124+ err = c.Store.DeleteFile(
125 bucket,
126 filepath.Join(projectName, file.Name()),
127 )
128@@ -105,22 +140,22 @@ func (c *Cmd) rmProjectAssets(projectName string) error {
129 }
130
131 func (c *Cmd) help() {
132- c.output(getHelpText(c.user.Name, "project-a"))
133+ c.output(getHelpText(c.User.Name, "project-a"))
134 }
135
136 func (c *Cmd) stats(maxSize int) error {
137- bucketName := shared.GetAssetBucketName(c.user.ID)
138- bucket, err := c.store.UpsertBucket(bucketName)
139+ bucketName := shared.GetAssetBucketName(c.User.ID)
140+ bucket, err := c.Store.UpsertBucket(bucketName)
141 if err != nil {
142 return err
143 }
144
145- totalFileSize, err := c.store.GetBucketQuota(bucket)
146+ totalFileSize, err := c.Store.GetBucketQuota(bucket)
147 if err != nil {
148 return err
149 }
150
151- projects, err := c.dbpool.FindProjectsByUser(c.user.ID)
152+ projects, err := c.Dbpool.FindProjectsByUser(c.User.ID)
153 if err != nil {
154 return err
155 }
156@@ -140,7 +175,7 @@ func (c *Cmd) stats(maxSize int) error {
157 }
158
159 func (c *Cmd) ls() error {
160- projects, err := c.dbpool.FindProjectsByUser(c.user.ID)
161+ projects, err := c.Dbpool.FindProjectsByUser(c.User.ID)
162 if err != nil {
163 return err
164 }
165@@ -161,13 +196,13 @@ func (c *Cmd) ls() error {
166 }
167
168 func (c *Cmd) unlink(projectName string) error {
169- c.log.Infof("user (%s) running `unlink` command with (%s)", c.user.Name, projectName)
170- project, err := c.dbpool.FindProjectByName(c.user.ID, projectName)
171+ c.Log.Infof("user (%s) running `unlink` command with (%s)", c.User.Name, projectName)
172+ project, err := c.Dbpool.FindProjectByName(c.User.ID, projectName)
173 if err != nil {
174 return errors.Join(err, fmt.Errorf("project (%s) does not exit", projectName))
175 }
176
177- err = c.dbpool.LinkToProject(c.user.ID, project.ID, project.Name, c.write)
178+ err = c.Dbpool.LinkToProject(c.User.ID, project.ID, project.Name, c.Write)
179 if err != nil {
180 return err
181 }
182@@ -177,40 +212,40 @@ func (c *Cmd) unlink(projectName string) error {
183 }
184
185 func (c *Cmd) link(projectName, linkTo string) error {
186- c.log.Infof("user (%s) running `link` command with (%s) (%s)", c.user.Name, projectName, linkTo)
187+ c.Log.Infof("user (%s) running `link` command with (%s) (%s)", c.User.Name, projectName, linkTo)
188
189 projectDir := linkTo
190- _, err := c.dbpool.FindProjectByName(c.user.ID, linkTo)
191+ _, err := c.Dbpool.FindProjectByName(c.User.ID, linkTo)
192 if err != nil {
193 e := fmt.Errorf("(%s) project doesn't exist", linkTo)
194 return e
195 }
196
197- project, err := c.dbpool.FindProjectByName(c.user.ID, projectName)
198+ project, err := c.Dbpool.FindProjectByName(c.User.ID, projectName)
199 projectID := ""
200 if err == nil {
201 projectID = project.ID
202- c.log.Infof("user (%s) already has project (%s), updating", c.user.Name, projectName)
203- err = c.dbpool.LinkToProject(c.user.ID, project.ID, projectDir, c.write)
204+ c.Log.Infof("user (%s) already has project (%s), updating", c.User.Name, projectName)
205+ err = c.Dbpool.LinkToProject(c.User.ID, project.ID, projectDir, c.Write)
206 if err != nil {
207 return err
208 }
209 } else {
210- c.log.Infof("user (%s) has no project record (%s), creating", c.user.Name, projectName)
211- if !c.write {
212+ c.Log.Infof("user (%s) has no project record (%s), creating", c.User.Name, projectName)
213+ if !c.Write {
214 out := fmt.Sprintf("(%s) cannot create a new project without `--write` permission, aborting", projectName)
215 c.output(out)
216 return nil
217 }
218- id, err := c.dbpool.InsertProject(c.user.ID, projectName, projectName)
219+ id, err := c.Dbpool.InsertProject(c.User.ID, projectName, projectName)
220 if err != nil {
221 return err
222 }
223 projectID = id
224 }
225
226- c.log.Infof("user (%s) linking (%s) to (%s)", c.user.Name, projectName, projectDir)
227- err = c.dbpool.LinkToProject(c.user.ID, projectID, projectDir, c.write)
228+ c.Log.Infof("user (%s) linking (%s) to (%s)", c.User.Name, projectName, projectDir)
229+ err = c.Dbpool.LinkToProject(c.User.ID, projectID, projectDir, c.Write)
230 if err != nil {
231 return err
232 }
233@@ -218,7 +253,7 @@ func (c *Cmd) link(projectName, linkTo string) error {
234 out := fmt.Sprintf("(%s) might have orphaned assets, removing", projectName)
235 c.output(out)
236
237- err = c.rmProjectAssets(projectName)
238+ err = c.RmProjectAssets(projectName)
239 if err != nil {
240 return err
241 }
242@@ -229,7 +264,7 @@ func (c *Cmd) link(projectName, linkTo string) error {
243 }
244
245 func (c *Cmd) depends(projectName string) error {
246- projects, err := c.dbpool.FindProjectLinks(c.user.ID, projectName)
247+ projects, err := c.Dbpool.FindProjectLinks(c.User.ID, projectName)
248 if err != nil {
249 return err
250 }
251@@ -254,7 +289,7 @@ func (c *Cmd) depends(projectName string) error {
252 // delete all the projects and associated assets matching prefix
253 // but keep the latest N records.
254 func (c *Cmd) prune(prefix string, keepNumLatest int) error {
255- c.log.Infof("user (%s) running `clean` command for (%s)", c.user.Name, prefix)
256+ c.Log.Infof("user (%s) running `clean` command for (%s)", c.User.Name, prefix)
257 c.output(fmt.Sprintf("searching for projects that match prefix (%s) and are not linked to other projects", prefix))
258
259 if prefix == "" || prefix == "*" {
260@@ -262,7 +297,7 @@ func (c *Cmd) prune(prefix string, keepNumLatest int) error {
261 return e
262 }
263
264- projects, err := c.dbpool.FindProjectsByPrefix(c.user.ID, prefix)
265+ projects, err := c.Dbpool.FindProjectsByPrefix(c.User.ID, prefix)
266 if err != nil {
267 return err
268 }
269@@ -274,7 +309,7 @@ func (c *Cmd) prune(prefix string, keepNumLatest int) error {
270
271 rmProjects := []*db.Project{}
272 for _, project := range projects {
273- links, err := c.dbpool.FindProjectLinks(c.user.ID, project.Name)
274+ links, err := c.Dbpool.FindProjectLinks(c.User.ID, project.Name)
275 if err != nil {
276 return err
277 }
278@@ -305,7 +340,7 @@ func (c *Cmd) prune(prefix string, keepNumLatest int) error {
279 for _, project := range goodbye {
280 out := fmt.Sprintf("project (%s) is available to be pruned", project.Name)
281 c.output(out)
282- err = c.rmProjectAssets(project.Name)
283+ err = c.RmProjectAssets(project.Name)
284 if err != nil {
285 return err
286 }
287@@ -313,9 +348,9 @@ func (c *Cmd) prune(prefix string, keepNumLatest int) error {
288 out = fmt.Sprintf("(%s) removing", project.Name)
289 c.output(out)
290
291- if c.write {
292- c.log.Infof("(%s) removing", project.Name)
293- err = c.dbpool.RemoveProject(project.ID)
294+ if c.Write {
295+ c.Log.Infof("(%s) removing", project.Name)
296+ err = c.Dbpool.RemoveProject(project.ID)
297 if err != nil {
298 return err
299 }
300@@ -332,12 +367,12 @@ func (c *Cmd) prune(prefix string, keepNumLatest int) error {
301 }
302
303 func (c *Cmd) rm(projectName string) error {
304- c.log.Infof("user (%s) running `rm` command for (%s)", c.user.Name, projectName)
305- project, err := c.dbpool.FindProjectByName(c.user.ID, projectName)
306+ c.Log.Infof("user (%s) running `rm` command for (%s)", c.User.Name, projectName)
307+ project, err := c.Dbpool.FindProjectByName(c.User.ID, projectName)
308 if err == nil {
309- c.log.Infof("found project (%s) (%s), checking dependencies", projectName, project.ID)
310+ c.Log.Infof("found project (%s) (%s), checking dependencies", projectName, project.ID)
311
312- links, err := c.dbpool.FindProjectLinks(c.user.ID, projectName)
313+ links, err := c.Dbpool.FindProjectLinks(c.User.ID, projectName)
314 if err != nil {
315 return err
316 }
317@@ -349,18 +384,18 @@ func (c *Cmd) rm(projectName string) error {
318
319 out := fmt.Sprintf("(%s) removing", project.Name)
320 c.output(out)
321- if c.write {
322- c.log.Infof("(%s) removing", project.Name)
323- err = c.dbpool.RemoveProject(project.ID)
324+ if c.Write {
325+ c.Log.Infof("(%s) removing", project.Name)
326+ err = c.Dbpool.RemoveProject(project.ID)
327 if err != nil {
328 return err
329 }
330 }
331 } else {
332- e := fmt.Errorf("(%s) project not found for user (%s)", projectName, c.user.Name)
333+ e := fmt.Errorf("(%s) project not found for user (%s)", projectName, c.User.Name)
334 return e
335 }
336
337- err = c.rmProjectAssets(project.Name)
338+ err = c.RmProjectAssets(project.Name)
339 return err
340 }
+8,
-8
1@@ -55,12 +55,12 @@ func WishMiddleware(handler *uploadassets.UploadAssetHandler) wish.Middleware {
2 args := session.Command()
3
4 opts := Cmd{
5- session: session,
6- user: user,
7- store: store,
8- log: log,
9- dbpool: dbpool,
10- write: false,
11+ Session: session,
12+ User: user,
13+ Store: store,
14+ Log: log,
15+ Dbpool: dbpool,
16+ Write: false,
17 }
18
19 cmd := strings.TrimSpace(args[0])
20@@ -98,7 +98,7 @@ func WishMiddleware(handler *uploadassets.UploadAssetHandler) wish.Middleware {
21 }
22 linkTo := strings.TrimSpace(args[2])
23 if len(args) >= 4 && strings.TrimSpace(args[3]) == "--write" {
24- opts.write = true
25+ opts.Write = true
26 }
27
28 err := opts.link(projectName, linkTo)
29@@ -110,7 +110,7 @@ func WishMiddleware(handler *uploadassets.UploadAssetHandler) wish.Middleware {
30 }
31
32 if len(args) >= 3 && strings.TrimSpace(args[2]) == "--write" {
33- opts.write = true
34+ opts.Write = true
35 }
36
37 if cmd == "unlink" {
1@@ -134,6 +134,10 @@ func (s *StorageFS) DeleteFile(bucket Bucket, fpath string) error {
2 return nil
3 }
4
5+func (s *StorageFS) ListBuckets() ([]string, error) {
6+ return []string{}, fmt.Errorf("not implemented")
7+}
8+
9 func (s *StorageFS) ListFiles(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error) {
10 var fileList []os.FileInfo
11
1@@ -97,6 +97,19 @@ func (s *StorageMinio) GetBucketQuota(bucket Bucket) (uint64, error) {
2 return 0, fmt.Errorf("%s bucket not found in account info", bucket.Name)
3 }
4
5+func (s *StorageMinio) ListBuckets() ([]string, error) {
6+ bcks := []string{}
7+ buckets, err := s.Client.ListBuckets(context.Background())
8+ if err != nil {
9+ return bcks, err
10+ }
11+ for _, bucket := range buckets {
12+ bcks = append(bcks, bucket.Name)
13+ }
14+
15+ return bcks, nil
16+}
17+
18 func (s *StorageMinio) ListFiles(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error) {
19 var fileList []os.FileInfo
20
1@@ -15,6 +15,7 @@ type Bucket struct {
2 type ObjectStorage interface {
3 GetBucket(name string) (Bucket, error)
4 UpsertBucket(name string) (Bucket, error)
5+ ListBuckets() ([]string, error)
6
7 DeleteBucket(bucket Bucket) error
8 GetBucketQuota(bucket Bucket) (uint64, error)