repos / pico

commit

c59051f

parent

7372600

author

Eric Bower

date

2024-12-13 04:39:25 +0000 UTC

refactor(feeds): remove feeds after 3 failed attempts

We have a bunch of feeds in our system that are failing in a loop.
Previously we did not notify the user when there was a failure, we just
logged it internally.

With this change, we will prepend errors to the email message body.
Further, if the feeds errors and doesn't return any feed items, we
increment a counter.  3 failed attempts and we remove the post and
notify the user.

1@@ -32,6 +32,7 @@ type User struct {
2 type PostData struct {
3 	ImgPath    string     `json:"img_path"`
4 	LastDigest *time.Time `json:"last_digest"`
5+	Attempts   int        `json:"attempts"`
6 }
7 
8 // Make the Attrs struct implement the driver.Valuer interface. This method

  1@@ -175,25 +175,77 @@ func (f *Fetcher) RunPost(logger *slog.Logger, user *db.User, post *db.Post) err
  2 		urls = append(urls, url)
  3 	}
  4 
  5-	msgBody, err := f.FetchAll(logger, urls, parsed.InlineContent, user.Name, post)
  6+	now := time.Now().UTC()
  7+	if post.ExpiresAt == nil {
  8+		expiresAt := time.Now().AddDate(0, 6, 0)
  9+		post.ExpiresAt = &expiresAt
 10+	}
 11+	post.Data.LastDigest = &now
 12+	_, err = f.db.UpdatePost(post)
 13 	if err != nil {
 14 		return err
 15 	}
 16 
 17 	subject := fmt.Sprintf("%s feed digest", post.Title)
 18-	err = f.SendEmail(logger, user.Name, parsed.Email, subject, msgBody)
 19+
 20+	msgBody, err := f.FetchAll(logger, urls, parsed.InlineContent, user.Name, post)
 21 	if err != nil {
 22-		return err
 23+		errForUser := err
 24+
 25+		// we don't want to increment in this case
 26+		if errors.Is(errForUser, ErrNoRecentArticles) {
 27+			return nil
 28+		}
 29+
 30+		post.Data.Attempts += 1
 31+		logger.Error("could not fetch urls", "err", err, "attempts", post.Data.Attempts)
 32+
 33+		errBody := fmt.Sprintf(`There was an error attempting to fetch your feeds (%d) times.  After (3) attempts we remove the file from our system.  Please check all the URLs and re-upload.
 34+Also, we have centralized logs in our pico.sh TUI that will display realtime feed errors so you can debug.
 35+
 36+
 37+%s
 38+
 39+
 40+%s`, post.Data.Attempts, errForUser.Error(), post.Text)
 41+		err = f.SendEmail(
 42+			logger, user.Name,
 43+			parsed.Email,
 44+			subject,
 45+			&MsgBody{Html: strings.ReplaceAll(errBody, "\n", "<br />"), Text: errBody},
 46+		)
 47+		if err != nil {
 48+			return err
 49+		}
 50+
 51+		if post.Data.Attempts >= 3 {
 52+			err = f.db.RemovePosts([]string{post.ID})
 53+			if err != nil {
 54+				return err
 55+			}
 56+		} else {
 57+			_, err = f.db.UpdatePost(post)
 58+			if err != nil {
 59+				return err
 60+			}
 61+		}
 62+		return errForUser
 63+	} else {
 64+		post.Data.Attempts = 0
 65+		_, err := f.db.UpdatePost(post)
 66+		if err != nil {
 67+			return err
 68+		}
 69 	}
 70 
 71-	now := time.Now().UTC()
 72-	if post.ExpiresAt == nil {
 73-		expiresAt := time.Now().AddDate(0, 6, 0)
 74-		post.ExpiresAt = &expiresAt
 75+	if msgBody != nil {
 76+		err = f.SendEmail(logger, user.Name, parsed.Email, subject, msgBody)
 77+		if err != nil {
 78+			return err
 79+		}
 80 	}
 81-	post.Data.LastDigest = &now
 82-	_, err = f.db.UpdatePost(post)
 83-	return err
 84+
 85+	return nil
 86 }
 87 
 88 func (f *Fetcher) RunUser(user *db.User) error {
 89@@ -353,12 +405,14 @@ func (f *Fetcher) FetchAll(logger *slog.Logger, urls []string, inlineContent boo
 90 		return nil, err
 91 	}
 92 
 93+	var allErrors error
 94 	for _, url := range urls {
 95 		feedTmpl, err := f.Fetch(logger, fp, url, username, feedItems)
 96 		if err != nil {
 97 			if errors.Is(err, ErrNoRecentArticles) {
 98 				logger.Info("no recent articles", "err", err)
 99 			} else {
100+				allErrors = errors.Join(allErrors, fmt.Errorf("%s: %w", url, err))
101 				logger.Error("fetch error", "err", err)
102 			}
103 			continue
104@@ -367,7 +421,10 @@ func (f *Fetcher) FetchAll(logger *slog.Logger, urls []string, inlineContent boo
105 	}
106 
107 	if len(feeds.Feeds) == 0 {
108-		return nil, fmt.Errorf("(%s) %w, skipping email", username, ErrNoRecentArticles)
109+		if allErrors != nil {
110+			return nil, allErrors
111+		}
112+		return nil, fmt.Errorf("%w, skipping email", ErrNoRecentArticles)
113 	}
114 
115 	fdi := []*db.FeedItem{}
116@@ -401,6 +458,11 @@ func (f *Fetcher) FetchAll(logger *slog.Logger, urls []string, inlineContent boo
117 		return nil, err
118 	}
119 
120+	if allErrors != nil {
121+		text = fmt.Sprintf("> %s\n\n%s", allErrors, text)
122+		html = fmt.Sprintf("<blockquote>%s</blockquote><br /><br/>%s", allErrors, html)
123+	}
124+
125 	return &MsgBody{
126 		Text: text,
127 		Html: html,