Commit 83a5183

Eric Bower  ·  2026-05-05 10:47:35 -0400 EDT
parent dc6ed61 
chore(visits): new indexes and parallel queries
3 files changed,  +162, -88
M Makefile
M pkg/db/postgres/storage.go
A sql/migrations/20260504_add_analytics_summary_indexes.sql
M Makefile
+2, -1
......@@ -151,10 +151,11 @@ migrate:
151151 $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20260116_add_analytics_filter_indexes.sql
152152 $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20260305_add_forms_table.sql
153153 $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20260503_add_analytics_summary_tables.sql
154+ $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20260504_add_analytics_summary_indexes.sql
154155 .PHONY: migrate
155156
156157 latest:
157- $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20260503_add_analytics_summary_tables.sql
158+ $(DOCKER_CMD) exec -i $(DB_CONTAINER) psql -U $(PGUSER) -d $(PGDATABASE) < ./sql/migrations/20260504_add_analytics_summary_indexes.sql
158159 .PHONY: latest
159160
160161 psql:
M pkg/db/postgres/storage.go
+153, -87
......@@ -9,6 +9,7 @@ import (
99 "slices"
1010 "sort"
1111 "strings"
12+ "sync"
1213 "time"
1314
1415 "github.com/jmoiron/sqlx"
......@@ -724,14 +725,28 @@ func visitFilterBy(opts *db.SummaryOpts) (string, string) {
724725 }
725726
726727 func (me *PsqlDB) visitUnique(opts *db.SummaryOpts) ([]*db.VisitInterval, error) {
727- intervals, err := me.visitUniqueFromSummary(opts)
728- if err != nil {
729- return nil, fmt.Errorf("query summary visits: %w", err)
730- }
728+ var intervals, currentIntervals []*db.VisitInterval
729+ var sumErr, rawErr error
731730
732- currentIntervals, err := me.visitUniqueFromRaw(opts)
733- if err != nil {
734- return nil, fmt.Errorf("query raw visits: %w", err)
731+ var wg sync.WaitGroup
732+ wg.Add(2)
733+
734+ go func() {
735+ defer wg.Done()
736+ intervals, sumErr = me.visitUniqueFromSummary(opts)
737+ }()
738+ go func() {
739+ defer wg.Done()
740+ currentIntervals, rawErr = me.visitUniqueFromRaw(opts)
741+ }()
742+
743+ wg.Wait()
744+
745+ if sumErr != nil {
746+ return nil, fmt.Errorf("query summary visits: %w", sumErr)
747+ }
748+ if rawErr != nil {
749+ return nil, fmt.Errorf("query raw visits: %w", rawErr)
735750 }
736751
737752 // Merge: current month data may overlap with summary data, combine counts
......@@ -744,23 +759,13 @@ func (me *PsqlDB) visitUniqueFromSummary(opts *db.SummaryOpts) ([]*db.VisitInter
744759 currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
745760 previousMonthStart := currentMonthStart.AddDate(0, -1, 0)
746761
747- // Always include at least the previous month in the summary query, since raw data
748- // for the previous month may have been deleted by aggregation. visitUniqueFromRaw
749- // covers the previous and current months, but once aggregation runs and deletes raw
750- // data, the summary table is the only source. Using min(origin, previousMonthStart)
751- // ensures we don't skip the previous month even when origin is set to the current month.
752- effectiveOrigin := opts.Origin
753- if opts.Origin.After(previousMonthStart) {
754- effectiveOrigin = previousMonthStart
755- }
756-
757- // If effective origin is in or after current month, no historical data to fetch
758- if !effectiveOrigin.Before(currentMonthStart) {
762+ // If origin is in the previous month or later, raw data covers it — no summary to fetch.
763+ if !opts.Origin.Before(previousMonthStart) {
759764 return nil, nil
760765 }
761766
762767 where := ""
763- args := []interface{}{opts.UserID, effectiveOrigin, currentMonthStart}
768+ args := []interface{}{opts.UserID, opts.Origin, currentMonthStart}
764769 argIdx := 4
765770 if opts.Host != "" {
766771 where = "AND host = $" + fmt.Sprintf("%d", argIdx)
......@@ -953,14 +958,28 @@ func mergeHosts(historical, current []*db.VisitUrl) []*db.VisitUrl {
953958 }
954959
955960 func (me *PsqlDB) visitReferer(opts *db.SummaryOpts) ([]*db.VisitUrl, error) {
956- historical, err := me.visitRefererFromSummary(opts)
957- if err != nil {
958- return nil, fmt.Errorf("query summary referers: %w", err)
959- }
961+ var historical, current []*db.VisitUrl
962+ var histErr, rawErr error
960963
961- current, err := me.visitRefererFromRaw(opts)
962- if err != nil {
963- return nil, fmt.Errorf("query raw referers: %w", err)
964+ var wg sync.WaitGroup
965+ wg.Add(2)
966+
967+ go func() {
968+ defer wg.Done()
969+ historical, histErr = me.visitRefererFromSummary(opts)
970+ }()
971+ go func() {
972+ defer wg.Done()
973+ current, rawErr = me.visitRefererFromRaw(opts)
974+ }()
975+
976+ wg.Wait()
977+
978+ if histErr != nil {
979+ return nil, fmt.Errorf("query summary referers: %w", histErr)
980+ }
981+ if rawErr != nil {
982+ return nil, fmt.Errorf("query raw referers: %w", rawErr)
964983 }
965984
966985 return mergeTopReferers(historical, current), nil
......@@ -972,18 +991,13 @@ func (me *PsqlDB) visitRefererFromSummary(opts *db.SummaryOpts) ([]*db.VisitUrl,
972991 currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
973992 previousMonthStart := currentMonthStart.AddDate(0, -1, 0)
974993
975- effectiveOrigin := opts.Origin
976- if opts.Origin.After(previousMonthStart) {
977- effectiveOrigin = previousMonthStart
978- }
979-
980- // If effective origin is in or after current month, no historical data to fetch
981- if !effectiveOrigin.Before(currentMonthStart) {
994+ // If origin is in the previous month or later, raw data covers it — no summary to fetch.
995+ if !opts.Origin.Before(previousMonthStart) {
982996 return nil, nil
983997 }
984998
985999 // Clamp origin to month boundary for summary table lookup
986- originMonthStart := time.Date(effectiveOrigin.Year(), effectiveOrigin.Month(), 1, 0, 0, 0, 0, time.UTC)
1000+ originMonthStart := time.Date(opts.Origin.Year(), opts.Origin.Month(), 1, 0, 0, 0, 0, time.UTC)
9871001
9881002 where := ""
9891003 args := []interface{}{opts.UserID, originMonthStart, currentMonthStart}
......@@ -1059,14 +1073,28 @@ func (me *PsqlDB) visitRefererFromRaw(opts *db.SummaryOpts) ([]*db.VisitUrl, err
10591073 }
10601074
10611075 func (me *PsqlDB) visitUrl(opts *db.SummaryOpts) ([]*db.VisitUrl, error) {
1062- historical, err := me.visitUrlFromSummary(opts)
1063- if err != nil {
1064- return nil, fmt.Errorf("query summary urls: %w", err)
1065- }
1076+ var historical, current []*db.VisitUrl
1077+ var histErr, rawErr error
10661078
1067- current, err := me.visitUrlFromRaw(opts)
1068- if err != nil {
1069- return nil, fmt.Errorf("query raw urls: %w", err)
1079+ var wg sync.WaitGroup
1080+ wg.Add(2)
1081+
1082+ go func() {
1083+ defer wg.Done()
1084+ historical, histErr = me.visitUrlFromSummary(opts)
1085+ }()
1086+ go func() {
1087+ defer wg.Done()
1088+ current, rawErr = me.visitUrlFromRaw(opts)
1089+ }()
1090+
1091+ wg.Wait()
1092+
1093+ if histErr != nil {
1094+ return nil, fmt.Errorf("query summary urls: %w", histErr)
1095+ }
1096+ if rawErr != nil {
1097+ return nil, fmt.Errorf("query raw urls: %w", rawErr)
10701098 }
10711099
10721100 return mergeTopUrls(historical, current), nil
......@@ -1078,18 +1106,13 @@ func (me *PsqlDB) visitUrlFromSummary(opts *db.SummaryOpts) ([]*db.VisitUrl, err
10781106 currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
10791107 previousMonthStart := currentMonthStart.AddDate(0, -1, 0)
10801108
1081- effectiveOrigin := opts.Origin
1082- if opts.Origin.After(previousMonthStart) {
1083- effectiveOrigin = previousMonthStart
1084- }
1085-
1086- // If effective origin is in or after current month, no historical data to fetch
1087- if !effectiveOrigin.Before(currentMonthStart) {
1109+ // If origin is in the previous month or later, raw data covers it — no summary to fetch.
1110+ if !opts.Origin.Before(previousMonthStart) {
10881111 return nil, nil
10891112 }
10901113
10911114 // Clamp origin to month boundary for summary table lookup
1092- originMonthStart := time.Date(effectiveOrigin.Year(), effectiveOrigin.Month(), 1, 0, 0, 0, 0, time.UTC)
1115+ originMonthStart := time.Date(opts.Origin.Year(), opts.Origin.Month(), 1, 0, 0, 0, 0, time.UTC)
10931116
10941117 where := ""
10951118 args := []interface{}{opts.UserID, originMonthStart, currentMonthStart}
......@@ -1170,14 +1193,28 @@ func (me *PsqlDB) VisitUrlNotFound(opts *db.SummaryOpts) ([]*db.VisitUrl, error)
11701193 limit = 10
11711194 }
11721195
1173- historical, err := me.visitUrlNotFoundFromSummary(opts, limit)
1174- if err != nil {
1175- return nil, fmt.Errorf("query summary 404 urls: %w", err)
1176- }
1196+ var historical, current []*db.VisitUrl
1197+ var histErr, rawErr error
11771198
1178- current, err := me.visitUrlNotFoundFromRaw(opts, limit)
1179- if err != nil {
1180- return nil, fmt.Errorf("query raw 404 urls: %w", err)
1199+ var wg sync.WaitGroup
1200+ wg.Add(2)
1201+
1202+ go func() {
1203+ defer wg.Done()
1204+ historical, histErr = me.visitUrlNotFoundFromSummary(opts, limit)
1205+ }()
1206+ go func() {
1207+ defer wg.Done()
1208+ current, rawErr = me.visitUrlNotFoundFromRaw(opts, limit)
1209+ }()
1210+
1211+ wg.Wait()
1212+
1213+ if histErr != nil {
1214+ return nil, fmt.Errorf("query summary 404 urls: %w", histErr)
1215+ }
1216+ if rawErr != nil {
1217+ return nil, fmt.Errorf("query raw 404 urls: %w", rawErr)
11811218 }
11821219
11831220 return mergeTopUrls(historical, current), nil
......@@ -1189,18 +1226,13 @@ func (me *PsqlDB) visitUrlNotFoundFromSummary(opts *db.SummaryOpts, limit int) (
11891226 currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
11901227 previousMonthStart := currentMonthStart.AddDate(0, -1, 0)
11911228
1192- effectiveOrigin := opts.Origin
1193- if opts.Origin.After(previousMonthStart) {
1194- effectiveOrigin = previousMonthStart
1195- }
1196-
1197- // If effective origin is in or after current month, no historical data to fetch
1198- if !effectiveOrigin.Before(currentMonthStart) {
1229+ // If origin is in the previous month or later, raw data covers it — no summary to fetch.
1230+ if !opts.Origin.Before(previousMonthStart) {
11991231 return nil, nil
12001232 }
12011233
12021234 // Clamp origin to month boundary for summary table lookup
1203- originMonthStart := time.Date(effectiveOrigin.Year(), effectiveOrigin.Month(), 1, 0, 0, 0, 0, time.UTC)
1235+ originMonthStart := time.Date(opts.Origin.Year(), opts.Origin.Month(), 1, 0, 0, 0, 0, time.UTC)
12041236
12051237 where := ""
12061238 args := []interface{}{opts.UserID, originMonthStart, currentMonthStart}
......@@ -1277,14 +1309,28 @@ func (me *PsqlDB) visitUrlNotFoundFromRaw(opts *db.SummaryOpts, limit int) ([]*d
12771309 }
12781310
12791311 func (me *PsqlDB) visitHost(opts *db.SummaryOpts) ([]*db.VisitUrl, error) {
1280- historical, err := me.visitHostFromSummary(opts)
1281- if err != nil {
1282- return nil, fmt.Errorf("query summary hosts: %w", err)
1283- }
1312+ var historical, current []*db.VisitUrl
1313+ var histErr, rawErr error
12841314
1285- current, err := me.visitHostFromRaw(opts)
1286- if err != nil {
1287- return nil, fmt.Errorf("query raw hosts: %w", err)
1315+ var wg sync.WaitGroup
1316+ wg.Add(2)
1317+
1318+ go func() {
1319+ defer wg.Done()
1320+ historical, histErr = me.visitHostFromSummary(opts)
1321+ }()
1322+ go func() {
1323+ defer wg.Done()
1324+ current, rawErr = me.visitHostFromRaw(opts)
1325+ }()
1326+
1327+ wg.Wait()
1328+
1329+ if histErr != nil {
1330+ return nil, fmt.Errorf("query summary hosts: %w", histErr)
1331+ }
1332+ if rawErr != nil {
1333+ return nil, fmt.Errorf("query raw hosts: %w", rawErr)
12881334 }
12891335
12901336 return mergeHosts(historical, current), nil
......@@ -1341,24 +1387,44 @@ func (me *PsqlDB) visitHostFromRaw(opts *db.SummaryOpts) ([]*db.VisitUrl, error)
13411387 }
13421388
13431389 func (me *PsqlDB) VisitSummary(opts *db.SummaryOpts) (*db.SummaryVisits, error) {
1344- visitors, err := me.visitUnique(opts)
1345- if err != nil {
1346- return nil, err
1347- }
1390+ var (
1391+ visitors []*db.VisitInterval
1392+ urls []*db.VisitUrl
1393+ refs []*db.VisitUrl
1394+ notFound []*db.VisitUrl
1395+ visitorsErr error
1396+ urlsErr error
1397+ refsErr error
1398+ nfErr error
1399+ )
13481400
1349- urls, err := me.visitUrl(opts)
1350- if err != nil {
1351- return nil, err
1352- }
1401+ var wg sync.WaitGroup
1402+ wg.Add(4)
13531403
1354- notFound, err := me.VisitUrlNotFound(opts)
1355- if err != nil {
1356- return nil, err
1357- }
1404+ go func() {
1405+ defer wg.Done()
1406+ visitors, visitorsErr = me.visitUnique(opts)
1407+ }()
1408+ go func() {
1409+ defer wg.Done()
1410+ urls, urlsErr = me.visitUrl(opts)
1411+ }()
1412+ go func() {
1413+ defer wg.Done()
1414+ refs, refsErr = me.visitReferer(opts)
1415+ }()
1416+ go func() {
1417+ defer wg.Done()
1418+ notFound, nfErr = me.VisitUrlNotFound(opts)
1419+ }()
13581420
1359- refs, err := me.visitReferer(opts)
1360- if err != nil {
1361- return nil, err
1421+ wg.Wait()
1422+
1423+ // Return the first error encountered
1424+ for _, err := range []error{visitorsErr, urlsErr, refsErr, nfErr} {
1425+ if err != nil {
1426+ return nil, err
1427+ }
13621428 }
13631429
13641430 return &db.SummaryVisits{
A sql/migrations/20260504_add_analytics_summary_indexes.sql
+7, -0
......@@ -0,0 +1,7 @@
1+-- Composite index for visitUniqueFromSummary: filters by (user_id, host, visit_date)
2+CREATE INDEX IF NOT EXISTS idx_monthly_visits_user_host_date
3+ON analytics_monthly_visits (user_id, host, visit_date);
4+
5+-- Covering index for visitHostFromRaw: filters by (user_id, created_at), groups by host, counts distinct ip_address
6+CREATE INDEX IF NOT EXISTS idx_analytics_visits_user_created_host_ip
7+ON analytics_visits (user_id, created_at, host, ip_address);