Repository: vanti-dev/sc-bos
Pull request #402: auto/exporthttp: use order by and page size = 1 to reduce the amount of pagination in ListMeterHistory calls to calculate consumption in meters
Author: login: dloveen-v
Created: 2025-05-15T10:31:51Z | Merged: 2025-05-27T11:36:15Z

This makes use of this fix https://github.com/vanti-dev/sc-bos/commit/bb723ae88593cc508f1886597484fc7fc979a8a0 which allows clients to calculate consumption between 2 timestamps without paginating through _**every meter history result between those 2 timestamps**_.

I have also added a fix because I don't trust

```
for {
  select {
    case <-time.After(...):
    ...
  }
}
```

won't execute case statements more than once when receiving from a chan dictated by a cron schedule.


Diff:
diff --git a/pkg/auto/exporthttp/auto.go b/pkg/auto/exporthttp/auto.go
index b525a85ee..3403cc939 100644
--- a/pkg/auto/exporthttp/auto.go
+++ b/pkg/auto/exporthttp/auto.go
@@ -10,6 +10,7 @@ package exporthttp
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"go.uber.org/zap"
 
@@ -69,7 +70,9 @@ func (a *autoImpl) applyConfig(ctx context.Context, cfg config.Root) error {
 				go func() {
 					if err := jb.Do(ctx, client.Post); err != nil {
 						logger.Warn(fmt.Sprintf("failed to run %s", jb.GetName()), zap.Error(err))
+						return
 					}
+					jb.SetPreviousExecution(time.Now())
 				}()
 			case <-ctx.Done():
 				return
diff --git a/pkg/auto/exporthttp/job/energy.go b/pkg/auto/exporthttp/job/energy.go
index 97eec23ef..eb64c93cc 100644
--- a/pkg/auto/exporthttp/job/energy.go
+++ b/pkg/auto/exporthttp/job/energy.go
@@ -40,7 +40,7 @@ func (e *EnergyJob) Do(ctx context.Context, sendFn sender) error {
 			e.Logger.Error("getting unit multiplier", zap.String("meter", meter), zap.Error(err))
 		}
 
-		earliest, latest, err := getRecordsByTime(cctx, e.client.ListMeterReadingHistory, meter, now, filterTime)
+		earliest, latest, err := getRecordsByTime(cctx, e.Logger, e.client.ListMeterReadingHistory, meter, now, filterTime)
 
 		cancel()
 
diff --git a/pkg/auto/exporthttp/job/job.go b/pkg/auto/exporthttp/job/job.go
index 046c07264..3cafe0c38 100644
--- a/pkg/auto/exporthttp/job/job.go
+++ b/pkg/auto/exporthttp/job/job.go
@@ -36,6 +36,7 @@ type Job interface {
 	GetUrl() string
 	GetSite() string
 	GetExecutionAfter(t time.Time) <-chan time.Time
+	SetPreviousExecution(t time.Time)
 
 	Do(ctx context.Context, sendFn sender) error
 }
@@ -102,10 +103,13 @@ func (b *BaseJob) GetExecutionAfter(t time.Time) <-chan time.Time {
 	if t.IsZero() {
 		t = time.Now().UTC()
 	}
-	b.PreviousExecution = t
 	return time.After(time.Until(b.Schedule.Next(t)))
 }
 
+func (b *BaseJob) SetPreviousExecution(t time.Time) {
+	b.PreviousExecution = t
+}
+
 func (b *BaseJob) GetSite() string {
 	return b.Site
 }
diff --git a/pkg/auto/exporthttp/job/meter.go b/pkg/auto/exporthttp/job/meter.go
index 146fa5e3d..facc02344 100644
--- a/pkg/auto/exporthttp/job/meter.go
+++ b/pkg/auto/exporthttp/job/meter.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"time"
 
+	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
@@ -14,46 +15,50 @@ import (
 
 type listMeterReadingFn func(ctx context.Context, in *gen.ListMeterReadingHistoryRequest, opts ...grpc.CallOption) (*gen.ListMeterReadingHistoryResponse, error)
 
-func getRecordsByTime(ctx context.Context, historyFn listMeterReadingFn, meter string, now time.Time, filterTime time.Duration) (earliest, latest *gen.MeterReadingRecord, err error) {
-	var (
-		pageToken string
-		resp      *gen.ListMeterReadingHistoryResponse
-	)
-
-	latest = &gen.MeterReadingRecord{RecordTime: timestamppb.New(time.Time{})}
-	earliest = &gen.MeterReadingRecord{RecordTime: timestamppb.New(now)}
-
-	for {
-		resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
-			Name: meter,
-			Period: &sctime.Period{
-				StartTime: timestamppb.New(now.Add(-filterTime - time.Second)),
-				EndTime:   timestamppb.New(now),
-			},
-			PageToken: pageToken,
-		})
-
-		if err != nil {
-			return nil, nil, err
-		}
-
-		for _, record := range resp.GetMeterReadingRecords() {
-			if record.GetRecordTime().AsTime().Before(earliest.GetRecordTime().AsTime()) {
-				earliest = record
-			}
-
-			if record.GetRecordTime().AsTime().After(latest.GetRecordTime().AsTime()) {
-				latest = record
-			}
-		}
-
-		if resp.GetNextPageToken() == "" {
-			break
-		}
-
-		pageToken = resp.GetNextPageToken()
+func getRecordsByTime(ctx context.Context, logger *zap.Logger, historyFn listMeterReadingFn, meter string, now time.Time, filterTime time.Duration) (earliest, latest *gen.MeterReadingRecord, err error) {
+	var resp *gen.ListMeterReadingHistoryResponse
+
+	start := now.Add(-filterTime)
+
+	resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
+		Name: meter,
+		Period: &sctime.Period{
+			StartTime: timestamppb.New(start),
+		},
+		PageSize: 1,
+	})
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if len(resp.GetMeterReadingRecords()) == 0 {
+		logger.Error("no records found in earliest", zap.String("meter", meter), zap.Time("start", start))
+		return earliest, latest, nil
+	}
+
+	earliest = resp.GetMeterReadingRecords()[0]
+
+	resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
+		Name: meter,
+		Period: &sctime.Period{
+			EndTime: timestamppb.New(now),
+		},
+		PageSize: 1,
+		OrderBy:  "record_time DESC",
+	})
+
+	if err != nil {
+		return nil, nil, err
 	}
 
+	if len(resp.GetMeterReadingRecords()) == 0 {
+		logger.Error("no records found in latest", zap.String("meter", meter), zap.Time("end", now))
+		return earliest, earliest, nil // make sure this resolves consumption to 0 by returning < earliest, earliest, nil >
+	}
+
+	latest = resp.GetMeterReadingRecords()[0]
+
 	return earliest, latest, nil
 }
 
diff --git a/pkg/auto/exporthttp/job/meter_test.go b/pkg/auto/exporthttp/job/meter_test.go
index c11048dde..d05e96441 100644
--- a/pkg/auto/exporthttp/job/meter_test.go
+++ b/pkg/auto/exporthttp/job/meter_test.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
@@ -21,6 +22,8 @@ func Test_processMeter(t *testing.T) {
 		filterTime time.Duration
 	}
 
+	logger := zap.NewNop()
+
 	start := time.Time{}.Add(time.Nanosecond)
 
 	tests := []struct {
@@ -74,7 +77,7 @@ func Test_processMeter(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			earliest, latest, err := getRecordsByTime(context.Background(), tt.args.historyFn, tt.args.meter, tt.args.now, tt.args.filterTime)
+			earliest, latest, err := getRecordsByTime(context.Background(), logger, tt.args.historyFn, tt.args.meter, tt.args.now, tt.args.filterTime)
 
 			assert.Equal(t, tt.wantEarliest, earliest, tt.name)
 			assert.Equal(t, tt.wantLatest, latest, tt.name)
@@ -88,7 +91,7 @@ func Test_processMeter(t *testing.T) {
 }
 
 func twoMeterReadingPages(_ context.Context, start time.Time, in *gen.ListMeterReadingHistoryRequest, _ ...grpc.CallOption) (*gen.ListMeterReadingHistoryResponse, error) {
-	if in.GetPageToken() == "abc" {
+	if in.GetOrderBy() == "record_time DESC" {
 		return &gen.ListMeterReadingHistoryResponse{
 			MeterReadingRecords: []*gen.MeterReadingRecord{
 				{
@@ -99,24 +102,7 @@ func twoMeterReadingPages(_ context.Context, start time.Time, in *gen.ListMeterR
 					},
 					RecordTime: timestamppb.New(start.Add(30 * time.Minute)),
 				},
-				{
-					MeterReading: &gen.MeterReading{
-						Usage:     10,
-						StartTime: timestamppb.New(start),
-						EndTime:   timestamppb.New(start.Add(8 * time.Minute)),
-					},
-					RecordTime: timestamppb.New(start.Add(8 * time.Minute)),
-				},
-				{
-					MeterReading: &gen.MeterReading{
-						Usage:     20,
-						StartTime: timestamppb.New(start.Add(10 * time.Minute)),
-						EndTime:   timestamppb.New(start.Add(17 * time.Minute)),
-					},
-					RecordTime: timestamppb.New(start.Add(17 * time.Minute)),
-				},
 			},
-			NextPageToken: "",
 		}, nil
 	}
 	return &gen.ListMeterReadingHistoryResponse{
@@ -129,23 +115,6 @@ func twoMeterReadingPages(_ context.Context, start time.Time, in *gen.ListMeterR
 				},
 				RecordTime: timestamppb.New(start.Add(time.Minute)),
 			},
-			{
-				MeterReading: &gen.MeterReading{
-					Usage:     6,
-					StartTime: timestamppb.New(start.Add(6 * time.Minute)),
-					EndTime:   timestamppb.New(start.Add(7 * time.Minute)),
-				},
-				RecordTime: timestamppb.New(start.Add(7 * time.Minute)),
-			},
-			{
-				MeterReading: &gen.MeterReading{
-					Usage:     2,
-					StartTime: timestamppb.New(start.Add(2 * time.Minute)),
-					EndTime:   timestamppb.New(start.Add(5 * time.Minute)),
-				},
-				RecordTime: timestamppb.New(start.Add(5 * time.Minute)),
-			},
 		},
-		NextPageToken: "abc",
 	}, nil
 }
diff --git a/pkg/auto/exporthttp/job/water.go b/pkg/auto/exporthttp/job/water.go
index 8c05c5ddb..ef318ffb2 100644
--- a/pkg/auto/exporthttp/job/water.go
+++ b/pkg/auto/exporthttp/job/water.go
@@ -38,7 +38,7 @@ func (w *WaterJob) Do(ctx context.Context, sendFn sender) error {
 			w.Logger.Error("getting unit multiplier", zap.String("meter", meter), zap.Error(err))
 		}
 
-		earliest, latest, err := getRecordsByTime(cctx, w.client.ListMeterReadingHistory, meter, now, filterTime)
+		earliest, latest, err := getRecordsByTime(cctx, w.Logger, w.client.ListMeterReadingHistory, meter, now, filterTime)
 
 		cancel()
 		if err != nil {


Reviews:
user: login: deanredfern26
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T10:52:08Z
user: login: dloveen-v
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T11:14:33Z
user: login: deanredfern26
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T11:55:08Z
user: login: deanredfern26
body: Based on the screenshot looks like this has been posting more sensible values than previously
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T11:05:53Z

Review comments:
diff_hunk: @@ -14,46 +15,50 @@ import (
 
 type listMeterReadingFn func(ctx context.Context, in *gen.ListMeterReadingHistoryRequest, opts ...grpc.CallOption) (*gen.ListMeterReadingHistoryResponse, error)
 
-func getRecordsByTime(ctx context.Context, historyFn listMeterReadingFn, meter string, now time.Time, filterTime time.Duration) (earliest, latest *gen.MeterReadingRecord, err error) {
-	var (
-		pageToken string
-		resp      *gen.ListMeterReadingHistoryResponse
-	)
-
-	latest = &gen.MeterReadingRecord{RecordTime: timestamppb.New(time.Time{})}
-	earliest = &gen.MeterReadingRecord{RecordTime: timestamppb.New(now)}
-
-	for {
-		resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
-			Name: meter,
-			Period: &sctime.Period{
-				StartTime: timestamppb.New(now.Add(-filterTime - time.Second)),
-				EndTime:   timestamppb.New(now),
-			},
-			PageToken: pageToken,
-		})
-
-		if err != nil {
-			return nil, nil, err
-		}
-
-		for _, record := range resp.GetMeterReadingRecords() {
-			if record.GetRecordTime().AsTime().Before(earliest.GetRecordTime().AsTime()) {
-				earliest = record
-			}
-
-			if record.GetRecordTime().AsTime().After(latest.GetRecordTime().AsTime()) {
-				latest = record
-			}
-		}
-
-		if resp.GetNextPageToken() == "" {
-			break
-		}
-
-		pageToken = resp.GetNextPageToken()
+func getRecordsByTime(ctx context.Context, logger *zap.Logger, historyFn listMeterReadingFn, meter string, now time.Time, filterTime time.Duration) (earliest, latest *gen.MeterReadingRecord, err error) {
+	var resp *gen.ListMeterReadingHistoryResponse
+
+	start := now.Add(-filterTime)
+
+	resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
+		Name: meter,
+		Period: &sctime.Period{
+			StartTime: timestamppb.New(start),
+		},
+		PageSize: 1,
+	})
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if len(resp.GetMeterReadingRecords()) == 0 {
+		logger.Error("no records found in earliest", zap.String("meter", meter), zap.Time("start", start))
+		return earliest, latest, nil
+	}
+
+	earliest = resp.GetMeterReadingRecords()[0]
+
+	resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
+		Name: meter,
+		Period: &sctime.Period{
+			StartTime: timestamppb.New(now),
path: pkg/auto/exporthttp/job/meter.go
user: login: deanredfern26
body: Shouldn't this be EndTime? Else you are looking for records in the future
created_at: 2025-05-15T10:51:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -14,46 +15,50 @@ import (
 
 type listMeterReadingFn func(ctx context.Context, in *gen.ListMeterReadingHistoryRequest, opts ...grpc.CallOption) (*gen.ListMeterReadingHistoryResponse, error)
 
-func getRecordsByTime(ctx context.Context, historyFn listMeterReadingFn, meter string, now time.Time, filterTime time.Duration) (earliest, latest *gen.MeterReadingRecord, err error) {
-	var (
-		pageToken string
-		resp      *gen.ListMeterReadingHistoryResponse
-	)
-
-	latest = &gen.MeterReadingRecord{RecordTime: timestamppb.New(time.Time{})}
-	earliest = &gen.MeterReadingRecord{RecordTime: timestamppb.New(now)}
-
-	for {
-		resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
-			Name: meter,
-			Period: &sctime.Period{
-				StartTime: timestamppb.New(now.Add(-filterTime - time.Second)),
-				EndTime:   timestamppb.New(now),
-			},
-			PageToken: pageToken,
-		})
-
-		if err != nil {
-			return nil, nil, err
-		}
-
-		for _, record := range resp.GetMeterReadingRecords() {
-			if record.GetRecordTime().AsTime().Before(earliest.GetRecordTime().AsTime()) {
-				earliest = record
-			}
-
-			if record.GetRecordTime().AsTime().After(latest.GetRecordTime().AsTime()) {
-				latest = record
-			}
-		}
-
-		if resp.GetNextPageToken() == "" {
-			break
-		}
-
-		pageToken = resp.GetNextPageToken()
+func getRecordsByTime(ctx context.Context, logger *zap.Logger, historyFn listMeterReadingFn, meter string, now time.Time, filterTime time.Duration) (earliest, latest *gen.MeterReadingRecord, err error) {
+	var resp *gen.ListMeterReadingHistoryResponse
+
+	start := now.Add(-filterTime)
+
+	resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
+		Name: meter,
+		Period: &sctime.Period{
+			StartTime: timestamppb.New(start),
+		},
+		PageSize: 1,
+	})
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if len(resp.GetMeterReadingRecords()) == 0 {
+		logger.Error("no records found in earliest", zap.String("meter", meter), zap.Time("start", start))
+		return earliest, latest, nil
+	}
+
+	earliest = resp.GetMeterReadingRecords()[0]
+
+	resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
+		Name: meter,
+		Period: &sctime.Period{
+			StartTime: timestamppb.New(now),
path: pkg/auto/exporthttp/job/meter.go
user: login: dloveen-v
body: It means "I want the nearest record (page_size = 1) in the future _**after**_ this time (start_time) but the first of them (record_time DESC)".
created_at: 2025-05-15T11:14:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -14,46 +15,50 @@ import (
 
 type listMeterReadingFn func(ctx context.Context, in *gen.ListMeterReadingHistoryRequest, opts ...grpc.CallOption) (*gen.ListMeterReadingHistoryResponse, error)
 
-func getRecordsByTime(ctx context.Context, historyFn listMeterReadingFn, meter string, now time.Time, filterTime time.Duration) (earliest, latest *gen.MeterReadingRecord, err error) {
-	var (
-		pageToken string
-		resp      *gen.ListMeterReadingHistoryResponse
-	)
-
-	latest = &gen.MeterReadingRecord{RecordTime: timestamppb.New(time.Time{})}
-	earliest = &gen.MeterReadingRecord{RecordTime: timestamppb.New(now)}
-
-	for {
-		resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
-			Name: meter,
-			Period: &sctime.Period{
-				StartTime: timestamppb.New(now.Add(-filterTime - time.Second)),
-				EndTime:   timestamppb.New(now),
-			},
-			PageToken: pageToken,
-		})
-
-		if err != nil {
-			return nil, nil, err
-		}
-
-		for _, record := range resp.GetMeterReadingRecords() {
-			if record.GetRecordTime().AsTime().Before(earliest.GetRecordTime().AsTime()) {
-				earliest = record
-			}
-
-			if record.GetRecordTime().AsTime().After(latest.GetRecordTime().AsTime()) {
-				latest = record
-			}
-		}
-
-		if resp.GetNextPageToken() == "" {
-			break
-		}
-
-		pageToken = resp.GetNextPageToken()
+func getRecordsByTime(ctx context.Context, logger *zap.Logger, historyFn listMeterReadingFn, meter string, now time.Time, filterTime time.Duration) (earliest, latest *gen.MeterReadingRecord, err error) {
+	var resp *gen.ListMeterReadingHistoryResponse
+
+	start := now.Add(-filterTime)
+
+	resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
+		Name: meter,
+		Period: &sctime.Period{
+			StartTime: timestamppb.New(start),
+		},
+		PageSize: 1,
+	})
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if len(resp.GetMeterReadingRecords()) == 0 {
+		logger.Error("no records found in earliest", zap.String("meter", meter), zap.Time("start", start))
+		return earliest, latest, nil
+	}
+
+	earliest = resp.GetMeterReadingRecords()[0]
+
+	resp, err = historyFn(ctx, &gen.ListMeterReadingHistoryRequest{
+		Name: meter,
+		Period: &sctime.Period{
+			StartTime: timestamppb.New(now),
path: pkg/auto/exporthttp/job/meter.go
user: login: deanredfern26
body: Does this call not resolve to an SQL query something like this?
SELECT *
FROM history
WHERE source==meter AND start_time > now

I might be wrong but if so that won't get any records because the start time would have to be in the future
created_at: 2025-05-15T11:55:08Z
author_association: CONTRIBUTOR

PR comments:
user: login: dloveen-v
created_at: 2025-05-19T12:21:47Z
body: @deanredfern26 the reason I hadn't used `endTime` is because using this field parameter in a site deployment was giving very stale meter readings (from 2024). I have resolved the issue in this particular deployment and can merge this now.
author_association: CONTRIBUTOR
user: login: deanredfern26
created_at: 2025-05-19T14:17:44Z
body: > @deanredfern26 the reason I hadn't used `endTime` is because using this field parameter in a site deployment was giving very stale meter readings (from 2024). I have resolved the issue in this particular deployment and can merge this now.

Yeah the end time might not have worked there either depending on how meter API was implemented, my comment was more about not trying to look for records with a start time after `time.now`
author_association: CONTRIBUTOR
user: login: deanredfern26
created_at: 2025-05-21T09:23:12Z
body: @dloveen-v LGTM, have you managed to test this? Happy to approve if it has fixed/improved the problem
author_association: CONTRIBUTOR
user: login: dloveen-v
created_at: 2025-05-21T11:20:57Z
body: Yes this is deployed on site and is successful in getting consumption values between 2 timestamps. I have DM'd you a screenshot.
author_association: CONTRIBUTOR