Repository: google/certificate-transparency-go
Pull request #904 (2022-04-07 18:56:41)

Remove unused packages


Patch:
diff --git a/ingestor/ranges/tracker.go b/ingestor/ranges/tracker.go
deleted file mode 100644
index 14327fd1..00000000
--- a/ingestor/ranges/tracker.go
+++ /dev/null
@@ -1,214 +0,0 @@
-// Copyright 2017 Google LLC. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package ranges provides tools to track the completeness of a range composed of a number of sub-ranges which may be added in any order.
-package ranges
-
-import (
-	"fmt"
-	"math"
-	"sort"
-	"strings"
-	"time"
-)
-
-// Tracker tracks a range of integer indices which may be split into a set
-// of non-overlapping sub-ranges.  The Tracker is defined over a range
-// [start, end] with 0 <= start <= end.
-//
-// As sub-ranges are added to the Tracker, the range being tracked
-// progresses towards becoming complete.  When the set of sub-ranges that have
-// been added form a contiguous range equal to [start, end] the Tracker is
-// complete.
-//
-// If the set of sub-ranges that have been added include a sub-range whose first
-// index is 'start' the Tracker is considered partially complete.  Then,
-// PartiallyCompleteUpto() can be used to determine the last index in a sub-
-// range contiguous to the start.
-//
-// Examples:
-// 1. Tracker for [0, 99]: sub-ranges [10, 19] and [30, 39] added
-//    -> neither complete nor partially complete.
-// 2. Tracker for [0, 99]: sub-ranges [0, 19] and [30, 39] added
-//    -> not complete, but partially complete upto 19.
-// 3. Tracker for [0, 99]: sub-ranges [0, 19] and [20, 39] added
-//    -> not complete, but partially complete upto 39.
-// 4. Tracker for [0, 99]: sub-ranges [50, 99] and [0, 49] added
-//    -> complete.
-type Tracker struct {
-	start, end int64         // the extent of the range being tracked [start, end]
-	wm         watermarks    // the low, mid and high water marks (see below)
-	subRanges  subRangeSlice // the ranges recorded so far - kept sorted by subRange.first
-	startTime  time.Time     // timestamp recorded when Tracker created
-}
-
-// watermarks encapsulates a set of three watermarks that are calculated when
-// subranges are added.  The watermarks are used to determine the completion
-// state of the tracker.  They are defined as follows:
-// * the 'low-water' mark indicates the lowest value in any sub-range added;
-// * the 'mid-water' mark indicates the high point in the lowest contiguous set
-//   of sub-ranges if we have a sub-range whose first index is 'start'; if not,
-//   it is set to -1.
-// * the 'high-water' mark indicates the highest value in any sub-range added.
-// When (lo, hi) == (start, end) of the expected range, and hi == mid, then the
-// full extent of the expected range has been recorded.
-type watermarks struct {
-	lo, mid, hi int64
-}
-
-func (wm *watermarks) update(start int64, srs subRangeSlice) {
-	lo, mid, hi := int64(math.MaxInt64), int64(-1), int64(-1)
-	for _, sr := range srs {
-		if sr.first < lo {
-			lo = sr.first
-		}
-		if sr.last > hi {
-			hi = sr.last
-		}
-	}
-	if lo == start {
-		// The range is partially complete as we have a subrange located at the
-		// range start.  The mid watermark is the highest entry in a contiguous
-		// subrange.
-		nextLo := lo
-		for _, sr := range srs {
-			if sr.first == nextLo {
-				mid = sr.last
-				nextLo = mid + 1
-			}
-		}
-	}
-	wm.lo, wm.mid, wm.hi = lo, mid, hi
-}
-
-func (wm watermarks) String() string {
-	return fmt.Sprintf("<lo %d, mid %d, hi %d>", wm.lo, wm.mid, wm.hi)
-}
-
-// subRange is defined as a pair of indices [first, last]
-type subRange struct {
-	first, last int64
-}
-
-// String conforms with fmt.Stringer for subRange.
-func (sr subRange) String() string {
-	return fmt.Sprintf("<first %d, last %d>", sr.first, sr.last)
-}
-
-type subRangeSlice []subRange
-
-// Len conforms with sort.Interface for subRangeSlice
-func (srs subRangeSlice) Len() int {
-	return len(srs)
-}
-
-// Less conforms with sort.Interface for subRangeSlice
-func (srs subRangeSlice) Less(i, j int) bool {
-	return srs[i].first < srs[j].first
-}
-
-// Swap conforms with sort.Interface for subRangeSlice
-func (srs subRangeSlice) Swap(i, j int) {
-	srs[i].first, srs[j].first = srs[j].first, srs[i].first
-	srs[i].last, srs[j].last = srs[j].last, srs[i].last
-}
-
-// timeNow may be replaced in tests.
-var timeNow = time.Now
-
-// NewTracker returns a Tracker for the range [start, end].
-func NewTracker(start, end int64) (*Tracker, error) {
-	if start > end {
-		return nil, fmt.Errorf("want start <= end, got [start %d, end %d]", start, end)
-	}
-	if start < 0 {
-		return nil, fmt.Errorf("want start >= 0, got %d]", start)
-	}
-	return &Tracker{
-		start:     start,
-		end:       end,
-		wm:        watermarks{lo: -1, mid: -1, hi: -1},
-		startTime: timeNow(),
-	}, nil
-}
-
-// AddSubRange records a sub-range.  As sub-ranges are recorded, the low/mid/high watermarks are
-// adjusted.
-func (rt *Tracker) AddSubRange(first, last int64) error {
-	if last < first {
-		return fmt.Errorf("subrange [%d, %d] should have last > first", first, last)
-	}
-	if first < rt.start || last > rt.end {
-		return fmt.Errorf("subrange [%d, %d] may not lie outside [%d, %d]", first, last, rt.start, rt.end)
-	}
-	// Check that [first, last] doesn't overlap any already-added subranges.
-	for _, sr := range rt.subRanges {
-		if (sr.first <= first && first <= sr.last) || (sr.first <= last && last <= sr.last) {
-			return fmt.Errorf("trying to add overlapping subrange [%d, %d] which collides with [%d, %d]",
-				first, last, sr.first, sr.last)
-		}
-	}
-
-	rt.subRanges = append(rt.subRanges, subRange{first: first, last: last})
-	sort.Sort(rt.subRanges)
-	rt.wm.update(rt.start, rt.subRanges)
-	return nil
-}
-
-// IsComplete returns true if the union of recorded sub-ranges equals the expected range.
-func (rt *Tracker) IsComplete() bool {
-	return rt.IsPartiallyComplete() && rt.wm.mid == rt.wm.hi && rt.wm.hi == rt.end
-}
-
-// IsPartiallyComplete returns true if a sub-range has been recorded whose first entry is located at the start of the expected range.
-func (rt *Tracker) IsPartiallyComplete() bool {
-	return rt.wm.mid >= 0
-}
-
-// PartiallyCompleteUpto returns the last entry of a set of contiguous recorded
-// subranges that begins at the Tracker's start, or -1 if there are no
-// subranges that satisfy this.
-func (rt *Tracker) PartiallyCompleteUpto() int64 {
-	return rt.wm.mid
-}
-
-func (rt *Tracker) progressSummary() string {
-	switch {
-	case rt.IsComplete():
-		return "100% complete"
-	case !rt.IsPartiallyComplete():
-		return "0% complete"
-	}
-	fractionComplete := float32(rt.wm.mid+1-rt.start) / float32(rt.end-rt.start)
-	elapsed := float32(timeNow().Sub(rt.startTime))
-	remain := elapsed/fractionComplete - elapsed
-	return fmt.Sprintf("%2.1f%% complete, done in %v", 100.0*fractionComplete, time.Duration(remain))
-}
-
-// String returns a printable representation of the Tracker.
-func (rt *Tracker) String() string {
-	return fmt.Sprintf("<expected range [start %d, end %d] watermarks %v #subranges %d %s>",
-		rt.start, rt.end, rt.wm, len(rt.subRanges), rt.progressSummary())
-}
-
-// DebugString returns a verbose printable representation of the Tracker, including details of all added subranges, for debug use.
-func (rt *Tracker) DebugString() string {
-	srs := make([]string, 0, len(rt.subRanges))
-	for _, sr := range rt.subRanges {
-		srs = append(srs, sr.String())
-	}
-
-	return fmt.Sprintf("<expected range [start %d, end %d] watermarks %v subranges [%v] %s>",
-		rt.start, rt.end, rt.wm, strings.Join(srs, " "), rt.progressSummary())
-}
diff --git a/watcher/diff.go b/watcher/diff.go
deleted file mode 100644
index 1bd5e878..00000000
--- a/watcher/diff.go
+++ /dev/null
@@ -1,111 +0,0 @@
-// Copyright 2018 Google LLC. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package watcher holds tools for loglist-files versioning and updates propagation.
-package watcher
-
-import (
-	"context"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/google/certificate-transparency-go/schedule"
-	"github.com/google/certificate-transparency-go/x509util"
-	"github.com/sergi/go-diff/diffmatchpatch"
-)
-
-// Diff regularly check data at path provided, notifies if changes detected.
-type Diff struct {
-	latest []byte
-
-	synced []byte
-	diffs  []diffmatchpatch.Diff
-	// mu guards all data-fields: latest, synced and diffs.
-	mu sync.Mutex
-
-	// Exactly one of url/filepath fields is specified.
-	url      string
-	filepath string
-
-	checkInterval time.Duration
-
-	events chan<- DiffEvent
-}
-
-// DiffEvent refelects diff/error detection.
-type DiffEvent struct {
-	Diffs []diffmatchpatch.Diff
-	Err   error
-}
-
-// NewDiff is factory for Diff.
-func NewDiff(ctx context.Context, path string, isPathURL bool, checkInterval time.Duration, events chan<- DiffEvent) *Diff {
-	var d Diff
-	if isPathURL {
-		d.url = path
-	} else {
-		d.filepath = path
-	}
-	d.checkInterval = checkInterval
-	d.events = events
-
-	d.init(ctx)
-	return &d
-}
-
-func (d *Diff) init(ctx context.Context) {
-	d.checkUpdate()
-	go schedule.Every(ctx, d.checkInterval, func(ctx context.Context) {
-		d.checkUpdate()
-	})
-}
-
-func (d *Diff) checkUpdate() {
-	var path string
-	if len(d.url) > 0 {
-		path = d.url
-	} else {
-		path = d.filepath
-	}
-	llData, err := x509util.ReadFileOrURL(path, &http.Client{Timeout: time.Second * 10})
-	if err != nil {
-		d.events <- DiffEvent{Err: err}
-		return
-	}
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	d.latest = llData
-	// Compare data as strings
-	dmp := diffmatchpatch.New()
-	diffs := dmp.DiffMain(string(d.synced), string(d.latest), false)
-	d.diffs = diffs
-	if len(diffs) > 0 && d.events != nil {
-		d.events <- DiffEvent{Diffs: d.diffs}
-	}
-}
-
-func (d *Diff) sync() { // nolint:unused
-	d.mu.Lock()
-	d.synced = d.latest
-	d.diffs = []diffmatchpatch.Diff{}
-	d.mu.Unlock()
-}
-
-// GetSyncedData provides acces to Diff data.
-func (d *Diff) GetSyncedData() []byte {
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	return d.synced
-}


Test patch:
diff --git a/ingestor/ranges/tracker_test.go b/ingestor/ranges/tracker_test.go
deleted file mode 100644
index e2dac867..00000000
--- a/ingestor/ranges/tracker_test.go
+++ /dev/null
@@ -1,193 +0,0 @@
-// Copyright 2017 Google LLC. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ranges
-
-import (
-	"testing"
-	"time"
-)
-
-func TestEmpty(t *testing.T) {
-	tr, err := NewTracker(0, 100)
-	if err != nil {
-		t.Errorf("NewTracker(%d, %d) err=%v, want nil", 0, 100, err)
-	}
-
-	if tr.IsComplete() {
-		t.Error("IsComplete()? true, want false")
-	}
-
-	if tr.IsPartiallyComplete() {
-		t.Error("IsPartiallyComplete()? true, want false")
-	}
-}
-
-func TestAddingSubRanges(t *testing.T) {
-	type toAdd struct {
-		first, last           int64
-		wantErr               bool
-		wantComplete          bool
-		wantPartiallyComplete bool
-	}
-
-	tests := []struct {
-		desc                  string
-		start, end            int64
-		additions             []toAdd
-		wantErr               bool
-		wantString, wantDebug string
-	}{
-		{
-			desc:       "Valid input (start == end) but no sub-ranges added.",
-			wantString: "<expected range [start 0, end 0] watermarks <lo -1, mid -1, hi -1> #subranges 0 0% complete>",
-			wantDebug:  "<expected range [start 0, end 0] watermarks <lo -1, mid -1, hi -1> subranges [] 0% complete>",
-		},
-		{
-			desc:       "Valid input (start < end) but no sub-ranges added.",
-			end:        1,
-			wantString: "<expected range [start 0, end 1] watermarks <lo -1, mid -1, hi -1> #subranges 0 0% complete>",
-			wantDebug:  "<expected range [start 0, end 1] watermarks <lo -1, mid -1, hi -1> subranges [] 0% complete>",
-		},
-		{
-			desc:    "Invalid input (start < 0)",
-			start:   -1,
-			wantErr: true,
-		},
-		{
-			desc:    "Invalid input (end < start)",
-			start:   1,
-			wantErr: true,
-		},
-		{
-			desc:       "Invalid subranges - last < first",
-			end:        100,
-			additions:  []toAdd{{10, 0, true, false, false}},
-			wantString: "<expected range [start 0, end 100] watermarks <lo -1, mid -1, hi -1> #subranges 0 0% complete>",
-			wantDebug:  "<expected range [start 0, end 100] watermarks <lo -1, mid -1, hi -1> subranges [] 0% complete>",
-		},
-		{
-			desc:       "Invalid subranges - last > end",
-			end:        100,
-			additions:  []toAdd{{0, 105, true, false, false}},
-			wantString: "<expected range [start 0, end 100] watermarks <lo -1, mid -1, hi -1> #subranges 0 0% complete>",
-			wantDebug:  "<expected range [start 0, end 100] watermarks <lo -1, mid -1, hi -1> subranges [] 0% complete>",
-		},
-		{
-			desc:       "Invalid subranges - first < start",
-			start:      10,
-			end:        90,
-			additions:  []toAdd{{9, 10, true, false, false}},
-			wantString: "<expected range [start 10, end 90] watermarks <lo -1, mid -1, hi -1> #subranges 0 0% complete>",
-			wantDebug:  "<expected range [start 10, end 90] watermarks <lo -1, mid -1, hi -1> subranges [] 0% complete>",
-		},
-		{
-			desc: "Invalid subranges - 2nd overlaps 1st",
-			end:  100,
-			additions: []toAdd{
-				{5, 10, false, false, false},
-				{8, 15, true, false, false},
-			},
-			wantString: "<expected range [start 0, end 100] watermarks <lo 5, mid -1, hi 10> #subranges 1 0% complete>",
-			wantDebug:  "<expected range [start 0, end 100] watermarks <lo 5, mid -1, hi 10> subranges [<first 5, last 10>] 0% complete>",
-		},
-		{
-			desc:       "Single subrange added located at range start, so we are partially complete.",
-			end:        100,
-			additions:  []toAdd{{0, 19, false, false, true}},
-			wantString: "<expected range [start 0, end 100] watermarks <lo 0, mid 19, hi 19> #subranges 1 20.0% complete, done in 4s>",
-			wantDebug:  "<expected range [start 0, end 100] watermarks <lo 0, mid 19, hi 19> subranges [<first 0, last 19>] 20.0% complete, done in 4s>",
-		},
-		{
-			desc: "Two subranges added, partially complete after the second.",
-			end:  100,
-			additions: []toAdd{
-				{10, 19, false, false, false},
-				{0, 9, false, false, true},
-			},
-			wantString: "<expected range [start 0, end 100] watermarks <lo 0, mid 19, hi 19> #subranges 2 20.0% complete, done in 8s>",
-			wantDebug:  "<expected range [start 0, end 100] watermarks <lo 0, mid 19, hi 19> subranges [<first 0, last 9> <first 10, last 19>] 20.0% complete, done in 8s>",
-		},
-		{
-			desc: "Four subranges added in fully reverse order, not partially complete until end when it is fully complete.",
-			end:  100,
-			additions: []toAdd{
-				{75, 100, false, false, false},
-				{50, 74, false, false, false},
-				{25, 49, false, false, false},
-				{0, 24, false, true, true},
-			},
-			wantString: "<expected range [start 0, end 100] watermarks <lo 0, mid 100, hi 100> #subranges 4 100% complete>",
-			wantDebug:  "<expected range [start 0, end 100] watermarks <lo 0, mid 100, hi 100> subranges [<first 0, last 24> <first 25, last 49> <first 50, last 74> <first 75, last 100>] 100% complete>",
-		},
-		{
-			desc:       "Single subrange the size of the full range, complete straight away.",
-			end:        100,
-			additions:  []toAdd{{0, 100, false, true, true}},
-			wantString: "<expected range [start 0, end 100] watermarks <lo 0, mid 100, hi 100> #subranges 1 100% complete>",
-			wantDebug:  "<expected range [start 0, end 100] watermarks <lo 0, mid 100, hi 100> subranges [<first 0, last 100>] 100% complete>",
-		},
-		{
-			desc: "Lots of subranges of size 1. Partially complete straight away; complete by the end.",
-			end:  4,
-			additions: []toAdd{
-				{0, 0, false, false, true},
-				{1, 1, false, false, true},
-				{2, 2, false, false, true},
-				{3, 3, false, false, true},
-				{4, 4, false, true, true},
-			},
-			wantString: "<expected range [start 0, end 4] watermarks <lo 0, mid 4, hi 4> #subranges 5 100% complete>",
-			wantDebug:  "<expected range [start 0, end 4] watermarks <lo 0, mid 4, hi 4> subranges [<first 0, last 0> <first 1, last 1> <first 2, last 2> <first 3, last 3> <first 4, last 4>] 100% complete>",
-		},
-	}
-
-	for _, test := range tests {
-		now := time.Unix(0, 0)
-		timeNow = func() time.Time { return now }
-
-		tr, err := NewTracker(test.start, test.end)
-		if gotErr := err != nil; gotErr != test.wantErr {
-			t.Errorf("%s: NewTracker(%d, %d): got err? %t, want? %t (err %v)", test.desc, test.start, test.end, gotErr, test.wantErr, err)
-		}
-		if err != nil {
-			continue
-		}
-
-		for i, a := range test.additions {
-			now = now.Add(time.Second)
-
-			err := tr.AddSubRange(a.first, a.last)
-			if gotErr := err != nil; gotErr != a.wantErr {
-				t.Errorf("%s: %d: AddSubRange(%d, %d): got err? %t, want? %t (err %v)", test.desc, i, a.first, a.last, gotErr, a.wantErr, err)
-			}
-
-			if got := tr.IsComplete(); got != a.wantComplete {
-				t.Errorf("%s: %d: IsComplete()? %t, want? %t (Tracker state %v)", test.desc, i, got, a.wantComplete, tr)
-			}
-
-			if got := tr.IsPartiallyComplete(); got != a.wantPartiallyComplete {
-				t.Errorf("%s: %d: IsPartiallyComplete()? %t, want? %t (Tracker state %v)", test.desc, i, got, a.wantPartiallyComplete, tr)
-			}
-		}
-
-		if got, want := tr.String(), test.wantString; got != want {
-			t.Errorf("%s: String(): got %q, want %q", test.desc, got, want)
-		}
-
-		if got, want := tr.DebugString(), test.wantDebug; got != want {
-			t.Errorf("%s: DebugString(): got %q, want %q", test.desc, got, want)
-		}
-	}
-}