Remove unnecessary Printf

Envelope sender and RFC5322.From address are set to $EMAIL if it's non-empty.

Requested in #87

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change Log
 
+## v0.18.0 (2023-11-13)
+- Fix bug with downloading entries that did not materialize in practice
+  with any of the current logs.
+- Include `Message-ID` and `Date` in outbound emails.
+
 ## v0.17.0 (2023-10-26)
 - Allow sendmail path to be configured with `$SENDMAIL_PATH`.
 - Minor improvements to documentation, efficiency.

README.md

@@ -46,7 +46,7 @@ Cert Spotter requires Go version 1.19 or higher.
 
 4. Configure your system to run `certspotter` as a daemon.  You may want to specify
    the `-start_at_end` command line option to tell certspotter to start monitoring
-   logs at the end instead of the beginning.  This saves significant bandwidth, but
+   new logs at the end instead of the beginning.  This saves significant bandwidth, but
    you won't be notified about certificates which were logged before you started
    using certspotter.
 

cmd/certspotter/main.go

@@ -16,7 +16,6 @@ import (
 	"flag"
 	"fmt"
 	"io/fs"
-	insecurerand "math/rand"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -152,8 +151,6 @@ func appendFunc(slice *[]string) func(string) error {
 }
 
 func main() {
-	insecurerand.Seed(time.Now().UnixNano()) // TODO: remove after upgrading to Go 1.20
-
 	loglist.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s)", certspotterVersion(), runtime.Version(), runtime.GOOS, runtime.GOARCH)
 
 	var flags struct {
@@ -176,7 +173,7 @@ func main() {
 	flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
 	flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory")
 	flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered")
-	flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring logs from the end rather than the beginning (saves considerable bandwidth)")
+	flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring new logs from the end rather than the beginning (saves considerable bandwidth)")
 	flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates")
 	flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout")
 	flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose")
@@ -193,38 +190,36 @@ func main() {
 		os.Exit(2)
 	}
 
+	fsstate := &monitor.FilesystemState{
+		StateDir:  flags.stateDir,
+		SaveCerts: !flags.noSave,
+		Script:    flags.script,
+		ScriptDir: defaultScriptDir(),
+		Email:     flags.email,
+		Stdout:    flags.stdout,
+	}
 	config := &monitor.Config{
 		LogListSource:       flags.logs,
-		StateDir:            flags.stateDir,
+		State:               fsstate,
-		SaveCerts:           !flags.noSave,
 		StartAtEnd:          flags.startAtEnd,
 		Verbose:             flags.verbose,
-		Script:              flags.script,
-		ScriptDir:           defaultScriptDir(),
-		SendmailPath:        "/usr/sbin/sendmail",
-		Email:               flags.email,
-		Stdout:              flags.stdout,
 		HealthCheckInterval: flags.healthcheck,
 	}
 
-	if envVar := os.Getenv("SENDMAIL_PATH"); envVar != "" {
-		config.SendmailPath = envVar
-	}
-
 	emailFileExists := false
 	if emailRecipients, err := readEmailFile(defaultEmailFile()); err == nil {
 		emailFileExists = true
-		config.Email = append(config.Email, emailRecipients...)
+		fsstate.Email = append(fsstate.Email, emailRecipients...)
 	} else if !errors.Is(err, fs.ErrNotExist) {
 		fmt.Fprintf(os.Stderr, "%s: error reading email recipients file %q: %s\n", programName, defaultEmailFile(), err)
 		os.Exit(1)
 	}
 
-	if len(config.Email) == 0 && !emailFileExists && config.Script == "" && !fileExists(config.ScriptDir) && config.Stdout == false {
+	if len(fsstate.Email) == 0 && !emailFileExists && fsstate.Script == "" && !fileExists(fsstate.ScriptDir) && fsstate.Stdout == false {
 		fmt.Fprintf(os.Stderr, "%s: no notification methods were specified\n", programName)
 		fmt.Fprintf(os.Stderr, "Please specify at least one of the following notification methods:\n")
 		fmt.Fprintf(os.Stderr, " - Place one or more email addresses in %s (one address per line)\n", defaultEmailFile())
-		fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", config.ScriptDir)
+		fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", fsstate.ScriptDir)
 		fmt.Fprintf(os.Stderr, " - Specify an email address using the -email flag\n")
 		fmt.Fprintf(os.Stderr, " - Specify the path to an executable script using the -script flag\n")
 		fmt.Fprintf(os.Stderr, " - Specify the -stdout flag\n")

cmd/submitct/main.go

@@ -158,18 +158,19 @@ func main() {
 
 	var logs []Log
 	for _, ctlog := range list.AllLogs() {
+		submissionURL := ctlog.GetSubmissionURL()
 		pubkey, err := x509.ParsePKIXPublicKey(ctlog.Key)
 		if err != nil {
-			log.Fatalf("%s: Failed to parse log public key: %s", ctlog.URL, err)
+			log.Fatalf("%s: Failed to parse log public key: %s", submissionURL, err)
 		}
 		verifier, err := ct.NewSignatureVerifier(pubkey)
 		if err != nil {
-			log.Fatalf("%s: Failed to create signature verifier for log: %s", ctlog.URL, err)
+			log.Fatalf("%s: Failed to create signature verifier for log: %s", submissionURL, err)
 		}
 		logs = append(logs, Log{
 			Log: ctlog,
 			SignatureVerifier: verifier,
-			LogClient: client.New(strings.TrimRight(ctlog.URL, "/")),
+			LogClient: client.New(strings.TrimRight(submissionURL, "/")),
 		})
 	}
 
@@ -212,11 +213,11 @@ func main() {
 			go func(fingerprint [32]byte, ctlog Log) {
 				sct, err := ctlog.SubmitChain(chain)
 				if err != nil {
-					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.URL, err)
+					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.GetSubmissionURL(), err)
 					atomic.AddUint32(&submitErrors, 1)
 				} else if *verbose {
 					timestamp := time.Unix(int64(sct.Timestamp)/1000, int64(sct.Timestamp%1000)*1000000)
-					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.URL, timestamp)
+					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.GetSubmissionURL(), timestamp)
 				}
 				wg.Done()
 			}(fingerprint, ctlog)

loglist/helpers.go

@@ -13,12 +13,16 @@ import (
 	"time"
 )
 
+// Return all tiled and non-tiled logs from all operators
 func (list *List) AllLogs() []*Log {
 	logs := []*Log{}
 	for operator := range list.Operators {
 		for log := range list.Operators[operator].Logs {
 			logs = append(logs, &list.Operators[operator].Logs[log])
 		}
+		for log := range list.Operators[operator].TiledLogs {
+			logs = append(logs, &list.Operators[operator].TiledLogs[log])
+		}
 	}
 	return logs
 }

loglist/schema.go

@@ -22,16 +22,19 @@ type List struct {
 }
 
 type Operator struct {
-	Name  string   `json:"name"`
+	Name      string   `json:"name"`
-	Email []string `json:"email"`
+	Email     []string `json:"email"`
-	Logs  []Log    `json:"logs"`
+	Logs      []Log    `json:"logs"`
+	TiledLogs []Log    `json:"tiled_logs"`
 }
 
 type Log struct {
 	Key              []byte        `json:"key"`
 	LogID            ct.SHA256Hash `json:"log_id"`
 	MMD              int           `json:"mmd"`
-	URL              string        `json:"url"`
+	URL              string        `json:"url,omitempty"`            // only for rfc6962 logs
+	SubmissionURL    string        `json:"submission_url,omitempty"` // only for static-ct-api logs
+	MonitoringURL    string        `json:"monitoring_url,omitempty"` // only for static-ct-api logs
 	Description      string        `json:"description"`
 	State            State         `json:"state"`
 	DNS              string        `json:"dns"`
@@ -44,6 +47,29 @@ type Log struct {
 	// TODO: add previous_operators
 }
 
+func (log *Log) IsRFC6962() bool     { return log.URL != "" }
+func (log *Log) IsStaticCTAPI() bool { return log.SubmissionURL != "" && log.MonitoringURL != "" }
+
+// Return URL prefix for submission using the RFC6962 protocol
+func (log *Log) GetSubmissionURL() string {
+	if log.SubmissionURL != "" {
+		return log.SubmissionURL
+	} else {
+		return log.URL
+	}
+}
+
+// Return URL prefix for monitoring.
+// Since the protocol understood by the URL might be either RFC6962 or static-ct-api, this URL is
+// only useful for informational purposes.
+func (log *Log) GetMonitoringURL() string {
+	if log.MonitoringURL != "" {
+		return log.MonitoringURL
+	} else {
+		return log.URL
+	}
+}
+
 type State struct {
 	Pending *struct {
 		Timestamp time.Time `json:"timestamp"`

loglist/validate.go

@@ -26,7 +26,12 @@ func (list *List) Validate() error {
 func (operator *Operator) Validate() error {
 	for i := range operator.Logs {
 		if err := operator.Logs[i].Validate(); err != nil {
-			return fmt.Errorf("problem with %dth log (%s): %w", i, operator.Logs[i].LogIDString(), err)
+			return fmt.Errorf("problem with %dth non-tiled log (%s): %w", i, operator.Logs[i].LogIDString(), err)
+		}
+	}
+	for i := range operator.TiledLogs {
+		if err := operator.TiledLogs[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth tiled log (%s): %w", i, operator.TiledLogs[i].LogIDString(), err)
 		}
 	}
 	return nil
@@ -37,5 +42,12 @@ func (log *Log) Validate() error {
 	if log.LogID != realLogID {
 		return fmt.Errorf("log ID does not match log key")
 	}
+
+	if !log.IsRFC6962() && !log.IsStaticCTAPI() {
+		return fmt.Errorf("URL(s) not provided")
+	} else if log.IsRFC6962() && log.IsStaticCTAPI() {
+		return fmt.Errorf("inconsistent URLs provided")
+	}
+
 	return nil
 }

man/certspotter.md

@@ -63,7 +63,9 @@ You can use Cert Spotter to detect:
 
 -no\_save
 
-:   Do not save a copy of matching certificates.
+:   Do not save a copy of matching certificates. Note that enabling this option
+    will cause you to receive duplicate notifications, since certspotter will
+    have no way of knowing if you've been previously notified about a certificate.
 
 -script *COMMAND*
 
@@ -213,6 +215,11 @@ and non-zero when a serious error occurs.
 :   Directory from which any configuration, such as the watch list, is read.
     Defaults to `~/.certspotter`.
 
+`EMAIL`
+
+:   Email address from which to send emails. If not set, certspotter lets sendmail pick
+    the address.
+
 `HTTPS_PROXY`
 
 :   URL of proxy server for making HTTPS requests.  `http://`, `https://`, and

merkletree/collapsed_tree.go

@@ -16,46 +16,76 @@ import (
 	"slices"
 )
 
+// CollapsedTree is an efficient representation of a Merkle (sub)tree that permits appending
+// nodes and calculating the root hash.
 type CollapsedTree struct {
-	nodes []Hash
+	offset uint64
-	size  uint64
+	nodes  []Hash
+	size   uint64
 }
 
 func calculateNumNodes(size uint64) int {
 	return bits.OnesCount64(size)
 }
 
+// TODO: phase out this function
 func EmptyCollapsedTree() *CollapsedTree {
 	return &CollapsedTree{nodes: []Hash{}, size: 0}
 }
 
+// TODO: phase out this function
 func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) {
-	if len(nodes) != calculateNumNodes(size) {
+	tree := new(CollapsedTree)
-		return nil, fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes))
+	if err := tree.Init(nodes, size); err != nil {
+		return nil, err
 	}
-	return &CollapsedTree{nodes: nodes, size: size}, nil
+	return tree, nil
-}
-
-func CloneCollapsedTree(source *CollapsedTree) *CollapsedTree {
-	nodes := make([]Hash, len(source.nodes))
-	copy(nodes, source.nodes)
-	return &CollapsedTree{nodes: nodes, size: source.size}
 }
 
 func (tree CollapsedTree) Equal(other CollapsedTree) bool {
-	return tree.size == other.size && slices.Equal(tree.nodes, other.nodes)
+	return tree.offset == other.offset && tree.size == other.size && slices.Equal(tree.nodes, other.nodes)
 }
 
-func (tree *CollapsedTree) Add(hash Hash) {
+func (tree CollapsedTree) Clone() CollapsedTree {
+	return CollapsedTree{
+		offset: tree.offset,
+		nodes:  slices.Clone(tree.nodes),
+		size:   tree.size,
+	}
+}
+
+// Add a new leaf hash to the end of the tree.
+// Returns an error if and only if the new tree would be too large for the subtree offset.
+// Always returns a nil error if tree.Offset() == 0.
+func (tree *CollapsedTree) Add(hash Hash) error {
+	if tree.offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if tree.size+1 > maxSize {
+			return fmt.Errorf("subtree at offset %d is already at maximum size %d", tree.offset, maxSize)
+		}
+	}
 	tree.nodes = append(tree.nodes, hash)
 	tree.size++
 	tree.collapse()
+	return nil
 }
 
-func (tree *CollapsedTree) Append(other *CollapsedTree) error {
+func (tree *CollapsedTree) Append(other CollapsedTree) error {
-	maxSize := uint64(1) << bits.TrailingZeros64(tree.size)
+	if tree.offset+tree.size != other.offset {
-	if other.size > maxSize {
+		return fmt.Errorf("subtree at offset %d cannot be appended to subtree ending at offset %d", other.offset, tree.offset+tree.size)
-		return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize)
+	}
+	if tree.offset > 0 {
+		newSize := tree.size + other.size
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if newSize > maxSize {
+			return fmt.Errorf("size of new subtree (%d) would exceed maximum size %d for a subtree at offset %d", newSize, maxSize, tree.offset)
+		}
+	}
+	if tree.size > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.size)
+		if other.size > maxSize {
+			return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize)
+		}
 	}
 
 	tree.nodes = append(tree.nodes, other.nodes...)
@@ -73,7 +103,7 @@ func (tree *CollapsedTree) collapse() {
 	}
 }
 
-func (tree *CollapsedTree) CalculateRoot() Hash {
+func (tree CollapsedTree) CalculateRoot() Hash {
 	if len(tree.nodes) == 0 {
 		return HashNothing()
 	}
@@ -86,29 +116,67 @@ func (tree *CollapsedTree) CalculateRoot() Hash {
 	return hash
 }
 
-func (tree *CollapsedTree) Size() uint64 {
+// Return the subtree offset (0 if this represents an entire tree)
+func (tree CollapsedTree) Offset() uint64 {
+	return tree.offset
+}
+
+// Return a non-nil slice containing the nodes.  The slice
+// must not be modified.
+func (tree CollapsedTree) Nodes() []Hash {
+	if tree.nodes == nil {
+		return []Hash{}
+	} else {
+		return tree.nodes
+	}
+}
+
+// Return the number of leaf nodes in the tree.
+func (tree CollapsedTree) Size() uint64 {
 	return tree.size
 }
 
+type collapsedTreeMessage struct {
+	Offset uint64 `json:"offset,omitempty"`
+	Nodes  []Hash `json:"nodes"` // never nil
+	Size   uint64 `json:"size"`
+}
+
 func (tree CollapsedTree) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
+	return json.Marshal(collapsedTreeMessage{
-		"nodes": tree.nodes,
+		Offset: tree.offset,
-		"size":  tree.size,
+		Nodes:  tree.Nodes(),
+		Size:   tree.size,
 	})
 }
 
 func (tree *CollapsedTree) UnmarshalJSON(b []byte) error {
-	var rawTree struct {
+	var rawTree collapsedTreeMessage
-		Nodes []Hash `json:"nodes"`
-		Size  uint64 `json:"size"`
-	}
 	if err := json.Unmarshal(b, &rawTree); err != nil {
 		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
 	}
-	if len(rawTree.Nodes) != calculateNumNodes(rawTree.Size) {
+	if err := tree.InitSubtree(rawTree.Offset, rawTree.Nodes, rawTree.Size); err != nil {
-		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: nodes has wrong length (should be %d, not %d)", calculateNumNodes(rawTree.Size), len(rawTree.Nodes))
+		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
 	}
-	tree.size = rawTree.Size
-	tree.nodes = rawTree.Nodes
 	return nil
 }
+
+func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error {
+	if len(nodes) != calculateNumNodes(size) {
+		return fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes))
+	}
+	tree.size = size
+	tree.nodes = nodes
+	return nil
+}
+
+func (tree *CollapsedTree) InitSubtree(offset uint64, nodes []Hash, size uint64) error {
+	if offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(offset)
+		if size > maxSize {
+			return fmt.Errorf("subtree size (%d) is too large for offset %d (maximum size is %d)", size, offset, maxSize)
+		}
+	}
+	tree.offset = offset
+	return tree.Init(nodes, size)
+}

merkletree/fragment.go

@@ -0,0 +1,121 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"encoding/json"
+	"fmt"
+	"slices"
+)
+
+// FragmentedCollapsedTree represents a sequence of non-overlapping subtrees
+type FragmentedCollapsedTree struct {
+	subtrees []CollapsedTree // sorted by offset
+}
+
+func (tree *FragmentedCollapsedTree) AddHash(position uint64, hash Hash) error {
+	return tree.Add(CollapsedTree{
+		offset: position,
+		nodes:  []Hash{hash},
+		size:   1,
+	})
+}
+
+func (tree *FragmentedCollapsedTree) Add(subtree CollapsedTree) error {
+	if subtree.size == 0 {
+		return nil
+	}
+	i := len(tree.subtrees)
+	for i > 0 && tree.subtrees[i-1].offset > subtree.offset {
+		i--
+	}
+	if i > 0 && tree.subtrees[i-1].offset+tree.subtrees[i-1].size > subtree.offset {
+		return fmt.Errorf("overlaps with subtree ending at %d", tree.subtrees[i-1].offset+tree.subtrees[i-1].size)
+	}
+	if i < len(tree.subtrees) && subtree.offset+subtree.size > tree.subtrees[i].offset {
+		return fmt.Errorf("overlaps with subtree starting at %d", tree.subtrees[i].offset)
+	}
+	if i == 0 || tree.subtrees[i-1].Append(subtree) != nil {
+		tree.subtrees = slices.Insert(tree.subtrees, i, subtree)
+		i++
+	}
+	for i < len(tree.subtrees) && tree.subtrees[i-1].Append(tree.subtrees[i]) == nil {
+		tree.subtrees = slices.Delete(tree.subtrees, i, i+1)
+	}
+	return nil
+}
+
+func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error {
+	for _, subtree := range other.subtrees {
+		if err := tree.Add(subtree); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) {
+	var prevEnd uint64
+	for i := range tree.subtrees {
+		if prevEnd != tree.subtrees[i].offset {
+			if !yield(prevEnd, tree.subtrees[i].offset) {
+				return
+			}
+		}
+		prevEnd = tree.subtrees[i].offset + tree.subtrees[i].size
+	}
+	yield(prevEnd, 0)
+}
+
+func (tree FragmentedCollapsedTree) NumSubtrees() int {
+	return len(tree.subtrees)
+}
+
+func (tree FragmentedCollapsedTree) Subtree(i int) CollapsedTree {
+	return tree.subtrees[i]
+}
+
+func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree {
+	if tree.subtrees == nil {
+		return []CollapsedTree{}
+	} else {
+		return tree.subtrees
+	}
+}
+
+// Return true iff the tree contains at least the first n nodes (without any gaps)
+func (tree FragmentedCollapsedTree) ContainsFirstN(n uint64) bool {
+	return len(tree.subtrees) >= 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size >= n
+}
+
+func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error {
+	for i := 1; i < len(subtrees); i++ {
+		if subtrees[i-1].offset+subtrees[i-1].size > subtrees[i].offset {
+			return fmt.Errorf("subtrees %d and %d overlap", i-1, i)
+		}
+	}
+	tree.subtrees = subtrees
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(tree.Subtrees())
+}
+
+func (tree *FragmentedCollapsedTree) UnmarshalJSON(b []byte) error {
+	var subtrees []CollapsedTree
+	if err := json.Unmarshal(b, &subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	if err := tree.Init(subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	return nil
+}

monitor/config.go

@@ -15,15 +15,9 @@ import (
 
 type Config struct {
 	LogListSource       string
-	StateDir            string
+	State               StateProvider
 	StartAtEnd          bool
 	WatchList           WatchList
 	Verbose             bool
-	SaveCerts           bool
-	SendmailPath        string
-	Script              string
-	ScriptDir           string
-	Email               []string
-	Stdout              bool
 	HealthCheckInterval time.Duration
 }

monitor/daemon.go

@@ -16,7 +16,6 @@ import (
 	"golang.org/x/sync/errgroup"
 	"log"
 	insecurerand "math/rand"
-	"path/filepath"
 	"software.sslmate.com/src/certspotter/loglist"
 	"time"
 )
@@ -51,18 +50,13 @@ type daemon struct {
 
 func (daemon *daemon) healthCheck(ctx context.Context) error {
 	if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval {
-		textPath := filepath.Join(daemon.config.StateDir, "healthchecks", healthCheckFilename())
+		info := &StaleLogListInfo{
-		event := &staleLogListEvent{
 			Source:        daemon.config.LogListSource,
 			LastSuccess:   daemon.logsLoadedAt,
 			LastError:     daemon.logListError,
 			LastErrorTime: daemon.logListErrorAt,
-			TextPath:      textPath,
 		}
-		if err := event.save(); err != nil {
+		if err := daemon.config.State.NotifyHealthCheckFailure(ctx, nil, info); err != nil {
-			return fmt.Errorf("error saving stale log list event: %w", err)
-		}
-		if err := notify(ctx, daemon.config, event); err != nil {
 			return fmt.Errorf("error notifying about stale log list: %w", err)
 		}
 	}
@@ -129,8 +123,8 @@ func (daemon *daemon) loadLogList(ctx context.Context) error {
 }
 
 func (daemon *daemon) run(ctx context.Context) error {
-	if err := prepareStateDir(daemon.config.StateDir); err != nil {
+	if err := daemon.config.State.Prepare(ctx); err != nil {
-		return fmt.Errorf("error preparing state directory: %w", err)
+		return fmt.Errorf("error preparing state: %w", err)
 	}
 
 	if err := daemon.loadLogList(ctx); err != nil {
@@ -150,7 +144,7 @@ func (daemon *daemon) run(ctx context.Context) error {
 			if err := daemon.loadLogList(ctx); err != nil {
 				daemon.logListError = err.Error()
 				daemon.logListErrorAt = time.Now()
-				recordError(fmt.Errorf("error reloading log list (will try again later): %w", err))
+				recordError(ctx, daemon.config, nil, fmt.Errorf("error reloading log list (will try again later): %w", err))
 			}
 			reloadLogListTicker.Reset(reloadLogListInterval())
 		case <-healthCheckTicker.C:

monitor/discoveredcert.go

@@ -21,21 +21,24 @@ import (
 	"software.sslmate.com/src/certspotter/ct"
 )
 
-type discoveredCert struct {
+type DiscoveredCert struct {
 	WatchItem    WatchItem
-	LogEntry     *logEntry
+	LogEntry     *LogEntry
 	Info         *certspotter.CertInfo
 	Chain        []ct.ASN1Cert // first entry is the leaf certificate or precertificate
 	TBSSHA256    [32]byte      // computed over Info.TBS.Raw
 	SHA256       [32]byte      // computed over Chain[0]
 	PubkeySHA256 [32]byte      // computed over Info.TBS.PublicKey.FullBytes
 	Identifiers  *certspotter.Identifiers
-	CertPath     string // empty if not saved on the filesystem
-	JSONPath     string // empty if not saved on the filesystem
-	TextPath     string // empty if not saved on the filesystem
 }
 
-func (cert *discoveredCert) pemChain() []byte {
+type certPaths struct {
+	certPath string
+	jsonPath string
+	textPath string
+}
+
+func (cert *DiscoveredCert) pemChain() []byte {
 	var buffer bytes.Buffer
 	for _, certBytes := range cert.Chain {
 		if err := pem.Encode(&buffer, &pem.Block{
@@ -48,7 +51,7 @@ func (cert *discoveredCert) pemChain() []byte {
 	return buffer.Bytes()
 }
 
-func (cert *discoveredCert) json() any {
+func (cert *DiscoveredCert) json() any {
 	object := map[string]any{
 		"tbs_sha256":    hex.EncodeToString(cert.TBSSHA256[:]),
 		"pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]),
@@ -67,23 +70,23 @@ func (cert *discoveredCert) json() any {
 	return object
 }
 
-func (cert *discoveredCert) save() error {
+func writeCertFiles(cert *DiscoveredCert, paths *certPaths) error {
-	if err := writeFile(cert.CertPath, cert.pemChain(), 0666); err != nil {
+	if err := writeFile(paths.certPath, cert.pemChain(), 0666); err != nil {
 		return err
 	}
-	if err := writeJSONFile(cert.JSONPath, cert.json(), 0666); err != nil {
+	if err := writeJSONFile(paths.jsonPath, cert.json(), 0666); err != nil {
 		return err
 	}
-	if err := writeTextFile(cert.TextPath, cert.Text(), 0666); err != nil {
+	if err := writeTextFile(paths.textPath, certNotificationText(cert, paths), 0666); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (cert *discoveredCert) Environ() []string {
+func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string {
 	env := []string{
 		"EVENT=discovered_cert",
-		"SUMMARY=" + cert.Summary(),
+		"SUMMARY=" + certNotificationSummary(cert),
 		"CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented
 		"LOG_URI=" + cert.LogEntry.Log.URL,
 		"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
@@ -93,9 +96,12 @@ func (cert *discoveredCert) Environ() []string {
 		"FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented
 		"PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]),
 		"PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented
-		"CERT_FILENAME=" + cert.CertPath,
+	}
-		"JSON_FILENAME=" + cert.JSONPath,
+
-		"TEXT_FILENAME=" + cert.TextPath,
+	if paths != nil {
+		env = append(env, "CERT_FILENAME="+paths.certPath)
+		env = append(env, "JSON_FILENAME="+paths.jsonPath)
+		env = append(env, "TEXT_FILENAME="+paths.textPath)
 	}
 
 	if cert.Info.ValidityParseError == nil {
@@ -130,7 +136,7 @@ func (cert *discoveredCert) Environ() []string {
 	return env
 }
 
-func (cert *discoveredCert) Text() string {
+func certNotificationText(cert *DiscoveredCert, paths *certPaths) string {
 	// TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration)
 
 	text := new(strings.Builder)
@@ -158,13 +164,13 @@ func (cert *discoveredCert) Text() string {
 	}
 	writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL))
 	writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:]))
-	if cert.CertPath != "" {
+	if paths != nil {
-		writeField("Filename", cert.CertPath)
+		writeField("Filename", paths.certPath)
 	}
 
 	return text.String()
 }
 
-func (cert *discoveredCert) Summary() string {
+func certNotificationSummary(cert *DiscoveredCert) string {
 	return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem)
 }

monitor/errors.go

@@ -10,9 +10,19 @@
 package monitor
 
 import (
+	"context"
 	"log"
+
+	"software.sslmate.com/src/certspotter/loglist"
 )
 
-func recordError(err error) {
+func recordError(ctx context.Context, config *Config, ctlog *loglist.Log, errToRecord error) {
-	log.Print(err)
+	if err := config.State.NotifyError(ctx, ctlog, errToRecord); err != nil {
+		log.Print("unable to notify about error: ", err)
+		if ctlog == nil {
+			log.Print(errToRecord)
+		} else {
+			log.Print(ctlog.URL, ": ", errToRecord)
+		}
+	}
 }

monitor/fsstate.go

@@ -0,0 +1,239 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+type FilesystemState struct {
+	StateDir  string
+	SaveCerts bool
+	Script    string
+	ScriptDir string
+	Email     []string
+	Stdout    bool
+}
+
+func (s *FilesystemState) logStateDir(logID LogID) string {
+	return filepath.Join(s.StateDir, "logs", logID.Base64URLString())
+}
+
+func (s *FilesystemState) Prepare(ctx context.Context) error {
+	return prepareStateDir(s.StateDir)
+}
+
+func (s *FilesystemState) PrepareLog(ctx context.Context, logID LogID) error {
+	var (
+		stateDirPath        = s.logStateDir(logID)
+		sthsDirPath         = filepath.Join(stateDirPath, "unverified_sths")
+		malformedDirPath    = filepath.Join(stateDirPath, "malformed_entries")
+		healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
+	)
+	for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} {
+		if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *FilesystemState) LoadLogState(ctx context.Context, logID LogID) (*LogState, error) {
+	filePath := filepath.Join(s.logStateDir(logID), "state.json")
+	fileBytes, err := os.ReadFile(filePath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	state := new(LogState)
+	if err := json.Unmarshal(fileBytes, state); err != nil {
+		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
+	}
+	return state, nil
+}
+
+func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state *LogState) error {
+	filePath := filepath.Join(s.logStateDir(logID), "state.json")
+	return writeJSONFile(filePath, state, 0666)
+}
+
+func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return storeSTHInDir(sthsDirPath, sth)
+}
+
+func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*ct.SignedTreeHead, error) {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return loadSTHsFromDir(sthsDirPath)
+}
+
+func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return removeSTHFromDir(sthsDirPath, sth)
+}
+
+func (s *FilesystemState) NotifyCert(ctx context.Context, cert *DiscoveredCert) error {
+	var notifiedPath string
+	var paths *certPaths
+	if s.SaveCerts {
+		hexFingerprint := hex.EncodeToString(cert.SHA256[:])
+		prefixPath := filepath.Join(s.StateDir, "certs", hexFingerprint[0:2])
+		var (
+			notifiedFilename      = "." + hexFingerprint + ".notified"
+			certFilename          = hexFingerprint + ".pem"
+			jsonFilename          = hexFingerprint + ".v1.json"
+			textFilename          = hexFingerprint + ".txt"
+			legacyCertFilename    = hexFingerprint + ".cert.pem"
+			legacyPrecertFilename = hexFingerprint + ".precert.pem"
+		)
+
+		for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} {
+			if fileExists(filepath.Join(prefixPath, filename)) {
+				return nil
+			}
+		}
+
+		if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
+		}
+
+		notifiedPath = filepath.Join(prefixPath, notifiedFilename)
+		paths = &certPaths{
+			certPath: filepath.Join(prefixPath, certFilename),
+			jsonPath: filepath.Join(prefixPath, jsonFilename),
+			textPath: filepath.Join(prefixPath, textFilename),
+		}
+		if err := writeCertFiles(cert, paths); err != nil {
+			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
+		}
+	} else {
+		// TODO-4: save cert to temporary files, and defer their unlinking
+	}
+
+	if err := s.notify(ctx, &notification{
+		summary: certNotificationSummary(cert),
+		environ: certNotificationEnviron(cert, paths),
+		text:    certNotificationText(cert, paths),
+	}); err != nil {
+		return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
+	}
+
+	if notifiedPath != "" {
+		if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
+			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
+		}
+	}
+
+	return nil
+}
+
+func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError error) error {
+	var (
+		dirPath   = filepath.Join(s.logStateDir(entry.Log.LogID), "malformed_entries")
+		entryPath = filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index))
+		textPath  = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index))
+	)
+
+	summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.URL)
+
+	entryJSON := struct {
+		LeafInput []byte `json:"leaf_input"`
+		ExtraData []byte `json:"extra_data"`
+	}{
+		LeafInput: entry.LeafInput,
+		ExtraData: entry.ExtraData,
+	}
+
+	text := new(strings.Builder)
+	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
+	fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
+	writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.URL))
+	writeField("Leaf Hash", entry.LeafHash.Base64String())
+	writeField("Error", parseError.Error())
+
+	if err := writeJSONFile(entryPath, entryJSON, 0666); err != nil {
+		return fmt.Errorf("error saving JSON file: %w", err)
+	}
+	if err := writeTextFile(textPath, text.String(), 0666); err != nil {
+		return fmt.Errorf("error saving texT file: %w", err)
+	}
+
+	environ := []string{
+		"EVENT=malformed_cert",
+		"SUMMARY=" + summary,
+		"LOG_URI=" + entry.Log.URL,
+		"ENTRY_INDEX=" + fmt.Sprint(entry.Index),
+		"LEAF_HASH=" + entry.LeafHash.Base64String(),
+		"PARSE_ERROR=" + parseError.Error(),
+		"ENTRY_FILENAME=" + entryPath,
+		"TEXT_FILENAME=" + textPath,
+		"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
+	}
+
+	if err := s.notify(ctx, &notification{
+		environ: environ,
+		summary: summary,
+		text:    text.String(),
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) healthCheckDir(ctlog *loglist.Log) string {
+	if ctlog == nil {
+		return filepath.Join(s.StateDir, "healthchecks")
+	} else {
+		return filepath.Join(s.logStateDir(ctlog.LogID), "healthchecks")
+	}
+}
+
+func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info HealthCheckFailure) error {
+	textPath := filepath.Join(s.healthCheckDir(ctlog), healthCheckFilename())
+	environ := []string{
+		"EVENT=error",
+		"SUMMARY=" + info.Summary(),
+		"TEXT_FILENAME=" + textPath,
+	}
+	text := info.Text()
+	if err := writeTextFile(textPath, text, 0666); err != nil {
+		return fmt.Errorf("error saving text file: %w", err)
+	}
+	if err := s.notify(ctx, &notification{
+		environ: environ,
+		summary: info.Summary(),
+		text:    text,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, err error) error {
+	if ctlog == nil {
+		log.Print(err)
+	} else {
+		log.Print(ctlog.URL, ":", err)
+	}
+	return nil
+}

monitor/healthcheck.go

@@ -11,10 +11,7 @@ package monitor
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"io/fs"
-	"path/filepath"
 	"strings"
 	"time"
 
@@ -27,52 +24,38 @@ func healthCheckFilename() string {
 }
 
 func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error {
-	var (
+	state, err := config.State.LoadLogState(ctx, ctlog.LogID)
-		stateDirPath  = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString())
+	if err != nil {
-		stateFilePath = filepath.Join(stateDirPath, "state.json")
+		return fmt.Errorf("error loading log state: %w", err)
-		sthsDirPath   = filepath.Join(stateDirPath, "unverified_sths")
+	} else if state == nil {
-		textPath      = filepath.Join(stateDirPath, "healthchecks", healthCheckFilename())
-	)
-	state, err := loadStateFile(stateFilePath)
-	if errors.Is(err, fs.ErrNotExist) {
 		return nil
-	} else if err != nil {
-		return fmt.Errorf("error loading state file: %w", err)
 	}
 
 	if time.Since(state.LastSuccess) < config.HealthCheckInterval {
 		return nil
 	}
 
-	sths, err := loadSTHsFromDir(sthsDirPath)
+	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
 	if err != nil {
-		return fmt.Errorf("error loading STHs directory: %w", err)
+		return fmt.Errorf("error loading STHs: %w", err)
 	}
 
 	if len(sths) == 0 {
-		event := &staleSTHEvent{
+		info := &StaleSTHInfo{
 			Log:         ctlog,
 			LastSuccess: state.LastSuccess,
 			LatestSTH:   state.VerifiedSTH,
-			TextPath:    textPath,
 		}
-		if err := event.save(); err != nil {
+		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
-			return fmt.Errorf("error saving stale STH event: %w", err)
-		}
-		if err := notify(ctx, config, event); err != nil {
 			return fmt.Errorf("error notifying about stale STH: %w", err)
 		}
 	} else {
-		event := &backlogEvent{
+		info := &BacklogInfo{
 			Log:       ctlog,
 			LatestSTH: sths[len(sths)-1],
 			Position:  state.DownloadPosition.Size(),
-			TextPath:  textPath,
 		}
-		if err := event.save(); err != nil {
+		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
-			return fmt.Errorf("error saving backlog event: %w", err)
-		}
-		if err := notify(ctx, config, event); err != nil {
 			return fmt.Errorf("error notifying about backlog: %w", err)
 		}
 	}
@@ -80,63 +63,45 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err
 	return nil
 }
 
-type staleSTHEvent struct {
+type HealthCheckFailure interface {
+	Summary() string
+	Text() string
+}
+
+type StaleSTHInfo struct {
 	Log         *loglist.Log
 	LastSuccess time.Time
 	LatestSTH   *ct.SignedTreeHead // may be nil
-	TextPath    string
 }
-type backlogEvent struct {
+
+type BacklogInfo struct {
 	Log       *loglist.Log
 	LatestSTH *ct.SignedTreeHead
 	Position  uint64
-	TextPath  string
 }
-type staleLogListEvent struct {
+
+type StaleLogListInfo struct {
 	Source        string
 	LastSuccess   time.Time
 	LastError     string
 	LastErrorTime time.Time
-	TextPath      string
 }
 
-func (e *backlogEvent) Backlog() uint64 {
+func (e *BacklogInfo) Backlog() uint64 {
 	return e.LatestSTH.TreeSize - e.Position
 }
 
-func (e *staleSTHEvent) Environ() []string {
+func (e *StaleSTHInfo) Summary() string {
-	return []string{
-		"EVENT=error",
-		"SUMMARY=" + e.Summary(),
-		"TEXT_FILENAME=" + e.TextPath,
-	}
-}
-func (e *backlogEvent) Environ() []string {
-	return []string{
-		"EVENT=error",
-		"SUMMARY=" + e.Summary(),
-		"TEXT_FILENAME=" + e.TextPath,
-	}
-}
-func (e *staleLogListEvent) Environ() []string {
-	return []string{
-		"EVENT=error",
-		"SUMMARY=" + e.Summary(),
-		"TEXT_FILENAME=" + e.TextPath,
-	}
-}
-
-func (e *staleSTHEvent) Summary() string {
 	return fmt.Sprintf("Unable to contact %s since %s", e.Log.URL, e.LastSuccess)
 }
-func (e *backlogEvent) Summary() string {
+func (e *BacklogInfo) Summary() string {
 	return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.URL)
 }
-func (e *staleLogListEvent) Summary() string {
+func (e *StaleLogListInfo) Summary() string {
 	return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess)
 }
 
-func (e *staleSTHEvent) Text() string {
+func (e *StaleSTHInfo) Text() string {
 	text := new(strings.Builder)
 	fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess)
 	fmt.Fprintf(text, "\n")
@@ -149,7 +114,7 @@ func (e *staleSTHEvent) Text() string {
 	}
 	return text.String()
 }
-func (e *backlogEvent) Text() string {
+func (e *BacklogInfo) Text() string {
 	text := new(strings.Builder)
 	fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL)
 	fmt.Fprintf(text, "\n")
@@ -160,7 +125,7 @@ func (e *backlogEvent) Text() string {
 	fmt.Fprintf(text, "         Backlog = %d\n", e.Backlog())
 	return text.String()
 }
-func (e *staleLogListEvent) Text() string {
+func (e *StaleLogListInfo) Text() string {
 	text := new(strings.Builder)
 	fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess)
 	fmt.Fprintf(text, "\n")
@@ -170,14 +135,4 @@ func (e *staleLogListEvent) Text() string {
 	return text.String()
 }
 
-func (e *staleSTHEvent) save() error {
-	return writeTextFile(e.TextPath, e.Text(), 0666)
-}
-func (e *backlogEvent) save() error {
-	return writeTextFile(e.TextPath, e.Text(), 0666)
-}
-func (e *staleLogListEvent) save() error {
-	return writeTextFile(e.TextPath, e.Text(), 0666)
-}
-
 // TODO-3: make the errors more actionable

monitor/mailutils.go

@@ -0,0 +1,34 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"os"
+)
+
+const mailDateFormat = "Mon, 2 Jan 2006 15:04:05 -0700"
+
+func generateMessageID() string {
+	var randomBytes [16]byte
+	if _, err := rand.Read(randomBytes[:]); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(randomBytes[:]) + "@selfhosted.certspotter.org"
+}
+
+func sendmailPath() string {
+	if envVar := os.Getenv("SENDMAIL_PATH"); envVar != "" {
+		return envVar
+	} else {
+		return "/usr/sbin/sendmail"
+	}
+}

monitor/malformed.go

@@ -1,72 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"fmt"
-	"strings"
-)
-
-type malformedLogEntry struct {
-	Entry     *logEntry
-	Error     string
-	EntryPath string
-	TextPath  string
-}
-
-func (malformed *malformedLogEntry) entryJSON() any {
-	return struct {
-		LeafInput []byte `json:"leaf_input"`
-		ExtraData []byte `json:"extra_data"`
-	}{
-		LeafInput: malformed.Entry.LeafInput,
-		ExtraData: malformed.Entry.ExtraData,
-	}
-}
-
-func (malformed *malformedLogEntry) save() error {
-	if err := writeJSONFile(malformed.EntryPath, malformed.entryJSON(), 0666); err != nil {
-		return err
-	}
-	if err := writeTextFile(malformed.TextPath, malformed.Text(), 0666); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (malformed *malformedLogEntry) Environ() []string {
-	return []string{
-		"EVENT=malformed_cert",
-		"SUMMARY=" + malformed.Summary(),
-		"LOG_URI=" + malformed.Entry.Log.URL,
-		"ENTRY_INDEX=" + fmt.Sprint(malformed.Entry.Index),
-		"LEAF_HASH=" + malformed.Entry.LeafHash.Base64String(),
-		"PARSE_ERROR=" + malformed.Error,
-		"ENTRY_FILENAME=" + malformed.EntryPath,
-		"TEXT_FILENAME=" + malformed.TextPath,
-		"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
-	}
-}
-
-func (malformed *malformedLogEntry) Text() string {
-	text := new(strings.Builder)
-	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
-
-	fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
-	writeField("Log Entry", fmt.Sprintf("%d @ %s", malformed.Entry.Index, malformed.Entry.Log.URL))
-	writeField("Leaf Hash", malformed.Entry.LeafHash.Base64String())
-	writeField("Error", malformed.Error)
-
-	return text.String()
-}
-
-func (malformed *malformedLogEntry) Summary() string {
-	return fmt.Sprintf("Unable to Parse Entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL)
-}

monitor/monitor.go

@@ -14,10 +14,7 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	"io/fs"
 	"log"
-	"os"
-	"path/filepath"
 	"strings"
 	"time"
 
@@ -73,17 +70,8 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	var (
+	if err := config.State.PrepareLog(ctx, ctlog.LogID); err != nil {
-		stateDirPath        = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString())
+		return fmt.Errorf("error preparing state: %w", err)
-		stateFilePath       = filepath.Join(stateDirPath, "state.json")
-		sthsDirPath         = filepath.Join(stateDirPath, "unverified_sths")
-		malformedDirPath    = filepath.Join(stateDirPath, "malformed_entries")
-		healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
-	)
-	for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} {
-		if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
-			return fmt.Errorf("error creating state directory: %w", err)
-		}
 	}
 
 	startTime := time.Now()
@@ -91,32 +79,35 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 	if isFatalLogError(err) {
 		return err
 	} else if err != nil {
-		recordError(fmt.Errorf("error fetching latest STH for %s: %w", ctlog.URL, err))
+		recordError(ctx, config, ctlog, fmt.Errorf("error fetching latest STH: %w", err))
 		return nil
 	}
 	latestSTH.LogID = ctlog.LogID
-	if err := storeSTHInDir(sthsDirPath, latestSTH); err != nil {
+	if err := config.State.StoreSTH(ctx, ctlog.LogID, latestSTH); err != nil {
 		return fmt.Errorf("error storing latest STH: %w", err)
 	}
 
-	state, err := loadStateFile(stateFilePath)
+	state, err := config.State.LoadLogState(ctx, ctlog.LogID)
-	if errors.Is(err, fs.ErrNotExist) {
+	if err != nil {
+		return fmt.Errorf("error loading log state: %w", err)
+	}
+	if state == nil {
 		if config.StartAtEnd {
 			tree, err := reconstructTree(ctx, logClient, latestSTH)
 			if isFatalLogError(err) {
 				return err
 			} else if err != nil {
-				recordError(fmt.Errorf("error reconstructing tree of size %d for %s: %w", latestSTH.TreeSize, ctlog.URL, err))
+				recordError(ctx, config, ctlog, fmt.Errorf("error reconstructing tree of size %d: %w", latestSTH.TreeSize, err))
 				return nil
 			}
-			state = &stateFile{
+			state = &LogState{
 				DownloadPosition: tree,
 				VerifiedPosition: tree,
 				VerifiedSTH:      latestSTH,
 				LastSuccess:      startTime.UTC(),
 			}
 		} else {
-			state = &stateFile{
+			state = &LogState{
 				DownloadPosition: merkletree.EmptyCollapsedTree(),
 				VerifiedPosition: merkletree.EmptyCollapsedTree(),
 				VerifiedSTH:      nil,
@@ -126,21 +117,19 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 		if config.Verbose {
 			log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size())
 		}
-		if err := state.store(stateFilePath); err != nil {
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
-			return fmt.Errorf("error storing state file: %w", err)
+			return fmt.Errorf("error storing log state: %w", err)
 		}
-	} else if err != nil {
-		return fmt.Errorf("error loading state file: %w", err)
 	}
 
-	sths, err := loadSTHsFromDir(sthsDirPath)
+	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
 	if err != nil {
-		return fmt.Errorf("error loading STHs directory: %w", err)
+		return fmt.Errorf("error loading STHs: %w", err)
 	}
 
 	for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() {
 		// TODO-4: audit sths[0] against state.VerifiedSTH
-		if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
+		if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
 			return fmt.Errorf("error removing STH: %w", err)
 		}
 		sths = sths[1:]
@@ -150,8 +139,8 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 		if config.Verbose {
 			log.Printf("saving state in defer for %s", ctlog.URL)
 		}
-		if err := state.store(stateFilePath); err != nil && returnedErr == nil {
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil && returnedErr == nil {
-			returnedErr = fmt.Errorf("error storing state file: %w", err)
+			returnedErr = fmt.Errorf("error storing log state: %w", err)
 		}
 	}()
 
@@ -174,7 +163,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 		downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd)
 	}()
 	for rawEntry := range entries {
-		entry := &logEntry{
+		entry := &LogEntry{
 			Log:       ctlog,
 			Index:     state.DownloadPosition.Size(),
 			LeafInput: rawEntry.LeafInput,
@@ -191,11 +180,11 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 
 		for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize {
 			if merkletree.Hash(sths[0].SHA256RootHash) != rootHash {
-				recordError(fmt.Errorf("error verifying %s at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", ctlog.URL, sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
+				recordError(ctx, config, ctlog, fmt.Errorf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
 
 				state.DownloadPosition = state.VerifiedPosition
-				if err := state.store(stateFilePath); err != nil {
+				if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
-					return fmt.Errorf("error storing state file: %w", err)
+					return fmt.Errorf("error storing log state: %w", err)
 				}
 				return nil
 			}
@@ -203,7 +192,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 			state.VerifiedPosition = state.DownloadPosition
 			state.VerifiedSTH = sths[0]
 			shouldSaveState = true
-			if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
+			if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
 				return fmt.Errorf("error removing verified STH: %w", err)
 			}
 
@@ -211,7 +200,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 		}
 
 		if shouldSaveState {
-			if err := state.store(stateFilePath); err != nil {
+			if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
 				return fmt.Errorf("error storing state file: %w", err)
 			}
 		}
@@ -220,7 +209,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 	if isFatalLogError(downloadErr) {
 		return downloadErr
 	} else if downloadErr != nil {
-		recordError(fmt.Errorf("error downloading entries from %s: %w", ctlog.URL, downloadErr))
+		recordError(ctx, config, ctlog, fmt.Errorf("error downloading entries: %w", downloadErr))
 		return nil
 	}
 
@@ -234,7 +223,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 
 func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error {
 	for begin < end && ctx.Err() == nil {
-		size := begin - end
+		size := end - begin
 		if size > maxGetEntriesSize {
 			size = maxGetEntriesSize
 		}

monitor/notify.go

@@ -20,35 +20,36 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 )
 
 var stdoutMu sync.Mutex
 
-type notification interface {
+type notification struct {
-	Environ() []string
+	environ []string
-	Summary() string
+	summary string
-	Text() string
+	text    string
 }
 
-func notify(ctx context.Context, config *Config, notif notification) error {
+func (s *FilesystemState) notify(ctx context.Context, notif *notification) error {
-	if config.Stdout {
+	if s.Stdout {
 		writeToStdout(notif)
 	}
 
-	if len(config.Email) > 0 {
+	if len(s.Email) > 0 {
-		if err := sendEmail(ctx, config.SendmailPath, config.Email, notif); err != nil {
+		if err := sendEmail(ctx, s.Email, notif); err != nil {
 			return err
 		}
 	}
 
-	if config.Script != "" {
+	if s.Script != "" {
-		if err := execScript(ctx, config.Script, notif); err != nil {
+		if err := execScript(ctx, s.Script, notif); err != nil {
 			return err
 		}
 	}
 
-	if config.ScriptDir != "" {
+	if s.ScriptDir != "" {
-		if err := execScriptDir(ctx, config.ScriptDir, notif); err != nil {
+		if err := execScriptDir(ctx, s.ScriptDir, notif); err != nil {
 			return err
 		}
 	}
@@ -56,28 +57,39 @@ func notify(ctx context.Context, config *Config, notif notification) error {
 	return nil
 }
 
-func writeToStdout(notif notification) {
+func writeToStdout(notif *notification) {
 	stdoutMu.Lock()
 	defer stdoutMu.Unlock()
-	os.Stdout.WriteString(notif.Text() + "\n")
+	os.Stdout.WriteString(notif.text + "\n")
 }
 
-func sendEmail(ctx context.Context, sendmailPath string, to []string, notif notification) error {
+func sendEmail(ctx context.Context, to []string, notif *notification) error {
 	stdin := new(bytes.Buffer)
 	stderr := new(bytes.Buffer)
 
+	from := os.Getenv("EMAIL")
+
+	if from != "" {
+		fmt.Fprintf(stdin, "From: %s\n", from)
+	}
 	fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", "))
-	fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.Summary())
+	fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.summary)
+	fmt.Fprintf(stdin, "Date: %s\n", time.Now().Format(mailDateFormat))
+	fmt.Fprintf(stdin, "Message-ID: <%s>\n", generateMessageID())
 	fmt.Fprintf(stdin, "Mime-Version: 1.0\n")
 	fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n")
 	fmt.Fprintf(stdin, "X-Mailer: certspotter\n")
 	fmt.Fprintf(stdin, "\n")
-	fmt.Fprint(stdin, notif.Text())
+	fmt.Fprint(stdin, notif.text)
 
-	args := []string{"-i", "--"}
+	args := []string{"-i"}
+	if from != "" {
+		args = append(args, "-f", from)
+	}
+	args = append(args, "--")
 	args = append(args, to...)
 
-	sendmail := exec.CommandContext(ctx, sendmailPath, args...)
+	sendmail := exec.CommandContext(ctx, sendmailPath(), args...)
 	sendmail.Stdin = stdin
 	sendmail.Stderr = stderr
 
@@ -92,12 +104,12 @@ func sendEmail(ctx context.Context, sendmailPath string, to []string, notif noti
 	}
 }
 
-func execScript(ctx context.Context, scriptName string, notif notification) error {
+func execScript(ctx context.Context, scriptName string, notif *notification) error {
 	stderr := new(bytes.Buffer)
 
 	cmd := exec.CommandContext(ctx, scriptName)
 	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, notif.Environ()...)
+	cmd.Env = append(cmd.Env, notif.environ...)
 	cmd.Stderr = stderr
 
 	if err := cmd.Run(); err == nil {
@@ -113,7 +125,7 @@ func execScript(ctx context.Context, scriptName string, notif notification) erro
 	}
 }
 
-func execScriptDir(ctx context.Context, dirPath string, notif notification) error {
+func execScriptDir(ctx context.Context, dirPath string, notif *notification) error {
 	dirents, err := os.ReadDir(dirPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil

monitor/process.go

@@ -13,19 +13,14 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
-	"encoding/hex"
-	"errors"
 	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
 	"software.sslmate.com/src/certspotter"
 	"software.sslmate.com/src/certspotter/ct"
 	"software.sslmate.com/src/certspotter/loglist"
 	"software.sslmate.com/src/certspotter/merkletree"
 )
 
-type logEntry struct {
+type LogEntry struct {
 	Log       *loglist.Log
 	Index     uint64
 	LeafInput []byte
@@ -33,7 +28,7 @@ type logEntry struct {
 	LeafHash  merkletree.Hash
 }
 
-func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error {
+func processLogEntry(ctx context.Context, config *Config, entry *LogEntry) error {
 	leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput))
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err))
@@ -48,7 +43,7 @@ func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error
 	}
 }
 
-func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, cert ct.ASN1Cert) error {
+func processX509LogEntry(ctx context.Context, config *Config, entry *LogEntry, cert ct.ASN1Cert) error {
 	certInfo, err := certspotter.MakeCertInfoFromRawCert(cert)
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err))
@@ -69,7 +64,7 @@ func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, c
 	return processCertificate(ctx, config, entry, certInfo, chain)
 }
 
-func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry, precert ct.PreCert) error {
+func processPrecertLogEntry(ctx context.Context, config *Config, entry *LogEntry, precert ct.PreCert) error {
 	certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate)
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err))
@@ -87,7 +82,7 @@ func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry
 	return processCertificate(ctx, config, entry, certInfo, chain)
 }
 
-func processCertificate(ctx context.Context, config *Config, entry *logEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
+func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
 	identifiers, err := certInfo.ParseIdentifiers()
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, err)
@@ -97,7 +92,7 @@ func processCertificate(ctx context.Context, config *Config, entry *logEntry, ce
 		return nil
 	}
 
-	cert := &discoveredCert{
+	cert := &DiscoveredCert{
 		WatchItem:    watchItem,
 		LogEntry:     entry,
 		Info:         certInfo,
@@ -108,68 +103,15 @@ func processCertificate(ctx context.Context, config *Config, entry *logEntry, ce
 		Identifiers:  identifiers,
 	}
 
-	var notifiedPath string
+	if err := config.State.NotifyCert(ctx, cert); err != nil {
-	if config.SaveCerts {
+		return fmt.Errorf("error notifying about certificate %x: %w", cert.SHA256, err)
-		hexFingerprint := hex.EncodeToString(cert.SHA256[:])
-		prefixPath := filepath.Join(config.StateDir, "certs", hexFingerprint[0:2])
-		var (
-			notifiedFilename      = "." + hexFingerprint + ".notified"
-			certFilename          = hexFingerprint + ".pem"
-			jsonFilename          = hexFingerprint + ".v1.json"
-			textFilename          = hexFingerprint + ".txt"
-			legacyCertFilename    = hexFingerprint + ".cert.pem"
-			legacyPrecertFilename = hexFingerprint + ".precert.pem"
-		)
-
-		for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} {
-			if fileExists(filepath.Join(prefixPath, filename)) {
-				return nil
-			}
-		}
-
-		if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
-			return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
-		}
-
-		notifiedPath = filepath.Join(prefixPath, notifiedFilename)
-		cert.CertPath = filepath.Join(prefixPath, certFilename)
-		cert.JSONPath = filepath.Join(prefixPath, jsonFilename)
-		cert.TextPath = filepath.Join(prefixPath, textFilename)
-
-		if err := cert.save(); err != nil {
-			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
-		}
-	} else {
-		// TODO-4: save cert to temporary files, and defer their unlinking
-	}
-
-	if err := notify(ctx, config, cert); err != nil {
-		return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
-	}
-
-	if notifiedPath != "" {
-		if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
-			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
-		}
 	}
 
 	return nil
 }
 
-func processMalformedLogEntry(ctx context.Context, config *Config, entry *logEntry, parseError error) error {
+func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error {
-	dirPath := filepath.Join(config.StateDir, "logs", entry.Log.LogID.Base64URLString(), "malformed_entries")
+	if err := config.State.NotifyMalformedEntry(ctx, entry, parseError); err != nil {
-	malformed := &malformedLogEntry{
-		Entry:     entry,
-		Error:     parseError.Error(),
-		EntryPath: filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index)),
-		TextPath:  filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index)),
-	}
-
-	if err := malformed.save(); err != nil {
-		return fmt.Errorf("error saving malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
-	}
-
-	if err := notify(ctx, config, malformed); err != nil {
 		return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
 	}
 	return nil

monitor/state.go

@@ -0,0 +1,68 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/merkletree"
+	"time"
+)
+
+type LogState struct {
+	DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
+	VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
+	VerifiedSTH      *ct.SignedTreeHead        `json:"verified_sth"`
+	LastSuccess      time.Time                 `json:"last_success"`
+}
+
+type StateProvider interface {
+	// Initialize the state.  Called before any other method in this interface.
+	// Idempotent: returns nil if the state is already initialized.
+	Prepare(context.Context) error
+
+	// Initialize the state for the given log.  Called before any other method
+	// with the log ID.  Idempotent: returns nil if log state already initialized.
+	PrepareLog(context.Context, LogID) error
+
+	// Store log state for retrieval by LoadLogState.
+	StoreLogState(context.Context, LogID, *LogState) error
+
+	// Load log state that was previously stored with StoreLogState.
+	// Returns nil, nil if StoreLogState has not been called yet for this log.
+	LoadLogState(context.Context, LogID) (*LogState, error)
+
+	// Store STH for retrieval by LoadSTHs.  If an STH with the same
+	// timestamp and root hash is already stored, this STH can be ignored.
+	StoreSTH(context.Context, LogID, *ct.SignedTreeHead) error
+
+	// Load all STHs for this log previously stored with StoreSTH.
+	// The returned slice must be sorted by tree size.
+	LoadSTHs(context.Context, LogID) ([]*ct.SignedTreeHead, error)
+
+	// Remove an STH so it is no longer returned by LoadSTHs.
+	RemoveSTH(context.Context, LogID, *ct.SignedTreeHead) error
+
+	// Called when a certificate matching the watch list is discovered.
+	NotifyCert(context.Context, *DiscoveredCert) error
+
+	// Called when certspotter fails to parse a log entry.
+	NotifyMalformedEntry(context.Context, *LogEntry, error) error
+
+	// Called when a health check fails.  The log is nil if the
+	// feailure is not associated with a log.
+	NotifyHealthCheckFailure(context.Context, *loglist.Log, HealthCheckFailure) error
+
+	// Called when a non-fatal error occurs.  The log is nil if the error is
+	// not associated with a log.  Note that most errors are transient, and
+	// certspotter will retry the failed operation later.
+	NotifyError(context.Context, *loglist.Log, error) error
+}

monitor/statedir.go

@@ -76,13 +76,13 @@ func migrateLogStateDirV1(dir string) error {
 		return fmt.Errorf("error unmarshaling %s: %w", treePath, err)
 	}
 
-	stateFile := stateFile{
+	stateFile := LogState{
 		DownloadPosition: &tree,
 		VerifiedPosition: &tree,
 		VerifiedSTH:      &sth,
 		LastSuccess:      time.Now().UTC(),
 	}
-	if stateFile.store(filepath.Join(dir, "state.json")); err != nil {
+	if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil {
 		return err
 	}
 

monitor/statefile.go

@@ -1,42 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/merkletree"
-	"time"
-)
-
-type stateFile struct {
-	DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
-	VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
-	VerifiedSTH      *ct.SignedTreeHead        `json:"verified_sth"`
-	LastSuccess      time.Time                 `json:"last_success"`
-}
-
-func loadStateFile(filePath string) (*stateFile, error) {
-	fileBytes, err := os.ReadFile(filePath)
-	if err != nil {
-		return nil, err
-	}
-	file := new(stateFile)
-	if err := json.Unmarshal(fileBytes, file); err != nil {
-		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
-	}
-	return file, nil
-}
-
-func (file *stateFile) store(filePath string) error {
-	return writeJSONFile(filePath, file, 0666)
-}

monitor/sthdir.go

@@ -17,10 +17,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"slices"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"slices"
 	"software.sslmate.com/src/certspotter/ct"
 	"strconv"
 	"strings"