Remove unnecessary Printf

Envelope sender and RFC5322.From address are set to $EMAIL if it's non-empty.

Requested in #87

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Change Log
 
+## v0.18.0 (2023-11-13)
+- Fix bug with downloading entries that did not materialize in practice
+  with any of the current logs.
+- Include `Message-ID` and `Date` in outbound emails.
+
+## v0.17.0 (2023-10-26)
+- Allow sendmail path to be configured with `$SENDMAIL_PATH`.
+- Minor improvements to documentation, efficiency.
+
+## v0.16.0 (2023-02-21)
+- Write malformed certs and failed healthchecks to filesystem so scripts
+  can access them.
+- Automatically execute scripts under `$CERTSPOTTER_CONFIG_DIR/hooks.d`
+  if it exists.
+- Automatically email addresses listed in `$CERTSPOTTER_CONFIG_DIR/email_recipients`
+  if it exists.
+
 ## v0.15.1 (2023-02-09)
 - Fix some typos in help and error messages.
 - Allow version to be set via linker flag, to facilitate distro package building.

README.md

@@ -32,34 +32,28 @@ Cert Spotter requires Go version 1.19 or higher.
    go install software.sslmate.com/src/certspotter/cmd/certspotter@latest
    ```
 
-2. Create a watch list file containing the DNS names you want to monitor,
+2. Create a watch list file `$HOME/.certspotter/watchlist` containing the DNS names you want to monitor,
    one per line.  To monitor an entire domain tree (including the domain itself
    and all sub-domains) prefix the domain name with a dot (e.g. `.example.com`).
    To monitor a single DNS name only, do not prefix the name with a dot.
 
-3. Configure your system to run `certspotter` as a daemon.  You should specify
-   the following command line options:
+3. Place one or more email addresses in the `$HOME/.certspotter/email_recipients`
+   file (one per line), and/or place one or more executable scripts in the
+   `$HOME/.certspotter/hooks.d` directory.  certspotter will email the listed
+   addresses (requires your system to have a working sendmail command) and
+   execute the provided scripts when it detects a certificate for a domain on
+   your watch list.
 
-   * `-watchlist PATH` to specify the path to your watch list file.
-
-   * `-email ADDRESS` to specify an email address which certspotter will contact
-     when it detects a domain on your watch list.  (Your system must have a
-     working sendmail command.)
-
-   * (Optional) `-start_at_end` to tell certspotter to start monitoring logs at the end
-     instead of the beginning.  This saves significant bandwidth, but you won't be
-     notified about certificates which were logged before you started using certspotter.
-
-   For example:
-
-   ```
-   certspotter -watchlist /etc/certspotter.watchlist -email pki@certspotteruser.example -start_at_end
-   ```
+4. Configure your system to run `certspotter` as a daemon.  You may want to specify
+   the `-start_at_end` command line option to tell certspotter to start monitoring
+   new logs at the end instead of the beginning.  This saves significant bandwidth, but
+   you won't be notified about certificates which were logged before you started
+   using certspotter.
 
 ## Documentation
 
 * Command line options and operational details: [certspotter(8) man page](man/certspotter.md)
-* The `-script` interface: [certspotter-script(8) man page](man/certspotter-script.md)
+* The script interface: [certspotter-script(8) man page](man/certspotter-script.md)
 * [Change Log](CHANGELOG.md)
 
 ## What certificates are detected by Cert Spotter?
@@ -87,8 +81,8 @@ For instance, if a DNS identifier contains a null byte, Cert Spotter
 interprets it as two identifiers: the complete identifier, and the
 identifier formed by truncating at the first null byte.  For example, a
 certificate for `example.org\0.example.com` will alert the owners of both
-`example.org` and `example.com`.  This defends against [null prefix attacks]
-(http://www.thoughtcrime.org/papers/null-prefix-attacks.pdf).
+`example.org` and `example.com`.  This defends against [null prefix attacks](
+http://www.thoughtcrime.org/papers/null-prefix-attacks.pdf).
 
 SSLMate continuously monitors CT logs to make sure every certificate's
 identifiers can be successfully parsed, and will release updates to

cmd/certspotter/main.go

@@ -10,12 +10,12 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"errors"
 	"flag"
 	"fmt"
 	"io/fs"
-	insecurerand "math/rand"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -64,6 +64,10 @@ func certspotterVersion() string {
 	return "unknown"
 }
 
+func fileExists(filename string) bool {
+	_, err := os.Lstat(filename)
+	return err == nil
+}
 func homedir() string {
 	homedir, err := os.UserHomeDir()
 	if err != nil {
@@ -85,20 +89,60 @@ func defaultConfigDir() string {
 		return filepath.Join(homedir(), ".certspotter")
 	}
 }
+func defaultWatchListPath() string {
+	return filepath.Join(defaultConfigDir(), "watchlist")
+}
+func defaultWatchListPathIfExists() string {
+	if fileExists(defaultWatchListPath()) {
+		return defaultWatchListPath()
+	} else {
+		return ""
+	}
+}
+func defaultScriptDir() string {
+	return filepath.Join(defaultConfigDir(), "hooks.d")
+}
+func defaultEmailFile() string {
+	return filepath.Join(defaultConfigDir(), "email_recipients")
+}
+
+func simplifyError(err error) error {
+	var pathErr *fs.PathError
+	if errors.As(err, &pathErr) {
+		return pathErr.Err
+	}
+
+	return err
+}
 
 func readWatchListFile(filename string) (monitor.WatchList, error) {
 	file, err := os.Open(filename)
 	if err != nil {
-		var pathErr *fs.PathError
-		if errors.As(err, &pathErr) {
-			err = pathErr.Err
-		}
-		return nil, err
+		return nil, simplifyError(err)
 	}
 	defer file.Close()
 	return monitor.ReadWatchList(file)
 }
 
+func readEmailFile(filename string) ([]string, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, simplifyError(err)
+	}
+	defer file.Close()
+
+	var emails []string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" {
+			continue
+		}
+		emails = append(emails, line)
+	}
+	return emails, err
+}
+
 func appendFunc(slice *[]string) func(string) error {
 	return func(value string) error {
 		*slice = append(*slice, value)
@@ -107,8 +151,6 @@ func appendFunc(slice *[]string) func(string) error {
 }
 
 func main() {
-	insecurerand.Seed(time.Now().UnixNano()) // TODO: remove after upgrading to Go 1.20
-
 	loglist.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s)", certspotterVersion(), runtime.Version(), runtime.GOOS, runtime.GOARCH)
 
 	var flags struct {
@@ -131,36 +173,59 @@ func main() {
 	flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
 	flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory")
 	flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered")
-	flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring logs from the end rather than the beginning (saves considerable bandwidth)")
+	flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring new logs from the end rather than the beginning (saves considerable bandwidth)")
 	flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates")
 	flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout")
 	flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose")
 	flag.BoolVar(&flags.version, "version", false, "Print version and exit")
-	flag.StringVar(&flags.watchlist, "watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing domain names to watch")
+	flag.StringVar(&flags.watchlist, "watchlist", defaultWatchListPathIfExists(), "File containing domain names to watch")
 	flag.Parse()
 
 	if flags.version {
 		fmt.Fprintf(os.Stdout, "certspotter version %s\n", certspotterVersion())
 		os.Exit(0)
 	}
-
-	if len(flags.email) == 0 && len(flags.script) == 0 && flags.stdout == false {
-		fmt.Fprintf(os.Stderr, "%s: at least one of -email, -script, or -stdout must be specified (see -help for details)\n", programName)
+	if flags.watchlist == "" {
+		fmt.Fprintf(os.Stderr, "%s: watch list not found: please create %s or specify alternative path using -watchlist\n", programName, defaultWatchListPath())
 		os.Exit(2)
 	}
 
+	fsstate := &monitor.FilesystemState{
+		StateDir:  flags.stateDir,
+		SaveCerts: !flags.noSave,
+		Script:    flags.script,
+		ScriptDir: defaultScriptDir(),
+		Email:     flags.email,
+		Stdout:    flags.stdout,
+	}
 	config := &monitor.Config{
 		LogListSource:       flags.logs,
-		StateDir:            flags.stateDir,
-		SaveCerts:           !flags.noSave,
+		State:               fsstate,
 		StartAtEnd:          flags.startAtEnd,
 		Verbose:             flags.verbose,
-		Script:              flags.script,
-		Email:               flags.email,
-		Stdout:              flags.stdout,
 		HealthCheckInterval: flags.healthcheck,
 	}
 
+	emailFileExists := false
+	if emailRecipients, err := readEmailFile(defaultEmailFile()); err == nil {
+		emailFileExists = true
+		fsstate.Email = append(fsstate.Email, emailRecipients...)
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		fmt.Fprintf(os.Stderr, "%s: error reading email recipients file %q: %s\n", programName, defaultEmailFile(), err)
+		os.Exit(1)
+	}
+
+	if len(fsstate.Email) == 0 && !emailFileExists && fsstate.Script == "" && !fileExists(fsstate.ScriptDir) && fsstate.Stdout == false {
+		fmt.Fprintf(os.Stderr, "%s: no notification methods were specified\n", programName)
+		fmt.Fprintf(os.Stderr, "Please specify at least one of the following notification methods:\n")
+		fmt.Fprintf(os.Stderr, " - Place one or more email addresses in %s (one address per line)\n", defaultEmailFile())
+		fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", fsstate.ScriptDir)
+		fmt.Fprintf(os.Stderr, " - Specify an email address using the -email flag\n")
+		fmt.Fprintf(os.Stderr, " - Specify the path to an executable script using the -script flag\n")
+		fmt.Fprintf(os.Stderr, " - Specify the -stdout flag\n")
+		os.Exit(2)
+	}
+
 	if flags.watchlist == "-" {
 		watchlist, err := monitor.ReadWatchList(os.Stdin)
 		if err != nil {

cmd/submitct/main.go

@@ -22,7 +22,7 @@ import (
 	"encoding/pem"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"os"
 	"strings"
@@ -146,7 +146,7 @@ func main() {
 	flag.Parse()
 	log.SetPrefix("submitct: ")
 
-	certsPem, err := ioutil.ReadAll(os.Stdin)
+	certsPem, err := io.ReadAll(os.Stdin)
 	if err != nil {
 		log.Fatalf("Error reading stdin: %s", err)
 	}
@@ -158,18 +158,19 @@ func main() {
 
 	var logs []Log
 	for _, ctlog := range list.AllLogs() {
+		submissionURL := ctlog.GetSubmissionURL()
 		pubkey, err := x509.ParsePKIXPublicKey(ctlog.Key)
 		if err != nil {
-			log.Fatalf("%s: Failed to parse log public key: %s", ctlog.URL, err)
+			log.Fatalf("%s: Failed to parse log public key: %s", submissionURL, err)
 		}
 		verifier, err := ct.NewSignatureVerifier(pubkey)
 		if err != nil {
-			log.Fatalf("%s: Failed to create signature verifier for log: %s", ctlog.URL, err)
+			log.Fatalf("%s: Failed to create signature verifier for log: %s", submissionURL, err)
 		}
 		logs = append(logs, Log{
 			Log: ctlog,
 			SignatureVerifier: verifier,
-			LogClient: client.New(strings.TrimRight(ctlog.URL, "/")),
+			LogClient: client.New(strings.TrimRight(submissionURL, "/")),
 		})
 	}
 
@@ -212,11 +213,11 @@ func main() {
 			go func(fingerprint [32]byte, ctlog Log) {
 				sct, err := ctlog.SubmitChain(chain)
 				if err != nil {
-					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.URL, err)
+					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.GetSubmissionURL(), err)
 					atomic.AddUint32(&submitErrors, 1)
 				} else if *verbose {
 					timestamp := time.Unix(int64(sct.Timestamp)/1000, int64(sct.Timestamp%1000)*1000000)
-					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.URL, timestamp)
+					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.GetSubmissionURL(), timestamp)
 				}
 				wg.Done()
 			}(fingerprint, ctlog)

ct/README

@@ -1,2 +1,4 @@
-The code in this directory is from https://github.com/google/certificate-transparency/tree/master/go
+The code in this directory is based on Google's Certificiate Transparency Go library
+(originally at <https://github.com/google/certificate-transparency/tree/master/go>;
+now at <https://github.com/google/certificate-transparency-go>).
 See AUTHORS for the copyright holders, and LICENSE for the license.

go.mod

@@ -1,11 +1,10 @@
 module software.sslmate.com/src/certspotter
 
-go 1.19
+go 1.21
 
 require (
-	golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9
-	golang.org/x/net v0.5.0
-	golang.org/x/sync v0.1.0
+	golang.org/x/net v0.17.0
+	golang.org/x/sync v0.4.0
 )
 
-require golang.org/x/text v0.6.0 // indirect
+require golang.org/x/text v0.13.0 // indirect

go.sum

@@ -1,8 +1,6 @@
-golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9 h1:frX3nT9RkKybPnjyI+yvZh6ZucTZatCCEm9D47sZ2zo=
-golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
-golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
-golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
-golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

loglist/helpers.go

@@ -13,12 +13,16 @@ import (
 	"time"
 )
 
+// Return all tiled and non-tiled logs from all operators
 func (list *List) AllLogs() []*Log {
 	logs := []*Log{}
 	for operator := range list.Operators {
 		for log := range list.Operators[operator].Logs {
 			logs = append(logs, &list.Operators[operator].Logs[log])
 		}
+		for log := range list.Operators[operator].TiledLogs {
+			logs = append(logs, &list.Operators[operator].TiledLogs[log])
+		}
 	}
 	return logs
 }

loglist/schema.go

@@ -22,16 +22,19 @@ type List struct {
 }
 
 type Operator struct {
-	Name  string   `json:"name"`
-	Email []string `json:"email"`
-	Logs  []Log    `json:"logs"`
+	Name      string   `json:"name"`
+	Email     []string `json:"email"`
+	Logs      []Log    `json:"logs"`
+	TiledLogs []Log    `json:"tiled_logs"`
 }
 
 type Log struct {
 	Key              []byte        `json:"key"`
 	LogID            ct.SHA256Hash `json:"log_id"`
 	MMD              int           `json:"mmd"`
-	URL              string        `json:"url"`
+	URL              string        `json:"url,omitempty"`            // only for rfc6962 logs
+	SubmissionURL    string        `json:"submission_url,omitempty"` // only for static-ct-api logs
+	MonitoringURL    string        `json:"monitoring_url,omitempty"` // only for static-ct-api logs
 	Description      string        `json:"description"`
 	State            State         `json:"state"`
 	DNS              string        `json:"dns"`
@@ -44,6 +47,29 @@ type Log struct {
 	// TODO: add previous_operators
 }
 
+func (log *Log) IsRFC6962() bool     { return log.URL != "" }
+func (log *Log) IsStaticCTAPI() bool { return log.SubmissionURL != "" && log.MonitoringURL != "" }
+
+// Return URL prefix for submission using the RFC6962 protocol
+func (log *Log) GetSubmissionURL() string {
+	if log.SubmissionURL != "" {
+		return log.SubmissionURL
+	} else {
+		return log.URL
+	}
+}
+
+// Return URL prefix for monitoring.
+// Since the protocol understood by the URL might be either RFC6962 or static-ct-api, this URL is
+// only useful for informational purposes.
+func (log *Log) GetMonitoringURL() string {
+	if log.MonitoringURL != "" {
+		return log.MonitoringURL
+	} else {
+		return log.URL
+	}
+}
+
 type State struct {
 	Pending *struct {
 		Timestamp time.Time `json:"timestamp"`

loglist/validate.go

@@ -26,7 +26,12 @@ func (list *List) Validate() error {
 func (operator *Operator) Validate() error {
 	for i := range operator.Logs {
 		if err := operator.Logs[i].Validate(); err != nil {
-			return fmt.Errorf("problem with %dth log (%s): %w", i, operator.Logs[i].LogIDString(), err)
+			return fmt.Errorf("problem with %dth non-tiled log (%s): %w", i, operator.Logs[i].LogIDString(), err)
+		}
+	}
+	for i := range operator.TiledLogs {
+		if err := operator.TiledLogs[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth tiled log (%s): %w", i, operator.TiledLogs[i].LogIDString(), err)
 		}
 	}
 	return nil
@@ -37,5 +42,12 @@ func (log *Log) Validate() error {
 	if log.LogID != realLogID {
 		return fmt.Errorf("log ID does not match log key")
 	}
+
+	if !log.IsRFC6962() && !log.IsStaticCTAPI() {
+		return fmt.Errorf("URL(s) not provided")
+	} else if log.IsRFC6962() && log.IsStaticCTAPI() {
+		return fmt.Errorf("inconsistent URLs provided")
+	}
+
 	return nil
 }

man/certspotter-script.md

@@ -4,9 +4,13 @@
 
 # DESCRIPTION
 
-**certspotter-script** is *any* program that is called using **certspotter(8)**'s
-*-script* argument. **certspotter** executes this program when it needs to notify
-you about an event, such as detecting a certificate for a domain on your watch list.
+**certspotter-script** is *any* program that is executed by **certspotter(8)**
+when it needs to notify you about an event, such as detecting a certificate for
+a domain on your watch list.
+
+Scripts are placed in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` directory
+(`~/.certspotter/hooks.d` by default), or specified on the command line
+using the `-script` argument.
 
 # ENVIRONMENT
 
@@ -33,7 +37,8 @@ The following environment variables are set for all types of events:
 
 `SUMMARY`
 
-:   A short human-readable string describing the event.
+:   A short human-readable string describing the event.  This is the same string
+    used in the subject line of emails sent by certspotter.
 
 
 ## Discovered certificate information
@@ -73,12 +78,12 @@ The following environment variables are set for `discovered_cert` events:
 
 `JSON_FILENAME`
 
-:    Path to a JSON containing additional information about the certificate.  See below for the format of the JSON file.
+:    Path to a JSON file containing additional information about the certificate.  See below for the format of the JSON file.
      Not set if `-no_save` was used.
 
 `TEXT_FILENAME`
 
-:    Path to a file containing a text representation of the certificate.  This file contains the same text that
+:    Path to a text file containing information about the certificate.  This file contains the same text that
      certspotter uses in emails.  You should not attempt to parse this file because its format may change in the future.
      Not set if `-no_save` was used.
 
@@ -138,6 +143,22 @@ The following environment variables are set for `malformed_cert` events:
 
 :    A human-readable string describing why the certificate is malformed.
 
+`ENTRY_FILENAME`
+
+:    Path to a file containing the JSON log entry.  The file contains a JSON object with two fields, `leaf_input` and `extra_data`, as described in RFC 6962 Section 4.6.
+
+`TEXT_FILENAME`
+
+:    Path to a text file containing a description of the malformed certificate.  This file contains the same text that certspotter uses in emails.
+
+## Error information
+
+The following environment variables are set for `error` events:
+
+`TEXT_FILENAME`
+
+:    Path to a text file containing a description of the error.  This file contains the same text that certspotter uses in emails.
+
 # JSON FILE FORMAT
 
 Unless `-no_save` is used, certspotter saves a JSON file for every discovered certificate

man/certspotter.md

@@ -41,6 +41,12 @@ You can use Cert Spotter to detect:
     an error occurs.  You can specify this option more than once to email
     multiple addresses.  Your system must have a working sendmail(1) command.
 
+    Regardless of the `-email` option, certspotter also emails any address listed
+    in `$CERTSPOTTER_CONFIG_DIR/email_recipients` file
+    (`~/.certspotter/email_recipients` by default).  (One address per line,
+    blank lines are ignored.)  This file is read only at startup, so you
+    must restart certspotter if you change it.
+
 -healthcheck *INTERVAL*
 
 :   Perform a health check at the given interval (default: "24h") as described
@@ -57,13 +63,19 @@ You can use Cert Spotter to detect:
 
 -no\_save
 
-:   Do not save a copy of matching certificates.
+:   Do not save a copy of matching certificates. Note that enabling this option
+    will cause you to receive duplicate notifications, since certspotter will
+    have no way of knowing if you've been previously notified about a certificate.
 
 -script *COMMAND*
 
 :   Command to execute when a matching certificate is found or an error occurs. See
     certspotter-script(8) for information about the interface to scripts.
 
+    Regardless of the `-script` option, certspotter also executes any executable
+    file in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` directory
+    (`~/.certspotter/hooks.d` by default).
+
 -start\_at\_end
 
 :   Start monitoring logs from the end rather than the beginning.
@@ -103,14 +115,36 @@ You can use Cert Spotter to detect:
     certspotter reads the watch list only when starting up, so you must restart
     certspotter if you change it.
 
+# NOTIFICATIONS
+
+When certspotter detects a certificate matching your watchlist, or encounters
+an error that is preventing it from discovering certificates, it notifies you
+as follows:
+
+* Emails any address specified by the `-email` command line flag.
+
+* Emails any address listed in the `$CERTSPOTTER_CONFIG_DIR/email_recipients`
+  file (`~/.certspotter/email_recipients` by default).  (One address per line,
+  blank lines are ignored.)  This file is read only at startup, so you
+  must restart certspotter if you change it.
+
+* Executes the script specified by the `-script` command line flag.
+
+* Executes every executable file in the `$CERTSPOTTER_CONFIG_DIR/hooks.d`
+ directory (`~/.certspotter/hooks.d` by default).
+
+* Writes the notification to standard out if the `-stdout` flag was specified.
+
+Sending email requires a working sendmail(1) command.  For details about
+the script interface, see certspotter-script(8).
+
 # OPERATION
 
 certspotter continuously monitors all browser-recognized Certificate
 Transparency logs looking for certificates which are valid for any domain
 on your watch list. When certspotter detects a matching certificate, it
-emails you (if `-email` is specified), executes a script (if `-script`
-is specified), and/or writes a report to standard out (if `-stdout`
-is specified).
+emails you, executes a script, and/or writes a report to standard out,
+as described above.
 
 certspotter also saves a copy of matching certificates in
 `$CERTSPOTTER_STATE_DIR/certs` ("~/.certspotter/certs" by default)
@@ -156,9 +190,8 @@ following health checks:
    since the previous health check.
  * Ensure that certspotter is not falling behind monitoring any logs.
 
-If any health check fails, certspotter notifies you by email (if `-email`
-is specified), script (if `-script` is specified), and/or standard out
-(if `-stdout` is specified).
+If any health check fails, certspotter notifies you by email, script, and/or
+standard out, as described above.
 
 Health check failures should be rare, and you should take them seriously because it means
 certspotter might not detect all certificates.  It might also be an indication
@@ -182,11 +215,20 @@ and non-zero when a serious error occurs.
 :   Directory from which any configuration, such as the watch list, is read.
     Defaults to `~/.certspotter`.
 
+`EMAIL`
+
+:   Email address from which to send emails. If not set, certspotter lets sendmail pick
+    the address.
+
 `HTTPS_PROXY`
 
 :   URL of proxy server for making HTTPS requests.  `http://`, `https://`, and
     `socks5://` URLs are supported.  By default, no proxy server is used.
 
+`SENDMAIL_PATH`
+
+:   Path to the sendmail binary used for sending emails. Defaults to `/usr/sbin/sendmail`.
+
 # SEE ALSO
 
 certspotter-script(8)

merkletree/collapsed_tree.go

@@ -12,52 +12,98 @@ package merkletree
 import (
 	"encoding/json"
 	"fmt"
+	"math/bits"
+	"slices"
 )
 
+// CollapsedTree is an efficient representation of a Merkle (sub)tree that permits appending
+// nodes and calculating the root hash.
 type CollapsedTree struct {
-	nodes []Hash
-	size  uint64
+	offset uint64
+	nodes  []Hash
+	size   uint64
 }
 
 func calculateNumNodes(size uint64) int {
-	numNodes := 0
-	for size > 0 {
-		numNodes += int(size & 1)
-		size >>= 1
-	}
-	return numNodes
+	return bits.OnesCount64(size)
 }
 
+// TODO: phase out this function
 func EmptyCollapsedTree() *CollapsedTree {
 	return &CollapsedTree{nodes: []Hash{}, size: 0}
 }
 
+// TODO: phase out this function
 func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) {
-	if len(nodes) != calculateNumNodes(size) {
-		return nil, fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes))
+	tree := new(CollapsedTree)
+	if err := tree.Init(nodes, size); err != nil {
+		return nil, err
 	}
-	return &CollapsedTree{nodes: nodes, size: size}, nil
+	return tree, nil
 }
 
-func CloneCollapsedTree(source *CollapsedTree) *CollapsedTree {
-	nodes := make([]Hash, len(source.nodes))
-	copy(nodes, source.nodes)
-	return &CollapsedTree{nodes: nodes, size: source.size}
+func (tree CollapsedTree) Equal(other CollapsedTree) bool {
+	return tree.offset == other.offset && tree.size == other.size && slices.Equal(tree.nodes, other.nodes)
 }
 
-func (tree *CollapsedTree) Add(hash Hash) {
+func (tree CollapsedTree) Clone() CollapsedTree {
+	return CollapsedTree{
+		offset: tree.offset,
+		nodes:  slices.Clone(tree.nodes),
+		size:   tree.size,
+	}
+}
+
+// Add a new leaf hash to the end of the tree.
+// Returns an error if and only if the new tree would be too large for the subtree offset.
+// Always returns a nil error if tree.Offset() == 0.
+func (tree *CollapsedTree) Add(hash Hash) error {
+	if tree.offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if tree.size+1 > maxSize {
+			return fmt.Errorf("subtree at offset %d is already at maximum size %d", tree.offset, maxSize)
+		}
+	}
 	tree.nodes = append(tree.nodes, hash)
 	tree.size++
-	size := tree.size
-	for size%2 == 0 {
+	tree.collapse()
+	return nil
+}
+
+func (tree *CollapsedTree) Append(other CollapsedTree) error {
+	if tree.offset+tree.size != other.offset {
+		return fmt.Errorf("subtree at offset %d cannot be appended to subtree ending at offset %d", other.offset, tree.offset+tree.size)
+	}
+	if tree.offset > 0 {
+		newSize := tree.size + other.size
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if newSize > maxSize {
+			return fmt.Errorf("size of new subtree (%d) would exceed maximum size %d for a subtree at offset %d", newSize, maxSize, tree.offset)
+		}
+	}
+	if tree.size > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.size)
+		if other.size > maxSize {
+			return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize)
+		}
+	}
+
+	tree.nodes = append(tree.nodes, other.nodes...)
+	tree.size += other.size
+	tree.collapse()
+	return nil
+}
+
+func (tree *CollapsedTree) collapse() {
+	numNodes := calculateNumNodes(tree.size)
+	for len(tree.nodes) > numNodes {
 		left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1]
 		tree.nodes = tree.nodes[:len(tree.nodes)-2]
 		tree.nodes = append(tree.nodes, HashChildren(left, right))
-		size /= 2
 	}
 }
 
-func (tree *CollapsedTree) CalculateRoot() Hash {
+func (tree CollapsedTree) CalculateRoot() Hash {
 	if len(tree.nodes) == 0 {
 		return HashNothing()
 	}
@@ -70,29 +116,67 @@ func (tree *CollapsedTree) CalculateRoot() Hash {
 	return hash
 }
 
-func (tree *CollapsedTree) Size() uint64 {
+// Return the subtree offset (0 if this represents an entire tree)
+func (tree CollapsedTree) Offset() uint64 {
+	return tree.offset
+}
+
+// Return a non-nil slice containing the nodes.  The slice
+// must not be modified.
+func (tree CollapsedTree) Nodes() []Hash {
+	if tree.nodes == nil {
+		return []Hash{}
+	} else {
+		return tree.nodes
+	}
+}
+
+// Return the number of leaf nodes in the tree.
+func (tree CollapsedTree) Size() uint64 {
 	return tree.size
 }
 
-func (tree *CollapsedTree) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
-		"nodes": tree.nodes,
-		"size":  tree.size,
+type collapsedTreeMessage struct {
+	Offset uint64 `json:"offset,omitempty"`
+	Nodes  []Hash `json:"nodes"` // never nil
+	Size   uint64 `json:"size"`
+}
+
+func (tree CollapsedTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(collapsedTreeMessage{
+		Offset: tree.offset,
+		Nodes:  tree.Nodes(),
+		Size:   tree.size,
 	})
 }
 
 func (tree *CollapsedTree) UnmarshalJSON(b []byte) error {
-	var rawTree struct {
-		Nodes []Hash `json:"nodes"`
-		Size  uint64 `json:"size"`
-	}
+	var rawTree collapsedTreeMessage
 	if err := json.Unmarshal(b, &rawTree); err != nil {
 		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
 	}
-	if len(rawTree.Nodes) != calculateNumNodes(rawTree.Size) {
-		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: nodes has wrong length (should be %d, not %d)", calculateNumNodes(rawTree.Size), len(rawTree.Nodes))
+	if err := tree.InitSubtree(rawTree.Offset, rawTree.Nodes, rawTree.Size); err != nil {
+		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
 	}
-	tree.size = rawTree.Size
-	tree.nodes = rawTree.Nodes
 	return nil
 }
+
+func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error {
+	if len(nodes) != calculateNumNodes(size) {
+		return fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes))
+	}
+	tree.size = size
+	tree.nodes = nodes
+	return nil
+}
+
+func (tree *CollapsedTree) InitSubtree(offset uint64, nodes []Hash, size uint64) error {
+	if offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(offset)
+		if size > maxSize {
+			return fmt.Errorf("subtree size (%d) is too large for offset %d (maximum size is %d)", size, offset, maxSize)
+		}
+	}
+	tree.offset = offset
+	return tree.Init(nodes, size)
+}

merkletree/fragment.go

@@ -0,0 +1,121 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"encoding/json"
+	"fmt"
+	"slices"
+)
+
+// FragmentedCollapsedTree represents a sequence of non-overlapping subtrees
+type FragmentedCollapsedTree struct {
+	subtrees []CollapsedTree // sorted by offset
+}
+
+func (tree *FragmentedCollapsedTree) AddHash(position uint64, hash Hash) error {
+	return tree.Add(CollapsedTree{
+		offset: position,
+		nodes:  []Hash{hash},
+		size:   1,
+	})
+}
+
+func (tree *FragmentedCollapsedTree) Add(subtree CollapsedTree) error {
+	if subtree.size == 0 {
+		return nil
+	}
+	i := len(tree.subtrees)
+	for i > 0 && tree.subtrees[i-1].offset > subtree.offset {
+		i--
+	}
+	if i > 0 && tree.subtrees[i-1].offset+tree.subtrees[i-1].size > subtree.offset {
+		return fmt.Errorf("overlaps with subtree ending at %d", tree.subtrees[i-1].offset+tree.subtrees[i-1].size)
+	}
+	if i < len(tree.subtrees) && subtree.offset+subtree.size > tree.subtrees[i].offset {
+		return fmt.Errorf("overlaps with subtree starting at %d", tree.subtrees[i].offset)
+	}
+	if i == 0 || tree.subtrees[i-1].Append(subtree) != nil {
+		tree.subtrees = slices.Insert(tree.subtrees, i, subtree)
+		i++
+	}
+	for i < len(tree.subtrees) && tree.subtrees[i-1].Append(tree.subtrees[i]) == nil {
+		tree.subtrees = slices.Delete(tree.subtrees, i, i+1)
+	}
+	return nil
+}
+
+func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error {
+	for _, subtree := range other.subtrees {
+		if err := tree.Add(subtree); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) {
+	var prevEnd uint64
+	for i := range tree.subtrees {
+		if prevEnd != tree.subtrees[i].offset {
+			if !yield(prevEnd, tree.subtrees[i].offset) {
+				return
+			}
+		}
+		prevEnd = tree.subtrees[i].offset + tree.subtrees[i].size
+	}
+	yield(prevEnd, 0)
+}
+
+func (tree FragmentedCollapsedTree) NumSubtrees() int {
+	return len(tree.subtrees)
+}
+
+func (tree FragmentedCollapsedTree) Subtree(i int) CollapsedTree {
+	return tree.subtrees[i]
+}
+
+func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree {
+	if tree.subtrees == nil {
+		return []CollapsedTree{}
+	} else {
+		return tree.subtrees
+	}
+}
+
+// Return true iff the tree contains at least the first n nodes (without any gaps)
+func (tree FragmentedCollapsedTree) ContainsFirstN(n uint64) bool {
+	return len(tree.subtrees) >= 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size >= n
+}
+
+func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error {
+	for i := 1; i < len(subtrees); i++ {
+		if subtrees[i-1].offset+subtrees[i-1].size > subtrees[i].offset {
+			return fmt.Errorf("subtrees %d and %d overlap", i-1, i)
+		}
+	}
+	tree.subtrees = subtrees
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(tree.Subtrees())
+}
+
+func (tree *FragmentedCollapsedTree) UnmarshalJSON(b []byte) error {
+	var subtrees []CollapsedTree
+	if err := json.Unmarshal(b, &subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	if err := tree.Init(subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	return nil
+}

merkletree/hash.go

@@ -28,11 +28,19 @@ func (h Hash) MarshalJSON() ([]byte, error) {
 	return json.Marshal(h[:])
 }
 
+func (h Hash) MarshalBinary() ([]byte, error) {
+	return h[:], nil
+}
+
 func (h *Hash) UnmarshalJSON(b []byte) error {
 	var hashBytes []byte
 	if err := json.Unmarshal(b, &hashBytes); err != nil {
 		return err
 	}
+	return h.UnmarshalBinary(hashBytes)
+}
+
+func (h *Hash) UnmarshalBinary(hashBytes []byte) error {
 	if len(hashBytes) != HashLen {
 		return fmt.Errorf("Merkle Tree hash has wrong length (should be %d bytes long, not %d)", HashLen, len(hashBytes))
 	}

monitor/config.go

@@ -15,13 +15,9 @@ import (
 
 type Config struct {
 	LogListSource       string
-	StateDir            string
+	State               StateProvider
 	StartAtEnd          bool
 	WatchList           WatchList
 	Verbose             bool
-	SaveCerts           bool
-	Script              string
-	Email               []string
-	Stdout              bool
 	HealthCheckInterval time.Duration
 }

monitor/daemon.go

@@ -50,12 +50,13 @@ type daemon struct {
 
 func (daemon *daemon) healthCheck(ctx context.Context) error {
 	if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval {
-		if err := notify(ctx, daemon.config, &staleLogListEvent{
+		info := &StaleLogListInfo{
 			Source:        daemon.config.LogListSource,
 			LastSuccess:   daemon.logsLoadedAt,
 			LastError:     daemon.logListError,
 			LastErrorTime: daemon.logListErrorAt,
-		}); err != nil {
+		}
+		if err := daemon.config.State.NotifyHealthCheckFailure(ctx, nil, info); err != nil {
 			return fmt.Errorf("error notifying about stale log list: %w", err)
 		}
 	}
@@ -122,8 +123,8 @@ func (daemon *daemon) loadLogList(ctx context.Context) error {
 }
 
 func (daemon *daemon) run(ctx context.Context) error {
-	if err := prepareStateDir(daemon.config.StateDir); err != nil {
-		return fmt.Errorf("error preparing state directory: %w", err)
+	if err := daemon.config.State.Prepare(ctx); err != nil {
+		return fmt.Errorf("error preparing state: %w", err)
 	}
 
 	if err := daemon.loadLogList(ctx); err != nil {
@@ -143,7 +144,7 @@ func (daemon *daemon) run(ctx context.Context) error {
 			if err := daemon.loadLogList(ctx); err != nil {
 				daemon.logListError = err.Error()
 				daemon.logListErrorAt = time.Now()
-				recordError(fmt.Errorf("error reloading log list (will try again later): %w", err))
+				recordError(ctx, daemon.config, nil, fmt.Errorf("error reloading log list (will try again later): %w", err))
 			}
 			reloadLogListTicker.Reset(reloadLogListInterval())
 		case <-healthCheckTicker.C:

monitor/discoveredcert.go

@@ -12,7 +12,6 @@ package monitor
 import (
 	"bytes"
 	"encoding/hex"
-	"encoding/json"
 	"encoding/pem"
 	"fmt"
 	"strings"
@@ -22,21 +21,24 @@ import (
 	"software.sslmate.com/src/certspotter/ct"
 )
 
-type discoveredCert struct {
+type DiscoveredCert struct {
 	WatchItem    WatchItem
-	LogEntry     *logEntry
+	LogEntry     *LogEntry
 	Info         *certspotter.CertInfo
 	Chain        []ct.ASN1Cert // first entry is the leaf certificate or precertificate
 	TBSSHA256    [32]byte      // computed over Info.TBS.Raw
 	SHA256       [32]byte      // computed over Chain[0]
 	PubkeySHA256 [32]byte      // computed over Info.TBS.PublicKey.FullBytes
 	Identifiers  *certspotter.Identifiers
-	CertPath     string // empty if not saved on the filesystem
-	JSONPath     string // empty if not saved on the filesystem
-	TextPath     string // empty if not saved on the filesystem
 }
 
-func (cert *discoveredCert) pemChain() []byte {
+type certPaths struct {
+	certPath string
+	jsonPath string
+	textPath string
+}
+
+func (cert *DiscoveredCert) pemChain() []byte {
 	var buffer bytes.Buffer
 	for _, certBytes := range cert.Chain {
 		if err := pem.Encode(&buffer, &pem.Block{
@@ -49,7 +51,7 @@ func (cert *discoveredCert) pemChain() []byte {
 	return buffer.Bytes()
 }
 
-func (cert *discoveredCert) json() []byte {
+func (cert *DiscoveredCert) json() any {
 	object := map[string]any{
 		"tbs_sha256":    hex.EncodeToString(cert.TBSSHA256[:]),
 		"pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]),
@@ -65,30 +67,26 @@ func (cert *discoveredCert) json() []byte {
 		object["not_after"] = nil
 	}
 
-	jsonBytes, err := json.Marshal(object)
-	if err != nil {
-		panic(fmt.Errorf("encoding certificate as JSON failed unexpectedly: %w", err))
-	}
-	return jsonBytes
+	return object
 }
 
-func (cert *discoveredCert) save() error {
-	if err := writeFile(cert.CertPath, cert.pemChain(), 0666); err != nil {
+func writeCertFiles(cert *DiscoveredCert, paths *certPaths) error {
+	if err := writeFile(paths.certPath, cert.pemChain(), 0666); err != nil {
 		return err
 	}
-	if err := writeFile(cert.JSONPath, cert.json(), 0666); err != nil {
+	if err := writeJSONFile(paths.jsonPath, cert.json(), 0666); err != nil {
 		return err
 	}
-	if err := writeFile(cert.TextPath, []byte(cert.Text()), 0666); err != nil {
+	if err := writeTextFile(paths.textPath, certNotificationText(cert, paths), 0666); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (cert *discoveredCert) Environ() []string {
+func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string {
 	env := []string{
 		"EVENT=discovered_cert",
-		"SUMMARY=certificate discovered for " + cert.WatchItem.String(),
+		"SUMMARY=" + certNotificationSummary(cert),
 		"CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented
 		"LOG_URI=" + cert.LogEntry.Log.URL,
 		"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
@@ -98,9 +96,12 @@ func (cert *discoveredCert) Environ() []string {
 		"FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented
 		"PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]),
 		"PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented
-		"CERT_FILENAME=" + cert.CertPath,
-		"JSON_FILENAME=" + cert.JSONPath,
-		"TEXT_FILENAME=" + cert.TextPath,
+	}
+
+	if paths != nil {
+		env = append(env, "CERT_FILENAME="+paths.certPath)
+		env = append(env, "JSON_FILENAME="+paths.jsonPath)
+		env = append(env, "TEXT_FILENAME="+paths.textPath)
 	}
 
 	if cert.Info.ValidityParseError == nil {
@@ -135,7 +136,7 @@ func (cert *discoveredCert) Environ() []string {
 	return env
 }
 
-func (cert *discoveredCert) Text() string {
+func certNotificationText(cert *DiscoveredCert, paths *certPaths) string {
 	// TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration)
 
 	text := new(strings.Builder)
@@ -163,13 +164,13 @@ func (cert *discoveredCert) Text() string {
 	}
 	writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL))
 	writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:]))
-	if cert.CertPath != "" {
-		writeField("Filename", cert.CertPath)
+	if paths != nil {
+		writeField("Filename", paths.certPath)
 	}
 
 	return text.String()
 }
 
-func (cert *discoveredCert) EmailSubject() string {
-	return fmt.Sprintf("[certspotter] Certificate Discovered for %s", cert.WatchItem)
+func certNotificationSummary(cert *DiscoveredCert) string {
+	return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem)
 }

monitor/errors.go

@@ -10,9 +10,19 @@
 package monitor
 
 import (
+	"context"
 	"log"
+
+	"software.sslmate.com/src/certspotter/loglist"
 )
 
-func recordError(err error) {
-	log.Print(err)
+func recordError(ctx context.Context, config *Config, ctlog *loglist.Log, errToRecord error) {
+	if err := config.State.NotifyError(ctx, ctlog, errToRecord); err != nil {
+		log.Print("unable to notify about error: ", err)
+		if ctlog == nil {
+			log.Print(errToRecord)
+		} else {
+			log.Print(ctlog.URL, ": ", errToRecord)
+		}
+	}
 }

monitor/fileutils.go

@@ -12,6 +12,7 @@ package monitor
 import (
 	"crypto/rand"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"os"
 )
@@ -36,6 +37,19 @@ func writeFile(filename string, data []byte, perm os.FileMode) error {
 	return nil
 }
 
+func writeTextFile(filename string, fileText string, perm os.FileMode) error {
+	return writeFile(filename, []byte(fileText), perm)
+}
+
+func writeJSONFile(filename string, data any, perm os.FileMode) error {
+	fileBytes, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	fileBytes = append(fileBytes, '\n')
+	return writeFile(filename, fileBytes, perm)
+}
+
 func fileExists(filename string) bool {
 	_, err := os.Lstat(filename)
 	return err == nil

monitor/fsstate.go

@@ -0,0 +1,239 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+type FilesystemState struct {
+	StateDir  string
+	SaveCerts bool
+	Script    string
+	ScriptDir string
+	Email     []string
+	Stdout    bool
+}
+
+func (s *FilesystemState) logStateDir(logID LogID) string {
+	return filepath.Join(s.StateDir, "logs", logID.Base64URLString())
+}
+
+func (s *FilesystemState) Prepare(ctx context.Context) error {
+	return prepareStateDir(s.StateDir)
+}
+
+func (s *FilesystemState) PrepareLog(ctx context.Context, logID LogID) error {
+	var (
+		stateDirPath        = s.logStateDir(logID)
+		sthsDirPath         = filepath.Join(stateDirPath, "unverified_sths")
+		malformedDirPath    = filepath.Join(stateDirPath, "malformed_entries")
+		healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
+	)
+	for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} {
+		if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *FilesystemState) LoadLogState(ctx context.Context, logID LogID) (*LogState, error) {
+	filePath := filepath.Join(s.logStateDir(logID), "state.json")
+	fileBytes, err := os.ReadFile(filePath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	state := new(LogState)
+	if err := json.Unmarshal(fileBytes, state); err != nil {
+		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
+	}
+	return state, nil
+}
+
+func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state *LogState) error {
+	filePath := filepath.Join(s.logStateDir(logID), "state.json")
+	return writeJSONFile(filePath, state, 0666)
+}
+
+func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return storeSTHInDir(sthsDirPath, sth)
+}
+
+func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*ct.SignedTreeHead, error) {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return loadSTHsFromDir(sthsDirPath)
+}
+
+func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return removeSTHFromDir(sthsDirPath, sth)
+}
+
+func (s *FilesystemState) NotifyCert(ctx context.Context, cert *DiscoveredCert) error {
+	var notifiedPath string
+	var paths *certPaths
+	if s.SaveCerts {
+		hexFingerprint := hex.EncodeToString(cert.SHA256[:])
+		prefixPath := filepath.Join(s.StateDir, "certs", hexFingerprint[0:2])
+		var (
+			notifiedFilename      = "." + hexFingerprint + ".notified"
+			certFilename          = hexFingerprint + ".pem"
+			jsonFilename          = hexFingerprint + ".v1.json"
+			textFilename          = hexFingerprint + ".txt"
+			legacyCertFilename    = hexFingerprint + ".cert.pem"
+			legacyPrecertFilename = hexFingerprint + ".precert.pem"
+		)
+
+		for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} {
+			if fileExists(filepath.Join(prefixPath, filename)) {
+				return nil
+			}
+		}
+
+		if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
+		}
+
+		notifiedPath = filepath.Join(prefixPath, notifiedFilename)
+		paths = &certPaths{
+			certPath: filepath.Join(prefixPath, certFilename),
+			jsonPath: filepath.Join(prefixPath, jsonFilename),
+			textPath: filepath.Join(prefixPath, textFilename),
+		}
+		if err := writeCertFiles(cert, paths); err != nil {
+			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
+		}
+	} else {
+		// TODO-4: save cert to temporary files, and defer their unlinking
+	}
+
+	if err := s.notify(ctx, &notification{
+		summary: certNotificationSummary(cert),
+		environ: certNotificationEnviron(cert, paths),
+		text:    certNotificationText(cert, paths),
+	}); err != nil {
+		return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
+	}
+
+	if notifiedPath != "" {
+		if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
+			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
+		}
+	}
+
+	return nil
+}
+
+func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError error) error {
+	var (
+		dirPath   = filepath.Join(s.logStateDir(entry.Log.LogID), "malformed_entries")
+		entryPath = filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index))
+		textPath  = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index))
+	)
+
+	summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.URL)
+
+	entryJSON := struct {
+		LeafInput []byte `json:"leaf_input"`
+		ExtraData []byte `json:"extra_data"`
+	}{
+		LeafInput: entry.LeafInput,
+		ExtraData: entry.ExtraData,
+	}
+
+	text := new(strings.Builder)
+	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
+	fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
+	writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.URL))
+	writeField("Leaf Hash", entry.LeafHash.Base64String())
+	writeField("Error", parseError.Error())
+
+	if err := writeJSONFile(entryPath, entryJSON, 0666); err != nil {
+		return fmt.Errorf("error saving JSON file: %w", err)
+	}
+	if err := writeTextFile(textPath, text.String(), 0666); err != nil {
+		return fmt.Errorf("error saving texT file: %w", err)
+	}
+
+	environ := []string{
+		"EVENT=malformed_cert",
+		"SUMMARY=" + summary,
+		"LOG_URI=" + entry.Log.URL,
+		"ENTRY_INDEX=" + fmt.Sprint(entry.Index),
+		"LEAF_HASH=" + entry.LeafHash.Base64String(),
+		"PARSE_ERROR=" + parseError.Error(),
+		"ENTRY_FILENAME=" + entryPath,
+		"TEXT_FILENAME=" + textPath,
+		"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
+	}
+
+	if err := s.notify(ctx, &notification{
+		environ: environ,
+		summary: summary,
+		text:    text.String(),
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) healthCheckDir(ctlog *loglist.Log) string {
+	if ctlog == nil {
+		return filepath.Join(s.StateDir, "healthchecks")
+	} else {
+		return filepath.Join(s.logStateDir(ctlog.LogID), "healthchecks")
+	}
+}
+
+func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info HealthCheckFailure) error {
+	textPath := filepath.Join(s.healthCheckDir(ctlog), healthCheckFilename())
+	environ := []string{
+		"EVENT=error",
+		"SUMMARY=" + info.Summary(),
+		"TEXT_FILENAME=" + textPath,
+	}
+	text := info.Text()
+	if err := writeTextFile(textPath, text, 0666); err != nil {
+		return fmt.Errorf("error saving text file: %w", err)
+	}
+	if err := s.notify(ctx, &notification{
+		environ: environ,
+		summary: info.Summary(),
+		text:    text,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, err error) error {
+	if ctlog == nil {
+		log.Print(err)
+	} else {
+		log.Print(ctlog.URL, ":", err)
+	}
+	return nil
+}

monitor/healthcheck.go

@@ -11,10 +11,7 @@ package monitor
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"io/fs"
-	"path/filepath"
 	"strings"
 	"time"
 
@@ -22,42 +19,43 @@ import (
 	"software.sslmate.com/src/certspotter/loglist"
 )
 
+func healthCheckFilename() string {
+	return time.Now().UTC().Format(time.RFC3339) + ".txt"
+}
+
 func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error {
-	var (
-		stateDirPath  = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString())
-		stateFilePath = filepath.Join(stateDirPath, "state.json")
-		sthsDirPath   = filepath.Join(stateDirPath, "unverified_sths")
-	)
-	state, err := loadStateFile(stateFilePath)
-	if errors.Is(err, fs.ErrNotExist) {
+	state, err := config.State.LoadLogState(ctx, ctlog.LogID)
+	if err != nil {
+		return fmt.Errorf("error loading log state: %w", err)
+	} else if state == nil {
 		return nil
-	} else if err != nil {
-		return fmt.Errorf("error loading state file: %w", err)
 	}
 
 	if time.Since(state.LastSuccess) < config.HealthCheckInterval {
 		return nil
 	}
 
-	sths, err := loadSTHsFromDir(sthsDirPath)
+	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
 	if err != nil {
-		return fmt.Errorf("error loading STHs directory: %w", err)
+		return fmt.Errorf("error loading STHs: %w", err)
 	}
 
 	if len(sths) == 0 {
-		if err := notify(ctx, config, &staleSTHEvent{
+		info := &StaleSTHInfo{
 			Log:         ctlog,
 			LastSuccess: state.LastSuccess,
 			LatestSTH:   state.VerifiedSTH,
-		}); err != nil {
+		}
+		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
 			return fmt.Errorf("error notifying about stale STH: %w", err)
 		}
 	} else {
-		if err := notify(ctx, config, &backlogEvent{
+		info := &BacklogInfo{
 			Log:       ctlog,
 			LatestSTH: sths[len(sths)-1],
 			Position:  state.DownloadPosition.Size(),
-		}); err != nil {
+		}
+		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
 			return fmt.Errorf("error notifying about backlog: %w", err)
 		}
 	}
@@ -65,57 +63,45 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err
 	return nil
 }
 
-type staleSTHEvent struct {
+type HealthCheckFailure interface {
+	Summary() string
+	Text() string
+}
+
+type StaleSTHInfo struct {
 	Log         *loglist.Log
 	LastSuccess time.Time
 	LatestSTH   *ct.SignedTreeHead // may be nil
 }
-type backlogEvent struct {
+
+type BacklogInfo struct {
 	Log       *loglist.Log
 	LatestSTH *ct.SignedTreeHead
 	Position  uint64
 }
-type staleLogListEvent struct {
+
+type StaleLogListInfo struct {
 	Source        string
 	LastSuccess   time.Time
 	LastError     string
 	LastErrorTime time.Time
 }
 
-func (e *backlogEvent) Backlog() uint64 {
+func (e *BacklogInfo) Backlog() uint64 {
 	return e.LatestSTH.TreeSize - e.Position
 }
 
-func (e *staleSTHEvent) Environ() []string {
-	return []string{
-		"EVENT=error",
-		"SUMMARY=" + fmt.Sprintf("unable to contact %s since %s", e.Log.URL, e.LastSuccess),
-	}
+func (e *StaleSTHInfo) Summary() string {
+	return fmt.Sprintf("Unable to contact %s since %s", e.Log.URL, e.LastSuccess)
 }
-func (e *backlogEvent) Environ() []string {
-	return []string{
-		"EVENT=error",
-		"SUMMARY=" + fmt.Sprintf("backlog of size %d from %s", e.Backlog(), e.Log.URL),
-	}
+func (e *BacklogInfo) Summary() string {
+	return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.URL)
 }
-func (e *staleLogListEvent) Environ() []string {
-	return []string{
-		"EVENT=error",
-		"SUMMARY=" + fmt.Sprintf("unable to retrieve log list since %s: %s", e.LastSuccess, e.LastError),
-	}
+func (e *StaleLogListInfo) Summary() string {
+	return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess)
 }
 
-func (e *staleSTHEvent) EmailSubject() string {
-	return fmt.Sprintf("[certspotter] Unable to contact %s since %s", e.Log.URL, e.LastSuccess)
-}
-func (e *backlogEvent) EmailSubject() string {
-	return fmt.Sprintf("[certspotter] Backlog of size %d from %s", e.Backlog(), e.Log.URL)
-}
-func (e *staleLogListEvent) EmailSubject() string {
-	return fmt.Sprintf("[certspotter] Unable to retrieve log list since %s", e.LastSuccess)
-}
-
-func (e *staleSTHEvent) Text() string {
+func (e *StaleSTHInfo) Text() string {
 	text := new(strings.Builder)
 	fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess)
 	fmt.Fprintf(text, "\n")
@@ -128,7 +114,7 @@ func (e *staleSTHEvent) Text() string {
 	}
 	return text.String()
 }
-func (e *backlogEvent) Text() string {
+func (e *BacklogInfo) Text() string {
 	text := new(strings.Builder)
 	fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL)
 	fmt.Fprintf(text, "\n")
@@ -139,7 +125,7 @@ func (e *backlogEvent) Text() string {
 	fmt.Fprintf(text, "         Backlog = %d\n", e.Backlog())
 	return text.String()
 }
-func (e *staleLogListEvent) Text() string {
+func (e *StaleLogListInfo) Text() string {
 	text := new(strings.Builder)
 	fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess)
 	fmt.Fprintf(text, "\n")

monitor/mailutils.go

@@ -0,0 +1,34 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"os"
+)
+
+const mailDateFormat = "Mon, 2 Jan 2006 15:04:05 -0700"
+
+func generateMessageID() string {
+	var randomBytes [16]byte
+	if _, err := rand.Read(randomBytes[:]); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(randomBytes[:]) + "@selfhosted.certspotter.org"
+}
+
+func sendmailPath() string {
+	if envVar := os.Getenv("SENDMAIL_PATH"); envVar != "" {
+		return envVar
+	} else {
+		return "/usr/sbin/sendmail"
+	}
+}

monitor/malformed.go

@@ -1,48 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"fmt"
-	"strings"
-)
-
-type malformedLogEntry struct {
-	Entry *logEntry
-	Error string
-}
-
-func (malformed *malformedLogEntry) Environ() []string {
-	return []string{
-		"EVENT=malformed_cert",
-		"SUMMARY=" + fmt.Sprintf("unable to parse entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL),
-		"LOG_URI=" + malformed.Entry.Log.URL,
-		"ENTRY_INDEX=" + fmt.Sprint(malformed.Entry.Index),
-		"LEAF_HASH=" + malformed.Entry.LeafHash.Base64String(),
-		"PARSE_ERROR=" + malformed.Error,
-		"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
-	}
-}
-
-func (malformed *malformedLogEntry) Text() string {
-	text := new(strings.Builder)
-	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
-
-	fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
-	writeField("Log Entry", fmt.Sprintf("%d @ %s", malformed.Entry.Index, malformed.Entry.Log.URL))
-	writeField("Leaf Hash", malformed.Entry.LeafHash.Base64String())
-	writeField("Error", malformed.Error)
-
-	return text.String()
-}
-
-func (malformed *malformedLogEntry) EmailSubject() string {
-	return fmt.Sprintf("[certspotter] Unable to Parse Entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL)
-}

monitor/monitor.go

@@ -14,10 +14,7 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	"io/fs"
 	"log"
-	"os"
-	"path/filepath"
 	"strings"
 	"time"
 
@@ -73,15 +70,8 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	var (
-		stateDirPath  = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString())
-		stateFilePath = filepath.Join(stateDirPath, "state.json")
-		sthsDirPath   = filepath.Join(stateDirPath, "unverified_sths")
-	)
-	for _, dirPath := range []string{stateDirPath, sthsDirPath} {
-		if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
-			return fmt.Errorf("error creating state directory: %w", err)
-		}
+	if err := config.State.PrepareLog(ctx, ctlog.LogID); err != nil {
+		return fmt.Errorf("error preparing state: %w", err)
 	}
 
 	startTime := time.Now()
@@ -89,32 +79,35 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 	if isFatalLogError(err) {
 		return err
 	} else if err != nil {
-		recordError(fmt.Errorf("error fetching latest STH for %s: %w", ctlog.URL, err))
+		recordError(ctx, config, ctlog, fmt.Errorf("error fetching latest STH: %w", err))
 		return nil
 	}
 	latestSTH.LogID = ctlog.LogID
-	if err := storeSTHInDir(sthsDirPath, latestSTH); err != nil {
+	if err := config.State.StoreSTH(ctx, ctlog.LogID, latestSTH); err != nil {
 		return fmt.Errorf("error storing latest STH: %w", err)
 	}
 
-	state, err := loadStateFile(stateFilePath)
-	if errors.Is(err, fs.ErrNotExist) {
+	state, err := config.State.LoadLogState(ctx, ctlog.LogID)
+	if err != nil {
+		return fmt.Errorf("error loading log state: %w", err)
+	}
+	if state == nil {
 		if config.StartAtEnd {
 			tree, err := reconstructTree(ctx, logClient, latestSTH)
 			if isFatalLogError(err) {
 				return err
 			} else if err != nil {
-				recordError(fmt.Errorf("error reconstructing tree of size %d for %s: %w", latestSTH.TreeSize, ctlog.URL, err))
+				recordError(ctx, config, ctlog, fmt.Errorf("error reconstructing tree of size %d: %w", latestSTH.TreeSize, err))
 				return nil
 			}
-			state = &stateFile{
+			state = &LogState{
 				DownloadPosition: tree,
 				VerifiedPosition: tree,
 				VerifiedSTH:      latestSTH,
 				LastSuccess:      startTime.UTC(),
 			}
 		} else {
-			state = &stateFile{
+			state = &LogState{
 				DownloadPosition: merkletree.EmptyCollapsedTree(),
 				VerifiedPosition: merkletree.EmptyCollapsedTree(),
 				VerifiedSTH:      nil,
@@ -124,21 +117,19 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 		if config.Verbose {
 			log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size())
 		}
-		if err := state.store(stateFilePath); err != nil {
-			return fmt.Errorf("error storing state file: %w", err)
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+			return fmt.Errorf("error storing log state: %w", err)
 		}
-	} else if err != nil {
-		return fmt.Errorf("error loading state file: %w", err)
 	}
 
-	sths, err := loadSTHsFromDir(sthsDirPath)
+	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
 	if err != nil {
-		return fmt.Errorf("error loading STHs directory: %w", err)
+		return fmt.Errorf("error loading STHs: %w", err)
 	}
 
 	for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() {
 		// TODO-4: audit sths[0] against state.VerifiedSTH
-		if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
+		if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
 			return fmt.Errorf("error removing STH: %w", err)
 		}
 		sths = sths[1:]
@@ -148,8 +139,8 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 		if config.Verbose {
 			log.Printf("saving state in defer for %s", ctlog.URL)
 		}
-		if err := state.store(stateFilePath); err != nil && returnedErr == nil {
-			returnedErr = fmt.Errorf("error storing state file: %w", err)
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil && returnedErr == nil {
+			returnedErr = fmt.Errorf("error storing log state: %w", err)
 		}
 	}()
 
@@ -172,7 +163,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 		downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd)
 	}()
 	for rawEntry := range entries {
-		entry := &logEntry{
+		entry := &LogEntry{
 			Log:       ctlog,
 			Index:     state.DownloadPosition.Size(),
 			LeafInput: rawEntry.LeafInput,
@@ -189,11 +180,11 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 
 		for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize {
 			if merkletree.Hash(sths[0].SHA256RootHash) != rootHash {
-				recordError(fmt.Errorf("error verifying %s at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", ctlog.URL, sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
+				recordError(ctx, config, ctlog, fmt.Errorf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
 
 				state.DownloadPosition = state.VerifiedPosition
-				if err := state.store(stateFilePath); err != nil {
-					return fmt.Errorf("error storing state file: %w", err)
+				if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+					return fmt.Errorf("error storing log state: %w", err)
 				}
 				return nil
 			}
@@ -201,7 +192,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 			state.VerifiedPosition = state.DownloadPosition
 			state.VerifiedSTH = sths[0]
 			shouldSaveState = true
-			if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
+			if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
 				return fmt.Errorf("error removing verified STH: %w", err)
 			}
 
@@ -209,7 +200,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 		}
 
 		if shouldSaveState {
-			if err := state.store(stateFilePath); err != nil {
+			if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
 				return fmt.Errorf("error storing state file: %w", err)
 			}
 		}
@@ -218,7 +209,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 	if isFatalLogError(downloadErr) {
 		return downloadErr
 	} else if downloadErr != nil {
-		recordError(fmt.Errorf("error downloading entries from %s: %w", ctlog.URL, downloadErr))
+		recordError(ctx, config, ctlog, fmt.Errorf("error downloading entries: %w", downloadErr))
 		return nil
 	}
 
@@ -232,7 +223,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
 
 func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error {
 	for begin < end && ctx.Err() == nil {
-		size := begin - end
+		size := end - begin
 		if size > maxGetEntriesSize {
 			size = maxGetEntriesSize
 		}
@@ -267,6 +258,7 @@ func reconstructTree(ctx context.Context, logClient *client.LogClient, sth *ct.S
 
 	var tree *merkletree.CollapsedTree
 	if sth.TreeSize > 1 {
+		// XXX: if leafHash is in the tree in more than one place, this might not return the proof that we need ... get-entry-and-proof avoids this problem but not all logs support it
 		auditPath, _, err := logClient.GetAuditProof(ctx, leafHash[:], sth.TreeSize)
 		if err != nil {
 			return nil, err

monitor/notify.go

@@ -12,34 +12,44 @@ package monitor
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 )
 
 var stdoutMu sync.Mutex
 
-type notification interface {
-	Environ() []string
-	EmailSubject() string
-	Text() string
+type notification struct {
+	environ []string
+	summary string
+	text    string
 }
 
-func notify(ctx context.Context, config *Config, notif notification) error {
-	if config.Stdout {
+func (s *FilesystemState) notify(ctx context.Context, notif *notification) error {
+	if s.Stdout {
 		writeToStdout(notif)
 	}
 
-	if len(config.Email) > 0 {
-		if err := sendEmail(ctx, config.Email, notif); err != nil {
+	if len(s.Email) > 0 {
+		if err := sendEmail(ctx, s.Email, notif); err != nil {
 			return err
 		}
 	}
 
-	if config.Script != "" {
-		if err := execScript(ctx, config.Script, notif); err != nil {
+	if s.Script != "" {
+		if err := execScript(ctx, s.Script, notif); err != nil {
+			return err
+		}
+	}
+
+	if s.ScriptDir != "" {
+		if err := execScriptDir(ctx, s.ScriptDir, notif); err != nil {
 			return err
 		}
 	}
@@ -47,28 +57,39 @@ func notify(ctx context.Context, config *Config, notif notification) error {
 	return nil
 }
 
-func writeToStdout(notif notification) {
+func writeToStdout(notif *notification) {
 	stdoutMu.Lock()
 	defer stdoutMu.Unlock()
-	os.Stdout.WriteString(notif.Text() + "\n")
+	os.Stdout.WriteString(notif.text + "\n")
 }
 
-func sendEmail(ctx context.Context, to []string, notif notification) error {
+func sendEmail(ctx context.Context, to []string, notif *notification) error {
 	stdin := new(bytes.Buffer)
 	stderr := new(bytes.Buffer)
 
+	from := os.Getenv("EMAIL")
+
+	if from != "" {
+		fmt.Fprintf(stdin, "From: %s\n", from)
+	}
 	fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", "))
-	fmt.Fprintf(stdin, "Subject: %s\n", notif.EmailSubject())
+	fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.summary)
+	fmt.Fprintf(stdin, "Date: %s\n", time.Now().Format(mailDateFormat))
+	fmt.Fprintf(stdin, "Message-ID: <%s>\n", generateMessageID())
 	fmt.Fprintf(stdin, "Mime-Version: 1.0\n")
 	fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n")
 	fmt.Fprintf(stdin, "X-Mailer: certspotter\n")
 	fmt.Fprintf(stdin, "\n")
-	fmt.Fprint(stdin, notif.Text())
+	fmt.Fprint(stdin, notif.text)
 
-	args := []string{"-i", "--"}
+	args := []string{"-i"}
+	if from != "" {
+		args = append(args, "-f", from)
+	}
+	args = append(args, "--")
 	args = append(args, to...)
 
-	sendmail := exec.CommandContext(ctx, "/usr/sbin/sendmail", args...)
+	sendmail := exec.CommandContext(ctx, sendmailPath(), args...)
 	sendmail.Stdin = stdin
 	sendmail.Stderr = stderr
 
@@ -83,12 +104,12 @@ func sendEmail(ctx context.Context, to []string, notif notification) error {
 	}
 }
 
-func execScript(ctx context.Context, scriptName string, notif notification) error {
+func execScript(ctx context.Context, scriptName string, notif *notification) error {
 	stderr := new(bytes.Buffer)
 
 	cmd := exec.CommandContext(ctx, scriptName)
 	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, notif.Environ()...)
+	cmd.Env = append(cmd.Env, notif.environ...)
 	cmd.Stderr = stderr
 
 	if err := cmd.Run(); err == nil {
@@ -104,6 +125,32 @@ func execScript(ctx context.Context, scriptName string, notif notification) erro
 	}
 }
 
+func execScriptDir(ctx context.Context, dirPath string, notif *notification) error {
+	dirents, err := os.ReadDir(dirPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("error executing scripts in directory %q: %w", dirPath, err)
+	}
+	for _, dirent := range dirents {
+		if strings.HasPrefix(dirent.Name(), ".") {
+			continue
+		}
+		scriptPath := filepath.Join(dirPath, dirent.Name())
+		info, err := os.Stat(scriptPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("error executing %q in directory %q: %w", dirent.Name(), dirPath, err)
+		} else if info.Mode().IsRegular() && isExecutable(info.Mode()) {
+			if err := execScript(ctx, scriptPath, notif); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 func isExecutable(mode os.FileMode) bool {
 	return mode&0111 != 0
 }

monitor/process.go

@@ -13,19 +13,14 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
-	"encoding/hex"
-	"errors"
 	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
 	"software.sslmate.com/src/certspotter"
 	"software.sslmate.com/src/certspotter/ct"
 	"software.sslmate.com/src/certspotter/loglist"
 	"software.sslmate.com/src/certspotter/merkletree"
 )
 
-type logEntry struct {
+type LogEntry struct {
 	Log       *loglist.Log
 	Index     uint64
 	LeafInput []byte
@@ -33,7 +28,7 @@ type logEntry struct {
 	LeafHash  merkletree.Hash
 }
 
-func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error {
+func processLogEntry(ctx context.Context, config *Config, entry *LogEntry) error {
 	leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput))
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err))
@@ -48,7 +43,7 @@ func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error
 	}
 }
 
-func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, cert ct.ASN1Cert) error {
+func processX509LogEntry(ctx context.Context, config *Config, entry *LogEntry, cert ct.ASN1Cert) error {
 	certInfo, err := certspotter.MakeCertInfoFromRawCert(cert)
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err))
@@ -69,7 +64,7 @@ func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, c
 	return processCertificate(ctx, config, entry, certInfo, chain)
 }
 
-func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry, precert ct.PreCert) error {
+func processPrecertLogEntry(ctx context.Context, config *Config, entry *LogEntry, precert ct.PreCert) error {
 	certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate)
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err))
@@ -87,7 +82,7 @@ func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry
 	return processCertificate(ctx, config, entry, certInfo, chain)
 }
 
-func processCertificate(ctx context.Context, config *Config, entry *logEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
+func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
 	identifiers, err := certInfo.ParseIdentifiers()
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, err)
@@ -97,7 +92,7 @@ func processCertificate(ctx context.Context, config *Config, entry *logEntry, ce
 		return nil
 	}
 
-	cert := &discoveredCert{
+	cert := &DiscoveredCert{
 		WatchItem:    watchItem,
 		LogEntry:     entry,
 		Info:         certInfo,
@@ -108,62 +103,15 @@ func processCertificate(ctx context.Context, config *Config, entry *logEntry, ce
 		Identifiers:  identifiers,
 	}
 
-	var notifiedPath string
-	if config.SaveCerts {
-		hexFingerprint := hex.EncodeToString(cert.SHA256[:])
-		prefixPath := filepath.Join(config.StateDir, "certs", hexFingerprint[0:2])
-		var (
-			notifiedFilename      = "." + hexFingerprint + ".notified"
-			certFilename          = hexFingerprint + ".pem"
-			jsonFilename          = hexFingerprint + ".v1.json"
-			textFilename          = hexFingerprint + ".txt"
-			legacyCertFilename    = hexFingerprint + ".cert.pem"
-			legacyPrecertFilename = hexFingerprint + ".precert.pem"
-		)
-
-		for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} {
-			if fileExists(filepath.Join(prefixPath, filename)) {
-				return nil
-			}
-		}
-
-		if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
-			return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
-		}
-
-		notifiedPath = filepath.Join(prefixPath, notifiedFilename)
-		cert.CertPath = filepath.Join(prefixPath, certFilename)
-		cert.JSONPath = filepath.Join(prefixPath, jsonFilename)
-		cert.TextPath = filepath.Join(prefixPath, textFilename)
-
-		if err := cert.save(); err != nil {
-			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
-		}
-	} else {
-		// TODO-4: save cert to temporary files, and defer their unlinking
-	}
-
-	if err := notify(ctx, config, cert); err != nil {
-		return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
-	}
-
-	if notifiedPath != "" {
-		if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
-			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
-		}
+	if err := config.State.NotifyCert(ctx, cert); err != nil {
+		return fmt.Errorf("error notifying about certificate %x: %w", cert.SHA256, err)
 	}
 
 	return nil
 }
 
-func processMalformedLogEntry(ctx context.Context, config *Config, entry *logEntry, parseError error) error {
-	// TODO-4: save the malformed entry (in get-entries format) in the state directory so user can inspect it
-
-	malformed := &malformedLogEntry{
-		Entry: entry,
-		Error: parseError.Error(),
-	}
-	if err := notify(ctx, config, malformed); err != nil {
+func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error {
+	if err := config.State.NotifyMalformedEntry(ctx, entry, parseError); err != nil {
 		return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
 	}
 	return nil

monitor/state.go

@@ -0,0 +1,68 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/merkletree"
+	"time"
+)
+
+type LogState struct {
+	DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
+	VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
+	VerifiedSTH      *ct.SignedTreeHead        `json:"verified_sth"`
+	LastSuccess      time.Time                 `json:"last_success"`
+}
+
+type StateProvider interface {
+	// Initialize the state.  Called before any other method in this interface.
+	// Idempotent: returns nil if the state is already initialized.
+	Prepare(context.Context) error
+
+	// Initialize the state for the given log.  Called before any other method
+	// with the log ID.  Idempotent: returns nil if log state already initialized.
+	PrepareLog(context.Context, LogID) error
+
+	// Store log state for retrieval by LoadLogState.
+	StoreLogState(context.Context, LogID, *LogState) error
+
+	// Load log state that was previously stored with StoreLogState.
+	// Returns nil, nil if StoreLogState has not been called yet for this log.
+	LoadLogState(context.Context, LogID) (*LogState, error)
+
+	// Store STH for retrieval by LoadSTHs.  If an STH with the same
+	// timestamp and root hash is already stored, this STH can be ignored.
+	StoreSTH(context.Context, LogID, *ct.SignedTreeHead) error
+
+	// Load all STHs for this log previously stored with StoreSTH.
+	// The returned slice must be sorted by tree size.
+	LoadSTHs(context.Context, LogID) ([]*ct.SignedTreeHead, error)
+
+	// Remove an STH so it is no longer returned by LoadSTHs.
+	RemoveSTH(context.Context, LogID, *ct.SignedTreeHead) error
+
+	// Called when a certificate matching the watch list is discovered.
+	NotifyCert(context.Context, *DiscoveredCert) error
+
+	// Called when certspotter fails to parse a log entry.
+	NotifyMalformedEntry(context.Context, *LogEntry, error) error
+
+	// Called when a health check fails.  The log is nil if the
+	// feailure is not associated with a log.
+	NotifyHealthCheckFailure(context.Context, *loglist.Log, HealthCheckFailure) error
+
+	// Called when a non-fatal error occurs.  The log is nil if the error is
+	// not associated with a log.  Note that most errors are transient, and
+	// certspotter will retry the failed operation later.
+	NotifyError(context.Context, *loglist.Log, error) error
+}

monitor/statedir.go

@@ -76,13 +76,13 @@ func migrateLogStateDirV1(dir string) error {
 		return fmt.Errorf("error unmarshaling %s: %w", treePath, err)
 	}
 
-	stateFile := stateFile{
+	stateFile := LogState{
 		DownloadPosition: &tree,
 		VerifiedPosition: &tree,
 		VerifiedSTH:      &sth,
 		LastSuccess:      time.Now().UTC(),
 	}
-	if stateFile.store(filepath.Join(dir, "state.json")); err != nil {
+	if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil {
 		return err
 	}
 
@@ -145,7 +145,7 @@ func prepareStateDir(stateDir string) error {
 		return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir)
 	}
 
-	for _, subdir := range []string{"certs", "logs"} {
+	for _, subdir := range []string{"certs", "logs", "healthchecks"} {
 		if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) {
 			return err
 		}

monitor/statefile.go

@@ -1,47 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/merkletree"
-	"time"
-)
-
-type stateFile struct {
-	DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
-	VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
-	VerifiedSTH      *ct.SignedTreeHead        `json:"verified_sth"`
-	LastSuccess      time.Time                 `json:"last_success"`
-}
-
-func loadStateFile(filePath string) (*stateFile, error) {
-	fileBytes, err := os.ReadFile(filePath)
-	if err != nil {
-		return nil, err
-	}
-	file := new(stateFile)
-	if err := json.Unmarshal(fileBytes, file); err != nil {
-		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
-	}
-	return file, nil
-}
-
-func (file *stateFile) store(filePath string) error {
-	fileBytes, err := json.Marshal(file)
-	if err != nil {
-		return err
-	}
-	fileBytes = append(fileBytes, '\n')
-	return writeFile(filePath, fileBytes, 0666)
-}

monitor/sthdir.go

@@ -10,16 +10,17 @@
 package monitor
 
 import (
+	"cmp"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"golang.org/x/exp/slices"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"slices"
 	"software.sslmate.com/src/certspotter/ct"
 	"strconv"
 	"strings"
@@ -44,7 +45,7 @@ func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) {
 		}
 		sths = append(sths, sth)
 	}
-	slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) bool { return a.TreeSize < b.TreeSize })
+	slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) int { return cmp.Compare(a.TreeSize, b.TreeSize) })
 	return sths, nil
 }
 
@@ -65,11 +66,7 @@ func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error {
 	if fileExists(filePath) {
 		return nil
 	}
-	fileBytes, err := json.Marshal(sth)
-	if err != nil {
-		return err
-	}
-	return writeFile(filePath, fileBytes, 0666)
+	return writeJSONFile(filePath, sth, 0666)
 }
 
 func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error {

x509.go

@@ -320,7 +320,7 @@ func (tbs *TBSCertificate) ParseSubjectAltNames() ([]SubjectAltName, error) {
 
 	for _, sanExt := range tbs.GetExtension(oidExtensionSubjectAltName) {
 		var err error
-		sans, err = parseSANExtension(sans, sanExt.Value)
+		sans, err = ParseSANExtension(sans, sanExt.Value)
 		if err != nil {
 			return nil, err
 		}
@@ -377,7 +377,7 @@ func (cert *Certificate) ParseSignatureValue() ([]byte, error) {
 	return signatureValue.RightAlign(), nil
 }
 
-func parseSANExtension(sans []SubjectAltName, value []byte) ([]SubjectAltName, error) {
+func ParseSANExtension(sans []SubjectAltName, value []byte) ([]SubjectAltName, error) {
 	var seq asn1.RawValue
 	if rest, err := asn1.Unmarshal(value, &seq); err != nil {
 		return nil, errors.New("failed to parse subjectAltName extension: " + err.Error())