Release v0.21.0

These are important and should not happen very often.

.github/workflows/test.yml

@@ -0,0 +1,35 @@
+name: Test and lint Go Code
+
+on:
+  push:
+  schedule:
+    - cron: '42 9 * * *' # Runs daily at 09:42 UTC
+  workflow_dispatch:     # Allows manual triggering
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Run tests
+        run: CGO_ENABLED=1 go test -race ./...
+
+      - name: Install staticcheck
+        run: go install honnef.co/go/tools/cmd/staticcheck@latest
+
+      - name: Run staticcheck
+        run: staticcheck ./...

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Change Log
 
+## v0.21.0 (2025-07-02)
+- Instead of writing log errors to stderr, save the last 7 days worth in the
+  state directory, and include recent errors in failed health check notifications.
+- Send a meaningful User-Agent string with HTTP requests.
+
+## v0.20.1 (2025-06-19)
+- Add resilience against sendmail hanging indefinitely.
+- Add resilience against hooks which fork and keep stderr open.
+- Upgrade dependencies to latest versions.
+- Minor improvements to error handling, code quality, and efficiency.
+
+## v0.20.0 (2025-06-13)
+- Remove -batch_size option, which is obsolete due to new parallel download system.
+- Only print log errors to stderr if -verbose is specified.
+- Fix bug that could cause unverified STHs to be deleted prematurely.
+- Fail health check if log has never been successfully contacted.
+- Improve -verbose.
+- Improve documentation.
+
+## v0.19.1 (2025-05-07)
+- Fix panic when retrying failed log requests.
+- Properly log failed log requests.
+
+## v0.19.0 (2025-05-07) (RETRACTED)
+- Add support for static-ct-api logs, the next generation of CT logs.
+- Add support for downloading entries in parallel, to avoid backlogs when
+  monitoring fast-growing logs.
+- Use $EMAIL environment variable to determine sender of emails.
+- Remove submitct command.
+- Make certspotter more resilient to system crashes such as power failures.
+
+## v0.18.0 (2023-11-13)
+- Fix bug with downloading entries that did not materialize in practice
+  with any of the current logs.
+- Include `Message-ID` and `Date` in outbound emails.
+
 ## v0.17.0 (2023-10-26)
 - Allow sendmail path to be configured with `$SENDMAIL_PATH`.
 - Minor improvements to documentation, efficiency.

README.md

@@ -24,7 +24,7 @@ You can use Cert Spotter to detect:
 
 ## Quickstart
 
-Cert Spotter requires Go version 1.19 or higher.
+The following instructions require you to have [Go version 1.21 or higher](https://go.dev/dl/) installed.
 
 1. Install the certspotter command using the `go` command:
 
@@ -46,7 +46,7 @@ Cert Spotter requires Go version 1.19 or higher.
 
 4. Configure your system to run `certspotter` as a daemon.  You may want to specify
    the `-start_at_end` command line option to tell certspotter to start monitoring
-   logs at the end instead of the beginning.  This saves significant bandwidth, but
+   new logs at the end instead of the beginning.  This saves significant bandwidth, but
    you won't be notified about certificates which were logged before you started
    using certspotter.
 
@@ -64,6 +64,8 @@ Cert Spotter.  By default, Google Chrome and Apple only accept certificates that
 are logged, so any certificate that works in Chrome or Safari will be detected
 by Cert Spotter.
 
+Cert Spotter will monitor both traditional RFC6962 logs, and modern static-ct-api logs.
+
 ## Security
 
 Cert Spotter assumes an adversarial model in which an attacker produces
@@ -100,6 +102,6 @@ to ensure the log is presenting a single view.
 
 ## Copyright
 
-Copyright © 2016-2023 Opsmate, Inc.
+Copyright © 2016-2025 Opsmate, Inc.
 
 Licensed under the [Mozilla Public License Version 2.0](LICENSE).

asn1.go

@@ -46,7 +46,7 @@ func decodeASN1String(value *asn1.RawValue) (string, error) {
 		if value.Tag == 12 {
 			// UTF8String
 			if !utf8.Valid(value.Bytes) {
-				return "", errors.New("Malformed UTF8String")
+				return "", errors.New("malformed UTF8String")
 			}
 			return string(value.Bytes), nil
 		} else if value.Tag == 19 || value.Tag == 22 || value.Tag == 20 || value.Tag == 26 {
@@ -74,5 +74,5 @@ func decodeASN1String(value *asn1.RawValue) (string, error) {
 			return stringFromUint32Slice(runes), nil
 		}
 	}
-	return "", errors.New("Not a string")
+	return "", errors.New("not a string")
 }

asn1time.go

@@ -253,5 +253,5 @@ func decodeASN1Time(value *asn1.RawValue) (time.Time, error) {
 			return parseGeneralizedTime(value.Bytes)
 		}
 	}
-	return time.Time{}, errors.New("Not a time value")
+	return time.Time{}, errors.New("not a time value")
 }

cmd/certspotter/main.go

@@ -16,7 +16,6 @@ import (
 	"flag"
 	"fmt"
 	"io/fs"
-	insecurerand "math/rand"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -26,43 +25,23 @@ import (
 	"syscall"
 	"time"
 
+	"software.sslmate.com/src/certspotter/ctclient"
 	"software.sslmate.com/src/certspotter/loglist"
 	"software.sslmate.com/src/certspotter/monitor"
 )
 
 var programName = os.Args[0]
-var Version = ""
+var Version = "unknown"
+var Source = "unknown"
 
 const defaultLogList = "https://loglist.certspotter.org/monitor.json"
 
-func certspotterVersion() string {
+func certspotterVersion() (string, string) {
-	if Version != "" {
+	if buildinfo, ok := debug.ReadBuildInfo(); ok && strings.HasPrefix(buildinfo.Main.Version, "v") {
-		return Version + "?"
+		return strings.TrimPrefix(buildinfo.Main.Version, "v"), buildinfo.Main.Path
+	} else {
+		return Version, Source
 	}
-	info, ok := debug.ReadBuildInfo()
-	if !ok {
-		return "unknown"
-	}
-	if strings.HasPrefix(info.Main.Version, "v") {
-		return info.Main.Version
-	}
-	var vcs, vcsRevision, vcsModified string
-	for _, s := range info.Settings {
-		switch s.Key {
-		case "vcs":
-			vcs = s.Value
-		case "vcs.revision":
-			vcsRevision = s.Value
-		case "vcs.modified":
-			vcsModified = s.Value
-		}
-	}
-	if vcs == "git" && vcsRevision != "" && vcsModified == "true" {
-		return vcsRevision + "+"
-	} else if vcs == "git" && vcsRevision != "" {
-		return vcsRevision
-	}
-	return "unknown"
 }
 
 func fileExists(filename string) bool {
@@ -90,6 +69,13 @@ func defaultConfigDir() string {
 		return filepath.Join(homedir(), ".certspotter")
 	}
 }
+func defaultCacheDir() string {
+	userCacheDir, err := os.UserCacheDir()
+	if err != nil {
+		panic(fmt.Errorf("unable to determine user cache directory: %w", err))
+	}
+	return filepath.Join(userCacheDir, "certspotter")
+}
 func defaultWatchListPath() string {
 	return filepath.Join(defaultConfigDir(), "watchlist")
 }
@@ -152,12 +138,13 @@ func appendFunc(slice *[]string) func(string) error {
 }
 
 func main() {
-	insecurerand.Seed(time.Now().UnixNano()) // TODO: remove after upgrading to Go 1.20
+	version, source := certspotterVersion()
 
-	loglist.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s)", certspotterVersion(), runtime.Version(), runtime.GOOS, runtime.GOARCH)
+	ctclient.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s; %s; +https://github.com/SSLMate/certspotter)", version, source, runtime.Version(), runtime.GOOS, runtime.GOARCH)
+	loglist.UserAgent = ctclient.UserAgent
 
 	var flags struct {
-		batchSize   int // TODO-4: respect this option
+		batchSize   bool
 		email       []string
 		healthcheck time.Duration
 		logs        string
@@ -170,22 +157,26 @@ func main() {
 		version     bool
 		watchlist   string
 	}
-	flag.IntVar(&flags.batchSize, "batch_size", 1000, "Max number of entries to request per call to get-entries (advanced)")
+	flag.Func("batch_size", "Obsolete; do not use", func(string) error { flags.batchSize = true; return nil }) // TODO: remove in 0.21.0
 	flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email))
 	flag.DurationVar(&flags.healthcheck, "healthcheck", 24*time.Hour, "How frequently to perform a health check")
 	flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
 	flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory")
 	flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered")
-	flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring logs from the end rather than the beginning (saves considerable bandwidth)")
+	flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring new logs from the end rather than the beginning (saves considerable bandwidth)")
 	flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates")
 	flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout")
-	flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose")
+	flag.BoolVar(&flags.verbose, "verbose", false, "Print detailed information about certspotter's operation to stderr")
 	flag.BoolVar(&flags.version, "version", false, "Print version and exit")
 	flag.StringVar(&flags.watchlist, "watchlist", defaultWatchListPathIfExists(), "File containing domain names to watch")
 	flag.Parse()
 
+	if flags.batchSize {
+		fmt.Fprintf(os.Stderr, "%s: -batch_size is obsolete; please remove it from your command line\n", programName)
+		os.Exit(2)
+	}
 	if flags.version {
-		fmt.Fprintf(os.Stdout, "certspotter version %s\n", certspotterVersion())
+		fmt.Fprintf(os.Stdout, "certspotter version %s (%s)\n", version, source)
 		os.Exit(0)
 	}
 	if flags.watchlist == "" {
@@ -193,38 +184,37 @@ func main() {
 		os.Exit(2)
 	}
 
+	fsstate := &monitor.FilesystemState{
+		StateDir:  flags.stateDir,
+		CacheDir:  defaultCacheDir(),
+		SaveCerts: !flags.noSave,
+		Script:    flags.script,
+		ScriptDir: defaultScriptDir(),
+		Email:     flags.email,
+		Stdout:    flags.stdout,
+	}
 	config := &monitor.Config{
 		LogListSource:       flags.logs,
-		StateDir:            flags.stateDir,
+		State:               fsstate,
-		SaveCerts:           !flags.noSave,
 		StartAtEnd:          flags.startAtEnd,
 		Verbose:             flags.verbose,
-		Script:              flags.script,
-		ScriptDir:           defaultScriptDir(),
-		SendmailPath:        "/usr/sbin/sendmail",
-		Email:               flags.email,
-		Stdout:              flags.stdout,
 		HealthCheckInterval: flags.healthcheck,
 	}
 
-	if envVar := os.Getenv("SENDMAIL_PATH"); envVar != "" {
-		config.SendmailPath = envVar
-	}
-
 	emailFileExists := false
 	if emailRecipients, err := readEmailFile(defaultEmailFile()); err == nil {
 		emailFileExists = true
-		config.Email = append(config.Email, emailRecipients...)
+		fsstate.Email = append(fsstate.Email, emailRecipients...)
 	} else if !errors.Is(err, fs.ErrNotExist) {
 		fmt.Fprintf(os.Stderr, "%s: error reading email recipients file %q: %s\n", programName, defaultEmailFile(), err)
 		os.Exit(1)
 	}
 
-	if len(config.Email) == 0 && !emailFileExists && config.Script == "" && !fileExists(config.ScriptDir) && config.Stdout == false {
+	if len(fsstate.Email) == 0 && !emailFileExists && fsstate.Script == "" && !fileExists(fsstate.ScriptDir) && fsstate.Stdout == false {
 		fmt.Fprintf(os.Stderr, "%s: no notification methods were specified\n", programName)
 		fmt.Fprintf(os.Stderr, "Please specify at least one of the following notification methods:\n")
 		fmt.Fprintf(os.Stderr, " - Place one or more email addresses in %s (one address per line)\n", defaultEmailFile())
-		fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", config.ScriptDir)
+		fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", fsstate.ScriptDir)
 		fmt.Fprintf(os.Stderr, " - Specify an email address using the -email flag\n")
 		fmt.Fprintf(os.Stderr, " - Specify the path to an executable script using the -script flag\n")
 		fmt.Fprintf(os.Stderr, " - Specify the -stdout flag\n")
@@ -250,7 +240,25 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()
 
-	if err := monitor.Run(ctx, config); err != nil && !errors.Is(err, context.Canceled) {
+	go func() {
+		ticker := time.NewTicker(24*time.Hour)
+		defer ticker.Stop()
+		for {
+			fsstate.PruneOldErrors()
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+
+	if err := monitor.Run(ctx, config); ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
+		if flags.verbose {
+			fmt.Fprintf(os.Stderr, "%s: exiting due to SIGINT or SIGTERM\n", programName)
+		}
+		os.Exit(0)
+	} else {
 		fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err)
 		os.Exit(1)
 	}

cmd/submitct/.gitignore

@@ -1 +0,0 @@
-/submitct

cmd/submitct/main.go

@@ -1,237 +0,0 @@
-// Copyright (C) 2017 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package main
-
-import (
-	"software.sslmate.com/src/certspotter"
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/ct/client"
-	"software.sslmate.com/src/certspotter/loglist"
-
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"crypto/x509"
-	"encoding/pem"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-const defaultLogList = "https://loglist.certspotter.org/submit.json"
-
-var verbose = flag.Bool("v", false, "Enable verbose output")
-var logsURL = flag.String("logs", defaultLogList, "File path or URL of JSON list of logs to submit to")
-
-type Certificate struct {
-	Subject []byte
-	Issuer  []byte
-	Raw     []byte
-	Expiration time.Time
-}
-
-func (cert *Certificate) Fingerprint() [32]byte {
-	return sha256.Sum256(cert.Raw)
-}
-
-func (cert *Certificate) CommonName() string {
-	subject, err := certspotter.ParseRDNSequence(cert.Subject)
-	if err != nil {
-		return "???"
-	}
-	cns, err := subject.ParseCNs()
-	if err != nil || len(cns) == 0 {
-		return "???"
-	}
-	return cns[0]
-}
-
-func parseCertificate(data []byte) (*Certificate, error) {
-	crt, err := certspotter.ParseCertificate(data)
-	if err != nil {
-		return nil, err
-	}
-
-	tbs, err := crt.ParseTBSCertificate()
-	if err != nil {
-		return nil, err
-	}
-
-	validity, err := tbs.ParseValidity()
-	if err != nil {
-		return nil, err
-	}
-
-	return &Certificate{
-		Subject: tbs.Subject.FullBytes,
-		Issuer:  tbs.Issuer.FullBytes,
-		Raw:     data,
-		Expiration: validity.NotAfter,
-	}, nil
-}
-
-type Chain []*Certificate
-
-func (c Chain) GetRawCerts() [][]byte {
-	rawCerts := make([][]byte, len(c))
-	for i := range c {
-		rawCerts[i] = c[i].Raw
-	}
-	return rawCerts
-}
-
-type CertificateBunch struct {
-	byFingerprint map[[32]byte]*Certificate
-	bySubject     map[[32]byte]*Certificate
-}
-
-func MakeCertificateBunch() CertificateBunch {
-	return CertificateBunch{
-		byFingerprint: make(map[[32]byte]*Certificate),
-		bySubject:     make(map[[32]byte]*Certificate),
-	}
-}
-
-func (certs *CertificateBunch) Add(cert *Certificate) {
-	certs.byFingerprint[cert.Fingerprint()] = cert
-	certs.bySubject[sha256.Sum256(cert.Subject)] = cert
-}
-
-func (certs *CertificateBunch) FindBySubject(subject []byte) *Certificate {
-	return certs.bySubject[sha256.Sum256(subject)]
-}
-
-type Log struct {
-	*loglist.Log
-	*ct.SignatureVerifier
-	*client.LogClient
-}
-
-func (ctlog *Log) SubmitChain(chain Chain) (*ct.SignedCertificateTimestamp, error) {
-	rawCerts := chain.GetRawCerts()
-	sct, err := ctlog.AddChain(context.Background(), rawCerts)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := certspotter.VerifyX509SCT(sct, rawCerts[0], ctlog.SignatureVerifier); err != nil {
-		return nil, fmt.Errorf("Bad SCT signature: %s", err)
-	}
-	return sct, nil
-}
-
-func buildChain(cert *Certificate, certs *CertificateBunch) Chain {
-	chain := make([]*Certificate, 0)
-	for len(chain) < 16 && cert != nil && !bytes.Equal(cert.Subject, cert.Issuer) {
-		chain = append(chain, cert)
-		cert = certs.FindBySubject(cert.Issuer)
-	}
-	return chain
-}
-
-func main() {
-	flag.Parse()
-	log.SetPrefix("submitct: ")
-
-	certsPem, err := io.ReadAll(os.Stdin)
-	if err != nil {
-		log.Fatalf("Error reading stdin: %s", err)
-	}
-
-	list, err := loglist.Load(context.Background(), *logsURL)
-	if err != nil {
-		log.Fatalf("Error loading log list: %s", err)
-	}
-
-	var logs []Log
-	for _, ctlog := range list.AllLogs() {
-		pubkey, err := x509.ParsePKIXPublicKey(ctlog.Key)
-		if err != nil {
-			log.Fatalf("%s: Failed to parse log public key: %s", ctlog.URL, err)
-		}
-		verifier, err := ct.NewSignatureVerifier(pubkey)
-		if err != nil {
-			log.Fatalf("%s: Failed to create signature verifier for log: %s", ctlog.URL, err)
-		}
-		logs = append(logs, Log{
-			Log: ctlog,
-			SignatureVerifier: verifier,
-			LogClient: client.New(strings.TrimRight(ctlog.URL, "/")),
-		})
-	}
-
-	certs := MakeCertificateBunch()
-	var parseErrors uint32
-	var submitErrors uint32
-
-	for len(certsPem) > 0 {
-		var pemBlock *pem.Block
-		pemBlock, certsPem = pem.Decode(certsPem)
-		if pemBlock == nil {
-			log.Fatalf("Invalid PEM read from stdin")
-		}
-		if pemBlock.Type != "CERTIFICATE" {
-			log.Printf("Ignoring non-certificate read from stdin")
-			continue
-		}
-
-		cert, err := parseCertificate(pemBlock.Bytes)
-		if err != nil {
-			log.Printf("Ignoring un-parseable certificate read from stdin: %s", err)
-			parseErrors++
-			continue
-		}
-		certs.Add(cert)
-	}
-
-	wg := sync.WaitGroup{}
-	for fingerprint, cert := range certs.byFingerprint {
-		cn := cert.CommonName()
-		chain := buildChain(cert, &certs)
-		if len(chain) == 0 {
-			continue
-		}
-		for _, ctlog := range logs {
-			if !ctlog.AcceptsExpiration(chain[0].Expiration) {
-				continue
-			}
-			wg.Add(1)
-			go func(fingerprint [32]byte, ctlog Log) {
-				sct, err := ctlog.SubmitChain(chain)
-				if err != nil {
-					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.URL, err)
-					atomic.AddUint32(&submitErrors, 1)
-				} else if *verbose {
-					timestamp := time.Unix(int64(sct.Timestamp)/1000, int64(sct.Timestamp%1000)*1000000)
-					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.URL, timestamp)
-				}
-				wg.Done()
-			}(fingerprint, ctlog)
-		}
-	}
-	wg.Wait()
-
-	exitStatus := 0
-	if parseErrors > 0 {
-		log.Printf("%d certificates failed to parse and were ignored", parseErrors)
-		exitStatus |= 4
-	}
-	if submitErrors > 0 {
-		log.Printf("%d submission errors occurred", submitErrors)
-		exitStatus |= 8
-	}
-	os.Exit(exitStatus)
-}

ct/AUTHORS

@@ -1,24 +0,0 @@
-# This is the official list of benchmark authors for copyright purposes.
-# This file is distinct from the CONTRIBUTORS files.
-# See the latter for an explanation.
-#
-# Names should be added to this file as:
-#	Name or Organization <email address>
-# The email address is not required for organizations.
-#
-# Please keep the list sorted.
-
-Comodo CA Limited
-Ed Maste <emaste@freebsd.org>
-Fiaz Hossain <fiaz.hossain@salesforce.com>
-Google Inc.
-Jeff Trawick <trawick@gmail.com>
-Katriel Cohn-Gordon <katriel.cohn-gordon@cybersecurity.ox.ac.uk>
-Mark Schloesser <ms@mwcollect.org>
-NORDUnet A/S
-Nicholas Galbreath <nickg@client9.com>
-Oliver Weidner <Oliver.Weidner@gmail.com>
-Ruslan Kovalov <ruslan.kovalyov@gmail.com>
-Venafi, Inc.
-Vladimir Rutsky <vladimir@rutsky.org>
-Ximin Luo <infinity0@gmx.com>

ct/LICENSE

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

ct/README

@@ -1,4 +0,0 @@
-The code in this directory is based on Google's Certificiate Transparency Go library
-(originally at <https://github.com/google/certificate-transparency/tree/master/go>;
-now at <https://github.com/google/certificate-transparency-go>).
-See AUTHORS for the copyright holders, and LICENSE for the license.

ct/client/logclient.go

@@ -1,416 +0,0 @@
-// Package client is a CT log client implementation and contains types and code
-// for interacting with RFC6962-compliant CT Log instances.
-// See http://tools.ietf.org/html/rfc6962 for details
-package client
-
-import (
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"crypto/tls"
-	"encoding/base64"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	insecurerand "math/rand"
-	"net/http"
-	"net/url"
-	"strconv"
-	"time"
-
-	"software.sslmate.com/src/certspotter/ct"
-)
-
-const (
-	baseRetryDelay = 1 * time.Second
-	maxRetryDelay  = 120 * time.Second
-	maxRetries     = 10
-)
-
-func isRetryableStatusCode(code int) bool {
-	return code/100 == 5 || code == http.StatusTooManyRequests
-}
-
-func randomDuration(min, max time.Duration) time.Duration {
-	return min + time.Duration(insecurerand.Int63n(int64(max)-int64(min)+1))
-}
-
-func getRetryAfter(resp *http.Response) (time.Duration, bool) {
-	if resp == nil {
-		return 0, false
-	}
-	seconds, err := strconv.ParseUint(resp.Header.Get("Retry-After"), 10, 16)
-	if err != nil {
-		return 0, false
-	}
-	return time.Duration(seconds) * time.Second, true
-}
-
-func sleep(ctx context.Context, duration time.Duration) {
-	timer := time.NewTimer(duration)
-	defer timer.Stop()
-	select {
-	case <-ctx.Done():
-	case <-timer.C:
-	}
-}
-
-// URI paths for CT Log endpoints
-const (
-	GetSTHPath            = "/ct/v1/get-sth"
-	GetEntriesPath        = "/ct/v1/get-entries"
-	GetSTHConsistencyPath = "/ct/v1/get-sth-consistency"
-	GetProofByHashPath    = "/ct/v1/get-proof-by-hash"
-	AddChainPath          = "/ct/v1/add-chain"
-)
-
-// LogClient represents a client for a given CT Log instance
-type LogClient struct {
-	uri        string       // the base URI of the log. e.g. http://ct.googleapis/pilot
-	httpClient *http.Client // used to interact with the log via HTTP
-	verifier   *ct.SignatureVerifier // if non-nil, used to verify STH signatures
-}
-
-//////////////////////////////////////////////////////////////////////////////////
-// JSON structures follow.
-// These represent the structures returned by the CT Log server.
-//////////////////////////////////////////////////////////////////////////////////
-
-// getSTHResponse represents the JSON response to the get-sth CT method
-type getSTHResponse struct {
-	TreeSize          uint64 `json:"tree_size"`           // Number of certs in the current tree
-	Timestamp         uint64 `json:"timestamp"`           // Time that the tree was created
-	SHA256RootHash    []byte `json:"sha256_root_hash"`    // Root hash of the tree
-	TreeHeadSignature []byte `json:"tree_head_signature"` // Log signature for this STH
-}
-
-// base64LeafEntry represents a Base64 encoded leaf entry
-type base64LeafEntry struct {
-	LeafInput []byte `json:"leaf_input"`
-	ExtraData []byte `json:"extra_data"`
-}
-
-// getEntriesReponse represents the JSON response to the CT get-entries method
-type getEntriesResponse struct {
-	Entries []base64LeafEntry `json:"entries"` // the list of returned entries
-}
-
-// getConsistencyProofResponse represents the JSON response to the CT get-consistency-proof method
-type getConsistencyProofResponse struct {
-	Consistency [][]byte `json:"consistency"`
-}
-
-// getAuditProofResponse represents the JSON response to the CT get-proof-by-hash method
-type getAuditProofResponse struct {
-	LeafIndex uint64   `json:"leaf_index"`
-	AuditPath [][]byte `json:"audit_path"`
-}
-
-type addChainRequest struct {
-	Chain [][]byte `json:"chain"`
-}
-
-type addChainResponse struct {
-	SCTVersion uint8  `json:"sct_version"`
-	ID         []byte `json:"id"`
-	Timestamp  uint64 `json:"timestamp"`
-	Extensions []byte `json:"extensions"`
-	Signature  []byte `json:"signature"`
-}
-
-// New constructs a new LogClient instance.
-// |uri| is the base URI of the CT log instance to interact with, e.g.
-// http://ct.googleapis.com/pilot
-func New(uri string) *LogClient {
-	return NewWithVerifier(uri, nil)
-}
-
-func NewWithVerifier(uri string, verifier *ct.SignatureVerifier) *LogClient {
-	var c LogClient
-	c.uri = uri
-	c.verifier = verifier
-	transport := &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		TLSHandshakeTimeout:   15 * time.Second,
-		ResponseHeaderTimeout: 30 * time.Second,
-		MaxIdleConnsPerHost:   10,
-		DisableKeepAlives:     false,
-		MaxIdleConns:          100,
-		IdleConnTimeout:       15 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig: &tls.Config{
-			// We have to disable TLS certificate validation because because several logs
-			// (WoSign, StartCom, GDCA) use certificates that are not widely trusted.
-			// Since we verify that every response we receive from the log is signed
-			// by the log's CT public key (either directly, or indirectly via the Merkle Tree),
-			// TLS certificate validation is not actually necessary.  (We don't want to ship
-			// our own trust store because that adds undesired complexity and would require
-			// updating should a log ever change to a different CA.)
-			InsecureSkipVerify: true,
-		},
-	}
-	c.httpClient = &http.Client{Timeout: 60 * time.Second, Transport: transport}
-	return &c
-}
-
-func (c *LogClient) fetchAndParse(ctx context.Context, uri string, respBody interface{}) error {
-	return c.doAndParse(ctx, "GET", uri, nil, respBody)
-}
-
-func (c *LogClient) postAndParse(ctx context.Context, uri string, body interface{}, respBody interface{}) error {
-	return c.doAndParse(ctx, "POST", uri, body, respBody)
-}
-
-func (c *LogClient) makeRequest(ctx context.Context, method string, uri string, body interface{}) (*http.Request, error) {
-	if body == nil {
-		return http.NewRequestWithContext(ctx, method, uri, nil)
-	} else {
-		bodyBytes, err := json.Marshal(body)
-		if err != nil {
-			return nil, err
-		}
-		req, err := http.NewRequestWithContext(ctx, method, uri, bytes.NewReader(bodyBytes))
-		if err != nil {
-			return nil, err
-		}
-		req.Header.Set("Content-Type", "application/json")
-		return req, nil
-	}
-}
-
-func (c *LogClient) doAndParse(ctx context.Context, method string, uri string, reqBody interface{}, respBody interface{}) error {
-	numRetries := 0
-retry:
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
-	req, err := c.makeRequest(ctx, method, uri, reqBody)
-	if err != nil {
-		return fmt.Errorf("%s %s: error creating request: %w", method, uri, err)
-	}
-	req.Header.Set("User-Agent", "") // Don't send a User-Agent to make life harder for malicious logs
-	resp, err := c.httpClient.Do(req)
-	if err != nil {
-		if c.shouldRetry(ctx, numRetries, nil) {
-			numRetries++
-			goto retry
-		}
-		return err
-	}
-	respBodyBytes, err := io.ReadAll(resp.Body)
-	resp.Body.Close()
-	if err != nil {
-		if c.shouldRetry(ctx, numRetries, nil) {
-			numRetries++
-			goto retry
-		}
-		return fmt.Errorf("%s %s: error reading response: %w", method, uri, err)
-	}
-	if resp.StatusCode/100 != 2 {
-		if c.shouldRetry(ctx, numRetries, resp) {
-			numRetries++
-			goto retry
-		}
-		return fmt.Errorf("%s %s: %s (%s)", method, uri, resp.Status, string(respBodyBytes))
-	}
-	if err := json.Unmarshal(respBodyBytes, respBody); err != nil {
-		return fmt.Errorf("%s %s: error parsing response JSON: %w", method, uri, err)
-	}
-	return nil
-}
-
-func (c *LogClient) shouldRetry(ctx context.Context, numRetries int, resp *http.Response) bool {
-	if numRetries == maxRetries {
-		return false
-	}
-
-	if resp != nil && !isRetryableStatusCode(resp.StatusCode) {
-		return false
-	}
-
-	var delay time.Duration
-	if retryAfter, hasRetryAfter := getRetryAfter(resp); hasRetryAfter {
-		delay = retryAfter
-	} else {
-		delay = baseRetryDelay * (1 << numRetries)
-		if delay > maxRetryDelay {
-			delay = maxRetryDelay
-		}
-		delay += randomDuration(0, delay/2)
-	}
-
-	if deadline, hasDeadline := ctx.Deadline(); hasDeadline && time.Now().Add(delay).After(deadline) {
-		return false
-	}
-
-	sleep(ctx, delay)
-	return true
-}
-
-// GetSTH retrieves the current STH from the log.
-// Returns a populated SignedTreeHead, or a non-nil error.
-func (c *LogClient) GetSTH(ctx context.Context) (sth *ct.SignedTreeHead, err error) {
-	var resp getSTHResponse
-	if err = c.fetchAndParse(ctx, c.uri+GetSTHPath, &resp); err != nil {
-		return
-	}
-	sth = &ct.SignedTreeHead{
-		TreeSize:  resp.TreeSize,
-		Timestamp: resp.Timestamp,
-	}
-
-	if len(resp.SHA256RootHash) != sha256.Size {
-		return nil, fmt.Errorf("STH returned by server has invalid sha256_root_hash (expected length %d got %d)", sha256.Size, len(resp.SHA256RootHash))
-	}
-	copy(sth.SHA256RootHash[:], resp.SHA256RootHash)
-
-	ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.TreeHeadSignature))
-	if err != nil {
-		return nil, err
-	}
-	sth.TreeHeadSignature = *ds
-	if c.verifier != nil {
-		if err := c.verifier.VerifySTHSignature(*sth); err != nil {
-			return nil, fmt.Errorf("STH returned by server has invalid signature: %w", err)
-		}
-	}
-	return
-}
-
-type GetEntriesItem struct {
-	LeafInput []byte `json:"leaf_input"`
-	ExtraData []byte `json:"extra_data"`
-}
-
-// Retrieve the entries in the sequence [start, end] from the CT log server.
-// If error is nil, at least one entry is returned, and no excess entries are returned.
-// Fewer entries than requested may be returned.
-func (c *LogClient) GetRawEntries(ctx context.Context, start, end uint64) ([]GetEntriesItem, error) {
-	if end < start {
-		panic("LogClient.GetRawEntries: end < start")
-	}
-	var response struct {
-		Entries []GetEntriesItem `json:"entries"`
-	}
-	uri := fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end)
-	err := c.fetchAndParse(ctx, uri, &response)
-	if err != nil {
-		return nil, err
-	}
-	if len(response.Entries) == 0 {
-		return nil, fmt.Errorf("GET %s: log server returned an empty get-entries response", uri)
-	}
-	if uint64(len(response.Entries)) > end-start+1 {
-		return nil, fmt.Errorf("GET %s: log server returned a get-entries response with extraneous entries", uri)
-	}
-	return response.Entries, nil
-}
-
-// GetEntries attempts to retrieve the entries in the sequence [|start|, |end|] from the CT
-// log server. (see section 4.6.)
-// Returns a slice of LeafInputs or a non-nil error.
-func (c *LogClient) GetEntries(ctx context.Context, start, end int64) ([]ct.LogEntry, error) {
-	if end < 0 {
-		return nil, errors.New("GetEntries: end should be >= 0")
-	}
-	if end < start {
-		return nil, errors.New("GetEntries: start should be <= end")
-	}
-	var resp getEntriesResponse
-	err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end), &resp)
-	if err != nil {
-		return nil, err
-	}
-	entries := make([]ct.LogEntry, len(resp.Entries))
-	for index, entry := range resp.Entries {
-		leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewBuffer(entry.LeafInput))
-		if err != nil {
-			return nil, fmt.Errorf("Reading Merkle Tree Leaf at index %d failed: %s", start+int64(index), err)
-		}
-		entries[index].LeafBytes = entry.LeafInput
-		entries[index].Leaf = *leaf
-
-		var chain []ct.ASN1Cert
-		switch leaf.TimestampedEntry.EntryType {
-		case ct.X509LogEntryType:
-			chain, err = ct.UnmarshalX509ChainArray(entry.ExtraData)
-
-		case ct.PrecertLogEntryType:
-			chain, err = ct.UnmarshalPrecertChainArray(entry.ExtraData)
-
-		default:
-			return nil, fmt.Errorf("Unknown entry type at index %d: %v", start+int64(index), leaf.TimestampedEntry.EntryType)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("Parsing entry of type %d at index %d failed: %s", leaf.TimestampedEntry.EntryType, start+int64(index), err)
-		}
-		entries[index].Chain = chain
-		entries[index].Index = start + int64(index)
-	}
-	return entries, nil
-}
-
-// GetConsistencyProof retrieves a Merkle Consistency Proof between two STHs (|first| and |second|)
-// from the log.  Returns a slice of MerkleTreeNodes (a ct.ConsistencyProof) or a non-nil error.
-func (c *LogClient) GetConsistencyProof(ctx context.Context, first, second int64) (ct.ConsistencyProof, error) {
-	if second < 0 {
-		return nil, errors.New("GetConsistencyProof: second should be >= 0")
-	}
-	if second < first {
-		return nil, errors.New("GetConsistencyProof: first should be <= second")
-	}
-	var resp getConsistencyProofResponse
-	err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?first=%d&second=%d", c.uri, GetSTHConsistencyPath, first, second), &resp)
-	if err != nil {
-		return nil, err
-	}
-	nodes := make([]ct.MerkleTreeNode, len(resp.Consistency))
-	for index, nodeBytes := range resp.Consistency {
-		nodes[index] = nodeBytes
-	}
-	return nodes, nil
-}
-
-// GetAuditProof retrieves a Merkle Audit Proof (aka Inclusion Proof) for the given
-// |hash| based on the STH at |treeSize| from the log.  Returns a slice of MerkleTreeNodes
-// and the index of the leaf.
-func (c *LogClient) GetAuditProof(ctx context.Context, hash ct.MerkleTreeNode, treeSize uint64) (ct.AuditPath, uint64, error) {
-	var resp getAuditProofResponse
-	err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?hash=%s&tree_size=%d", c.uri, GetProofByHashPath, url.QueryEscape(base64.StdEncoding.EncodeToString(hash)), treeSize), &resp)
-	if err != nil {
-		return nil, 0, err
-	}
-	path := make([]ct.MerkleTreeNode, len(resp.AuditPath))
-	for index, nodeBytes := range resp.AuditPath {
-		path[index] = nodeBytes
-	}
-	return path, resp.LeafIndex, nil
-}
-
-func (c *LogClient) AddChain(ctx context.Context, chain [][]byte) (*ct.SignedCertificateTimestamp, error) {
-	req := addChainRequest{Chain: chain}
-
-	var resp addChainResponse
-	if err := c.postAndParse(ctx, c.uri+AddChainPath, &req, &resp); err != nil {
-		return nil, err
-	}
-
-	sct := &ct.SignedCertificateTimestamp{
-		SCTVersion: ct.Version(resp.SCTVersion),
-		Timestamp:  resp.Timestamp,
-		Extensions: resp.Extensions,
-	}
-
-	if len(resp.ID) != sha256.Size {
-		return nil, fmt.Errorf("SCT returned by server has invalid id (expected length %d got %d)", sha256.Size, len(resp.ID))
-	}
-	copy(sct.LogID[:], resp.ID)
-
-	ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.Signature))
-	if err != nil {
-		return nil, err
-	}
-	sct.Signature = *ds
-	return sct, nil
-}

ct/serialization.go

@@ -1,462 +0,0 @@
-package ct
-
-import (
-	"bytes"
-	"container/list"
-	"crypto"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"io"
-)
-
-// Variable size structure prefix-header byte lengths
-const (
-	CertificateLengthBytes      = 3
-	PreCertificateLengthBytes   = 3
-	ExtensionsLengthBytes       = 2
-	CertificateChainLengthBytes = 3
-	SignatureLengthBytes        = 2
-)
-
-// Max lengths
-const (
-	MaxCertificateLength = (1 << 24) - 1
-	MaxExtensionsLength  = (1 << 16) - 1
-)
-
-func writeUint(w io.Writer, value uint64, numBytes int) error {
-	buf := make([]uint8, numBytes)
-	for i := 0; i < numBytes; i++ {
-		buf[numBytes-i-1] = uint8(value & 0xff)
-		value >>= 8
-	}
-	if value != 0 {
-		return errors.New("numBytes was insufficiently large to represent value")
-	}
-	if _, err := w.Write(buf); err != nil {
-		return err
-	}
-	return nil
-}
-
-func writeVarBytes(w io.Writer, value []byte, numLenBytes int) error {
-	if err := writeUint(w, uint64(len(value)), numLenBytes); err != nil {
-		return err
-	}
-	if _, err := w.Write(value); err != nil {
-		return err
-	}
-	return nil
-}
-
-func readUint(r io.Reader, numBytes int) (uint64, error) {
-	var l uint64
-	for i := 0; i < numBytes; i++ {
-		l <<= 8
-		var t uint8
-		if err := binary.Read(r, binary.BigEndian, &t); err != nil {
-			return 0, err
-		}
-		l |= uint64(t)
-	}
-	return l, nil
-}
-
-// Reads a variable length array of bytes from |r|. |numLenBytes| specifies the
-// number of (BigEndian) prefix-bytes which contain the length of the actual
-// array data bytes that follow.
-// Allocates an array to hold the contents and returns a slice view into it if
-// the read was successful, or an error otherwise.
-func readVarBytes(r io.Reader, numLenBytes int) ([]byte, error) {
-	switch {
-	case numLenBytes > 8:
-		return nil, fmt.Errorf("numLenBytes too large (%d)", numLenBytes)
-	case numLenBytes == 0:
-		return nil, errors.New("numLenBytes should be > 0")
-	}
-	l, err := readUint(r, numLenBytes)
-	if err != nil {
-		return nil, err
-	}
-	data := make([]byte, l)
-	if n, err := io.ReadFull(r, data); err != nil {
-		if err == io.EOF || err == io.ErrUnexpectedEOF {
-			return nil, fmt.Errorf("short read: expected %d but got %d", l, n)
-		}
-		return nil, err
-	}
-	return data, nil
-}
-
-// Reads a list of ASN1Cert types from |r|
-func readASN1CertList(r io.Reader, totalLenBytes int, elementLenBytes int) ([]ASN1Cert, error) {
-	listBytes, err := readVarBytes(r, totalLenBytes)
-	if err != nil {
-		return []ASN1Cert{}, err
-	}
-	list := list.New()
-	listReader := bytes.NewReader(listBytes)
-	var entry []byte
-	for err == nil {
-		entry, err = readVarBytes(listReader, elementLenBytes)
-		if err != nil {
-			if err != io.EOF {
-				return []ASN1Cert{}, err
-			}
-		} else {
-			list.PushBack(entry)
-		}
-	}
-	ret := make([]ASN1Cert, list.Len())
-	i := 0
-	for e := list.Front(); e != nil; e = e.Next() {
-		ret[i] = e.Value.([]byte)
-		i++
-	}
-	return ret, nil
-}
-
-// ReadTimestampedEntryInto parses the byte-stream representation of a
-// TimestampedEntry from |r| and populates the struct |t| with the data.  See
-// RFC section 3.4 for details on the format.
-// Returns a non-nil error if there was a problem.
-func ReadTimestampedEntryInto(r io.Reader, t *TimestampedEntry) error {
-	var err error
-	if err = binary.Read(r, binary.BigEndian, &t.Timestamp); err != nil {
-		return err
-	}
-	if err = binary.Read(r, binary.BigEndian, &t.EntryType); err != nil {
-		return err
-	}
-	switch t.EntryType {
-	case X509LogEntryType:
-		if t.X509Entry, err = readVarBytes(r, CertificateLengthBytes); err != nil {
-			return err
-		}
-	case PrecertLogEntryType:
-		if err := binary.Read(r, binary.BigEndian, &t.PrecertEntry.IssuerKeyHash); err != nil {
-			return err
-		}
-		if t.PrecertEntry.TBSCertificate, err = readVarBytes(r, PreCertificateLengthBytes); err != nil {
-			return err
-		}
-	default:
-		return fmt.Errorf("unknown EntryType: %d", t.EntryType)
-	}
-	t.Extensions, err = readVarBytes(r, ExtensionsLengthBytes)
-	return nil
-}
-
-// ReadMerkleTreeLeaf parses the byte-stream representation of a MerkleTreeLeaf
-// and returns a pointer to a new MerkleTreeLeaf structure containing the
-// parsed data.
-// See RFC section 3.4 for details on the format.
-// Returns a pointer to a new MerkleTreeLeaf or non-nil error if there was a
-// problem
-func ReadMerkleTreeLeaf(r io.Reader) (*MerkleTreeLeaf, error) {
-	var m MerkleTreeLeaf
-	if err := binary.Read(r, binary.BigEndian, &m.Version); err != nil {
-		return nil, err
-	}
-	if m.Version != V1 {
-		return nil, fmt.Errorf("unknown Version %d", m.Version)
-	}
-	if err := binary.Read(r, binary.BigEndian, &m.LeafType); err != nil {
-		return nil, err
-	}
-	if m.LeafType != TimestampedEntryLeafType {
-		return nil, fmt.Errorf("unknown LeafType %d", m.LeafType)
-	}
-	if err := ReadTimestampedEntryInto(r, &m.TimestampedEntry); err != nil {
-		return nil, err
-	}
-	return &m, nil
-}
-
-// UnmarshalX509ChainArray unmarshalls the contents of the "chain:" entry in a
-// GetEntries response in the case where the entry refers to an X509 leaf.
-func UnmarshalX509ChainArray(b []byte) ([]ASN1Cert, error) {
-	return readASN1CertList(bytes.NewReader(b), CertificateChainLengthBytes, CertificateLengthBytes)
-}
-
-// UnmarshalPrecertChainArray unmarshalls the contents of the "chain:" entry in
-// a GetEntries response in the case where the entry refers to a Precertificate
-// leaf.
-func UnmarshalPrecertChainArray(b []byte) ([]ASN1Cert, error) {
-	var chain []ASN1Cert
-
-	reader := bytes.NewReader(b)
-	// read the pre-cert entry:
-	precert, err := readVarBytes(reader, CertificateLengthBytes)
-	if err != nil {
-		return chain, err
-	}
-	chain = append(chain, precert)
-	// and then read and return the chain up to the root:
-	remainingChain, err := readASN1CertList(reader, CertificateChainLengthBytes, CertificateLengthBytes)
-	if err != nil {
-		return chain, err
-	}
-	chain = append(chain, remainingChain...)
-	return chain, nil
-}
-
-// UnmarshalDigitallySigned reconstructs a DigitallySigned structure from a Reader
-func UnmarshalDigitallySigned(r io.Reader) (*DigitallySigned, error) {
-	var h byte
-	if err := binary.Read(r, binary.BigEndian, &h); err != nil {
-		return nil, fmt.Errorf("failed to read HashAlgorithm: %v", err)
-	}
-
-	var s byte
-	if err := binary.Read(r, binary.BigEndian, &s); err != nil {
-		return nil, fmt.Errorf("failed to read SignatureAlgorithm: %v", err)
-	}
-
-	sig, err := readVarBytes(r, SignatureLengthBytes)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read Signature bytes: %v", err)
-	}
-
-	return &DigitallySigned{
-		HashAlgorithm:      HashAlgorithm(h),
-		SignatureAlgorithm: SignatureAlgorithm(s),
-		Signature:          sig,
-	}, nil
-}
-
-// MarshalDigitallySigned marshalls a DigitallySigned structure into a byte array
-func MarshalDigitallySigned(ds DigitallySigned) ([]byte, error) {
-	var b bytes.Buffer
-	if err := b.WriteByte(byte(ds.HashAlgorithm)); err != nil {
-		return nil, fmt.Errorf("failed to write HashAlgorithm: %v", err)
-	}
-	if err := b.WriteByte(byte(ds.SignatureAlgorithm)); err != nil {
-		return nil, fmt.Errorf("failed to write SignatureAlgorithm: %v", err)
-	}
-	if err := writeVarBytes(&b, ds.Signature, SignatureLengthBytes); err != nil {
-		return nil, fmt.Errorf("failed to write HashAlgorithm: %v", err)
-	}
-	return b.Bytes(), nil
-}
-
-func checkCertificateFormat(cert ASN1Cert) error {
-	if len(cert) == 0 {
-		return errors.New("certificate is zero length")
-	}
-	if len(cert) > MaxCertificateLength {
-		return errors.New("certificate too large")
-	}
-	return nil
-}
-
-func checkExtensionsFormat(ext CTExtensions) error {
-	if len(ext) > MaxExtensionsLength {
-		return errors.New("extensions too large")
-	}
-	return nil
-}
-
-func serializeV1CertSCTSignatureInput(timestamp uint64, cert ASN1Cert, ext CTExtensions) ([]byte, error) {
-	if err := checkCertificateFormat(cert); err != nil {
-		return nil, err
-	}
-	if err := checkExtensionsFormat(ext); err != nil {
-		return nil, err
-	}
-	var buf bytes.Buffer
-	if err := binary.Write(&buf, binary.BigEndian, V1); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, CertificateTimestampSignatureType); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, timestamp); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, X509LogEntryType); err != nil {
-		return nil, err
-	}
-	if err := writeVarBytes(&buf, cert, CertificateLengthBytes); err != nil {
-		return nil, err
-	}
-	if err := writeVarBytes(&buf, ext, ExtensionsLengthBytes); err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-func serializeV1PrecertSCTSignatureInput(timestamp uint64, issuerKeyHash [issuerKeyHashLength]byte, tbs []byte, ext CTExtensions) ([]byte, error) {
-	if err := checkCertificateFormat(tbs); err != nil {
-		return nil, err
-	}
-	if err := checkExtensionsFormat(ext); err != nil {
-		return nil, err
-	}
-	var buf bytes.Buffer
-	if err := binary.Write(&buf, binary.BigEndian, V1); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, CertificateTimestampSignatureType); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, timestamp); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, PrecertLogEntryType); err != nil {
-		return nil, err
-	}
-	if _, err := buf.Write(issuerKeyHash[:]); err != nil {
-		return nil, err
-	}
-	if err := writeVarBytes(&buf, tbs, CertificateLengthBytes); err != nil {
-		return nil, err
-	}
-	if err := writeVarBytes(&buf, ext, ExtensionsLengthBytes); err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-func serializeV1SCTSignatureInput(sct SignedCertificateTimestamp, entry LogEntry) ([]byte, error) {
-	if sct.SCTVersion != V1 {
-		return nil, fmt.Errorf("unsupported SCT version, expected V1, but got %s", sct.SCTVersion)
-	}
-	if entry.Leaf.LeafType != TimestampedEntryLeafType {
-		return nil, fmt.Errorf("Unsupported leaf type %s", entry.Leaf.LeafType)
-	}
-	switch entry.Leaf.TimestampedEntry.EntryType {
-	case X509LogEntryType:
-		return serializeV1CertSCTSignatureInput(sct.Timestamp, entry.Leaf.TimestampedEntry.X509Entry, entry.Leaf.TimestampedEntry.Extensions)
-	case PrecertLogEntryType:
-		return serializeV1PrecertSCTSignatureInput(sct.Timestamp, entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash,
-			entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate,
-			entry.Leaf.TimestampedEntry.Extensions)
-	default:
-		return nil, fmt.Errorf("unknown TimestampedEntryLeafType %s", entry.Leaf.TimestampedEntry.EntryType)
-	}
-}
-
-// SerializeSCTSignatureInput serializes the passed in sct and log entry into
-// the correct format for signing.
-func SerializeSCTSignatureInput(sct SignedCertificateTimestamp, entry LogEntry) ([]byte, error) {
-	switch sct.SCTVersion {
-	case V1:
-		return serializeV1SCTSignatureInput(sct, entry)
-	default:
-		return nil, fmt.Errorf("unknown SCT version %d", sct.SCTVersion)
-	}
-}
-
-func serializeV1SCT(sct SignedCertificateTimestamp) ([]byte, error) {
-	if err := checkExtensionsFormat(sct.Extensions); err != nil {
-		return nil, err
-	}
-	var buf bytes.Buffer
-	if err := binary.Write(&buf, binary.BigEndian, V1); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, sct.LogID); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, sct.Timestamp); err != nil {
-		return nil, err
-	}
-	if err := writeVarBytes(&buf, sct.Extensions, ExtensionsLengthBytes); err != nil {
-		return nil, err
-	}
-	sig, err := MarshalDigitallySigned(sct.Signature)
-	if err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, sig); err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-// SerializeSCT serializes the passed in sct into the format specified
-// by RFC6962 section 3.2
-func SerializeSCT(sct SignedCertificateTimestamp) ([]byte, error) {
-	switch sct.SCTVersion {
-	case V1:
-		return serializeV1SCT(sct)
-	default:
-		return nil, fmt.Errorf("unknown SCT version %d", sct.SCTVersion)
-	}
-}
-
-func deserializeSCTV1(r io.Reader, sct *SignedCertificateTimestamp) error {
-	if err := binary.Read(r, binary.BigEndian, &sct.LogID); err != nil {
-		return err
-	}
-	if err := binary.Read(r, binary.BigEndian, &sct.Timestamp); err != nil {
-		return err
-	}
-	ext, err := readVarBytes(r, ExtensionsLengthBytes)
-	if err != nil {
-		return err
-	}
-	sct.Extensions = ext
-	ds, err := UnmarshalDigitallySigned(r)
-	if err != nil {
-		return err
-	}
-	sct.Signature = *ds
-	return nil
-}
-
-func DeserializeSCT(r io.Reader) (*SignedCertificateTimestamp, error) {
-	var sct SignedCertificateTimestamp
-	if err := binary.Read(r, binary.BigEndian, &sct.SCTVersion); err != nil {
-		return nil, err
-	}
-	switch sct.SCTVersion {
-	case V1:
-		return &sct, deserializeSCTV1(r, &sct)
-	default:
-		return nil, fmt.Errorf("unknown SCT version %d", sct.SCTVersion)
-	}
-}
-
-func serializeV1STHSignatureInput(sth SignedTreeHead) ([]byte, error) {
-	if sth.Version != V1 {
-		return nil, fmt.Errorf("invalid STH version %d", sth.Version)
-	}
-	if sth.TreeSize < 0 {
-		return nil, fmt.Errorf("invalid tree size %d", sth.TreeSize)
-	}
-	if len(sth.SHA256RootHash) != crypto.SHA256.Size() {
-		return nil, fmt.Errorf("invalid TreeHash length, got %d expected %d", len(sth.SHA256RootHash), crypto.SHA256.Size())
-	}
-
-	var buf bytes.Buffer
-	if err := binary.Write(&buf, binary.BigEndian, V1); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, TreeHashSignatureType); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, sth.Timestamp); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, sth.TreeSize); err != nil {
-		return nil, err
-	}
-	if err := binary.Write(&buf, binary.BigEndian, sth.SHA256RootHash); err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-// SerializeSTHSignatureInput serializes the passed in sth into the correct
-// format for signing.
-func SerializeSTHSignatureInput(sth SignedTreeHead) ([]byte, error) {
-	switch sth.Version {
-	case V1:
-		return serializeV1STHSignatureInput(sth)
-	default:
-		return nil, fmt.Errorf("unsupported STH version %d", sth.Version)
-	}
-}

ct/signatures.go

@@ -1,109 +0,0 @@
-package ct
-
-import (
-	"crypto"
-	"crypto/ecdsa"
-	"crypto/rsa"
-	"crypto/sha256"
-	"crypto/x509"
-	"encoding/asn1"
-	"encoding/pem"
-	"errors"
-	"fmt"
-	"math/big"
-)
-
-// PublicKeyFromPEM parses a PEM formatted block and returns the public key contained within and any remaining unread bytes, or an error.
-func PublicKeyFromPEM(b []byte) (crypto.PublicKey, SHA256Hash, []byte, error) {
-	p, rest := pem.Decode(b)
-	if p == nil {
-		return nil, [sha256.Size]byte{}, rest, fmt.Errorf("no PEM block found in %s", string(b))
-	}
-	k, err := x509.ParsePKIXPublicKey(p.Bytes)
-	return k, sha256.Sum256(p.Bytes), rest, err
-}
-
-// SignatureVerifier can verify signatures on SCTs and STHs
-type SignatureVerifier struct {
-	pubKey crypto.PublicKey
-}
-
-// NewSignatureVerifier creates a new SignatureVerifier using the passed in PublicKey.
-func NewSignatureVerifier(pk crypto.PublicKey) (*SignatureVerifier, error) {
-	switch pkType := pk.(type) {
-	case *rsa.PublicKey:
-	case *ecdsa.PublicKey:
-	default:
-		return nil, fmt.Errorf("Unsupported public key type %v", pkType)
-	}
-
-	return &SignatureVerifier{
-		pubKey: pk,
-	}, nil
-}
-
-// verifySignature verifies that the passed in signature over data was created by our PublicKey.
-// Currently, only SHA256 is supported as a HashAlgorithm, and only ECDSA and RSA signatures are supported.
-func (s SignatureVerifier) verifySignature(data []byte, sig DigitallySigned) error {
-	if sig.HashAlgorithm != SHA256 {
-		return fmt.Errorf("unsupported HashAlgorithm in signature: %v", sig.HashAlgorithm)
-	}
-
-	hasherType := crypto.SHA256
-	hasher := hasherType.New()
-	if _, err := hasher.Write(data); err != nil {
-		return fmt.Errorf("failed to write to hasher: %v", err)
-	}
-	hash := hasher.Sum([]byte{})
-
-	switch sig.SignatureAlgorithm {
-	case RSA:
-		rsaKey, ok := s.pubKey.(*rsa.PublicKey)
-		if !ok {
-			return fmt.Errorf("cannot verify RSA signature with %T key", s.pubKey)
-		}
-		if err := rsa.VerifyPKCS1v15(rsaKey, hasherType, hash, sig.Signature); err != nil {
-			return fmt.Errorf("failed to verify rsa signature: %v", err)
-		}
-	case ECDSA:
-		ecdsaKey, ok := s.pubKey.(*ecdsa.PublicKey)
-		if !ok {
-			return fmt.Errorf("cannot verify ECDSA signature with %T key", s.pubKey)
-		}
-		var ecdsaSig struct {
-			R, S *big.Int
-		}
-		rest, err := asn1.Unmarshal(sig.Signature, &ecdsaSig)
-		if err != nil {
-			return fmt.Errorf("failed to unmarshal ECDSA signature: %v", err)
-		}
-		if len(rest) != 0 {
-			return fmt.Errorf("Garbage following signature %v", rest)
-		}
-
-		if !ecdsa.Verify(ecdsaKey, hash, ecdsaSig.R, ecdsaSig.S) {
-			return errors.New("failed to verify ecdsa signature")
-		}
-	default:
-		return fmt.Errorf("unsupported signature type %v", sig.SignatureAlgorithm)
-	}
-	return nil
-}
-
-// VerifySCTSignature verifies that the SCT's signature is valid for the given LogEntry
-func (s SignatureVerifier) VerifySCTSignature(sct SignedCertificateTimestamp, entry LogEntry) error {
-	sctData, err := SerializeSCTSignatureInput(sct, entry)
-	if err != nil {
-		return err
-	}
-	return s.verifySignature(sctData, sct.Signature)
-}
-
-// VerifySTHSignature verifies that the STH's signature is valid.
-func (s SignatureVerifier) VerifySTHSignature(sth SignedTreeHead) error {
-	sthData, err := SerializeSTHSignatureInput(sth)
-	if err != nil {
-		return err
-	}
-	return s.verifySignature(sthData, sth.TreeHeadSignature)
-}

ct/types.go

@@ -1,333 +0,0 @@
-package ct
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"encoding/base64"
-	"encoding/json"
-	"fmt"
-	"time"
-)
-
-const (
-	issuerKeyHashLength = 32
-)
-
-///////////////////////////////////////////////////////////////////////////////
-// The following structures represent those outlined in the RFC6962 document:
-///////////////////////////////////////////////////////////////////////////////
-
-// LogEntryType represents the LogEntryType enum from section 3.1 of the RFC:
-//   enum { x509_entry(0), precert_entry(1), (65535) } LogEntryType;
-type LogEntryType uint16
-
-func (e LogEntryType) String() string {
-	switch e {
-	case X509LogEntryType:
-		return "X509LogEntryType"
-	case PrecertLogEntryType:
-		return "PrecertLogEntryType"
-	}
-	panic(fmt.Sprintf("No string defined for LogEntryType constant value %d", e))
-}
-
-// LogEntryType constants, see section 3.1 of RFC6962.
-const (
-	X509LogEntryType    LogEntryType = 0
-	PrecertLogEntryType LogEntryType = 1
-)
-
-// MerkleLeafType represents the MerkleLeafType enum from section 3.4 of the
-// RFC: enum { timestamped_entry(0), (255) } MerkleLeafType;
-type MerkleLeafType uint8
-
-func (m MerkleLeafType) String() string {
-	switch m {
-	case TimestampedEntryLeafType:
-		return "TimestampedEntryLeafType"
-	default:
-		return fmt.Sprintf("UnknownLeafType(%d)", m)
-	}
-}
-
-// MerkleLeafType constants, see section 3.4 of the RFC.
-const (
-	TimestampedEntryLeafType MerkleLeafType = 0 // Entry type for an SCT
-)
-
-// Version represents the Version enum from section 3.2 of the RFC:
-// enum { v1(0), (255) } Version;
-type Version uint8
-
-func (v Version) String() string {
-	switch v {
-	case V1:
-		return "V1"
-	default:
-		return fmt.Sprintf("UnknownVersion(%d)", v)
-	}
-}
-
-// CT Version constants, see section 3.2 of the RFC.
-const (
-	V1 Version = 0
-)
-
-// SignatureType differentiates STH signatures from SCT signatures, see RFC
-// section 3.2
-type SignatureType uint8
-
-func (st SignatureType) String() string {
-	switch st {
-	case CertificateTimestampSignatureType:
-		return "CertificateTimestamp"
-	case TreeHashSignatureType:
-		return "TreeHash"
-	default:
-		return fmt.Sprintf("UnknownSignatureType(%d)", st)
-	}
-}
-
-// SignatureType constants, see RFC section 3.2
-const (
-	CertificateTimestampSignatureType SignatureType = 0
-	TreeHashSignatureType             SignatureType = 1
-)
-
-// ASN1Cert type for holding the raw DER bytes of an ASN.1 Certificate
-// (section 3.1)
-type ASN1Cert []byte
-
-// PreCert represents a Precertificate (section 3.2)
-type PreCert struct {
-	IssuerKeyHash  [issuerKeyHashLength]byte
-	TBSCertificate []byte
-}
-
-// CTExtensions is a representation of the raw bytes of any CtExtension
-// structure (see section 3.2)
-type CTExtensions []byte
-
-// MerkleTreeNode represents an internal node in the CT tree
-type MerkleTreeNode []byte
-
-// ConsistencyProof represents a CT consistency proof (see sections 2.1.2 and
-// 4.4)
-type ConsistencyProof []MerkleTreeNode
-
-// AuditPath represents a CT inclusion proof (see sections 2.1.1 and 4.5)
-type AuditPath []MerkleTreeNode
-
-// LeafInput represents a serialized MerkleTreeLeaf structure
-type LeafInput []byte
-
-// HashAlgorithm from the DigitallySigned struct
-type HashAlgorithm byte
-
-// HashAlgorithm constants
-const (
-	None   HashAlgorithm = 0
-	MD5    HashAlgorithm = 1
-	SHA1   HashAlgorithm = 2
-	SHA224 HashAlgorithm = 3
-	SHA256 HashAlgorithm = 4
-	SHA384 HashAlgorithm = 5
-	SHA512 HashAlgorithm = 6
-)
-
-func (h HashAlgorithm) String() string {
-	switch h {
-	case None:
-		return "None"
-	case MD5:
-		return "MD5"
-	case SHA1:
-		return "SHA1"
-	case SHA224:
-		return "SHA224"
-	case SHA256:
-		return "SHA256"
-	case SHA384:
-		return "SHA384"
-	case SHA512:
-		return "SHA512"
-	default:
-		return fmt.Sprintf("UNKNOWN(%d)", h)
-	}
-}
-
-// SignatureAlgorithm from the DigitallySigned struct
-type SignatureAlgorithm byte
-
-// SignatureAlgorithm constants
-const (
-	Anonymous SignatureAlgorithm = 0
-	RSA       SignatureAlgorithm = 1
-	DSA       SignatureAlgorithm = 2
-	ECDSA     SignatureAlgorithm = 3
-)
-
-func (s SignatureAlgorithm) String() string {
-	switch s {
-	case Anonymous:
-		return "Anonymous"
-	case RSA:
-		return "RSA"
-	case DSA:
-		return "DSA"
-	case ECDSA:
-		return "ECDSA"
-	default:
-		return fmt.Sprintf("UNKNOWN(%d)", s)
-	}
-}
-
-// DigitallySigned represents an RFC5246 DigitallySigned structure
-type DigitallySigned struct {
-	HashAlgorithm      HashAlgorithm
-	SignatureAlgorithm SignatureAlgorithm
-	Signature          []byte
-}
-
-// FromBase64String populates the DigitallySigned structure from the base64 data passed in.
-// Returns an error if the base64 data is invalid.
-func (d *DigitallySigned) FromBase64String(b64 string) error {
-	raw, err := base64.StdEncoding.DecodeString(b64)
-	if err != nil {
-		return fmt.Errorf("failed to unbase64 DigitallySigned: %v", err)
-	}
-	ds, err := UnmarshalDigitallySigned(bytes.NewReader(raw))
-	if err != nil {
-		return fmt.Errorf("failed to unmarshal DigitallySigned: %v", err)
-	}
-	*d = *ds
-	return nil
-}
-
-// Base64String returns the base64 representation of the DigitallySigned struct.
-func (d DigitallySigned) Base64String() (string, error) {
-	b, err := MarshalDigitallySigned(d)
-	if err != nil {
-		return "", err
-	}
-	return base64.StdEncoding.EncodeToString(b), nil
-}
-
-// MarshalJSON implements the json.Marshaller interface.
-func (d DigitallySigned) MarshalJSON() ([]byte, error) {
-	b64, err := d.Base64String()
-	if err != nil {
-		return []byte{}, err
-	}
-	return []byte(`"` + b64 + `"`), nil
-}
-
-// UnmarshalJSON implements the json.Unmarshaler interface.
-func (d *DigitallySigned) UnmarshalJSON(b []byte) error {
-	var content string
-	if err := json.Unmarshal(b, &content); err != nil {
-		return fmt.Errorf("failed to unmarshal DigitallySigned: %v", err)
-	}
-	return d.FromBase64String(content)
-}
-
-// LogEntry represents the contents of an entry in a CT log, see section 3.1.
-type LogEntry struct {
-	Index     int64
-	Leaf      MerkleTreeLeaf
-	Chain     []ASN1Cert
-	LeafBytes []byte
-}
-
-// SHA256Hash represents the output from the SHA256 hash function.
-type SHA256Hash [sha256.Size]byte
-
-// FromBase64String populates the SHA256 struct with the contents of the base64 data passed in.
-func (s *SHA256Hash) FromBase64String(b64 string) error {
-	bs, err := base64.StdEncoding.DecodeString(b64)
-	if err != nil {
-		return fmt.Errorf("failed to unbase64 LogID: %v", err)
-	}
-	if len(bs) != sha256.Size {
-		return fmt.Errorf("invalid SHA256 length, expected 32 but got %d", len(bs))
-	}
-	copy(s[:], bs)
-	return nil
-}
-
-// Base64String returns the base64 representation of this SHA256Hash.
-func (s SHA256Hash) Base64String() string {
-	return base64.StdEncoding.EncodeToString(s[:])
-}
-
-// Returns the raw base64url representation of this SHA256Hash.
-func (s SHA256Hash) Base64URLString() string {
-	return base64.RawURLEncoding.EncodeToString(s[:])
-}
-
-// MarshalJSON implements the json.Marshaller interface for SHA256Hash.
-func (s SHA256Hash) MarshalJSON() ([]byte, error) {
-	return []byte(`"` + s.Base64String() + `"`), nil
-}
-
-// UnmarshalJSON implements the json.Unmarshaller interface.
-func (s *SHA256Hash) UnmarshalJSON(b []byte) error {
-	var content string
-	if err := json.Unmarshal(b, &content); err != nil {
-		return fmt.Errorf("failed to unmarshal SHA256Hash: %v", err)
-	}
-	return s.FromBase64String(content)
-}
-
-// SignedTreeHead represents the structure returned by the get-sth CT method
-// after base64 decoding. See sections 3.5 and 4.3 in the RFC)
-type SignedTreeHead struct {
-	Version           Version         `json:"sth_version"`         // The version of the protocol to which the STH conforms
-	TreeSize          uint64          `json:"tree_size"`           // The number of entries in the new tree
-	Timestamp         uint64          `json:"timestamp"`           // The time at which the STH was created
-	SHA256RootHash    SHA256Hash      `json:"sha256_root_hash"`    // The root hash of the log's Merkle tree
-	TreeHeadSignature DigitallySigned `json:"tree_head_signature"` // The Log's signature for this STH (see RFC section 3.5)
-	LogID             SHA256Hash      `json:"log_id"`              // The SHA256 hash of the log's public key
-}
-
-func (sth *SignedTreeHead) TimestampTime() time.Time {
-	return time.Unix(int64(sth.Timestamp/1000), int64(sth.Timestamp%1000)*1_000_000).UTC()
-}
-
-// SignedCertificateTimestamp represents the structure returned by the
-// add-chain and add-pre-chain methods after base64 decoding. (see RFC sections
-// 3.2 ,4.1 and 4.2)
-type SignedCertificateTimestamp struct {
-	SCTVersion Version          `json:"sct_version"`    // The version of the protocol to which the SCT conforms
-	LogID      SHA256Hash       `json:"id"` // the SHA-256 hash of the log's public key, calculated over
-	// the DER encoding of the key represented as SubjectPublicKeyInfo.
-	Timestamp  uint64           `json:"timestamp"`          // Timestamp (in ms since unix epoch) at which the SCT was issued
-	Extensions CTExtensions     `json:"extensions"`    // For future extensions to the protocol
-	Signature  DigitallySigned  `json:"signature"` // The Log's signature for this SCT
-}
-
-func (s SignedCertificateTimestamp) String() string {
-	return fmt.Sprintf("{Version:%d LogId:%s Timestamp:%d Extensions:'%s' Signature:%v}", s.SCTVersion,
-		base64.StdEncoding.EncodeToString(s.LogID[:]),
-		s.Timestamp,
-		s.Extensions,
-		s.Signature)
-}
-
-// TimestampedEntry is part of the MerkleTreeLeaf structure.
-// See RFC section 3.4
-type TimestampedEntry struct {
-	Timestamp    uint64
-	EntryType    LogEntryType
-	X509Entry    ASN1Cert
-	PrecertEntry PreCert
-	Extensions   CTExtensions
-}
-
-// MerkleTreeLeaf represents the deserialized structure of the hash input for the
-// leaves of a log's Merkle tree. See RFC section 3.4
-type MerkleTreeLeaf struct {
-	Version          Version          // the version of the protocol to which the MerkleTreeLeaf corresponds
-	LeafType         MerkleLeafType   // The type of the leaf input, currently only TimestampedEntry can exist
-	TimestampedEntry TimestampedEntry // The entry data itself
-}

ctclient/client.go

@@ -0,0 +1,110 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+// Package ctclient implements a client for monitoring RFC6962 and static-ct-api Certificate Transparency logs
+package ctclient
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+var UserAgent = ""
+
+// Create an HTTP client suitable for communicating with CT logs.  dialContext, if non-nil, is used for dialing.
+func NewHTTPClient(dialContext func(context.Context, string, string) (net.Conn, error)) *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy:                 http.ProxyFromEnvironment,
+			TLSHandshakeTimeout:   15 * time.Second,
+			ResponseHeaderTimeout: 30 * time.Second,
+			MaxIdleConnsPerHost:   10,
+			IdleConnTimeout:       90 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+			TLSClientConfig: &tls.Config{
+				// We have to disable TLS certificate validation because because several logs
+				// (WoSign, StartCom, GDCA) use certificates that are not widely trusted.
+				// Since we verify that every response we receive from the log is signed
+				// by the log's CT public key (either directly, or indirectly via the Merkle Tree),
+				// TLS certificate validation is not actually necessary.  (We don't want to manage
+				// our own trust store because that adds undesired complexity and would require
+				// updating should a log ever change to a different CA.)
+				InsecureSkipVerify: true,
+			},
+			DialContext:       dialContext,
+			ForceAttemptHTTP2: true,
+		},
+		CheckRedirect: func(*http.Request, []*http.Request) error {
+			return errors.New("redirects not followed")
+		},
+		Timeout: 60 * time.Second,
+	}
+}
+
+var defaultHTTPClient = NewHTTPClient(nil)
+
+func get(ctx context.Context, httpClient *http.Client, fullURL string) ([]byte, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("User-Agent", UserAgent)
+
+	if httpClient == nil {
+		httpClient = defaultHTTPClient
+	}
+
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+
+	responseBody, err := io.ReadAll(response.Body)
+	response.Body.Close()
+	if err != nil {
+		return nil, fmt.Errorf("Get %q: error reading response: %w", fullURL, err)
+	}
+
+	if response.StatusCode != 200 {
+		return nil, fmt.Errorf("Get %q: %s (%q)", fullURL, response.Status, bytes.TrimSpace(responseBody))
+	}
+
+	return responseBody, nil
+}
+
+func getJSON(ctx context.Context, httpClient *http.Client, fullURL string, response any) error {
+	responseBytes, err := get(ctx, httpClient, fullURL)
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(responseBytes, response); err != nil {
+		return fmt.Errorf("Get %q: error parsing response JSON: %w", fullURL, err)
+	}
+	return nil
+}
+
+func getRoots(ctx context.Context, httpClient *http.Client, logURL *url.URL) ([][]byte, error) {
+	fullURL := logURL.JoinPath("/ct/v1/get-roots").String()
+	var parsedResponse struct {
+		Certificates [][]byte `json:"certificates"`
+	}
+	if err := getJSON(ctx, httpClient, fullURL, &parsedResponse); err != nil {
+		return nil, err
+	}
+	return parsedResponse.Certificates, nil
+}

ctclient/log.go

@@ -0,0 +1,53 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type Log interface {
+	GetSTH(context.Context) (*cttypes.SignedTreeHead, string, error)
+	GetRoots(context.Context) ([][]byte, error)
+	GetEntries(ctx context.Context, startInclusive, endInclusive uint64) ([]Entry, error)
+	ReconstructTree(context.Context, *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error)
+}
+
+// IssuerGetter represents a source of issuer certificates.
+//
+// If a [Log] also implements IssuerGetter, then it is mandatory to provide
+// an IssuerGetter when using [Entry]s returned by the [Log].  The IssuerGetter
+// may be the Log itself, or your own implementation which retrieves issuers
+// from a different source, such as a cache.
+//
+// If a Log doesn't implement IssuerGetter, then you may pass a nil IssuerGetter
+// when using the Log's Entrys.
+type IssuerGetter interface {
+	GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error)
+}
+
+type Entry interface {
+	LeafInput() []byte
+
+	// Returns error from IssuerGetter, otherwise infallible
+	ExtraData(context.Context, IssuerGetter) ([]byte, error)
+
+	// Returns an error if this is not a well-formed precert entry
+	Precertificate() (cttypes.ASN1Cert, error)
+
+	// Returns an error if this is not a well-formed x509 or precert entry
+	ChainFingerprints() ([][32]byte, error)
+
+	// Returns an error if this is not a well-formed x509 or precert entry, or if IssuerGetter failed
+	GetChain(context.Context, IssuerGetter) (cttypes.ASN1CertChain, error)
+}

ctclient/rfc6962.go

@@ -0,0 +1,219 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"net/http"
+	"net/url"
+	"slices"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type RFC6962Log struct {
+	URL        *url.URL
+	HTTPClient *http.Client // nil to use default client
+}
+
+type RFC6962LogEntry struct {
+	Leaf_input []byte `json:"leaf_input"`
+	Extra_data []byte `json:"extra_data"`
+}
+
+func (ctlog *RFC6962Log) GetSTH(ctx context.Context) (*cttypes.SignedTreeHead, string, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-sth").String()
+	sth := new(cttypes.SignedTreeHead)
+	if err := getJSON(ctx, ctlog.HTTPClient, fullURL, sth); err != nil {
+		return nil, fullURL, err
+	}
+	return sth, fullURL, nil
+}
+
+func (ctlog *RFC6962Log) GetRoots(ctx context.Context) ([][]byte, error) {
+	return getRoots(ctx, ctlog.HTTPClient, ctlog.URL)
+}
+
+func (ctlog *RFC6962Log) getEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]RFC6962LogEntry, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-entries").String()
+	fullURL += fmt.Sprintf("?start=%d&end=%d", startInclusive, endInclusive)
+
+	var parsedResponse struct {
+		Entries []RFC6962LogEntry `json:"entries"`
+	}
+	if err := getJSON(ctx, ctlog.HTTPClient, fullURL, &parsedResponse); err != nil {
+		return nil, err
+	}
+	if len(parsedResponse.Entries) == 0 {
+		return nil, fmt.Errorf("Get %q: zero entries returned", fullURL)
+	}
+	if uint64(len(parsedResponse.Entries)) > endInclusive-startInclusive+1 {
+		return nil, fmt.Errorf("Get %q: extraneous entries returned", fullURL)
+	}
+	return parsedResponse.Entries, nil
+}
+
+func (ctlog *RFC6962Log) GetEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]Entry, error) {
+	nativeEntries, err := ctlog.getEntries(ctx, startInclusive, endInclusive)
+	if err != nil {
+		return nil, err
+	}
+	entries := make([]Entry, len(nativeEntries))
+	for i := range nativeEntries {
+		entries[i] = &nativeEntries[i]
+	}
+	return entries, nil
+}
+
+type entryAndProofResponse struct {
+	LeafInput []byte            `json:"leaf_input"`
+	ExtraData []byte            `json:"extra_data"`
+	AuditPath []merkletree.Hash `json:"audit_path"`
+}
+
+func (ctlog *RFC6962Log) getEntryAndProof(ctx context.Context, leafIndex uint64, treeSize uint64) (*entryAndProofResponse, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-entry-and-proof").String()
+	fullURL += fmt.Sprintf("?leaf_index=%d&tree_size=%d", leafIndex, treeSize)
+	response := new(entryAndProofResponse)
+	if err := getJSON(ctx, ctlog.HTTPClient, fullURL, response); err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+type proofResponse struct {
+	LeafIndex uint64            `json:"leaf_index"`
+	AuditPath []merkletree.Hash `json:"audit_path"`
+}
+
+func (ctlog *RFC6962Log) getProofByHash(ctx context.Context, hash *merkletree.Hash, treeSize uint64) (*proofResponse, error) {
+	fullURL := ctlog.URL.JoinPath("/ct/v1/get-proof-by-hash").String()
+	fullURL += fmt.Sprintf("?hash=%s&tree_size=%d", url.QueryEscape(hash.Base64String()), treeSize)
+	response := new(proofResponse)
+	if err := getJSON(ctx, ctlog.HTTPClient, fullURL, response); err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+func (ctlog *RFC6962Log) reconstructTree(ctx context.Context, treeSize uint64) (*merkletree.CollapsedTree, error) {
+	if treeSize == 0 {
+		return new(merkletree.CollapsedTree), nil
+	}
+	if entryAndProof, err := ctlog.getEntryAndProof(ctx, treeSize-1, treeSize); err == nil {
+		tree := new(merkletree.CollapsedTree)
+		slices.Reverse(entryAndProof.AuditPath)
+		if err := tree.Init(entryAndProof.AuditPath, treeSize-1); err != nil {
+			return nil, fmt.Errorf("log returned invalid audit proof for entry %d to STH %d: %w", treeSize-1, treeSize, err)
+		}
+		tree.Add(merkletree.HashLeaf(entryAndProof.LeafInput))
+		return tree, nil
+	}
+	entries, err := ctlog.getEntries(ctx, treeSize-1, treeSize-1)
+	if err != nil {
+		return nil, err
+	}
+	leafHash := merkletree.HashLeaf(entries[0].Leaf_input)
+	tree := new(merkletree.CollapsedTree)
+	if treeSize > 1 {
+		response, err := ctlog.getProofByHash(ctx, &leafHash, treeSize)
+		if err != nil {
+			return nil, err
+		}
+		if response.LeafIndex != treeSize-1 {
+			// This can happen if the leaf hash is present in the tree in more than one place. Unfortunately, we can't reconstruct when tree if this happens. Fortunately, this is really unlikely, and most logs support get-entry-and-proof anyways.
+			return nil, fmt.Errorf("unable to reconstruct tree because leaf hash %s is present in tree at more than one index (need proof for index %d but get-proof-by-hash returned proof for index %d)", leafHash.Base64String(), treeSize-1, response.LeafIndex)
+		}
+		slices.Reverse(response.AuditPath)
+		if err := tree.Init(response.AuditPath, treeSize-1); err != nil {
+			return nil, fmt.Errorf("log returned invalid audit proof for hash %s to STH %d: %w", leafHash.Base64String(), treeSize, err)
+		}
+	}
+	tree.Add(leafHash)
+	return tree, nil
+}
+
+func (ctlog *RFC6962Log) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error) {
+	tree, err := ctlog.reconstructTree(ctx, sth.TreeSize)
+	if err != nil {
+		return nil, err
+	}
+
+	if rootHash := tree.CalculateRoot(); rootHash != sth.RootHash {
+		return nil, fmt.Errorf("calculated root hash (%s) does not match STH (%s) at size %d", rootHash.Base64String(), sth.RootHash.Base64String(), sth.TreeSize)
+	}
+
+	return tree, nil
+}
+
+func (entry *RFC6962LogEntry) isX509() bool {
+	return len(entry.Leaf_input) >= 12 && entry.Leaf_input[0] == 0 && entry.Leaf_input[1] == 0 && entry.Leaf_input[10] == 0 && entry.Leaf_input[11] == 0
+}
+
+func (entry *RFC6962LogEntry) isPrecert() bool {
+	return len(entry.Leaf_input) >= 12 && entry.Leaf_input[0] == 0 && entry.Leaf_input[1] == 0 && entry.Leaf_input[10] == 0 && entry.Leaf_input[11] == 1
+}
+
+func (entry *RFC6962LogEntry) LeafInput() []byte {
+	return entry.Leaf_input
+}
+
+func (entry *RFC6962LogEntry) ExtraData(context.Context, IssuerGetter) ([]byte, error) {
+	return entry.Extra_data, nil
+}
+
+func (entry *RFC6962LogEntry) Precertificate() (cttypes.ASN1Cert, error) {
+	if !entry.isPrecert() {
+		return nil, fmt.Errorf("not a precertificate entry")
+	}
+	extraData, err := cttypes.ParseExtraDataForPrecertEntry(entry.Extra_data)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing extra_data: %w", err)
+	}
+	return extraData.PreCertificate, nil
+}
+
+func (entry *RFC6962LogEntry) ChainFingerprints() ([][32]byte, error) {
+	chain, err := entry.parseChain()
+	if err != nil {
+		return nil, err
+	}
+	fingerprints := make([][32]byte, len(chain))
+	for i := range chain {
+		fingerprints[i] = sha256.Sum256(chain[i])
+	}
+	return fingerprints, nil
+}
+
+func (entry *RFC6962LogEntry) GetChain(context.Context, IssuerGetter) (cttypes.ASN1CertChain, error) {
+	return entry.parseChain()
+}
+
+func (entry *RFC6962LogEntry) parseChain() (cttypes.ASN1CertChain, error) {
+	switch {
+	case entry.isX509():
+		extraData, err := cttypes.ParseExtraDataForX509Entry(entry.Extra_data)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing extra_data for X509 entry: %w", err)
+		}
+		return extraData, nil
+	case entry.isPrecert():
+		extraData, err := cttypes.ParseExtraDataForPrecertEntry(entry.Extra_data)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing extra_data for precert entry: %w", err)
+		}
+		return extraData.PrecertificateChain, nil
+	default:
+		return nil, fmt.Errorf("unknown entry type")
+	}
+}

ctclient/static.go

@@ -0,0 +1,397 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"sync"
+
+	"golang.org/x/crypto/cryptobyte"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+const (
+	staticTileHeight = 8
+	StaticTileWidth  = 1 << staticTileHeight
+)
+
+func staticSubtreeSize(level uint64) uint64 { return 1 << (level * staticTileHeight) }
+
+type StaticLog struct {
+	SubmissionURL *url.URL
+	MonitoringURL *url.URL
+	ID            cttypes.LogID
+	HTTPClient    *http.Client // nil to use default client
+}
+
+type StaticLogEntry struct {
+	timestampedEntry []byte
+	precertificate   []byte // nil iff x509 entry; non-nil iff precert entry
+	chain            [][32]byte
+}
+
+func (ctlog *StaticLog) GetSTH(ctx context.Context) (*cttypes.SignedTreeHead, string, error) {
+	fullURL := ctlog.MonitoringURL.JoinPath("/checkpoint").String()
+	responseBody, err := get(ctx, ctlog.HTTPClient, fullURL)
+	if err != nil {
+		return nil, fullURL, err
+	}
+	sth, err := cttypes.ParseCheckpoint(responseBody, ctlog.ID)
+	if err != nil {
+		return nil, fullURL, err
+	}
+	return sth, fullURL, nil
+}
+
+func (ctlog *StaticLog) GetRoots(ctx context.Context) ([][]byte, error) {
+	return getRoots(ctx, ctlog.HTTPClient, ctlog.SubmissionURL)
+}
+
+func (ctlog *StaticLog) getEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]StaticLogEntry, error) {
+	var (
+		tile       = startInclusive / StaticTileWidth
+		skip       = startInclusive % StaticTileWidth
+		tileWidth  = min(StaticTileWidth, endInclusive+1-tile*StaticTileWidth)
+		numEntries = tileWidth - skip
+	)
+
+	data, err := ctlog.getDataTile(ctx, tile, tileWidth)
+	if err != nil {
+		return nil, err
+	}
+	var skippedEntry StaticLogEntry
+	for i := range skip {
+		if rest, err := skippedEntry.parse(data); err != nil {
+			return nil, fmt.Errorf("error parsing skipped entry %d in tile %d: %w", i, tile, err)
+		} else {
+			data = rest
+		}
+	}
+	entries := make([]StaticLogEntry, numEntries)
+	for i := range numEntries {
+		if rest, err := entries[i].parse(data); err != nil {
+			return nil, fmt.Errorf("error parsing entry %d in tile %d: %w", skip+i, tile, err)
+		} else {
+			data = rest
+		}
+	}
+	return entries, nil
+}
+
+func (ctlog *StaticLog) GetEntries(ctx context.Context, startInclusive uint64, endInclusive uint64) ([]Entry, error) {
+	nativeEntries, err := ctlog.getEntries(ctx, startInclusive, endInclusive)
+	if err != nil {
+		return nil, err
+	}
+	entries := make([]Entry, len(nativeEntries))
+	for i := range nativeEntries {
+		entries[i] = &nativeEntries[i]
+	}
+	return entries, nil
+}
+
+func (ctlog *StaticLog) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (*merkletree.CollapsedTree, error) {
+	type job struct {
+		level  uint64
+		offset uint64
+		width  uint64
+		tree   *merkletree.CollapsedTree
+		err    error
+	}
+	var jobs []job
+	for level, size := uint64(0), sth.TreeSize; size > 0; level++ {
+		fullTiles := size / StaticTileWidth
+		remainder := size % StaticTileWidth
+		size = fullTiles
+		if remainder > 0 {
+			jobs = append(jobs, job{
+				level:  level,
+				offset: fullTiles,
+				width:  remainder,
+			})
+		}
+	}
+	var wg sync.WaitGroup
+	for i := range jobs {
+		job := &jobs[i]
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			job.tree, job.err = ctlog.getTileCollapsedTree(ctx, job.level, job.offset, job.width)
+		}()
+	}
+	wg.Wait()
+
+	var errs []error
+	tree := new(merkletree.CollapsedTree)
+	for i := range jobs {
+		job := &jobs[len(jobs)-1-i]
+		if job.err != nil {
+			errs = append(errs, job.err)
+			continue
+		}
+		if err := tree.Append(*job.tree); err != nil {
+			panic(err)
+		}
+	}
+	if len(errs) > 0 {
+		return nil, errors.Join(errs...)
+	}
+
+	if rootHash := tree.CalculateRoot(); rootHash != sth.RootHash {
+		return nil, fmt.Errorf("calculated root hash (%s) does not match STH (%s) at size %d", rootHash.Base64String(), sth.RootHash.Base64String(), sth.TreeSize)
+	}
+
+	return tree, nil
+}
+
+func (ctlog *StaticLog) getDataTile(ctx context.Context, tile uint64, width uint64) ([]byte, error) {
+	if width == 0 || width > StaticTileWidth {
+		panic("width is out of range")
+	}
+	var partialErr error
+	if width < StaticTileWidth {
+		fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath("data", tile, width)).String()
+		if data, err := get(ctx, ctlog.HTTPClient, fullURL); err != nil {
+			partialErr = err
+		} else {
+			return data, nil
+		}
+	}
+
+	fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath("data", tile, 0)).String()
+	if data, err := get(ctx, ctlog.HTTPClient, fullURL); err != nil {
+		if partialErr != nil {
+			return nil, partialErr
+		} else {
+			return nil, err
+		}
+	} else {
+		return data, nil
+	}
+}
+
+// returned slice is numHashes*merkletree.HashLen bytes long
+func (ctlog *StaticLog) getTile(ctx context.Context, level uint64, tile uint64, numHashes uint64) ([]byte, error) {
+	if numHashes == 0 || numHashes > StaticTileWidth {
+		panic("numHashes is out of range")
+	}
+
+	var partialErr error
+	if numHashes < StaticTileWidth {
+		fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath(strconv.FormatUint(level, 10), tile, numHashes)).String()
+		if data, err := get(ctx, ctlog.HTTPClient, fullURL); err != nil {
+			partialErr = err
+		} else if expectedLen := merkletree.HashLen * int(numHashes); len(data) != expectedLen {
+			return nil, fmt.Errorf("%s returned %d bytes instead of expected %d", fullURL, len(data), expectedLen)
+		} else {
+			return data, nil
+		}
+	}
+
+	fullURL := ctlog.MonitoringURL.JoinPath(formatTilePath(strconv.FormatUint(level, 10), tile, 0)).String()
+	if data, err := get(ctx, ctlog.HTTPClient, fullURL); err != nil {
+		if partialErr != nil {
+			return nil, partialErr
+		} else {
+			return nil, err
+		}
+	} else if expectedLen := merkletree.HashLen * StaticTileWidth; len(data) != expectedLen {
+		return nil, fmt.Errorf("%s returned %d bytes instead of expected %d", fullURL, len(data), expectedLen)
+	} else {
+		desiredLen := merkletree.HashLen * int(numHashes)
+		return data[:desiredLen], nil
+	}
+}
+
+func (ctlog *StaticLog) getTileCollapsedTree(ctx context.Context, level uint64, tile uint64, numHashes uint64) (*merkletree.CollapsedTree, error) {
+	data, err := ctlog.getTile(ctx, level, tile, numHashes)
+	if err != nil {
+		return nil, err
+	}
+	subtreeSize := staticSubtreeSize(level)
+	offset := staticSubtreeSize(level+1) * tile
+
+	tree := new(merkletree.CollapsedTree)
+	if err := tree.InitSubtree(offset, nil, 0); err != nil {
+		panic(err)
+	}
+	for i := uint64(0); i < numHashes; i++ {
+		hash := (merkletree.Hash)(data[i*merkletree.HashLen : (i+1)*merkletree.HashLen])
+		var subtree merkletree.CollapsedTree
+		if err := subtree.InitSubtree(offset+i*subtreeSize, []merkletree.Hash{hash}, subtreeSize); err != nil {
+			panic(err)
+		}
+		if err := tree.Append(subtree); err != nil {
+			panic(err)
+		}
+	}
+	return tree, nil
+}
+
+func (ctlog *StaticLog) GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) {
+	fullURL := ctlog.MonitoringURL.JoinPath("/issuer/" + hex.EncodeToString(fingerprint[:])).String()
+	data, err := get(ctx, ctlog.HTTPClient, fullURL)
+	if err != nil {
+		return nil, err
+	}
+	if gotFingerprint := sha256.Sum256(data); gotFingerprint != *fingerprint {
+		return nil, fmt.Errorf("%s returned incorrect data with fingerprint %x", fullURL, gotFingerprint[:])
+	}
+	return data, nil
+}
+
+func (entry *StaticLogEntry) parse(input []byte) ([]byte, error) {
+	var skipped cryptobyte.String
+	str := cryptobyte.String(input)
+
+	// TimestampedEntry.timestamp
+	if !str.Skip(8) {
+		return nil, fmt.Errorf("error reading timestamp")
+	}
+	// TimestampedEntry.entry_type
+	var entryType uint16
+	if !str.ReadUint16(&entryType) {
+		return nil, fmt.Errorf("error reading entry type")
+	}
+	// TimestampedEntry.signed_entry
+	if entryType == 0 {
+		if !str.ReadUint24LengthPrefixed(&skipped) {
+			return nil, fmt.Errorf("error reading certificate")
+		}
+	} else if entryType == 1 {
+		if !str.Skip(32) {
+			return nil, fmt.Errorf("error reading issuer_key_hash")
+		}
+		if !str.ReadUint24LengthPrefixed(&skipped) {
+			return nil, fmt.Errorf("error reading tbs_certificate")
+		}
+	} else {
+		return nil, fmt.Errorf("invalid entry type %d", entryType)
+	}
+
+	// TimestampedEntry.extensions
+	if !str.ReadUint16LengthPrefixed(&skipped) {
+		return nil, fmt.Errorf("error reading extensions")
+	}
+
+	timestampedEntryLen := len(input) - len(str)
+	entry.timestampedEntry = input[:timestampedEntryLen]
+
+	// precertificate
+	if entryType == 1 {
+		var precertificate cryptobyte.String
+		if !str.ReadUint24LengthPrefixed(&precertificate) {
+			return nil, fmt.Errorf("error reading precertificate")
+		}
+		entry.precertificate = precertificate
+	} else {
+		entry.precertificate = nil
+	}
+
+	// certificate_chain
+	var chainBytes cryptobyte.String
+	if !str.ReadUint16LengthPrefixed(&chainBytes) {
+		return nil, fmt.Errorf("error reading certificate_chain")
+	}
+	entry.chain = make([][32]byte, 0, len(chainBytes)/32)
+	for !chainBytes.Empty() {
+		var fingerprint [32]byte
+		if !chainBytes.CopyBytes(fingerprint[:]) {
+			return nil, fmt.Errorf("error reading fingerprint in certificate_chain")
+		}
+		entry.chain = append(entry.chain, fingerprint)
+	}
+
+	return str, nil
+}
+
+func (entry *StaticLogEntry) LeafInput() []byte {
+	return append([]byte{0, 0}, entry.timestampedEntry...)
+}
+
+func (entry *StaticLogEntry) ExtraData(ctx context.Context, issuerGetter IssuerGetter) ([]byte, error) {
+	b := cryptobyte.NewBuilder(nil)
+	if entry.precertificate != nil {
+		b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+			b.AddBytes(entry.precertificate)
+		})
+	}
+	b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+		for _, fingerprint := range entry.chain {
+			b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+				cert, err := issuerGetter.GetIssuer(ctx, &fingerprint)
+				if err != nil {
+					panic(cryptobyte.BuildError{Err: fmt.Errorf("error getting issuer %x: %w", fingerprint, err)})
+				}
+				b.AddBytes(cert)
+			})
+		}
+	})
+	return b.Bytes()
+}
+
+func (entry *StaticLogEntry) Precertificate() (cttypes.ASN1Cert, error) {
+	if entry.precertificate == nil {
+		return nil, fmt.Errorf("not a precertificate entry")
+	}
+	return entry.precertificate, nil
+}
+
+func (entry *StaticLogEntry) ChainFingerprints() ([][32]byte, error) {
+	return entry.chain, nil
+}
+
+func (entry *StaticLogEntry) GetChain(ctx context.Context, issuerGetter IssuerGetter) (cttypes.ASN1CertChain, error) {
+	var (
+		chain = make(cttypes.ASN1CertChain, len(entry.chain))
+		errs  = make([]error, len(entry.chain))
+	)
+	var wg sync.WaitGroup
+	for i, fingerprint := range entry.chain {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			chain[i], errs[i] = issuerGetter.GetIssuer(ctx, &fingerprint)
+		}()
+	}
+	wg.Wait()
+	if err := errors.Join(errs...); err != nil {
+		return nil, err
+	}
+	return chain, nil
+}
+
+func formatTilePath(level string, tile uint64, partial uint64) string {
+	path := "tile/" + level + "/" + formatTileIndex(tile)
+	if partial != 0 {
+		path += fmt.Sprintf(".p/%d", partial)
+	}
+	return path
+}
+
+func formatTileIndex(tile uint64) string {
+	const base = 1000
+	str := fmt.Sprintf("%03d", tile%base)
+	for tile >= base {
+		tile = tile / base
+		str = fmt.Sprintf("x%03d/%s", tile%base, str)
+	}
+	return str
+}

ctclient/static_test.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctclient
+
+import (
+	"testing"
+)
+
+func TestFormatTileIndex(t *testing.T) {
+	tests := []struct {
+		in  uint64
+		out string
+	}{
+		{0, "000"},
+		{1, "001"},
+		{12, "012"},
+		{105, "105"},
+		{1000, "x001/000"},
+		{1050, "x001/050"},
+		{52123, "x052/123"},
+		{999001, "x999/001"},
+		{1999001, "x001/x999/001"},
+		{15999001, "x015/x999/001"},
+	}
+	for i, test := range tests {
+		result := formatTileIndex(test.in)
+		if result != test.out {
+			t.Errorf("#%d: formatTileIndex(%q) = %q, want %q", i, test.in, result, test.out)
+		}
+	}
+}

ctcrypto/key.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctcrypto
+
+import (
+	"bytes"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/rsa"
+	"crypto/x509"
+	"fmt"
+
+	"software.sslmate.com/src/certspotter/tlstypes"
+)
+
+type PublicKey []byte
+
+func (key PublicKey) Verify(input SignatureInput, signature tlstypes.DigitallySigned) error {
+	parsedKey, err := x509.ParsePKIXPublicKey(key)
+	if err != nil {
+		return fmt.Errorf("error parsing log key: %w", err)
+	}
+	switch key := parsedKey.(type) {
+	case *rsa.PublicKey:
+		if signature.Algorithm.Signature != tlstypes.RSA {
+			return fmt.Errorf("log key is RSA but this is not an RSA signature")
+		}
+		if signature.Algorithm.Hash != tlstypes.SHA256 {
+			return fmt.Errorf("unsupported hash algorithm %v (only SHA-256 is allowed in CT)", signature.Algorithm.Hash)
+		}
+		if rsa.VerifyPKCS1v15((*rsa.PublicKey)(key), crypto.SHA256, input[:], signature.Signature) != nil {
+			return fmt.Errorf("RSA signature is incorrect")
+		}
+		return nil
+
+	case *ecdsa.PublicKey:
+		if signature.Algorithm.Signature != tlstypes.ECDSA {
+			return fmt.Errorf("log key is ECDSA but this is not an ECDSA signature")
+		}
+		if signature.Algorithm.Hash != tlstypes.SHA256 {
+			return fmt.Errorf("unsupported hash algorithm %v (only SHA-256 is allowed in CT)", signature.Algorithm.Hash)
+		}
+		if !ecdsa.VerifyASN1((*ecdsa.PublicKey)(key), input[:], signature.Signature) {
+			return fmt.Errorf("ECDSA signature is incorrect")
+		}
+
+	default:
+		return fmt.Errorf("unsupported public key type %T (CT only allows RSA and ECDSA)", key)
+	}
+	return nil
+}
+
+func (key PublicKey) MarshalBinary() ([]byte, error) {
+	return bytes.Clone(key), nil
+}
+
+func (key *PublicKey) UnmarshalBinary(data []byte) error {
+	*key = bytes.Clone(data)
+	return nil
+}

ctcrypto/signatures.go

@@ -0,0 +1,55 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package ctcrypto
+
+import (
+	"crypto/sha256"
+	"golang.org/x/crypto/cryptobyte"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+)
+
+type SignatureInput [32]byte
+
+func MakeSignatureInput(message []byte) SignatureInput {
+	return sha256.Sum256(message)
+}
+
+func SignatureInputForPrecertSCT(sct *cttypes.SignedCertificateTimestamp, precert cttypes.PreCert) SignatureInput {
+	var builder cryptobyte.Builder
+	builder.AddValue(sct.SCTVersion)
+	builder.AddValue(cttypes.CertificateTimestampSignatureType)
+	builder.AddUint64(sct.Timestamp)
+	builder.AddValue(cttypes.PrecertEntryType)
+	builder.AddValue(&precert)
+	builder.AddValue(sct.Extensions)
+	return MakeSignatureInput(builder.BytesOrPanic())
+}
+
+func SignatureInputForCertSCT(sct *cttypes.SignedCertificateTimestamp, cert cttypes.ASN1Cert) SignatureInput {
+	var builder cryptobyte.Builder
+	builder.AddValue(sct.SCTVersion)
+	builder.AddValue(cttypes.CertificateTimestampSignatureType)
+	builder.AddUint64(sct.Timestamp)
+	builder.AddValue(cttypes.X509EntryType)
+	builder.AddValue(cert)
+	builder.AddValue(sct.Extensions)
+	return MakeSignatureInput(builder.BytesOrPanic())
+}
+
+func SignatureInputForSTH(sth *cttypes.SignedTreeHead) SignatureInput {
+	var builder cryptobyte.Builder
+	builder.AddValue(cttypes.V1)
+	builder.AddValue(cttypes.TreeHashSignatureType)
+	builder.AddUint64(sth.Timestamp)
+	builder.AddUint64(sth.TreeSize)
+	builder.AddBytes(sth.RootHash[:])
+	return MakeSignatureInput(builder.BytesOrPanic())
+}

cttypes/certs.go

@@ -0,0 +1,121 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"fmt"
+	"golang.org/x/crypto/cryptobyte"
+)
+
+type TBSCertificate []byte
+
+type ASN1Cert []byte
+
+type ASN1CertChain []ASN1Cert
+
+// Corresponds to the PreCert structure in RFC 6962.  PreCert is a misnomer because this is really a TBSCertificate, not a precertificate.
+type PreCert struct {
+	IssuerKeyHash  [32]byte
+	TBSCertificate TBSCertificate
+}
+
+type PrecertChainEntry struct {
+	PreCertificate      ASN1Cert
+	PrecertificateChain ASN1CertChain
+}
+
+func (v *TBSCertificate) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint24LengthPrefixed((*cryptobyte.String)(v))
+}
+func (v TBSCertificate) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint24LengthPrefixed(addBytesFunc(v))
+	return nil
+}
+
+func (v *ASN1Cert) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint24LengthPrefixed((*cryptobyte.String)(v))
+}
+func (v ASN1Cert) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint24LengthPrefixed(addBytesFunc(v))
+	return nil
+}
+
+func (v *ASN1CertChain) Unmarshal(s *cryptobyte.String) bool {
+	chainBytes := new(cryptobyte.String)
+	if !s.ReadUint24LengthPrefixed(chainBytes) {
+		return false
+	}
+	*v = []ASN1Cert{}
+	for !chainBytes.Empty() {
+		var cert ASN1Cert
+		if !cert.Unmarshal(chainBytes) {
+			return false
+		}
+		*v = append(*v, cert)
+	}
+	return true
+}
+func (v ASN1CertChain) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+		for _, cert := range v {
+			b.AddValue(cert)
+		}
+	})
+	return nil
+}
+
+func (precert *PreCert) Unmarshal(s *cryptobyte.String) error {
+	if !s.CopyBytes(precert.IssuerKeyHash[:]) {
+		return fmt.Errorf("error reading PreCert issuer_key_hash")
+	}
+	if !precert.TBSCertificate.Unmarshal(s) {
+		return fmt.Errorf("error reading PreCert tbs_certificate")
+	}
+	return nil
+}
+func (v *PreCert) Marshal(b *cryptobyte.Builder) error {
+	b.AddBytes(v.IssuerKeyHash[:])
+	b.AddValue(v.TBSCertificate)
+	return nil
+}
+
+func (entry *PrecertChainEntry) Unmarshal(s *cryptobyte.String) error {
+	if !entry.PreCertificate.Unmarshal(s) {
+		return fmt.Errorf("error reading PrecertChainEntry pre_certificate")
+	}
+	if !entry.PrecertificateChain.Unmarshal(s) {
+		return fmt.Errorf("error reading PrecertChainEntry preeertificate_chain")
+	}
+	return nil
+}
+
+func ParseExtraDataForX509Entry(extraData []byte) (ASN1CertChain, error) {
+	str := cryptobyte.String(extraData)
+	var chain ASN1CertChain
+	if !chain.Unmarshal(&str) {
+		return nil, fmt.Errorf("error reading ASN.1Cert chain")
+	}
+	if !str.Empty() {
+		return nil, fmt.Errorf("trailing garbage after ASN.1Cert chain")
+	}
+	return chain, nil
+}
+
+func ParseExtraDataForPrecertEntry(extraData []byte) (*PrecertChainEntry, error) {
+	str := cryptobyte.String(extraData)
+	entry := new(PrecertChainEntry)
+	if err := entry.Unmarshal(&str); err != nil {
+		return nil, err
+	}
+	if !str.Empty() {
+		return nil, fmt.Errorf("trailing garbage after PrecertChainEntry")
+	}
+	return entry, nil
+}

cttypes/checkpoint.go

@@ -0,0 +1,112 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"software.sslmate.com/src/certspotter/merkletree"
+	"software.sslmate.com/src/certspotter/tlstypes"
+)
+
+func chompLine(input []byte) (string, []byte, bool) {
+	newline := bytes.IndexByte(input, '\n')
+	if newline == -1 {
+		return "", nil, false
+	}
+	return string(input[:newline]), input[newline+1:], true
+}
+
+func makeCheckpointKeyID(origin string, logID LogID) [4]byte {
+	h := sha256.New()
+	h.Write([]byte(origin))
+	h.Write([]byte{'\n', 0x05})
+	h.Write(logID[:])
+
+	var digest [sha256.Size]byte
+	h.Sum(digest[:0])
+	return [4]byte(digest[:4])
+}
+
+func ParseCheckpoint(input []byte, logID LogID) (*SignedTreeHead, error) {
+	// origin
+	origin, input, _ := chompLine(input)
+
+	// tree size
+	sizeLine, input, _ := chompLine(input)
+	treeSize, err := strconv.ParseUint(sizeLine, 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("malformed tree size: %w", err)
+	}
+
+	// root hash
+	hashLine, input, _ := chompLine(input)
+	rootHash, err := base64.StdEncoding.DecodeString(hashLine)
+	if err != nil {
+		return nil, fmt.Errorf("malformed root hash: %w", err)
+	}
+	if len(rootHash) != merkletree.HashLen {
+		return nil, fmt.Errorf("root hash has wrong length (should be %d bytes long, not %d)", merkletree.HashLen, len(rootHash))
+	}
+
+	// 0 or more non-empty extension lines (ignored)
+	for {
+		line, rest, ok := chompLine(input)
+		if !ok {
+			return nil, errors.New("signed note ended prematurely")
+		}
+		input = rest
+		if len(line) == 0 {
+			break
+		}
+	}
+
+	// signature lines
+	signaturePrefix := "\u2014 " + origin + " "
+	keyID := makeCheckpointKeyID(origin, logID)
+	for {
+		signatureLine, rest, ok := chompLine(input)
+		if !ok {
+			return nil, errors.New("signed note is missing signature from the log")
+		}
+		input = rest
+		if !strings.HasPrefix(signatureLine, signaturePrefix) {
+			continue
+		}
+		signatureBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(signatureLine, signaturePrefix))
+		if err != nil {
+			return nil, fmt.Errorf("malformed signature: %w", err)
+		}
+		if !bytes.HasPrefix(signatureBytes, keyID[:]) {
+			continue
+		}
+		if len(signatureBytes) < 12 {
+			return nil, errors.New("malformed signature: too short")
+		}
+		timestamp := binary.BigEndian.Uint64(signatureBytes[4:12])
+		signature, err := tlstypes.ParseDigitallySigned(signatureBytes[12:])
+		if err != nil {
+			return nil, fmt.Errorf("malformed signature: %w", err)
+		}
+		return &SignedTreeHead{
+			TreeSize:  treeSize,
+			Timestamp: timestamp,
+			RootHash:  (merkletree.Hash)(rootHash),
+			Signature: *signature,
+		}, nil
+	}
+}

cttypes/checkpoint_test.go

@@ -0,0 +1,89 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"bytes"
+	"software.sslmate.com/src/certspotter/tlstypes"
+	"testing"
+)
+
+func TestParseCheckpoint(t *testing.T) {
+	logID := LogID{0x49, 0x4d, 0x90, 0x49, 0xb8, 0xaf, 0x3e, 0x5a, 0xca, 0xba, 0x99, 0x3e, 0x4c, 0x1f, 0x30, 0x56, 0x73, 0x7a, 0xa9, 0xf9, 0x6d, 0x00, 0xf7, 0xb0, 0xb9, 0xb2, 0x51, 0x06, 0xf7, 0xbe, 0x1a, 0x8f}
+	expected := SignedTreeHead{
+		TreeSize:  820495916,
+		Timestamp: 1719511711300,
+		RootHash:  [...]byte{0xc0, 0x3a, 0x6b, 0xe9, 0xf1, 0x1d, 0xc9, 0xcc, 0x20, 0x89, 0xe4, 0x45, 0xa7, 0x3f, 0x61, 0x41, 0xee, 0x82, 0xe8, 0x0d, 0x2d, 0x83, 0xa2, 0xb6, 0x65, 0x53, 0xd4, 0x96, 0xd7, 0x1e, 0xd2, 0x12},
+		Signature: tlstypes.DigitallySigned{
+			Algorithm: tlstypes.SignatureAndHashAlgorithm{
+				Hash:      tlstypes.SHA256,
+				Signature: tlstypes.ECDSA,
+			},
+			Signature: []byte{0x30, 0x46, 0x02, 0x21, 0x00, 0xd0, 0x0d, 0x31, 0x91, 0x50, 0x80, 0x62, 0xfa, 0xb0, 0xf9, 0xf7, 0x63, 0x61, 0x78, 0x95, 0x2b, 0x9c, 0x19, 0x22, 0x3a, 0x1d, 0x08, 0xc5, 0x68, 0x0e, 0xd0, 0x8b, 0x3b, 0x79, 0x18, 0x88, 0x86, 0x02, 0x21, 0x00, 0xd5, 0x74, 0xae, 0x8c, 0x1d, 0x1d, 0x7a, 0x4e, 0x80, 0x6c, 0x36, 0x46, 0x81, 0xb4, 0x7c, 0x91, 0x78, 0xc0, 0x3f, 0xdc, 0xc0, 0xab, 0xa5, 0x90, 0x40, 0x8d, 0x0e, 0xf6, 0x2c, 0x83, 0xa9, 0x34},
+		},
+	}
+
+	sthStrings := []string{
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\nextension line\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\nextension line\nanother extension line\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n\u2014 someoneelse.example c2lnbmF0dXJlCg==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 someoneelse.example c2lnbmF0dXJlCg==\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\nextension line\nanother extension line\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2c2lnbmF0dXJlCg==\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+	}
+
+	for i, str := range sthStrings {
+		parsed, err := ParseCheckpoint([]byte(str), logID)
+		if err != nil {
+			t.Errorf("%d: Unexpected error: %s", i, err)
+			return
+		}
+		if parsed.TreeSize != expected.TreeSize {
+			t.Errorf("%d: wrong tree size", i)
+			return
+		}
+		if parsed.Timestamp != expected.Timestamp {
+			t.Errorf("%d: wrong timestamp", i)
+			return
+		}
+		if !bytes.Equal(parsed.RootHash[:], expected.RootHash[:]) {
+			t.Errorf("%d: wrong root hash", i)
+			return
+		}
+		if !(parsed.Signature.Algorithm == expected.Signature.Algorithm && bytes.Equal(parsed.Signature.Signature, expected.Signature.Signature)) {
+			t.Errorf("%d: wrong signature", i)
+			return
+		}
+	}
+}
+
+func TestParseCheckpointFailure(t *testing.T) {
+	logID := LogID{0x49, 0x4d, 0x90, 0x49, 0xb8, 0xaf, 0x3e, 0x5a, 0xca, 0xba, 0x99, 0x3e, 0x4c, 0x1f, 0x30, 0x56, 0x73, 0x7a, 0xa9, 0xf9, 0x6d, 0x00, 0xf7, 0xb0, 0xb9, 0xb2, 0x51, 0x06, 0xf7, 0xbe, 0x1a, 0x8f}
+	sthStrings := []string{
+		"",
+		"\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 oak.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 bm90YXNpZ25hdHVyZQo=\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 notbase64!\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916!\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEd!ycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n\u2014 sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+		"sycamore.ct.letsencrypt.org/2024h2\n820495916\nwDpr6fEdycwgieRFpz9hQe6C6A0tg6K2ZVPUltce0hI=\n\n- sycamore.ct.letsencrypt.org/2024h2 PdkIagAAAZBa4n5EBAMASDBGAiEA0A0xkVCAYvqw+fdjYXiVK5wZIjodCMVoDtCLO3kYiIYCIQDVdK6MHR16ToBsNkaBtHyReMA/3MCrpZBAjQ72LIOpNA==\n",
+	}
+
+	for i, str := range sthStrings {
+		_, err := ParseCheckpoint([]byte(str), logID)
+		if err == nil {
+			t.Errorf("%d: Unexpected success", i)
+			return
+		}
+	}
+}

cttypes/helpers.go

@@ -0,0 +1,20 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"golang.org/x/crypto/cryptobyte"
+)
+
+func addBytesFunc(v []byte) cryptobyte.BuilderContinuation {
+	return func(b *cryptobyte.Builder) {
+		b.AddBytes(v)
+	}
+}

cttypes/logid.go

@@ -0,0 +1,69 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"golang.org/x/crypto/cryptobyte"
+)
+
+type LogID [32]byte
+
+func (id LogID) Compare(other LogID) int {
+	return bytes.Compare(id[:], other[:])
+}
+
+func (v *LogID) Unmarshal(s *cryptobyte.String) bool {
+	return s.CopyBytes((*v)[:])
+}
+func (v LogID) Marshal(b *cryptobyte.Builder) error {
+	b.AddBytes(v[:])
+	return nil
+}
+
+func (id *LogID) UnmarshalBinary(bytes []byte) error {
+	if len(bytes) != len(*id) {
+		return fmt.Errorf("LogID has wrong length (should be %d, not %d)", len(*id), len(bytes))
+	}
+	*id = (LogID)(bytes)
+	return nil
+}
+func (id LogID) MarshalBinary() ([]byte, error) {
+	return id[:], nil
+}
+
+func (id *LogID) UnmarshalText(textData []byte) error {
+	if len(textData) != 44 {
+		return fmt.Errorf("LogID has wrong length (should be %d, not %d)", 44, len(textData))
+	}
+	var bytes [33]byte
+	if n, err := base64.StdEncoding.Decode(bytes[:], textData); err != nil {
+		return fmt.Errorf("LogID contains invalid base64: %w", err)
+	} else if n != 32 {
+		return fmt.Errorf("LogID has wrong length (should be %d bytes, not %d)", 32, n)
+	}
+	copy(id[:], bytes[:])
+	return nil
+}
+func (id LogID) MarshalText() ([]byte, error) {
+	encodedBytes := make([]byte, 44)
+	base64.StdEncoding.Encode(encodedBytes, id[:])
+	return encodedBytes, nil
+}
+
+func (id LogID) Base64String() string {
+	return base64.StdEncoding.EncodeToString(id[:])
+}
+
+func (id LogID) Base64URLString() string {
+	return base64.RawURLEncoding.EncodeToString(id[:])
+}

cttypes/merkleleaf.go

@@ -0,0 +1,213 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"fmt"
+	"golang.org/x/crypto/cryptobyte"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type MerkleLeafType uint8
+
+const (
+	TimestampedEntryType MerkleLeafType = 0
+)
+
+type LogEntryType uint16
+
+const (
+	X509EntryType    LogEntryType = 0
+	PrecertEntryType LogEntryType = 1
+)
+
+type CTExtensions []byte
+
+type MerkleTreeLeaf struct {
+	Version          Version
+	LeafType         MerkleLeafType
+	TimestampedEntry *TimestampedEntry
+}
+
+type TimestampedEntry struct {
+	Timestamp           uint64
+	EntryType           LogEntryType
+	SignedEntryASN1Cert *ASN1Cert
+	SignedEntryPreCert  *PreCert
+	Extensions          CTExtensions
+}
+
+func (v *MerkleLeafType) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint8((*uint8)(v))
+}
+func (v MerkleLeafType) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint8(uint8(v))
+	return nil
+}
+
+func (v *LogEntryType) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint16((*uint16)(v))
+}
+func (v LogEntryType) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint16(uint16(v))
+	return nil
+}
+
+func (v *CTExtensions) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint16LengthPrefixed((*cryptobyte.String)(v))
+}
+func (v CTExtensions) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint16LengthPrefixed(addBytesFunc(v))
+	return nil
+}
+
+func (leaf *MerkleTreeLeaf) Unmarshal(s *cryptobyte.String) error {
+	if !leaf.Version.Unmarshal(s) {
+		return fmt.Errorf("error reading MerkleTreeLeaf version")
+	}
+	if leaf.Version != V1 {
+		return fmt.Errorf("unsupported Version 0x%02x", leaf.Version)
+	}
+	if !leaf.LeafType.Unmarshal(s) {
+		return fmt.Errorf("error reading MerkleTreeLeaf leaf_type")
+	}
+	switch leaf.LeafType {
+	case TimestampedEntryType:
+		leaf.TimestampedEntry = new(TimestampedEntry)
+		if err := leaf.TimestampedEntry.Unmarshal(s); err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("unrecognized MerkleLeafType 0x%02x", leaf.LeafType)
+	}
+	return nil
+}
+func (v *MerkleTreeLeaf) Marshal(b *cryptobyte.Builder) error {
+	b.AddValue(v.Version)
+	b.AddValue(v.LeafType)
+	switch v.LeafType {
+	case TimestampedEntryType:
+		b.AddValue(v.TimestampedEntry)
+	}
+	return nil
+}
+func (v *MerkleTreeLeaf) Bytes() ([]byte, error) {
+	var builder cryptobyte.Builder
+	builder.AddValue(v)
+	return builder.Bytes()
+}
+func (v *MerkleTreeLeaf) Hash() merkletree.Hash {
+	var builder cryptobyte.Builder
+	builder.AddValue(v)
+	return merkletree.HashLeaf(builder.BytesOrPanic())
+}
+
+func (entry *TimestampedEntry) Unmarshal(s *cryptobyte.String) error {
+	if !s.ReadUint64(&entry.Timestamp) {
+		return fmt.Errorf("error reading TimestampedEntry timestamp")
+	}
+	if !entry.EntryType.Unmarshal(s) {
+		return fmt.Errorf("error reading TimestampedEntry entry_type")
+	}
+	switch entry.EntryType {
+	case X509EntryType:
+		entry.SignedEntryASN1Cert = new(ASN1Cert)
+		if !entry.SignedEntryASN1Cert.Unmarshal(s) {
+			return fmt.Errorf("error reading TimestampedEntry signed_entry ASN.1Cert")
+		}
+	case PrecertEntryType:
+		entry.SignedEntryPreCert = new(PreCert)
+		if err := entry.SignedEntryPreCert.Unmarshal(s); err != nil {
+			return fmt.Errorf("error reading TimestampedEntryType signed_entry: %w", err)
+		}
+	default:
+		return fmt.Errorf("unrecognized TimestampedEntryType 0x%02x", entry.EntryType)
+	}
+	if !entry.Extensions.Unmarshal(s) {
+		return fmt.Errorf("error reading TimestampedEntry extensions")
+	}
+	return nil
+}
+func (v *TimestampedEntry) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint64(v.Timestamp)
+	b.AddValue(v.EntryType)
+	switch v.EntryType {
+	case X509EntryType:
+		b.AddValue(v.SignedEntryASN1Cert)
+	case PrecertEntryType:
+		b.AddValue(v.SignedEntryPreCert)
+	}
+	b.AddValue(v.Extensions)
+	return nil
+}
+
+func ParseLeafInput(leafInput []byte) (*MerkleTreeLeaf, error) {
+	str := cryptobyte.String(leafInput)
+	leaf := new(MerkleTreeLeaf)
+	if err := leaf.Unmarshal(&str); err != nil {
+		return nil, err
+	}
+	if !str.Empty() {
+		return nil, fmt.Errorf("trailing garbage after MerkleTreeLeaf")
+	}
+	return leaf, nil
+}
+
+func MerkleTreeLeafForCert(timestamp uint64, extensions []byte, cert ASN1Cert) *MerkleTreeLeaf {
+	return &MerkleTreeLeaf{
+		Version:  V1,
+		LeafType: TimestampedEntryType,
+		TimestampedEntry: &TimestampedEntry{
+			Timestamp:           timestamp,
+			EntryType:           X509EntryType,
+			SignedEntryASN1Cert: &cert,
+			Extensions:          extensions,
+		},
+	}
+}
+
+func MerkleTreeLeafForCertSCT(sct *SignedCertificateTimestamp, cert ASN1Cert) *MerkleTreeLeaf {
+	return &MerkleTreeLeaf{
+		Version:  sct.SCTVersion,
+		LeafType: TimestampedEntryType,
+		TimestampedEntry: &TimestampedEntry{
+			Timestamp:           sct.Timestamp,
+			EntryType:           X509EntryType,
+			SignedEntryASN1Cert: &cert,
+			Extensions:          sct.Extensions,
+		},
+	}
+}
+
+func MerkleTreeLeafForPrecert(timestamp uint64, extensions []byte, precert PreCert) *MerkleTreeLeaf {
+	return &MerkleTreeLeaf{
+		Version:  V1,
+		LeafType: TimestampedEntryType,
+		TimestampedEntry: &TimestampedEntry{
+			Timestamp:          timestamp,
+			EntryType:          PrecertEntryType,
+			SignedEntryPreCert: &precert,
+			Extensions:         extensions,
+		},
+	}
+}
+
+func MerkleTreeLeafForPrecertSCT(sct *SignedCertificateTimestamp, precert PreCert) *MerkleTreeLeaf {
+	return &MerkleTreeLeaf{
+		Version:  sct.SCTVersion,
+		LeafType: TimestampedEntryType,
+		TimestampedEntry: &TimestampedEntry{
+			Timestamp:          sct.Timestamp,
+			EntryType:          PrecertEntryType,
+			SignedEntryPreCert: &precert,
+			Extensions:         sct.Extensions,
+		},
+	}
+}

cttypes/sct.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"fmt"
+	"golang.org/x/crypto/cryptobyte"
+	"software.sslmate.com/src/certspotter/tlstypes"
+)
+
+type SignedCertificateTimestamp struct {
+	SCTVersion Version                  `json:"sct_version"`
+	ID         LogID                    `json:"id"`
+	Timestamp  uint64                   `json:"timestamp"`
+	Extensions CTExtensions             `json:"extensions"`
+	Signature  tlstypes.DigitallySigned `json:"signature"`
+}
+
+func (sct *SignedCertificateTimestamp) Unmarshal(s *cryptobyte.String) error {
+	if !sct.SCTVersion.Unmarshal(s) {
+		return fmt.Errorf("error reading SCT version")
+	}
+	if sct.SCTVersion != V1 {
+		return fmt.Errorf("unsupported SCT version 0x%02x", sct.SCTVersion)
+	}
+	if !sct.ID.Unmarshal(s) {
+		return fmt.Errorf("error reading SCT id")
+	}
+	if !s.ReadUint64(&sct.Timestamp) {
+		return fmt.Errorf("error reading SCT timestamp")
+	}
+	if !sct.Extensions.Unmarshal(s) {
+		return fmt.Errorf("error reading SCT extensions")
+	}
+	if !sct.Signature.Unmarshal(s) {
+		return fmt.Errorf("error reading SCT signature")
+	}
+	return nil
+}
+
+func ParseSignedCertificateTimestamp(data []byte) (*SignedCertificateTimestamp, error) {
+	str := cryptobyte.String(data)
+	sct := new(SignedCertificateTimestamp)
+	if err := sct.Unmarshal(&str); err != nil {
+		return nil, err
+	}
+	if !str.Empty() {
+		return nil, fmt.Errorf("trailing garbage after SignedCertificateTimestamp")
+	}
+	return sct, nil
+}

cttypes/signatures.go

@@ -0,0 +1,29 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"golang.org/x/crypto/cryptobyte"
+)
+
+type SignatureType uint8
+
+const (
+	CertificateTimestampSignatureType SignatureType = 0
+	TreeHashSignatureType             SignatureType = 1
+)
+
+func (v *SignatureType) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint8((*uint8)(v))
+}
+func (v SignatureType) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint8(uint8(v))
+	return nil
+}

cttypes/sth.go

@@ -0,0 +1,37 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"software.sslmate.com/src/certspotter/merkletree"
+	"software.sslmate.com/src/certspotter/tlstypes"
+	"time"
+)
+
+type SignedTreeHead struct {
+	TreeSize  uint64                   `json:"tree_size"`
+	Timestamp uint64                   `json:"timestamp"`
+	RootHash  merkletree.Hash          `json:"sha256_root_hash"`
+	Signature tlstypes.DigitallySigned `json:"tree_head_signature"`
+}
+
+type GossipedSignedTreeHead struct {
+	SignedTreeHead
+	STHVersion Version `json:"sth_version"`
+	LogID      LogID   `json:"log_id"`
+}
+
+func (sth *SignedTreeHead) TimestampTime() time.Time {
+	return time.UnixMilli(int64(sth.Timestamp))
+}
+
+func (sth *SignedTreeHead) Same(other *SignedTreeHead) bool {
+	return sth.TreeSize == other.TreeSize && sth.Timestamp == other.Timestamp && sth.RootHash == other.RootHash
+}

cttypes/version.go

@@ -0,0 +1,29 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cttypes
+
+import (
+	"golang.org/x/crypto/cryptobyte"
+)
+
+type Version uint8
+
+const (
+	V1 Version = 0
+)
+
+func (v Version) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint8(uint8(v))
+	return nil
+}
+
+func (v *Version) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint8((*uint8)(v))
+}

go.mod

@@ -1,10 +1,13 @@
 module software.sslmate.com/src/certspotter
 
-go 1.21
+go 1.24.4
 
 require (
-	golang.org/x/net v0.17.0
+	golang.org/x/crypto v0.39.0
-	golang.org/x/sync v0.4.0
+	golang.org/x/net v0.41.0
+	golang.org/x/sync v0.15.0
 )
 
-require golang.org/x/text v0.13.0 // indirect
+require golang.org/x/text v0.26.0 // indirect
+
+retract v0.19.0 // Contains serious bugs.

go.sum

@@ -1,6 +1,8 @@
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
-golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
+golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=

helpers.go

@@ -10,16 +10,9 @@
 package certspotter
 
 import (
-	"fmt"
 	"math/big"
-
-	"software.sslmate.com/src/certspotter/ct"
 )
 
-func IsPrecert(entry *ct.LogEntry) bool {
-	return entry.Leaf.TimestampedEntry.EntryType == ct.PrecertLogEntryType
-}
-
 type CertInfo struct {
 	TBS *TBSCertificate
 
@@ -68,19 +61,6 @@ func MakeCertInfoFromRawCert(certBytes []byte) (*CertInfo, error) {
 	return MakeCertInfoFromRawTBS(cert.GetRawTBSCertificate())
 }
 
-func MakeCertInfoFromLogEntry(entry *ct.LogEntry) (*CertInfo, error) {
-	switch entry.Leaf.TimestampedEntry.EntryType {
-	case ct.X509LogEntryType:
-		return MakeCertInfoFromRawCert(entry.Leaf.TimestampedEntry.X509Entry)
-
-	case ct.PrecertLogEntryType:
-		return MakeCertInfoFromRawTBS(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
-
-	default:
-		return nil, fmt.Errorf("MakeCertInfoFromCTEntry: unknown CT entry type (neither X509 nor precert)")
-	}
-}
-
 func MatchesWildcard(dnsName string, pattern string) bool {
 	for len(pattern) > 0 {
 		if pattern[0] == '*' {

identifiers.go

@@ -262,21 +262,6 @@ func (ids *Identifiers) AddIPAddress(value net.IP) {
 	ids.appendIPAddress(value)
 }
 
-func (ids *Identifiers) dnsNamesString(sep string) string {
-	return strings.Join(ids.DNSNames, sep)
-}
-
-func (ids *Identifiers) ipAddrsString(sep string) string {
-	str := ""
-	for _, ipAddr := range ids.IPAddrs {
-		if str != "" {
-			str += sep
-		}
-		str += ipAddr.String()
-	}
-	return str
-}
-
 func (cert *CertInfo) ParseIdentifiers() (*Identifiers, error) {
 	ids := NewIdentifiers()
 

loglist/helpers.go

@@ -13,12 +13,16 @@ import (
 	"time"
 )
 
+// Return all tiled and non-tiled logs from all operators
 func (list *List) AllLogs() []*Log {
 	logs := []*Log{}
 	for operator := range list.Operators {
 		for log := range list.Operators[operator].Logs {
 			logs = append(logs, &list.Operators[operator].Logs[log])
 		}
+		for log := range list.Operators[operator].TiledLogs {
+			logs = append(logs, &list.Operators[operator].TiledLogs[log])
+		}
 	}
 	return logs
 }

loglist/load.go

@@ -21,7 +21,7 @@ import (
 	"time"
 )
 
-var UserAgent = "certspotter"
+var UserAgent = ""
 
 type ModificationToken struct {
 	etag     string
@@ -112,7 +112,7 @@ func Unmarshal(jsonBytes []byte) (*List, error) {
 		return nil, err
 	}
 	if err := list.Validate(); err != nil {
-		return nil, fmt.Errorf("Invalid log list: %s", err)
+		return nil, fmt.Errorf("invalid log list: %s", err)
 	}
 	return list, nil
 }

loglist/schema.go

@@ -12,7 +12,7 @@ package loglist
 import (
 	"time"
 
-	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/cttypes"
 )
 
 type List struct {
@@ -22,16 +22,19 @@ type List struct {
 }
 
 type Operator struct {
-	Name  string   `json:"name"`
+	Name      string   `json:"name"`
-	Email []string `json:"email"`
+	Email     []string `json:"email"`
-	Logs  []Log    `json:"logs"`
+	Logs      []Log    `json:"logs"`
+	TiledLogs []Log    `json:"tiled_logs"`
 }
 
 type Log struct {
 	Key              []byte        `json:"key"`
-	LogID            ct.SHA256Hash `json:"log_id"`
+	LogID            cttypes.LogID `json:"log_id"`
 	MMD              int           `json:"mmd"`
-	URL              string        `json:"url"`
+	URL              string        `json:"url,omitempty"`            // only for rfc6962 logs
+	SubmissionURL    string        `json:"submission_url,omitempty"` // only for static-ct-api logs
+	MonitoringURL    string        `json:"monitoring_url,omitempty"` // only for static-ct-api logs
 	Description      string        `json:"description"`
 	State            State         `json:"state"`
 	DNS              string        `json:"dns"`
@@ -41,9 +44,36 @@ type Log struct {
 		EndExclusive   time.Time `json:"end_exclusive"`
 	} `json:"temporal_interval"`
 
+	// certspotter-specific extensions
+	CertspotterDownloadSize int `json:"certspotter_download_size,omitempty"`
+	CertspotterDownloadJobs int `json:"certspotter_download_jobs,omitempty"`
+
 	// TODO: add previous_operators
 }
 
+func (log *Log) IsRFC6962() bool     { return log.URL != "" }
+func (log *Log) IsStaticCTAPI() bool { return log.SubmissionURL != "" && log.MonitoringURL != "" }
+
+// Return URL prefix for submission using the RFC6962 protocol
+func (log *Log) GetSubmissionURL() string {
+	if log.SubmissionURL != "" {
+		return log.SubmissionURL
+	} else {
+		return log.URL
+	}
+}
+
+// Return URL prefix for monitoring.
+// Since the protocol understood by the URL might be either RFC6962 or static-ct-api, this URL is
+// only useful for informational purposes.
+func (log *Log) GetMonitoringURL() string {
+	if log.MonitoringURL != "" {
+		return log.MonitoringURL
+	} else {
+		return log.URL
+	}
+}
+
 type State struct {
 	Pending *struct {
 		Timestamp time.Time `json:"timestamp"`

loglist/validate.go

@@ -26,7 +26,12 @@ func (list *List) Validate() error {
 func (operator *Operator) Validate() error {
 	for i := range operator.Logs {
 		if err := operator.Logs[i].Validate(); err != nil {
-			return fmt.Errorf("problem with %dth log (%s): %w", i, operator.Logs[i].LogIDString(), err)
+			return fmt.Errorf("problem with %dth non-tiled log (%s): %w", i, operator.Logs[i].LogIDString(), err)
+		}
+	}
+	for i := range operator.TiledLogs {
+		if err := operator.TiledLogs[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth tiled log (%s): %w", i, operator.TiledLogs[i].LogIDString(), err)
 		}
 	}
 	return nil
@@ -37,5 +42,12 @@ func (log *Log) Validate() error {
 	if log.LogID != realLogID {
 		return fmt.Errorf("log ID does not match log key")
 	}
+
+	if !log.IsRFC6962() && !log.IsStaticCTAPI() {
+		return fmt.Errorf("URL(s) not provided")
+	} else if log.IsRFC6962() && log.IsStaticCTAPI() {
+		return fmt.Errorf("inconsistent URLs provided")
+	}
+
 	return nil
 }

man/certspotter-script.md

@@ -65,7 +65,7 @@ The following environment variables are set for `discovered_cert` events:
 
 `CERT_SHA256`
 
-:    The hex-encoded SHA-256 digest (sometimes called fingerprint) of the certificate.
+:    The hex-encoded SHA-256 digest (sometimes called fingerprint) of the certificate or precertificate.
      The digest is computed over the ASN.1 DER encoding. 
 
 `PUBKEY_SHA256`
@@ -123,6 +123,10 @@ The following environment variables are set for `discovered_cert` events:
 
 :    Error parsing the serial number, if any.  If this variable is set, then `SERIAL` is unset.
 
+`CHAIN_ERROR`
+
+:    Error building or verifying the certificate chain, if any.  If this variable is set, then the certificate chain in `CERT_FILENAME` may be incomplete or invalid.
+
 ## Malformed certificate information
 
 The following environment variables are set for `malformed_cert` events:
@@ -251,7 +255,7 @@ certspotter(8)
 
 # COPYRIGHT
 
-Copyright (c) 2016-2023 Opsmate, Inc.
+Copyright (c) 2016-2025 Opsmate, Inc.
 
 # BUGS
 

man/certspotter.md

@@ -30,11 +30,6 @@ You can use Cert Spotter to detect:
 
 # OPTIONS
 
--batch_size *NUMBER*
-
-:   Maximum number of entries to request per call to get-entries.
-    You should not generally need to change this. Defaults to 1000.
-
 -email *ADDRESS*
 
 :   Email address to contact when a matching certificate is discovered, or
@@ -58,12 +53,14 @@ You can use Cert Spotter to detect:
 :   Filename or HTTPS URL of a v2 or v3 JSON log list containing logs to monitor.
     The schema for this file can be found at <https://www.gstatic.com/ct/log_list/v3/log_list_schema.json>.
     Defaults to <https://loglist.certspotter.org/monitor.json>, which includes
-    the union of active logs recognized by Chrome and Apple.  certspotter periodically
+    the union of active logs recognized by Chrome and Apple.  certspotter loads the
-    reloads the log list in case it has changed.
+    log list when starting up, and periodically reloads it in case it has changed.
 
 -no\_save
 
-:   Do not save a copy of matching certificates.
+:   Do not save a copy of matching certificates. Note that enabling this option
+    will cause you to receive duplicate notifications, since certspotter will
+    have no way of knowing if you've been previously notified about a certificate.
 
 -script *COMMAND*
 
@@ -93,7 +90,7 @@ You can use Cert Spotter to detect:
 
 -verbose
 
-:   Be verbose.
+:   Print detailed information about certspotter's operation to stderr.
 
 -version
 
@@ -139,10 +136,10 @@ the script interface, see certspotter-script(8).
 # OPERATION
 
 certspotter continuously monitors all browser-recognized Certificate
-Transparency logs looking for certificates which are valid for any domain
+Transparency logs (both RFC6962 and static-ct-api) looking for certificates (including precertificates)
-on your watch list. When certspotter detects a matching certificate, it
+which are valid for any domain on your watch list. When certspotter
-emails you, executes a script, and/or writes a report to standard out,
+detects a matching certificate, it emails you, executes a script, and/or
-as described above.
+writes a report to standard out, as described above.
 
 certspotter also saves a copy of matching certificates in
 `$CERTSPOTTER_STATE_DIR/certs` ("~/.certspotter/certs" by default)
@@ -172,12 +169,17 @@ API <https://sslmate.com/ct_search_api>, or a CT search engine such as
 # ERROR HANDLING
 
 When certspotter encounters a problem with the local system (e.g. failure
-to write a file or execute a script), it prints a message to stderr and
+to write a file, send an email, or execute a script), it prints a message to stderr and
 exits with a non-zero status.
 
-When certspotter encounters a problem monitoring a log, it prints a message
+When certspotter encounters a problem loading the log list during startup, it
-to stderr and continues running.  It will try monitoring the log again later;
+prints a message to stderr and exits with a non-zero status.  When certspotter encounters a problem
-most log errors are transient.
+reloading the log list, it prints a message to stderr and continues running with the previously-loaded
+log list.  It will try reloading the log list again later.
+
+When certspotter encounters a problem contacting a log, it writes the error to a file in
+the state directory and continues running.  It will try contacting the log again later;
+most log errors are transient.  The last 7 days of errors are kept.
 
 Every 24 hours (unless overridden by `-healthcheck`), certspotter performs the
 following health checks:
@@ -189,11 +191,12 @@ following health checks:
  * Ensure that certspotter is not falling behind monitoring any logs.
 
 If any health check fails, certspotter notifies you by email, script, and/or
-standard out, as described above.
+standard out, as described above.  The notification includes the last several errors
+encountered when contacting the log.
 
 Health check failures should be rare, and you should take them seriously because it means
 certspotter might not detect all certificates.  It might also be an indication
-of CT log misbehavior.  Consult certspotter's stderr output for details, and if
+of CT log misbehavior.  Check the error files for details, and if
 you need help, file an issue at <https://github.com/SSLMate/certspotter>.
 
 # EXIT STATUS
@@ -213,6 +216,11 @@ and non-zero when a serious error occurs.
 :   Directory from which any configuration, such as the watch list, is read.
     Defaults to `~/.certspotter`.
 
+`EMAIL`
+
+:   Email address from which to send emails. If not set, certspotter lets sendmail pick
+    the address.
+
 `HTTPS_PROXY`
 
 :   URL of proxy server for making HTTPS requests.  `http://`, `https://`, and
@@ -222,13 +230,27 @@ and non-zero when a serious error occurs.
 
 :   Path to the sendmail binary used for sending emails. Defaults to `/usr/sbin/sendmail`.
 
+# DIRECTORIES
+
+Config directory
+
+: Stores configuration, such as the watch list. The location is: (1) the `CERTSPOTTER_CONFIG_DIR` environment variable, if set, or (2) the default location `~/.certspotter`. certspotter does not write to this directory.
+
+State directory
+
+: Stores state, such as the position of each log and a store of discovered certificates. The location is: (1) the `-state_dir` command line flag, if provided, (2) the `CERTSPOTTER_STATE_DIR` environment variable, if set, or (3) the default location `~/.certspotter`.  certspotter creates this directory if necessary.
+
+Cache directory
+
+: Stores cached data. The location is `$XDG_CACHE_HOME/certspotter` (which on Linux is `~/.cache/certspotter` by default).  You can delete this directory without without impacting functionality, but certspotter may need to perform additional computation or network requests.
+
 # SEE ALSO
 
 certspotter-script(8)
 
 # COPYRIGHT
 
-Copyright (c) 2016-2023 Opsmate, Inc.
+Copyright (c) 2016-2025 Opsmate, Inc.
 
 # BUGS
 

merkletree/collapsed_tree.go

@@ -16,46 +16,76 @@ import (
 	"slices"
 )
 
+// CollapsedTree is an efficient representation of a Merkle (sub)tree that permits appending
+// nodes and calculating the root hash.
 type CollapsedTree struct {
-	nodes []Hash
+	offset uint64
-	size  uint64
+	nodes  []Hash
+	size   uint64
 }
 
 func calculateNumNodes(size uint64) int {
 	return bits.OnesCount64(size)
 }
 
+// TODO: phase out this function
 func EmptyCollapsedTree() *CollapsedTree {
 	return &CollapsedTree{nodes: []Hash{}, size: 0}
 }
 
+// TODO: phase out this function
 func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) {
-	if len(nodes) != calculateNumNodes(size) {
+	tree := new(CollapsedTree)
-		return nil, fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes))
+	if err := tree.Init(nodes, size); err != nil {
+		return nil, err
 	}
-	return &CollapsedTree{nodes: nodes, size: size}, nil
+	return tree, nil
-}
-
-func CloneCollapsedTree(source *CollapsedTree) *CollapsedTree {
-	nodes := make([]Hash, len(source.nodes))
-	copy(nodes, source.nodes)
-	return &CollapsedTree{nodes: nodes, size: source.size}
 }
 
 func (tree CollapsedTree) Equal(other CollapsedTree) bool {
-	return tree.size == other.size && slices.Equal(tree.nodes, other.nodes)
+	return tree.offset == other.offset && tree.size == other.size && slices.Equal(tree.nodes, other.nodes)
 }
 
-func (tree *CollapsedTree) Add(hash Hash) {
+func (tree CollapsedTree) Clone() CollapsedTree {
+	return CollapsedTree{
+		offset: tree.offset,
+		nodes:  slices.Clone(tree.nodes),
+		size:   tree.size,
+	}
+}
+
+// Add a new leaf hash to the end of the tree.
+// Returns an error if and only if the new tree would be too large for the subtree offset.
+// Always returns a nil error if tree.Offset() == 0.
+func (tree *CollapsedTree) Add(hash Hash) error {
+	if tree.offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if tree.size+1 > maxSize {
+			return fmt.Errorf("subtree at offset %d is already at maximum size %d", tree.offset, maxSize)
+		}
+	}
 	tree.nodes = append(tree.nodes, hash)
 	tree.size++
 	tree.collapse()
+	return nil
 }
 
-func (tree *CollapsedTree) Append(other *CollapsedTree) error {
+func (tree *CollapsedTree) Append(other CollapsedTree) error {
-	maxSize := uint64(1) << bits.TrailingZeros64(tree.size)
+	if tree.offset+tree.size != other.offset {
-	if other.size > maxSize {
+		return fmt.Errorf("subtree at offset %d cannot be appended to subtree ending at offset %d", other.offset, tree.offset+tree.size)
-		return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize)
+	}
+	if tree.offset > 0 {
+		newSize := tree.size + other.size
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if newSize > maxSize {
+			return fmt.Errorf("size of new subtree (%d) would exceed maximum size %d for a subtree at offset %d", newSize, maxSize, tree.offset)
+		}
+	}
+	if tree.size > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.size)
+		if other.size > maxSize {
+			return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize)
+		}
 	}
 
 	tree.nodes = append(tree.nodes, other.nodes...)
@@ -73,7 +103,7 @@ func (tree *CollapsedTree) collapse() {
 	}
 }
 
-func (tree *CollapsedTree) CalculateRoot() Hash {
+func (tree CollapsedTree) CalculateRoot() Hash {
 	if len(tree.nodes) == 0 {
 		return HashNothing()
 	}
@@ -86,29 +116,67 @@ func (tree *CollapsedTree) CalculateRoot() Hash {
 	return hash
 }
 
-func (tree *CollapsedTree) Size() uint64 {
+// Return the subtree offset (0 if this represents an entire tree)
+func (tree CollapsedTree) Offset() uint64 {
+	return tree.offset
+}
+
+// Return a non-nil slice containing the nodes.  The slice
+// must not be modified.
+func (tree CollapsedTree) Nodes() []Hash {
+	if tree.nodes == nil {
+		return []Hash{}
+	} else {
+		return tree.nodes
+	}
+}
+
+// Return the number of leaf nodes in the tree.
+func (tree CollapsedTree) Size() uint64 {
 	return tree.size
 }
 
+type collapsedTreeMessage struct {
+	Offset uint64 `json:"offset,omitempty"`
+	Nodes  []Hash `json:"nodes"` // never nil
+	Size   uint64 `json:"size"`
+}
+
 func (tree CollapsedTree) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
+	return json.Marshal(collapsedTreeMessage{
-		"nodes": tree.nodes,
+		Offset: tree.offset,
-		"size":  tree.size,
+		Nodes:  tree.Nodes(),
+		Size:   tree.size,
 	})
 }
 
 func (tree *CollapsedTree) UnmarshalJSON(b []byte) error {
-	var rawTree struct {
+	var rawTree collapsedTreeMessage
-		Nodes []Hash `json:"nodes"`
-		Size  uint64 `json:"size"`
-	}
 	if err := json.Unmarshal(b, &rawTree); err != nil {
 		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
 	}
-	if len(rawTree.Nodes) != calculateNumNodes(rawTree.Size) {
+	if err := tree.InitSubtree(rawTree.Offset, rawTree.Nodes, rawTree.Size); err != nil {
-		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: nodes has wrong length (should be %d, not %d)", calculateNumNodes(rawTree.Size), len(rawTree.Nodes))
+		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
 	}
-	tree.size = rawTree.Size
-	tree.nodes = rawTree.Nodes
 	return nil
 }
+
+func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error {
+	if len(nodes) != calculateNumNodes(size) {
+		return fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes))
+	}
+	tree.size = size
+	tree.nodes = nodes
+	return nil
+}
+
+func (tree *CollapsedTree) InitSubtree(offset uint64, nodes []Hash, size uint64) error {
+	if offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(offset)
+		if size > maxSize {
+			return fmt.Errorf("subtree size (%d) is too large for offset %d (maximum size is %d)", size, offset, maxSize)
+		}
+	}
+	tree.offset = offset
+	return tree.Init(nodes, size)
+}

merkletree/fragment.go

@@ -0,0 +1,122 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"encoding/json"
+	"fmt"
+	"slices"
+)
+
+// FragmentedCollapsedTree represents a sequence of non-overlapping subtrees
+type FragmentedCollapsedTree struct {
+	subtrees []CollapsedTree // sorted by offset
+}
+
+func (tree *FragmentedCollapsedTree) AddHash(position uint64, hash Hash) error {
+	return tree.Add(CollapsedTree{
+		offset: position,
+		nodes:  []Hash{hash},
+		size:   1,
+	})
+}
+
+func (tree *FragmentedCollapsedTree) Add(subtree CollapsedTree) error {
+	if subtree.size == 0 {
+		return nil
+	}
+	i := len(tree.subtrees)
+	for i > 0 && tree.subtrees[i-1].offset > subtree.offset {
+		i--
+	}
+	if i > 0 && tree.subtrees[i-1].offset+tree.subtrees[i-1].size > subtree.offset {
+		return fmt.Errorf("overlaps with subtree ending at %d", tree.subtrees[i-1].offset+tree.subtrees[i-1].size)
+	}
+	if i < len(tree.subtrees) && subtree.offset+subtree.size > tree.subtrees[i].offset {
+		return fmt.Errorf("overlaps with subtree starting at %d", tree.subtrees[i].offset)
+	}
+	if i == 0 || tree.subtrees[i-1].Append(subtree) != nil {
+		tree.subtrees = slices.Insert(tree.subtrees, i, subtree)
+		i++
+	}
+	for i < len(tree.subtrees) && tree.subtrees[i-1].Append(tree.subtrees[i]) == nil {
+		tree.subtrees = slices.Delete(tree.subtrees, i, i+1)
+	}
+	return nil
+}
+
+func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error {
+	// TODO: try to make this linear time instead of quadratic; it should be possible since the subtrees are sorted by offset
+	for _, subtree := range other.subtrees {
+		if err := tree.Add(subtree); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) {
+	var prevEnd uint64
+	for i := range tree.subtrees {
+		if prevEnd != tree.subtrees[i].offset {
+			if !yield(prevEnd, tree.subtrees[i].offset) {
+				return
+			}
+		}
+		prevEnd = tree.subtrees[i].offset + tree.subtrees[i].size
+	}
+	yield(prevEnd, 0)
+}
+
+func (tree FragmentedCollapsedTree) NumSubtrees() int {
+	return len(tree.subtrees)
+}
+
+func (tree FragmentedCollapsedTree) Subtree(i int) CollapsedTree {
+	return tree.subtrees[i]
+}
+
+func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree {
+	if tree.subtrees == nil {
+		return []CollapsedTree{}
+	} else {
+		return tree.subtrees
+	}
+}
+
+// Return true iff the tree contains at least the first n nodes (without any gaps)
+func (tree FragmentedCollapsedTree) ContainsFirstN(n uint64) bool {
+	return len(tree.subtrees) >= 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size >= n
+}
+
+func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error {
+	for i := 1; i < len(subtrees); i++ {
+		if subtrees[i-1].offset+subtrees[i-1].size > subtrees[i].offset {
+			return fmt.Errorf("subtrees %d and %d overlap", i-1, i)
+		}
+	}
+	tree.subtrees = subtrees
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(tree.Subtrees())
+}
+
+func (tree *FragmentedCollapsedTree) UnmarshalJSON(b []byte) error {
+	var subtrees []CollapsedTree
+	if err := json.Unmarshal(b, &subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	if err := tree.Init(subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	return nil
+}

merkletree/hash.go

@@ -10,6 +10,7 @@
 package merkletree
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
@@ -20,6 +21,10 @@ const HashLen = 32
 
 type Hash [HashLen]byte
 
+func (h Hash) Compare(other Hash) int {
+	return bytes.Compare(h[:], other[:])
+}
+
 func (h Hash) Base64String() string {
 	return base64.StdEncoding.EncodeToString(h[:])
 }

monitor/config.go

@@ -15,15 +15,9 @@ import (
 
 type Config struct {
 	LogListSource       string
-	StateDir            string
+	State               StateProvider
 	StartAtEnd          bool
 	WatchList           WatchList
 	Verbose             bool
-	SaveCerts           bool
-	SendmailPath        string
-	Script              string
-	ScriptDir           string
-	Email               []string
-	Stdout              bool
 	HealthCheckInterval time.Duration
 }

monitor/daemon.go

@@ -16,7 +16,6 @@ import (
 	"golang.org/x/sync/errgroup"
 	"log"
 	insecurerand "math/rand"
-	"path/filepath"
 	"software.sslmate.com/src/certspotter/loglist"
 	"time"
 )
@@ -45,31 +44,32 @@ type daemon struct {
 	tasks          map[LogID]task
 	logsLoadedAt   time.Time
 	logListToken   *loglist.ModificationToken
-	logListError   string
-	logListErrorAt time.Time
 }
 
 func (daemon *daemon) healthCheck(ctx context.Context) error {
 	if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval {
-		textPath := filepath.Join(daemon.config.StateDir, "healthchecks", healthCheckFilename())
+		errors, err := daemon.config.State.GetErrors(ctx, nil, recentErrorCount)
-		event := &staleLogListEvent{
+		if err != nil {
+			return fmt.Errorf("error getting recent errors: %w", err)
+		}
+		var errorsDir string
+		if fsstate, ok := daemon.config.State.(*FilesystemState); ok {
+			errorsDir = fsstate.errorDir(nil)
+		}
+		info := &StaleLogListInfo{
 			Source:        daemon.config.LogListSource,
 			LastSuccess:   daemon.logsLoadedAt,
-			LastError:     daemon.logListError,
+			RecentErrors:  errors,
-			LastErrorTime: daemon.logListErrorAt,
+			ErrorsDir:     errorsDir,
-			TextPath:      textPath,
 		}
-		if err := event.save(); err != nil {
+		if err := daemon.config.State.NotifyHealthCheckFailure(ctx, nil, info); err != nil {
-			return fmt.Errorf("error saving stale log list event: %w", err)
-		}
-		if err := notify(ctx, daemon.config, event); err != nil {
 			return fmt.Errorf("error notifying about stale log list: %w", err)
 		}
 	}
 
 	for _, task := range daemon.tasks {
 		if err := healthCheckLog(ctx, daemon.config, task.log); err != nil {
-			return fmt.Errorf("error checking health of log %q: %w", task.log.URL, err)
+			return fmt.Errorf("error checking health of log %q: %w", task.log.GetMonitoringURL(), err)
 		}
 	}
 	return nil
@@ -81,12 +81,12 @@ func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task {
 		defer cancel()
 		err := monitorLogContinously(ctx, daemon.config, ctlog)
 		if daemon.config.Verbose {
-			log.Printf("task for log %s stopped with error %s", ctlog.URL, err)
+			log.Printf("%s: task stopped with error: %s", ctlog.GetMonitoringURL(), err)
 		}
 		if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
 			return nil
 		} else {
-			return fmt.Errorf("error while monitoring %s: %w", ctlog.URL, err)
+			return fmt.Errorf("error while monitoring %s: %w", ctlog.GetMonitoringURL(), err)
 		}
 	})
 	return task{log: ctlog, stop: cancel}
@@ -119,7 +119,7 @@ func (daemon *daemon) loadLogList(ctx context.Context) error {
 			continue
 		}
 		if daemon.config.Verbose {
-			log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.URL)
+			log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.GetMonitoringURL())
 		}
 		daemon.tasks[logID] = daemon.startTask(ctx, ctlog)
 	}
@@ -129,8 +129,8 @@ func (daemon *daemon) loadLogList(ctx context.Context) error {
 }
 
 func (daemon *daemon) run(ctx context.Context) error {
-	if err := prepareStateDir(daemon.config.StateDir); err != nil {
+	if err := daemon.config.State.Prepare(ctx); err != nil {
-		return fmt.Errorf("error preparing state directory: %w", err)
+		return fmt.Errorf("error preparing state: %w", err)
 	}
 
 	if err := daemon.loadLogList(ctx); err != nil {
@@ -143,14 +143,13 @@ func (daemon *daemon) run(ctx context.Context) error {
 	healthCheckTicker := time.NewTicker(daemon.config.HealthCheckInterval)
 	defer healthCheckTicker.Stop()
 
-	for ctx.Err() == nil {
+	for {
 		select {
 		case <-ctx.Done():
+			return ctx.Err()
 		case <-reloadLogListTicker.C:
 			if err := daemon.loadLogList(ctx); err != nil {
-				daemon.logListError = err.Error()
+				recordError(ctx, daemon.config, nil, fmt.Errorf("error reloading log list (will try again later): %w", err))
-				daemon.logListErrorAt = time.Now()
-				recordError(fmt.Errorf("error reloading log list (will try again later): %w", err))
 			}
 			reloadLogListTicker.Reset(reloadLogListInterval())
 		case <-healthCheckTicker.C:
@@ -159,7 +158,6 @@ func (daemon *daemon) run(ctx context.Context) error {
 			}
 		}
 	}
-	return ctx.Err()
 }
 
 func Run(ctx context.Context, config *Config) error {

monitor/discoveredcert.go

@@ -18,25 +18,32 @@ import (
 	"time"
 
 	"software.sslmate.com/src/certspotter"
-	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/cttypes"
 )
 
-type discoveredCert struct {
+type DiscoveredCert struct {
 	WatchItem    WatchItem
-	LogEntry     *logEntry
+	LogEntry     *LogEntry
 	Info         *certspotter.CertInfo
-	Chain        []ct.ASN1Cert // first entry is the leaf certificate or precertificate
+	Chain        []cttypes.ASN1Cert // first entry is the leaf certificate or precertificate
-	TBSSHA256    [32]byte      // computed over Info.TBS.Raw
+	ChainError   error              // any error generating or validating Chain; if non-nil, Chain may be partial or incorrect
-	SHA256       [32]byte      // computed over Chain[0]
+	TBSSHA256    [32]byte           // computed over Info.TBS.Raw
-	PubkeySHA256 [32]byte      // computed over Info.TBS.PublicKey.FullBytes
+	SHA256       [32]byte           // computed over Chain[0]
+	PubkeySHA256 [32]byte           // computed over Info.TBS.PublicKey.FullBytes
 	Identifiers  *certspotter.Identifiers
-	CertPath     string // empty if not saved on the filesystem
-	JSONPath     string // empty if not saved on the filesystem
-	TextPath     string // empty if not saved on the filesystem
 }
 
-func (cert *discoveredCert) pemChain() []byte {
+type certPaths struct {
+	certPath string
+	jsonPath string
+	textPath string
+}
+
+func (cert *DiscoveredCert) pemChain() []byte {
 	var buffer bytes.Buffer
+	if cert.ChainError != nil {
+		fmt.Fprintln(&buffer, "Warning: this chain may be incomplete or invalid: ", cert.ChainError)
+	}
 	for _, certBytes := range cert.Chain {
 		if err := pem.Encode(&buffer, &pem.Block{
 			Type:  "CERTIFICATE",
@@ -48,7 +55,7 @@ func (cert *discoveredCert) pemChain() []byte {
 	return buffer.Bytes()
 }
 
-func (cert *discoveredCert) json() any {
+func (cert *DiscoveredCert) json() any {
 	object := map[string]any{
 		"tbs_sha256":    hex.EncodeToString(cert.TBSSHA256[:]),
 		"pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]),
@@ -67,25 +74,25 @@ func (cert *discoveredCert) json() any {
 	return object
 }
 
-func (cert *discoveredCert) save() error {
+func writeCertFiles(cert *DiscoveredCert, paths *certPaths) error {
-	if err := writeFile(cert.CertPath, cert.pemChain(), 0666); err != nil {
+	if err := writeFile(paths.certPath, cert.pemChain(), 0666); err != nil {
 		return err
 	}
-	if err := writeJSONFile(cert.JSONPath, cert.json(), 0666); err != nil {
+	if err := writeJSONFile(paths.jsonPath, cert.json(), 0666); err != nil {
 		return err
 	}
-	if err := writeTextFile(cert.TextPath, cert.Text(), 0666); err != nil {
+	if err := writeTextFile(paths.textPath, certNotificationText(cert, paths), 0666); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (cert *discoveredCert) Environ() []string {
+func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string {
 	env := []string{
 		"EVENT=discovered_cert",
-		"SUMMARY=" + cert.Summary(),
+		"SUMMARY=" + certNotificationSummary(cert),
 		"CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented
-		"LOG_URI=" + cert.LogEntry.Log.URL,
+		"LOG_URI=" + cert.LogEntry.Log.GetMonitoringURL(),
 		"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
 		"WATCH_ITEM=" + cert.WatchItem.String(),
 		"TBS_SHA256=" + hex.EncodeToString(cert.TBSSHA256[:]),
@@ -93,9 +100,12 @@ func (cert *discoveredCert) Environ() []string {
 		"FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented
 		"PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]),
 		"PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented
-		"CERT_FILENAME=" + cert.CertPath,
+	}
-		"JSON_FILENAME=" + cert.JSONPath,
+
-		"TEXT_FILENAME=" + cert.TextPath,
+	if paths != nil {
+		env = append(env, "CERT_FILENAME="+paths.certPath)
+		env = append(env, "JSON_FILENAME="+paths.jsonPath)
+		env = append(env, "TEXT_FILENAME="+paths.textPath)
 	}
 
 	if cert.Info.ValidityParseError == nil {
@@ -127,10 +137,14 @@ func (cert *discoveredCert) Environ() []string {
 		env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error())
 	}
 
+	if cert.ChainError != nil {
+		env = append(env, "CHAIN_ERROR="+cert.ChainError.Error())
+	}
+
 	return env
 }
 
-func (cert *discoveredCert) Text() string {
+func certNotificationText(cert *DiscoveredCert, paths *certPaths) string {
 	// TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration)
 
 	text := new(strings.Builder)
@@ -156,15 +170,18 @@ func (cert *discoveredCert) Text() string {
 		writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
 		writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
 	}
-	writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL))
+	writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.GetMonitoringURL()))
 	writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:]))
-	if cert.CertPath != "" {
+	if cert.ChainError != nil {
-		writeField("Filename", cert.CertPath)
+		writeField("Error Building Chain", cert.ChainError.Error())
+	}
+	if paths != nil {
+		writeField("Filename", paths.certPath)
 	}
 
 	return text.String()
 }
 
-func (cert *discoveredCert) Summary() string {
+func certNotificationSummary(cert *DiscoveredCert) string {
 	return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem)
 }

monitor/errors.go

@@ -10,9 +10,19 @@
 package monitor
 
 import (
+	"context"
 	"log"
+
+	"software.sslmate.com/src/certspotter/loglist"
 )
 
-func recordError(err error) {
+func recordError(ctx context.Context, config *Config, ctlog *loglist.Log, errToRecord error) {
-	log.Print(err)
+	if err := config.State.NotifyError(ctx, ctlog, errToRecord); err != nil {
+		log.Print("unable to notify about error: ", err)
+		if ctlog == nil {
+			log.Print(errToRecord)
+		} else {
+			log.Print(ctlog.GetMonitoringURL(), ": ", errToRecord)
+		}
+	}
 }

monitor/fileutils.go

@@ -14,7 +14,9 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
+	"slices"
 )
 
 func randomFileSuffix() string {
@@ -25,9 +27,24 @@ func randomFileSuffix() string {
 	return hex.EncodeToString(randomBytes[:])
 }
 
+func writeSyncFile(filename string, data []byte, perm os.FileMode) error {
+	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(data)
+	if err2 := f.Sync(); err2 != nil && err == nil {
+		err = err2
+	}
+	if err2 := f.Close(); err2 != nil && err == nil {
+		err = err2
+	}
+	return err
+}
+
 func writeFile(filename string, data []byte, perm os.FileMode) error {
 	tempname := filename + ".tmp." + randomFileSuffix()
-	if err := os.WriteFile(tempname, data, perm); err != nil {
+	if err := writeSyncFile(tempname, data, perm); err != nil {
 		return fmt.Errorf("error writing %s: %w", filename, err)
 	}
 	if err := os.Rename(tempname, filename); err != nil {
@@ -54,3 +71,47 @@ func fileExists(filename string) bool {
 	_, err := os.Lstat(filename)
 	return err == nil
 }
+
+func tailFile(filename string, linesWanted int) ([]byte, int, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer file.Close()
+	return tail(file, linesWanted, 4096)
+}
+
+func tail(r io.ReadSeeker, linesWanted int, chunkSize int) ([]byte, int, error) {
+	var buf []byte
+	linesGot := 0
+
+	offset, err := r.Seek(0, io.SeekEnd)
+	if err != nil {
+		return nil, 0, err
+	}
+	for offset > 0 {
+		readSize := chunkSize
+		if offset < int64(readSize) {
+			readSize = int(offset)
+		}
+		offset -= int64(readSize)
+		if _, err := r.Seek(offset, io.SeekStart); err != nil {
+			return nil, 0, err
+		}
+		buf = slices.Grow(buf, readSize)
+		copy(buf[readSize:len(buf)+readSize], buf)
+		buf = buf[:len(buf)+readSize]
+		if _, err := io.ReadFull(r, buf[:readSize]); err != nil {
+			return nil, 0, err
+		}
+		for i := readSize; i > 0; i-- {
+			if buf[i-1] == '\n' {
+				if linesGot == linesWanted {
+					return buf[i:], linesGot, nil
+				}
+				linesGot++
+			}
+		}
+	}
+	return buf, linesGot, nil
+}

monitor/fsstate.go

@@ -0,0 +1,344 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+const keepErrorDays = 7
+const errorDateFormat = "2006-01-02"
+
+type FilesystemState struct {
+	StateDir  string
+	CacheDir  string
+	SaveCerts bool
+	Script    string
+	ScriptDir string
+	Email     []string
+	Stdout    bool
+	errorMu   sync.Mutex
+}
+
+func (s *FilesystemState) logStateDir(logID LogID) string {
+	return filepath.Join(s.StateDir, "logs", logID.Base64URLString())
+}
+
+func (s *FilesystemState) Prepare(ctx context.Context) error {
+	if err := prepareStateDir(s.StateDir); err != nil {
+		return err
+	}
+	if err := prepareCacheDir(s.CacheDir); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) PrepareLog(ctx context.Context, logID LogID) error {
+	var (
+		stateDirPath        = s.logStateDir(logID)
+		sthsDirPath         = filepath.Join(stateDirPath, "unverified_sths")
+		malformedDirPath    = filepath.Join(stateDirPath, "malformed_entries")
+		healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
+		errorsDirPath       = filepath.Join(stateDirPath, "errors")
+	)
+	for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath, errorsDirPath} {
+		if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *FilesystemState) LoadLogState(ctx context.Context, logID LogID) (*LogState, error) {
+	filePath := filepath.Join(s.logStateDir(logID), "state.json")
+	fileBytes, err := os.ReadFile(filePath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	state := new(LogState)
+	if err := json.Unmarshal(fileBytes, state); err != nil {
+		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
+	}
+	return state, nil
+}
+
+func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state *LogState) error {
+	filePath := filepath.Join(s.logStateDir(logID), "state.json")
+	return writeJSONFile(filePath, state, 0666)
+}
+
+func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) (*StoredSTH, error) {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return storeSTHInDir(sthsDirPath, sth)
+}
+
+func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*StoredSTH, error) {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return loadSTHsFromDir(sthsDirPath)
+}
+
+func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) error {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return removeSTHFromDir(sthsDirPath, sth)
+}
+
+func (s *FilesystemState) StoreIssuer(ctx context.Context, fingerprint *[32]byte, issuer []byte) error {
+	filePath := filepath.Join(s.CacheDir, "issuers", hex.EncodeToString(fingerprint[:]))
+	return writeFile(filePath, issuer, 0666)
+}
+
+func (s *FilesystemState) LoadIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) {
+	filePath := filepath.Join(s.CacheDir, "issuers", hex.EncodeToString(fingerprint[:]))
+	issuer, err := os.ReadFile(filePath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	} else {
+		return issuer, err
+	}
+}
+
+func (s *FilesystemState) NotifyCert(ctx context.Context, cert *DiscoveredCert) error {
+	var notifiedPath string
+	var paths *certPaths
+	if s.SaveCerts {
+		hexFingerprint := hex.EncodeToString(cert.SHA256[:])
+		prefixPath := filepath.Join(s.StateDir, "certs", hexFingerprint[0:2])
+		var (
+			notifiedFilename      = "." + hexFingerprint + ".notified"
+			certFilename          = hexFingerprint + ".pem"
+			jsonFilename          = hexFingerprint + ".v1.json"
+			textFilename          = hexFingerprint + ".txt"
+			legacyCertFilename    = hexFingerprint + ".cert.pem"
+			legacyPrecertFilename = hexFingerprint + ".precert.pem"
+		)
+
+		for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} {
+			if fileExists(filepath.Join(prefixPath, filename)) {
+				return nil
+			}
+		}
+
+		if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
+		}
+
+		notifiedPath = filepath.Join(prefixPath, notifiedFilename)
+		paths = &certPaths{
+			certPath: filepath.Join(prefixPath, certFilename),
+			jsonPath: filepath.Join(prefixPath, jsonFilename),
+			textPath: filepath.Join(prefixPath, textFilename),
+		}
+		if err := writeCertFiles(cert, paths); err != nil {
+			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
+		}
+	} else {
+		// TODO-4: save cert to temporary files, and defer their unlinking
+	}
+
+	if err := s.notify(ctx, &notification{
+		summary: certNotificationSummary(cert),
+		environ: certNotificationEnviron(cert, paths),
+		text:    certNotificationText(cert, paths),
+	}); err != nil {
+		return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
+	}
+
+	if notifiedPath != "" {
+		if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
+			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
+		}
+	}
+
+	return nil
+}
+
+func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError error) error {
+	var (
+		dirPath   = filepath.Join(s.logStateDir(entry.Log.LogID), "malformed_entries")
+		entryPath = filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index))
+		textPath  = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index))
+	)
+
+	summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.GetMonitoringURL())
+	leafHash := merkletree.HashLeaf(entry.LeafInput())
+
+	text := new(strings.Builder)
+	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
+	fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
+	writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.GetMonitoringURL()))
+	writeField("Leaf Hash", leafHash.Base64String())
+	writeField("Error", parseError.Error())
+
+	if err := writeJSONFile(entryPath, entry.Entry, 0666); err != nil {
+		return fmt.Errorf("error saving JSON file: %w", err)
+	}
+	if err := writeTextFile(textPath, text.String(), 0666); err != nil {
+		return fmt.Errorf("error saving texT file: %w", err)
+	}
+
+	environ := []string{
+		"EVENT=malformed_cert",
+		"SUMMARY=" + summary,
+		"LOG_URI=" + entry.Log.GetMonitoringURL(),
+		"ENTRY_INDEX=" + fmt.Sprint(entry.Index),
+		"LEAF_HASH=" + leafHash.Base64String(),
+		"PARSE_ERROR=" + parseError.Error(),
+		"ENTRY_FILENAME=" + entryPath,
+		"TEXT_FILENAME=" + textPath,
+		"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
+	}
+
+	if err := s.notify(ctx, &notification{
+		environ: environ,
+		summary: summary,
+		text:    text.String(),
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) healthCheckDir(ctlog *loglist.Log) string {
+	if ctlog == nil {
+		return filepath.Join(s.StateDir, "healthchecks")
+	} else {
+		return filepath.Join(s.logStateDir(ctlog.LogID), "healthchecks")
+	}
+}
+
+func (s *FilesystemState) errorDir(ctlog *loglist.Log) string {
+	if ctlog == nil {
+		return filepath.Join(s.StateDir, "errors")
+	}
+	return filepath.Join(s.logStateDir(ctlog.LogID), "errors")
+}
+
+func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info HealthCheckFailure) error {
+	textPath := filepath.Join(s.healthCheckDir(ctlog), healthCheckFilename())
+	environ := []string{
+		"EVENT=error",
+		"SUMMARY=" + info.Summary(),
+		"TEXT_FILENAME=" + textPath,
+	}
+	text := info.Text()
+	if err := writeTextFile(textPath, text, 0666); err != nil {
+		return fmt.Errorf("error saving text file: %w", err)
+	}
+	if err := s.notify(ctx, &notification{
+		environ: environ,
+		summary: info.Summary(),
+		text:    text,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, notifyErr error) error {
+	if ctlog == nil {
+		log.Print(notifyErr)
+	}
+
+	var (
+		now      = time.Now()
+		filePath = filepath.Join(s.errorDir(ctlog), now.Format(errorDateFormat))
+		line     = now.Format(time.RFC3339) + " " + notifyErr.Error() + "\n"
+	)
+
+	s.errorMu.Lock()
+	defer s.errorMu.Unlock()
+	file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	if _, err := file.WriteString(line); err != nil {
+		return err
+	}
+	return file.Close()
+}
+
+func (s *FilesystemState) GetErrors(ctx context.Context, ctlog *loglist.Log, count int) (string, error) {
+	dir := s.errorDir(ctlog)
+	now := time.Now()
+	var buf []byte
+	for daysBack := 0; count > 0 && daysBack < keepErrorDays; daysBack++ {
+		datePath := filepath.Join(dir, now.AddDate(0, 0, -daysBack).Format(errorDateFormat))
+		dateBuf, dateLines, err := tailFile(datePath, count)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		} else if err != nil {
+			return "", err
+		}
+		buf = append(dateBuf, buf...)
+		count -= dateLines
+	}
+	return string(buf), nil
+}
+
+func (s *FilesystemState) PruneOldErrors() {
+	cutoff := time.Now().AddDate(0, 0, -keepErrorDays)
+	pruneDir := func(dir string) {
+		entries, err := os.ReadDir(dir)
+		if errors.Is(err, fs.ErrNotExist) {
+			return
+		} else if err != nil {
+			log.Printf("unable to read error directory: %s", err)
+			return
+		}
+		for _, entry := range entries {
+			if entry.IsDir() {
+				continue
+			}
+			date, err := time.Parse(errorDateFormat, entry.Name())
+			if err != nil {
+				continue
+			}
+			if date.Before(cutoff) {
+				if err := os.Remove(filepath.Join(dir, entry.Name())); err != nil && !errors.Is(err, fs.ErrNotExist) {
+					log.Printf("unable to remove old error file: %s", err)
+				}
+			}
+		}
+	}
+	pruneDir(filepath.Join(s.StateDir, "errors"))
+	logsDir := filepath.Join(s.StateDir, "logs")
+	logDirs, err := os.ReadDir(logsDir)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		log.Printf("unable to read logs directory: %s", err)
+		return
+	}
+	for _, d := range logDirs {
+		if !d.IsDir() {
+			continue
+		}
+		pruneDir(filepath.Join(logsDir, d.Name(), "errors"))
+	}
+}

monitor/healthcheck.go

@@ -11,68 +11,78 @@ package monitor
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"io/fs"
-	"path/filepath"
 	"strings"
 	"time"
 
-	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/cttypes"
 	"software.sslmate.com/src/certspotter/loglist"
 )
 
+const recentErrorCount = 10
+
 func healthCheckFilename() string {
 	return time.Now().UTC().Format(time.RFC3339) + ".txt"
 }
 
 func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error {
 	var (
-		stateDirPath  = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString())
+		position    uint64
-		stateFilePath = filepath.Join(stateDirPath, "state.json")
+		lastSuccess time.Time
-		sthsDirPath   = filepath.Join(stateDirPath, "unverified_sths")
+		verifiedSTH *cttypes.SignedTreeHead
-		textPath      = filepath.Join(stateDirPath, "healthchecks", healthCheckFilename())
 	)
-	state, err := loadStateFile(stateFilePath)
+
-	if errors.Is(err, fs.ErrNotExist) {
+	if state, err := config.State.LoadLogState(ctx, ctlog.LogID); err != nil {
-		return nil
+		return fmt.Errorf("error loading log state: %w", err)
-	} else if err != nil {
+	} else if state != nil {
-		return fmt.Errorf("error loading state file: %w", err)
+		if time.Since(state.LastSuccess) < config.HealthCheckInterval {
+			// log is healthy
+			return nil
+		}
+
+		position = state.DownloadPosition.Size()
+		lastSuccess = state.LastSuccess
+		verifiedSTH = state.VerifiedSTH
 	}
 
-	if time.Since(state.LastSuccess) < config.HealthCheckInterval {
+	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
-		return nil
-	}
-
-	sths, err := loadSTHsFromDir(sthsDirPath)
 	if err != nil {
-		return fmt.Errorf("error loading STHs directory: %w", err)
+		return fmt.Errorf("error loading STHs: %w", err)
+	}
+
+	var errorsDir string
+	if fsstate, ok := config.State.(*FilesystemState); ok {
+		errorsDir = fsstate.errorDir(ctlog)
 	}
 
 	if len(sths) == 0 {
-		event := &staleSTHEvent{
+		errors, err := config.State.GetErrors(ctx, ctlog, recentErrorCount)
-			Log:         ctlog,
+		if err != nil {
-			LastSuccess: state.LastSuccess,
+			return fmt.Errorf("error getting recent errors: %w", err)
-			LatestSTH:   state.VerifiedSTH,
-			TextPath:    textPath,
 		}
-		if err := event.save(); err != nil {
+		info := &StaleSTHInfo{
-			return fmt.Errorf("error saving stale STH event: %w", err)
+			Log:          ctlog,
+			LastSuccess:  lastSuccess,
+			LatestSTH:    verifiedSTH,
+			RecentErrors: errors,
+			ErrorsDir:    errorsDir,
 		}
-		if err := notify(ctx, config, event); err != nil {
+		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
 			return fmt.Errorf("error notifying about stale STH: %w", err)
 		}
 	} else {
-		event := &backlogEvent{
+		errors, err := config.State.GetErrors(ctx, ctlog, recentErrorCount)
-			Log:       ctlog,
+		if err != nil {
-			LatestSTH: sths[len(sths)-1],
+			return fmt.Errorf("error getting recent errors: %w", err)
-			Position:  state.DownloadPosition.Size(),
-			TextPath:  textPath,
 		}
-		if err := event.save(); err != nil {
+		info := &BacklogInfo{
-			return fmt.Errorf("error saving backlog event: %w", err)
+			Log:          ctlog,
+			LatestSTH:    sths[len(sths)-1],
+			Position:     position,
+			RecentErrors: errors,
+			ErrorsDir:    errorsDir,
 		}
-		if err := notify(ctx, config, event); err != nil {
+		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
 			return fmt.Errorf("error notifying about backlog: %w", err)
 		}
 	}
@@ -80,104 +90,99 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err
 	return nil
 }
 
-type staleSTHEvent struct {
+type HealthCheckFailure interface {
-	Log         *loglist.Log
+	Summary() string
-	LastSuccess time.Time
+	Text() string
-	LatestSTH   *ct.SignedTreeHead // may be nil
-	TextPath    string
-}
-type backlogEvent struct {
-	Log       *loglist.Log
-	LatestSTH *ct.SignedTreeHead
-	Position  uint64
-	TextPath  string
-}
-type staleLogListEvent struct {
-	Source        string
-	LastSuccess   time.Time
-	LastError     string
-	LastErrorTime time.Time
-	TextPath      string
 }
 
-func (e *backlogEvent) Backlog() uint64 {
+type StaleSTHInfo struct {
+	Log          *loglist.Log
+	LastSuccess  time.Time               // may be zero
+	LatestSTH    *cttypes.SignedTreeHead // may be nil
+	RecentErrors string
+	ErrorsDir    string
+}
+
+type BacklogInfo struct {
+	Log          *loglist.Log
+	LatestSTH    *StoredSTH
+	Position     uint64
+	RecentErrors string
+	ErrorsDir    string
+}
+
+type StaleLogListInfo struct {
+	Source        string
+	LastSuccess   time.Time
+	RecentErrors  string
+	ErrorsDir     string
+}
+
+func (e *StaleSTHInfo) LastSuccessString() string {
+	if e.LastSuccess.IsZero() {
+		return "never"
+	} else {
+		return e.LastSuccess.String()
+	}
+}
+func (e *BacklogInfo) Backlog() uint64 {
 	return e.LatestSTH.TreeSize - e.Position
 }
 
-func (e *staleSTHEvent) Environ() []string {
+func (e *StaleSTHInfo) Summary() string {
-	return []string{
+	return fmt.Sprintf("Unable to contact %s since %s", e.Log.GetMonitoringURL(), e.LastSuccessString())
-		"EVENT=error",
-		"SUMMARY=" + e.Summary(),
-		"TEXT_FILENAME=" + e.TextPath,
-	}
 }
-func (e *backlogEvent) Environ() []string {
+func (e *BacklogInfo) Summary() string {
-	return []string{
+	return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.GetMonitoringURL())
-		"EVENT=error",
-		"SUMMARY=" + e.Summary(),
-		"TEXT_FILENAME=" + e.TextPath,
-	}
 }
-func (e *staleLogListEvent) Environ() []string {
+func (e *StaleLogListInfo) Summary() string {
-	return []string{
-		"EVENT=error",
-		"SUMMARY=" + e.Summary(),
-		"TEXT_FILENAME=" + e.TextPath,
-	}
-}
-
-func (e *staleSTHEvent) Summary() string {
-	return fmt.Sprintf("Unable to contact %s since %s", e.Log.URL, e.LastSuccess)
-}
-func (e *backlogEvent) Summary() string {
-	return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.URL)
-}
-func (e *staleLogListEvent) Summary() string {
 	return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess)
 }
 
-func (e *staleSTHEvent) Text() string {
+func (e *StaleSTHInfo) Text() string {
 	text := new(strings.Builder)
-	fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess)
+	fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.GetMonitoringURL(), e.LastSuccessString())
-	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "For details, see certspotter's stderr output.\n")
 	fmt.Fprintf(text, "\n")
 	if e.LatestSTH != nil {
-		fmt.Fprintf(text, "Latest known log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.TimestampTime())
+		fmt.Fprintf(text, "Latest known log size = %d\n", e.LatestSTH.TreeSize)
 	} else {
 		fmt.Fprintf(text, "Latest known log size = none\n")
 	}
+	if e.RecentErrors != "" {
+		fmt.Fprintf(text, "\n")
+		fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir)
+		fmt.Fprintf(text, "\n")
+		fmt.Fprint(text, e.RecentErrors)
+	}
 	return text.String()
 }
-func (e *backlogEvent) Text() string {
+func (e *BacklogInfo) Text() string {
 	text := new(strings.Builder)
-	fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL)
+	fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.GetMonitoringURL())
 	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n")
+	fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.StoredAt)
-	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.TimestampTime())
 	fmt.Fprintf(text, "Current position = %d\n", e.Position)
 	fmt.Fprintf(text, "         Backlog = %d\n", e.Backlog())
+	if e.RecentErrors != "" {
+		fmt.Fprintf(text, "\n")
+		fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir)
+		fmt.Fprintf(text, "\n")
+		fmt.Fprint(text, e.RecentErrors)
+	}
 	return text.String()
 }
-func (e *staleLogListEvent) Text() string {
+func (e *StaleLogListInfo) Text() string {
 	text := new(strings.Builder)
 	fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess)
 	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "Last error (at %s): %s\n", e.LastErrorTime, e.LastError)
-	fmt.Fprintf(text, "\n")
 	fmt.Fprintf(text, "Consequentially, certspotter may not be monitoring all logs, and might fail to detect certificates.\n")
+	if e.RecentErrors != "" {
+		fmt.Fprintf(text, "\n")
+		fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir)
+		fmt.Fprintf(text, "\n")
+		fmt.Fprint(text, e.RecentErrors)
+	}
 	return text.String()
 }
 
-func (e *staleSTHEvent) save() error {
-	return writeTextFile(e.TextPath, e.Text(), 0666)
-}
-func (e *backlogEvent) save() error {
-	return writeTextFile(e.TextPath, e.Text(), 0666)
-}
-func (e *staleLogListEvent) save() error {
-	return writeTextFile(e.TextPath, e.Text(), 0666)
-}
-
 // TODO-3: make the errors more actionable

monitor/loglist.go

@@ -12,11 +12,11 @@ package monitor
 import (
 	"context"
 	"fmt"
-	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/cttypes"
 	"software.sslmate.com/src/certspotter/loglist"
 )
 
-type LogID = ct.SHA256Hash
+type LogID = cttypes.LogID
 
 func getLogList(ctx context.Context, source string, token *loglist.ModificationToken) (map[LogID]*loglist.Log, *loglist.ModificationToken, error) {
 	list, newToken, err := loglist.LoadIfModified(ctx, source, token)
@@ -33,6 +33,13 @@ func getLogList(ctx context.Context, source string, token *loglist.ModificationT
 			}
 			logs[log.LogID] = log
 		}
+		for logIndex := range list.Operators[operatorIndex].TiledLogs {
+			log := &list.Operators[operatorIndex].TiledLogs[logIndex]
+			if _, exists := logs[log.LogID]; exists {
+				return nil, nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String())
+			}
+			logs[log.LogID] = log
+		}
 	}
 	return logs, newToken, nil
 }

monitor/mailutils.go

@@ -0,0 +1,34 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"os"
+)
+
+const mailDateFormat = "Mon, 2 Jan 2006 15:04:05 -0700"
+
+func generateMessageID() string {
+	var randomBytes [16]byte
+	if _, err := rand.Read(randomBytes[:]); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(randomBytes[:]) + "@selfhosted.certspotter.org"
+}
+
+func sendmailPath() string {
+	if envVar := os.Getenv("SENDMAIL_PATH"); envVar != "" {
+		return envVar
+	} else {
+		return "/usr/sbin/sendmail"
+	}
+}

monitor/malformed.go

@@ -1,72 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"fmt"
-	"strings"
-)
-
-type malformedLogEntry struct {
-	Entry     *logEntry
-	Error     string
-	EntryPath string
-	TextPath  string
-}
-
-func (malformed *malformedLogEntry) entryJSON() any {
-	return struct {
-		LeafInput []byte `json:"leaf_input"`
-		ExtraData []byte `json:"extra_data"`
-	}{
-		LeafInput: malformed.Entry.LeafInput,
-		ExtraData: malformed.Entry.ExtraData,
-	}
-}
-
-func (malformed *malformedLogEntry) save() error {
-	if err := writeJSONFile(malformed.EntryPath, malformed.entryJSON(), 0666); err != nil {
-		return err
-	}
-	if err := writeTextFile(malformed.TextPath, malformed.Text(), 0666); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (malformed *malformedLogEntry) Environ() []string {
-	return []string{
-		"EVENT=malformed_cert",
-		"SUMMARY=" + malformed.Summary(),
-		"LOG_URI=" + malformed.Entry.Log.URL,
-		"ENTRY_INDEX=" + fmt.Sprint(malformed.Entry.Index),
-		"LEAF_HASH=" + malformed.Entry.LeafHash.Base64String(),
-		"PARSE_ERROR=" + malformed.Error,
-		"ENTRY_FILENAME=" + malformed.EntryPath,
-		"TEXT_FILENAME=" + malformed.TextPath,
-		"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
-	}
-}
-
-func (malformed *malformedLogEntry) Text() string {
-	text := new(strings.Builder)
-	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
-
-	fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
-	writeField("Log Entry", fmt.Sprintf("%d @ %s", malformed.Entry.Index, malformed.Entry.Log.URL))
-	writeField("Leaf Hash", malformed.Entry.LeafHash.Base64String())
-	writeField("Error", malformed.Error)
-
-	return text.String()
-}
-
-func (malformed *malformedLogEntry) Summary() string {
-	return fmt.Sprintf("Unable to Parse Entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL)
-}

monitor/monitor.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2023 Opsmate, Inc.
+// Copyright (C) 2025 Opsmate, Inc.
 //
 // This Source Code Form is subject to the terms of the Mozilla
 // Public License, v. 2.0. If a copy of the MPL was not distributed
@@ -11,286 +11,548 @@ package monitor
 
 import (
 	"context"
-	"crypto/x509"
 	"errors"
 	"fmt"
-	"io/fs"
+	"golang.org/x/sync/errgroup"
 	"log"
-	"os"
+	mathrand "math/rand/v2"
-	"path/filepath"
+	"net/url"
-	"strings"
+	"slices"
 	"time"
 
-	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/ctclient"
-	"software.sslmate.com/src/certspotter/ct/client"
+	"software.sslmate.com/src/certspotter/ctcrypto"
+	"software.sslmate.com/src/certspotter/cttypes"
 	"software.sslmate.com/src/certspotter/loglist"
 	"software.sslmate.com/src/certspotter/merkletree"
+	"software.sslmate.com/src/certspotter/sequencer"
 )
 
 const (
-	maxGetEntriesSize  = 1000
+	getSTHInterval    = 5 * time.Minute
-	monitorLogInterval = 5 * time.Minute
+	maxPartialTileAge = 5 * time.Minute
 )
 
-func isFatalLogError(err error) bool {
+func downloadJobSize(ctlog *loglist.Log) uint64 {
-	return errors.Is(err, context.Canceled)
+	if ctlog.IsStaticCTAPI() {
+		return ctclient.StaticTileWidth
+	} else if ctlog.CertspotterDownloadSize != 0 {
+		return uint64(ctlog.CertspotterDownloadSize)
+	} else {
+		return 1000
+	}
 }
 
-func newLogClient(ctlog *loglist.Log) (*client.LogClient, error) {
+func downloadWorkers(ctlog *loglist.Log) int {
-	logKey, err := x509.ParsePKIXPublicKey(ctlog.Key)
+	if ctlog.CertspotterDownloadJobs != 0 {
-	if err != nil {
+		return ctlog.CertspotterDownloadJobs
-		return nil, fmt.Errorf("error parsing log key: %w", err)
+	} else {
+		return 1
 	}
-	verifier, err := ct.NewSignatureVerifier(logKey)
-	if err != nil {
-		return nil, fmt.Errorf("error with log key: %w", err)
-	}
-	return client.NewWithVerifier(strings.TrimRight(ctlog.URL, "/"), verifier), nil
 }
 
-func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) error {
+type verifyEntriesError struct {
-	logClient, err := newLogClient(ctlog)
+	sth             *cttypes.SignedTreeHead
+	entriesRootHash merkletree.Hash
+}
+
+func (e *verifyEntriesError) Error() string {
+	return fmt.Sprintf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", e.sth.TreeSize, e.sth.RootHash, e.entriesRootHash)
+}
+
+func withRetry(ctx context.Context, config *Config, ctlog *loglist.Log, maxRetries int, f func() error) error {
+	minSleep := 1 * time.Second
+	numRetries := 0
+	for ctx.Err() == nil {
+		err := f()
+		if err == nil || errors.Is(err, context.Canceled) {
+			return err
+		}
+		if maxRetries != -1 && numRetries >= maxRetries {
+			return fmt.Errorf("%w (retried %d times)", err, numRetries)
+		}
+		recordError(ctx, config, ctlog, err)
+		sleepTime := minSleep + mathrand.N(minSleep)
+		if err := sleep(ctx, sleepTime); err != nil {
+			return err
+		}
+		minSleep = min(minSleep*2, 5*time.Minute)
+		numRetries++
+	}
+	return ctx.Err()
+}
+
+func getEntriesFull(ctx context.Context, client ctclient.Log, startInclusive, endExclusive uint64) ([]ctclient.Entry, error) {
+	allEntries := make([]ctclient.Entry, 0, endExclusive-startInclusive)
+	for startInclusive < endExclusive {
+		entries, err := client.GetEntries(ctx, startInclusive, endExclusive-1)
+		if err != nil {
+			return nil, err
+		}
+		allEntries = append(allEntries, entries...)
+		startInclusive += uint64(len(entries))
+	}
+	return allEntries, nil
+}
+
+func getAndVerifySTH(ctx context.Context, ctlog *loglist.Log, client ctclient.Log) (*cttypes.SignedTreeHead, string, error) {
+	sth, url, err := client.GetSTH(ctx)
+	if err != nil {
+		return nil, "", err
+	}
+	if err := ctcrypto.PublicKey(ctlog.Key).Verify(ctcrypto.SignatureInputForSTH(sth), sth.Signature); err != nil {
+		return nil, "", fmt.Errorf("STH has invalid signature: %w", err)
+	}
+	return sth, url, nil
+}
+
+type logClient struct {
+	config *Config
+	log    *loglist.Log
+	client ctclient.Log
+}
+
+func (client *logClient) GetSTH(ctx context.Context) (sth *cttypes.SignedTreeHead, url string, err error) {
+	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		sth, url, err = getAndVerifySTH(ctx, client.log, client.client)
+		return err
+	})
+	return
+}
+func (client *logClient) GetRoots(ctx context.Context) (roots [][]byte, err error) {
+	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		roots, err = client.client.GetRoots(ctx)
+		return err
+	})
+	return
+}
+func (client *logClient) GetEntries(ctx context.Context, startInclusive, endInclusive uint64) (entries []ctclient.Entry, err error) {
+	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		entries, err = client.client.GetEntries(ctx, startInclusive, endInclusive)
+		return err
+	})
+	return
+}
+func (client *logClient) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (tree *merkletree.CollapsedTree, err error) {
+	err = withRetry(ctx, client.config, client.log, -1, func() error {
+		tree, err = client.client.ReconstructTree(ctx, sth)
+		return err
+	})
+	return
+}
+
+type issuerGetter struct {
+	config    *Config
+	log       *loglist.Log
+	logGetter ctclient.IssuerGetter
+}
+
+func (ig *issuerGetter) GetIssuer(ctx context.Context, fingerprint *[32]byte) ([]byte, error) {
+	if issuer, err := ig.config.State.LoadIssuer(ctx, fingerprint); err != nil {
+		log.Printf("error loading cached issuer %x (issuer will be retrieved from log instead): %s", *fingerprint, err)
+	} else if issuer != nil {
+		return issuer, nil
+	}
+
+	var issuer []byte
+	if err := withRetry(ctx, ig.config, ig.log, 7, func() error {
+		var err error
+		issuer, err = ig.logGetter.GetIssuer(ctx, fingerprint)
+		return err
+	}); err != nil {
+		return nil, err
+	}
+
+	if err := ig.config.State.StoreIssuer(ctx, fingerprint, issuer); err != nil {
+		log.Printf("error caching issuer %x (issuer will be re-retrieved from log in the future): %s", *fingerprint, err)
+	}
+
+	return issuer, nil
+}
+
+func newLogClient(config *Config, ctlog *loglist.Log) (ctclient.Log, ctclient.IssuerGetter, error) {
+	switch {
+	case ctlog.IsRFC6962():
+		logURL, err := url.Parse(ctlog.URL)
+		if err != nil {
+			return nil, nil, fmt.Errorf("log has invalid URL: %w", err)
+		}
+		return &logClient{
+			config: config,
+			log:    ctlog,
+			client: &ctclient.RFC6962Log{URL: logURL},
+		}, nil, nil
+	case ctlog.IsStaticCTAPI():
+		submissionURL, err := url.Parse(ctlog.SubmissionURL)
+		if err != nil {
+			return nil, nil, fmt.Errorf("log has invalid submission URL: %w", err)
+		}
+		monitoringURL, err := url.Parse(ctlog.MonitoringURL)
+		if err != nil {
+			return nil, nil, fmt.Errorf("log has invalid monitoring URL: %w", err)
+		}
+		client := &ctclient.StaticLog{
+			SubmissionURL: submissionURL,
+			MonitoringURL: monitoringURL,
+			ID:            ctlog.LogID,
+		}
+		return &logClient{
+				config: config,
+				log:    ctlog,
+				client: client,
+			}, &issuerGetter{
+				config:    config,
+				log:       ctlog,
+				logGetter: client,
+			}, nil
+	default:
+		return nil, nil, errors.New("log uses unknown protocol")
+	}
+}
+
+func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) (returnedErr error) {
+	client, issuerGetter, err := newLogClient(config, ctlog)
 	if err != nil {
 		return err
 	}
+	if err := config.State.PrepareLog(ctx, ctlog.LogID); err != nil {
+		return fmt.Errorf("error preparing state: %w", err)
+	}
+	state, err := config.State.LoadLogState(ctx, ctlog.LogID)
+	if err != nil {
+		return fmt.Errorf("error loading log state: %w", err)
+	}
+	if state == nil {
+		if config.StartAtEnd {
+			sth, _, err := client.GetSTH(ctx)
+			if err != nil {
+				return err
+			}
+			tree, err := client.ReconstructTree(ctx, sth)
+			if err != nil {
+				return err
+			}
+			state = &LogState{
+				DownloadPosition: tree,
+				VerifiedPosition: tree,
+				VerifiedSTH:      sth,
+				LastSuccess:      time.Now(),
+			}
+		} else {
+			state = &LogState{
+				DownloadPosition: merkletree.EmptyCollapsedTree(),
+				VerifiedPosition: merkletree.EmptyCollapsedTree(),
+				VerifiedSTH:      nil,
+				LastSuccess:      time.Now(),
+			}
+		}
+		if config.Verbose {
+			log.Printf("%s: monitoring brand new log starting from position %d", ctlog.GetMonitoringURL(), state.DownloadPosition.Size())
+		}
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+			return fmt.Errorf("error storing log state: %w", err)
+		}
+	} else {
+		if config.Verbose {
+			log.Printf("%s: resuming monitoring from position %d", ctlog.GetMonitoringURL(), state.DownloadPosition.Size())
+		}
+	}
 
-	ticker := time.NewTicker(monitorLogInterval)
+	defer func() {
+		storeCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		if err := config.State.StoreLogState(storeCtx, ctlog.LogID, state); err != nil && returnedErr == nil {
+			returnedErr = fmt.Errorf("error storing log state: %w", err)
+		}
+	}()
+
+retry:
+	position := state.DownloadPosition.Size()
+
+	// logs are monitored using the following pipeline of workers, with each worker sending results to the next worker:
+	//     1 getSTHWorker ==> 1 generateBatchesWorker ==> multiple downloadWorkers ==> multiple processWorkers ==> 1 saveStateWorker
+	// getSTHWorker          - periodically download STHs from the log
+	// generateBatchesWorker - generate batches of work
+	// downloadWorkers       - download the entries in each batch
+	// processWorkers        - process the certificates (store/notify if matches watch list) in each batch
+	// saveStateWorker       - builds the Merkle Tree and compares against STHs
+
+	sths := make(chan *cttypes.SignedTreeHead, 1)
+	batches := make(chan *batch, downloadWorkers(ctlog))
+	processedBatches := sequencer.New[batch](0, uint64(downloadWorkers(ctlog))*10)
+
+	group, gctx := errgroup.WithContext(ctx)
+	group.Go(func() error { return getSTHWorker(gctx, config, ctlog, client, sths) })
+	group.Go(func() error { return generateBatchesWorker(gctx, config, ctlog, position, sths, batches) })
+	for range downloadWorkers(ctlog) {
+		downloadedBatches := make(chan *batch, 1)
+		group.Go(func() error { return downloadWorker(gctx, config, ctlog, client, batches, downloadedBatches) })
+		group.Go(func() error {
+			return processWorker(gctx, config, ctlog, issuerGetter, downloadedBatches, processedBatches)
+		})
+	}
+	group.Go(func() error { return saveStateWorker(gctx, config, ctlog, state, processedBatches) })
+
+	err = group.Wait()
+	if verifyErr := (*verifyEntriesError)(nil); errors.As(err, &verifyErr) {
+		recordError(ctx, config, ctlog, verifyErr)
+		state.rewindDownloadPosition()
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+			return fmt.Errorf("error storing log state: %w", err)
+		}
+		if err := sleep(ctx, 5*time.Minute); err != nil {
+			return err
+		}
+		goto retry
+	}
+	return err
+}
+
+func getSTHWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log, sthsOut chan<- *cttypes.SignedTreeHead) error {
+	ticker := time.NewTicker(getSTHInterval)
 	defer ticker.Stop()
-
+	for {
-	for ctx.Err() == nil {
+		sth, _, err := client.GetSTH(ctx)
-		if err := monitorLog(ctx, config, ctlog, logClient); err != nil {
+		if err != nil {
 			return err
 		}
 		select {
 		case <-ctx.Done():
+			return ctx.Err()
+		case sthsOut <- sth:
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
 		case <-ticker.C:
 		}
 	}
-	return ctx.Err()
 }
 
-func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClient *client.LogClient) (returnedErr error) {
+type batch struct {
-	ctx, cancel := context.WithCancel(ctx)
+	number       uint64
-	defer cancel()
+	begin, end   uint64
+	discoveredAt time.Time        // time at which we became aware of the log having entries in range [begin,end)
+	sths         []*StoredSTH     // STHs with sizes in range [begin,end], sorted by TreeSize
+	entries      []ctclient.Entry // in range [begin,end)
+}
 
-	var (
+// Create a batch starting from begin, based on sths (which must be non-empty, sorted by TreeSize, and contain only STHs with TreeSize >= begin).  Returns the batch, plus the remaining STHs.
-		stateDirPath        = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString())
+func newBatch(number uint64, begin uint64, sths []*StoredSTH, downloadJobSize uint64) (*batch, []*StoredSTH) {
-		stateFilePath       = filepath.Join(stateDirPath, "state.json")
+	batch := &batch{
-		sthsDirPath         = filepath.Join(stateDirPath, "unverified_sths")
+		number:       number,
-		malformedDirPath    = filepath.Join(stateDirPath, "malformed_entries")
+		begin:        begin,
-		healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
+		discoveredAt: sths[0].StoredAt,
-	)
+	}
-	for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} {
+	maxEnd := (begin/downloadJobSize + 1) * downloadJobSize
-		if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+	for _, sth := range sths {
-			return fmt.Errorf("error creating state directory: %w", err)
+		if sth.StoredAt.Before(batch.discoveredAt) {
+			batch.discoveredAt = sth.StoredAt
 		}
-	}
+		if sth.TreeSize <= maxEnd {
-
+			batch.end = sth.TreeSize
-	startTime := time.Now()
+			batch.sths = append(batch.sths, sth)
-	latestSTH, err := logClient.GetSTH(ctx)
-	if isFatalLogError(err) {
-		return err
-	} else if err != nil {
-		recordError(fmt.Errorf("error fetching latest STH for %s: %w", ctlog.URL, err))
-		return nil
-	}
-	latestSTH.LogID = ctlog.LogID
-	if err := storeSTHInDir(sthsDirPath, latestSTH); err != nil {
-		return fmt.Errorf("error storing latest STH: %w", err)
-	}
-
-	state, err := loadStateFile(stateFilePath)
-	if errors.Is(err, fs.ErrNotExist) {
-		if config.StartAtEnd {
-			tree, err := reconstructTree(ctx, logClient, latestSTH)
-			if isFatalLogError(err) {
-				return err
-			} else if err != nil {
-				recordError(fmt.Errorf("error reconstructing tree of size %d for %s: %w", latestSTH.TreeSize, ctlog.URL, err))
-				return nil
-			}
-			state = &stateFile{
-				DownloadPosition: tree,
-				VerifiedPosition: tree,
-				VerifiedSTH:      latestSTH,
-				LastSuccess:      startTime.UTC(),
-			}
 		} else {
-			state = &stateFile{
+			batch.end = maxEnd
-				DownloadPosition: merkletree.EmptyCollapsedTree(),
+			break
-				VerifiedPosition: merkletree.EmptyCollapsedTree(),
-				VerifiedSTH:      nil,
-				LastSuccess:      startTime.UTC(),
-			}
 		}
-		if config.Verbose {
-			log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size())
-		}
-		if err := state.store(stateFilePath); err != nil {
-			return fmt.Errorf("error storing state file: %w", err)
-		}
-	} else if err != nil {
-		return fmt.Errorf("error loading state file: %w", err)
 	}
+	return batch, sths[len(batch.sths):]
+}
 
-	sths, err := loadSTHsFromDir(sthsDirPath)
+// insert sth into sths, which is sorted by TreeSize, and return a new, still-sorted slice.
+// if an equivalent STH is already in sths, it is returned unchanged.
+func insertSTH(sths []*StoredSTH, sth *StoredSTH) []*StoredSTH {
+	i := len(sths)
+	for i > 0 {
+		if sths[i-1].Same(&sth.SignedTreeHead) {
+			return sths
+		}
+		if sths[i-1].TreeSize < sth.TreeSize {
+			break
+		}
+		i--
+	}
+	return slices.Insert(sths, i, sth)
+}
+
+func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.Log, position uint64, sthsIn <-chan *cttypes.SignedTreeHead, batchesOut chan<- *batch) error {
+	downloadJobSize := downloadJobSize(ctlog)
+
+	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
 	if err != nil {
-		return fmt.Errorf("error loading STHs directory: %w", err)
+		return fmt.Errorf("error loading STHs: %w", err)
 	}
-
+	// sths is sorted by TreeSize but may contain STHs with TreeSize < position; get rid of them
-	for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() {
+	for len(sths) > 0 && sths[0].TreeSize < position {
-		// TODO-4: audit sths[0] against state.VerifiedSTH
+		// TODO-4: audit sths[0] against log's verified STH
-		if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
+		if err := config.State.RemoveSTH(ctx, ctlog.LogID, &sths[0].SignedTreeHead); err != nil {
 			return fmt.Errorf("error removing STH: %w", err)
 		}
 		sths = sths[1:]
 	}
-
+	// from this point, sths is sorted by TreeSize and contains only STHs with TreeSize >= position
-	defer func() {
+	handleSTH := func(sth *cttypes.SignedTreeHead) error {
-		if config.Verbose {
+		if sth.TreeSize < position {
-			log.Printf("saving state in defer for %s", ctlog.URL)
+			// TODO-4: audit against log's verified STH
+		} else {
+			storedSTH, err := config.State.StoreSTH(ctx, ctlog.LogID, sth)
+			if err != nil {
+				return fmt.Errorf("error storing STH: %w", err)
+			}
+			sths = insertSTH(sths, storedSTH)
 		}
-		if err := state.store(stateFilePath); err != nil && returnedErr == nil {
-			returnedErr = fmt.Errorf("error storing state file: %w", err)
-		}
-	}()
-
-	if len(sths) == 0 {
-		state.LastSuccess = startTime.UTC()
 		return nil
 	}
 
-	var (
+	var number uint64
-		downloadBegin = state.DownloadPosition.Size()
+	for {
-		downloadEnd   = sths[len(sths)-1].TreeSize
+		for len(sths) == 0 {
-		entries       = make(chan client.GetEntriesItem, maxGetEntriesSize)
-		downloadErr   error
-	)
-	if config.Verbose {
-		log.Printf("downloading entries from %s in range [%d, %d)", ctlog.URL, downloadBegin, downloadEnd)
-	}
-	go func() {
-		defer close(entries)
-		downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd)
-	}()
-	for rawEntry := range entries {
-		entry := &logEntry{
-			Log:       ctlog,
-			Index:     state.DownloadPosition.Size(),
-			LeafInput: rawEntry.LeafInput,
-			ExtraData: rawEntry.ExtraData,
-			LeafHash:  merkletree.HashLeaf(rawEntry.LeafInput),
-		}
-		if err := processLogEntry(ctx, config, entry); err != nil {
-			return fmt.Errorf("error processing entry %d: %w", entry.Index, err)
-		}
-
-		state.DownloadPosition.Add(entry.LeafHash)
-		rootHash := state.DownloadPosition.CalculateRoot()
-		shouldSaveState := state.DownloadPosition.Size()%10000 == 0
-
-		for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize {
-			if merkletree.Hash(sths[0].SHA256RootHash) != rootHash {
-				recordError(fmt.Errorf("error verifying %s at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", ctlog.URL, sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
-
-				state.DownloadPosition = state.VerifiedPosition
-				if err := state.store(stateFilePath); err != nil {
-					return fmt.Errorf("error storing state file: %w", err)
-				}
-				return nil
-			}
-
-			state.VerifiedPosition = state.DownloadPosition
-			state.VerifiedSTH = sths[0]
-			shouldSaveState = true
-			if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
-				return fmt.Errorf("error removing verified STH: %w", err)
-			}
-
-			sths = sths[1:]
-		}
-
-		if shouldSaveState {
-			if err := state.store(stateFilePath); err != nil {
-				return fmt.Errorf("error storing state file: %w", err)
-			}
-		}
-	}
-
-	if isFatalLogError(downloadErr) {
-		return downloadErr
-	} else if downloadErr != nil {
-		recordError(fmt.Errorf("error downloading entries from %s: %w", ctlog.URL, downloadErr))
-		return nil
-	}
-
-	if config.Verbose {
-		log.Printf("finished downloading entries from %s", ctlog.URL)
-	}
-
-	state.LastSuccess = startTime.UTC()
-	return nil
-}
-
-func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error {
-	for begin < end && ctx.Err() == nil {
-		size := begin - end
-		if size > maxGetEntriesSize {
-			size = maxGetEntriesSize
-		}
-		entries, err := logClient.GetRawEntries(ctx, begin, begin+size-1)
-		if err != nil {
-			return err
-		}
-		for _, entry := range entries {
-			if ctx.Err() != nil {
-				return ctx.Err()
-			}
 			select {
 			case <-ctx.Done():
 				return ctx.Err()
-			case entriesChan <- entry:
+			case sth := <-sthsIn:
+				if err := handleSTH(sth); err != nil {
+					return err
+				}
 			}
 		}
-		begin += uint64(len(entries))
+
+		batch, remainingSTHs := newBatch(number, position, sths, downloadJobSize)
+
+		if ctlog.IsStaticCTAPI() && batch.end%downloadJobSize != 0 {
+			// Wait to download this partial tile until it's old enough
+			if age := time.Since(batch.discoveredAt); age < maxPartialTileAge {
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				case <-time.After(maxPartialTileAge - age):
+				case sth := <-sthsIn:
+					if err := handleSTH(sth); err != nil {
+						return err
+					}
+					continue
+				}
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case sth := <-sthsIn:
+			if err := handleSTH(sth); err != nil {
+				return err
+			}
+		case batchesOut <- batch:
+			number = batch.number + 1
+			position = batch.end
+			sths = remainingSTHs
+		}
 	}
-	return ctx.Err()
 }
 
-func reconstructTree(ctx context.Context, logClient *client.LogClient, sth *ct.SignedTreeHead) (*merkletree.CollapsedTree, error) {
+func downloadWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log, batchesIn <-chan *batch, batchesOut chan<- *batch) error {
-	if sth.TreeSize == 0 {
+	for {
-		return merkletree.EmptyCollapsedTree(), nil
+		var batch *batch
-	}
+		select {
-	entries, err := logClient.GetRawEntries(ctx, sth.TreeSize-1, sth.TreeSize-1)
+		case <-ctx.Done():
-	if err != nil {
+			return ctx.Err()
-		return nil, err
+		case batch = <-batchesIn:
-	}
+		}
-	leafHash := merkletree.HashLeaf(entries[0].LeafInput)
 
-	var tree *merkletree.CollapsedTree
+		entries, err := getEntriesFull(ctx, client, batch.begin, batch.end)
-	if sth.TreeSize > 1 {
-		// XXX: if leafHash is in the tree in more than one place, this might not return the proof that we need ... get-entry-and-proof avoids this problem but not all logs support it
-		auditPath, _, err := logClient.GetAuditProof(ctx, leafHash[:], sth.TreeSize)
 		if err != nil {
-			return nil, err
+			return err
 		}
-		hashes := make([]merkletree.Hash, len(auditPath))
+		batch.entries = entries
-		for i := range hashes {
-			copy(hashes[i][:], auditPath[len(auditPath)-i-1])
-		}
-		tree, err = merkletree.NewCollapsedTree(hashes, sth.TreeSize-1)
-		if err != nil {
-			return nil, fmt.Errorf("log returned invalid audit proof for %x to %d: %w", leafHash, sth.TreeSize, err)
-		}
-	} else {
-		tree = merkletree.EmptyCollapsedTree()
-	}
 
-	tree.Add(leafHash)
+		select {
-	rootHash := tree.CalculateRoot()
+		case <-ctx.Done():
-	if rootHash != merkletree.Hash(sth.SHA256RootHash) {
+			return ctx.Err()
-		return nil, fmt.Errorf("calculated root hash (%x) does not match signed tree head (%x) at size %d", rootHash, sth.SHA256RootHash, sth.TreeSize)
+		case batchesOut <- batch:
+		}
+	}
+}
+
+func processWorker(ctx context.Context, config *Config, ctlog *loglist.Log, issuerGetter ctclient.IssuerGetter, batchesIn <-chan *batch, batchesOut *sequencer.Channel[batch]) error {
+	for {
+		var batch *batch
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case batch = <-batchesIn:
+		}
+		for offset, entry := range batch.entries {
+			index := batch.begin + uint64(offset)
+			if err := processLogEntry(ctx, config, issuerGetter, &LogEntry{
+				Entry: entry,
+				Index: index,
+				Log:   ctlog,
+			}); err != nil {
+				return fmt.Errorf("error processing entry %d: %w", index, err)
+			}
+		}
+		if err := batchesOut.Add(ctx, batch.number, batch); err != nil {
+			return err
+		}
+	}
+}
+
+func saveStateWorker(ctx context.Context, config *Config, ctlog *loglist.Log, state *LogState, batchesIn *sequencer.Channel[batch]) error {
+	for {
+		batch, err := batchesIn.Next(ctx)
+		if err != nil {
+			return err
+		}
+		if batch.begin != state.DownloadPosition.Size() {
+			panic(fmt.Errorf("saveStateWorker: expected batch to start at %d but got %d instead", state.DownloadPosition.Size(), batch.begin))
+		}
+		for {
+			for len(batch.sths) > 0 && batch.sths[0].TreeSize == state.DownloadPosition.Size() {
+				sth := batch.sths[0]
+				batch.sths = batch.sths[1:]
+				if rootHash := state.DownloadPosition.CalculateRoot(); sth.RootHash != rootHash {
+					return &verifyEntriesError{
+						sth:             &sth.SignedTreeHead,
+						entriesRootHash: rootHash,
+					}
+				}
+				state.advanceVerifiedPosition()
+				state.LastSuccess = sth.StoredAt
+				state.VerifiedSTH = &sth.SignedTreeHead
+				if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+					return fmt.Errorf("error storing log state: %w", err)
+				}
+				// don't remove the STH until state has been durably stored
+				if err := config.State.RemoveSTH(ctx, ctlog.LogID, &sth.SignedTreeHead); err != nil {
+					return fmt.Errorf("error removing verified STH: %w", err)
+				}
+				if config.Verbose {
+					log.Printf("%s: verified position is now %d", ctlog.GetMonitoringURL(), sth.SignedTreeHead.TreeSize)
+				}
+			}
+			if len(batch.entries) == 0 {
+				break
+			}
+			entry := batch.entries[0]
+			batch.entries = batch.entries[1:]
+			leafHash := merkletree.HashLeaf(entry.LeafInput())
+			state.DownloadPosition.Add(leafHash)
+		}
+
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+			return fmt.Errorf("error storing log state: %w", err)
+		}
+	}
+}
+
+func sleep(ctx context.Context, duration time.Duration) error {
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
 	}
-
-	return tree, nil
 }

monitor/notify.go

@@ -20,35 +20,36 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 )
 
 var stdoutMu sync.Mutex
 
-type notification interface {
+type notification struct {
-	Environ() []string
+	environ []string
-	Summary() string
+	summary string
-	Text() string
+	text    string
 }
 
-func notify(ctx context.Context, config *Config, notif notification) error {
+func (s *FilesystemState) notify(ctx context.Context, notif *notification) error {
-	if config.Stdout {
+	if s.Stdout {
 		writeToStdout(notif)
 	}
 
-	if len(config.Email) > 0 {
+	if len(s.Email) > 0 {
-		if err := sendEmail(ctx, config.SendmailPath, config.Email, notif); err != nil {
+		if err := sendEmail(ctx, s.Email, notif); err != nil {
 			return err
 		}
 	}
 
-	if config.Script != "" {
+	if s.Script != "" {
-		if err := execScript(ctx, config.Script, notif); err != nil {
+		if err := execScript(ctx, s.Script, notif); err != nil {
 			return err
 		}
 	}
 
-	if config.ScriptDir != "" {
+	if s.ScriptDir != "" {
-		if err := execScriptDir(ctx, config.ScriptDir, notif); err != nil {
+		if err := execScriptDir(ctx, s.ScriptDir, notif); err != nil {
 			return err
 		}
 	}
@@ -56,53 +57,74 @@ func notify(ctx context.Context, config *Config, notif notification) error {
 	return nil
 }
 
-func writeToStdout(notif notification) {
+func writeToStdout(notif *notification) {
 	stdoutMu.Lock()
 	defer stdoutMu.Unlock()
-	os.Stdout.WriteString(notif.Text() + "\n")
+	os.Stdout.WriteString(notif.text + "\n")
 }
 
-func sendEmail(ctx context.Context, sendmailPath string, to []string, notif notification) error {
+func sendEmail(ctx context.Context, to []string, notif *notification) error {
 	stdin := new(bytes.Buffer)
 	stderr := new(bytes.Buffer)
 
+	from := os.Getenv("EMAIL")
+
+	if from != "" {
+		fmt.Fprintf(stdin, "From: %s\n", from)
+	}
 	fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", "))
-	fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.Summary())
+	fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.summary)
+	fmt.Fprintf(stdin, "Date: %s\n", time.Now().Format(mailDateFormat))
+	fmt.Fprintf(stdin, "Message-ID: <%s>\n", generateMessageID())
 	fmt.Fprintf(stdin, "Mime-Version: 1.0\n")
 	fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n")
 	fmt.Fprintf(stdin, "X-Mailer: certspotter\n")
 	fmt.Fprintf(stdin, "\n")
-	fmt.Fprint(stdin, notif.Text())
+	fmt.Fprint(stdin, notif.text)
 
-	args := []string{"-i", "--"}
+	args := []string{"-i"}
+	if from != "" {
+		args = append(args, "-f", from)
+	}
+	args = append(args, "--")
 	args = append(args, to...)
 
-	sendmail := exec.CommandContext(ctx, sendmailPath, args...)
+	sendmailCtx, cancel := context.WithDeadline(ctx, time.Now().Add(2*time.Minute))
+	defer cancel()
+	sendmail := exec.CommandContext(sendmailCtx, sendmailPath(), args...)
 	sendmail.Stdin = stdin
 	sendmail.Stderr = stderr
+	sendmail.WaitDelay = 5 * time.Second
 
-	if err := sendmail.Run(); err == nil {
+	if err := sendmail.Run(); err == nil || err == exec.ErrWaitDelay {
 		return nil
+	} else if sendmailCtx.Err() != nil && ctx.Err() == nil {
+		return fmt.Errorf("error sending email to %v: sendmail command timed out", to)
 	} else if ctx.Err() != nil {
+		// if the context was canceled, we can't be sure that the error is the fault of sendmail, so ignore it
 		return ctx.Err()
 	} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
 		return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
+	} else if isExitError {
+		return fmt.Errorf("error sending email to %v: sendmail terminated by signal with error %q", to, strings.TrimSpace(stderr.String()))
 	} else {
-		return fmt.Errorf("error sending email to %v: %w", to, err)
+		return fmt.Errorf("error sending email to %v: error running sendmail command: %w", to, err)
 	}
 }
 
-func execScript(ctx context.Context, scriptName string, notif notification) error {
+func execScript(ctx context.Context, scriptName string, notif *notification) error {
 	stderr := new(bytes.Buffer)
 
 	cmd := exec.CommandContext(ctx, scriptName)
 	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, notif.Environ()...)
+	cmd.Env = append(cmd.Env, notif.environ...)
 	cmd.Stderr = stderr
+	cmd.WaitDelay = 5 * time.Second
 
-	if err := cmd.Run(); err == nil {
+	if err := cmd.Run(); err == nil || err == exec.ErrWaitDelay {
 		return nil
 	} else if ctx.Err() != nil {
+		// if the context was canceled, we can't be sure that the error is the fault of the script, so ignore it
 		return ctx.Err()
 	} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
 		return fmt.Errorf("script %q exited with code %d and error %q", scriptName, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
@@ -113,7 +135,7 @@ func execScript(ctx context.Context, scriptName string, notif notification) erro
 	}
 }
 
-func execScriptDir(ctx context.Context, dirPath string, notif notification) error {
+func execScriptDir(ctx context.Context, dirPath string, notif *notification) error {
 	dirents, err := os.ReadDir(dirPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil

monitor/process.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2023 Opsmate, Inc.
+// Copyright (C) 2025 Opsmate, Inc.
 //
 // This Source Code Form is subject to the terms of the Mozilla
 // Public License, v. 2.0. If a copy of the MPL was not distributed
@@ -10,84 +10,93 @@
 package monitor
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha256"
-	"encoding/hex"
 	"errors"
 	"fmt"
-	"io/fs"
+
-	"os"
-	"path/filepath"
 	"software.sslmate.com/src/certspotter"
-	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/ctclient"
+	"software.sslmate.com/src/certspotter/cttypes"
 	"software.sslmate.com/src/certspotter/loglist"
-	"software.sslmate.com/src/certspotter/merkletree"
 )
 
-type logEntry struct {
+type LogEntry struct {
-	Log       *loglist.Log
+	ctclient.Entry
-	Index     uint64
+	Index uint64
-	LeafInput []byte
+	Log   *loglist.Log
-	ExtraData []byte
-	LeafHash  merkletree.Hash
 }
 
-func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error {
+func processLogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry) error {
-	leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput))
+	leaf, err := cttypes.ParseLeafInput(entry.LeafInput())
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err))
 	}
 	switch leaf.TimestampedEntry.EntryType {
-	case ct.X509LogEntryType:
+	case cttypes.X509EntryType:
-		return processX509LogEntry(ctx, config, entry, leaf.TimestampedEntry.X509Entry)
+		return processX509LogEntry(ctx, config, issuerGetter, entry, leaf.TimestampedEntry.SignedEntryASN1Cert)
-	case ct.PrecertLogEntryType:
+	case cttypes.PrecertEntryType:
-		return processPrecertLogEntry(ctx, config, entry, leaf.TimestampedEntry.PrecertEntry)
+		return processPrecertLogEntry(ctx, config, issuerGetter, entry, leaf.TimestampedEntry.SignedEntryPreCert)
 	default:
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType))
 	}
 }
 
-func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, cert ct.ASN1Cert) error {
+func processX509LogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry, cert *cttypes.ASN1Cert) error {
-	certInfo, err := certspotter.MakeCertInfoFromRawCert(cert)
+	certInfo, err := certspotter.MakeCertInfoFromRawCert(*cert)
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err))
 	}
-
-	chain, err := ct.UnmarshalX509ChainArray(entry.ExtraData)
-	if err != nil {
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for X.509 entry: %w", err))
-	}
-	chain = append([]ct.ASN1Cert{cert}, chain...)
-
 	if precertTBS, err := certspotter.ReconstructPrecertTBS(certInfo.TBS); err == nil {
 		certInfo.TBS = precertTBS
 	} else {
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error reconstructing precertificate TBSCertificate: %w", err))
 	}
 
-	return processCertificate(ctx, config, entry, certInfo, chain)
+	getChain := func(ctx context.Context) ([]cttypes.ASN1Cert, error) {
+		var (
+			chain = []cttypes.ASN1Cert{*cert}
+			errs  = []error{}
+		)
+		if issuers, err := entry.GetChain(ctx, issuerGetter); err == nil {
+			chain = append(chain, issuers...)
+		} else {
+			errs = append(errs, err)
+		}
+		return chain, errors.Join(errs...)
+	}
+	return processCertificate(ctx, config, entry, certInfo, getChain)
 }
 
-func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry, precert ct.PreCert) error {
+func processPrecertLogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry, precert *cttypes.PreCert) error {
 	certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate)
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err))
 	}
-
+	precertBytes, err := entry.Precertificate()
-	chain, err := ct.UnmarshalPrecertChainArray(entry.ExtraData)
 	if err != nil {
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for precert entry: %w", err))
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error getting precert entry's precertificate: %w", err))
 	}
 
-	if _, err := certspotter.ValidatePrecert(chain[0], precert.TBSCertificate); err != nil {
+	getChain := func(ctx context.Context) ([]cttypes.ASN1Cert, error) {
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err))
+		var (
+			chain = []cttypes.ASN1Cert{precertBytes}
+			errs  = []error{}
+		)
+		if issuers, err := entry.GetChain(ctx, issuerGetter); err == nil {
+			chain = append(chain, issuers...)
+		} else {
+			errs = append(errs, err)
+		}
+		if _, err := certspotter.ValidatePrecert(precertBytes, precert.TBSCertificate); err != nil {
+			errs = append(errs, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err))
+		}
+		return chain, errors.Join(errs...)
 	}
-
+	return processCertificate(ctx, config, entry, certInfo, getChain)
-	return processCertificate(ctx, config, entry, certInfo, chain)
 }
 
-func processCertificate(ctx context.Context, config *Config, entry *logEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
+func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, getChain func(context.Context) ([]cttypes.ASN1Cert, error)) error {
 	identifiers, err := certInfo.ParseIdentifiers()
 	if err != nil {
 		return processMalformedLogEntry(ctx, config, entry, err)
@@ -97,80 +106,40 @@ func processCertificate(ctx context.Context, config *Config, entry *logEntry, ce
 		return nil
 	}
 
-	cert := &discoveredCert{
+	chain, chainErr := getChain(ctx)
+	if chainErr != nil {
+		if ctx.Err() != nil {
+			// Getting chain failed, but it was probably because our context
+			// has been canceled, so just act like we never called getChain.
+			return ctx.Err()
+		}
+		// Although getting the chain failed, we still want to notify
+		// the user about the matching certificate. We'll include chainErr in the
+		// notification so the user knows why the chain is missing or incorrect.
+	}
+
+	cert := &DiscoveredCert{
 		WatchItem:    watchItem,
 		LogEntry:     entry,
 		Info:         certInfo,
 		Chain:        chain,
+		ChainError:   chainErr,
 		TBSSHA256:    sha256.Sum256(certInfo.TBS.Raw),
 		SHA256:       sha256.Sum256(chain[0]),
 		PubkeySHA256: sha256.Sum256(certInfo.TBS.PublicKey.FullBytes),
 		Identifiers:  identifiers,
 	}
 
-	var notifiedPath string
+	if err := config.State.NotifyCert(ctx, cert); err != nil {
-	if config.SaveCerts {
+		return fmt.Errorf("error notifying about certificate %x: %w", cert.SHA256, err)
-		hexFingerprint := hex.EncodeToString(cert.SHA256[:])
-		prefixPath := filepath.Join(config.StateDir, "certs", hexFingerprint[0:2])
-		var (
-			notifiedFilename      = "." + hexFingerprint + ".notified"
-			certFilename          = hexFingerprint + ".pem"
-			jsonFilename          = hexFingerprint + ".v1.json"
-			textFilename          = hexFingerprint + ".txt"
-			legacyCertFilename    = hexFingerprint + ".cert.pem"
-			legacyPrecertFilename = hexFingerprint + ".precert.pem"
-		)
-
-		for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} {
-			if fileExists(filepath.Join(prefixPath, filename)) {
-				return nil
-			}
-		}
-
-		if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
-			return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
-		}
-
-		notifiedPath = filepath.Join(prefixPath, notifiedFilename)
-		cert.CertPath = filepath.Join(prefixPath, certFilename)
-		cert.JSONPath = filepath.Join(prefixPath, jsonFilename)
-		cert.TextPath = filepath.Join(prefixPath, textFilename)
-
-		if err := cert.save(); err != nil {
-			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
-		}
-	} else {
-		// TODO-4: save cert to temporary files, and defer their unlinking
-	}
-
-	if err := notify(ctx, config, cert); err != nil {
-		return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
-	}
-
-	if notifiedPath != "" {
-		if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
-			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
-		}
 	}
 
 	return nil
 }
 
-func processMalformedLogEntry(ctx context.Context, config *Config, entry *logEntry, parseError error) error {
+func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error {
-	dirPath := filepath.Join(config.StateDir, "logs", entry.Log.LogID.Base64URLString(), "malformed_entries")
+	if err := config.State.NotifyMalformedEntry(ctx, entry, parseError); err != nil {
-	malformed := &malformedLogEntry{
+		return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.GetMonitoringURL(), parseError, err)
-		Entry:     entry,
-		Error:     parseError.Error(),
-		EntryPath: filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index)),
-		TextPath:  filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index)),
-	}
-
-	if err := malformed.save(); err != nil {
-		return fmt.Errorf("error saving malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
-	}
-
-	if err := notify(ctx, config, malformed); err != nil {
-		return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
 	}
 	return nil
 }

monitor/state.go

@@ -0,0 +1,91 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"time"
+
+	"software.sslmate.com/src/certspotter/cttypes"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type LogState struct {
+	DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
+	VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
+	VerifiedSTH      *cttypes.SignedTreeHead   `json:"verified_sth"`
+	LastSuccess      time.Time                 `json:"last_success"`
+}
+
+func (state *LogState) rewindDownloadPosition() {
+	position := state.VerifiedPosition.Clone()
+	state.DownloadPosition = &position
+}
+
+func (state *LogState) advanceVerifiedPosition() {
+	position := state.DownloadPosition.Clone()
+	state.VerifiedPosition = &position
+}
+
+// Methods are safe to call concurrently.
+type StateProvider interface {
+	// Initialize the state.  Called before any other method in this interface.
+	// Idempotent: returns nil if the state is already initialized.
+	Prepare(context.Context) error
+
+	// Initialize the state for the given log.  Called before any other method
+	// with the log ID.  Idempotent: returns nil if log state already initialized.
+	PrepareLog(context.Context, LogID) error
+
+	// Store log state for retrieval by LoadLogState.
+	StoreLogState(context.Context, LogID, *LogState) error
+
+	// Load log state that was previously stored with StoreLogState.
+	// Returns nil, nil if StoreLogState has not been called yet for this log.
+	LoadLogState(context.Context, LogID) (*LogState, error)
+
+	// Store STH for retrieval by LoadSTHs.  If an STH with the same
+	// timestamp and root hash is already stored, this STH can be ignored.
+	StoreSTH(context.Context, LogID, *cttypes.SignedTreeHead) (*StoredSTH, error)
+
+	// Load all STHs for this log previously stored with StoreSTH.
+	// The returned slice must be sorted by tree size.
+	LoadSTHs(context.Context, LogID) ([]*StoredSTH, error)
+
+	// Remove an STH so it is no longer returned by LoadSTHs.
+	RemoveSTH(context.Context, LogID, *cttypes.SignedTreeHead) error
+
+	// Store a DER-encoded issuer certificate with the given fingerprint for
+	// retrieval by LoadIssuer.  Returns nil if the issuer has already been stored.
+	StoreIssuer(context.Context, *[32]byte, []byte) error
+
+	// Retrieve a DER-encoded issuer certificate previously stored with StoreIssuer.
+	// Returns nil, nil if this issuer certificate has not been stored.
+	LoadIssuer(context.Context, *[32]byte) ([]byte, error)
+
+	// Called when a certificate matching the watch list is discovered.
+	NotifyCert(context.Context, *DiscoveredCert) error
+
+	// Called when certspotter fails to parse a log entry.
+	NotifyMalformedEntry(context.Context, *LogEntry, error) error
+
+	// Called when a health check fails.  The log is nil if the
+	// feailure is not associated with a log.
+	NotifyHealthCheckFailure(context.Context, *loglist.Log, HealthCheckFailure) error
+
+	// Called when a non-fatal error occurs.  The log is nil if the error is
+	// not associated with a log.  Note that most errors are transient, and
+	// certspotter will retry the failed operation later.
+	NotifyError(context.Context, *loglist.Log, error) error
+
+	// Retrieve the specified number of most recent errors.
+	GetErrors(context.Context, *loglist.Log, int) (string, error)
+}

monitor/statedir.go

@@ -16,7 +16,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/cttypes"
 	"software.sslmate.com/src/certspotter/merkletree"
 	"strconv"
 	"strings"
@@ -50,7 +50,7 @@ func writeVersion(stateDir string) error {
 }
 
 func migrateLogStateDirV1(dir string) error {
-	var sth ct.SignedTreeHead
+	var sth cttypes.SignedTreeHead
 	var tree merkletree.CollapsedTree
 
 	sthPath := filepath.Join(dir, "sth.json")
@@ -76,13 +76,13 @@ func migrateLogStateDirV1(dir string) error {
 		return fmt.Errorf("error unmarshaling %s: %w", treePath, err)
 	}
 
-	stateFile := stateFile{
+	stateFile := LogState{
 		DownloadPosition: &tree,
 		VerifiedPosition: &tree,
 		VerifiedSTH:      &sth,
-		LastSuccess:      time.Now().UTC(),
+		LastSuccess:      time.Now(),
 	}
-	if stateFile.store(filepath.Join(dir, "state.json")); err != nil {
+	if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil {
 		return err
 	}
 
@@ -145,7 +145,7 @@ func prepareStateDir(stateDir string) error {
 		return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir)
 	}
 
-	for _, subdir := range []string{"certs", "logs", "healthchecks"} {
+	for _, subdir := range []string{"certs", "logs", "healthchecks", "errors"} {
 		if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) {
 			return err
 		}
@@ -153,3 +153,15 @@ func prepareStateDir(stateDir string) error {
 
 	return nil
 }
+
+func prepareCacheDir(cacheDir string) error {
+	if err := os.MkdirAll(cacheDir, 0777); err != nil {
+		return err
+	}
+	for _, subdir := range []string{"issuers"} {
+		if err := os.Mkdir(filepath.Join(cacheDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return err
+		}
+	}
+	return nil
+}

monitor/statefile.go

@@ -1,42 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/merkletree"
-	"time"
-)
-
-type stateFile struct {
-	DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
-	VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
-	VerifiedSTH      *ct.SignedTreeHead        `json:"verified_sth"`
-	LastSuccess      time.Time                 `json:"last_success"`
-}
-
-func loadStateFile(filePath string) (*stateFile, error) {
-	fileBytes, err := os.ReadFile(filePath)
-	if err != nil {
-		return nil, err
-	}
-	file := new(stateFile)
-	if err := json.Unmarshal(fileBytes, file); err != nil {
-		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
-	}
-	return file, nil
-}
-
-func (file *stateFile) store(filePath string) error {
-	return writeJSONFile(filePath, file, 0666)
-}

monitor/sthdir.go

@@ -17,59 +17,95 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"slices"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
-	"software.sslmate.com/src/certspotter/ct"
+	"slices"
+	"software.sslmate.com/src/certspotter/cttypes"
 	"strconv"
 	"strings"
+	"time"
 )
 
-func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) {
+type StoredSTH struct {
+	cttypes.SignedTreeHead
+	StoredAt time.Time `json:"stored_at"` // time at which the STH was first stored
+}
+
+func loadSTHsFromDir(dirPath string) ([]*StoredSTH, error) {
 	entries, err := os.ReadDir(dirPath)
 	if errors.Is(err, fs.ErrNotExist) {
-		return []*ct.SignedTreeHead{}, nil
+		return []*StoredSTH{}, nil
 	} else if err != nil {
 		return nil, err
 	}
-	sths := make([]*ct.SignedTreeHead, 0, len(entries))
+	sths := make([]*StoredSTH, 0, len(entries))
 	for _, entry := range entries {
 		filename := entry.Name()
 		if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") {
 			continue
 		}
 		sth, err := readSTHFile(filepath.Join(dirPath, filename))
-		if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		} else if err != nil {
 			return nil, err
 		}
 		sths = append(sths, sth)
 	}
-	slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) int { return cmp.Compare(a.TreeSize, b.TreeSize) })
+	slices.SortFunc(sths, func(a, b *StoredSTH) int { return cmp.Compare(a.TreeSize, b.TreeSize) })
 	return sths, nil
 }
 
-func readSTHFile(filePath string) (*ct.SignedTreeHead, error) {
+func readSTHFile(filePath string) (*StoredSTH, error) {
-	fileBytes, err := os.ReadFile(filePath)
+	file, err := os.Open(filePath)
 	if err != nil {
 		return nil, err
 	}
-	sth := new(ct.SignedTreeHead)
+	defer file.Close()
-	if err := json.Unmarshal(fileBytes, sth); err != nil {
+
+	info, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+	sth := &StoredSTH{
+		StoredAt: info.ModTime(),
+	}
+
+	fileBytes, err := io.ReadAll(file)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %s: %w", filePath, err)
+	}
+	if err := json.Unmarshal(fileBytes, &sth.SignedTreeHead); err != nil {
 		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
 	}
 	return sth, nil
 }
 
-func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error {
+func storeSTHInDir(dirPath string, sth *cttypes.SignedTreeHead) (*StoredSTH, error) {
 	filePath := filepath.Join(dirPath, sthFilename(sth))
-	if fileExists(filePath) {
+
-		return nil
+	if info, err := os.Lstat(filePath); err == nil {
+		return &StoredSTH{
+			SignedTreeHead: *sth,
+			StoredAt:       info.ModTime(),
+		}, nil
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return nil, err
 	}
-	return writeJSONFile(filePath, sth, 0666)
+
+	if err := writeJSONFile(filePath, sth, 0666); err != nil {
+		return nil, err
+	}
+
+	return &StoredSTH{
+		SignedTreeHead: *sth,
+		StoredAt:       time.Now(), // not the exact modtime of the file, but close enough for our purposes
+	}, nil
 }
 
-func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error {
+func removeSTHFromDir(dirPath string, sth *cttypes.SignedTreeHead) error {
 	filePath := filepath.Join(dirPath, sthFilename(sth))
 	err := os.Remove(filePath)
 	if err != nil && !errors.Is(err, fs.ErrNotExist) {
@@ -79,15 +115,9 @@ func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error {
 }
 
 // generate a filename that uniquely identifies the STH (within the context of a particular log)
-func sthFilename(sth *ct.SignedTreeHead) string {
+func sthFilename(sth *cttypes.SignedTreeHead) string {
 	hasher := sha256.New()
-	switch sth.Version {
+	binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
-	case ct.V1:
+	hasher.Write(sth.RootHash[:])
-		binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
-		binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash)
-	default:
-		panic(fmt.Errorf("sthFilename: invalid STH version %d", sth.Version))
-	}
-	// For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic)
 	return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json"
 }

precerts.go

@@ -27,9 +27,9 @@ var (
 )
 
 type PrecertInfo struct {
-	SameIssuer		bool	// The pre-certificate was issued from the same CA as the final certificate
+	SameIssuer bool   // The pre-certificate was issued from the same CA as the final certificate
-	Issuer			[]byte	// The pre-certificate's issuer, if different from the final certificate
+	Issuer     []byte // The pre-certificate's issuer, if different from the final certificate
-	AKI			[]byte	// The pre-certificate's AKI, if present and different from the final certificate
+	AKI        []byte // The pre-certificate's AKI, if present and different from the final certificate
 }
 
 func ValidatePrecert(precertBytes []byte, tbsBytes []byte) (*PrecertInfo, error) {

sct.go

@@ -1,46 +0,0 @@
-// Copyright (C) 2017 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package certspotter
-
-import (
-	"software.sslmate.com/src/certspotter/ct"
-)
-
-func VerifyX509SCT(sct *ct.SignedCertificateTimestamp, cert []byte, verify *ct.SignatureVerifier) error {
-	entry := ct.LogEntry{
-		Leaf: ct.MerkleTreeLeaf{
-			Version:  0,
-			LeafType: ct.TimestampedEntryLeafType,
-			TimestampedEntry: ct.TimestampedEntry{
-				Timestamp:  sct.Timestamp,
-				EntryType:  ct.X509LogEntryType,
-				X509Entry:  cert,
-				Extensions: sct.Extensions,
-			},
-		},
-	}
-	return verify.VerifySCTSignature(*sct, entry)
-}
-
-func VerifyPrecertSCT(sct *ct.SignedCertificateTimestamp, precert ct.PreCert, verify *ct.SignatureVerifier) error {
-	entry := ct.LogEntry{
-		Leaf: ct.MerkleTreeLeaf{
-			Version:  0,
-			LeafType: ct.TimestampedEntryLeafType,
-			TimestampedEntry: ct.TimestampedEntry{
-				Timestamp:  sct.Timestamp,
-				EntryType:  ct.PrecertLogEntryType,
-				PrecertEntry:  precert,
-				Extensions: sct.Extensions,
-			},
-		},
-	}
-	return verify.VerifySCTSignature(*sct, entry)
-}

sequencer/sequencer.go

@@ -0,0 +1,151 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package sequencer
+
+import (
+	"context"
+	"slices"
+	"sync"
+)
+
+type seqWriter struct {
+	seqNbr uint64
+	ready  chan<- struct{}
+}
+
+// Channel[T] is a multi-producer, single-consumer channel of items with monotonicaly-increasing sequence numbers.
+// Items can be sent in any order, but they are always received in order of their sequence number.
+// It is unsafe to call Next concurrently with itself, or to call Add/Reserve concurrently with another Add/Reserve
+// call for the same sequence number.  Otherwise, methods are safe to call concurrently.
+type Channel[T any] struct {
+	mu          sync.Mutex
+	next        uint64
+	buf         []*T
+	writers     []seqWriter
+	readWaiting bool
+	readReady   chan struct{}
+}
+
+func New[T any](initialSequenceNumber uint64, capacity uint64) *Channel[T] {
+	return &Channel[T]{
+		buf:       make([]*T, capacity),
+		next:      initialSequenceNumber,
+		readReady: make(chan struct{}, 1),
+	}
+}
+
+func (seq *Channel[T]) parkWriter(seqNbr uint64) <-chan struct{} {
+	ready := make(chan struct{})
+	seq.writers = append(seq.writers, seqWriter{seqNbr: seqNbr, ready: ready})
+	return ready
+}
+
+func (seq *Channel[T]) signalWriter(seqNbr uint64) {
+	for i := range seq.writers {
+		if seq.writers[i].seqNbr == seqNbr {
+			close(seq.writers[i].ready)
+			seq.writers = slices.Delete(seq.writers, i, i+1)
+			return
+		}
+	}
+}
+
+func (seq *Channel[T]) forgetWriter(seqNbr uint64) {
+	for i := range seq.writers {
+		if seq.writers[i].seqNbr == seqNbr {
+			seq.writers = slices.Delete(seq.writers, i, i+1)
+			return
+		}
+	}
+}
+
+func (seq *Channel[T]) Cap() uint64 {
+	return uint64(len(seq.buf))
+}
+
+func (seq *Channel[T]) index(seqNbr uint64) int {
+	return int(seqNbr % seq.Cap())
+}
+
+// Wait until the channel has capacity for an item with the given sequence number.
+// After this function returns nil, calling Add with the same sequence number will not block.
+func (seq *Channel[T]) Reserve(ctx context.Context, sequenceNumber uint64) error {
+	seq.mu.Lock()
+	if sequenceNumber >= seq.next+seq.Cap() {
+		ready := seq.parkWriter(sequenceNumber)
+		seq.mu.Unlock()
+		select {
+		case <-ctx.Done():
+			seq.mu.Lock()
+			seq.forgetWriter(sequenceNumber)
+			seq.mu.Unlock()
+			return ctx.Err()
+		case <-ready:
+		}
+	} else {
+		seq.mu.Unlock()
+	}
+	return nil
+}
+
+// Send an item with the given sequence number.  Blocks if the channel does not have capacity for the item.
+// It is undefined behavior to send a sequence number that has previously been sent.
+func (seq *Channel[T]) Add(ctx context.Context, sequenceNumber uint64, item *T) error {
+	seq.mu.Lock()
+	if sequenceNumber >= seq.next+seq.Cap() {
+		ready := seq.parkWriter(sequenceNumber)
+		seq.mu.Unlock()
+		select {
+		case <-ctx.Done():
+			seq.mu.Lock()
+			seq.forgetWriter(sequenceNumber)
+			seq.mu.Unlock()
+			return ctx.Err()
+		case <-ready:
+		}
+		seq.mu.Lock()
+	}
+	seq.buf[seq.index(sequenceNumber)] = item
+	if sequenceNumber == seq.next && seq.readWaiting {
+		seq.readReady <- struct{}{}
+	}
+	seq.mu.Unlock()
+	return nil
+}
+
+// Return the item with the next sequence number, blocking if necessary.
+// Not safe to call concurrently with other Next calls.
+func (seq *Channel[T]) Next(ctx context.Context) (*T, error) {
+	seq.mu.Lock()
+	if seq.buf[seq.index(seq.next)] == nil {
+		seq.readWaiting = true
+		seq.mu.Unlock()
+		select {
+		case <-ctx.Done():
+			seq.mu.Lock()
+			select {
+			case <-seq.readReady:
+			default:
+			}
+			seq.readWaiting = false
+			seq.mu.Unlock()
+			return nil, ctx.Err()
+		case <-seq.readReady:
+		}
+		seq.mu.Lock()
+		seq.readWaiting = false
+	}
+	item := seq.buf[seq.index(seq.next)]
+	seq.buf[seq.index(seq.next)] = nil
+	seq.signalWriter(seq.next + seq.Cap())
+	seq.next++
+	seq.mu.Unlock()
+	return item, nil
+}

sequencer/sequencer_test.go

@@ -0,0 +1,195 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package sequencer
+
+import (
+	"context"
+	"fmt"
+	mathrand "math/rand/v2"
+	"testing"
+	"time"
+)
+
+func TestSequencerBasic(t *testing.T) {
+	ctx := context.Background()
+	seq := New[uint64](0, 100)
+	go func() {
+		for i := range uint64(10_000) {
+			err := seq.Add(ctx, i, &i)
+			if err != nil {
+				panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
+			}
+		}
+	}()
+
+	for i := range uint64(10_000) {
+		next, err := seq.Next(ctx)
+		if err != nil {
+			t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
+		}
+		if *next != i {
+			t.Fatalf("%d: got unexpected value %d", i, *next)
+		}
+	}
+}
+
+func TestSequencerNonZeroStart(t *testing.T) {
+	ctx := context.Background()
+	seq := New[uint64](10, 100)
+	go func() {
+		for i := range uint64(10_000) {
+			err := seq.Add(ctx, i+10, &i)
+			if err != nil {
+				panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
+			}
+		}
+	}()
+
+	for i := range uint64(10_000) {
+		next, err := seq.Next(ctx)
+		if err != nil {
+			t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
+		}
+		if *next != i {
+			t.Fatalf("%d: got unexpected value %d", i, *next)
+		}
+	}
+}
+
+func TestSequencerCapacity1(t *testing.T) {
+	ctx := context.Background()
+	seq := New[uint64](0, 1)
+	go func() {
+		for i := range uint64(10_000) {
+			err := seq.Add(ctx, i, &i)
+			if err != nil {
+				panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
+			}
+		}
+	}()
+
+	for i := range uint64(10_000) {
+		next, err := seq.Next(ctx)
+		if err != nil {
+			t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
+		}
+		if *next != i {
+			t.Fatalf("%d: got unexpected value %d", i, *next)
+		}
+	}
+}
+
+func TestSequencerTimeout(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	seq := New[uint64](0, 10_000)
+	go func() {
+		var i uint64
+		for {
+			newI := i
+			err := seq.Add(ctx, i, &newI)
+			if err != nil {
+				break
+			}
+			i++
+		}
+	}()
+
+	var i uint64
+	for {
+		next, err := seq.Next(ctx)
+		if err != nil {
+			break
+		}
+		if *next != i {
+			t.Fatalf("%d: got unexpected value %d", i, *next)
+		}
+		i++
+	}
+}
+
+func TestSequencerOutOfOrder(t *testing.T) {
+	ctx := context.Background()
+	seq := New[uint64](0, 100)
+	ch := make(chan uint64)
+	go func() {
+		for i := range uint64(10_000) {
+			ch <- i
+		}
+	}()
+	for range 4 {
+		go func() {
+			for i := range ch {
+				time.Sleep(mathrand.N(10 * time.Millisecond))
+				//t.Logf("seq.Add %d", i)
+				err := seq.Add(ctx, i, &i)
+				if err != nil {
+					panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
+				}
+			}
+		}()
+	}
+	for i := range uint64(10_000) {
+		next, err := seq.Next(ctx)
+		if err != nil {
+			t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
+		}
+		if *next != i {
+			t.Fatalf("%d: got unexpected value %d", i, *next)
+		}
+		//t.Logf("seq.Next %d", i)
+	}
+}
+
+func TestSequencerOutOfOrderReserve(t *testing.T) {
+	ctx := context.Background()
+	seq := New[uint64](0, 10)
+	ch := make(chan uint64)
+	go func() {
+		for i := range uint64(10_000) {
+			ch <- i
+		}
+	}()
+	ch2 := make(chan uint64)
+	for job := range 4 {
+		go func() {
+			for i := range ch {
+				time.Sleep(mathrand.N(11 * time.Duration(job+1) * time.Millisecond))
+				err := seq.Reserve(ctx, i)
+				if err != nil {
+					panic(fmt.Sprintf("%d: seq.Reserve returned unexpected error %v", i, err))
+				}
+				ch2 <- i
+			}
+		}()
+	}
+	for range 4 {
+		go func() {
+			for i := range ch2 {
+				time.Sleep(mathrand.N(7 * time.Millisecond))
+				t.Logf("seq.Add %d", i)
+				err := seq.Add(ctx, i, &i)
+				if err != nil {
+					panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
+				}
+			}
+		}()
+	}
+	for i := range uint64(10_000) {
+		next, err := seq.Next(ctx)
+		if err != nil {
+			t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
+		}
+		if *next != i {
+			t.Fatalf("%d: got unexpected value %d", i, *next)
+		}
+		t.Logf("seq.Next %d", i)
+	}
+}

staticcheck.conf

@@ -0,0 +1 @@
+checks = ["inherit", "-ST1005", "-S1002"]

tlstypes/signatures.go

@@ -0,0 +1,134 @@
+// Copyright (C) 2025 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package tlstypes
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"golang.org/x/crypto/cryptobyte"
+)
+
+type HashAlgorithm uint8
+
+const (
+	SHA224 HashAlgorithm = 3
+	SHA256 HashAlgorithm = 4
+	SHA384 HashAlgorithm = 5
+	SHA512 HashAlgorithm = 6
+)
+
+type SignatureAlgorithm uint8
+
+const (
+	RSA   SignatureAlgorithm = 1
+	ECDSA SignatureAlgorithm = 3
+)
+
+type SignatureAndHashAlgorithm struct {
+	Hash      HashAlgorithm
+	Signature SignatureAlgorithm
+}
+
+type DigitallySigned struct {
+	Algorithm SignatureAndHashAlgorithm
+	Signature []byte
+}
+
+func (v HashAlgorithm) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint8(uint8(v))
+	return nil
+}
+func (v *HashAlgorithm) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint8((*uint8)(v))
+}
+
+func (v SignatureAlgorithm) Marshal(b *cryptobyte.Builder) error {
+	b.AddUint8(uint8(v))
+	return nil
+}
+func (v *SignatureAlgorithm) Unmarshal(s *cryptobyte.String) bool {
+	return s.ReadUint8((*uint8)(v))
+}
+
+func (v SignatureAndHashAlgorithm) Marshal(b *cryptobyte.Builder) error {
+	b.AddValue(v.Hash)
+	b.AddValue(v.Signature)
+	return nil
+}
+func (v *SignatureAndHashAlgorithm) Unmarshal(s *cryptobyte.String) bool {
+	return v.Hash.Unmarshal(s) && v.Signature.Unmarshal(s)
+}
+
+func (v DigitallySigned) Marshal(b *cryptobyte.Builder) error {
+	b.AddValue(v.Algorithm)
+	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { b.AddBytes(v.Signature) })
+	return nil
+}
+func (v *DigitallySigned) Unmarshal(s *cryptobyte.String) bool {
+	return v.Algorithm.Unmarshal(s) && s.ReadUint16LengthPrefixed((*cryptobyte.String)(&v.Signature))
+}
+
+func (v DigitallySigned) Bytes() []byte {
+	b := cryptobyte.NewBuilder(make([]byte, 0, 4+len(v.Signature)))
+	b.AddValue(v)
+	return b.BytesOrPanic()
+}
+
+func (v DigitallySigned) MarshalBinary() ([]byte, error) {
+	b := cryptobyte.NewBuilder(make([]byte, 0, 4+len(v.Signature)))
+	b.AddValue(v)
+	return b.Bytes()
+}
+
+func (v *DigitallySigned) UnmarshalBinary(data []byte) error {
+	str := cryptobyte.String(bytes.Clone(data))
+	if !v.Unmarshal(&str) {
+		return fmt.Errorf("DigitallySigned bytes are malformed")
+	}
+	if !str.Empty() {
+		return fmt.Errorf("trailing bytes after DigitallySigned")
+	}
+	return nil
+}
+
+func (v DigitallySigned) MarshalJSON() ([]byte, error) {
+	b := cryptobyte.NewBuilder(make([]byte, 0, 4+len(v.Signature)))
+	b.AddValue(v)
+	if bytes, err := b.Bytes(); err != nil {
+		return nil, err
+	} else {
+		return json.Marshal(bytes)
+	}
+}
+func (v *DigitallySigned) UnmarshalJSON(data []byte) error {
+	str := new(cryptobyte.String)
+	if err := json.Unmarshal(data, (*[]byte)(str)); err != nil {
+		return fmt.Errorf("unable to unmarshal DigitallySigned JSON: %w", err)
+	}
+	if !v.Unmarshal(str) {
+		return fmt.Errorf("DigitallySigned bytes are malformed")
+	}
+	if !str.Empty() {
+		return fmt.Errorf("trailing bytes after DigitallySigned")
+	}
+	return nil
+}
+func ParseDigitallySigned(bytes []byte) (*DigitallySigned, error) {
+	ds := new(DigitallySigned)
+	str := cryptobyte.String(bytes)
+	if !ds.Unmarshal(&str) {
+		return nil, fmt.Errorf("DigitallySigned bytes are malformed")
+	}
+	if !str.Empty() {
+		return nil, fmt.Errorf("trailing bytes after DigitallySigned")
+	}
+	return ds, nil
+}