Compare commits

..

No commits in common. "master" and "v0.19.1" have entirely different histories.

26 changed files with 224 additions and 420 deletions

View File

@ -1,35 +0,0 @@
name: Test and lint Go Code
on:
push:
schedule:
- cron: '42 9 * * *' # Runs daily at 09:42 UTC
workflow_dispatch: # Allows manual triggering
permissions:
contents: read
jobs:
test:
name: Test
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run tests
run: CGO_ENABLED=1 go test -race ./...
- name: Install staticcheck
run: go install honnef.co/go/tools/cmd/staticcheck@latest
- name: Run staticcheck
run: staticcheck ./...

View File

@ -1,19 +1,5 @@
# Change Log
## v0.20.1 (2025-06-19)
- Add resilience against sendmail hanging indefinitely.
- Add resilience against hooks which fork and keep stderr open.
- Upgrade dependencies to latest versions.
- Minor improvements to error handling, code quality, and efficiency.
## v0.20.0 (2025-06-13)
- Remove -batch_size option, which is obsolete due to new parallel download system.
- Only print log errors to stderr if -verbose is specified.
- Fix bug that could cause unverified STHs to be deleted prematurely.
- Fail health check if log has never been successfully contacted.
- Improve -verbose.
- Improve documentation.
## v0.19.1 (2025-05-07)
- Fix panic when retrying failed log requests.
- Properly log failed log requests.

View File

@ -46,7 +46,7 @@ func decodeASN1String(value *asn1.RawValue) (string, error) {
if value.Tag == 12 {
// UTF8String
if !utf8.Valid(value.Bytes) {
return "", errors.New("malformed UTF8String")
return "", errors.New("Malformed UTF8String")
}
return string(value.Bytes), nil
} else if value.Tag == 19 || value.Tag == 22 || value.Tag == 20 || value.Tag == 26 {
@ -74,5 +74,5 @@ func decodeASN1String(value *asn1.RawValue) (string, error) {
return stringFromUint32Slice(runes), nil
}
}
return "", errors.New("not a string")
return "", errors.New("Not a string")
}

View File

@ -253,5 +253,5 @@ func decodeASN1Time(value *asn1.RawValue) (time.Time, error) {
return parseGeneralizedTime(value.Bytes)
}
}
return time.Time{}, errors.New("not a time value")
return time.Time{}, errors.New("Not a time value")
}

View File

@ -25,7 +25,6 @@ import (
"syscall"
"time"
"software.sslmate.com/src/certspotter/ctclient"
"software.sslmate.com/src/certspotter/loglist"
"software.sslmate.com/src/certspotter/monitor"
)
@ -38,11 +37,31 @@ const defaultLogList = "https://loglist.certspotter.org/monitor.json"
func certspotterVersion() string {
if Version != "" {
return Version + "?"
} else if info, ok := debug.ReadBuildInfo(); ok && strings.HasPrefix(info.Main.Version, "v") {
return info.Main.Version
} else {
}
info, ok := debug.ReadBuildInfo()
if !ok {
return "unknown"
}
if strings.HasPrefix(info.Main.Version, "v") {
return info.Main.Version
}
var vcs, vcsRevision, vcsModified string
for _, s := range info.Settings {
switch s.Key {
case "vcs":
vcs = s.Value
case "vcs.revision":
vcsRevision = s.Value
case "vcs.modified":
vcsModified = s.Value
}
}
if vcs == "git" && vcsRevision != "" && vcsModified == "true" {
return vcsRevision + "+"
} else if vcs == "git" && vcsRevision != "" {
return vcsRevision
}
return "unknown"
}
func fileExists(filename string) bool {
@ -140,10 +159,9 @@ func appendFunc(slice *[]string) func(string) error {
func main() {
loglist.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s)", certspotterVersion(), runtime.Version(), runtime.GOOS, runtime.GOARCH)
ctclient.UserAgent = fmt.Sprintf("certspotter/%s (+https://github.com/SSLMate/certspotter)", certspotterVersion())
var flags struct {
batchSize bool
batchSize int // TODO-4: respect this option
email []string
healthcheck time.Duration
logs string
@ -156,7 +174,7 @@ func main() {
version bool
watchlist string
}
flag.Func("batch_size", "Obsolete; do not use", func(string) error { flags.batchSize = true; return nil }) // TODO: remove in 0.21.0
flag.IntVar(&flags.batchSize, "batch_size", 1000, "Max number of entries to request per call to get-entries (advanced)")
flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email))
flag.DurationVar(&flags.healthcheck, "healthcheck", 24*time.Hour, "How frequently to perform a health check")
flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
@ -165,15 +183,11 @@ func main() {
flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring new logs from the end rather than the beginning (saves considerable bandwidth)")
flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates")
flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout")
flag.BoolVar(&flags.verbose, "verbose", false, "Print detailed information about certspotter's operation to stderr")
flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose")
flag.BoolVar(&flags.version, "version", false, "Print version and exit")
flag.StringVar(&flags.watchlist, "watchlist", defaultWatchListPathIfExists(), "File containing domain names to watch")
flag.Parse()
if flags.batchSize {
fmt.Fprintf(os.Stderr, "%s: -batch_size is obsolete; please remove it from your command line\n", programName)
os.Exit(2)
}
if flags.version {
fmt.Fprintf(os.Stdout, "certspotter version %s\n", certspotterVersion())
os.Exit(0)
@ -191,7 +205,6 @@ func main() {
ScriptDir: defaultScriptDir(),
Email: flags.email,
Stdout: flags.stdout,
Quiet: !flags.verbose,
}
config := &monitor.Config{
LogListSource: flags.logs,
@ -240,12 +253,7 @@ func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := monitor.Run(ctx, config); ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
if flags.verbose {
fmt.Fprintf(os.Stderr, "%s: exiting due to SIGINT or SIGTERM\n", programName)
}
os.Exit(0)
} else {
if err := monitor.Run(ctx, config); err != nil && !errors.Is(err, context.Canceled) {
fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err)
os.Exit(1)
}

View File

@ -11,7 +11,6 @@
package ctclient
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
@ -24,8 +23,6 @@ import (
"time"
)
var UserAgent = "software.sslmate.com/src/certspotter"
// Create an HTTP client suitable for communicating with CT logs. dialContext, if non-nil, is used for dialing.
func NewHTTPClient(dialContext func(context.Context, string, string) (net.Conn, error)) *http.Client {
return &http.Client{
@ -34,7 +31,9 @@ func NewHTTPClient(dialContext func(context.Context, string, string) (net.Conn,
TLSHandshakeTimeout: 15 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
DisableKeepAlives: false,
MaxIdleConns: 100,
IdleConnTimeout: 15 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{
// We have to disable TLS certificate validation because because several logs
@ -46,8 +45,7 @@ func NewHTTPClient(dialContext func(context.Context, string, string) (net.Conn,
// updating should a log ever change to a different CA.)
InsecureSkipVerify: true,
},
DialContext: dialContext,
ForceAttemptHTTP2: true,
DialContext: dialContext,
},
CheckRedirect: func(*http.Request, []*http.Request) error {
return errors.New("redirects not followed")
@ -63,7 +61,7 @@ func get(ctx context.Context, httpClient *http.Client, fullURL string) ([]byte,
if err != nil {
return nil, err
}
request.Header.Set("User-Agent", UserAgent)
request.Header.Set("User-Agent", "") // Don't send a User-Agent to make life harder for malicious logs
if httpClient == nil {
httpClient = defaultHTTPClient
@ -81,7 +79,7 @@ func get(ctx context.Context, httpClient *http.Client, fullURL string) ([]byte,
}
if response.StatusCode != 200 {
return nil, fmt.Errorf("Get %q: %s (%q)", fullURL, response.Status, bytes.TrimSpace(responseBody))
return nil, fmt.Errorf("Get %q: %s (%q)", fullURL, response.Status, string(responseBody))
}
return responseBody, nil

View File

@ -10,7 +10,6 @@
package cttypes
import (
"bytes"
"encoding/base64"
"fmt"
"golang.org/x/crypto/cryptobyte"
@ -18,10 +17,6 @@ import (
type LogID [32]byte
func (id LogID) Compare(other LogID) int {
return bytes.Compare(id[:], other[:])
}
func (v *LogID) Unmarshal(s *cryptobyte.String) bool {
return s.CopyBytes((*v)[:])
}

View File

@ -31,7 +31,3 @@ type GossipedSignedTreeHead struct {
func (sth *SignedTreeHead) TimestampTime() time.Time {
return time.UnixMilli(int64(sth.Timestamp))
}
func (sth *SignedTreeHead) Same(other *SignedTreeHead) bool {
return sth.TreeSize == other.TreeSize && sth.Timestamp == other.Timestamp && sth.RootHash == other.RootHash
}

10
go.mod
View File

@ -1,13 +1,13 @@
module software.sslmate.com/src/certspotter
go 1.24.4
go 1.24
require (
golang.org/x/crypto v0.39.0
golang.org/x/net v0.41.0
golang.org/x/sync v0.15.0
golang.org/x/crypto v0.37.0
golang.org/x/net v0.39.0
golang.org/x/sync v0.13.0
)
require golang.org/x/text v0.26.0 // indirect
require golang.org/x/text v0.24.0 // indirect
retract v0.19.0 // Contains serious bugs.

16
go.sum
View File

@ -1,8 +1,8 @@
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=

View File

@ -262,6 +262,21 @@ func (ids *Identifiers) AddIPAddress(value net.IP) {
ids.appendIPAddress(value)
}
func (ids *Identifiers) dnsNamesString(sep string) string {
return strings.Join(ids.DNSNames, sep)
}
func (ids *Identifiers) ipAddrsString(sep string) string {
str := ""
for _, ipAddr := range ids.IPAddrs {
if str != "" {
str += sep
}
str += ipAddr.String()
}
return str
}
func (cert *CertInfo) ParseIdentifiers() (*Identifiers, error) {
ids := NewIdentifiers()

View File

@ -21,7 +21,7 @@ import (
"time"
)
var UserAgent = "software.sslmate.com/src/certspotter"
var UserAgent = "certspotter"
type ModificationToken struct {
etag string
@ -112,7 +112,7 @@ func Unmarshal(jsonBytes []byte) (*List, error) {
return nil, err
}
if err := list.Validate(); err != nil {
return nil, fmt.Errorf("invalid log list: %s", err)
return nil, fmt.Errorf("Invalid log list: %s", err)
}
return list, nil
}

View File

@ -65,7 +65,7 @@ The following environment variables are set for `discovered_cert` events:
`CERT_SHA256`
: The hex-encoded SHA-256 digest (sometimes called fingerprint) of the certificate or precertificate.
: The hex-encoded SHA-256 digest (sometimes called fingerprint) of the certificate.
The digest is computed over the ASN.1 DER encoding.
`PUBKEY_SHA256`

View File

@ -30,6 +30,11 @@ You can use Cert Spotter to detect:
# OPTIONS
-batch_size *NUMBER*
: Maximum number of entries to request per call to get-entries.
You should not generally need to change this. Defaults to 1000.
-email *ADDRESS*
: Email address to contact when a matching certificate is discovered, or
@ -90,7 +95,7 @@ You can use Cert Spotter to detect:
-verbose
: Print detailed information about certspotter's operation (such as errors contacting logs) to stderr.
: Be verbose.
-version
@ -136,10 +141,10 @@ the script interface, see certspotter-script(8).
# OPERATION
certspotter continuously monitors all browser-recognized Certificate
Transparency logs looking for certificates (including precertificates)
which are valid for any domain on your watch list. When certspotter
detects a matching certificate, it emails you, executes a script, and/or
writes a report to standard out, as described above.
Transparency logs looking for certificates which are valid for any domain
on your watch list. When certspotter detects a matching certificate, it
emails you, executes a script, and/or writes a report to standard out,
as described above.
certspotter also saves a copy of matching certificates in
`$CERTSPOTTER_STATE_DIR/certs` ("~/.certspotter/certs" by default)
@ -173,7 +178,7 @@ to write a file or execute a script), it prints a message to stderr and
exits with a non-zero status.
When certspotter encounters a problem monitoring a log, it prints a message
to stderr if `-verbose` is specified and continues running. It will try monitoring the log again later;
to stderr and continues running. It will try monitoring the log again later;
most log errors are transient.
Every 24 hours (unless overridden by `-healthcheck`), certspotter performs the
@ -190,7 +195,7 @@ standard out, as described above.
Health check failures should be rare, and you should take them seriously because it means
certspotter might not detect all certificates. It might also be an indication
of CT log misbehavior. Enable the `-verbose` flag and consult stderr for details, and if
of CT log misbehavior. Consult certspotter's stderr output for details, and if
you need help, file an issue at <https://github.com/SSLMate/certspotter>.
# EXIT STATUS
@ -224,20 +229,6 @@ and non-zero when a serious error occurs.
: Path to the sendmail binary used for sending emails. Defaults to `/usr/sbin/sendmail`.
# DIRECTORIES
Config directory
: Stores configuration, such as the watch list. The location is: (1) the `CERTSPOTTER_CONFIG_DIR` environment variable, if set, or (2) the default location `~/.certspotter`. certspotter does not write to this directory.
State directory
: Stores state, such as the position of each log and a store of discovered certificates. The location is: (1) the `-state_dir` command line flag, if provided, (2) the `CERTSPOTTER_STATE_DIR` environment variable, if set, or (3) the default location `~/.certspotter`. certspotter creates this directory if necessary.
Cache directory
: Stores cached data. The location is `$XDG_CACHE_HOME/certspotter` (which on Linux is `~/.cache/certspotter` by default). You can delete this directory without without impacting functionality, but certspotter may need to perform additional computation or network requests.
# SEE ALSO
certspotter-script(8)

View File

@ -10,7 +10,6 @@
package merkletree
import (
"bytes"
"crypto/sha256"
"encoding/base64"
"encoding/json"
@ -21,10 +20,6 @@ const HashLen = 32
type Hash [HashLen]byte
func (h Hash) Compare(other Hash) int {
return bytes.Compare(h[:], other[:])
}
func (h Hash) Base64String() string {
return base64.StdEncoding.EncodeToString(h[:])
}

View File

@ -75,7 +75,7 @@ func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task {
defer cancel()
err := monitorLogContinously(ctx, daemon.config, ctlog)
if daemon.config.Verbose {
log.Printf("%s: task stopped with error: %s", ctlog.GetMonitoringURL(), err)
log.Printf("task for log %s stopped with error %s", ctlog.GetMonitoringURL(), err)
}
if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
return nil
@ -137,10 +137,9 @@ func (daemon *daemon) run(ctx context.Context) error {
healthCheckTicker := time.NewTicker(daemon.config.HealthCheckInterval)
defer healthCheckTicker.Stop()
for {
for ctx.Err() == nil {
select {
case <-ctx.Done():
return ctx.Err()
case <-reloadLogListTicker.C:
if err := daemon.loadLogList(ctx); err != nil {
daemon.logListError = err.Error()
@ -154,6 +153,7 @@ func (daemon *daemon) run(ctx context.Context) error {
}
}
}
return ctx.Err()
}
func Run(ctx context.Context, config *Config) error {

View File

@ -34,7 +34,6 @@ type FilesystemState struct {
ScriptDir string
Email []string
Stdout bool
Quiet bool
}
func (s *FilesystemState) logStateDir(logID LogID) string {
@ -86,7 +85,7 @@ func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state
return writeJSONFile(filePath, state, 0666)
}
func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) (*StoredSTH, error) {
func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) error {
sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
return storeSTHInDir(sthsDirPath, sth)
}
@ -249,12 +248,10 @@ func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, ctlog *l
}
func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, err error) error {
if !s.Quiet {
if ctlog == nil {
log.Print(err)
} else {
log.Print(ctlog.GetMonitoringURL(), ": ", err)
}
if ctlog == nil {
log.Print(err)
} else {
log.Print(ctlog.GetMonitoringURL(), ": ", err)
}
return nil
}

View File

@ -24,23 +24,15 @@ func healthCheckFilename() string {
}
func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error {
var (
position uint64
lastSuccess time.Time
verifiedSTH *cttypes.SignedTreeHead
)
if state, err := config.State.LoadLogState(ctx, ctlog.LogID); err != nil {
state, err := config.State.LoadLogState(ctx, ctlog.LogID)
if err != nil {
return fmt.Errorf("error loading log state: %w", err)
} else if state != nil {
if time.Since(state.LastSuccess) < config.HealthCheckInterval {
// log is healthy
return nil
}
} else if state == nil {
return nil
}
position = state.DownloadPosition.Size()
lastSuccess = state.LastSuccess
verifiedSTH = state.VerifiedSTH
if time.Since(state.LastSuccess) < config.HealthCheckInterval {
return nil
}
sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
@ -51,8 +43,8 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err
if len(sths) == 0 {
info := &StaleSTHInfo{
Log: ctlog,
LastSuccess: lastSuccess,
LatestSTH: verifiedSTH,
LastSuccess: state.LastSuccess,
LatestSTH: state.VerifiedSTH,
}
if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
return fmt.Errorf("error notifying about stale STH: %w", err)
@ -61,7 +53,7 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err
info := &BacklogInfo{
Log: ctlog,
LatestSTH: sths[len(sths)-1],
Position: position,
Position: state.DownloadPosition.Size(),
}
if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
return fmt.Errorf("error notifying about backlog: %w", err)
@ -78,7 +70,7 @@ type HealthCheckFailure interface {
type StaleSTHInfo struct {
Log *loglist.Log
LastSuccess time.Time // may be zero
LastSuccess time.Time
LatestSTH *cttypes.SignedTreeHead // may be nil
}
@ -95,19 +87,12 @@ type StaleLogListInfo struct {
LastErrorTime time.Time
}
func (e *StaleSTHInfo) LastSuccessString() string {
if e.LastSuccess.IsZero() {
return "never"
} else {
return e.LastSuccess.String()
}
}
func (e *BacklogInfo) Backlog() uint64 {
return e.LatestSTH.TreeSize - e.Position
}
func (e *StaleSTHInfo) Summary() string {
return fmt.Sprintf("Unable to contact %s since %s", e.Log.GetMonitoringURL(), e.LastSuccessString())
return fmt.Sprintf("Unable to contact %s since %s", e.Log.GetMonitoringURL(), e.LastSuccess)
}
func (e *BacklogInfo) Summary() string {
return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.GetMonitoringURL())
@ -118,9 +103,9 @@ func (e *StaleLogListInfo) Summary() string {
func (e *StaleSTHInfo) Text() string {
text := new(strings.Builder)
fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.GetMonitoringURL(), e.LastSuccessString())
fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.GetMonitoringURL(), e.LastSuccess)
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "For details, enable -verbose and see certspotter's stderr output.\n")
fmt.Fprintf(text, "For details, see certspotter's stderr output.\n")
fmt.Fprintf(text, "\n")
if e.LatestSTH != nil {
fmt.Fprintf(text, "Latest known log size = %d\n", e.LatestSTH.TreeSize)
@ -133,7 +118,7 @@ func (e *BacklogInfo) Text() string {
text := new(strings.Builder)
fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.GetMonitoringURL())
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "For details, enable -verbose and see certspotter's stderr output.\n")
fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n")
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.StoredAt)
fmt.Fprintf(text, "Current position = %d\n", e.Position)

View File

@ -29,8 +29,7 @@ import (
)
const (
getSTHInterval = 5 * time.Minute
maxPartialTileAge = 5 * time.Minute
getSTHInterval = 5 * time.Minute
)
func downloadJobSize(ctlog *loglist.Log) uint64 {
@ -206,7 +205,7 @@ func newLogClient(config *Config, ctlog *loglist.Log) (ctclient.Log, ctclient.Is
logGetter: client,
}, nil
default:
return nil, nil, errors.New("log uses unknown protocol")
return nil, nil, fmt.Errorf("log uses unknown protocol")
}
}
@ -247,18 +246,17 @@ func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.L
}
}
if config.Verbose {
log.Printf("%s: monitoring brand new log starting from position %d", ctlog.GetMonitoringURL(), state.DownloadPosition.Size())
log.Printf("brand new log %s (starting from %d)", ctlog.GetMonitoringURL(), state.DownloadPosition.Size())
}
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
return fmt.Errorf("error storing log state: %w", err)
}
} else {
if config.Verbose {
log.Printf("%s: resuming monitoring from position %d", ctlog.GetMonitoringURL(), state.DownloadPosition.Size())
}
}
defer func() {
if config.Verbose {
log.Printf("saving state in defer for %s", ctlog.GetMonitoringURL())
}
storeCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
if err := config.State.StoreLogState(storeCtx, ctlog.LogID, state); err != nil && returnedErr == nil {
@ -269,21 +267,14 @@ func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.L
retry:
position := state.DownloadPosition.Size()
// logs are monitored using the following pipeline of workers, with each worker sending results to the next worker:
// 1 getSTHWorker ==> 1 generateBatchesWorker ==> multiple downloadWorkers ==> multiple processWorkers ==> 1 saveStateWorker
// getSTHWorker - periodically download STHs from the log
// generateBatchesWorker - generate batches of work
// downloadWorkers - download the entries in each batch
// processWorkers - process the certificates (store/notify if matches watch list) in each batch
// saveStateWorker - builds the Merkle Tree and compares against STHs
// generateBatchesWorker ==> downloadWorker ==> processWorker ==> saveStateWorker
sths := make(chan *cttypes.SignedTreeHead, 1)
batches := make(chan *batch, downloadWorkers(ctlog))
processedBatches := sequencer.New[batch](0, uint64(downloadWorkers(ctlog))*10)
group, gctx := errgroup.WithContext(ctx)
group.Go(func() error { return getSTHWorker(gctx, config, ctlog, client, sths) })
group.Go(func() error { return generateBatchesWorker(gctx, config, ctlog, position, sths, batches) })
group.Go(func() error { return getSTHWorker(gctx, config, ctlog, client) })
group.Go(func() error { return generateBatchesWorker(gctx, config, ctlog, position, batches) })
for range downloadWorkers(ctlog) {
downloadedBatches := make(chan *batch, 1)
group.Go(func() error { return downloadWorker(gctx, config, ctlog, client, batches, downloadedBatches) })
@ -308,18 +299,47 @@ retry:
return err
}
func getSTHWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log, sthsOut chan<- *cttypes.SignedTreeHead) error {
ticker := time.NewTicker(getSTHInterval)
defer ticker.Stop()
for {
func getSTHWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log) error {
for ctx.Err() == nil {
sth, _, err := client.GetSTH(ctx)
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case sthsOut <- sth:
if err := config.State.StoreSTH(ctx, ctlog.LogID, sth); err != nil {
return fmt.Errorf("error storing STH: %w", err)
}
if err := sleep(ctx, getSTHInterval); err != nil {
return err
}
}
return ctx.Err()
}
type batch struct {
number uint64
begin, end uint64
sths []*StoredSTH // STHs with sizes in range [begin,end], sorted by TreeSize
entries []ctclient.Entry // in range [begin,end)
}
func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.Log, position uint64, batches chan<- *batch) error {
ticker := time.NewTicker(15 * time.Second)
var number uint64
for ctx.Err() == nil {
sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
if err != nil {
return fmt.Errorf("error loading STHs: %w", err)
}
for len(sths) > 0 && sths[0].TreeSize < position {
// TODO-4: audit sths[0] against log's verified STH
if err := config.State.RemoveSTH(ctx, ctlog.LogID, &sths[0].SignedTreeHead); err != nil {
return fmt.Errorf("error removing STH: %w", err)
}
sths = sths[1:]
}
position, number, err = generateBatches(ctx, ctlog, position, number, sths, batches)
if err != nil {
return err
}
select {
case <-ctx.Done():
@ -327,132 +347,77 @@ func getSTHWorker(ctx context.Context, config *Config, ctlog *loglist.Log, clien
case <-ticker.C:
}
}
return ctx.Err()
}
type batch struct {
number uint64
begin, end uint64
discoveredAt time.Time // time at which we became aware of the log having entries in range [begin,end)
sths []*StoredSTH // STHs with sizes in range [begin,end], sorted by TreeSize
entries []ctclient.Entry // in range [begin,end)
}
// Create a batch starting from begin, based on sths (which must be non-empty, sorted by TreeSize, and contain only STHs with TreeSize >= begin). Returns the batch, plus the remaining STHs.
func newBatch(number uint64, begin uint64, sths []*StoredSTH, downloadJobSize uint64) (*batch, []*StoredSTH) {
batch := &batch{
number: number,
begin: begin,
discoveredAt: sths[0].StoredAt,
}
maxEnd := (begin/downloadJobSize + 1) * downloadJobSize
for _, sth := range sths {
if sth.StoredAt.Before(batch.discoveredAt) {
batch.discoveredAt = sth.StoredAt
}
if sth.TreeSize <= maxEnd {
batch.end = sth.TreeSize
batch.sths = append(batch.sths, sth)
} else {
batch.end = maxEnd
// return the time at which the right-most tile indicated by sths was discovered
func tileDiscoveryTime(sths []*StoredSTH) time.Time {
largestSTH, sths := sths[len(sths)-1], sths[:len(sths)-1]
tileNumber := largestSTH.TreeSize / ctclient.StaticTileWidth
storedAt := largestSTH.StoredAt
for _, sth := range slices.Backward(sths) {
if sth.TreeSize/ctclient.StaticTileWidth != tileNumber {
break
}
if sth.StoredAt.Before(storedAt) {
storedAt = sth.StoredAt
}
}
return batch, sths[len(batch.sths):]
return storedAt
}
// insert sth into sths, which is sorted by TreeSize, and return a new, still-sorted slice.
// if an equivalent STH is already in sths, it is returned unchanged.
func insertSTH(sths []*StoredSTH, sth *StoredSTH) []*StoredSTH {
i := len(sths)
for i > 0 {
if sths[i-1].Same(&sth.SignedTreeHead) {
return sths
}
if sths[i-1].TreeSize < sth.TreeSize {
break
}
i--
}
return slices.Insert(sths, i, sth)
}
func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.Log, position uint64, sthsIn <-chan *cttypes.SignedTreeHead, batchesOut chan<- *batch) error {
func generateBatches(ctx context.Context, ctlog *loglist.Log, position uint64, number uint64, sths []*StoredSTH, batches chan<- *batch) (uint64, uint64, error) {
downloadJobSize := downloadJobSize(ctlog)
sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
if err != nil {
return fmt.Errorf("error loading STHs: %w", err)
if len(sths) == 0 {
return position, number, nil
}
// sths is sorted by TreeSize but may contain STHs with TreeSize < position; get rid of them
for len(sths) > 0 && sths[0].TreeSize < position {
// TODO-4: audit sths[0] against log's verified STH
if err := config.State.RemoveSTH(ctx, ctlog.LogID, &sths[0].SignedTreeHead); err != nil {
return fmt.Errorf("error removing STH: %w", err)
largestSTH := sths[len(sths)-1]
treeSize := largestSTH.TreeSize
if ctlog.IsStaticCTAPI() && time.Since(tileDiscoveryTime(sths)) < 5*time.Minute {
// Round down to the tile boundary to avoid downloading a partial tile that was recently discovered
// In a future invocation of this function, either enough time will have passed that this code path will be skipped, or the log will have grown and treeSize will be rounded to a larger tile boundary
treeSize -= treeSize % ctclient.StaticTileWidth
if treeSize < position {
// This can arise with a brand new log when config.StartAtEnd is true
return position, number, nil
}
sths = sths[1:]
}
// from this point, sths is sorted by TreeSize and contains only STHs with TreeSize >= position
handleSTH := func(sth *cttypes.SignedTreeHead) error {
if sth.TreeSize < position {
// TODO-4: audit against log's verified STH
} else {
storedSTH, err := config.State.StoreSTH(ctx, ctlog.LogID, sth)
if err != nil {
return fmt.Errorf("error storing STH: %w", err)
}
sths = insertSTH(sths, storedSTH)
}
return nil
}
var number uint64
for {
for len(sths) == 0 {
select {
case <-ctx.Done():
return ctx.Err()
case sth := <-sthsIn:
if err := handleSTH(sth); err != nil {
return err
}
}
batch := &batch{
number: number,
begin: position,
end: min(treeSize, (position/downloadJobSize+1)*downloadJobSize),
}
batch, remainingSTHs := newBatch(number, position, sths, downloadJobSize)
if ctlog.IsStaticCTAPI() && batch.end%downloadJobSize != 0 {
// Wait to download this partial tile until it's old enough
if age := time.Since(batch.discoveredAt); age < maxPartialTileAge {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(maxPartialTileAge - age):
case sth := <-sthsIn:
if err := handleSTH(sth); err != nil {
return err
}
continue
}
}
for len(sths) > 0 && sths[0].TreeSize <= batch.end {
batch.sths = append(batch.sths, sths[0])
sths = sths[1:]
}
select {
case <-ctx.Done():
return ctx.Err()
case sth := <-sthsIn:
if err := handleSTH(sth); err != nil {
return err
}
case batchesOut <- batch:
number = batch.number + 1
position = batch.end
sths = remainingSTHs
return position, number, ctx.Err()
default:
}
select {
case <-ctx.Done():
return position, number, ctx.Err()
case batches <- batch:
}
number++
if position == batch.end {
break
}
position = batch.end
}
return position, number, nil
}
func downloadWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log, batchesIn <-chan *batch, batchesOut chan<- *batch) error {
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
var batch *batch
select {
case <-ctx.Done():
@ -466,6 +431,11 @@ func downloadWorker(ctx context.Context, config *Config, ctlog *loglist.Log, cli
}
batch.entries = entries
select {
case <-ctx.Done():
return ctx.Err()
default:
}
select {
case <-ctx.Done():
return ctx.Err()
@ -476,6 +446,11 @@ func downloadWorker(ctx context.Context, config *Config, ctlog *loglist.Log, cli
func processWorker(ctx context.Context, config *Config, ctlog *loglist.Log, issuerGetter ctclient.IssuerGetter, batchesIn <-chan *batch, batchesOut *sequencer.Channel[batch]) error {
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
var batch *batch
select {
case <-ctx.Done():
@ -507,11 +482,12 @@ func saveStateWorker(ctx context.Context, config *Config, ctlog *loglist.Log, st
if batch.begin != state.DownloadPosition.Size() {
panic(fmt.Errorf("saveStateWorker: expected batch to start at %d but got %d instead", state.DownloadPosition.Size(), batch.begin))
}
rootHash := state.DownloadPosition.CalculateRoot()
for {
for len(batch.sths) > 0 && batch.sths[0].TreeSize == state.DownloadPosition.Size() {
sth := batch.sths[0]
batch.sths = batch.sths[1:]
if rootHash := state.DownloadPosition.CalculateRoot(); sth.RootHash != rootHash {
if sth.RootHash != rootHash {
return &verifyEntriesError{
sth: &sth.SignedTreeHead,
entriesRootHash: rootHash,
@ -527,9 +503,6 @@ func saveStateWorker(ctx context.Context, config *Config, ctlog *loglist.Log, st
if err := config.State.RemoveSTH(ctx, ctlog.LogID, &sth.SignedTreeHead); err != nil {
return fmt.Errorf("error removing verified STH: %w", err)
}
if config.Verbose {
log.Printf("%s: verified position is now %d", ctlog.GetMonitoringURL(), sth.SignedTreeHead.TreeSize)
}
}
if len(batch.entries) == 0 {
break
@ -538,6 +511,7 @@ func saveStateWorker(ctx context.Context, config *Config, ctlog *loglist.Log, st
batch.entries = batch.entries[1:]
leafHash := merkletree.HashLeaf(entry.LeafInput())
state.DownloadPosition.Add(leafHash)
rootHash = state.DownloadPosition.CalculateRoot()
}
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {

View File

@ -89,26 +89,18 @@ func sendEmail(ctx context.Context, to []string, notif *notification) error {
args = append(args, "--")
args = append(args, to...)
sendmailCtx, cancel := context.WithDeadline(ctx, time.Now().Add(2*time.Minute))
defer cancel()
sendmail := exec.CommandContext(sendmailCtx, sendmailPath(), args...)
sendmail := exec.CommandContext(ctx, sendmailPath(), args...)
sendmail.Stdin = stdin
sendmail.Stderr = stderr
sendmail.WaitDelay = 5 * time.Second
if err := sendmail.Run(); err == nil || err == exec.ErrWaitDelay {
if err := sendmail.Run(); err == nil {
return nil
} else if sendmailCtx.Err() != nil && ctx.Err() == nil {
return fmt.Errorf("error sending email to %v: sendmail command timed out", to)
} else if ctx.Err() != nil {
// if the context was canceled, we can't be sure that the error is the fault of sendmail, so ignore it
return ctx.Err()
} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
} else if isExitError {
return fmt.Errorf("error sending email to %v: sendmail terminated by signal with error %q", to, strings.TrimSpace(stderr.String()))
} else {
return fmt.Errorf("error sending email to %v: error running sendmail command: %w", to, err)
return fmt.Errorf("error sending email to %v: %w", to, err)
}
}
@ -119,12 +111,10 @@ func execScript(ctx context.Context, scriptName string, notif *notification) err
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, notif.environ...)
cmd.Stderr = stderr
cmd.WaitDelay = 5 * time.Second
if err := cmd.Run(); err == nil || err == exec.ErrWaitDelay {
if err := cmd.Run(); err == nil {
return nil
} else if ctx.Err() != nil {
// if the context was canceled, we can't be sure that the error is the fault of the script, so ignore it
return ctx.Err()
} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
return fmt.Errorf("script %q exited with code %d and error %q", scriptName, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))

View File

@ -107,15 +107,8 @@ func processCertificate(ctx context.Context, config *Config, entry *LogEntry, ce
}
chain, chainErr := getChain(ctx)
if chainErr != nil {
if ctx.Err() != nil {
// Getting chain failed, but it was probably because our context
// has been canceled, so just act like we never called getChain.
return ctx.Err()
}
// Although getting the chain failed, we still want to notify
// the user about the matching certificate. We'll include chainErr in the
// notification so the user knows why the chain is missing or incorrect.
if errors.Is(chainErr, context.Canceled) {
return chainErr
}
cert := &DiscoveredCert{

View File

@ -54,7 +54,7 @@ type StateProvider interface {
// Store STH for retrieval by LoadSTHs. If an STH with the same
// timestamp and root hash is already stored, this STH can be ignored.
StoreSTH(context.Context, LogID, *cttypes.SignedTreeHead) (*StoredSTH, error)
StoreSTH(context.Context, LogID, *cttypes.SignedTreeHead) error
// Load all STHs for this log previously stored with StoreSTH.
// The returned slice must be sorted by tree size.

View File

@ -47,9 +47,7 @@ func loadSTHsFromDir(dirPath string) ([]*StoredSTH, error) {
continue
}
sth, err := readSTHFile(filepath.Join(dirPath, filename))
if errors.Is(err, fs.ErrNotExist) {
continue
} else if err != nil {
if err != nil {
return nil, err
}
sths = append(sths, sth)
@ -83,26 +81,14 @@ func readSTHFile(filePath string) (*StoredSTH, error) {
return sth, nil
}
func storeSTHInDir(dirPath string, sth *cttypes.SignedTreeHead) (*StoredSTH, error) {
func storeSTHInDir(dirPath string, sth *cttypes.SignedTreeHead) error {
filePath := filepath.Join(dirPath, sthFilename(sth))
if info, err := os.Lstat(filePath); err == nil {
return &StoredSTH{
SignedTreeHead: *sth,
StoredAt: info.ModTime(),
}, nil
} else if !errors.Is(err, fs.ErrNotExist) {
return nil, err
if fileExists(filePath) {
// If the file already exists, we don't want its mtime to change
// because StoredSTH.StoredAt needs to be the time the STH was *first* stored.
return nil
}
if err := writeJSONFile(filePath, sth, 0666); err != nil {
return nil, err
}
return &StoredSTH{
SignedTreeHead: *sth,
StoredAt: time.Now(), // not the exact modtime of the file, but close enough for our purposes
}, nil
return writeJSONFile(filePath, sth, 0666)
}
func removeSTHFromDir(dirPath string, sth *cttypes.SignedTreeHead) error {

View File

@ -22,8 +22,6 @@ type seqWriter struct {
// Channel[T] is a multi-producer, single-consumer channel of items with monotonicaly-increasing sequence numbers.
// Items can be sent in any order, but they are always received in order of their sequence number.
// It is unsafe to call Next concurrently with itself, or to call Add/Reserve concurrently with another Add/Reserve
// call for the same sequence number. Otherwise, methods are safe to call concurrently.
type Channel[T any] struct {
mu sync.Mutex
next uint64
@ -74,27 +72,6 @@ func (seq *Channel[T]) index(seqNbr uint64) int {
return int(seqNbr % seq.Cap())
}
// Wait until the channel has capacity for an item with the given sequence number.
// After this function returns nil, calling Add with the same sequence number will not block.
func (seq *Channel[T]) Reserve(ctx context.Context, sequenceNumber uint64) error {
seq.mu.Lock()
if sequenceNumber >= seq.next+seq.Cap() {
ready := seq.parkWriter(sequenceNumber)
seq.mu.Unlock()
select {
case <-ctx.Done():
seq.mu.Lock()
seq.forgetWriter(sequenceNumber)
seq.mu.Unlock()
return ctx.Err()
case <-ready:
}
} else {
seq.mu.Unlock()
}
return nil
}
// Send an item with the given sequence number. Blocks if the channel does not have capacity for the item.
// It is undefined behavior to send a sequence number that has previously been sent.
func (seq *Channel[T]) Add(ctx context.Context, sequenceNumber uint64, item *T) error {

View File

@ -147,49 +147,3 @@ func TestSequencerOutOfOrder(t *testing.T) {
//t.Logf("seq.Next %d", i)
}
}
func TestSequencerOutOfOrderReserve(t *testing.T) {
ctx := context.Background()
seq := New[uint64](0, 10)
ch := make(chan uint64)
go func() {
for i := range uint64(10_000) {
ch <- i
}
}()
ch2 := make(chan uint64)
for job := range 4 {
go func() {
for i := range ch {
time.Sleep(mathrand.N(11 * time.Duration(job+1) * time.Millisecond))
err := seq.Reserve(ctx, i)
if err != nil {
panic(fmt.Sprintf("%d: seq.Reserve returned unexpected error %v", i, err))
}
ch2 <- i
}
}()
}
for range 4 {
go func() {
for i := range ch2 {
time.Sleep(mathrand.N(7 * time.Millisecond))
t.Logf("seq.Add %d", i)
err := seq.Add(ctx, i, &i)
if err != nil {
panic(fmt.Sprintf("%d: seq.Add returned unexpected error %v", i, err))
}
}
}()
}
for i := range uint64(10_000) {
next, err := seq.Next(ctx)
if err != nil {
t.Fatalf("%d: seq.Next returned unexpected error %v", i, err)
}
if *next != i {
t.Fatalf("%d: got unexpected value %d", i, *next)
}
t.Logf("seq.Next %d", i)
}
}

View File

@ -1 +0,0 @@
checks = ["inherit", "-ST1005", "-S1002"]