Remove unnecessary Printf

Envelope sender and RFC5322.From address are set to $EMAIL if it's non-empty.

Requested in #87

CHANGELOG.md

@@ -0,0 +1,196 @@
+# Change Log
+
+## v0.18.0 (2023-11-13)
+- Fix bug with downloading entries that did not materialize in practice
+  with any of the current logs.
+- Include `Message-ID` and `Date` in outbound emails.
+
+## v0.17.0 (2023-10-26)
+- Allow sendmail path to be configured with `$SENDMAIL_PATH`.
+- Minor improvements to documentation, efficiency.
+
+## v0.16.0 (2023-02-21)
+- Write malformed certs and failed healthchecks to filesystem so scripts
+  can access them.
+- Automatically execute scripts under `$CERTSPOTTER_CONFIG_DIR/hooks.d`
+  if it exists.
+- Automatically email addresses listed in `$CERTSPOTTER_CONFIG_DIR/email_recipients`
+  if it exists.
+
+## v0.15.1 (2023-02-09)
+- Fix some typos in help and error messages.
+- Allow version to be set via linker flag, to facilitate distro package building.
+
+## v0.15.0 (2023-02-08)
+- **Significant behavior change**: certspotter is now intended to run as
+  a daemon instead of a cron job.  Specifically, certspotter no longer
+  terminates unless it receives SIGTERM or SIGINT or there is a serious error.
+  You should remove certspotter from your crontab and arrange to run it as a
+  daemon, passing either the `-email` option or `-script` option to configure
+  how you want to be notified about certificates.
+
+  Reason for this change: although using cron made sense in the early days of
+  Certificate Transparency, certspotter now needs to run continuously to reliably
+  keep up with the high growth rate of contemporary CT logs, and to gracefully
+  handle the many transient errors that can arise when monitoring CT.
+  See <https://github.com/SSLMate/certspotter/issues/63> for background.
+
+- `-script` is now officially supported and can be used to execute
+  a command when a certificate is discovered or there is an error.  For details,
+  see the [certspotter-script(8) man page](man/certspotter-script.md).
+
+  Note the following changes from the experimental, undocumented `-script`
+  option found in previous versions:
+  - The script is also executed when there is an error.  Consult the `$EVENT`
+    variable to determine why the script was executed.
+  - The `$DNS_NAMES` and `$IP_ADDRESSES` variables have been removed because
+    the OS limits the size of environment variables and some certificates have
+    too many identifiers.  To determine a certificate's identifiers, you can
+    read the JSON file specified by the `$JSON_FILENAME` variable, as explained
+    in the [certspotter-script(8) man page](man/certspotter-script.md).
+  - The `$CERT_TYPE` variable has been removed because it is almost always
+    a serious mistake (that can make you miss malicious certificates) to treat
+    certificates and precertificates differently.  If you are currently
+    using this variable to skip precertificates, stop doing that because
+    precertificates imply the existence of a corresponding certificate that you
+    **might not** be separately notified about.  For more details, see
+    <https://github.com/SSLMate/certspotter/commit/cd2bb429fc2f4060a33ec8eb8b71a3eb12e9ba93>.
+  - New variable `$WATCH_ITEM` contains the first watch list item which
+    matched the certificate.
+
+- New `-email` option can be used to send an email when a certificate is
+  discovered or there is an error.  Your system must have a working `sendmail`
+  command.
+
+- (Behavior change) You must specify the `-stdout` option if you want discovered
+  certificates to be written to stdout.  This only makes sense when running
+  certspotter from the terminal; when running as a daemon you probably want to
+  use `-email` or `-script` instead.
+
+- Once a day, certspotter will send you a notification (per `-email` or
+  `-script`) if any problems are preventing it from detecting all certificates.
+  As in previous versions of certspotter, errors are written to stderr when they
+  occur, but since most errors are transient, you can now ignore stderr and rely
+  on the daily health check to notify you about any persistent problems that
+  require your attention.
+
+- certspotter now saves `.json` and `.txt` files alongside the `.pem` files
+  containing parsed representations of the certificate.
+
+- `.pem` files no longer have `.cert` or `.precert` in the filename.
+
+- certspotter will save its state periodically, and before terminating due to
+  SIGTERM or SIGINT, meaning it can resume monitoring without having to
+  re-download entries it has already processed.
+
+- The experimental "BygoneSSL" feature has been removed due to limited utility.
+
+- The `-num_workers` option has been removed.
+
+- The `-all_time` option has been removed. You can remove the certspotter state
+  directory if you want to re-download all entries.
+
+- The minimum supported Go version is now 1.19.
+
+## v0.14.0 (2022-06-13)
+- Switch to Go module versioning conventions.
+
+## v0.13 (2022-06-13)
+- Reduce minimum Go version to 1.17.
+- Update install instructions.
+
+## v0.12 (2022-06-07)
+- Retry failed log requests.  This should make certspotter resilient
+  to rate limiting by logs.
+- Add `-version` flag.
+- Eliminate unnecessary dependency. certspotter now depends only on
+  golang.org/x packages.
+- Switch to Go modules.
+
+## v0.11 (2021-08-17)
+- Add support for contacting logs via HTTP proxies;
+  just set the appropriate environment variable as documented at
+  <https://golang.org/pkg/net/http/#ProxyFromEnvironment>.
+- Work around RFC 6962 ambiguity related to consistency proofs
+  for empty trees.
+
+## v0.10 (2020-04-29)
+- Improve speed by processing logs in parallel
+- Add `-start_at_end` option to begin monitoring new logs at the end,
+  which significantly speeds up Cert Spotter, at the cost of missing
+  certificates that were added to a log before Cert Spotter starts
+  monitoring it
+- (Behavior change) Scan logs in their entirety the first time Cert
+  Spotter is run, unless `-start_at_end` specified (behavior change)
+- The log list is now retrieved from certspotter.org at startup instead
+  of being embedded in the source. This will allow Cert Spotter to react
+  more quickly to the frequent changes in logs.
+- (Behavior change) the `-logs` option now expects a JSON file in the v2
+  log list format. See <https://www.certificate-transparency.org/known-logs>
+  and <https://www.gstatic.com/ct/log_list/v2/log_list_schema.json>.
+- `-logs` now accepts an HTTPS URL in addition to a file path.
+- (Behavior change) the `-underwater` option has been removed. If you want
+  its behavior, specify `https://loglist.certspotter.org/underwater.json` to
+  the `-logs` option.
+
+## v0.9 (2018-04-19)
+- Add Cloudflare Nimbus logs
+- Remove Google Argon 2017 log
+- Remove WoSign and StartCom logs due to disqualification by Chromium
+  and extended downtime
+
+## v0.8 (2017-12-08)
+- Add Symantec Sirius log
+- Add DigiCert 2 log
+
+## v0.7 (2017-11-13)
+- Add Google Argon logs
+- Fix bug that caused crash on 32 bit architectures
+
+## v0.6 (2017-10-19)
+- Add Comodo Mammoth and Comodo Sabre logs
+- Minor bug fixes and improvements
+
+## v0.5 (2017-05-18)
+- Remove PuChuangSiDa 1 log due to excessive downtime and presumptive
+  disqualification from Chrome
+- Add Venafi Gen2 log
+- Improve monitoring robustness under certain pathological behavior
+  by logs
+- Minor documentation improvements
+
+## v0.4 (2017-04-03)
+- Add PuChuangSiDa 1 log
+- Remove Venafi log due to fork and disqualification from Chrome
+
+## v0.3 (2017-02-20)
+- Revise `-all_time` flag (behavior change):
+  - If `-all_time` is specified, scan the entirety of all logs, even
+    existing logs
+  - When a new log is added, scan it in its entirety even if `-all_time`
+    is not specified
+- Add new logs:
+  - Google Icarus
+  - Google Skydiver
+  - StartCom
+  - WoSign
+- Overhaul log processing and auditing logic:
+  - STHs are never deleted unless they can be verified
+  - Multiple unverified STHs can be queued per log, laying groundwork
+    for STH pollination support
+  - New state directory layout; current state directories will be
+    migrated, but migration will be removed in a future version
+  - Persist condensed Merkle Tree state between runs, instead of
+    reconstructing it from consistency proof every time
+- Use a lock file to prevent multiple instances of Cert Spotter from
+  running concurrently (which could clobber the state directory).
+- Minor bug fixes and improvements
+
+## v0.2 (2016-08-25)
+- Suppress duplicate identifiers in output.
+- Fix "EOF" error when running under Go 1.7.
+- Fix bug where hook script could fail silently.
+- Fix compilation under Go 1.5.
+
+## v0.1 (2016-07-27)
+- Initial release.

NEWS

@@ -1,43 +0,0 @@
-v0.5 (2017-05-18)
-  * Remove PuChuangSiDa 1 log due to excessive downtime and presumptive
-    disqualification from Chrome
-  * Add Venafi Gen2 log
-  * Improve monitoring robustness under certain pathological behavior
-    by logs
-  * Minor documentation improvements
-
-v0.4 (2017-04-03)
-  * Add PuChuangSiDa 1 log
-  * Remove Venafi log due to fork and disqualification from Chrome
-
-v0.3 (2017-02-20)
-  * Revise -all_time flag (behavior change):
-    - If -all_time is specified, scan the entirety of all logs, even
-      existing logs
-    - When a new log is added, scan it in its entirety even if -all_time
-      is not specified
-  * Add new logs:
-    - Google Icarus
-    - Google Skydiver
-    - StartCom
-    - WoSign
-  * Overhaul log processing and auditing logic:
-    - STHs are never deleted unless they can be verified
-    - Multiple unverified STHs can be queued per log, laying groundwork
-      for STH pollination support
-    - New state directory layout; current state directories will be
-      migrated, but migration will be removed in a future version
-    - Persist condensed Merkle Tree state between runs, instead of
-      reconstructing it from consistency proof every time
-  * Use a lock file to prevent multiple instances of Cert Spotter from
-    running concurrently (which could clobber the state directory).
-  * Minor bug fixes and improvements
-
-v0.2 (2016-08-25)
-  * Suppress duplicate identifiers in output.
-  * Fix "EOF" error when running under Go 1.7.
-  * Fix bug where hook script could fail silently.
-  * Fix compilation under Go 1.5.
-
-v0.1 (2016-07-27)
-  * Initial release.

README

@@ -1,144 +0,0 @@
-Cert Spotter is a Certificate Transparency log monitor from SSLMate that
-alerts you when a SSL/TLS certificate is issued for one of your domains.
-Cert Spotter is easier than other open source CT monitors, since it does
-not require a database.  It's also more robust, since it uses a special
-certificate parser that ensures it won't miss certificates.
-
-Cert Spotter is also available as a hosted service by SSLMate that
-requires zero setup and provides an easy web dashboard to centrally
-manage your certificates.  Visit <https://sslmate.com/certspotter>
-to sign up.
-
-You can use Cert Spotter to detect:
-
-* Certificates issued to attackers who have compromised a certificate
-  authority and want to impersonate your site.
-
-* Certificates issued to attackers who are using your infrastructure
-  to serve malware.
-
-* Certificates issued in violation of your corporate policy
-  or outside of your centralized certificate procurement process.
-
-* Certificates issued to your infrastructure providers without your
-  consent.
-
-
-USING CERT SPOTTER
-
-The easiest way to use Cert Spotter is to sign up for an account at
-<https://sslmate.com/certspotter>.  If you want to run Cert Spotter on
-your own server, follow these instructions.
-
-Cert Spotter requires Go version 1.5 or higher.
-
-1. Install Cert Spotter using go get:
-
-	go get software.sslmate.com/src/certspotter/cmd/certspotter
-
-2. Create a file called ~/.certspotter/watchlist listing the DNS names
-   you want to monitor, one per line.  To monitor an entire domain tree
-   (including the domain itself and all sub-domains) prefix the domain
-   name with a dot (e.g. ".example.com").  To monitor a single DNS name
-   only, do not prefix the name with a dot.
-
-3. Create a cron job to periodically run:
-
-	certspotter
-
-   When Cert Spotter detects a certificate for a name on your watchlist,
-   it writes a report to standard out, which the Cron daemon emails
-   to you.  Make sure you are able to receive emails sent by Cron.
-
-   Cert Spotter also saves a copy of matching certificates in
-   ~/.certspotter/certs.
-
-You can add and remove domains on your watchlist at any time.  However,
-the certspotter command only notifies you of certificates that were
-logged since adding a domain to the watchlist, unless you specify the
--all_time option, which requires scanning the entirety of every log
-and takes several hours to complete with a fast Internet connection.
-To examine preexisting certificates, it's better to use the Cert
-Spotter service <https://sslmate.com/certspotter>, the Cert Spotter
-API <https://sslmate.com/certspotter/api>, or a CT search engine such
-as <https://crt.sh>.
-
-
-COMMAND LINE FLAGS
-
-  -watchlist FILENAME
-	File containing identifiers to watch, one per line, as described
-	above (use - to read from stdin).  Default: ~/.certspotter/watchlist
-  -no_save
-	Do not save a copy of matching certificates.
-  -all_time
-	Scan for certificates from all time, not just those added since
-	the last run of Cert Spotter.  Unless this option is specified,
-	no certificates are scanned the first time Cert Spotter is run.
-  -logs FILENAME
-	JSON file containing logs to scan, in the format documented at
-	<https://www.certificate-transparency.org/known-logs>.
-	Default: use the logs trusted by Chromium.
-  -state_dir PATH
-	Directory for storing state. Default: ~/.certspotter
-  -verbose
-	Be verbose.
-
-
-WHAT CERTIFICATES ARE DETECTED BY CERT SPOTTER?
-
-Any certificate that is logged to a Certificate Transparency log trusted
-by Chromium will be detected by Cert Spotter.  Currently, the following
-certificates are logged:
-
-* EV certificates
-
-* All certificates issued by the following CAs:
-
-	* Let's Encrypt <https://letsencrypt.org/certificates/#certificate-transparency>
-	* StartCom <https://www.startssl.com/NewsDetails?date=20160323>
-	* Symantec <https://security.googleblog.com/2015/10/sustaining-digital-certificate-security.html>
-	* WoSign <https://www.wosign.com/english/News/2016_wosign_CT.htm>
-
-* All DV certificates issued by GlobalSign <https://www.globalsign.com/en/blog/google-updates-certificate-transparency-policy/>.
-
-* Certificates that are detected when crawling web pages and doing
-  Internet-wide scans.
-
-Starting from April 2018, all new certificates must be logged (and
-therefore detectable by Cert Spotter) to be trusted by Google Chrome.
-
-
-SECURITY
-
-Cert Spotter assumes an adversarial model in which an attacker produces
-a certificate that is accepted by at least some clients but goes
-undetected because of an encoding error that prevents CT monitors from
-understanding it.  To defend against this attack, Cert Spotter uses a
-special certificate parser that keeps the certificate unparsed except
-for the identifiers.  If one of the identifiers matches a domain on your
-watchlist, you will be notified, even if other parts of the certificate
-are unparsable.
-
-Cert Spotter takes special precautions to ensure identifiers are parsed
-correctly, and implements defenses against identifier-based attacks.
-For instance, if a DNS identifier contains a null byte, Cert Spotter
-interprets it as two identifiers: the complete identifier, and the
-identifier formed by truncating at the first null byte.  For example, a
-certificate for example.org\0.example.com will alert the owners of both
-example.org and example.com.  This defends against null prefix attacks
-<http://www.thoughtcrime.org/papers/null-prefix-attacks.pdf>.
-
-SSLMate continuously monitors CT logs to make sure every certificate's
-identifiers can be successfully parsed, and will release updates to
-Cert Spotter as necessary to fix parsing failures.
-
-Cert Spotter understands wildcard and redacted DNS names, and will alert
-you if a wildcard or redacted certificate might match an identifier on
-your watchlist.  For example, a watchlist entry for sub.example.com would
-match certificates for *.example.com or ?.example.com.
-
-Cert Spotter is not just a log monitor, but also a log auditor which
-checks that the log is obeying its append-only property.  A future
-release of Cert Spotter will support gossiping with other log monitors
-to ensure the log is presenting a single view.

README.md

@@ -0,0 +1,105 @@
+# Cert Spotter - Certificate Transparency Monitor
+
+**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that
+alerts you when an SSL/TLS certificate is issued for one of your domains.
+Cert Spotter is easier to use than other open source CT monitors, since it does not require
+a database. It's also more robust, since it uses a special certificate parser
+that ensures it won't miss certificates.
+
+Cert Spotter is also available as a hosted service by SSLMate that
+requires zero setup and provides an easy web dashboard to centrally
+manage your certificates.  Visit <https://sslmate.com/certspotter>
+to sign up.
+
+You can use Cert Spotter to detect:
+
+ * Certificates issued to attackers who have compromised your DNS and
+   are redirecting your visitors to their malicious site.
+ * Certificates issued to attackers who have taken over an abandoned
+   sub-domain in order to serve malware under your name.
+ * Certificates issued to attackers who have compromised a certificate
+   authority and want to impersonate your site.
+ * Certificates issued in violation of your corporate policy
+   or outside of your centralized certificate procurement process.
+
+## Quickstart
+
+Cert Spotter requires Go version 1.19 or higher.
+
+1. Install the certspotter command using the `go` command:
+
+   ```
+   go install software.sslmate.com/src/certspotter/cmd/certspotter@latest
+   ```
+
+2. Create a watch list file `$HOME/.certspotter/watchlist` containing the DNS names you want to monitor,
+   one per line.  To monitor an entire domain tree (including the domain itself
+   and all sub-domains) prefix the domain name with a dot (e.g. `.example.com`).
+   To monitor a single DNS name only, do not prefix the name with a dot.
+
+3. Place one or more email addresses in the `$HOME/.certspotter/email_recipients`
+   file (one per line), and/or place one or more executable scripts in the
+   `$HOME/.certspotter/hooks.d` directory.  certspotter will email the listed
+   addresses (requires your system to have a working sendmail command) and
+   execute the provided scripts when it detects a certificate for a domain on
+   your watch list.
+
+4. Configure your system to run `certspotter` as a daemon.  You may want to specify
+   the `-start_at_end` command line option to tell certspotter to start monitoring
+   new logs at the end instead of the beginning.  This saves significant bandwidth, but
+   you won't be notified about certificates which were logged before you started
+   using certspotter.
+
+## Documentation
+
+* Command line options and operational details: [certspotter(8) man page](man/certspotter.md)
+* The script interface: [certspotter-script(8) man page](man/certspotter-script.md)
+* [Change Log](CHANGELOG.md)
+
+## What certificates are detected by Cert Spotter?
+
+In the default configuration, any certificate that is logged to a Certificate
+Transparency log recognized by Google Chrome or Apple will be detected by
+Cert Spotter.  By default, Google Chrome and Apple only accept certificates that
+are logged, so any certificate that works in Chrome or Safari will be detected
+by Cert Spotter.
+
+## Security
+
+Cert Spotter assumes an adversarial model in which an attacker produces
+a certificate that is accepted by at least some clients but goes
+undetected because of an encoding error that prevents CT monitors from
+understanding it.  To defend against this attack, Cert Spotter uses a
+special certificate parser that keeps the certificate unparsed except
+for the identifiers.  If one of the identifiers matches a domain on your
+watchlist, you will be notified, even if other parts of the certificate
+are unparsable.
+
+Cert Spotter takes special precautions to ensure identifiers are parsed
+correctly, and implements defenses against identifier-based attacks.
+For instance, if a DNS identifier contains a null byte, Cert Spotter
+interprets it as two identifiers: the complete identifier, and the
+identifier formed by truncating at the first null byte.  For example, a
+certificate for `example.org\0.example.com` will alert the owners of both
+`example.org` and `example.com`.  This defends against [null prefix attacks](
+http://www.thoughtcrime.org/papers/null-prefix-attacks.pdf).
+
+SSLMate continuously monitors CT logs to make sure every certificate's
+identifiers can be successfully parsed, and will release updates to
+Cert Spotter as necessary to fix parsing failures.
+
+Cert Spotter understands wildcard DNS names, and will alert
+you if a wildcard certificate might match an identifier on
+your watchlist.  For example, a watchlist entry for `sub.example.com` would
+match certificates for `*.example.com`.
+
+Cert Spotter is not just a log monitor, but also a log auditor which
+checks that the log is obeying its append-only property.  A future
+release of Cert Spotter will support gossiping with other log monitors
+to ensure the log is presenting a single view.
+
+## Copyright
+
+Copyright © 2016-2023 Opsmate, Inc.
+
+Licensed under the [Mozilla Public License Version 2.0](LICENSE).

asn1.go

@@ -49,10 +49,11 @@ func decodeASN1String(value *asn1.RawValue) (string, error) {
 				return "", errors.New("Malformed UTF8String")
 			}
 			return string(value.Bytes), nil
-		} else if value.Tag == 19 || value.Tag == 22 || value.Tag == 20 {
+		} else if value.Tag == 19 || value.Tag == 22 || value.Tag == 20 || value.Tag == 26 {
 			// * PrintableString - subset of ASCII
 			// * IA5String - ASCII
 			// * TeletexString - 8 bit charset; not quite ISO-8859-1, but often treated as such
+			// * VisibleString - subset of ASCII
 
 			// Don't enforce character set rules. Allow any 8 bit character, since
 			// CAs routinely mess this up

auditing.go

@@ -1,213 +0,0 @@
-// Copyright (C) 2016 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package certspotter
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"encoding/json"
-	"errors"
-	"software.sslmate.com/src/certspotter/ct"
-)
-
-func reverseHashes(hashes []ct.MerkleTreeNode) {
-	for i := 0; i < len(hashes)/2; i++ {
-		j := len(hashes) - i - 1
-		hashes[i], hashes[j] = hashes[j], hashes[i]
-	}
-}
-
-func VerifyConsistencyProof(proof ct.ConsistencyProof, first *ct.SignedTreeHead, second *ct.SignedTreeHead) bool {
-	// TODO: make sure every hash in proof is right length? otherwise input to hashChildren is ambiguous
-	if second.TreeSize < first.TreeSize {
-		// Can't be consistent if tree got smaller
-		return false
-	}
-	if first.TreeSize == second.TreeSize {
-		if !(bytes.Equal(first.SHA256RootHash[:], second.SHA256RootHash[:]) && len(proof) == 0) {
-			return false
-		}
-		return true
-	}
-	if first.TreeSize == 0 {
-		// The purpose of the consistency proof is to ensure the append-only
-		// nature of the tree; i.e. that the first tree is a "prefix" of the
-		// second tree.  If the first tree is empty, then it's trivially a prefix
-		// of the second tree, so no proof is needed.
-		if len(proof) != 0 {
-			return false
-		}
-		return true
-	}
-	// Guaranteed that 0 < first.TreeSize < second.TreeSize
-
-	node := first.TreeSize - 1
-	lastNode := second.TreeSize - 1
-
-	// While we're the right child, everything is in both trees, so move one level up.
-	for node%2 == 1 {
-		node /= 2
-		lastNode /= 2
-	}
-
-	var newHash ct.MerkleTreeNode
-	var oldHash ct.MerkleTreeNode
-	if node > 0 {
-		if len(proof) == 0 {
-			return false
-		}
-		newHash = proof[0]
-		proof = proof[1:]
-	} else {
-		// The old tree was balanced, so we already know the first hash to use
-		newHash = first.SHA256RootHash[:]
-	}
-	oldHash = newHash
-
-	for node > 0 {
-		if node%2 == 1 {
-			// node is a right child; left sibling exists in both trees
-			if len(proof) == 0 {
-				return false
-			}
-			newHash = hashChildren(proof[0], newHash)
-			oldHash = hashChildren(proof[0], oldHash)
-			proof = proof[1:]
-		} else if node < lastNode {
-			// node is a left child; rigth sibling only exists in the new tree
-			if len(proof) == 0 {
-				return false
-			}
-			newHash = hashChildren(newHash, proof[0])
-			proof = proof[1:]
-		} // else node == lastNode: node is a left child with no sibling in either tree
-		node /= 2
-		lastNode /= 2
-	}
-
-	if !bytes.Equal(oldHash, first.SHA256RootHash[:]) {
-		return false
-	}
-
-	// If trees have different height, continue up the path to reach the new root
-	for lastNode > 0 {
-		if len(proof) == 0 {
-			return false
-		}
-		newHash = hashChildren(newHash, proof[0])
-		proof = proof[1:]
-		lastNode /= 2
-	}
-
-	if !bytes.Equal(newHash, second.SHA256RootHash[:]) {
-		return false
-	}
-
-	return true
-}
-
-func hashNothing() ct.MerkleTreeNode {
-	return sha256.New().Sum(nil)
-}
-
-func hashLeaf(leafBytes []byte) ct.MerkleTreeNode {
-	hasher := sha256.New()
-	hasher.Write([]byte{0x00})
-	hasher.Write(leafBytes)
-	return hasher.Sum(nil)
-}
-
-func hashChildren(left ct.MerkleTreeNode, right ct.MerkleTreeNode) ct.MerkleTreeNode {
-	hasher := sha256.New()
-	hasher.Write([]byte{0x01})
-	hasher.Write(left)
-	hasher.Write(right)
-	return hasher.Sum(nil)
-}
-
-type CollapsedMerkleTree struct {
-	nodes []ct.MerkleTreeNode
-	size  uint64
-}
-
-func calculateNumNodes(size uint64) int {
-	numNodes := 0
-	for size > 0 {
-		numNodes += int(size & 1)
-		size >>= 1
-	}
-	return numNodes
-}
-func EmptyCollapsedMerkleTree() *CollapsedMerkleTree {
-	return &CollapsedMerkleTree{}
-}
-func NewCollapsedMerkleTree(nodes []ct.MerkleTreeNode, size uint64) (*CollapsedMerkleTree, error) {
-	if len(nodes) != calculateNumNodes(size) {
-		return nil, errors.New("NewCollapsedMerkleTree: nodes has incorrect size")
-	}
-	return &CollapsedMerkleTree{nodes: nodes, size: size}, nil
-}
-func CloneCollapsedMerkleTree(source *CollapsedMerkleTree) *CollapsedMerkleTree {
-	nodes := make([]ct.MerkleTreeNode, len(source.nodes))
-	copy(nodes, source.nodes)
-	return &CollapsedMerkleTree{nodes: nodes, size: source.size}
-}
-
-func (tree *CollapsedMerkleTree) Add(hash ct.MerkleTreeNode) {
-	tree.nodes = append(tree.nodes, hash)
-	tree.size++
-	size := tree.size
-	for size%2 == 0 {
-		left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1]
-		tree.nodes = tree.nodes[:len(tree.nodes)-2]
-		tree.nodes = append(tree.nodes, hashChildren(left, right))
-		size /= 2
-	}
-}
-
-func (tree *CollapsedMerkleTree) CalculateRoot() ct.MerkleTreeNode {
-	if len(tree.nodes) == 0 {
-		return hashNothing()
-	}
-	i := len(tree.nodes) - 1
-	hash := tree.nodes[i]
-	for i > 0 {
-		i -= 1
-		hash = hashChildren(tree.nodes[i], hash)
-	}
-	return hash
-}
-
-func (tree *CollapsedMerkleTree) GetSize() uint64 {
-	return tree.size
-}
-
-func (tree *CollapsedMerkleTree) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
-		"nodes": tree.nodes,
-		"size":  tree.size,
-	})
-}
-
-func (tree *CollapsedMerkleTree) UnmarshalJSON(b []byte) error {
-	var rawTree struct {
-		Nodes []ct.MerkleTreeNode `json:"nodes"`
-		Size  uint64              `json:"size"`
-	}
-	if err := json.Unmarshal(b, &rawTree); err != nil {
-		return errors.New("Failed to unmarshal CollapsedMerkleTree: " + err.Error())
-	}
-	if len(rawTree.Nodes) != calculateNumNodes(rawTree.Size) {
-		return errors.New("Failed to unmarshal CollapsedMerkleTree: nodes has incorrect length")
-	}
-	tree.size = rawTree.Size
-	tree.nodes = rawTree.Nodes
-	return nil
-}

canonicalize.go

@@ -0,0 +1,92 @@
+// Copyright (C) 2019 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"encoding/asn1"
+)
+
+func canonicalizeRDNString(fromStr string) string {
+	from := []byte(fromStr)
+	to := []byte{}
+	inWhitespace := true
+	for _, ch := range from {
+		if ch == ' ' || ch == '\f' || ch == '\n' || ch == '\r' || ch == '\t' || ch == '\v' {
+			if !inWhitespace {
+				to = append(to, ' ')
+			}
+			inWhitespace = true
+		} else {
+			if ch >= 'A' && ch <= 'Z' {
+				to = append(to, ch+32) // convert to lowercase
+			} else {
+				to = append(to, ch)
+			}
+			inWhitespace = false
+		}
+	}
+	if inWhitespace && len(to) > 0 {
+		// whack off the space character that we appended
+		to = to[:len(to)-1]
+	}
+	return string(to)
+}
+
+func shouldCanonicalizeASN1String(value *asn1.RawValue) bool {
+	if !value.IsCompound && value.Class == 0 {
+		return value.Tag == 12 || value.Tag == 19 || value.Tag == 22 || value.Tag == 20 || value.Tag == 26 || value.Tag == 30 || value.Tag == 28
+	}
+	return false
+}
+
+func canonicalizeATV(oldATV AttributeTypeAndValue) (AttributeTypeAndValue, error) {
+	if shouldCanonicalizeASN1String(&oldATV.Value) {
+		str, err := decodeASN1String(&oldATV.Value)
+		if err != nil {
+			return AttributeTypeAndValue{}, err
+		}
+		str = canonicalizeRDNString(str)
+		return AttributeTypeAndValue{
+			Type: oldATV.Type,
+			Value: asn1.RawValue{
+				Class:      0,
+				Tag:        asn1.TagUTF8String,
+				IsCompound: false,
+				Bytes:      []byte(str),
+			},
+		}, nil
+	} else {
+		return oldATV, nil
+	}
+}
+
+func canonicalizeRDNSet(oldSet RelativeDistinguishedNameSET) (RelativeDistinguishedNameSET, error) {
+	newSet := make([]AttributeTypeAndValue, len(oldSet))
+	for i := range oldSet {
+		var err error
+		newSet[i], err = canonicalizeATV(oldSet[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+	return newSet, nil
+}
+
+func CanonicalizeRDNSequence(oldSequence RDNSequence) (RDNSequence, error) {
+	newSequence := make([]RelativeDistinguishedNameSET, len(oldSequence))
+	for i := range oldSequence {
+		var err error
+		newSequence[i], err = canonicalizeRDNSet(oldSequence[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+	return newSequence, nil
+}

canonicalize_test.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2019 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"testing"
+)
+
+type stringCanonTest struct {
+	in  string
+	out string
+}
+
+var stringCanonTests = []stringCanonTest{
+	{"", ""},
+	{" ", ""},
+	{"  ", ""},
+	{"abc", "abc"},
+	{"aBc", "abc"},
+	{"ab c", "ab c"},
+	{"ab  c", "ab c"},
+	{"ab\n c", "ab c"},
+	{" ab c ", "ab c"},
+	{" ab  c ", "ab c"},
+	{" ab  c", "ab c"},
+	{"ab  c ", "ab c"},
+	{"abc ", "abc"},
+	{"abc  ", "abc"},
+	{"  abc  ", "abc"},
+	{"  abc ", "abc"},
+	{"  abc", "abc"},
+	{"  aBc de  f      g\n", "abc de f g"},
+}
+
+func TestCanonicalizeRDNString(t *testing.T) {
+	for i, test := range stringCanonTests {
+		ret := canonicalizeRDNString(test.in)
+		if test.out != ret {
+			t.Errorf("#%d: canonicalizeRDNString(%q) = %q, want %q", i, test.in, ret, test.out)
+		}
+	}
+}

cmd/certspotter/main.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2016 Opsmate, Inc.
+// Copyright (C) 2016, 2023 Opsmate, Inc.
 //
 // This Source Code Form is subject to the terms of the Mozilla
 // Public License, v. 2.0. If a copy of the MPL was not distributed
@@ -11,185 +11,242 @@ package main
 
 import (
 	"bufio"
+	"context"
+	"errors"
 	"flag"
 	"fmt"
-	"io"
+	"io/fs"
 	"os"
+	"os/signal"
 	"path/filepath"
+	"runtime"
+	"runtime/debug"
 	"strings"
+	"syscall"
+	"time"
 
-	"golang.org/x/net/idna"
-
-	"software.sslmate.com/src/certspotter"
-	"software.sslmate.com/src/certspotter/cmd"
-	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/monitor"
 )
 
+var programName = os.Args[0]
+var Version = ""
+
+const defaultLogList = "https://loglist.certspotter.org/monitor.json"
+
+func certspotterVersion() string {
+	if Version != "" {
+		return Version + "?"
+	}
+	info, ok := debug.ReadBuildInfo()
+	if !ok {
+		return "unknown"
+	}
+	if strings.HasPrefix(info.Main.Version, "v") {
+		return info.Main.Version
+	}
+	var vcs, vcsRevision, vcsModified string
+	for _, s := range info.Settings {
+		switch s.Key {
+		case "vcs":
+			vcs = s.Value
+		case "vcs.revision":
+			vcsRevision = s.Value
+		case "vcs.modified":
+			vcsModified = s.Value
+		}
+	}
+	if vcs == "git" && vcsRevision != "" && vcsModified == "true" {
+		return vcsRevision + "+"
+	} else if vcs == "git" && vcsRevision != "" {
+		return vcsRevision
+	}
+	return "unknown"
+}
+
+func fileExists(filename string) bool {
+	_, err := os.Lstat(filename)
+	return err == nil
+}
+func homedir() string {
+	homedir, err := os.UserHomeDir()
+	if err != nil {
+		panic(fmt.Errorf("unable to determine home directory: %w", err))
+	}
+	return homedir
+}
 func defaultStateDir() string {
 	if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" {
 		return envVar
 	} else {
-		return cmd.DefaultStateDir("certspotter")
+		return filepath.Join(homedir(), ".certspotter")
 	}
 }
 func defaultConfigDir() string {
 	if envVar := os.Getenv("CERTSPOTTER_CONFIG_DIR"); envVar != "" {
 		return envVar
 	} else {
-		return cmd.DefaultConfigDir("certspotter")
+		return filepath.Join(homedir(), ".certspotter")
 	}
 }
-
-func trimTrailingDots(value string) string {
-	length := len(value)
-	for length > 0 && value[length-1] == '.' {
-		length--
-	}
-	return value[0:length]
+func defaultWatchListPath() string {
+	return filepath.Join(defaultConfigDir(), "watchlist")
 }
-
-var stateDir = flag.String("state_dir", defaultStateDir(), "Directory for storing state")
-var watchlistFilename = flag.String("watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing identifiers to watch (- for stdin)")
-
-type watchlistItem struct {
-	Domain       []string
-	AcceptSuffix bool
-}
-
-var watchlist []watchlistItem
-
-func parseWatchlistItem(str string) (watchlistItem, error) {
-	if str == "." { // "." as in root zone (matches everything)
-		return watchlistItem{
-			Domain:       []string{},
-			AcceptSuffix: true,
-		}, nil
+func defaultWatchListPathIfExists() string {
+	if fileExists(defaultWatchListPath()) {
+		return defaultWatchListPath()
 	} else {
-		acceptSuffix := false
-		if strings.HasPrefix(str, ".") {
-			acceptSuffix = true
-			str = str[1:]
-		}
-		asciiDomain, err := idna.ToASCII(strings.ToLower(trimTrailingDots(str)))
-		if err != nil {
-			return watchlistItem{}, fmt.Errorf("Invalid domain `%s': %s", str, err)
-		}
-		return watchlistItem{
-			Domain:       strings.Split(asciiDomain, "."),
-			AcceptSuffix: acceptSuffix,
-		}, nil
+		return ""
 	}
 }
+func defaultScriptDir() string {
+	return filepath.Join(defaultConfigDir(), "hooks.d")
+}
+func defaultEmailFile() string {
+	return filepath.Join(defaultConfigDir(), "email_recipients")
+}
 
-func readWatchlist(reader io.Reader) ([]watchlistItem, error) {
-	items := []watchlistItem{}
-	scanner := bufio.NewScanner(reader)
+func simplifyError(err error) error {
+	var pathErr *fs.PathError
+	if errors.As(err, &pathErr) {
+		return pathErr.Err
+	}
+
+	return err
+}
+
+func readWatchListFile(filename string) (monitor.WatchList, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, simplifyError(err)
+	}
+	defer file.Close()
+	return monitor.ReadWatchList(file)
+}
+
+func readEmailFile(filename string) ([]string, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, simplifyError(err)
+	}
+	defer file.Close()
+
+	var emails []string
+	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		line := scanner.Text()
-		if line == "" || strings.HasPrefix(line, "#") {
+		if line == "" {
 			continue
 		}
-		item, err := parseWatchlistItem(line)
-		if err != nil {
-			return nil, err
-		}
-		items = append(items, item)
+		emails = append(emails, line)
 	}
-	return items, scanner.Err()
+	return emails, err
 }
 
-func dnsLabelMatches(certLabel string, watchLabel string) bool {
-	// For fail-safe behavior, if a label was unparsable, it matches everything.
-	// Similarly, redacted labels match everything, since the label _might_ be
-	// for a name we're interested in.
-
-	return certLabel == "*" ||
-		certLabel == "?" ||
-		certLabel == certspotter.UnparsableDNSLabelPlaceholder ||
-		certspotter.MatchesWildcard(watchLabel, certLabel)
-}
-
-func dnsNameMatches(dnsName []string, watchDomain []string, acceptSuffix bool) bool {
-	for len(dnsName) > 0 && len(watchDomain) > 0 {
-		certLabel := dnsName[len(dnsName)-1]
-		watchLabel := watchDomain[len(watchDomain)-1]
-
-		if !dnsLabelMatches(certLabel, watchLabel) {
-			return false
-		}
-
-		dnsName = dnsName[:len(dnsName)-1]
-		watchDomain = watchDomain[:len(watchDomain)-1]
-	}
-
-	return len(watchDomain) == 0 && (acceptSuffix || len(dnsName) == 0)
-}
-
-func dnsNameIsWatched(dnsName string) bool {
-	labels := strings.Split(dnsName, ".")
-	for _, item := range watchlist {
-		if dnsNameMatches(labels, item.Domain, item.AcceptSuffix) {
-			return true
-		}
-	}
-	return false
-}
-
-func anyDnsNameIsWatched(dnsNames []string) bool {
-	for _, dnsName := range dnsNames {
-		if dnsNameIsWatched(dnsName) {
-			return true
-		}
-	}
-	return false
-}
-
-func processEntry(scanner *certspotter.Scanner, entry *ct.LogEntry) {
-	info := certspotter.EntryInfo{
-		LogUri:    scanner.LogUri,
-		Entry:     entry,
-		IsPrecert: certspotter.IsPrecert(entry),
-		FullChain: certspotter.GetFullChain(entry),
-	}
-
-	info.CertInfo, info.ParseError = certspotter.MakeCertInfoFromLogEntry(entry)
-
-	if info.CertInfo != nil {
-		info.Identifiers, info.IdentifiersParseError = info.CertInfo.ParseIdentifiers()
-	}
-
-	// Fail safe behavior: if info.Identifiers is nil (which is caused by a
-	// parse error), report the certificate because we can't say for sure it
-	// doesn't match a domain we care about.  We try very hard to make sure
-	// parsing identifiers always succeeds, so false alarms should be rare.
-	if info.Identifiers == nil || anyDnsNameIsWatched(info.Identifiers.DNSNames) {
-		cmd.LogEntry(&info)
+func appendFunc(slice *[]string) func(string) error {
+	return func(value string) error {
+		*slice = append(*slice, value)
+		return nil
 	}
 }
 
 func main() {
+	loglist.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s)", certspotterVersion(), runtime.Version(), runtime.GOOS, runtime.GOARCH)
+
+	var flags struct {
+		batchSize   int // TODO-4: respect this option
+		email       []string
+		healthcheck time.Duration
+		logs        string
+		noSave      bool
+		script      string
+		startAtEnd  bool
+		stateDir    string
+		stdout      bool
+		verbose     bool
+		version     bool
+		watchlist   string
+	}
+	flag.IntVar(&flags.batchSize, "batch_size", 1000, "Max number of entries to request per call to get-entries (advanced)")
+	flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email))
+	flag.DurationVar(&flags.healthcheck, "healthcheck", 24*time.Hour, "How frequently to perform a health check")
+	flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
+	flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory")
+	flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered")
+	flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring new logs from the end rather than the beginning (saves considerable bandwidth)")
+	flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates")
+	flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout")
+	flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose")
+	flag.BoolVar(&flags.version, "version", false, "Print version and exit")
+	flag.StringVar(&flags.watchlist, "watchlist", defaultWatchListPathIfExists(), "File containing domain names to watch")
 	flag.Parse()
 
-	if *watchlistFilename == "-" {
-		var err error
-		watchlist, err = readWatchlist(os.Stdin)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s: (stdin): %s\n", os.Args[0], err)
-			os.Exit(1)
-		}
-	} else {
-		file, err := os.Open(*watchlistFilename)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err)
-			os.Exit(1)
-		}
-		defer file.Close()
-		watchlist, err = readWatchlist(file)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err)
-			os.Exit(1)
-		}
+	if flags.version {
+		fmt.Fprintf(os.Stdout, "certspotter version %s\n", certspotterVersion())
+		os.Exit(0)
+	}
+	if flags.watchlist == "" {
+		fmt.Fprintf(os.Stderr, "%s: watch list not found: please create %s or specify alternative path using -watchlist\n", programName, defaultWatchListPath())
+		os.Exit(2)
 	}
 
-	os.Exit(cmd.Main(*stateDir, processEntry))
+	fsstate := &monitor.FilesystemState{
+		StateDir:  flags.stateDir,
+		SaveCerts: !flags.noSave,
+		Script:    flags.script,
+		ScriptDir: defaultScriptDir(),
+		Email:     flags.email,
+		Stdout:    flags.stdout,
+	}
+	config := &monitor.Config{
+		LogListSource:       flags.logs,
+		State:               fsstate,
+		StartAtEnd:          flags.startAtEnd,
+		Verbose:             flags.verbose,
+		HealthCheckInterval: flags.healthcheck,
+	}
+
+	emailFileExists := false
+	if emailRecipients, err := readEmailFile(defaultEmailFile()); err == nil {
+		emailFileExists = true
+		fsstate.Email = append(fsstate.Email, emailRecipients...)
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		fmt.Fprintf(os.Stderr, "%s: error reading email recipients file %q: %s\n", programName, defaultEmailFile(), err)
+		os.Exit(1)
+	}
+
+	if len(fsstate.Email) == 0 && !emailFileExists && fsstate.Script == "" && !fileExists(fsstate.ScriptDir) && fsstate.Stdout == false {
+		fmt.Fprintf(os.Stderr, "%s: no notification methods were specified\n", programName)
+		fmt.Fprintf(os.Stderr, "Please specify at least one of the following notification methods:\n")
+		fmt.Fprintf(os.Stderr, " - Place one or more email addresses in %s (one address per line)\n", defaultEmailFile())
+		fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", fsstate.ScriptDir)
+		fmt.Fprintf(os.Stderr, " - Specify an email address using the -email flag\n")
+		fmt.Fprintf(os.Stderr, " - Specify the path to an executable script using the -script flag\n")
+		fmt.Fprintf(os.Stderr, " - Specify the -stdout flag\n")
+		os.Exit(2)
+	}
+
+	if flags.watchlist == "-" {
+		watchlist, err := monitor.ReadWatchList(os.Stdin)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: error reading watchlist from standard in: %s\n", programName, err)
+			os.Exit(1)
+		}
+		config.WatchList = watchlist
+	} else {
+		watchlist, err := readWatchListFile(flags.watchlist)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: error reading watchlist from %q: %s\n", programName, flags.watchlist, err)
+			os.Exit(1)
+		}
+		config.WatchList = watchlist
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	if err := monitor.Run(ctx, config); err != nil && !errors.Is(err, context.Canceled) {
+		fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err)
+		os.Exit(1)
+	}
 }

cmd/common.go

@@ -1,351 +0,0 @@
-// Copyright (C) 2016-2017 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package cmd
-
-import (
-	"bytes"
-	"flag"
-	"fmt"
-	"log"
-	"os"
-	"os/user"
-	"path/filepath"
-	"sync"
-
-	"software.sslmate.com/src/certspotter"
-	"software.sslmate.com/src/certspotter/ct"
-)
-
-var batchSize = flag.Int("batch_size", 1000, "Max number of entries to request at per call to get-entries (advanced)")
-var numWorkers = flag.Int("num_workers", 2, "Number of concurrent matchers (advanced)")
-var script = flag.String("script", "", "Script to execute when a matching certificate is found")
-var logsFilename = flag.String("logs", "", "JSON file containing log information")
-var underwater = flag.Bool("underwater", false, "Monitor certificates from distrusted CAs instead of trusted CAs")
-var noSave = flag.Bool("no_save", false, "Do not save a copy of matching certificates")
-var verbose = flag.Bool("verbose", false, "Be verbose")
-var allTime = flag.Bool("all_time", false, "Scan certs from all time, not just since last scan")
-var state *State
-
-var printMutex sync.Mutex
-
-func homedir() string {
-	home := os.Getenv("HOME")
-	if home != "" {
-		return home
-	}
-	user, err := user.Current()
-	if err == nil {
-		return user.HomeDir
-	}
-	panic("Unable to determine home directory")
-}
-
-func DefaultStateDir(programName string) string {
-	return filepath.Join(homedir(), "."+programName)
-}
-
-func DefaultConfigDir(programName string) string {
-	return filepath.Join(homedir(), "."+programName)
-}
-
-func LogEntry(info *certspotter.EntryInfo) {
-	if !*noSave {
-		var alreadyPresent bool
-		var err error
-		alreadyPresent, info.Filename, err = state.SaveCert(info.IsPrecert, info.FullChain)
-		if err != nil {
-			log.Print(err)
-		}
-		if alreadyPresent {
-			return
-		}
-	}
-
-	if *script != "" {
-		if err := info.InvokeHookScript(*script); err != nil {
-			log.Print(err)
-		}
-	} else {
-		printMutex.Lock()
-		info.Write(os.Stdout)
-		fmt.Fprintf(os.Stdout, "\n")
-		printMutex.Unlock()
-	}
-}
-
-func loadLogList() ([]certspotter.LogInfo, error) {
-	if *logsFilename != "" {
-		var logFileObj certspotter.LogInfoFile
-		if err := readJSONFile(*logsFilename, &logFileObj); err != nil {
-			return nil, fmt.Errorf("Error reading logs file: %s: %s", *logsFilename, err)
-		}
-		return logFileObj.Logs, nil
-	} else if *underwater {
-		return certspotter.UnderwaterLogs, nil
-	} else {
-		return certspotter.DefaultLogs, nil
-	}
-}
-
-type logHandle struct {
-	scanner     *certspotter.Scanner
-	state       *LogState
-	tree        *certspotter.CollapsedMerkleTree
-	verifiedSTH *ct.SignedTreeHead
-}
-
-func makeLogHandle(logInfo *certspotter.LogInfo) (*logHandle, error) {
-	ctlog := new(logHandle)
-
-	logKey, err := logInfo.ParsedPublicKey()
-	if err != nil {
-		return nil, fmt.Errorf("Bad public key: %s", err)
-	}
-	ctlog.scanner = certspotter.NewScanner(logInfo.FullURI(), logInfo.ID(), logKey, &certspotter.ScannerOptions{
-		BatchSize:  *batchSize,
-		NumWorkers: *numWorkers,
-		Quiet:      !*verbose,
-	})
-
-	ctlog.state, err = state.OpenLogState(logInfo)
-	if err != nil {
-		return nil, fmt.Errorf("Error opening state directory: %s", err)
-	}
-	ctlog.tree, err = ctlog.state.GetTree()
-	if err != nil {
-		return nil, fmt.Errorf("Error loading tree: %s", err)
-	}
-	ctlog.verifiedSTH, err = ctlog.state.GetVerifiedSTH()
-	if err != nil {
-		return nil, fmt.Errorf("Error loading verified STH: %s", err)
-	}
-
-	if ctlog.tree == nil && ctlog.verifiedSTH == nil { // This branch can be removed eventually
-		legacySTH, err := state.GetLegacySTH(logInfo)
-		if err != nil {
-			return nil, fmt.Errorf("Error loading legacy STH: %s", err)
-		}
-		if legacySTH != nil {
-			log.Printf("Initializing log state from legacy state directory")
-			ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(legacySTH)
-			if err != nil {
-				return nil, fmt.Errorf("Error reconstructing Merkle Tree for legacy STH: %s", err)
-			}
-			if err := ctlog.state.StoreTree(ctlog.tree); err != nil {
-				return nil, fmt.Errorf("Error storing tree: %s", err)
-			}
-			if err := ctlog.state.StoreVerifiedSTH(legacySTH); err != nil {
-				return nil, fmt.Errorf("Error storing verified STH: %s", err)
-			}
-			state.RemoveLegacySTH(logInfo)
-		}
-	}
-
-	return ctlog, nil
-}
-
-func (ctlog *logHandle) refresh() error {
-	if *verbose {
-		log.Printf("Retrieving latest STH from log")
-	}
-	latestSTH, err := ctlog.scanner.GetSTH()
-	if err != nil {
-		return fmt.Errorf("Error retrieving STH from log: %s", err)
-	}
-	if ctlog.verifiedSTH == nil {
-		if *verbose {
-			log.Printf("No existing STH is known; presuming latest STH (%d) is valid", latestSTH.TreeSize)
-		}
-		ctlog.verifiedSTH = latestSTH
-		if err := ctlog.state.StoreVerifiedSTH(ctlog.verifiedSTH); err != nil {
-			return fmt.Errorf("Error storing verified STH: %s", err)
-		}
-	} else {
-		if err := ctlog.state.StoreUnverifiedSTH(latestSTH); err != nil {
-			return fmt.Errorf("Error storing unverified STH: %s", err)
-		}
-	}
-	return nil
-}
-
-func (ctlog *logHandle) verifySTH(sth *ct.SignedTreeHead) error {
-	isValid, err := ctlog.scanner.CheckConsistency(ctlog.verifiedSTH, sth)
-	if err != nil {
-		return fmt.Errorf("Error fetching consistency proof: %s", err)
-	}
-	if !isValid {
-		return fmt.Errorf("Consistency proof between %d and %d is invalid", ctlog.verifiedSTH.TreeSize, sth.TreeSize)
-	}
-	return nil
-}
-
-func (ctlog *logHandle) audit() error {
-	sths, err := ctlog.state.GetUnverifiedSTHs()
-	if err != nil {
-		return fmt.Errorf("Error loading unverified STHs: %s", err)
-	}
-
-	for _, sth := range sths {
-		if *verbose {
-			log.Printf("Verifying consistency of STH %d (%x) with previously-verified STH %d (%x)", sth.TreeSize, sth.SHA256RootHash, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash)
-		}
-		if err := ctlog.verifySTH(sth); err != nil {
-			log.Printf("Unable to verify consistency of STH %d (%s) (if this error persists, it should be construed as misbehavior by the log): %s", sth.TreeSize, ctlog.state.UnverifiedSTHFilename(sth), err)
-			continue
-		}
-		if sth.TreeSize > ctlog.verifiedSTH.TreeSize {
-			if *verbose {
-				log.Printf("STH %d (%x) is now the latest verified STH", sth.TreeSize, sth.SHA256RootHash)
-			}
-			ctlog.verifiedSTH = sth
-			if err := ctlog.state.StoreVerifiedSTH(ctlog.verifiedSTH); err != nil {
-				return fmt.Errorf("Error storing verified STH: %s", err)
-			}
-		}
-		if err := ctlog.state.RemoveUnverifiedSTH(sth); err != nil {
-			return fmt.Errorf("Error removing redundant STH: %s", err)
-		}
-	}
-
-	return nil
-}
-
-func (ctlog *logHandle) scan(processCallback certspotter.ProcessCallback) error {
-	startIndex := int64(ctlog.tree.GetSize())
-	endIndex := int64(ctlog.verifiedSTH.TreeSize)
-
-	if endIndex > startIndex {
-		tree := certspotter.CloneCollapsedMerkleTree(ctlog.tree)
-
-		if err := ctlog.scanner.Scan(startIndex, endIndex, processCallback, tree); err != nil {
-			return fmt.Errorf("Error scanning log (if this error persists, it should be construed as misbehavior by the log): %s", err)
-		}
-
-		rootHash := tree.CalculateRoot()
-		if !bytes.Equal(rootHash, ctlog.verifiedSTH.SHA256RootHash[:]) {
-			return fmt.Errorf("Log has misbehaved: log entries at tree size %d do not correspond to signed tree root", ctlog.verifiedSTH.TreeSize)
-		}
-
-		ctlog.tree = tree
-		if err := ctlog.state.StoreTree(ctlog.tree); err != nil {
-			return fmt.Errorf("Error storing tree: %s", err)
-		}
-	}
-
-	return nil
-}
-
-func processLog(logInfo *certspotter.LogInfo, processCallback certspotter.ProcessCallback) int {
-	log.SetPrefix(os.Args[0] + ": " + logInfo.Url + ": ")
-
-	ctlog, err := makeLogHandle(logInfo)
-	if err != nil {
-		log.Printf("%s\n", err)
-		return 1
-	}
-
-	if err := ctlog.refresh(); err != nil {
-		log.Printf("%s\n", err)
-		return 1
-	}
-
-	if err := ctlog.audit(); err != nil {
-		log.Printf("%s\n", err)
-		return 1
-	}
-
-	if *allTime {
-		ctlog.tree = certspotter.EmptyCollapsedMerkleTree()
-		if *verbose {
-			log.Printf("Scanning all %d entries in the log because -all_time option specified", ctlog.verifiedSTH.TreeSize)
-		}
-	} else if ctlog.tree != nil {
-		if *verbose {
-			log.Printf("Existing log; scanning %d new entries since previous scan", ctlog.verifiedSTH.TreeSize-ctlog.tree.GetSize())
-		}
-	} else if state.IsFirstRun() {
-		ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(ctlog.verifiedSTH)
-		if err != nil {
-			log.Printf("Error reconstructing Merkle Tree: %s", err)
-			return 1
-		}
-		if *verbose {
-			log.Printf("First run of Cert Spotter; not scanning %d existing entries because -all_time option not specified", ctlog.verifiedSTH.TreeSize)
-		}
-	} else {
-		ctlog.tree = certspotter.EmptyCollapsedMerkleTree()
-		if *verbose {
-			log.Printf("New log; scanning all %d entries in the log", ctlog.verifiedSTH.TreeSize)
-		}
-	}
-	if err := ctlog.state.StoreTree(ctlog.tree); err != nil {
-		log.Printf("Error storing tree: %s\n", err)
-		return 1
-	}
-
-	if err := ctlog.scan(processCallback); err != nil {
-		log.Printf("%s\n", err)
-		return 1
-	}
-
-	if *verbose {
-		log.Printf("Final log size = %d, final root hash = %x", ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash)
-	}
-
-	return 0
-}
-
-func Main(statePath string, processCallback certspotter.ProcessCallback) int {
-	var err error
-
-	logs, err := loadLogList()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err)
-		return 1
-	}
-
-	state, err = OpenState(statePath)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err)
-		return 1
-	}
-	locked, err := state.Lock()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s: Error locking state directory: %s\n", os.Args[0], err)
-		return 1
-	}
-	if !locked {
-		var otherPidInfo string
-		if otherPid := state.LockingPid(); otherPid != 0 {
-			otherPidInfo = fmt.Sprintf(" (as process ID %d)", otherPid)
-		}
-		fmt.Fprintf(os.Stderr, "%s: Another instance of %s is already running%s; remove the file %s if this is not the case\n", os.Args[0], os.Args[0], otherPidInfo, state.LockFilename())
-		return 1
-	}
-
-	exitCode := 0
-	for i := range logs {
-		exitCode |= processLog(&logs[i], processCallback)
-	}
-
-	if state.IsFirstRun() && exitCode == 0 {
-		if err := state.WriteOnceFile(); err != nil {
-			fmt.Fprintf(os.Stderr, "%s: Error writing once file: %s\n", os.Args[0], err)
-			exitCode |= 1
-		}
-	}
-
-	if err := state.Unlock(); err != nil {
-		fmt.Fprintf(os.Stderr, "%s: Error unlocking state directory: %s\n", os.Args[0], err)
-		exitCode |= 1
-	}
-
-	return exitCode
-}

cmd/ctparsewatch/.gitignore

@@ -1 +0,0 @@
-/ctparsewatch

cmd/ctparsewatch/main.go

@@ -1,52 +0,0 @@
-// Copyright (C) 2016 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package main
-
-import (
-	"flag"
-	"os"
-
-	"software.sslmate.com/src/certspotter"
-	"software.sslmate.com/src/certspotter/cmd"
-	"software.sslmate.com/src/certspotter/ct"
-)
-
-func DefaultStateDir() string {
-	if envVar := os.Getenv("CTPARSEWATCH_STATE_DIR"); envVar != "" {
-		return envVar
-	} else {
-		return cmd.DefaultStateDir("ctparsewatch")
-	}
-}
-
-var stateDir = flag.String("state_dir", DefaultStateDir(), "Directory for storing state")
-
-func processEntry(scanner *certspotter.Scanner, entry *ct.LogEntry) {
-	info := certspotter.EntryInfo{
-		LogUri:    scanner.LogUri,
-		Entry:     entry,
-		IsPrecert: certspotter.IsPrecert(entry),
-		FullChain: certspotter.GetFullChain(entry),
-	}
-
-	info.CertInfo, info.ParseError = certspotter.MakeCertInfoFromLogEntry(entry)
-	if info.CertInfo != nil {
-		info.Identifiers, info.IdentifiersParseError = info.CertInfo.ParseIdentifiers()
-	}
-
-	if info.HasParseErrors() {
-		cmd.LogEntry(&info)
-	}
-}
-
-func main() {
-	flag.Parse()
-	os.Exit(cmd.Main(*stateDir, processEntry))
-}

cmd/helpers.go

@@ -1,87 +0,0 @@
-// Copyright (C) 2017 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package cmd
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
-	"io/ioutil"
-	"os"
-
-	"software.sslmate.com/src/certspotter/ct"
-)
-
-func fileExists(path string) bool {
-	_, err := os.Lstat(path)
-	return err == nil
-}
-
-func writeFile(filename string, data []byte, perm os.FileMode) error {
-	tempname := filename + ".new"
-	if err := ioutil.WriteFile(tempname, data, perm); err != nil {
-		return err
-	}
-	if err := os.Rename(tempname, filename); err != nil {
-		os.Remove(tempname)
-		return err
-	}
-	return nil
-}
-
-func writeJSONFile(filename string, obj interface{}, perm os.FileMode) error {
-	tempname := filename + ".new"
-	f, err := os.OpenFile(tempname, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
-	if err != nil {
-		return err
-	}
-	if err := json.NewEncoder(f).Encode(obj); err != nil {
-		f.Close()
-		os.Remove(tempname)
-		return err
-	}
-	if err := f.Close(); err != nil {
-		os.Remove(tempname)
-		return err
-	}
-	if err := os.Rename(tempname, filename); err != nil {
-		os.Remove(tempname)
-		return err
-	}
-	return nil
-}
-
-func readJSONFile(filename string, obj interface{}) error {
-	bytes, err := ioutil.ReadFile(filename)
-	if err != nil {
-		return err
-	}
-	if err = json.Unmarshal(bytes, obj); err != nil {
-		return err
-	}
-	return nil
-}
-
-func readSTHFile(filename string) (*ct.SignedTreeHead, error) {
-	sth := new(ct.SignedTreeHead)
-	if err := readJSONFile(filename, sth); err != nil {
-		return nil, err
-	}
-	return sth, nil
-}
-
-func sha256sum(data []byte) []byte {
-	sum := sha256.Sum256(data)
-	return sum[:]
-}
-
-func sha256hex(data []byte) string {
-	return hex.EncodeToString(sha256sum(data))
-}

cmd/log_state.go

@@ -1,145 +0,0 @@
-// Copyright (C) 2017 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package cmd
-
-import (
-	"crypto/sha256"
-	"encoding/base64"
-	"encoding/binary"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-
-	"software.sslmate.com/src/certspotter"
-	"software.sslmate.com/src/certspotter/ct"
-)
-
-type LogState struct {
-	path string
-}
-
-// generate a filename that uniquely identifies the STH (within the context of a particular log)
-func sthFilename(sth *ct.SignedTreeHead) string {
-	hasher := sha256.New()
-	switch sth.Version {
-	case ct.V1:
-		binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
-		binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash)
-	default:
-		panic(fmt.Sprintf("Unsupported STH version %d", sth.Version))
-	}
-	// For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic)
-	return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json"
-}
-
-func makeLogStateDir(logStatePath string) error {
-	if err := os.Mkdir(logStatePath, 0777); err != nil && !os.IsExist(err) {
-		return fmt.Errorf("%s: %s", logStatePath, err)
-	}
-	for _, subdir := range []string{"unverified_sths"} {
-		path := filepath.Join(logStatePath, subdir)
-		if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) {
-			return fmt.Errorf("%s: %s", path, err)
-		}
-	}
-	return nil
-}
-
-func OpenLogState(logStatePath string) (*LogState, error) {
-	if err := makeLogStateDir(logStatePath); err != nil {
-		return nil, fmt.Errorf("Error creating log state directory: %s", err)
-	}
-	return &LogState{path: logStatePath}, nil
-}
-
-func (logState *LogState) VerifiedSTHFilename() string {
-	return filepath.Join(logState.path, "sth.json")
-}
-
-func (logState *LogState) GetVerifiedSTH() (*ct.SignedTreeHead, error) {
-	sth, err := readSTHFile(logState.VerifiedSTHFilename())
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		} else {
-			return nil, err
-		}
-	}
-	return sth, nil
-}
-
-func (logState *LogState) StoreVerifiedSTH(sth *ct.SignedTreeHead) error {
-	return writeJSONFile(logState.VerifiedSTHFilename(), sth, 0666)
-}
-
-func (logState *LogState) GetUnverifiedSTHs() ([]*ct.SignedTreeHead, error) {
-	dir, err := os.Open(filepath.Join(logState.path, "unverified_sths"))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return []*ct.SignedTreeHead{}, nil
-		} else {
-			return nil, err
-		}
-	}
-	filenames, err := dir.Readdirnames(0)
-	if err != nil {
-		return nil, err
-	}
-
-	sths := make([]*ct.SignedTreeHead, 0, len(filenames))
-	for _, filename := range filenames {
-		if !strings.HasPrefix(filename, ".") {
-			sth, _ := readSTHFile(filepath.Join(dir.Name(), filename))
-			if sth != nil {
-				sths = append(sths, sth)
-			}
-		}
-	}
-	return sths, nil
-}
-
-func (logState *LogState) UnverifiedSTHFilename(sth *ct.SignedTreeHead) string {
-	return filepath.Join(logState.path, "unverified_sths", sthFilename(sth))
-}
-
-func (logState *LogState) StoreUnverifiedSTH(sth *ct.SignedTreeHead) error {
-	filename := logState.UnverifiedSTHFilename(sth)
-	if fileExists(filename) {
-		return nil
-	}
-	return writeJSONFile(filename, sth, 0666)
-}
-
-func (logState *LogState) RemoveUnverifiedSTH(sth *ct.SignedTreeHead) error {
-	filename := logState.UnverifiedSTHFilename(sth)
-	err := os.Remove(filepath.Join(filename))
-	if err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	return nil
-}
-
-func (logState *LogState) GetTree() (*certspotter.CollapsedMerkleTree, error) {
-	tree := new(certspotter.CollapsedMerkleTree)
-	if err := readJSONFile(filepath.Join(logState.path, "tree.json"), tree); err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		} else {
-			return nil, err
-		}
-	}
-	return tree, nil
-}
-
-func (logState *LogState) StoreTree(tree *certspotter.CollapsedMerkleTree) error {
-	return writeJSONFile(filepath.Join(logState.path, "tree.json"), tree, 0666)
-}

cmd/state.go

@@ -1,220 +0,0 @@
-// Copyright (C) 2017 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package cmd
-
-import (
-	"bytes"
-	"encoding/base64"
-	"encoding/pem"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-
-	"software.sslmate.com/src/certspotter"
-	"software.sslmate.com/src/certspotter/ct"
-)
-
-type State struct {
-	path string
-}
-
-func legacySTHFilename(logInfo *certspotter.LogInfo) string {
-	return strings.Replace(strings.Replace(logInfo.FullURI(), "://", "_", 1), "/", "_", -1)
-}
-
-func readVersionFile(statePath string) (int, error) {
-	versionFilePath := filepath.Join(statePath, "version")
-	versionBytes, err := ioutil.ReadFile(versionFilePath)
-	if err == nil {
-		version, err := strconv.Atoi(string(bytes.TrimSpace(versionBytes)))
-		if err != nil {
-			return -1, fmt.Errorf("%s: contains invalid integer: %s", versionFilePath, err)
-		}
-		if version < 0 {
-			return -1, fmt.Errorf("%s: contains negative integer", versionFilePath)
-		}
-		return version, nil
-	} else if os.IsNotExist(err) {
-		if fileExists(filepath.Join(statePath, "sths")) {
-			// Original version of certspotter had no version file.
-			// Infer version 0 if "sths" directory is present.
-			return 0, nil
-		}
-		return -1, nil
-	} else {
-		return -1, fmt.Errorf("%s: %s", versionFilePath, err)
-	}
-}
-
-func writeVersionFile(statePath string) error {
-	version := 1
-	versionString := fmt.Sprintf("%d\n", version)
-	versionFilePath := filepath.Join(statePath, "version")
-	if err := ioutil.WriteFile(versionFilePath, []byte(versionString), 0666); err != nil {
-		return fmt.Errorf("%s: %s\n", versionFilePath, err)
-	}
-	return nil
-}
-
-func makeStateDir(statePath string) error {
-	if err := os.Mkdir(statePath, 0777); err != nil && !os.IsExist(err) {
-		return fmt.Errorf("%s: %s", statePath, err)
-	}
-	for _, subdir := range []string{"certs", "logs"} {
-		path := filepath.Join(statePath, subdir)
-		if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) {
-			return fmt.Errorf("%s: %s", path, err)
-		}
-	}
-	return nil
-}
-
-func OpenState(statePath string) (*State, error) {
-	version, err := readVersionFile(statePath)
-	if err != nil {
-		return nil, fmt.Errorf("Error reading version file: %s", err)
-	}
-
-	if version < 1 {
-		if err := makeStateDir(statePath); err != nil {
-			return nil, fmt.Errorf("Error creating state directory: %s", err)
-		}
-		if version == 0 {
-			log.Printf("Migrating state directory (%s) to new layout...", statePath)
-			if err := os.Rename(filepath.Join(statePath, "sths"), filepath.Join(statePath, "legacy_sths")); err != nil {
-				return nil, fmt.Errorf("Error migrating STHs directory: %s", err)
-			}
-			for _, subdir := range []string{"evidence", "legacy_sths"} {
-				os.Remove(filepath.Join(statePath, subdir))
-			}
-			if err := ioutil.WriteFile(filepath.Join(statePath, "once"), []byte{}, 0666); err != nil {
-				return nil, fmt.Errorf("Error creating once file: %s", err)
-			}
-		}
-		if err := writeVersionFile(statePath); err != nil {
-			return nil, fmt.Errorf("Error writing version file: %s", err)
-		}
-	} else if version > 1 {
-		return nil, fmt.Errorf("%s was created by a newer version of Cert Spotter; please remove this directory or upgrade Cert Spotter", statePath)
-	}
-
-	return &State{path: statePath}, nil
-}
-
-func (state *State) IsFirstRun() bool {
-	return !fileExists(filepath.Join(state.path, "once"))
-}
-
-func (state *State) WriteOnceFile() error {
-	if err := ioutil.WriteFile(filepath.Join(state.path, "once"), []byte{}, 0666); err != nil {
-		return fmt.Errorf("Error writing once file: %s", err)
-	}
-	return nil
-}
-
-func (state *State) SaveCert(isPrecert bool, certs [][]byte) (bool, string, error) {
-	if len(certs) == 0 {
-		return false, "", fmt.Errorf("Cannot write an empty certificate chain")
-	}
-
-	fingerprint := sha256hex(certs[0])
-	prefixPath := filepath.Join(state.path, "certs", fingerprint[0:2])
-	var filenameSuffix string
-	if isPrecert {
-		filenameSuffix = ".precert.pem"
-	} else {
-		filenameSuffix = ".cert.pem"
-	}
-	if err := os.Mkdir(prefixPath, 0777); err != nil && !os.IsExist(err) {
-		return false, "", fmt.Errorf("Failed to create prefix directory %s: %s", prefixPath, err)
-	}
-	path := filepath.Join(prefixPath, fingerprint+filenameSuffix)
-	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
-	if err != nil {
-		if os.IsExist(err) {
-			return true, path, nil
-		} else {
-			return false, path, fmt.Errorf("Failed to open %s for writing: %s", path, err)
-		}
-	}
-	for _, cert := range certs {
-		if err := pem.Encode(file, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil {
-			file.Close()
-			return false, path, fmt.Errorf("Error writing to %s: %s", path, err)
-		}
-	}
-	if err := file.Close(); err != nil {
-		return false, path, fmt.Errorf("Error writing to %s: %s", path, err)
-	}
-
-	return false, path, nil
-}
-
-func (state *State) OpenLogState(logInfo *certspotter.LogInfo) (*LogState, error) {
-	return OpenLogState(filepath.Join(state.path, "logs", base64.RawURLEncoding.EncodeToString(logInfo.ID())))
-}
-
-func (state *State) GetLegacySTH(logInfo *certspotter.LogInfo) (*ct.SignedTreeHead, error) {
-	sth, err := readSTHFile(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo)))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		} else {
-			return nil, err
-		}
-	}
-	return sth, nil
-}
-func (state *State) RemoveLegacySTH(logInfo *certspotter.LogInfo) error {
-	err := os.Remove(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo)))
-	os.Remove(filepath.Join(state.path, "legacy_sths"))
-	return err
-}
-func (state *State) LockFilename() string {
-	return filepath.Join(state.path, "lock")
-}
-func (state *State) Lock() (bool, error) {
-	file, err := os.OpenFile(state.LockFilename(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
-	if err != nil {
-		if os.IsExist(err) {
-			return false, nil
-		} else {
-			return false, err
-		}
-	}
-	if _, err := fmt.Fprintf(file, "%d\n", os.Getpid()); err != nil {
-		file.Close()
-		os.Remove(state.LockFilename())
-		return false, err
-	}
-	if err := file.Close(); err != nil {
-		os.Remove(state.LockFilename())
-		return false, err
-	}
-	return true, nil
-}
-func (state *State) Unlock() error {
-	return os.Remove(state.LockFilename())
-}
-func (state *State) LockingPid() int {
-	pidBytes, err := ioutil.ReadFile(state.LockFilename())
-	if err != nil {
-		return 0
-	}
-	pid, err := strconv.Atoi(string(bytes.TrimSpace(pidBytes)))
-	if err != nil {
-		return 0
-	}
-	return pid
-}

cmd/submitct/.gitignore

@@ -0,0 +1 @@
+/submitct

cmd/submitct/main.go

@@ -13,26 +13,34 @@ import (
 	"software.sslmate.com/src/certspotter"
 	"software.sslmate.com/src/certspotter/ct"
 	"software.sslmate.com/src/certspotter/ct/client"
+	"software.sslmate.com/src/certspotter/loglist"
 
 	"bytes"
+	"context"
 	"crypto/sha256"
+	"crypto/x509"
 	"encoding/pem"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"os"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 )
 
+const defaultLogList = "https://loglist.certspotter.org/submit.json"
+
 var verbose = flag.Bool("v", false, "Enable verbose output")
+var logsURL = flag.String("logs", defaultLogList, "File path or URL of JSON list of logs to submit to")
 
 type Certificate struct {
 	Subject []byte
 	Issuer  []byte
 	Raw     []byte
+	Expiration time.Time
 }
 
 func (cert *Certificate) Fingerprint() [32]byte {
@@ -62,10 +70,16 @@ func parseCertificate(data []byte) (*Certificate, error) {
 		return nil, err
 	}
 
+	validity, err := tbs.ParseValidity()
+	if err != nil {
+		return nil, err
+	}
+
 	return &Certificate{
 		Subject: tbs.Subject.FullBytes,
 		Issuer:  tbs.Issuer.FullBytes,
 		Raw:     data,
+		Expiration: validity.NotAfter,
 	}, nil
 }
 
@@ -101,32 +115,19 @@ func (certs *CertificateBunch) FindBySubject(subject []byte) *Certificate {
 }
 
 type Log struct {
-	info   certspotter.LogInfo
-	verify *ct.SignatureVerifier
-	client *client.LogClient
+	*loglist.Log
+	*ct.SignatureVerifier
+	*client.LogClient
 }
 
 func (ctlog *Log) SubmitChain(chain Chain) (*ct.SignedCertificateTimestamp, error) {
 	rawCerts := chain.GetRawCerts()
-	sct, err := ctlog.client.AddChain(rawCerts)
+	sct, err := ctlog.AddChain(context.Background(), rawCerts)
 	if err != nil {
 		return nil, err
 	}
 
-	entry := ct.LogEntry{
-		Leaf: ct.MerkleTreeLeaf{
-			Version:  0,
-			LeafType: ct.TimestampedEntryLeafType,
-			TimestampedEntry: ct.TimestampedEntry{
-				Timestamp:  sct.Timestamp,
-				EntryType:  ct.X509LogEntryType,
-				X509Entry:  rawCerts[0],
-				Extensions: sct.Extensions,
-			},
-		},
-	}
-
-	if err := ctlog.verify.VerifySCTSignature(*sct, entry); err != nil {
+	if err := certspotter.VerifyX509SCT(sct, rawCerts[0], ctlog.SignatureVerifier); err != nil {
 		return nil, fmt.Errorf("Bad SCT signature: %s", err)
 	}
 	return sct, nil
@@ -145,25 +146,31 @@ func main() {
 	flag.Parse()
 	log.SetPrefix("submitct: ")
 
-	certsPem, err := ioutil.ReadAll(os.Stdin)
+	certsPem, err := io.ReadAll(os.Stdin)
 	if err != nil {
 		log.Fatalf("Error reading stdin: %s", err)
 	}
 
-	logs := make([]Log, 0, len(certspotter.OpenLogs))
-	for _, loginfo := range certspotter.OpenLogs {
-		pubkey, err := loginfo.ParsedPublicKey()
+	list, err := loglist.Load(context.Background(), *logsURL)
+	if err != nil {
+		log.Fatalf("Error loading log list: %s", err)
+	}
+
+	var logs []Log
+	for _, ctlog := range list.AllLogs() {
+		submissionURL := ctlog.GetSubmissionURL()
+		pubkey, err := x509.ParsePKIXPublicKey(ctlog.Key)
 		if err != nil {
-			log.Fatalf("%s: Failed to parse log public key: %s", loginfo.Url, err)
+			log.Fatalf("%s: Failed to parse log public key: %s", submissionURL, err)
 		}
-		verify, err := ct.NewSignatureVerifier(pubkey)
+		verifier, err := ct.NewSignatureVerifier(pubkey)
 		if err != nil {
-			log.Fatalf("%s: Failed to create signature verifier for log: %s", loginfo.Url, err)
+			log.Fatalf("%s: Failed to create signature verifier for log: %s", submissionURL, err)
 		}
 		logs = append(logs, Log{
-			info:   loginfo,
-			verify: verify,
-			client: client.New(loginfo.FullURI()),
+			Log: ctlog,
+			SignatureVerifier: verifier,
+			LogClient: client.New(strings.TrimRight(submissionURL, "/")),
 		})
 	}
 
@@ -199,15 +206,18 @@ func main() {
 			continue
 		}
 		for _, ctlog := range logs {
+			if !ctlog.AcceptsExpiration(chain[0].Expiration) {
+				continue
+			}
 			wg.Add(1)
 			go func(fingerprint [32]byte, ctlog Log) {
 				sct, err := ctlog.SubmitChain(chain)
 				if err != nil {
-					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.info.Url, err)
+					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.GetSubmissionURL(), err)
 					atomic.AddUint32(&submitErrors, 1)
 				} else if *verbose {
 					timestamp := time.Unix(int64(sct.Timestamp)/1000, int64(sct.Timestamp%1000)*1000000)
-					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.info.Url, timestamp)
+					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.GetSubmissionURL(), timestamp)
 				}
 				wg.Done()
 			}(fingerprint, ctlog)

ct/README

@@ -1,2 +1,4 @@
-The code in this directory is from https://github.com/google/certificate-transparency/tree/master/go
+The code in this directory is based on Google's Certificiate Transparency Go library
+(originally at <https://github.com/google/certificate-transparency/tree/master/go>;
+now at <https://github.com/google/certificate-transparency-go>).
 See AUTHORS for the copyright holders, and LICENSE for the license.

ct/client/logclient.go

@@ -5,21 +5,57 @@ package client
 
 import (
 	"bytes"
+	"context"
 	"crypto/sha256"
+	"crypto/tls"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
+	insecurerand "math/rand"
 	"net/http"
 	"net/url"
+	"strconv"
 	"time"
 
-	"github.com/mreiferson/go-httpclient"
 	"software.sslmate.com/src/certspotter/ct"
 )
 
+const (
+	baseRetryDelay = 1 * time.Second
+	maxRetryDelay  = 120 * time.Second
+	maxRetries     = 10
+)
+
+func isRetryableStatusCode(code int) bool {
+	return code/100 == 5 || code == http.StatusTooManyRequests
+}
+
+func randomDuration(min, max time.Duration) time.Duration {
+	return min + time.Duration(insecurerand.Int63n(int64(max)-int64(min)+1))
+}
+
+func getRetryAfter(resp *http.Response) (time.Duration, bool) {
+	if resp == nil {
+		return 0, false
+	}
+	seconds, err := strconv.ParseUint(resp.Header.Get("Retry-After"), 10, 16)
+	if err != nil {
+		return 0, false
+	}
+	return time.Duration(seconds) * time.Second, true
+}
+
+func sleep(ctx context.Context, duration time.Duration) {
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+	case <-timer.C:
+	}
+}
+
 // URI paths for CT Log endpoints
 const (
 	GetSTHPath            = "/ct/v1/get-sth"
@@ -33,6 +69,7 @@ const (
 type LogClient struct {
 	uri        string       // the base URI of the log. e.g. http://ct.googleapis/pilot
 	httpClient *http.Client // used to interact with the log via HTTP
+	verifier   *ct.SignatureVerifier // if non-nil, used to verify STH signatures
 }
 
 //////////////////////////////////////////////////////////////////////////////////
@@ -40,7 +77,7 @@ type LogClient struct {
 // These represent the structures returned by the CT Log server.
 //////////////////////////////////////////////////////////////////////////////////
 
-// getSTHResponse respresents the JSON response to the get-sth CT method
+// getSTHResponse represents the JSON response to the get-sth CT method
 type getSTHResponse struct {
 	TreeSize          uint64 `json:"tree_size"`           // Number of certs in the current tree
 	Timestamp         uint64 `json:"timestamp"`           // Time that the tree was created
@@ -48,13 +85,13 @@ type getSTHResponse struct {
 	TreeHeadSignature []byte `json:"tree_head_signature"` // Log signature for this STH
 }
 
-// base64LeafEntry respresents a Base64 encoded leaf entry
+// base64LeafEntry represents a Base64 encoded leaf entry
 type base64LeafEntry struct {
 	LeafInput []byte `json:"leaf_input"`
 	ExtraData []byte `json:"extra_data"`
 }
 
-// getEntriesReponse respresents the JSON response to the CT get-entries method
+// getEntriesReponse represents the JSON response to the CT get-entries method
 type getEntriesResponse struct {
 	Entries []base64LeafEntry `json:"entries"` // the list of returned entries
 }
@@ -86,72 +123,136 @@ type addChainResponse struct {
 // |uri| is the base URI of the CT log instance to interact with, e.g.
 // http://ct.googleapis.com/pilot
 func New(uri string) *LogClient {
+	return NewWithVerifier(uri, nil)
+}
+
+func NewWithVerifier(uri string, verifier *ct.SignatureVerifier) *LogClient {
 	var c LogClient
 	c.uri = uri
-	transport := &httpclient.Transport{
-		ConnectTimeout:        10 * time.Second,
-		RequestTimeout:        60 * time.Second,
+	c.verifier = verifier
+	transport := &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		TLSHandshakeTimeout:   15 * time.Second,
 		ResponseHeaderTimeout: 30 * time.Second,
 		MaxIdleConnsPerHost:   10,
 		DisableKeepAlives:     false,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       15 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig: &tls.Config{
+			// We have to disable TLS certificate validation because because several logs
+			// (WoSign, StartCom, GDCA) use certificates that are not widely trusted.
+			// Since we verify that every response we receive from the log is signed
+			// by the log's CT public key (either directly, or indirectly via the Merkle Tree),
+			// TLS certificate validation is not actually necessary.  (We don't want to ship
+			// our own trust store because that adds undesired complexity and would require
+			// updating should a log ever change to a different CA.)
+			InsecureSkipVerify: true,
+		},
 	}
-	c.httpClient = &http.Client{Transport: transport}
+	c.httpClient = &http.Client{Timeout: 60 * time.Second, Transport: transport}
 	return &c
 }
 
-// Makes a HTTP call to |uri|, and attempts to parse the response as a JSON
-// representation of the structure in |res|.
-// Returns a non-nil |error| if there was a problem.
-func (c *LogClient) fetchAndParse(uri string, respBody interface{}) error {
-	req, err := http.NewRequest("GET", uri, nil)
-	if err != nil {
-		return fmt.Errorf("GET %s: Sending request failed: %s", uri, err)
-	}
-	return c.doAndParse(req, respBody)
+func (c *LogClient) fetchAndParse(ctx context.Context, uri string, respBody interface{}) error {
+	return c.doAndParse(ctx, "GET", uri, nil, respBody)
 }
 
-func (c *LogClient) postAndParse(uri string, body interface{}, respBody interface{}) error {
-	bodyReader, bodyWriter := io.Pipe()
-	go func() {
-		json.NewEncoder(bodyWriter).Encode(body)
-		bodyWriter.Close()
-	}()
-	req, err := http.NewRequest("POST", uri, bodyReader)
-	if err != nil {
-		return fmt.Errorf("POST %s: Sending request failed: %s", uri, err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-	return c.doAndParse(req, respBody)
+func (c *LogClient) postAndParse(ctx context.Context, uri string, body interface{}, respBody interface{}) error {
+	return c.doAndParse(ctx, "POST", uri, body, respBody)
 }
 
-func (c *LogClient) doAndParse(req *http.Request, respBody interface{}) error {
-	//	req.Header.Set("Keep-Alive", "timeout=15, max=100")
-	resp, err := c.httpClient.Do(req)
-	var respBodyBytes []byte
-	if resp != nil {
-		respBodyBytes, err = ioutil.ReadAll(resp.Body)
-		resp.Body.Close()
+func (c *LogClient) makeRequest(ctx context.Context, method string, uri string, body interface{}) (*http.Request, error) {
+	if body == nil {
+		return http.NewRequestWithContext(ctx, method, uri, nil)
+	} else {
+		bodyBytes, err := json.Marshal(body)
 		if err != nil {
-			return fmt.Errorf("%s %s: Reading response failed: %s", req.Method, req.URL, err)
+			return nil, err
 		}
+		req, err := http.NewRequestWithContext(ctx, method, uri, bytes.NewReader(bodyBytes))
+		if err != nil {
+			return nil, err
+		}
+		req.Header.Set("Content-Type", "application/json")
+		return req, nil
 	}
+}
+
+func (c *LogClient) doAndParse(ctx context.Context, method string, uri string, reqBody interface{}, respBody interface{}) error {
+	numRetries := 0
+retry:
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+	req, err := c.makeRequest(ctx, method, uri, reqBody)
 	if err != nil {
+		return fmt.Errorf("%s %s: error creating request: %w", method, uri, err)
+	}
+	req.Header.Set("User-Agent", "") // Don't send a User-Agent to make life harder for malicious logs
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		if c.shouldRetry(ctx, numRetries, nil) {
+			numRetries++
+			goto retry
+		}
 		return err
 	}
-	if resp.StatusCode/100 != 2 {
-		return fmt.Errorf("%s %s: %s (%s)", req.Method, req.URL, resp.Status, string(respBodyBytes))
+	respBodyBytes, err := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		if c.shouldRetry(ctx, numRetries, nil) {
+			numRetries++
+			goto retry
+		}
+		return fmt.Errorf("%s %s: error reading response: %w", method, uri, err)
 	}
-	if err = json.Unmarshal(respBodyBytes, &respBody); err != nil {
-		return fmt.Errorf("%s %s: Parsing response JSON failed: %s", req.Method, req.URL, err)
+	if resp.StatusCode/100 != 2 {
+		if c.shouldRetry(ctx, numRetries, resp) {
+			numRetries++
+			goto retry
+		}
+		return fmt.Errorf("%s %s: %s (%s)", method, uri, resp.Status, string(respBodyBytes))
+	}
+	if err := json.Unmarshal(respBodyBytes, respBody); err != nil {
+		return fmt.Errorf("%s %s: error parsing response JSON: %w", method, uri, err)
 	}
 	return nil
 }
 
+func (c *LogClient) shouldRetry(ctx context.Context, numRetries int, resp *http.Response) bool {
+	if numRetries == maxRetries {
+		return false
+	}
+
+	if resp != nil && !isRetryableStatusCode(resp.StatusCode) {
+		return false
+	}
+
+	var delay time.Duration
+	if retryAfter, hasRetryAfter := getRetryAfter(resp); hasRetryAfter {
+		delay = retryAfter
+	} else {
+		delay = baseRetryDelay * (1 << numRetries)
+		if delay > maxRetryDelay {
+			delay = maxRetryDelay
+		}
+		delay += randomDuration(0, delay/2)
+	}
+
+	if deadline, hasDeadline := ctx.Deadline(); hasDeadline && time.Now().Add(delay).After(deadline) {
+		return false
+	}
+
+	sleep(ctx, delay)
+	return true
+}
+
 // GetSTH retrieves the current STH from the log.
 // Returns a populated SignedTreeHead, or a non-nil error.
-func (c *LogClient) GetSTH() (sth *ct.SignedTreeHead, err error) {
+func (c *LogClient) GetSTH(ctx context.Context) (sth *ct.SignedTreeHead, err error) {
 	var resp getSTHResponse
-	if err = c.fetchAndParse(c.uri+GetSTHPath, &resp); err != nil {
+	if err = c.fetchAndParse(ctx, c.uri+GetSTHPath, &resp); err != nil {
 		return
 	}
 	sth = &ct.SignedTreeHead{
@@ -168,15 +269,48 @@ func (c *LogClient) GetSTH() (sth *ct.SignedTreeHead, err error) {
 	if err != nil {
 		return nil, err
 	}
-	// TODO(alcutter): Verify signature
 	sth.TreeHeadSignature = *ds
+	if c.verifier != nil {
+		if err := c.verifier.VerifySTHSignature(*sth); err != nil {
+			return nil, fmt.Errorf("STH returned by server has invalid signature: %w", err)
+		}
+	}
 	return
 }
 
+type GetEntriesItem struct {
+	LeafInput []byte `json:"leaf_input"`
+	ExtraData []byte `json:"extra_data"`
+}
+
+// Retrieve the entries in the sequence [start, end] from the CT log server.
+// If error is nil, at least one entry is returned, and no excess entries are returned.
+// Fewer entries than requested may be returned.
+func (c *LogClient) GetRawEntries(ctx context.Context, start, end uint64) ([]GetEntriesItem, error) {
+	if end < start {
+		panic("LogClient.GetRawEntries: end < start")
+	}
+	var response struct {
+		Entries []GetEntriesItem `json:"entries"`
+	}
+	uri := fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end)
+	err := c.fetchAndParse(ctx, uri, &response)
+	if err != nil {
+		return nil, err
+	}
+	if len(response.Entries) == 0 {
+		return nil, fmt.Errorf("GET %s: log server returned an empty get-entries response", uri)
+	}
+	if uint64(len(response.Entries)) > end-start+1 {
+		return nil, fmt.Errorf("GET %s: log server returned a get-entries response with extraneous entries", uri)
+	}
+	return response.Entries, nil
+}
+
 // GetEntries attempts to retrieve the entries in the sequence [|start|, |end|] from the CT
 // log server. (see section 4.6.)
 // Returns a slice of LeafInputs or a non-nil error.
-func (c *LogClient) GetEntries(start, end int64) ([]ct.LogEntry, error) {
+func (c *LogClient) GetEntries(ctx context.Context, start, end int64) ([]ct.LogEntry, error) {
 	if end < 0 {
 		return nil, errors.New("GetEntries: end should be >= 0")
 	}
@@ -184,7 +318,7 @@ func (c *LogClient) GetEntries(start, end int64) ([]ct.LogEntry, error) {
 		return nil, errors.New("GetEntries: start should be <= end")
 	}
 	var resp getEntriesResponse
-	err := c.fetchAndParse(fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end), &resp)
+	err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end), &resp)
 	if err != nil {
 		return nil, err
 	}
@@ -219,7 +353,7 @@ func (c *LogClient) GetEntries(start, end int64) ([]ct.LogEntry, error) {
 
 // GetConsistencyProof retrieves a Merkle Consistency Proof between two STHs (|first| and |second|)
 // from the log.  Returns a slice of MerkleTreeNodes (a ct.ConsistencyProof) or a non-nil error.
-func (c *LogClient) GetConsistencyProof(first, second int64) (ct.ConsistencyProof, error) {
+func (c *LogClient) GetConsistencyProof(ctx context.Context, first, second int64) (ct.ConsistencyProof, error) {
 	if second < 0 {
 		return nil, errors.New("GetConsistencyProof: second should be >= 0")
 	}
@@ -227,7 +361,7 @@ func (c *LogClient) GetConsistencyProof(first, second int64) (ct.ConsistencyProo
 		return nil, errors.New("GetConsistencyProof: first should be <= second")
 	}
 	var resp getConsistencyProofResponse
-	err := c.fetchAndParse(fmt.Sprintf("%s%s?first=%d&second=%d", c.uri, GetSTHConsistencyPath, first, second), &resp)
+	err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?first=%d&second=%d", c.uri, GetSTHConsistencyPath, first, second), &resp)
 	if err != nil {
 		return nil, err
 	}
@@ -241,9 +375,9 @@ func (c *LogClient) GetConsistencyProof(first, second int64) (ct.ConsistencyProo
 // GetAuditProof retrieves a Merkle Audit Proof (aka Inclusion Proof) for the given
 // |hash| based on the STH at |treeSize| from the log.  Returns a slice of MerkleTreeNodes
 // and the index of the leaf.
-func (c *LogClient) GetAuditProof(hash ct.MerkleTreeNode, treeSize uint64) (ct.AuditPath, uint64, error) {
+func (c *LogClient) GetAuditProof(ctx context.Context, hash ct.MerkleTreeNode, treeSize uint64) (ct.AuditPath, uint64, error) {
 	var resp getAuditProofResponse
-	err := c.fetchAndParse(fmt.Sprintf("%s%s?hash=%s&tree_size=%d", c.uri, GetProofByHashPath, url.QueryEscape(base64.StdEncoding.EncodeToString(hash)), treeSize), &resp)
+	err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?hash=%s&tree_size=%d", c.uri, GetProofByHashPath, url.QueryEscape(base64.StdEncoding.EncodeToString(hash)), treeSize), &resp)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -254,11 +388,11 @@ func (c *LogClient) GetAuditProof(hash ct.MerkleTreeNode, treeSize uint64) (ct.A
 	return path, resp.LeafIndex, nil
 }
 
-func (c *LogClient) AddChain(chain [][]byte) (*ct.SignedCertificateTimestamp, error) {
+func (c *LogClient) AddChain(ctx context.Context, chain [][]byte) (*ct.SignedCertificateTimestamp, error) {
 	req := addChainRequest{Chain: chain}
 
 	var resp addChainResponse
-	if err := c.postAndParse(c.uri+AddChainPath, &req, &resp); err != nil {
+	if err := c.postAndParse(ctx, c.uri+AddChainPath, &req, &resp); err != nil {
 		return nil, err
 	}
 

ct/signatures.go

@@ -10,7 +10,6 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"log"
 	"math/big"
 )
 
@@ -79,7 +78,7 @@ func (s SignatureVerifier) verifySignature(data []byte, sig DigitallySigned) err
 			return fmt.Errorf("failed to unmarshal ECDSA signature: %v", err)
 		}
 		if len(rest) != 0 {
-			log.Printf("Garbage following signature %v", rest)
+			return fmt.Errorf("Garbage following signature %v", rest)
 		}
 
 		if !ecdsa.Verify(ecdsaKey, hash, ecdsaSig.R, ecdsaSig.S) {

ct/types.go

@@ -6,6 +6,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"time"
 )
 
 const (
@@ -155,7 +156,7 @@ func (h HashAlgorithm) String() string {
 	}
 }
 
-// SignatureAlgorithm from the the DigitallySigned struct
+// SignatureAlgorithm from the DigitallySigned struct
 type SignatureAlgorithm byte
 
 // SignatureAlgorithm constants
@@ -259,6 +260,11 @@ func (s SHA256Hash) Base64String() string {
 	return base64.StdEncoding.EncodeToString(s[:])
 }
 
+// Returns the raw base64url representation of this SHA256Hash.
+func (s SHA256Hash) Base64URLString() string {
+	return base64.RawURLEncoding.EncodeToString(s[:])
+}
+
 // MarshalJSON implements the json.Marshaller interface for SHA256Hash.
 func (s SHA256Hash) MarshalJSON() ([]byte, error) {
 	return []byte(`"` + s.Base64String() + `"`), nil
@@ -284,16 +290,20 @@ type SignedTreeHead struct {
 	LogID             SHA256Hash      `json:"log_id"`              // The SHA256 hash of the log's public key
 }
 
+func (sth *SignedTreeHead) TimestampTime() time.Time {
+	return time.Unix(int64(sth.Timestamp/1000), int64(sth.Timestamp%1000)*1_000_000).UTC()
+}
+
 // SignedCertificateTimestamp represents the structure returned by the
 // add-chain and add-pre-chain methods after base64 decoding. (see RFC sections
 // 3.2 ,4.1 and 4.2)
 type SignedCertificateTimestamp struct {
-	SCTVersion Version    // The version of the protocol to which the SCT conforms
-	LogID      SHA256Hash // the SHA-256 hash of the log's public key, calculated over
+	SCTVersion Version          `json:"sct_version"`    // The version of the protocol to which the SCT conforms
+	LogID      SHA256Hash       `json:"id"` // the SHA-256 hash of the log's public key, calculated over
 	// the DER encoding of the key represented as SubjectPublicKeyInfo.
-	Timestamp  uint64          // Timestamp (in ms since unix epoc) at which the SCT was issued
-	Extensions CTExtensions    // For future extensions to the protocol
-	Signature  DigitallySigned // The Log's signature for this SCT
+	Timestamp  uint64           `json:"timestamp"`          // Timestamp (in ms since unix epoch) at which the SCT was issued
+	Extensions CTExtensions     `json:"extensions"`    // For future extensions to the protocol
+	Signature  DigitallySigned  `json:"signature"` // The Log's signature for this SCT
 }
 
 func (s SignedCertificateTimestamp) String() string {
@@ -314,7 +324,7 @@ type TimestampedEntry struct {
 	Extensions   CTExtensions
 }
 
-// MerkleTreeLeaf represents the deserialized sructure of the hash input for the
+// MerkleTreeLeaf represents the deserialized structure of the hash input for the
 // leaves of a log's Merkle tree. See RFC section 3.4
 type MerkleTreeLeaf struct {
 	Version          Version          // the version of the protocol to which the MerkleTreeLeaf corresponds

go.mod

@@ -0,0 +1,10 @@
+module software.sslmate.com/src/certspotter
+
+go 1.21
+
+require (
+	golang.org/x/net v0.17.0
+	golang.org/x/sync v0.4.0
+)
+
+require golang.org/x/text v0.13.0 // indirect

go.sum

@@ -0,0 +1,6 @@
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

helpers.go

@@ -10,104 +10,16 @@
 package certspotter
 
 import (
-	"bytes"
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"math/big"
-	"os"
-	"os/exec"
-	"strconv"
-	"strings"
-	"time"
 
 	"software.sslmate.com/src/certspotter/ct"
 )
 
-func ReadSTHFile(path string) (*ct.SignedTreeHead, error) {
-	content, err := ioutil.ReadFile(path)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
-		return nil, err
-	}
-
-	var sth ct.SignedTreeHead
-	if err := json.Unmarshal(content, &sth); err != nil {
-		return nil, err
-	}
-
-	return &sth, nil
-}
-
-func WriteSTHFile(path string, sth *ct.SignedTreeHead) error {
-	sthJson, err := json.MarshalIndent(sth, "", "\t")
-	if err != nil {
-		return err
-	}
-	sthJson = append(sthJson, byte('\n'))
-	return ioutil.WriteFile(path, sthJson, 0666)
-}
-
-func WriteProofFile(path string, proof ct.ConsistencyProof) error {
-	proofJson, err := json.MarshalIndent(proof, "", "\t")
-	if err != nil {
-		return err
-	}
-	proofJson = append(proofJson, byte('\n'))
-	return ioutil.WriteFile(path, proofJson, 0666)
-}
-
 func IsPrecert(entry *ct.LogEntry) bool {
 	return entry.Leaf.TimestampedEntry.EntryType == ct.PrecertLogEntryType
 }
 
-func GetFullChain(entry *ct.LogEntry) [][]byte {
-	certs := make([][]byte, 0, len(entry.Chain)+1)
-
-	if entry.Leaf.TimestampedEntry.EntryType == ct.X509LogEntryType {
-		certs = append(certs, entry.Leaf.TimestampedEntry.X509Entry)
-	}
-	for _, cert := range entry.Chain {
-		certs = append(certs, cert)
-	}
-
-	return certs
-}
-
-func formatSerialNumber(serial *big.Int) string {
-	if serial != nil {
-		return fmt.Sprintf("%x", serial)
-	} else {
-		return ""
-	}
-}
-
-func sha256sum(data []byte) []byte {
-	sum := sha256.Sum256(data)
-	return sum[:]
-}
-
-func sha256hex(data []byte) string {
-	return hex.EncodeToString(sha256sum(data))
-}
-
-type EntryInfo struct {
-	LogUri                string
-	Entry                 *ct.LogEntry
-	IsPrecert             bool
-	FullChain             [][]byte // first entry is logged X509 cert or pre-cert
-	CertInfo              *CertInfo
-	ParseError            error // set iff CertInfo is nil
-	Identifiers           *Identifiers
-	IdentifiersParseError error
-	Filename              string
-}
-
 type CertInfo struct {
 	TBS *TBSCertificate
 
@@ -123,6 +35,7 @@ type CertInfo struct {
 	ValidityParseError     error
 	IsCA                   *bool
 	IsCAParseError         error
+	IsPreCert              bool
 }
 
 func MakeCertInfoFromTBS(tbs *TBSCertificate) *CertInfo {
@@ -134,6 +47,7 @@ func MakeCertInfoFromTBS(tbs *TBSCertificate) *CertInfo {
 	info.SerialNumber, info.SerialNumberParseError = tbs.ParseSerialNumber()
 	info.Validity, info.ValidityParseError = tbs.ParseValidity()
 	info.IsCA, info.IsCAParseError = tbs.ParseBasicConstraints()
+	info.IsPreCert = len(tbs.GetExtension(oidExtensionCTPoison)) > 0
 
 	return info
 }
@@ -167,199 +81,6 @@ func MakeCertInfoFromLogEntry(entry *ct.LogEntry) (*CertInfo, error) {
 	}
 }
 
-func (info *CertInfo) NotBefore() *time.Time {
-	if info.ValidityParseError == nil {
-		return &info.Validity.NotBefore
-	} else {
-		return nil
-	}
-}
-
-func (info *CertInfo) NotAfter() *time.Time {
-	if info.ValidityParseError == nil {
-		return &info.Validity.NotAfter
-	} else {
-		return nil
-	}
-}
-
-func (info *CertInfo) PubkeyHash() string {
-	return sha256hex(info.TBS.GetRawPublicKey())
-}
-
-func (info *CertInfo) PubkeyHashBytes() []byte {
-	return sha256sum(info.TBS.GetRawPublicKey())
-}
-
-func (info *CertInfo) Environ() []string {
-	env := make([]string, 0, 10)
-
-	env = append(env, "PUBKEY_HASH="+info.PubkeyHash())
-
-	if info.SerialNumberParseError != nil {
-		env = append(env, "SERIAL_PARSE_ERROR="+info.SerialNumberParseError.Error())
-	} else {
-		env = append(env, "SERIAL="+formatSerialNumber(info.SerialNumber))
-	}
-
-	if info.ValidityParseError != nil {
-		env = append(env, "VALIDITY_PARSE_ERROR="+info.ValidityParseError.Error())
-	} else {
-		env = append(env, "NOT_BEFORE="+info.Validity.NotBefore.String())
-		env = append(env, "NOT_BEFORE_UNIXTIME="+strconv.FormatInt(info.Validity.NotBefore.Unix(), 10))
-		env = append(env, "NOT_AFTER="+info.Validity.NotAfter.String())
-		env = append(env, "NOT_AFTER_UNIXTIME="+strconv.FormatInt(info.Validity.NotAfter.Unix(), 10))
-	}
-
-	if info.SubjectParseError != nil {
-		env = append(env, "SUBJECT_PARSE_ERROR="+info.SubjectParseError.Error())
-	} else {
-		env = append(env, "SUBJECT_DN="+info.Subject.String())
-	}
-
-	if info.IssuerParseError != nil {
-		env = append(env, "ISSUER_PARSE_ERROR="+info.IssuerParseError.Error())
-	} else {
-		env = append(env, "ISSUER_DN="+info.Issuer.String())
-	}
-
-	// TODO: include SANs in environment
-
-	return env
-}
-
-func (info *EntryInfo) HasParseErrors() bool {
-	return info.ParseError != nil ||
-		info.IdentifiersParseError != nil ||
-		info.CertInfo.SubjectParseError != nil ||
-		info.CertInfo.IssuerParseError != nil ||
-		info.CertInfo.SANsParseError != nil ||
-		info.CertInfo.SerialNumberParseError != nil ||
-		info.CertInfo.ValidityParseError != nil ||
-		info.CertInfo.IsCAParseError != nil
-}
-
-func (info *EntryInfo) Fingerprint() string {
-	if len(info.FullChain) > 0 {
-		return sha256hex(info.FullChain[0])
-	} else {
-		return ""
-	}
-}
-
-func (info *EntryInfo) FingerprintBytes() []byte {
-	if len(info.FullChain) > 0 {
-		return sha256sum(info.FullChain[0])
-	} else {
-		return []byte{}
-	}
-}
-
-func (info *EntryInfo) typeString() string {
-	if info.IsPrecert {
-		return "precert"
-	} else {
-		return "cert"
-	}
-}
-
-func (info *EntryInfo) typeFriendlyString() string {
-	if info.IsPrecert {
-		return "Pre-certificate"
-	} else {
-		return "Certificate"
-	}
-}
-
-func yesnoString(value bool) string {
-	if value {
-		return "yes"
-	} else {
-		return "no"
-	}
-}
-
-func (info *EntryInfo) Environ() []string {
-	env := []string{
-		"FINGERPRINT=" + info.Fingerprint(),
-		"CERT_TYPE=" + info.typeString(),
-		"CERT_PARSEABLE=" + yesnoString(info.ParseError == nil),
-		"LOG_URI=" + info.LogUri,
-		"ENTRY_INDEX=" + strconv.FormatInt(info.Entry.Index, 10),
-	}
-
-	if info.Filename != "" {
-		env = append(env, "CERT_FILENAME="+info.Filename)
-	}
-	if info.ParseError != nil {
-		env = append(env, "PARSE_ERROR="+info.ParseError.Error())
-	} else if info.CertInfo != nil {
-		certEnv := info.CertInfo.Environ()
-		env = append(env, certEnv...)
-	}
-	if info.IdentifiersParseError != nil {
-		env = append(env, "IDENTIFIERS_PARSE_ERROR="+info.IdentifiersParseError.Error())
-	} else if info.Identifiers != nil {
-		env = append(env, "DNS_NAMES="+info.Identifiers.dnsNamesString(","))
-		env = append(env, "IP_ADDRESSES="+info.Identifiers.ipAddrsString(","))
-	}
-
-	return env
-}
-
-func writeField(out io.Writer, name string, value interface{}, err error) {
-	if err == nil {
-		fmt.Fprintf(out, "\t%13s = %s\n", name, value)
-	} else {
-		fmt.Fprintf(out, "\t%13s = *** UNKNOWN (%s) ***\n", name, err)
-	}
-}
-
-func (info *EntryInfo) Write(out io.Writer) {
-	fingerprint := info.Fingerprint()
-	fmt.Fprintf(out, "%s:\n", fingerprint)
-	if info.IdentifiersParseError != nil {
-		writeField(out, "Identifiers", nil, info.IdentifiersParseError)
-	} else if info.Identifiers != nil {
-		for _, dnsName := range info.Identifiers.DNSNames {
-			writeField(out, "DNS Name", dnsName, nil)
-		}
-		for _, ipaddr := range info.Identifiers.IPAddrs {
-			writeField(out, "IP Address", ipaddr, nil)
-		}
-	}
-	if info.ParseError != nil {
-		writeField(out, "Parse Error", "*** "+info.ParseError.Error()+" ***", nil)
-	} else if info.CertInfo != nil {
-		writeField(out, "Pubkey", info.CertInfo.PubkeyHash(), nil)
-		writeField(out, "Issuer", info.CertInfo.Issuer, info.CertInfo.IssuerParseError)
-		writeField(out, "Not Before", info.CertInfo.NotBefore(), info.CertInfo.ValidityParseError)
-		writeField(out, "Not After", info.CertInfo.NotAfter(), info.CertInfo.ValidityParseError)
-	}
-	writeField(out, "Log Entry", fmt.Sprintf("%d @ %s (%s)", info.Entry.Index, info.LogUri, info.typeFriendlyString()), nil)
-	writeField(out, "crt.sh", "https://crt.sh/?sha256="+fingerprint, nil)
-	if info.Filename != "" {
-		writeField(out, "Filename", info.Filename, nil)
-	}
-}
-
-func (info *EntryInfo) InvokeHookScript(command string) error {
-	cmd := exec.Command(command)
-	cmd.Env = os.Environ()
-	infoEnv := info.Environ()
-	cmd.Env = append(cmd.Env, infoEnv...)
-	stderrBuffer := bytes.Buffer{}
-	cmd.Stderr = &stderrBuffer
-	if err := cmd.Run(); err != nil {
-		if _, isExitError := err.(*exec.ExitError); isExitError {
-			return fmt.Errorf("Script failed: %s: %s", command, strings.TrimSpace(stderrBuffer.String()))
-		} else {
-			return fmt.Errorf("Failed to execute script: %s: %s", command, err)
-		}
-	}
-	return nil
-}
-
 func MatchesWildcard(dnsName string, pattern string) bool {
 	for len(pattern) > 0 {
 		if pattern[0] == '*' {

loglist/helpers.go

@@ -0,0 +1,40 @@
+// Copyright (C) 2020 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"time"
+)
+
+// Return all tiled and non-tiled logs from all operators
+func (list *List) AllLogs() []*Log {
+	logs := []*Log{}
+	for operator := range list.Operators {
+		for log := range list.Operators[operator].Logs {
+			logs = append(logs, &list.Operators[operator].Logs[log])
+		}
+		for log := range list.Operators[operator].TiledLogs {
+			logs = append(logs, &list.Operators[operator].TiledLogs[log])
+		}
+	}
+	return logs
+}
+
+func (log *Log) LogIDString() string {
+	return log.LogID.Base64String()
+}
+
+func (log *Log) AcceptsExpiration(expiration time.Time) bool {
+	return log.TemporalInterval == nil || withinInterval(expiration, log.TemporalInterval.StartInclusive, log.TemporalInterval.EndExclusive)
+}
+
+func withinInterval(expiration, startInclusive, endExclusive time.Time) bool {
+	return !expiration.Before(startInclusive) && expiration.Before(endExclusive)
+}

loglist/load.go

@@ -0,0 +1,118 @@
+// Copyright (C) 2020, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+var UserAgent = "certspotter"
+
+type ModificationToken struct {
+	etag     string
+	modified time.Time
+}
+
+var ErrNotModified = errors.New("loglist has not been modified")
+
+func newModificationToken(response *http.Response) *ModificationToken {
+	token := &ModificationToken{
+		etag: response.Header.Get("ETag"),
+	}
+	if t, err := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified")); err == nil {
+		token.modified = t
+	}
+	return token
+}
+
+func (token *ModificationToken) setRequestHeaders(request *http.Request) {
+	if token.etag != "" {
+		request.Header.Set("If-None-Match", token.etag)
+	} else if !token.modified.IsZero() {
+		request.Header.Set("If-Modified-Since", token.modified.Format(http.TimeFormat))
+	}
+}
+
+func Load(ctx context.Context, urlOrFile string) (*List, error) {
+	list, _, err := LoadIfModified(ctx, urlOrFile, nil)
+	return list, err
+}
+
+func LoadIfModified(ctx context.Context, urlOrFile string, token *ModificationToken) (*List, *ModificationToken, error) {
+	if strings.HasPrefix(urlOrFile, "https://") {
+		return FetchIfModified(ctx, urlOrFile, token)
+	} else {
+		list, err := ReadFile(urlOrFile)
+		return list, nil, err
+	}
+}
+
+func Fetch(ctx context.Context, url string) (*List, error) {
+	list, _, err := FetchIfModified(ctx, url, nil)
+	return list, err
+}
+
+func FetchIfModified(ctx context.Context, url string, token *ModificationToken) (*List, *ModificationToken, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	request.Header.Set("User-Agent", UserAgent)
+	if token != nil {
+		token.setRequestHeaders(request)
+	}
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return nil, nil, err
+	}
+	content, err := io.ReadAll(response.Body)
+	response.Body.Close()
+	if err != nil {
+		return nil, nil, err
+	}
+	if token != nil && response.StatusCode == http.StatusNotModified {
+		return nil, nil, ErrNotModified
+	}
+	if response.StatusCode != 200 {
+		return nil, nil, fmt.Errorf("%s: %s", url, response.Status)
+	}
+	list, err := Unmarshal(content)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parsing %s: %w", url, err)
+	}
+	return list, newModificationToken(response), err
+}
+
+func ReadFile(filename string) (*List, error) {
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return Unmarshal(content)
+}
+
+func Unmarshal(jsonBytes []byte) (*List, error) {
+	list := new(List)
+	if err := json.Unmarshal(jsonBytes, list); err != nil {
+		return nil, err
+	}
+	if err := list.Validate(); err != nil {
+		return nil, fmt.Errorf("Invalid log list: %s", err)
+	}
+	return list, nil
+}

loglist/schema.go

@@ -0,0 +1,116 @@
+// Copyright (C) 2020 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"time"
+
+	"software.sslmate.com/src/certspotter/ct"
+)
+
+type List struct {
+	Version          string     `json:"version"`
+	LogListTimestamp time.Time  `json:"log_list_timestamp"` // Only present in v3 of schema
+	Operators        []Operator `json:"operators"`
+}
+
+type Operator struct {
+	Name      string   `json:"name"`
+	Email     []string `json:"email"`
+	Logs      []Log    `json:"logs"`
+	TiledLogs []Log    `json:"tiled_logs"`
+}
+
+type Log struct {
+	Key              []byte        `json:"key"`
+	LogID            ct.SHA256Hash `json:"log_id"`
+	MMD              int           `json:"mmd"`
+	URL              string        `json:"url,omitempty"`            // only for rfc6962 logs
+	SubmissionURL    string        `json:"submission_url,omitempty"` // only for static-ct-api logs
+	MonitoringURL    string        `json:"monitoring_url,omitempty"` // only for static-ct-api logs
+	Description      string        `json:"description"`
+	State            State         `json:"state"`
+	DNS              string        `json:"dns"`
+	LogType          LogType       `json:"log_type"`
+	TemporalInterval *struct {
+		StartInclusive time.Time `json:"start_inclusive"`
+		EndExclusive   time.Time `json:"end_exclusive"`
+	} `json:"temporal_interval"`
+
+	// TODO: add previous_operators
+}
+
+func (log *Log) IsRFC6962() bool     { return log.URL != "" }
+func (log *Log) IsStaticCTAPI() bool { return log.SubmissionURL != "" && log.MonitoringURL != "" }
+
+// Return URL prefix for submission using the RFC6962 protocol
+func (log *Log) GetSubmissionURL() string {
+	if log.SubmissionURL != "" {
+		return log.SubmissionURL
+	} else {
+		return log.URL
+	}
+}
+
+// Return URL prefix for monitoring.
+// Since the protocol understood by the URL might be either RFC6962 or static-ct-api, this URL is
+// only useful for informational purposes.
+func (log *Log) GetMonitoringURL() string {
+	if log.MonitoringURL != "" {
+		return log.MonitoringURL
+	} else {
+		return log.URL
+	}
+}
+
+type State struct {
+	Pending *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"pending"`
+
+	Qualified *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"qualified"`
+
+	Usable *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"usable"`
+
+	Readonly *struct {
+		Timestamp     time.Time `json:"timestamp"`
+		FinalTreeHead struct {
+			TreeSize       int64  `json:"tree_size"`
+			SHA256RootHash []byte `json:"sha256_root_hash"`
+		} `json:"final_tree_head"`
+	} `json:"readonly"`
+
+	Retired *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"retired"`
+
+	Rejected *struct {
+		Timestamp time.Time `json:"timestamp"`
+	} `json:"rejected"`
+}
+
+func (state *State) IsApproved() bool {
+	return state.Qualified != nil || state.Usable != nil || state.Readonly != nil
+}
+
+func (state *State) WasApprovedAt(t time.Time) bool {
+	return state.Retired != nil && t.Before(state.Retired.Timestamp)
+}
+
+type LogType string
+
+const (
+	LogTypeProd = "prod"
+	LogTypeTest = "test"
+)

loglist/validate.go

@@ -0,0 +1,53 @@
+// Copyright (C) 2020 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package loglist
+
+import (
+	"crypto/sha256"
+	"fmt"
+)
+
+func (list *List) Validate() error {
+	for i := range list.Operators {
+		if err := list.Operators[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth operator (%s): %w", i, list.Operators[i].Name, err)
+		}
+	}
+	return nil
+}
+
+func (operator *Operator) Validate() error {
+	for i := range operator.Logs {
+		if err := operator.Logs[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth non-tiled log (%s): %w", i, operator.Logs[i].LogIDString(), err)
+		}
+	}
+	for i := range operator.TiledLogs {
+		if err := operator.TiledLogs[i].Validate(); err != nil {
+			return fmt.Errorf("problem with %dth tiled log (%s): %w", i, operator.TiledLogs[i].LogIDString(), err)
+		}
+	}
+	return nil
+}
+
+func (log *Log) Validate() error {
+	realLogID := sha256.Sum256(log.Key)
+	if log.LogID != realLogID {
+		return fmt.Errorf("log ID does not match log key")
+	}
+
+	if !log.IsRFC6962() && !log.IsStaticCTAPI() {
+		return fmt.Errorf("URL(s) not provided")
+	} else if log.IsRFC6962() && log.IsStaticCTAPI() {
+		return fmt.Errorf("inconsistent URLs provided")
+	}
+
+	return nil
+}

logs.go

@@ -1,139 +0,0 @@
-// Copyright (C) 2016 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package certspotter
-
-import (
-	"crypto"
-	"crypto/sha256"
-	"crypto/x509"
-	"encoding/base64"
-)
-
-type LogInfoFile struct {
-	Logs []LogInfo `json:"logs"`
-}
-type LogInfo struct {
-	Description string `json:"description"`
-	Key         []byte `json:"key"`
-	Url         string `json:"url"`
-	MMD         int    `json:"maximum_merge_delay"`
-}
-
-func (info *LogInfo) FullURI() string {
-	return "https://" + info.Url
-}
-
-func (info *LogInfo) ParsedPublicKey() (crypto.PublicKey, error) {
-	if info.Key != nil {
-		return x509.ParsePKIXPublicKey(info.Key)
-	} else {
-		return nil, nil
-	}
-}
-
-func (info *LogInfo) ID() []byte {
-	sum := sha256.Sum256(info.Key)
-	return sum[:]
-}
-
-var DefaultLogs = []LogInfo{
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA=="),
-		Url: "ct.googleapis.com/pilot",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1/TMabLkDpCjiupacAlP7xNi0I1JYP8bQFAHDG1xhtolSY1l4QgNRzRrvSe8liE+NPWHdjGxfx3JhTsN9x8/6Q=="),
-		Url: "ct.googleapis.com/aviator",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A=="),
-		Url: "ct1.digicert-ct.com/log",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg=="),
-		Url: "ct.googleapis.com/rocketeer",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEluqsHEYMG1XcDfy1lCdGV0JwOmkY4r87xNuroPS2bMBTP01CEDPwWJePa75y9CrsHEKqAy8afig1dpkIPSEUhg=="),
-		Url: "ct.ws.symantec.com",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE6pWeAv/u8TNtS4e8zf0ZF2L/lNPQWQc/Ai0ckP7IRzA78d0NuBEMXR2G3avTK0Zm+25ltzv9WWis36b4ztIYTQ=="),
-		Url: "vega.ws.symantec.com",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv7UIYZopMgTTJWPp2IXhhuAf1l6a9zM7gBvntj5fLaFm9pVKhKYhVnno94XuXeN8EsDgiSIJIj66FpUGvai5samyetZhLocRuXhAiXXbDNyQ4KR51tVebtEq2zT0mT9liTtGwiksFQccyUsaVPhsHq9gJ2IKZdWauVA2Fm5x9h8B9xKn/L/2IaMpkIYtd967TNTP/dLPgixN1PLCLaypvurDGSVDsuWabA3FHKWL9z8wr7kBkbdpEhLlg2H+NAC+9nGKx+tQkuhZ/hWR65aX+CNUPy2OB9/u2rNPyDydb988LENXoUcMkQT0dU3aiYGkFAY0uZjD2vH97TM20xYtNQIDAQAB"),
-		Url: "ctserver.cnnic.cn",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAETtK8v7MICve56qTHHDhhBOuV4IlUaESxZryCfk9QbG9co/CqPvTsgPDbCpp6oFtyAHwlDhnvr7JijXRD9Cb2FA=="),
-		Url: "ct.googleapis.com/icarus",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEmyGDvYXsRJsNyXSrYc9DjHsIa2xzb4UR7ZxVoV6mrc9iZB7xjI6+NrOiwH+P/xxkRmOFG6Jel20q37hTh58rA=="),
-		Url: "ct.googleapis.com/skydiver",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAESPNZ8/YFGNPbsu1Gfs/IEbVXsajWTOaft0oaFIZDqUiwy1o/PErK38SCFFWa+PeOQFXc9NKv6nV0+05/YIYuUQ=="),
-		Url: "ct.startssl.com",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzBGIey1my66PTTBmJxklIpMhRrQvAdPG+SvVyLpzmwai8IoCnNBrRhgwhbrpJIsO0VtwKAx+8TpFf1rzgkJgMQ=="),
-		Url: "ctlog.wosign.com",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEjicnerZVCXTrbEuUhGW85BXx6lrYfA43zro/bAna5ymW00VQb94etBzSg4j/KS/Oqf/fNN51D8DMGA2ULvw3AQ=="),
-		Url: "ctlog-gen2.api.venafi.com",
-		MMD: 86400,
-	},
-}
-
-// Logs which monitor certs from distrusted roots
-var UnderwaterLogs = []LogInfo{
-	{
-		Description: "Google 'Submariner' log",
-		Key:         mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEOfifIGLUV1Voou9JLfA5LZreRLSUMOCeeic8q3Dw0fpRkGMWV0Gtq20fgHQweQJeLVmEByQj9p81uIW4QkWkTw=="),
-		Url:         "ct.googleapis.com/submariner",
-		MMD:         86400,
-	},
-}
-
-// Logs which accept submissions from anyone
-var OpenLogs = []LogInfo{
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA=="),
-		Url: "ct.googleapis.com/pilot",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg=="),
-		Url: "ct.googleapis.com/rocketeer",
-		MMD: 86400,
-	},
-}
-
-func mustDecodeBase64(str string) []byte {
-	bytes, err := base64.StdEncoding.DecodeString(str)
-	if err != nil {
-		panic("MustDecodeBase64: " + err.Error())
-	}
-	return bytes
-}

man/.gitignore

@@ -0,0 +1 @@
+*.8

man/Makefile

@@ -0,0 +1,9 @@
+all: certspotter-script.8 certspotter.8
+
+%.8: %.md
+	lowdown -s -Tman \
+		-M title:$(basename $(notdir $@)) \
+		-M section:$(subst .,,$(suffix $@)) \
+		-M date:$(if $(SOURCE_DATE_EPOCH),$(shell date -I -u -d "@$(SOURCE_DATE_EPOCH)"),$(shell date -I -u)) \
+		-o $@ $<
+

man/certspotter-script.md

@@ -0,0 +1,258 @@
+# NAME
+
+**certspotter-script** - Certificate Transparency Log Monitor (hook script)
+
+# DESCRIPTION
+
+**certspotter-script** is *any* program that is executed by **certspotter(8)**
+when it needs to notify you about an event, such as detecting a certificate for
+a domain on your watch list.
+
+Scripts are placed in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` directory
+(`~/.certspotter/hooks.d` by default), or specified on the command line
+using the `-script` argument.
+
+# ENVIRONMENT
+
+## Event information
+
+The following environment variables are set for all types of events:
+
+`EVENT`
+
+:   One of the following values, indicating the type of event:
+
+      * `discovered_cert` - certspotter has discovered a certificate for a
+      domain on your watch list.
+
+      * `malformed_cert` - certspotter can't determine if a certificate
+      matches your watch list because the certificate or the log entry
+      is malformed.
+
+      * `error` - a problem is preventing certspotter from monitoring all
+      logs.
+
+    Additional event types may be defined in the future, so your script should
+    be able to handle unknown values.
+
+`SUMMARY`
+
+:   A short human-readable string describing the event.  This is the same string
+    used in the subject line of emails sent by certspotter.
+
+
+## Discovered certificate information
+
+The following environment variables are set for `discovered_cert` events:
+
+`WATCH_ITEM`
+
+:    The item from your watch list which matches this certificate.
+     (If more than one item matches, the first one is used.)
+
+`LOG_URI`
+
+:    The URI of the log containing the certificate.
+
+`ENTRY_INDEX`
+
+:    The index of the log entry containing the certificate.
+
+`TBS_SHA256`
+
+:    The hex-encoded SHA-256 digest of the TBSCertificate, as defined in RFC 6962 Section 3.2.
+     Certificates and their corresponding precertificates have the same `TBS_SHA256` value.
+
+`CERT_SHA256`
+
+:    The hex-encoded SHA-256 digest (sometimes called fingerprint) of the certificate.
+     The digest is computed over the ASN.1 DER encoding. 
+
+`PUBKEY_SHA256`
+
+:    The hex-encoded SHA-256 digest of the certificate's Subject Public Key Info.
+
+`CERT_FILENAME`
+
+:    Path to a file containing the PEM-encoded certificate chain.  Not set if `-no_save` was used.
+
+`JSON_FILENAME`
+
+:    Path to a JSON file containing additional information about the certificate.  See below for the format of the JSON file.
+     Not set if `-no_save` was used.
+
+`TEXT_FILENAME`
+
+:    Path to a text file containing information about the certificate.  This file contains the same text that
+     certspotter uses in emails.  You should not attempt to parse this file because its format may change in the future.
+     Not set if `-no_save` was used.
+
+`NOT_BEFORE`, `NOT_BEFORE_UNIXTIME`, `NOT_BEFORE_RFC3339`
+
+:    The not before time of the certificate, in a human-readable format, seconds since the UNIX epoch, and RFC3339, respectively.  These variables may be unset if there was a parse error, in which case `VALIDITY_PARSE_ERROR` is set.
+
+`NOT_AFTER`, `NOT_AFTER_UNIXTIME`, `NOT_AFTER_RFC3339`
+
+:    The not after (expiration) time of the certificate, in a human-readable format, seconds since the UNIX epoch, and RFC3339, respectively.  These variables may be unset if there was a parse error, in which case `VALIDITY_PARSE_ERROR` is set.
+
+`VALIDITY_PARSE_ERROR`
+
+:    Error parsing not before and not after, if any.  If this variable is set, then the `NOT_BEFORE` and `NOT_AFTER` family of variables are unset.
+
+`SUBJECT_DN`
+
+:    The distinguished name of the certificate's subject.  This variable may be unset if there was a parse error, in which case `SUBJECT_PARSE_ERROR` is set.
+
+`SUBJECT_PARSE_ERROR`
+
+:    Error parsing the subject, if any.  If this variable is set, then `SUBJECT_DN` is unset.
+
+`ISSUER_DN`
+
+:    The distinguished name of the certificate's issuer.  This variable may be unset if there was a parse error, in which case `ISSUER_PARSE_ERROR` is set.
+
+`ISSUER_PARSE_ERROR`
+
+:    Error parsing the issuer, if any.  If this variable is set, then `ISSUER_DN` is unset.
+
+`SERIAL`
+
+:    The hex-encoded serial number of the certificate.  Prefixed with a minus (-) sign if negative.  This variable may be unset if there was a parse error, in which case `SERIAL_PARSE_ERROR` is set.
+
+`SERIAL_PARSE_ERROR`
+
+:    Error parsing the serial number, if any.  If this variable is set, then `SERIAL` is unset.
+
+## Malformed certificate information
+
+The following environment variables are set for `malformed_cert` events:
+
+`LOG_URI`
+
+:    The URI of the log containing the malformed certificate.
+
+`ENTRY_INDEX`
+
+:    The index of the log entry containing the malformed certificate.
+
+`LEAF_HASH`
+
+:    The base64-encoded Merkle hash of the leaf containing the malformed certificate.
+
+`PARSE_ERROR`
+
+:    A human-readable string describing why the certificate is malformed.
+
+`ENTRY_FILENAME`
+
+:    Path to a file containing the JSON log entry.  The file contains a JSON object with two fields, `leaf_input` and `extra_data`, as described in RFC 6962 Section 4.6.
+
+`TEXT_FILENAME`
+
+:    Path to a text file containing a description of the malformed certificate.  This file contains the same text that certspotter uses in emails.
+
+## Error information
+
+The following environment variables are set for `error` events:
+
+`TEXT_FILENAME`
+
+:    Path to a text file containing a description of the error.  This file contains the same text that certspotter uses in emails.
+
+# JSON FILE FORMAT
+
+Unless `-no_save` is used, certspotter saves a JSON file for every discovered certificate
+under `$CERTSPOTTER_STATE_DIR`, and puts the path to the file in `$JSON_FILENAME`.  Your
+script can read the JSON file, such as with the jq(1) command, to get additional information
+about the certificate which isn't appropriate for environment variables.
+
+The JSON file contains an object with the following fields:
+
+`tbs_sha256`
+
+:    A string containing the hex-encoded SHA-256 digest of the TBSCertificate, as defined in RFC 6962 Section 3.2.
+     Certificates and their corresponding precertificates have the same `tbs_sha256` value.
+
+`pubkey_sha256`
+
+:    A string containing the hex-encoded SHA-256 digest of the certificate's Subject Public Key Info.
+
+`dns_names`
+
+:    An array of strings containing the DNS names for which the
+     certificate is valid, taken from both the DNS subject alternative names
+     (SANs) and the subject common name (CN). Internationalized domain names
+     are encoded in Punycode.
+
+`ip_addresses`
+
+:    An array of strings containing the IP addresses for which the certificate is valid,
+     taken from both the IP subject alternative names (SANs) and the subject common name (CN).
+
+`not_before`
+
+:    A string containing the not before time of the certificate in RFC3339 format.
+     Null if there was an error parsing the certificate's validity.
+
+`not_after`
+
+:    A string containing the not after (expiration) time of the certificate in RFC3339 format.
+     Null if there was an error parsing the certificate's validity.
+
+Additional fields will be added in the future based on user feedback. Please open
+an issue at <https://github.com/SSLMate/certspotter> if you have a use case for another field.
+
+# EXAMPLES
+
+Example environment variables for a `discovered_cert` event:
+
+```
+CERT_FILENAME=/home/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.pem
+CERT_SHA256=3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a
+ENTRY_INDEX=6464843
+EVENT=discovered_cert
+ISSUER_DN=C=GB, ST=Greater Manchester, L=Salford, O=Sectigo Limited, CN=Sectigo RSA Domain Validation Secure Server CA
+JSON_FILENAME=/usr2/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.v1.json
+LOG_URI=https://ct.cloudflare.com/logs/nimbus2024/
+NOT_AFTER='2024-01-26 03:47:26 +0000 UTC'
+NOT_AFTER_RFC3339=2024-01-26T03:47:26Z
+NOT_AFTER_UNIXTIME=1706240846
+NOT_BEFORE='2023-01-31 03:47:26 +0000 UTC'
+NOT_BEFORE_RFC3339=2023-01-31T03:47:26Z
+NOT_BEFORE_UNIXTIME=1675136846
+PUBKEY_SHA256=33ac1d9b9e56005ccac045eac2398b3e9dd6b3f5b66ae6260f2d478c7c0d82c8
+SERIAL=c170fbf3bf27481e5c351a4db6f2dc5f
+SUBJECT_DN=CN=sslmate.com
+SUMMARY='certificate discovered for .sslmate.com'
+TBS_SHA256=2388ee81c6f45cffc73e68a35fa8921e839e20acc9a98e8e6dcaea07cbfbdef8
+TEXT_FILENAME=/usr2/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.txt
+WATCH_ITEM=.sslmate.com
+```
+
+Example JSON file for a discovered certificate:
+
+```
+{
+  "dns_names": [
+    "sslmate.com",
+    "www.sslmate.com"
+  ],
+  "ip_addresses": [],
+  "not_after": "2024-01-26T03:47:26Z",
+  "not_before": "2023-01-31T03:47:26Z",
+  "pubkey_sha256": "33ac1d9b9e56005ccac045eac2398b3e9dd6b3f5b66ae6260f2d478c7c0d82c8",
+  "tbs_sha256": "2388ee81c6f45cffc73e68a35fa8921e839e20acc9a98e8e6dcaea07cbfbdef8"
+}
+```
+
+# SEE ALSO
+
+certspotter(8)
+
+# COPYRIGHT
+
+Copyright (c) 2016-2023 Opsmate, Inc.
+
+# BUGS
+
+Report bugs to <https://github.com/SSLMate/certspotter>.

man/certspotter.md

@@ -0,0 +1,242 @@
+# NAME
+
+**certspotter** - Certificate Transparency Log Monitor
+
+# SYNOPSIS
+
+**certspotter** [`-start_at_end`] [`-watchlist` *FILENAME*] [`-email` *ADDRESS*] `...`
+
+# DESCRIPTION
+
+**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that
+alerts you when a SSL/TLS certificate is issued for one of your domains.
+Cert Spotter is easier to use than other open source CT monitors, since it does not require
+a database. It's also more robust, since it uses a special certificate parser
+that ensures it won't miss certificates.
+
+Cert Spotter is also available as a hosted service by SSLMate,
+<https://sslmate.com/certspotter>.
+
+You can use Cert Spotter to detect:
+
+ * Certificates issued to attackers who have compromised your DNS and
+   are redirecting your visitors to their malicious site.
+ * Certificates issued to attackers who have taken over an abandoned
+   sub-domain in order to serve malware under your name.
+ * Certificates issued to attackers who have compromised a certificate
+   authority and want to impersonate your site.
+ * Certificates issued in violation of your corporate policy
+   or outside of your centralized certificate procurement process.
+
+# OPTIONS
+
+-batch_size *NUMBER*
+
+:   Maximum number of entries to request per call to get-entries.
+    You should not generally need to change this. Defaults to 1000.
+
+-email *ADDRESS*
+
+:   Email address to contact when a matching certificate is discovered, or
+    an error occurs.  You can specify this option more than once to email
+    multiple addresses.  Your system must have a working sendmail(1) command.
+
+    Regardless of the `-email` option, certspotter also emails any address listed
+    in `$CERTSPOTTER_CONFIG_DIR/email_recipients` file
+    (`~/.certspotter/email_recipients` by default).  (One address per line,
+    blank lines are ignored.)  This file is read only at startup, so you
+    must restart certspotter if you change it.
+
+-healthcheck *INTERVAL*
+
+:   Perform a health check at the given interval (default: "24h") as described
+    below.  *INTERVAL* must be a decimal number followed by "h" for hours or
+    "m" for minutes.
+
+-logs *ADDRESS*
+
+:   Filename or HTTPS URL of a v2 or v3 JSON log list containing logs to monitor.
+    The schema for this file can be found at <https://www.gstatic.com/ct/log_list/v3/log_list_schema.json>.
+    Defaults to <https://loglist.certspotter.org/monitor.json>, which includes
+    the union of active logs recognized by Chrome and Apple.  certspotter periodically
+    reloads the log list in case it has changed.
+
+-no\_save
+
+:   Do not save a copy of matching certificates. Note that enabling this option
+    will cause you to receive duplicate notifications, since certspotter will
+    have no way of knowing if you've been previously notified about a certificate.
+
+-script *COMMAND*
+
+:   Command to execute when a matching certificate is found or an error occurs. See
+    certspotter-script(8) for information about the interface to scripts.
+
+    Regardless of the `-script` option, certspotter also executes any executable
+    file in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` directory
+    (`~/.certspotter/hooks.d` by default).
+
+-start\_at\_end
+
+:   Start monitoring logs from the end rather than the beginning.
+
+    **WARNING**: monitoring from the beginning guarantees detection of all
+    certificates, but requires downloading hundreds of millions of
+    certificates, which takes days.
+
+-state\_dir *PATH*
+
+:   Directory for storing state. Defaults to `$CERTSPOTTER_STATE_DIR`, which is
+    "~/.certspotter" by default.
+
+-stdout
+
+:   Write matching certificates and errors to stdout.
+
+-verbose
+
+:   Be verbose.
+
+-version
+
+:   Print version and exit.
+
+-watchlist *PATH*
+
+:   File containing DNS names to monitor, one per line.  To monitor an entire
+    domain namespace (including the domain itself and all sub-domains) prefix
+    the domain name with a dot (e.g. ".example.com").  To monitor a single DNS
+    name only, do not prefix the name with a dot.
+    
+    Defaults to `$CERTSPOTTER_CONFIG_DIR/watchlist`, which is
+    "~/.certspotter/watchlist" by default.
+    Specify `-` to read the watch list from stdin.
+    
+    certspotter reads the watch list only when starting up, so you must restart
+    certspotter if you change it.
+
+# NOTIFICATIONS
+
+When certspotter detects a certificate matching your watchlist, or encounters
+an error that is preventing it from discovering certificates, it notifies you
+as follows:
+
+* Emails any address specified by the `-email` command line flag.
+
+* Emails any address listed in the `$CERTSPOTTER_CONFIG_DIR/email_recipients`
+  file (`~/.certspotter/email_recipients` by default).  (One address per line,
+  blank lines are ignored.)  This file is read only at startup, so you
+  must restart certspotter if you change it.
+
+* Executes the script specified by the `-script` command line flag.
+
+* Executes every executable file in the `$CERTSPOTTER_CONFIG_DIR/hooks.d`
+ directory (`~/.certspotter/hooks.d` by default).
+
+* Writes the notification to standard out if the `-stdout` flag was specified.
+
+Sending email requires a working sendmail(1) command.  For details about
+the script interface, see certspotter-script(8).
+
+# OPERATION
+
+certspotter continuously monitors all browser-recognized Certificate
+Transparency logs looking for certificates which are valid for any domain
+on your watch list. When certspotter detects a matching certificate, it
+emails you, executes a script, and/or writes a report to standard out,
+as described above.
+
+certspotter also saves a copy of matching certificates in
+`$CERTSPOTTER_STATE_DIR/certs` ("~/.certspotter/certs" by default)
+unless you specify the `-no_save` option.
+
+When certspotter has not previously monitored a log, it can either start
+monitoring the log from the beginning, or seek to the end of the log and
+start monitoring from there. Monitoring from the beginning guarantees
+detection of all certificates, but requires downloading hundreds of
+millions of certificates, which takes days. The default behavior is to
+monitor from the beginning. To start monitoring new logs from the end,
+specify the `-start_at_end` option.
+
+If certspotter has previously monitored a log, it resumes monitoring
+the log from the previous position.  This means that if you add
+a domain to your watch list, certspotter will not detect any certificates
+that were logged prior to the addition.  To detect such certificates,
+you must delete `$CERTSPOTTER_STATE_DIR/logs`, which will cause certspotter
+to restart monitoring from the very beginning of each log (provided
+`-start_at_end` is not specified).  This will cause certspotter to download
+hundreds of millions of certificates, which takes days.  To find preexisting
+certificates, it's faster to use the Cert Spotter service
+<https://sslmate.com/certspotter>, SSLMate's Certificate Transparency Search
+API <https://sslmate.com/ct_search_api>, or a CT search engine such as
+<https://crt.sh>.
+
+# ERROR HANDLING
+
+When certspotter encounters a problem with the local system (e.g. failure
+to write a file or execute a script), it prints a message to stderr and
+exits with a non-zero status.
+
+When certspotter encounters a problem monitoring a log, it prints a message
+to stderr and continues running.  It will try monitoring the log again later;
+most log errors are transient.
+
+Every 24 hours (unless overridden by `-healthcheck`), certspotter performs the
+following health checks:
+
+ * Ensure that the log list has been successfully retrieved at least once
+   since the previous health check.
+ * Ensure that every log has been successfully contacted at least once
+   since the previous health check.
+ * Ensure that certspotter is not falling behind monitoring any logs.
+
+If any health check fails, certspotter notifies you by email, script, and/or
+standard out, as described above.
+
+Health check failures should be rare, and you should take them seriously because it means
+certspotter might not detect all certificates.  It might also be an indication
+of CT log misbehavior.  Consult certspotter's stderr output for details, and if
+you need help, file an issue at <https://github.com/SSLMate/certspotter>.
+
+# EXIT STATUS
+
+certspotter exits 0 when it receives `SIGTERM` or `SIGINT`,
+and non-zero when a serious error occurs.
+
+# ENVIRONMENT
+
+`CERTSPOTTER_STATE_DIR`
+
+:   Directory for storing state. Overridden by `-state_dir`. Defaults to
+    `~/.certspotter`.
+
+`CERTSPOTTER_CONFIG_DIR`
+
+:   Directory from which any configuration, such as the watch list, is read.
+    Defaults to `~/.certspotter`.
+
+`EMAIL`
+
+:   Email address from which to send emails. If not set, certspotter lets sendmail pick
+    the address.
+
+`HTTPS_PROXY`
+
+:   URL of proxy server for making HTTPS requests.  `http://`, `https://`, and
+    `socks5://` URLs are supported.  By default, no proxy server is used.
+
+`SENDMAIL_PATH`
+
+:   Path to the sendmail binary used for sending emails. Defaults to `/usr/sbin/sendmail`.
+
+# SEE ALSO
+
+certspotter-script(8)
+
+# COPYRIGHT
+
+Copyright (c) 2016-2023 Opsmate, Inc.
+
+# BUGS
+
+Report bugs to <https://github.com/SSLMate/certspotter>.

merkletree/collapsed_tree.go

@@ -0,0 +1,182 @@
+// Copyright (C) 2022 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/bits"
+	"slices"
+)
+
+// CollapsedTree is an efficient representation of a Merkle (sub)tree that permits appending
+// nodes and calculating the root hash.
+type CollapsedTree struct {
+	offset uint64
+	nodes  []Hash
+	size   uint64
+}
+
+func calculateNumNodes(size uint64) int {
+	return bits.OnesCount64(size)
+}
+
+// TODO: phase out this function
+func EmptyCollapsedTree() *CollapsedTree {
+	return &CollapsedTree{nodes: []Hash{}, size: 0}
+}
+
+// TODO: phase out this function
+func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) {
+	tree := new(CollapsedTree)
+	if err := tree.Init(nodes, size); err != nil {
+		return nil, err
+	}
+	return tree, nil
+}
+
+func (tree CollapsedTree) Equal(other CollapsedTree) bool {
+	return tree.offset == other.offset && tree.size == other.size && slices.Equal(tree.nodes, other.nodes)
+}
+
+func (tree CollapsedTree) Clone() CollapsedTree {
+	return CollapsedTree{
+		offset: tree.offset,
+		nodes:  slices.Clone(tree.nodes),
+		size:   tree.size,
+	}
+}
+
+// Add a new leaf hash to the end of the tree.
+// Returns an error if and only if the new tree would be too large for the subtree offset.
+// Always returns a nil error if tree.Offset() == 0.
+func (tree *CollapsedTree) Add(hash Hash) error {
+	if tree.offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if tree.size+1 > maxSize {
+			return fmt.Errorf("subtree at offset %d is already at maximum size %d", tree.offset, maxSize)
+		}
+	}
+	tree.nodes = append(tree.nodes, hash)
+	tree.size++
+	tree.collapse()
+	return nil
+}
+
+func (tree *CollapsedTree) Append(other CollapsedTree) error {
+	if tree.offset+tree.size != other.offset {
+		return fmt.Errorf("subtree at offset %d cannot be appended to subtree ending at offset %d", other.offset, tree.offset+tree.size)
+	}
+	if tree.offset > 0 {
+		newSize := tree.size + other.size
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
+		if newSize > maxSize {
+			return fmt.Errorf("size of new subtree (%d) would exceed maximum size %d for a subtree at offset %d", newSize, maxSize, tree.offset)
+		}
+	}
+	if tree.size > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(tree.size)
+		if other.size > maxSize {
+			return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize)
+		}
+	}
+
+	tree.nodes = append(tree.nodes, other.nodes...)
+	tree.size += other.size
+	tree.collapse()
+	return nil
+}
+
+func (tree *CollapsedTree) collapse() {
+	numNodes := calculateNumNodes(tree.size)
+	for len(tree.nodes) > numNodes {
+		left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1]
+		tree.nodes = tree.nodes[:len(tree.nodes)-2]
+		tree.nodes = append(tree.nodes, HashChildren(left, right))
+	}
+}
+
+func (tree CollapsedTree) CalculateRoot() Hash {
+	if len(tree.nodes) == 0 {
+		return HashNothing()
+	}
+	i := len(tree.nodes) - 1
+	hash := tree.nodes[i]
+	for i > 0 {
+		i -= 1
+		hash = HashChildren(tree.nodes[i], hash)
+	}
+	return hash
+}
+
+// Return the subtree offset (0 if this represents an entire tree)
+func (tree CollapsedTree) Offset() uint64 {
+	return tree.offset
+}
+
+// Return a non-nil slice containing the nodes.  The slice
+// must not be modified.
+func (tree CollapsedTree) Nodes() []Hash {
+	if tree.nodes == nil {
+		return []Hash{}
+	} else {
+		return tree.nodes
+	}
+}
+
+// Return the number of leaf nodes in the tree.
+func (tree CollapsedTree) Size() uint64 {
+	return tree.size
+}
+
+type collapsedTreeMessage struct {
+	Offset uint64 `json:"offset,omitempty"`
+	Nodes  []Hash `json:"nodes"` // never nil
+	Size   uint64 `json:"size"`
+}
+
+func (tree CollapsedTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(collapsedTreeMessage{
+		Offset: tree.offset,
+		Nodes:  tree.Nodes(),
+		Size:   tree.size,
+	})
+}
+
+func (tree *CollapsedTree) UnmarshalJSON(b []byte) error {
+	var rawTree collapsedTreeMessage
+	if err := json.Unmarshal(b, &rawTree); err != nil {
+		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
+	}
+	if err := tree.InitSubtree(rawTree.Offset, rawTree.Nodes, rawTree.Size); err != nil {
+		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
+	}
+	return nil
+}
+
+func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error {
+	if len(nodes) != calculateNumNodes(size) {
+		return fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes))
+	}
+	tree.size = size
+	tree.nodes = nodes
+	return nil
+}
+
+func (tree *CollapsedTree) InitSubtree(offset uint64, nodes []Hash, size uint64) error {
+	if offset > 0 {
+		maxSize := uint64(1) << bits.TrailingZeros64(offset)
+		if size > maxSize {
+			return fmt.Errorf("subtree size (%d) is too large for offset %d (maximum size is %d)", size, offset, maxSize)
+		}
+	}
+	tree.offset = offset
+	return tree.Init(nodes, size)
+}

merkletree/fragment.go

@@ -0,0 +1,121 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"encoding/json"
+	"fmt"
+	"slices"
+)
+
+// FragmentedCollapsedTree represents a sequence of non-overlapping subtrees
+type FragmentedCollapsedTree struct {
+	subtrees []CollapsedTree // sorted by offset
+}
+
+func (tree *FragmentedCollapsedTree) AddHash(position uint64, hash Hash) error {
+	return tree.Add(CollapsedTree{
+		offset: position,
+		nodes:  []Hash{hash},
+		size:   1,
+	})
+}
+
+func (tree *FragmentedCollapsedTree) Add(subtree CollapsedTree) error {
+	if subtree.size == 0 {
+		return nil
+	}
+	i := len(tree.subtrees)
+	for i > 0 && tree.subtrees[i-1].offset > subtree.offset {
+		i--
+	}
+	if i > 0 && tree.subtrees[i-1].offset+tree.subtrees[i-1].size > subtree.offset {
+		return fmt.Errorf("overlaps with subtree ending at %d", tree.subtrees[i-1].offset+tree.subtrees[i-1].size)
+	}
+	if i < len(tree.subtrees) && subtree.offset+subtree.size > tree.subtrees[i].offset {
+		return fmt.Errorf("overlaps with subtree starting at %d", tree.subtrees[i].offset)
+	}
+	if i == 0 || tree.subtrees[i-1].Append(subtree) != nil {
+		tree.subtrees = slices.Insert(tree.subtrees, i, subtree)
+		i++
+	}
+	for i < len(tree.subtrees) && tree.subtrees[i-1].Append(tree.subtrees[i]) == nil {
+		tree.subtrees = slices.Delete(tree.subtrees, i, i+1)
+	}
+	return nil
+}
+
+func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error {
+	for _, subtree := range other.subtrees {
+		if err := tree.Add(subtree); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) {
+	var prevEnd uint64
+	for i := range tree.subtrees {
+		if prevEnd != tree.subtrees[i].offset {
+			if !yield(prevEnd, tree.subtrees[i].offset) {
+				return
+			}
+		}
+		prevEnd = tree.subtrees[i].offset + tree.subtrees[i].size
+	}
+	yield(prevEnd, 0)
+}
+
+func (tree FragmentedCollapsedTree) NumSubtrees() int {
+	return len(tree.subtrees)
+}
+
+func (tree FragmentedCollapsedTree) Subtree(i int) CollapsedTree {
+	return tree.subtrees[i]
+}
+
+func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree {
+	if tree.subtrees == nil {
+		return []CollapsedTree{}
+	} else {
+		return tree.subtrees
+	}
+}
+
+// Return true iff the tree contains at least the first n nodes (without any gaps)
+func (tree FragmentedCollapsedTree) ContainsFirstN(n uint64) bool {
+	return len(tree.subtrees) >= 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size >= n
+}
+
+func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error {
+	for i := 1; i < len(subtrees); i++ {
+		if subtrees[i-1].offset+subtrees[i-1].size > subtrees[i].offset {
+			return fmt.Errorf("subtrees %d and %d overlap", i-1, i)
+		}
+	}
+	tree.subtrees = subtrees
+	return nil
+}
+
+func (tree FragmentedCollapsedTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(tree.Subtrees())
+}
+
+func (tree *FragmentedCollapsedTree) UnmarshalJSON(b []byte) error {
+	var subtrees []CollapsedTree
+	if err := json.Unmarshal(b, &subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	if err := tree.Init(subtrees); err != nil {
+		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
+	}
+	return nil
+}

merkletree/hash.go

@@ -0,0 +1,72 @@
+// Copyright (C) 2022 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package merkletree
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+)
+
+const HashLen = 32
+
+type Hash [HashLen]byte
+
+func (h Hash) Base64String() string {
+	return base64.StdEncoding.EncodeToString(h[:])
+}
+
+func (h Hash) MarshalJSON() ([]byte, error) {
+	return json.Marshal(h[:])
+}
+
+func (h Hash) MarshalBinary() ([]byte, error) {
+	return h[:], nil
+}
+
+func (h *Hash) UnmarshalJSON(b []byte) error {
+	var hashBytes []byte
+	if err := json.Unmarshal(b, &hashBytes); err != nil {
+		return err
+	}
+	return h.UnmarshalBinary(hashBytes)
+}
+
+func (h *Hash) UnmarshalBinary(hashBytes []byte) error {
+	if len(hashBytes) != HashLen {
+		return fmt.Errorf("Merkle Tree hash has wrong length (should be %d bytes long, not %d)", HashLen, len(hashBytes))
+	}
+	copy(h[:], hashBytes)
+	return nil
+}
+
+func HashNothing() Hash {
+	return sha256.Sum256(nil)
+}
+
+func HashLeaf(leafBytes []byte) Hash {
+	var hash Hash
+	hasher := sha256.New()
+	hasher.Write([]byte{0x00})
+	hasher.Write(leafBytes)
+	hasher.Sum(hash[:0])
+	return hash
+}
+
+func HashChildren(left Hash, right Hash) Hash {
+	var hash Hash
+	hasher := sha256.New()
+	hasher.Write([]byte{0x01})
+	hasher.Write(left[:])
+	hasher.Write(right[:])
+	hasher.Sum(hash[:0])
+	return hash
+}

monitor/config.go

@@ -0,0 +1,23 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"time"
+)
+
+type Config struct {
+	LogListSource       string
+	State               StateProvider
+	StartAtEnd          bool
+	WatchList           WatchList
+	Verbose             bool
+	HealthCheckInterval time.Duration
+}

monitor/daemon.go

@@ -0,0 +1,168 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"golang.org/x/sync/errgroup"
+	"log"
+	insecurerand "math/rand"
+	"software.sslmate.com/src/certspotter/loglist"
+	"time"
+)
+
+const (
+	reloadLogListIntervalMin = 30 * time.Minute
+	reloadLogListIntervalMax = 90 * time.Minute
+)
+
+func randomDuration(min, max time.Duration) time.Duration {
+	return min + time.Duration(insecurerand.Int63n(int64(max-min+1)))
+}
+
+func reloadLogListInterval() time.Duration {
+	return randomDuration(reloadLogListIntervalMin, reloadLogListIntervalMax)
+}
+
+type task struct {
+	log  *loglist.Log
+	stop context.CancelFunc
+}
+
+type daemon struct {
+	config         *Config
+	taskgroup      *errgroup.Group
+	tasks          map[LogID]task
+	logsLoadedAt   time.Time
+	logListToken   *loglist.ModificationToken
+	logListError   string
+	logListErrorAt time.Time
+}
+
+func (daemon *daemon) healthCheck(ctx context.Context) error {
+	if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval {
+		info := &StaleLogListInfo{
+			Source:        daemon.config.LogListSource,
+			LastSuccess:   daemon.logsLoadedAt,
+			LastError:     daemon.logListError,
+			LastErrorTime: daemon.logListErrorAt,
+		}
+		if err := daemon.config.State.NotifyHealthCheckFailure(ctx, nil, info); err != nil {
+			return fmt.Errorf("error notifying about stale log list: %w", err)
+		}
+	}
+
+	for _, task := range daemon.tasks {
+		if err := healthCheckLog(ctx, daemon.config, task.log); err != nil {
+			return fmt.Errorf("error checking health of log %q: %w", task.log.URL, err)
+		}
+	}
+	return nil
+}
+
+func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task {
+	ctx, cancel := context.WithCancel(ctx)
+	daemon.taskgroup.Go(func() error {
+		defer cancel()
+		err := monitorLogContinously(ctx, daemon.config, ctlog)
+		if daemon.config.Verbose {
+			log.Printf("task for log %s stopped with error %s", ctlog.URL, err)
+		}
+		if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
+			return nil
+		} else {
+			return fmt.Errorf("error while monitoring %s: %w", ctlog.URL, err)
+		}
+	})
+	return task{log: ctlog, stop: cancel}
+}
+
+func (daemon *daemon) loadLogList(ctx context.Context) error {
+	newLogList, newToken, err := getLogList(ctx, daemon.config.LogListSource, daemon.logListToken)
+	if errors.Is(err, loglist.ErrNotModified) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	if daemon.config.Verbose {
+		log.Printf("fetched %d logs from %q", len(newLogList), daemon.config.LogListSource)
+	}
+
+	for logID, task := range daemon.tasks {
+		if _, exists := newLogList[logID]; exists {
+			continue
+		}
+		if daemon.config.Verbose {
+			log.Printf("stopping task for log %s", logID.Base64String())
+		}
+		task.stop()
+		delete(daemon.tasks, logID)
+	}
+	for logID, ctlog := range newLogList {
+		if _, isRunning := daemon.tasks[logID]; isRunning {
+			continue
+		}
+		if daemon.config.Verbose {
+			log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.URL)
+		}
+		daemon.tasks[logID] = daemon.startTask(ctx, ctlog)
+	}
+	daemon.logsLoadedAt = time.Now()
+	daemon.logListToken = newToken
+	return nil
+}
+
+func (daemon *daemon) run(ctx context.Context) error {
+	if err := daemon.config.State.Prepare(ctx); err != nil {
+		return fmt.Errorf("error preparing state: %w", err)
+	}
+
+	if err := daemon.loadLogList(ctx); err != nil {
+		return fmt.Errorf("error loading log list: %w", err)
+	}
+
+	reloadLogListTicker := time.NewTicker(reloadLogListInterval())
+	defer reloadLogListTicker.Stop()
+
+	healthCheckTicker := time.NewTicker(daemon.config.HealthCheckInterval)
+	defer healthCheckTicker.Stop()
+
+	for ctx.Err() == nil {
+		select {
+		case <-ctx.Done():
+		case <-reloadLogListTicker.C:
+			if err := daemon.loadLogList(ctx); err != nil {
+				daemon.logListError = err.Error()
+				daemon.logListErrorAt = time.Now()
+				recordError(ctx, daemon.config, nil, fmt.Errorf("error reloading log list (will try again later): %w", err))
+			}
+			reloadLogListTicker.Reset(reloadLogListInterval())
+		case <-healthCheckTicker.C:
+			if err := daemon.healthCheck(ctx); err != nil {
+				return err
+			}
+		}
+	}
+	return ctx.Err()
+}
+
+func Run(ctx context.Context, config *Config) error {
+	group, ctx := errgroup.WithContext(ctx)
+	daemon := &daemon{
+		config:    config,
+		taskgroup: group,
+		tasks:     make(map[LogID]task),
+	}
+	group.Go(func() error { return daemon.run(ctx) })
+	return group.Wait()
+}

monitor/discoveredcert.go

@@ -0,0 +1,176 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/pem"
+	"fmt"
+	"strings"
+	"time"
+
+	"software.sslmate.com/src/certspotter"
+	"software.sslmate.com/src/certspotter/ct"
+)
+
+type DiscoveredCert struct {
+	WatchItem    WatchItem
+	LogEntry     *LogEntry
+	Info         *certspotter.CertInfo
+	Chain        []ct.ASN1Cert // first entry is the leaf certificate or precertificate
+	TBSSHA256    [32]byte      // computed over Info.TBS.Raw
+	SHA256       [32]byte      // computed over Chain[0]
+	PubkeySHA256 [32]byte      // computed over Info.TBS.PublicKey.FullBytes
+	Identifiers  *certspotter.Identifiers
+}
+
+type certPaths struct {
+	certPath string
+	jsonPath string
+	textPath string
+}
+
+func (cert *DiscoveredCert) pemChain() []byte {
+	var buffer bytes.Buffer
+	for _, certBytes := range cert.Chain {
+		if err := pem.Encode(&buffer, &pem.Block{
+			Type:  "CERTIFICATE",
+			Bytes: certBytes,
+		}); err != nil {
+			panic(fmt.Errorf("encoding certificate as PEM failed unexpectedly: %w", err))
+		}
+	}
+	return buffer.Bytes()
+}
+
+func (cert *DiscoveredCert) json() any {
+	object := map[string]any{
+		"tbs_sha256":    hex.EncodeToString(cert.TBSSHA256[:]),
+		"pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]),
+		"dns_names":     cert.Identifiers.DNSNames,
+		"ip_addresses":  cert.Identifiers.IPAddrs,
+	}
+
+	if cert.Info.ValidityParseError == nil {
+		object["not_before"] = cert.Info.Validity.NotBefore
+		object["not_after"] = cert.Info.Validity.NotAfter
+	} else {
+		object["not_before"] = nil
+		object["not_after"] = nil
+	}
+
+	return object
+}
+
+func writeCertFiles(cert *DiscoveredCert, paths *certPaths) error {
+	if err := writeFile(paths.certPath, cert.pemChain(), 0666); err != nil {
+		return err
+	}
+	if err := writeJSONFile(paths.jsonPath, cert.json(), 0666); err != nil {
+		return err
+	}
+	if err := writeTextFile(paths.textPath, certNotificationText(cert, paths), 0666); err != nil {
+		return err
+	}
+	return nil
+}
+
+func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string {
+	env := []string{
+		"EVENT=discovered_cert",
+		"SUMMARY=" + certNotificationSummary(cert),
+		"CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented
+		"LOG_URI=" + cert.LogEntry.Log.URL,
+		"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
+		"WATCH_ITEM=" + cert.WatchItem.String(),
+		"TBS_SHA256=" + hex.EncodeToString(cert.TBSSHA256[:]),
+		"CERT_SHA256=" + hex.EncodeToString(cert.SHA256[:]),
+		"FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented
+		"PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]),
+		"PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented
+	}
+
+	if paths != nil {
+		env = append(env, "CERT_FILENAME="+paths.certPath)
+		env = append(env, "JSON_FILENAME="+paths.jsonPath)
+		env = append(env, "TEXT_FILENAME="+paths.textPath)
+	}
+
+	if cert.Info.ValidityParseError == nil {
+		env = append(env, "NOT_BEFORE="+cert.Info.Validity.NotBefore.String())
+		env = append(env, "NOT_BEFORE_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotBefore.Unix()))
+		env = append(env, "NOT_BEFORE_RFC3339="+cert.Info.Validity.NotBefore.Format(time.RFC3339))
+		env = append(env, "NOT_AFTER="+cert.Info.Validity.NotAfter.String())
+		env = append(env, "NOT_AFTER_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotAfter.Unix()))
+		env = append(env, "NOT_AFTER_RFC3339="+cert.Info.Validity.NotAfter.Format(time.RFC3339))
+	} else {
+		env = append(env, "VALIDITY_PARSE_ERROR="+cert.Info.ValidityParseError.Error())
+	}
+
+	if cert.Info.SubjectParseError == nil {
+		env = append(env, "SUBJECT_DN="+cert.Info.Subject.String())
+	} else {
+		env = append(env, "SUBJECT_PARSE_ERROR="+cert.Info.SubjectParseError.Error())
+	}
+
+	if cert.Info.IssuerParseError == nil {
+		env = append(env, "ISSUER_DN="+cert.Info.Issuer.String())
+	} else {
+		env = append(env, "ISSUER_PARSE_ERROR="+cert.Info.IssuerParseError.Error())
+	}
+
+	if cert.Info.SerialNumberParseError == nil {
+		env = append(env, "SERIAL="+fmt.Sprintf("%x", cert.Info.SerialNumber))
+	} else {
+		env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error())
+	}
+
+	return env
+}
+
+func certNotificationText(cert *DiscoveredCert, paths *certPaths) string {
+	// TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration)
+
+	text := new(strings.Builder)
+	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
+
+	fmt.Fprintf(text, "%x:\n", cert.SHA256)
+	for _, dnsName := range cert.Identifiers.DNSNames {
+		writeField("DNS Name", dnsName)
+	}
+	for _, ipaddr := range cert.Identifiers.IPAddrs {
+		writeField("IP Address", ipaddr)
+	}
+	writeField("Pubkey", hex.EncodeToString(cert.PubkeySHA256[:]))
+	if cert.Info.IssuerParseError == nil {
+		writeField("Issuer", cert.Info.Issuer)
+	} else {
+		writeField("Issuer", fmt.Sprintf("[unable to parse: %s]", cert.Info.IssuerParseError))
+	}
+	if cert.Info.ValidityParseError == nil {
+		writeField("Not Before", cert.Info.Validity.NotBefore)
+		writeField("Not After", cert.Info.Validity.NotAfter)
+	} else {
+		writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
+		writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
+	}
+	writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL))
+	writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:]))
+	if paths != nil {
+		writeField("Filename", paths.certPath)
+	}
+
+	return text.String()
+}
+
+func certNotificationSummary(cert *DiscoveredCert) string {
+	return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem)
+}

monitor/errors.go

@@ -0,0 +1,28 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"log"
+
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+func recordError(ctx context.Context, config *Config, ctlog *loglist.Log, errToRecord error) {
+	if err := config.State.NotifyError(ctx, ctlog, errToRecord); err != nil {
+		log.Print("unable to notify about error: ", err)
+		if ctlog == nil {
+			log.Print(errToRecord)
+		} else {
+			log.Print(ctlog.URL, ": ", errToRecord)
+		}
+	}
+}

monitor/fileutils.go

@@ -0,0 +1,56 @@
+// Copyright (C) 2017, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+)
+
+func randomFileSuffix() string {
+	var randomBytes [12]byte
+	if _, err := rand.Read(randomBytes[:]); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(randomBytes[:])
+}
+
+func writeFile(filename string, data []byte, perm os.FileMode) error {
+	tempname := filename + ".tmp." + randomFileSuffix()
+	if err := os.WriteFile(tempname, data, perm); err != nil {
+		return fmt.Errorf("error writing %s: %w", filename, err)
+	}
+	if err := os.Rename(tempname, filename); err != nil {
+		os.Remove(tempname)
+		return fmt.Errorf("error writing %s: %w", filename, err)
+	}
+	return nil
+}
+
+func writeTextFile(filename string, fileText string, perm os.FileMode) error {
+	return writeFile(filename, []byte(fileText), perm)
+}
+
+func writeJSONFile(filename string, data any, perm os.FileMode) error {
+	fileBytes, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	fileBytes = append(fileBytes, '\n')
+	return writeFile(filename, fileBytes, perm)
+}
+
+func fileExists(filename string) bool {
+	_, err := os.Lstat(filename)
+	return err == nil
+}

monitor/fsstate.go

@@ -0,0 +1,239 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+type FilesystemState struct {
+	StateDir  string
+	SaveCerts bool
+	Script    string
+	ScriptDir string
+	Email     []string
+	Stdout    bool
+}
+
+func (s *FilesystemState) logStateDir(logID LogID) string {
+	return filepath.Join(s.StateDir, "logs", logID.Base64URLString())
+}
+
+func (s *FilesystemState) Prepare(ctx context.Context) error {
+	return prepareStateDir(s.StateDir)
+}
+
+func (s *FilesystemState) PrepareLog(ctx context.Context, logID LogID) error {
+	var (
+		stateDirPath        = s.logStateDir(logID)
+		sthsDirPath         = filepath.Join(stateDirPath, "unverified_sths")
+		malformedDirPath    = filepath.Join(stateDirPath, "malformed_entries")
+		healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
+	)
+	for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} {
+		if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *FilesystemState) LoadLogState(ctx context.Context, logID LogID) (*LogState, error) {
+	filePath := filepath.Join(s.logStateDir(logID), "state.json")
+	fileBytes, err := os.ReadFile(filePath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	state := new(LogState)
+	if err := json.Unmarshal(fileBytes, state); err != nil {
+		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
+	}
+	return state, nil
+}
+
+func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state *LogState) error {
+	filePath := filepath.Join(s.logStateDir(logID), "state.json")
+	return writeJSONFile(filePath, state, 0666)
+}
+
+func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return storeSTHInDir(sthsDirPath, sth)
+}
+
+func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*ct.SignedTreeHead, error) {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return loadSTHsFromDir(sthsDirPath)
+}
+
+func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
+	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
+	return removeSTHFromDir(sthsDirPath, sth)
+}
+
+func (s *FilesystemState) NotifyCert(ctx context.Context, cert *DiscoveredCert) error {
+	var notifiedPath string
+	var paths *certPaths
+	if s.SaveCerts {
+		hexFingerprint := hex.EncodeToString(cert.SHA256[:])
+		prefixPath := filepath.Join(s.StateDir, "certs", hexFingerprint[0:2])
+		var (
+			notifiedFilename      = "." + hexFingerprint + ".notified"
+			certFilename          = hexFingerprint + ".pem"
+			jsonFilename          = hexFingerprint + ".v1.json"
+			textFilename          = hexFingerprint + ".txt"
+			legacyCertFilename    = hexFingerprint + ".cert.pem"
+			legacyPrecertFilename = hexFingerprint + ".precert.pem"
+		)
+
+		for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} {
+			if fileExists(filepath.Join(prefixPath, filename)) {
+				return nil
+			}
+		}
+
+		if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
+		}
+
+		notifiedPath = filepath.Join(prefixPath, notifiedFilename)
+		paths = &certPaths{
+			certPath: filepath.Join(prefixPath, certFilename),
+			jsonPath: filepath.Join(prefixPath, jsonFilename),
+			textPath: filepath.Join(prefixPath, textFilename),
+		}
+		if err := writeCertFiles(cert, paths); err != nil {
+			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
+		}
+	} else {
+		// TODO-4: save cert to temporary files, and defer their unlinking
+	}
+
+	if err := s.notify(ctx, &notification{
+		summary: certNotificationSummary(cert),
+		environ: certNotificationEnviron(cert, paths),
+		text:    certNotificationText(cert, paths),
+	}); err != nil {
+		return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
+	}
+
+	if notifiedPath != "" {
+		if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
+			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
+		}
+	}
+
+	return nil
+}
+
+func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError error) error {
+	var (
+		dirPath   = filepath.Join(s.logStateDir(entry.Log.LogID), "malformed_entries")
+		entryPath = filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index))
+		textPath  = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index))
+	)
+
+	summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.URL)
+
+	entryJSON := struct {
+		LeafInput []byte `json:"leaf_input"`
+		ExtraData []byte `json:"extra_data"`
+	}{
+		LeafInput: entry.LeafInput,
+		ExtraData: entry.ExtraData,
+	}
+
+	text := new(strings.Builder)
+	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
+	fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
+	writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.URL))
+	writeField("Leaf Hash", entry.LeafHash.Base64String())
+	writeField("Error", parseError.Error())
+
+	if err := writeJSONFile(entryPath, entryJSON, 0666); err != nil {
+		return fmt.Errorf("error saving JSON file: %w", err)
+	}
+	if err := writeTextFile(textPath, text.String(), 0666); err != nil {
+		return fmt.Errorf("error saving texT file: %w", err)
+	}
+
+	environ := []string{
+		"EVENT=malformed_cert",
+		"SUMMARY=" + summary,
+		"LOG_URI=" + entry.Log.URL,
+		"ENTRY_INDEX=" + fmt.Sprint(entry.Index),
+		"LEAF_HASH=" + entry.LeafHash.Base64String(),
+		"PARSE_ERROR=" + parseError.Error(),
+		"ENTRY_FILENAME=" + entryPath,
+		"TEXT_FILENAME=" + textPath,
+		"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
+	}
+
+	if err := s.notify(ctx, &notification{
+		environ: environ,
+		summary: summary,
+		text:    text.String(),
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) healthCheckDir(ctlog *loglist.Log) string {
+	if ctlog == nil {
+		return filepath.Join(s.StateDir, "healthchecks")
+	} else {
+		return filepath.Join(s.logStateDir(ctlog.LogID), "healthchecks")
+	}
+}
+
+func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info HealthCheckFailure) error {
+	textPath := filepath.Join(s.healthCheckDir(ctlog), healthCheckFilename())
+	environ := []string{
+		"EVENT=error",
+		"SUMMARY=" + info.Summary(),
+		"TEXT_FILENAME=" + textPath,
+	}
+	text := info.Text()
+	if err := writeTextFile(textPath, text, 0666); err != nil {
+		return fmt.Errorf("error saving text file: %w", err)
+	}
+	if err := s.notify(ctx, &notification{
+		environ: environ,
+		summary: info.Summary(),
+		text:    text,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, err error) error {
+	if ctlog == nil {
+		log.Print(err)
+	} else {
+		log.Print(ctlog.URL, ":", err)
+	}
+	return nil
+}

monitor/healthcheck.go

@@ -0,0 +1,138 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+func healthCheckFilename() string {
+	return time.Now().UTC().Format(time.RFC3339) + ".txt"
+}
+
+func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error {
+	state, err := config.State.LoadLogState(ctx, ctlog.LogID)
+	if err != nil {
+		return fmt.Errorf("error loading log state: %w", err)
+	} else if state == nil {
+		return nil
+	}
+
+	if time.Since(state.LastSuccess) < config.HealthCheckInterval {
+		return nil
+	}
+
+	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
+	if err != nil {
+		return fmt.Errorf("error loading STHs: %w", err)
+	}
+
+	if len(sths) == 0 {
+		info := &StaleSTHInfo{
+			Log:         ctlog,
+			LastSuccess: state.LastSuccess,
+			LatestSTH:   state.VerifiedSTH,
+		}
+		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
+			return fmt.Errorf("error notifying about stale STH: %w", err)
+		}
+	} else {
+		info := &BacklogInfo{
+			Log:       ctlog,
+			LatestSTH: sths[len(sths)-1],
+			Position:  state.DownloadPosition.Size(),
+		}
+		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
+			return fmt.Errorf("error notifying about backlog: %w", err)
+		}
+	}
+
+	return nil
+}
+
+type HealthCheckFailure interface {
+	Summary() string
+	Text() string
+}
+
+type StaleSTHInfo struct {
+	Log         *loglist.Log
+	LastSuccess time.Time
+	LatestSTH   *ct.SignedTreeHead // may be nil
+}
+
+type BacklogInfo struct {
+	Log       *loglist.Log
+	LatestSTH *ct.SignedTreeHead
+	Position  uint64
+}
+
+type StaleLogListInfo struct {
+	Source        string
+	LastSuccess   time.Time
+	LastError     string
+	LastErrorTime time.Time
+}
+
+func (e *BacklogInfo) Backlog() uint64 {
+	return e.LatestSTH.TreeSize - e.Position
+}
+
+func (e *StaleSTHInfo) Summary() string {
+	return fmt.Sprintf("Unable to contact %s since %s", e.Log.URL, e.LastSuccess)
+}
+func (e *BacklogInfo) Summary() string {
+	return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.URL)
+}
+func (e *StaleLogListInfo) Summary() string {
+	return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess)
+}
+
+func (e *StaleSTHInfo) Text() string {
+	text := new(strings.Builder)
+	fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess)
+	fmt.Fprintf(text, "\n")
+	fmt.Fprintf(text, "For details, see certspotter's stderr output.\n")
+	fmt.Fprintf(text, "\n")
+	if e.LatestSTH != nil {
+		fmt.Fprintf(text, "Latest known log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.TimestampTime())
+	} else {
+		fmt.Fprintf(text, "Latest known log size = none\n")
+	}
+	return text.String()
+}
+func (e *BacklogInfo) Text() string {
+	text := new(strings.Builder)
+	fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL)
+	fmt.Fprintf(text, "\n")
+	fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n")
+	fmt.Fprintf(text, "\n")
+	fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.TimestampTime())
+	fmt.Fprintf(text, "Current position = %d\n", e.Position)
+	fmt.Fprintf(text, "         Backlog = %d\n", e.Backlog())
+	return text.String()
+}
+func (e *StaleLogListInfo) Text() string {
+	text := new(strings.Builder)
+	fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess)
+	fmt.Fprintf(text, "\n")
+	fmt.Fprintf(text, "Last error (at %s): %s\n", e.LastErrorTime, e.LastError)
+	fmt.Fprintf(text, "\n")
+	fmt.Fprintf(text, "Consequentially, certspotter may not be monitoring all logs, and might fail to detect certificates.\n")
+	return text.String()
+}
+
+// TODO-3: make the errors more actionable

monitor/loglist.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"fmt"
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+type LogID = ct.SHA256Hash
+
+func getLogList(ctx context.Context, source string, token *loglist.ModificationToken) (map[LogID]*loglist.Log, *loglist.ModificationToken, error) {
+	list, newToken, err := loglist.LoadIfModified(ctx, source, token)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	logs := make(map[LogID]*loglist.Log)
+	for operatorIndex := range list.Operators {
+		for logIndex := range list.Operators[operatorIndex].Logs {
+			log := &list.Operators[operatorIndex].Logs[logIndex]
+			if _, exists := logs[log.LogID]; exists {
+				return nil, nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String())
+			}
+			logs[log.LogID] = log
+		}
+	}
+	return logs, newToken, nil
+}

monitor/mailutils.go

@@ -0,0 +1,34 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"os"
+)
+
+const mailDateFormat = "Mon, 2 Jan 2006 15:04:05 -0700"
+
+func generateMessageID() string {
+	var randomBytes [16]byte
+	if _, err := rand.Read(randomBytes[:]); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(randomBytes[:]) + "@selfhosted.certspotter.org"
+}
+
+func sendmailPath() string {
+	if envVar := os.Getenv("SENDMAIL_PATH"); envVar != "" {
+		return envVar
+	} else {
+		return "/usr/sbin/sendmail"
+	}
+}

monitor/monitor.go

@@ -0,0 +1,285 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"log"
+	"strings"
+	"time"
+
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/ct/client"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+const (
+	maxGetEntriesSize  = 1000
+	monitorLogInterval = 5 * time.Minute
+)
+
+func isFatalLogError(err error) bool {
+	return errors.Is(err, context.Canceled)
+}
+
+func newLogClient(ctlog *loglist.Log) (*client.LogClient, error) {
+	logKey, err := x509.ParsePKIXPublicKey(ctlog.Key)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing log key: %w", err)
+	}
+	verifier, err := ct.NewSignatureVerifier(logKey)
+	if err != nil {
+		return nil, fmt.Errorf("error with log key: %w", err)
+	}
+	return client.NewWithVerifier(strings.TrimRight(ctlog.URL, "/"), verifier), nil
+}
+
+func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) error {
+	logClient, err := newLogClient(ctlog)
+	if err != nil {
+		return err
+	}
+
+	ticker := time.NewTicker(monitorLogInterval)
+	defer ticker.Stop()
+
+	for ctx.Err() == nil {
+		if err := monitorLog(ctx, config, ctlog, logClient); err != nil {
+			return err
+		}
+		select {
+		case <-ctx.Done():
+		case <-ticker.C:
+		}
+	}
+	return ctx.Err()
+}
+
+func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClient *client.LogClient) (returnedErr error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	if err := config.State.PrepareLog(ctx, ctlog.LogID); err != nil {
+		return fmt.Errorf("error preparing state: %w", err)
+	}
+
+	startTime := time.Now()
+	latestSTH, err := logClient.GetSTH(ctx)
+	if isFatalLogError(err) {
+		return err
+	} else if err != nil {
+		recordError(ctx, config, ctlog, fmt.Errorf("error fetching latest STH: %w", err))
+		return nil
+	}
+	latestSTH.LogID = ctlog.LogID
+	if err := config.State.StoreSTH(ctx, ctlog.LogID, latestSTH); err != nil {
+		return fmt.Errorf("error storing latest STH: %w", err)
+	}
+
+	state, err := config.State.LoadLogState(ctx, ctlog.LogID)
+	if err != nil {
+		return fmt.Errorf("error loading log state: %w", err)
+	}
+	if state == nil {
+		if config.StartAtEnd {
+			tree, err := reconstructTree(ctx, logClient, latestSTH)
+			if isFatalLogError(err) {
+				return err
+			} else if err != nil {
+				recordError(ctx, config, ctlog, fmt.Errorf("error reconstructing tree of size %d: %w", latestSTH.TreeSize, err))
+				return nil
+			}
+			state = &LogState{
+				DownloadPosition: tree,
+				VerifiedPosition: tree,
+				VerifiedSTH:      latestSTH,
+				LastSuccess:      startTime.UTC(),
+			}
+		} else {
+			state = &LogState{
+				DownloadPosition: merkletree.EmptyCollapsedTree(),
+				VerifiedPosition: merkletree.EmptyCollapsedTree(),
+				VerifiedSTH:      nil,
+				LastSuccess:      startTime.UTC(),
+			}
+		}
+		if config.Verbose {
+			log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size())
+		}
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+			return fmt.Errorf("error storing log state: %w", err)
+		}
+	}
+
+	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
+	if err != nil {
+		return fmt.Errorf("error loading STHs: %w", err)
+	}
+
+	for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() {
+		// TODO-4: audit sths[0] against state.VerifiedSTH
+		if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
+			return fmt.Errorf("error removing STH: %w", err)
+		}
+		sths = sths[1:]
+	}
+
+	defer func() {
+		if config.Verbose {
+			log.Printf("saving state in defer for %s", ctlog.URL)
+		}
+		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil && returnedErr == nil {
+			returnedErr = fmt.Errorf("error storing log state: %w", err)
+		}
+	}()
+
+	if len(sths) == 0 {
+		state.LastSuccess = startTime.UTC()
+		return nil
+	}
+
+	var (
+		downloadBegin = state.DownloadPosition.Size()
+		downloadEnd   = sths[len(sths)-1].TreeSize
+		entries       = make(chan client.GetEntriesItem, maxGetEntriesSize)
+		downloadErr   error
+	)
+	if config.Verbose {
+		log.Printf("downloading entries from %s in range [%d, %d)", ctlog.URL, downloadBegin, downloadEnd)
+	}
+	go func() {
+		defer close(entries)
+		downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd)
+	}()
+	for rawEntry := range entries {
+		entry := &LogEntry{
+			Log:       ctlog,
+			Index:     state.DownloadPosition.Size(),
+			LeafInput: rawEntry.LeafInput,
+			ExtraData: rawEntry.ExtraData,
+			LeafHash:  merkletree.HashLeaf(rawEntry.LeafInput),
+		}
+		if err := processLogEntry(ctx, config, entry); err != nil {
+			return fmt.Errorf("error processing entry %d: %w", entry.Index, err)
+		}
+
+		state.DownloadPosition.Add(entry.LeafHash)
+		rootHash := state.DownloadPosition.CalculateRoot()
+		shouldSaveState := state.DownloadPosition.Size()%10000 == 0
+
+		for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize {
+			if merkletree.Hash(sths[0].SHA256RootHash) != rootHash {
+				recordError(ctx, config, ctlog, fmt.Errorf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
+
+				state.DownloadPosition = state.VerifiedPosition
+				if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+					return fmt.Errorf("error storing log state: %w", err)
+				}
+				return nil
+			}
+
+			state.VerifiedPosition = state.DownloadPosition
+			state.VerifiedSTH = sths[0]
+			shouldSaveState = true
+			if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
+				return fmt.Errorf("error removing verified STH: %w", err)
+			}
+
+			sths = sths[1:]
+		}
+
+		if shouldSaveState {
+			if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
+				return fmt.Errorf("error storing state file: %w", err)
+			}
+		}
+	}
+
+	if isFatalLogError(downloadErr) {
+		return downloadErr
+	} else if downloadErr != nil {
+		recordError(ctx, config, ctlog, fmt.Errorf("error downloading entries: %w", downloadErr))
+		return nil
+	}
+
+	if config.Verbose {
+		log.Printf("finished downloading entries from %s", ctlog.URL)
+	}
+
+	state.LastSuccess = startTime.UTC()
+	return nil
+}
+
+func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error {
+	for begin < end && ctx.Err() == nil {
+		size := end - begin
+		if size > maxGetEntriesSize {
+			size = maxGetEntriesSize
+		}
+		entries, err := logClient.GetRawEntries(ctx, begin, begin+size-1)
+		if err != nil {
+			return err
+		}
+		for _, entry := range entries {
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case entriesChan <- entry:
+			}
+		}
+		begin += uint64(len(entries))
+	}
+	return ctx.Err()
+}
+
+func reconstructTree(ctx context.Context, logClient *client.LogClient, sth *ct.SignedTreeHead) (*merkletree.CollapsedTree, error) {
+	if sth.TreeSize == 0 {
+		return merkletree.EmptyCollapsedTree(), nil
+	}
+	entries, err := logClient.GetRawEntries(ctx, sth.TreeSize-1, sth.TreeSize-1)
+	if err != nil {
+		return nil, err
+	}
+	leafHash := merkletree.HashLeaf(entries[0].LeafInput)
+
+	var tree *merkletree.CollapsedTree
+	if sth.TreeSize > 1 {
+		// XXX: if leafHash is in the tree in more than one place, this might not return the proof that we need ... get-entry-and-proof avoids this problem but not all logs support it
+		auditPath, _, err := logClient.GetAuditProof(ctx, leafHash[:], sth.TreeSize)
+		if err != nil {
+			return nil, err
+		}
+		hashes := make([]merkletree.Hash, len(auditPath))
+		for i := range hashes {
+			copy(hashes[i][:], auditPath[len(auditPath)-i-1])
+		}
+		tree, err = merkletree.NewCollapsedTree(hashes, sth.TreeSize-1)
+		if err != nil {
+			return nil, fmt.Errorf("log returned invalid audit proof for %x to %d: %w", leafHash, sth.TreeSize, err)
+		}
+	} else {
+		tree = merkletree.EmptyCollapsedTree()
+	}
+
+	tree.Add(leafHash)
+	rootHash := tree.CalculateRoot()
+	if rootHash != merkletree.Hash(sth.SHA256RootHash) {
+		return nil, fmt.Errorf("calculated root hash (%x) does not match signed tree head (%x) at size %d", rootHash, sth.SHA256RootHash, sth.TreeSize)
+	}
+
+	return tree, nil
+}

monitor/notify.go

@@ -0,0 +1,156 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+var stdoutMu sync.Mutex
+
+type notification struct {
+	environ []string
+	summary string
+	text    string
+}
+
+func (s *FilesystemState) notify(ctx context.Context, notif *notification) error {
+	if s.Stdout {
+		writeToStdout(notif)
+	}
+
+	if len(s.Email) > 0 {
+		if err := sendEmail(ctx, s.Email, notif); err != nil {
+			return err
+		}
+	}
+
+	if s.Script != "" {
+		if err := execScript(ctx, s.Script, notif); err != nil {
+			return err
+		}
+	}
+
+	if s.ScriptDir != "" {
+		if err := execScriptDir(ctx, s.ScriptDir, notif); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func writeToStdout(notif *notification) {
+	stdoutMu.Lock()
+	defer stdoutMu.Unlock()
+	os.Stdout.WriteString(notif.text + "\n")
+}
+
+func sendEmail(ctx context.Context, to []string, notif *notification) error {
+	stdin := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
+
+	from := os.Getenv("EMAIL")
+
+	if from != "" {
+		fmt.Fprintf(stdin, "From: %s\n", from)
+	}
+	fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", "))
+	fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.summary)
+	fmt.Fprintf(stdin, "Date: %s\n", time.Now().Format(mailDateFormat))
+	fmt.Fprintf(stdin, "Message-ID: <%s>\n", generateMessageID())
+	fmt.Fprintf(stdin, "Mime-Version: 1.0\n")
+	fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n")
+	fmt.Fprintf(stdin, "X-Mailer: certspotter\n")
+	fmt.Fprintf(stdin, "\n")
+	fmt.Fprint(stdin, notif.text)
+
+	args := []string{"-i"}
+	if from != "" {
+		args = append(args, "-f", from)
+	}
+	args = append(args, "--")
+	args = append(args, to...)
+
+	sendmail := exec.CommandContext(ctx, sendmailPath(), args...)
+	sendmail.Stdin = stdin
+	sendmail.Stderr = stderr
+
+	if err := sendmail.Run(); err == nil {
+		return nil
+	} else if ctx.Err() != nil {
+		return ctx.Err()
+	} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
+		return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
+	} else {
+		return fmt.Errorf("error sending email to %v: %w", to, err)
+	}
+}
+
+func execScript(ctx context.Context, scriptName string, notif *notification) error {
+	stderr := new(bytes.Buffer)
+
+	cmd := exec.CommandContext(ctx, scriptName)
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, notif.environ...)
+	cmd.Stderr = stderr
+
+	if err := cmd.Run(); err == nil {
+		return nil
+	} else if ctx.Err() != nil {
+		return ctx.Err()
+	} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
+		return fmt.Errorf("script %q exited with code %d and error %q", scriptName, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
+	} else if isExitError {
+		return fmt.Errorf("script %q terminated by signal with error %q", scriptName, strings.TrimSpace(stderr.String()))
+	} else {
+		return fmt.Errorf("error executing script: %w", err)
+	}
+}
+
+func execScriptDir(ctx context.Context, dirPath string, notif *notification) error {
+	dirents, err := os.ReadDir(dirPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("error executing scripts in directory %q: %w", dirPath, err)
+	}
+	for _, dirent := range dirents {
+		if strings.HasPrefix(dirent.Name(), ".") {
+			continue
+		}
+		scriptPath := filepath.Join(dirPath, dirent.Name())
+		info, err := os.Stat(scriptPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("error executing %q in directory %q: %w", dirent.Name(), dirPath, err)
+		} else if info.Mode().IsRegular() && isExecutable(info.Mode()) {
+			if err := execScript(ctx, scriptPath, notif); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func isExecutable(mode os.FileMode) bool {
+	return mode&0111 != 0
+}

monitor/process.go

@@ -0,0 +1,118 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"software.sslmate.com/src/certspotter"
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/merkletree"
+)
+
+type LogEntry struct {
+	Log       *loglist.Log
+	Index     uint64
+	LeafInput []byte
+	ExtraData []byte
+	LeafHash  merkletree.Hash
+}
+
+func processLogEntry(ctx context.Context, config *Config, entry *LogEntry) error {
+	leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput))
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err))
+	}
+	switch leaf.TimestampedEntry.EntryType {
+	case ct.X509LogEntryType:
+		return processX509LogEntry(ctx, config, entry, leaf.TimestampedEntry.X509Entry)
+	case ct.PrecertLogEntryType:
+		return processPrecertLogEntry(ctx, config, entry, leaf.TimestampedEntry.PrecertEntry)
+	default:
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType))
+	}
+}
+
+func processX509LogEntry(ctx context.Context, config *Config, entry *LogEntry, cert ct.ASN1Cert) error {
+	certInfo, err := certspotter.MakeCertInfoFromRawCert(cert)
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err))
+	}
+
+	chain, err := ct.UnmarshalX509ChainArray(entry.ExtraData)
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for X.509 entry: %w", err))
+	}
+	chain = append([]ct.ASN1Cert{cert}, chain...)
+
+	if precertTBS, err := certspotter.ReconstructPrecertTBS(certInfo.TBS); err == nil {
+		certInfo.TBS = precertTBS
+	} else {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error reconstructing precertificate TBSCertificate: %w", err))
+	}
+
+	return processCertificate(ctx, config, entry, certInfo, chain)
+}
+
+func processPrecertLogEntry(ctx context.Context, config *Config, entry *LogEntry, precert ct.PreCert) error {
+	certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate)
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err))
+	}
+
+	chain, err := ct.UnmarshalPrecertChainArray(entry.ExtraData)
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for precert entry: %w", err))
+	}
+
+	if _, err := certspotter.ValidatePrecert(chain[0], precert.TBSCertificate); err != nil {
+		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err))
+	}
+
+	return processCertificate(ctx, config, entry, certInfo, chain)
+}
+
+func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
+	identifiers, err := certInfo.ParseIdentifiers()
+	if err != nil {
+		return processMalformedLogEntry(ctx, config, entry, err)
+	}
+	matched, watchItem := config.WatchList.Matches(identifiers)
+	if !matched {
+		return nil
+	}
+
+	cert := &DiscoveredCert{
+		WatchItem:    watchItem,
+		LogEntry:     entry,
+		Info:         certInfo,
+		Chain:        chain,
+		TBSSHA256:    sha256.Sum256(certInfo.TBS.Raw),
+		SHA256:       sha256.Sum256(chain[0]),
+		PubkeySHA256: sha256.Sum256(certInfo.TBS.PublicKey.FullBytes),
+		Identifiers:  identifiers,
+	}
+
+	if err := config.State.NotifyCert(ctx, cert); err != nil {
+		return fmt.Errorf("error notifying about certificate %x: %w", cert.SHA256, err)
+	}
+
+	return nil
+}
+
+func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error {
+	if err := config.State.NotifyMalformedEntry(ctx, entry, parseError); err != nil {
+		return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
+	}
+	return nil
+}

monitor/state.go

@@ -0,0 +1,68 @@
+// Copyright (C) 2024 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"context"
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+	"software.sslmate.com/src/certspotter/merkletree"
+	"time"
+)
+
+type LogState struct {
+	DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
+	VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
+	VerifiedSTH      *ct.SignedTreeHead        `json:"verified_sth"`
+	LastSuccess      time.Time                 `json:"last_success"`
+}
+
+type StateProvider interface {
+	// Initialize the state.  Called before any other method in this interface.
+	// Idempotent: returns nil if the state is already initialized.
+	Prepare(context.Context) error
+
+	// Initialize the state for the given log.  Called before any other method
+	// with the log ID.  Idempotent: returns nil if log state already initialized.
+	PrepareLog(context.Context, LogID) error
+
+	// Store log state for retrieval by LoadLogState.
+	StoreLogState(context.Context, LogID, *LogState) error
+
+	// Load log state that was previously stored with StoreLogState.
+	// Returns nil, nil if StoreLogState has not been called yet for this log.
+	LoadLogState(context.Context, LogID) (*LogState, error)
+
+	// Store STH for retrieval by LoadSTHs.  If an STH with the same
+	// timestamp and root hash is already stored, this STH can be ignored.
+	StoreSTH(context.Context, LogID, *ct.SignedTreeHead) error
+
+	// Load all STHs for this log previously stored with StoreSTH.
+	// The returned slice must be sorted by tree size.
+	LoadSTHs(context.Context, LogID) ([]*ct.SignedTreeHead, error)
+
+	// Remove an STH so it is no longer returned by LoadSTHs.
+	RemoveSTH(context.Context, LogID, *ct.SignedTreeHead) error
+
+	// Called when a certificate matching the watch list is discovered.
+	NotifyCert(context.Context, *DiscoveredCert) error
+
+	// Called when certspotter fails to parse a log entry.
+	NotifyMalformedEntry(context.Context, *LogEntry, error) error
+
+	// Called when a health check fails.  The log is nil if the
+	// feailure is not associated with a log.
+	NotifyHealthCheckFailure(context.Context, *loglist.Log, HealthCheckFailure) error
+
+	// Called when a non-fatal error occurs.  The log is nil if the error is
+	// not associated with a log.  Note that most errors are transient, and
+	// certspotter will retry the failed operation later.
+	NotifyError(context.Context, *loglist.Log, error) error
+}

monitor/statedir.go

@@ -0,0 +1,155 @@
+// Copyright (C) 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/merkletree"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func readVersion(stateDir string) (int, error) {
+	path := filepath.Join(stateDir, "version")
+
+	fileBytes, err := os.ReadFile(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		if fileExists(filepath.Join(stateDir, "evidence")) {
+			return 0, nil
+		} else {
+			return -1, nil
+		}
+	} else if err != nil {
+		return -1, err
+	}
+
+	version, err := strconv.Atoi(strings.TrimSpace(string(fileBytes)))
+	if err != nil {
+		return -1, fmt.Errorf("version file %q is malformed: %w", path, err)
+	}
+
+	return version, nil
+}
+
+func writeVersion(stateDir string) error {
+	return writeFile(filepath.Join(stateDir, "version"), []byte{'2', '\n'}, 0666)
+}
+
+func migrateLogStateDirV1(dir string) error {
+	var sth ct.SignedTreeHead
+	var tree merkletree.CollapsedTree
+
+	sthPath := filepath.Join(dir, "sth.json")
+	sthData, err := os.ReadFile(sthPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	treePath := filepath.Join(dir, "tree.json")
+	treeData, err := os.ReadFile(treePath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	if err := json.Unmarshal(sthData, &sth); err != nil {
+		return fmt.Errorf("error unmarshaling %s: %w", sthPath, err)
+	}
+	if err := json.Unmarshal(treeData, &tree); err != nil {
+		return fmt.Errorf("error unmarshaling %s: %w", treePath, err)
+	}
+
+	stateFile := LogState{
+		DownloadPosition: &tree,
+		VerifiedPosition: &tree,
+		VerifiedSTH:      &sth,
+		LastSuccess:      time.Now().UTC(),
+	}
+	if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil {
+		return err
+	}
+
+	if err := os.Remove(sthPath); err != nil {
+		return err
+	}
+	if err := os.Remove(treePath); err != nil {
+		return err
+	}
+	return nil
+}
+
+func migrateStateDirV1(stateDir string) error {
+	if lockfile := filepath.Join(stateDir, "lock"); fileExists(lockfile) {
+		return fmt.Errorf("directory is locked by another instance of certspotter; remove %s if this is not the case", lockfile)
+	}
+
+	if logDirs, err := os.ReadDir(filepath.Join(stateDir, "logs")); err == nil {
+		for _, logDir := range logDirs {
+			if strings.HasPrefix(logDir.Name(), ".") || !logDir.IsDir() {
+				continue
+			}
+			if err := migrateLogStateDirV1(filepath.Join(stateDir, "logs", logDir.Name())); err != nil {
+				return fmt.Errorf("error migrating log state: %w", err)
+			}
+		}
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+
+	if err := writeVersion(stateDir); err != nil {
+		return err
+	}
+
+	if err := os.Remove(filepath.Join(stateDir, "once")); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+
+	return nil
+}
+
+func prepareStateDir(stateDir string) error {
+	if err := os.Mkdir(stateDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+		return err
+	}
+
+	if version, err := readVersion(stateDir); err != nil {
+		return err
+	} else if version == -1 {
+		if err := writeVersion(stateDir); err != nil {
+			return err
+		}
+	} else if version == 0 {
+		return fmt.Errorf("%s was created by a very old version of certspotter; run any version of certspotter after 0.2 and before 0.15.0 to upgrade this directory, or remove it to start from scratch", stateDir)
+	} else if version == 1 {
+		if err := migrateStateDirV1(stateDir); err != nil {
+			return err
+		}
+	} else if version > 2 {
+		return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir)
+	}
+
+	for _, subdir := range []string{"certs", "logs", "healthchecks"} {
+		if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+			return err
+		}
+	}
+
+	return nil
+}

monitor/sthdir.go

@@ -0,0 +1,93 @@
+// Copyright (C) 2017, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"cmp"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"software.sslmate.com/src/certspotter/ct"
+	"strconv"
+	"strings"
+)
+
+func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) {
+	entries, err := os.ReadDir(dirPath)
+	if errors.Is(err, fs.ErrNotExist) {
+		return []*ct.SignedTreeHead{}, nil
+	} else if err != nil {
+		return nil, err
+	}
+	sths := make([]*ct.SignedTreeHead, 0, len(entries))
+	for _, entry := range entries {
+		filename := entry.Name()
+		if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") {
+			continue
+		}
+		sth, err := readSTHFile(filepath.Join(dirPath, filename))
+		if err != nil {
+			return nil, err
+		}
+		sths = append(sths, sth)
+	}
+	slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) int { return cmp.Compare(a.TreeSize, b.TreeSize) })
+	return sths, nil
+}
+
+func readSTHFile(filePath string) (*ct.SignedTreeHead, error) {
+	fileBytes, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+	sth := new(ct.SignedTreeHead)
+	if err := json.Unmarshal(fileBytes, sth); err != nil {
+		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
+	}
+	return sth, nil
+}
+
+func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error {
+	filePath := filepath.Join(dirPath, sthFilename(sth))
+	if fileExists(filePath) {
+		return nil
+	}
+	return writeJSONFile(filePath, sth, 0666)
+}
+
+func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error {
+	filePath := filepath.Join(dirPath, sthFilename(sth))
+	err := os.Remove(filePath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+	return nil
+}
+
+// generate a filename that uniquely identifies the STH (within the context of a particular log)
+func sthFilename(sth *ct.SignedTreeHead) string {
+	hasher := sha256.New()
+	switch sth.Version {
+	case ct.V1:
+		binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
+		binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash)
+	default:
+		panic(fmt.Errorf("sthFilename: invalid STH version %d", sth.Version))
+	}
+	// For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic)
+	return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json"
+}

monitor/watchlist.go

@@ -0,0 +1,135 @@
+// Copyright (C) 2016, 2023 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package monitor
+
+import (
+	"bufio"
+	"fmt"
+	"golang.org/x/net/idna"
+	"io"
+	"software.sslmate.com/src/certspotter"
+	"strings"
+)
+
+type WatchItem struct {
+	domain       []string
+	acceptSuffix bool
+}
+
+type WatchList []WatchItem
+
+func ParseWatchItem(str string) (WatchItem, error) {
+	fields := strings.Fields(str)
+	if len(fields) == 0 {
+		return WatchItem{}, fmt.Errorf("empty domain")
+	}
+	domain := fields[0]
+
+	for _, field := range fields[1:] {
+		switch {
+		case strings.HasPrefix(field, "valid_at:"):
+			// Ignore for backwards compatibility
+		default:
+			return WatchItem{}, fmt.Errorf("unknown parameter %q", field)
+		}
+	}
+
+	if domain == "." {
+		// "." as in root zone -> matches everything
+		return WatchItem{
+			domain:       []string{},
+			acceptSuffix: true,
+		}, nil
+	}
+
+	acceptSuffix := false
+	if strings.HasPrefix(domain, ".") {
+		acceptSuffix = true
+		domain = domain[1:]
+	}
+
+	asciiDomain, err := idna.ToASCII(strings.ToLower(strings.TrimRight(domain, ".")))
+	if err != nil {
+		return WatchItem{}, fmt.Errorf("invalid domain %q (%w)", domain, err)
+	}
+	return WatchItem{
+		domain:       strings.Split(asciiDomain, "."),
+		acceptSuffix: acceptSuffix,
+	}, nil
+}
+
+func ReadWatchList(reader io.Reader) (WatchList, error) {
+	items := make(WatchList, 0, 50)
+	scanner := bufio.NewScanner(reader)
+	lineNo := 0
+	for scanner.Scan() {
+		line := scanner.Text()
+		lineNo++
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		item, err := ParseWatchItem(line)
+		if err != nil {
+			return nil, fmt.Errorf("%w on line %d", err, lineNo)
+		}
+		items = append(items, item)
+	}
+	return items, scanner.Err()
+}
+
+func (item WatchItem) String() string {
+	if item.acceptSuffix {
+		return "." + strings.Join(item.domain, ".")
+	} else {
+		return strings.Join(item.domain, ".")
+	}
+}
+
+func (item WatchItem) matchesDNSName(dnsName []string) bool {
+	watchDomain := item.domain
+	for len(dnsName) > 0 && len(watchDomain) > 0 {
+		certLabel := dnsName[len(dnsName)-1]
+		watchLabel := watchDomain[len(watchDomain)-1]
+
+		if !dnsLabelMatches(certLabel, watchLabel) {
+			return false
+		}
+
+		dnsName = dnsName[:len(dnsName)-1]
+		watchDomain = watchDomain[:len(watchDomain)-1]
+	}
+	return len(watchDomain) == 0 && (item.acceptSuffix || len(dnsName) == 0)
+}
+
+func dnsLabelMatches(certLabel string, watchLabel string) bool {
+	// For fail-safe behavior, if a label was unparsable, it matches everything.
+	// Similarly, redacted labels match everything, since the label _might_ be
+	// for a name we're interested in.
+
+	return certLabel == "*" ||
+		certLabel == "?" ||
+		certLabel == certspotter.UnparsableDNSLabelPlaceholder ||
+		certspotter.MatchesWildcard(watchLabel, certLabel)
+}
+
+func (list WatchList) Matches(identifiers *certspotter.Identifiers) (bool, WatchItem) {
+	dnsNames := make([][]string, len(identifiers.DNSNames))
+	for i, dnsName := range identifiers.DNSNames {
+		dnsNames[i] = strings.Split(dnsName, ".")
+	}
+	for _, item := range list {
+		for _, dnsName := range dnsNames {
+			if item.matchesDNSName(dnsName) {
+				return true, item
+			}
+		}
+	}
+	return false, WatchItem{}
+}

scanner.go

@@ -1,324 +0,0 @@
-// Copyright (C) 2016 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-//
-// This file contains code from https://github.com/google/certificate-transparency/tree/master/go
-// See ct/AUTHORS and ct/LICENSE for copyright and license information.
-
-package certspotter
-
-import (
-	//	"container/list"
-	"bytes"
-	"crypto"
-	"errors"
-	"fmt"
-	"log"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/ct/client"
-)
-
-type ProcessCallback func(*Scanner, *ct.LogEntry)
-
-const (
-	FETCH_RETRIES    = 10
-	FETCH_RETRY_WAIT = 1
-)
-
-// ScannerOptions holds configuration options for the Scanner
-type ScannerOptions struct {
-	// Number of entries to request in one batch from the Log
-	BatchSize int
-
-	// Number of concurrent proecssors to run
-	NumWorkers int
-
-	// Don't print any status messages to stdout
-	Quiet bool
-}
-
-// Creates a new ScannerOptions struct with sensible defaults
-func DefaultScannerOptions() *ScannerOptions {
-	return &ScannerOptions{
-		BatchSize:  1000,
-		NumWorkers: 1,
-		Quiet:      false,
-	}
-}
-
-// Scanner is a tool to scan all the entries in a CT Log.
-type Scanner struct {
-	// Base URI of CT log
-	LogUri string
-
-	// Public key of the log
-	publicKey crypto.PublicKey
-	LogId     []byte
-
-	// Client used to talk to the CT log instance
-	logClient *client.LogClient
-
-	// Configuration options for this Scanner instance
-	opts ScannerOptions
-
-	// Stats
-	certsProcessed int64
-}
-
-// fetchRange represents a range of certs to fetch from a CT log
-type fetchRange struct {
-	start int64
-	end   int64
-}
-
-// Worker function to process certs.
-// Accepts ct.LogEntries over the |entries| channel, and invokes processCert on them.
-// Returns true over the |done| channel when the |entries| channel is closed.
-func (s *Scanner) processerJob(id int, entries <-chan ct.LogEntry, processCert ProcessCallback, wg *sync.WaitGroup) {
-	for entry := range entries {
-		atomic.AddInt64(&s.certsProcessed, 1)
-		processCert(s, &entry)
-	}
-	wg.Done()
-}
-
-func (s *Scanner) fetch(r fetchRange, entries chan<- ct.LogEntry, tree *CollapsedMerkleTree) error {
-	success := false
-	retries := FETCH_RETRIES
-	retryWait := FETCH_RETRY_WAIT
-	for !success {
-		s.Log(fmt.Sprintf("Fetching entries %d to %d", r.start, r.end))
-		logEntries, err := s.logClient.GetEntries(r.start, r.end)
-		if err != nil {
-			if retries == 0 {
-				s.Warn(fmt.Sprintf("Problem fetching entries %d to %d from log: %s", r.start, r.end, err.Error()))
-				return err
-			} else {
-				s.Log(fmt.Sprintf("Problem fetching entries %d to %d from log (will retry): %s", r.start, r.end, err.Error()))
-				time.Sleep(time.Duration(retryWait) * time.Second)
-				retries--
-				retryWait *= 2
-				continue
-			}
-		}
-		retries = FETCH_RETRIES
-		retryWait = FETCH_RETRY_WAIT
-		for _, logEntry := range logEntries {
-			if tree != nil {
-				tree.Add(hashLeaf(logEntry.LeafBytes))
-			}
-			logEntry.Index = r.start
-			entries <- logEntry
-			r.start++
-		}
-		if r.start > r.end {
-			// Only complete if we actually got all the leaves we were
-			// expecting -- Logs MAY return fewer than the number of
-			// leaves requested.
-			success = true
-		}
-	}
-	return nil
-}
-
-// Worker function for fetcher jobs.
-// Accepts cert ranges to fetch over the |ranges| channel, and if the fetch is
-// successful sends the individual LeafInputs out into the
-// |entries| channel for the processors to chew on.
-// Will retry failed attempts to retrieve ranges indefinitely.
-// Sends true over the |done| channel when the |ranges| channel is closed.
-/* disabled becuase error handling is broken
-func (s *Scanner) fetcherJob(id int, ranges <-chan fetchRange, entries chan<- ct.LogEntry, wg *sync.WaitGroup) {
-	for r := range ranges {
-		s.fetch(r, entries, nil)
-	}
-	wg.Done()
-}
-*/
-
-// Returns the smaller of |a| and |b|
-func min(a int64, b int64) int64 {
-	if a < b {
-		return a
-	} else {
-		return b
-	}
-}
-
-// Returns the larger of |a| and |b|
-func max(a int64, b int64) int64 {
-	if a > b {
-		return a
-	} else {
-		return b
-	}
-}
-
-// Pretty prints the passed in number of |seconds| into a more human readable
-// string.
-func humanTime(seconds int) string {
-	nanos := time.Duration(seconds) * time.Second
-	hours := int(nanos / (time.Hour))
-	nanos %= time.Hour
-	minutes := int(nanos / time.Minute)
-	nanos %= time.Minute
-	seconds = int(nanos / time.Second)
-	s := ""
-	if hours > 0 {
-		s += fmt.Sprintf("%d hours ", hours)
-	}
-	if minutes > 0 {
-		s += fmt.Sprintf("%d minutes ", minutes)
-	}
-	if seconds > 0 {
-		s += fmt.Sprintf("%d seconds ", seconds)
-	}
-	return s
-}
-
-func (s Scanner) Log(msg string) {
-	if !s.opts.Quiet {
-		log.Print(msg)
-	}
-}
-
-func (s Scanner) Warn(msg string) {
-	log.Print(msg)
-}
-
-func (s *Scanner) GetSTH() (*ct.SignedTreeHead, error) {
-	latestSth, err := s.logClient.GetSTH()
-	if err != nil {
-		return nil, err
-	}
-	if s.publicKey != nil {
-		verifier, err := ct.NewSignatureVerifier(s.publicKey)
-		if err != nil {
-			return nil, err
-		}
-		if err := verifier.VerifySTHSignature(*latestSth); err != nil {
-			return nil, errors.New("STH signature is invalid: " + err.Error())
-		}
-	}
-	copy(latestSth.LogID[:], s.LogId)
-	return latestSth, nil
-}
-
-func (s *Scanner) CheckConsistency(first *ct.SignedTreeHead, second *ct.SignedTreeHead) (bool, error) {
-	if first.TreeSize < second.TreeSize {
-		proof, err := s.logClient.GetConsistencyProof(int64(first.TreeSize), int64(second.TreeSize))
-		if err != nil {
-			return false, err
-		}
-		return VerifyConsistencyProof(proof, first, second), nil
-	} else if first.TreeSize > second.TreeSize {
-		proof, err := s.logClient.GetConsistencyProof(int64(second.TreeSize), int64(first.TreeSize))
-		if err != nil {
-			return false, err
-		}
-		return VerifyConsistencyProof(proof, second, first), nil
-	} else {
-		// There is no need to ask the server for a consistency proof if the trees
-		// are the same size, and the DigiCert log returns a 400 error if we try.
-		return bytes.Equal(first.SHA256RootHash[:], second.SHA256RootHash[:]), nil
-	}
-}
-
-func (s *Scanner) MakeCollapsedMerkleTree(sth *ct.SignedTreeHead) (*CollapsedMerkleTree, error) {
-	if sth.TreeSize == 0 {
-		return &CollapsedMerkleTree{}, nil
-	}
-
-	entries, err := s.logClient.GetEntries(int64(sth.TreeSize-1), int64(sth.TreeSize-1))
-	if err != nil {
-		return nil, err
-	}
-	if len(entries) == 0 {
-		return nil, fmt.Errorf("Log did not return entry %d", sth.TreeSize-1)
-	}
-	leafHash := hashLeaf(entries[0].LeafBytes)
-
-	var tree *CollapsedMerkleTree
-	if sth.TreeSize > 1 {
-		auditPath, _, err := s.logClient.GetAuditProof(leafHash, sth.TreeSize)
-		if err != nil {
-			return nil, err
-		}
-		reverseHashes(auditPath)
-		tree, err = NewCollapsedMerkleTree(auditPath, sth.TreeSize-1)
-		if err != nil {
-			return nil, fmt.Errorf("Error returned bad audit proof for %x to %d", leafHash, sth.TreeSize)
-		}
-	} else {
-		tree = EmptyCollapsedMerkleTree()
-	}
-
-	tree.Add(leafHash)
-	if !bytes.Equal(tree.CalculateRoot(), sth.SHA256RootHash[:]) {
-		return nil, fmt.Errorf("Calculated root hash does not match signed tree head at size %d", sth.TreeSize)
-	}
-
-	return tree, nil
-}
-
-func (s *Scanner) Scan(startIndex int64, endIndex int64, processCert ProcessCallback, tree *CollapsedMerkleTree) error {
-	s.Log("Starting scan...")
-
-	s.certsProcessed = 0
-	startTime := time.Now()
-	/* TODO: only launch ticker goroutine if in verbose mode; kill the goroutine when the scanner finishes
-	ticker := time.NewTicker(time.Second)
-	go func() {
-		for range ticker.C {
-			throughput := float64(s.certsProcessed) / time.Since(startTime).Seconds()
-			remainingCerts := int64(endIndex) - int64(startIndex) - s.certsProcessed
-			remainingSeconds := int(float64(remainingCerts) / throughput)
-			remainingString := humanTime(remainingSeconds)
-			s.Log(fmt.Sprintf("Processed: %d certs (to index %d). Throughput: %3.2f ETA: %s", s.certsProcessed,
-				startIndex+int64(s.certsProcessed), throughput, remainingString))
-		}
-	}()
-	*/
-
-	// Start processor workers
-	jobs := make(chan ct.LogEntry, 100)
-	var processorWG sync.WaitGroup
-	for w := 0; w < s.opts.NumWorkers; w++ {
-		processorWG.Add(1)
-		go s.processerJob(w, jobs, processCert, &processorWG)
-	}
-
-	for start := startIndex; start < int64(endIndex); {
-		end := min(start+int64(s.opts.BatchSize), int64(endIndex)) - 1
-		if err := s.fetch(fetchRange{start, end}, jobs, tree); err != nil {
-			return err
-		}
-		start = end + 1
-	}
-	close(jobs)
-	processorWG.Wait()
-	s.Log(fmt.Sprintf("Completed %d certs in %s", s.certsProcessed, humanTime(int(time.Since(startTime).Seconds()))))
-
-	return nil
-}
-
-// Creates a new Scanner instance using |client| to talk to the log, and taking
-// configuration options from |opts|.
-func NewScanner(logUri string, logId []byte, publicKey crypto.PublicKey, opts *ScannerOptions) *Scanner {
-	var scanner Scanner
-	scanner.LogUri = logUri
-	scanner.LogId = logId
-	scanner.publicKey = publicKey
-	scanner.logClient = client.New(logUri)
-	scanner.opts = *opts
-	return &scanner
-}

sct.go

@@ -0,0 +1,46 @@
+// Copyright (C) 2017 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"software.sslmate.com/src/certspotter/ct"
+)
+
+func VerifyX509SCT(sct *ct.SignedCertificateTimestamp, cert []byte, verify *ct.SignatureVerifier) error {
+	entry := ct.LogEntry{
+		Leaf: ct.MerkleTreeLeaf{
+			Version:  0,
+			LeafType: ct.TimestampedEntryLeafType,
+			TimestampedEntry: ct.TimestampedEntry{
+				Timestamp:  sct.Timestamp,
+				EntryType:  ct.X509LogEntryType,
+				X509Entry:  cert,
+				Extensions: sct.Extensions,
+			},
+		},
+	}
+	return verify.VerifySCTSignature(*sct, entry)
+}
+
+func VerifyPrecertSCT(sct *ct.SignedCertificateTimestamp, precert ct.PreCert, verify *ct.SignatureVerifier) error {
+	entry := ct.LogEntry{
+		Leaf: ct.MerkleTreeLeaf{
+			Version:  0,
+			LeafType: ct.TimestampedEntryLeafType,
+			TimestampedEntry: ct.TimestampedEntry{
+				Timestamp:  sct.Timestamp,
+				EntryType:  ct.PrecertLogEntryType,
+				PrecertEntry:  precert,
+				Extensions: sct.Extensions,
+			},
+		},
+	}
+	return verify.VerifySCTSignature(*sct, entry)
+}

x509.go

@@ -11,6 +11,7 @@ package certspotter
 
 import (
 	"bytes"
+	"crypto/x509/pkix"
 	"encoding/asn1"
 	"errors"
 	"fmt"
@@ -83,6 +84,10 @@ func ParseRDNSequence(rdnsBytes []byte) (RDNSequence, error) {
 	return rdns, nil
 }
 
+func MarshalRDNSequence(rdns RDNSequence) ([]byte, error) {
+	return asn1.Marshal(rdns)
+}
+
 type TBSCertificate struct {
 	Raw asn1.RawContent
 
@@ -315,7 +320,7 @@ func (tbs *TBSCertificate) ParseSubjectAltNames() ([]SubjectAltName, error) {
 
 	for _, sanExt := range tbs.GetExtension(oidExtensionSubjectAltName) {
 		var err error
-		sans, err = parseSANExtension(sans, sanExt.Value)
+		sans, err = ParseSANExtension(sans, sanExt.Value)
 		if err != nil {
 			return nil, err
 		}
@@ -352,7 +357,27 @@ func (cert *Certificate) ParseTBSCertificate() (*TBSCertificate, error) {
 	return ParseTBSCertificate(cert.GetRawTBSCertificate())
 }
 
-func parseSANExtension(sans []SubjectAltName, value []byte) ([]SubjectAltName, error) {
+func (cert *Certificate) ParseSignatureAlgorithm() (*pkix.AlgorithmIdentifier, error) {
+	signatureAlgorithm := new(pkix.AlgorithmIdentifier)
+	if rest, err := asn1.Unmarshal(cert.SignatureAlgorithm.FullBytes, signatureAlgorithm); err != nil {
+		return nil, errors.New("failed to parse signature algorithm: " + err.Error())
+	} else if len(rest) > 0 {
+		return nil, fmt.Errorf("trailing data after signature algorithm: %v", rest)
+	}
+	return signatureAlgorithm, nil
+}
+
+func (cert *Certificate) ParseSignatureValue() ([]byte, error) {
+	var signatureValue asn1.BitString
+	if rest, err := asn1.Unmarshal(cert.SignatureValue.FullBytes, &signatureValue); err != nil {
+		return nil, errors.New("failed to parse signature value: " + err.Error())
+	} else if len(rest) > 0 {
+		return nil, fmt.Errorf("trailing data after signature value: %v", rest)
+	}
+	return signatureValue.RightAlign(), nil
+}
+
+func ParseSANExtension(sans []SubjectAltName, value []byte) ([]SubjectAltName, error) {
 	var seq asn1.RawValue
 	if rest, err := asn1.Unmarshal(value, &seq); err != nil {
 		return nil, errors.New("failed to parse subjectAltName extension: " + err.Error())