CHANGELOG.md

@@ -1,196 +0,0 @@
-# Change Log
-
-## v0.18.0 (2023-11-13)
-- Fix bug with downloading entries that did not materialize in practice
-  with any of the current logs.
-- Include `Message-ID` and `Date` in outbound emails.
-
-## v0.17.0 (2023-10-26)
-- Allow sendmail path to be configured with `$SENDMAIL_PATH`.
-- Minor improvements to documentation, efficiency.
-
-## v0.16.0 (2023-02-21)
-- Write malformed certs and failed healthchecks to filesystem so scripts
-  can access them.
-- Automatically execute scripts under `$CERTSPOTTER_CONFIG_DIR/hooks.d`
-  if it exists.
-- Automatically email addresses listed in `$CERTSPOTTER_CONFIG_DIR/email_recipients`
-  if it exists.
-
-## v0.15.1 (2023-02-09)
-- Fix some typos in help and error messages.
-- Allow version to be set via linker flag, to facilitate distro package building.
-
-## v0.15.0 (2023-02-08)
-- **Significant behavior change**: certspotter is now intended to run as
-  a daemon instead of a cron job.  Specifically, certspotter no longer
-  terminates unless it receives SIGTERM or SIGINT or there is a serious error.
-  You should remove certspotter from your crontab and arrange to run it as a
-  daemon, passing either the `-email` option or `-script` option to configure
-  how you want to be notified about certificates.
-
-  Reason for this change: although using cron made sense in the early days of
-  Certificate Transparency, certspotter now needs to run continuously to reliably
-  keep up with the high growth rate of contemporary CT logs, and to gracefully
-  handle the many transient errors that can arise when monitoring CT.
-  See <https://github.com/SSLMate/certspotter/issues/63> for background.
-
-- `-script` is now officially supported and can be used to execute
-  a command when a certificate is discovered or there is an error.  For details,
-  see the [certspotter-script(8) man page](man/certspotter-script.md).
-
-  Note the following changes from the experimental, undocumented `-script`
-  option found in previous versions:
-  - The script is also executed when there is an error.  Consult the `$EVENT`
-    variable to determine why the script was executed.
-  - The `$DNS_NAMES` and `$IP_ADDRESSES` variables have been removed because
-    the OS limits the size of environment variables and some certificates have
-    too many identifiers.  To determine a certificate's identifiers, you can
-    read the JSON file specified by the `$JSON_FILENAME` variable, as explained
-    in the [certspotter-script(8) man page](man/certspotter-script.md).
-  - The `$CERT_TYPE` variable has been removed because it is almost always
-    a serious mistake (that can make you miss malicious certificates) to treat
-    certificates and precertificates differently.  If you are currently
-    using this variable to skip precertificates, stop doing that because
-    precertificates imply the existence of a corresponding certificate that you
-    **might not** be separately notified about.  For more details, see
-    <https://github.com/SSLMate/certspotter/commit/cd2bb429fc2f4060a33ec8eb8b71a3eb12e9ba93>.
-  - New variable `$WATCH_ITEM` contains the first watch list item which
-    matched the certificate.
-
-- New `-email` option can be used to send an email when a certificate is
-  discovered or there is an error.  Your system must have a working `sendmail`
-  command.
-
-- (Behavior change) You must specify the `-stdout` option if you want discovered
-  certificates to be written to stdout.  This only makes sense when running
-  certspotter from the terminal; when running as a daemon you probably want to
-  use `-email` or `-script` instead.
-
-- Once a day, certspotter will send you a notification (per `-email` or
-  `-script`) if any problems are preventing it from detecting all certificates.
-  As in previous versions of certspotter, errors are written to stderr when they
-  occur, but since most errors are transient, you can now ignore stderr and rely
-  on the daily health check to notify you about any persistent problems that
-  require your attention.
-
-- certspotter now saves `.json` and `.txt` files alongside the `.pem` files
-  containing parsed representations of the certificate.
-
-- `.pem` files no longer have `.cert` or `.precert` in the filename.
-
-- certspotter will save its state periodically, and before terminating due to
-  SIGTERM or SIGINT, meaning it can resume monitoring without having to
-  re-download entries it has already processed.
-
-- The experimental "BygoneSSL" feature has been removed due to limited utility.
-
-- The `-num_workers` option has been removed.
-
-- The `-all_time` option has been removed. You can remove the certspotter state
-  directory if you want to re-download all entries.
-
-- The minimum supported Go version is now 1.19.
-
-## v0.14.0 (2022-06-13)
-- Switch to Go module versioning conventions.
-
-## v0.13 (2022-06-13)
-- Reduce minimum Go version to 1.17.
-- Update install instructions.
-
-## v0.12 (2022-06-07)
-- Retry failed log requests.  This should make certspotter resilient
-  to rate limiting by logs.
-- Add `-version` flag.
-- Eliminate unnecessary dependency. certspotter now depends only on
-  golang.org/x packages.
-- Switch to Go modules.
-
-## v0.11 (2021-08-17)
-- Add support for contacting logs via HTTP proxies;
-  just set the appropriate environment variable as documented at
-  <https://golang.org/pkg/net/http/#ProxyFromEnvironment>.
-- Work around RFC 6962 ambiguity related to consistency proofs
-  for empty trees.
-
-## v0.10 (2020-04-29)
-- Improve speed by processing logs in parallel
-- Add `-start_at_end` option to begin monitoring new logs at the end,
-  which significantly speeds up Cert Spotter, at the cost of missing
-  certificates that were added to a log before Cert Spotter starts
-  monitoring it
-- (Behavior change) Scan logs in their entirety the first time Cert
-  Spotter is run, unless `-start_at_end` specified (behavior change)
-- The log list is now retrieved from certspotter.org at startup instead
-  of being embedded in the source. This will allow Cert Spotter to react
-  more quickly to the frequent changes in logs.
-- (Behavior change) the `-logs` option now expects a JSON file in the v2
-  log list format. See <https://www.certificate-transparency.org/known-logs>
-  and <https://www.gstatic.com/ct/log_list/v2/log_list_schema.json>.
-- `-logs` now accepts an HTTPS URL in addition to a file path.
-- (Behavior change) the `-underwater` option has been removed. If you want
-  its behavior, specify `https://loglist.certspotter.org/underwater.json` to
-  the `-logs` option.
-
-## v0.9 (2018-04-19)
-- Add Cloudflare Nimbus logs
-- Remove Google Argon 2017 log
-- Remove WoSign and StartCom logs due to disqualification by Chromium
-  and extended downtime
-
-## v0.8 (2017-12-08)
-- Add Symantec Sirius log
-- Add DigiCert 2 log
-
-## v0.7 (2017-11-13)
-- Add Google Argon logs
-- Fix bug that caused crash on 32 bit architectures
-
-## v0.6 (2017-10-19)
-- Add Comodo Mammoth and Comodo Sabre logs
-- Minor bug fixes and improvements
-
-## v0.5 (2017-05-18)
-- Remove PuChuangSiDa 1 log due to excessive downtime and presumptive
-  disqualification from Chrome
-- Add Venafi Gen2 log
-- Improve monitoring robustness under certain pathological behavior
-  by logs
-- Minor documentation improvements
-
-## v0.4 (2017-04-03)
-- Add PuChuangSiDa 1 log
-- Remove Venafi log due to fork and disqualification from Chrome
-
-## v0.3 (2017-02-20)
-- Revise `-all_time` flag (behavior change):
-  - If `-all_time` is specified, scan the entirety of all logs, even
-    existing logs
-  - When a new log is added, scan it in its entirety even if `-all_time`
-    is not specified
-- Add new logs:
-  - Google Icarus
-  - Google Skydiver
-  - StartCom
-  - WoSign
-- Overhaul log processing and auditing logic:
-  - STHs are never deleted unless they can be verified
-  - Multiple unverified STHs can be queued per log, laying groundwork
-    for STH pollination support
-  - New state directory layout; current state directories will be
-    migrated, but migration will be removed in a future version
-  - Persist condensed Merkle Tree state between runs, instead of
-    reconstructing it from consistency proof every time
-- Use a lock file to prevent multiple instances of Cert Spotter from
-  running concurrently (which could clobber the state directory).
-- Minor bug fixes and improvements
-
-## v0.2 (2016-08-25)
-- Suppress duplicate identifiers in output.
-- Fix "EOF" error when running under Go 1.7.
-- Fix bug where hook script could fail silently.
-- Fix compilation under Go 1.5.
-
-## v0.1 (2016-07-27)
-- Initial release.

NEWS

@@ -0,0 +1,87 @@
+v0.11 (2021-08-17)
+  * Add support for contacting logs via HTTP proxies;
+    just set the appropriate environment variable as documented at
+    https://golang.org/pkg/net/http/#ProxyFromEnvironment
+  * Work around RFC 6962 ambiguity related to consistency proofs
+    for empty trees.
+
+v0.10 (2020-04-29)
+  * Improve speed by processing logs in parallel
+  * Add -start_at_end option to begin monitoring new logs at the end,
+    which significantly speeds up Cert Spotter, at the cost of missing
+    certificates that were added to a log before Cert Spotter starts
+    monitoring it
+  * (Behavior change) Scan logs in their entirety the first time Cert
+    Spotter is run, unless -start_at_end specified (behavior change)
+  * The log list is now retrieved from certspotter.org at startup instead
+    of being embedded in the source. This will allow Cert Spotter to react
+    more quickly to the frequent changes in logs.
+  * (Behavior change) the -logs option now expects a JSON file in the v2
+    log list format. See <https://www.certificate-transparency.org/known-logs>
+    and <https://www.gstatic.com/ct/log_list/v2/log_list_schema.json>.
+  * -logs now accepts an HTTPS URL in addition to a file path.
+  * (Behavior change) the -underwater option has been removed. If you want
+    its behavior, specify https://loglist.certspotter.org/underwater.json to
+    the -logs option.
+
+v0.9 (2018-04-19)
+  * Add Cloudflare Nimbus logs
+  * Remove Google Argon 2017 log
+  * Remove WoSign and StartCom logs due to disqualification by Chromium
+    and extended downtime
+
+v0.8 (2017-12-08)
+  * Add Symantec Sirius log
+  * Add DigiCert 2 log
+
+v0.7 (2017-11-13)
+  * Add Google Argon logs
+  * Fix bug that caused crash on 32 bit architectures
+
+v0.6 (2017-10-19)
+  * Add Comodo Mammoth and Comodo Sabre logs
+  * Minor bug fixes and improvements
+
+v0.5 (2017-05-18)
+  * Remove PuChuangSiDa 1 log due to excessive downtime and presumptive
+    disqualification from Chrome
+  * Add Venafi Gen2 log
+  * Improve monitoring robustness under certain pathological behavior
+    by logs
+  * Minor documentation improvements
+
+v0.4 (2017-04-03)
+  * Add PuChuangSiDa 1 log
+  * Remove Venafi log due to fork and disqualification from Chrome
+
+v0.3 (2017-02-20)
+  * Revise -all_time flag (behavior change):
+    - If -all_time is specified, scan the entirety of all logs, even
+      existing logs
+    - When a new log is added, scan it in its entirety even if -all_time
+      is not specified
+  * Add new logs:
+    - Google Icarus
+    - Google Skydiver
+    - StartCom
+    - WoSign
+  * Overhaul log processing and auditing logic:
+    - STHs are never deleted unless they can be verified
+    - Multiple unverified STHs can be queued per log, laying groundwork
+      for STH pollination support
+    - New state directory layout; current state directories will be
+      migrated, but migration will be removed in a future version
+    - Persist condensed Merkle Tree state between runs, instead of
+      reconstructing it from consistency proof every time
+  * Use a lock file to prevent multiple instances of Cert Spotter from
+    running concurrently (which could clobber the state directory).
+  * Minor bug fixes and improvements
+
+v0.2 (2016-08-25)
+  * Suppress duplicate identifiers in output.
+  * Fix "EOF" error when running under Go 1.7.
+  * Fix bug where hook script could fail silently.
+  * Fix compilation under Go 1.5.
+
+v0.1 (2016-07-27)
+  * Initial release.

README

@@ -0,0 +1,162 @@
+Cert Spotter is a Certificate Transparency log monitor from SSLMate that
+alerts you when a SSL/TLS certificate is issued for one of your domains.
+Cert Spotter is easier than other open source CT monitors, since it does
+not require a database.  It's also more robust, since it uses a special
+certificate parser that ensures it won't miss certificates.
+
+Cert Spotter is also available as a hosted service by SSLMate that
+requires zero setup and provides an easy web dashboard to centrally
+manage your certificates.  Visit <https://sslmate.com/certspotter>
+to sign up.
+
+You can use Cert Spotter to detect:
+
+* Certificates issued to attackers who have compromised your DNS and
+  are redirecting your visitors to their malicious site.
+
+* Certificates issued to attackers who have taken over an abandoned
+  sub-domain in order to serve malware under your name.
+
+* Certificates issued to attackers who have compromised a certificate
+  authority and want to impersonate your site.
+
+* Certificates issued in violation of your corporate policy
+  or outside of your centralized certificate procurement process.
+
+
+USING CERT SPOTTER
+
+The easiest way to use Cert Spotter is to sign up for an account at
+<https://sslmate.com/certspotter>.  If you want to run Cert Spotter on
+your own server, follow these instructions.
+
+Cert Spotter requires Go version 1.5 or higher.
+
+1. Install Cert Spotter using go get:
+
+	go get software.sslmate.com/src/certspotter/cmd/certspotter
+
+2. Create a file called ~/.certspotter/watchlist listing the DNS names
+   you want to monitor, one per line.  To monitor an entire domain tree
+   (including the domain itself and all sub-domains) prefix the domain
+   name with a dot (e.g. ".example.com").  To monitor a single DNS name
+   only, do not prefix the name with a dot.
+
+3. Create a cron job to periodically run `certspotter`.  See below for
+   command line options.
+
+Every time you run Cert Spotter, it scans all browser-recognized
+Certificate Transparency logs for certificates matching domains on
+your watch list.  When Cert Spotter detects a matching certificate, it
+writes a report to standard out, which the Cron daemon emails to you.
+Make sure you are able to receive emails sent by Cron.
+
+Cert Spotter also saves a copy of matching certificates in
+~/.certspotter/certs (unless you specify the -no_save option).
+
+When Cert Spotter has previously monitored a log, it scans the log
+from the previous position, to avoid downloading the same log entry
+more than once.  (To override this behavior and scan all logs from the
+beginning, specify the -all_time option.)
+
+When Cert Spotter has not previously monitored a log, it can either start
+monitoring the log from the beginning, or seek to the end of the log and
+start monitoring from there.  Monitoring from the beginning guarantees
+detection of all certificates, but requires downloading hundreds of
+millions of certificates, which takes days.  The default behavior is to
+monitor from the beginning.  To start monitoring new logs from the end,
+specify the -start_at_end option.
+
+You can add and remove domains on your watchlist at any time.  However,
+the certspotter command only notifies you of certificates that were
+logged since adding a domain to the watchlist, unless you specify the
+-all_time option, which requires scanning the entirety of every log
+and takes many days to complete with a fast Internet connection.
+To examine preexisting certificates, it's better to use the Cert
+Spotter service <https://sslmate.com/certspotter>, the Cert Spotter
+API <https://sslmate.com/certspotter/api>, or a CT search engine such
+as <https://crt.sh>.
+
+
+COMMAND LINE FLAGS
+
+  -watchlist FILENAME
+	File containing identifiers to watch, one per line, as described
+	above (use - to read from stdin).  Default: ~/.certspotter/watchlist
+  -no_save
+	Do not save a copy of matching certificates.
+  -start_at_end
+	Start monitoring logs from the end, rather than the beginning.
+	This significantly reduces the time to run Cert Spotter, but
+	you will miss certificates that were added to a log before Cert
+	Spotter started monitoring it.
+  -all_time
+	Scan for certificates from all time, not just those logged since
+	the previous run of Cert Spotter.
+  -logs FILENAME_OR_URL
+	Filename of HTTPS URL of a JSON file containing logs to monitor, in the format
+	documented at <https://www.certificate-transparency.org/known-logs>.
+	Default: https://loglist.certspotter.org/monitor.json which includes the union
+	of active logs recognized by Chrome and Apple.
+  -state_dir PATH
+	Directory for storing state. Default: ~/.certspotter
+  -verbose
+	Be verbose.
+
+
+WHAT CERTIFICATES ARE DETECTED BY CERT SPOTTER?
+
+Any certificate that is logged to a Certificate Transparency log trusted
+by Chromium will be detected by Cert Spotter.  All certificates issued
+after April 30, 2018 must be logged to such a log to be trusted by Chromium.
+
+Generally, certificate authorities will automatically submit certificates
+to logs so that they will work in Chromium.  In addition, certificates
+that are discovered during Internet-wide scans are submitted to Certificate
+Transparency logs.
+
+
+SECURITY
+
+Cert Spotter assumes an adversarial model in which an attacker produces
+a certificate that is accepted by at least some clients but goes
+undetected because of an encoding error that prevents CT monitors from
+understanding it.  To defend against this attack, Cert Spotter uses a
+special certificate parser that keeps the certificate unparsed except
+for the identifiers.  If one of the identifiers matches a domain on your
+watchlist, you will be notified, even if other parts of the certificate
+are unparsable.
+
+Cert Spotter takes special precautions to ensure identifiers are parsed
+correctly, and implements defenses against identifier-based attacks.
+For instance, if a DNS identifier contains a null byte, Cert Spotter
+interprets it as two identifiers: the complete identifier, and the
+identifier formed by truncating at the first null byte.  For example, a
+certificate for example.org\0.example.com will alert the owners of both
+example.org and example.com.  This defends against null prefix attacks
+<http://www.thoughtcrime.org/papers/null-prefix-attacks.pdf>.
+
+SSLMate continuously monitors CT logs to make sure every certificate's
+identifiers can be successfully parsed, and will release updates to
+Cert Spotter as necessary to fix parsing failures.
+
+Cert Spotter understands wildcard and redacted DNS names, and will alert
+you if a wildcard or redacted certificate might match an identifier on
+your watchlist.  For example, a watchlist entry for sub.example.com would
+match certificates for *.example.com or ?.example.com.
+
+Cert Spotter is not just a log monitor, but also a log auditor which
+checks that the log is obeying its append-only property.  A future
+release of Cert Spotter will support gossiping with other log monitors
+to ensure the log is presenting a single view.
+
+
+BygoneSSL
+
+Cert Spotter can also notify users of bygone SSL certificates, which are SSL
+certificates that outlived their prior domain owner's registration into the
+next owners registration. To detect these certificates add a valid_at
+argument to each domain in the watchlist followed by the date the domain was
+registered in the following format YYYY-MM-DD. For example:
+example.com valid_at:2014-05-02
+

README.md

@@ -1,105 +0,0 @@
-# Cert Spotter - Certificate Transparency Monitor
-
-**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that
-alerts you when an SSL/TLS certificate is issued for one of your domains.
-Cert Spotter is easier to use than other open source CT monitors, since it does not require
-a database. It's also more robust, since it uses a special certificate parser
-that ensures it won't miss certificates.
-
-Cert Spotter is also available as a hosted service by SSLMate that
-requires zero setup and provides an easy web dashboard to centrally
-manage your certificates.  Visit <https://sslmate.com/certspotter>
-to sign up.
-
-You can use Cert Spotter to detect:
-
- * Certificates issued to attackers who have compromised your DNS and
-   are redirecting your visitors to their malicious site.
- * Certificates issued to attackers who have taken over an abandoned
-   sub-domain in order to serve malware under your name.
- * Certificates issued to attackers who have compromised a certificate
-   authority and want to impersonate your site.
- * Certificates issued in violation of your corporate policy
-   or outside of your centralized certificate procurement process.
-
-## Quickstart
-
-Cert Spotter requires Go version 1.19 or higher.
-
-1. Install the certspotter command using the `go` command:
-
-   ```
-   go install software.sslmate.com/src/certspotter/cmd/certspotter@latest
-   ```
-
-2. Create a watch list file `$HOME/.certspotter/watchlist` containing the DNS names you want to monitor,
-   one per line.  To monitor an entire domain tree (including the domain itself
-   and all sub-domains) prefix the domain name with a dot (e.g. `.example.com`).
-   To monitor a single DNS name only, do not prefix the name with a dot.
-
-3. Place one or more email addresses in the `$HOME/.certspotter/email_recipients`
-   file (one per line), and/or place one or more executable scripts in the
-   `$HOME/.certspotter/hooks.d` directory.  certspotter will email the listed
-   addresses (requires your system to have a working sendmail command) and
-   execute the provided scripts when it detects a certificate for a domain on
-   your watch list.
-
-4. Configure your system to run `certspotter` as a daemon.  You may want to specify
-   the `-start_at_end` command line option to tell certspotter to start monitoring
-   new logs at the end instead of the beginning.  This saves significant bandwidth, but
-   you won't be notified about certificates which were logged before you started
-   using certspotter.
-
-## Documentation
-
-* Command line options and operational details: [certspotter(8) man page](man/certspotter.md)
-* The script interface: [certspotter-script(8) man page](man/certspotter-script.md)
-* [Change Log](CHANGELOG.md)
-
-## What certificates are detected by Cert Spotter?
-
-In the default configuration, any certificate that is logged to a Certificate
-Transparency log recognized by Google Chrome or Apple will be detected by
-Cert Spotter.  By default, Google Chrome and Apple only accept certificates that
-are logged, so any certificate that works in Chrome or Safari will be detected
-by Cert Spotter.
-
-## Security
-
-Cert Spotter assumes an adversarial model in which an attacker produces
-a certificate that is accepted by at least some clients but goes
-undetected because of an encoding error that prevents CT monitors from
-understanding it.  To defend against this attack, Cert Spotter uses a
-special certificate parser that keeps the certificate unparsed except
-for the identifiers.  If one of the identifiers matches a domain on your
-watchlist, you will be notified, even if other parts of the certificate
-are unparsable.
-
-Cert Spotter takes special precautions to ensure identifiers are parsed
-correctly, and implements defenses against identifier-based attacks.
-For instance, if a DNS identifier contains a null byte, Cert Spotter
-interprets it as two identifiers: the complete identifier, and the
-identifier formed by truncating at the first null byte.  For example, a
-certificate for `example.org\0.example.com` will alert the owners of both
-`example.org` and `example.com`.  This defends against [null prefix attacks](
-http://www.thoughtcrime.org/papers/null-prefix-attacks.pdf).
-
-SSLMate continuously monitors CT logs to make sure every certificate's
-identifiers can be successfully parsed, and will release updates to
-Cert Spotter as necessary to fix parsing failures.
-
-Cert Spotter understands wildcard DNS names, and will alert
-you if a wildcard certificate might match an identifier on
-your watchlist.  For example, a watchlist entry for `sub.example.com` would
-match certificates for `*.example.com`.
-
-Cert Spotter is not just a log monitor, but also a log auditor which
-checks that the log is obeying its append-only property.  A future
-release of Cert Spotter will support gossiping with other log monitors
-to ensure the log is presenting a single view.
-
-## Copyright
-
-Copyright © 2016-2023 Opsmate, Inc.
-
-Licensed under the [Mozilla Public License Version 2.0](LICENSE).

auditing.go

@@ -0,0 +1,213 @@
+// Copyright (C) 2016 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package certspotter
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"software.sslmate.com/src/certspotter/ct"
+)
+
+func reverseHashes(hashes []ct.MerkleTreeNode) {
+	for i := 0; i < len(hashes)/2; i++ {
+		j := len(hashes) - i - 1
+		hashes[i], hashes[j] = hashes[j], hashes[i]
+	}
+}
+
+func VerifyConsistencyProof(proof ct.ConsistencyProof, first *ct.SignedTreeHead, second *ct.SignedTreeHead) bool {
+	// TODO: make sure every hash in proof is right length? otherwise input to hashChildren is ambiguous
+	if second.TreeSize < first.TreeSize {
+		// Can't be consistent if tree got smaller
+		return false
+	}
+	if first.TreeSize == second.TreeSize {
+		if !(bytes.Equal(first.SHA256RootHash[:], second.SHA256RootHash[:]) && len(proof) == 0) {
+			return false
+		}
+		return true
+	}
+	if first.TreeSize == 0 {
+		// The purpose of the consistency proof is to ensure the append-only
+		// nature of the tree; i.e. that the first tree is a "prefix" of the
+		// second tree.  If the first tree is empty, then it's trivially a prefix
+		// of the second tree, so no proof is needed.
+		if len(proof) != 0 {
+			return false
+		}
+		return true
+	}
+	// Guaranteed that 0 < first.TreeSize < second.TreeSize
+
+	node := first.TreeSize - 1
+	lastNode := second.TreeSize - 1
+
+	// While we're the right child, everything is in both trees, so move one level up.
+	for node%2 == 1 {
+		node /= 2
+		lastNode /= 2
+	}
+
+	var newHash ct.MerkleTreeNode
+	var oldHash ct.MerkleTreeNode
+	if node > 0 {
+		if len(proof) == 0 {
+			return false
+		}
+		newHash = proof[0]
+		proof = proof[1:]
+	} else {
+		// The old tree was balanced, so we already know the first hash to use
+		newHash = first.SHA256RootHash[:]
+	}
+	oldHash = newHash
+
+	for node > 0 {
+		if node%2 == 1 {
+			// node is a right child; left sibling exists in both trees
+			if len(proof) == 0 {
+				return false
+			}
+			newHash = hashChildren(proof[0], newHash)
+			oldHash = hashChildren(proof[0], oldHash)
+			proof = proof[1:]
+		} else if node < lastNode {
+			// node is a left child; rigth sibling only exists in the new tree
+			if len(proof) == 0 {
+				return false
+			}
+			newHash = hashChildren(newHash, proof[0])
+			proof = proof[1:]
+		} // else node == lastNode: node is a left child with no sibling in either tree
+		node /= 2
+		lastNode /= 2
+	}
+
+	if !bytes.Equal(oldHash, first.SHA256RootHash[:]) {
+		return false
+	}
+
+	// If trees have different height, continue up the path to reach the new root
+	for lastNode > 0 {
+		if len(proof) == 0 {
+			return false
+		}
+		newHash = hashChildren(newHash, proof[0])
+		proof = proof[1:]
+		lastNode /= 2
+	}
+
+	if !bytes.Equal(newHash, second.SHA256RootHash[:]) {
+		return false
+	}
+
+	return true
+}
+
+func hashNothing() ct.MerkleTreeNode {
+	return sha256.New().Sum(nil)
+}
+
+func hashLeaf(leafBytes []byte) ct.MerkleTreeNode {
+	hasher := sha256.New()
+	hasher.Write([]byte{0x00})
+	hasher.Write(leafBytes)
+	return hasher.Sum(nil)
+}
+
+func hashChildren(left ct.MerkleTreeNode, right ct.MerkleTreeNode) ct.MerkleTreeNode {
+	hasher := sha256.New()
+	hasher.Write([]byte{0x01})
+	hasher.Write(left)
+	hasher.Write(right)
+	return hasher.Sum(nil)
+}
+
+type CollapsedMerkleTree struct {
+	nodes []ct.MerkleTreeNode
+	size  uint64
+}
+
+func calculateNumNodes(size uint64) int {
+	numNodes := 0
+	for size > 0 {
+		numNodes += int(size & 1)
+		size >>= 1
+	}
+	return numNodes
+}
+func EmptyCollapsedMerkleTree() *CollapsedMerkleTree {
+	return &CollapsedMerkleTree{}
+}
+func NewCollapsedMerkleTree(nodes []ct.MerkleTreeNode, size uint64) (*CollapsedMerkleTree, error) {
+	if len(nodes) != calculateNumNodes(size) {
+		return nil, errors.New("NewCollapsedMerkleTree: nodes has incorrect size")
+	}
+	return &CollapsedMerkleTree{nodes: nodes, size: size}, nil
+}
+func CloneCollapsedMerkleTree(source *CollapsedMerkleTree) *CollapsedMerkleTree {
+	nodes := make([]ct.MerkleTreeNode, len(source.nodes))
+	copy(nodes, source.nodes)
+	return &CollapsedMerkleTree{nodes: nodes, size: source.size}
+}
+
+func (tree *CollapsedMerkleTree) Add(hash ct.MerkleTreeNode) {
+	tree.nodes = append(tree.nodes, hash)
+	tree.size++
+	size := tree.size
+	for size%2 == 0 {
+		left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1]
+		tree.nodes = tree.nodes[:len(tree.nodes)-2]
+		tree.nodes = append(tree.nodes, hashChildren(left, right))
+		size /= 2
+	}
+}
+
+func (tree *CollapsedMerkleTree) CalculateRoot() ct.MerkleTreeNode {
+	if len(tree.nodes) == 0 {
+		return hashNothing()
+	}
+	i := len(tree.nodes) - 1
+	hash := tree.nodes[i]
+	for i > 0 {
+		i -= 1
+		hash = hashChildren(tree.nodes[i], hash)
+	}
+	return hash
+}
+
+func (tree *CollapsedMerkleTree) GetSize() uint64 {
+	return tree.size
+}
+
+func (tree *CollapsedMerkleTree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(map[string]interface{}{
+		"nodes": tree.nodes,
+		"size":  tree.size,
+	})
+}
+
+func (tree *CollapsedMerkleTree) UnmarshalJSON(b []byte) error {
+	var rawTree struct {
+		Nodes []ct.MerkleTreeNode `json:"nodes"`
+		Size  uint64              `json:"size"`
+	}
+	if err := json.Unmarshal(b, &rawTree); err != nil {
+		return errors.New("Failed to unmarshal CollapsedMerkleTree: " + err.Error())
+	}
+	if len(rawTree.Nodes) != calculateNumNodes(rawTree.Size) {
+		return errors.New("Failed to unmarshal CollapsedMerkleTree: nodes has incorrect length")
+	}
+	tree.size = rawTree.Size
+	tree.nodes = rawTree.Nodes
+	return nil
+}

cmd/certspotter/main.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2016, 2023 Opsmate, Inc.
+// Copyright (C) 2016 Opsmate, Inc.
 //
 // This Source Code Form is subject to the terms of the Mozilla
 // Public License, v. 2.0. If a copy of the MPL was not distributed
@@ -11,242 +11,223 @@ package main
 
 import (
 	"bufio"
-	"context"
-	"errors"
 	"flag"
 	"fmt"
-	"io/fs"
+	"io"
 	"os"
-	"os/signal"
 	"path/filepath"
-	"runtime"
-	"runtime/debug"
 	"strings"
-	"syscall"
 	"time"
 
-	"software.sslmate.com/src/certspotter/loglist"
+	"golang.org/x/net/idna"
-	"software.sslmate.com/src/certspotter/monitor"
+
+	"software.sslmate.com/src/certspotter"
+	"software.sslmate.com/src/certspotter/cmd"
+	"software.sslmate.com/src/certspotter/ct"
 )
 
-var programName = os.Args[0]
-var Version = ""
-
-const defaultLogList = "https://loglist.certspotter.org/monitor.json"
-
-func certspotterVersion() string {
-	if Version != "" {
-		return Version + "?"
-	}
-	info, ok := debug.ReadBuildInfo()
-	if !ok {
-		return "unknown"
-	}
-	if strings.HasPrefix(info.Main.Version, "v") {
-		return info.Main.Version
-	}
-	var vcs, vcsRevision, vcsModified string
-	for _, s := range info.Settings {
-		switch s.Key {
-		case "vcs":
-			vcs = s.Value
-		case "vcs.revision":
-			vcsRevision = s.Value
-		case "vcs.modified":
-			vcsModified = s.Value
-		}
-	}
-	if vcs == "git" && vcsRevision != "" && vcsModified == "true" {
-		return vcsRevision + "+"
-	} else if vcs == "git" && vcsRevision != "" {
-		return vcsRevision
-	}
-	return "unknown"
-}
-
-func fileExists(filename string) bool {
-	_, err := os.Lstat(filename)
-	return err == nil
-}
-func homedir() string {
-	homedir, err := os.UserHomeDir()
-	if err != nil {
-		panic(fmt.Errorf("unable to determine home directory: %w", err))
-	}
-	return homedir
-}
 func defaultStateDir() string {
 	if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" {
 		return envVar
 	} else {
-		return filepath.Join(homedir(), ".certspotter")
+		return cmd.DefaultStateDir("certspotter")
 	}
 }
 func defaultConfigDir() string {
 	if envVar := os.Getenv("CERTSPOTTER_CONFIG_DIR"); envVar != "" {
 		return envVar
 	} else {
-		return filepath.Join(homedir(), ".certspotter")
+		return cmd.DefaultConfigDir("certspotter")
 	}
 }
-func defaultWatchListPath() string {
+
-	return filepath.Join(defaultConfigDir(), "watchlist")
+func trimTrailingDots(value string) string {
-}
+	length := len(value)
-func defaultWatchListPathIfExists() string {
+	for length > 0 && value[length-1] == '.' {
-	if fileExists(defaultWatchListPath()) {
+		length--
-		return defaultWatchListPath()
-	} else {
-		return ""
 	}
-}
+	return value[0:length]
-func defaultScriptDir() string {
-	return filepath.Join(defaultConfigDir(), "hooks.d")
-}
-func defaultEmailFile() string {
-	return filepath.Join(defaultConfigDir(), "email_recipients")
 }
 
-func simplifyError(err error) error {
+var stateDir = flag.String("state_dir", defaultStateDir(), "Directory for storing state")
-	var pathErr *fs.PathError
+var watchlistFilename = flag.String("watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing identifiers to watch (- for stdin)")
-	if errors.As(err, &pathErr) {
+
-		return pathErr.Err
+type watchlistItem struct {
+	Domain       []string
+	AcceptSuffix bool
+	ValidAt      *time.Time // optional
+}
+
+var watchlist []watchlistItem
+
+func parseWatchlistItem(str string) (watchlistItem, error) {
+	fields := strings.Fields(str)
+	if len(fields) == 0 {
+		return watchlistItem{}, fmt.Errorf("Empty domain")
+	}
+	domain := fields[0]
+	var validAt *time.Time = nil
+
+	// parse options
+	for i := 1; i < len(fields); i++ {
+		chunks := strings.SplitN(fields[i], ":", 2)
+		if len(chunks) != 2 {
+			return watchlistItem{}, fmt.Errorf("Missing Value `%s'", fields[i])
+		}
+		switch chunks[0] {
+		case "valid_at":
+			validAtTime, err := time.Parse("2006-01-02", chunks[1])
+			if err != nil {
+				return watchlistItem{}, fmt.Errorf("Invalid Date `%s': %s", chunks[1], err)
+			}
+			validAt = &validAtTime
+		default:
+			return watchlistItem{}, fmt.Errorf("Unknown Option `%s'", fields[i])
+		}
 	}
 
-	return err
+	// parse domain
-}
+	// "." as in root zone (matches everything)
+	if domain == "." {
+		return watchlistItem{
+			Domain:       []string{},
+			AcceptSuffix: true,
+			ValidAt:      validAt,
+		}, nil
+	}
 
-func readWatchListFile(filename string) (monitor.WatchList, error) {
+	acceptSuffix := false
-	file, err := os.Open(filename)
+	if strings.HasPrefix(domain, ".") {
+		acceptSuffix = true
+		domain = domain[1:]
+	}
+
+	asciiDomain, err := idna.ToASCII(strings.ToLower(trimTrailingDots(domain)))
 	if err != nil {
-		return nil, simplifyError(err)
+		return watchlistItem{}, fmt.Errorf("Invalid domain `%s': %s", domain, err)
 	}
-	defer file.Close()
+	return watchlistItem{
-	return monitor.ReadWatchList(file)
+		Domain:       strings.Split(asciiDomain, "."),
+		AcceptSuffix: acceptSuffix,
+		ValidAt:      validAt,
+	}, nil
 }
 
-func readEmailFile(filename string) ([]string, error) {
+func readWatchlist(reader io.Reader) ([]watchlistItem, error) {
-	file, err := os.Open(filename)
+	items := []watchlistItem{}
-	if err != nil {
+	scanner := bufio.NewScanner(reader)
-		return nil, simplifyError(err)
-	}
-	defer file.Close()
-
-	var emails []string
-	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		line := scanner.Text()
-		if line == "" {
+		if line == "" || strings.HasPrefix(line, "#") {
 			continue
 		}
-		emails = append(emails, line)
+		item, err := parseWatchlistItem(line)
+		if err != nil {
+			return nil, err
+		}
+		items = append(items, item)
 	}
-	return emails, err
+	return items, scanner.Err()
 }
 
-func appendFunc(slice *[]string) func(string) error {
+func dnsLabelMatches(certLabel string, watchLabel string) bool {
-	return func(value string) error {
+	// For fail-safe behavior, if a label was unparsable, it matches everything.
-		*slice = append(*slice, value)
+	// Similarly, redacted labels match everything, since the label _might_ be
-		return nil
+	// for a name we're interested in.
+
+	return certLabel == "*" ||
+		certLabel == "?" ||
+		certLabel == certspotter.UnparsableDNSLabelPlaceholder ||
+		certspotter.MatchesWildcard(watchLabel, certLabel)
+}
+
+func dnsNameMatches(dnsName []string, watchDomain []string, acceptSuffix bool) bool {
+	for len(dnsName) > 0 && len(watchDomain) > 0 {
+		certLabel := dnsName[len(dnsName)-1]
+		watchLabel := watchDomain[len(watchDomain)-1]
+
+		if !dnsLabelMatches(certLabel, watchLabel) {
+			return false
+		}
+
+		dnsName = dnsName[:len(dnsName)-1]
+		watchDomain = watchDomain[:len(watchDomain)-1]
+	}
+
+	return len(watchDomain) == 0 && (acceptSuffix || len(dnsName) == 0)
+}
+
+func anyDnsNameIsWatched(info *certspotter.EntryInfo) bool {
+	dnsNames := info.Identifiers.DNSNames
+	matched := false
+	for _, dnsName := range dnsNames {
+		labels := strings.Split(dnsName, ".")
+		for _, item := range watchlist {
+			if dnsNameMatches(labels, item.Domain, item.AcceptSuffix) {
+				if item.ValidAt != nil {
+					// BygoneSSL Check
+					// was the SSL certificate issued before the domain was registered
+					// and valid after
+					if item.ValidAt.Before(*info.CertInfo.NotAfter()) &&
+						item.ValidAt.After(*info.CertInfo.NotBefore()) {
+						info.Bygone = true
+						return true
+					}
+				}
+				// keep iterating in case another domain watched matches valid_at
+				matched = true
+			}
+		}
+	}
+	return matched
+}
+
+func processEntry(scanner *certspotter.Scanner, entry *ct.LogEntry) {
+	info := certspotter.EntryInfo{
+		LogUri:    scanner.LogUri,
+		Entry:     entry,
+		IsPrecert: certspotter.IsPrecert(entry),
+		FullChain: certspotter.GetFullChain(entry),
+	}
+
+	info.CertInfo, info.ParseError = certspotter.MakeCertInfoFromLogEntry(entry)
+
+	if info.CertInfo != nil {
+		info.Identifiers, info.IdentifiersParseError = info.CertInfo.ParseIdentifiers()
+	}
+
+	// Fail safe behavior: if info.Identifiers is nil (which is caused by a
+	// parse error), report the certificate because we can't say for sure it
+	// doesn't match a domain we care about.  We try very hard to make sure
+	// parsing identifiers always succeeds, so false alarms should be rare.
+	if info.Identifiers == nil || anyDnsNameIsWatched(&info) {
+		cmd.LogEntry(&info)
 	}
 }
 
 func main() {
-	loglist.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s)", certspotterVersion(), runtime.Version(), runtime.GOOS, runtime.GOARCH)
-
-	var flags struct {
-		batchSize   int // TODO-4: respect this option
-		email       []string
-		healthcheck time.Duration
-		logs        string
-		noSave      bool
-		script      string
-		startAtEnd  bool
-		stateDir    string
-		stdout      bool
-		verbose     bool
-		version     bool
-		watchlist   string
-	}
-	flag.IntVar(&flags.batchSize, "batch_size", 1000, "Max number of entries to request per call to get-entries (advanced)")
-	flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email))
-	flag.DurationVar(&flags.healthcheck, "healthcheck", 24*time.Hour, "How frequently to perform a health check")
-	flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
-	flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory")
-	flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered")
-	flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring new logs from the end rather than the beginning (saves considerable bandwidth)")
-	flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates")
-	flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout")
-	flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose")
-	flag.BoolVar(&flags.version, "version", false, "Print version and exit")
-	flag.StringVar(&flags.watchlist, "watchlist", defaultWatchListPathIfExists(), "File containing domain names to watch")
 	flag.Parse()
 
-	if flags.version {
+	if *watchlistFilename == "-" {
-		fmt.Fprintf(os.Stdout, "certspotter version %s\n", certspotterVersion())
+		var err error
-		os.Exit(0)
+		watchlist, err = readWatchlist(os.Stdin)
-	}
-	if flags.watchlist == "" {
-		fmt.Fprintf(os.Stderr, "%s: watch list not found: please create %s or specify alternative path using -watchlist\n", programName, defaultWatchListPath())
-		os.Exit(2)
-	}
-
-	fsstate := &monitor.FilesystemState{
-		StateDir:  flags.stateDir,
-		SaveCerts: !flags.noSave,
-		Script:    flags.script,
-		ScriptDir: defaultScriptDir(),
-		Email:     flags.email,
-		Stdout:    flags.stdout,
-	}
-	config := &monitor.Config{
-		LogListSource:       flags.logs,
-		State:               fsstate,
-		StartAtEnd:          flags.startAtEnd,
-		Verbose:             flags.verbose,
-		HealthCheckInterval: flags.healthcheck,
-	}
-
-	emailFileExists := false
-	if emailRecipients, err := readEmailFile(defaultEmailFile()); err == nil {
-		emailFileExists = true
-		fsstate.Email = append(fsstate.Email, emailRecipients...)
-	} else if !errors.Is(err, fs.ErrNotExist) {
-		fmt.Fprintf(os.Stderr, "%s: error reading email recipients file %q: %s\n", programName, defaultEmailFile(), err)
-		os.Exit(1)
-	}
-
-	if len(fsstate.Email) == 0 && !emailFileExists && fsstate.Script == "" && !fileExists(fsstate.ScriptDir) && fsstate.Stdout == false {
-		fmt.Fprintf(os.Stderr, "%s: no notification methods were specified\n", programName)
-		fmt.Fprintf(os.Stderr, "Please specify at least one of the following notification methods:\n")
-		fmt.Fprintf(os.Stderr, " - Place one or more email addresses in %s (one address per line)\n", defaultEmailFile())
-		fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", fsstate.ScriptDir)
-		fmt.Fprintf(os.Stderr, " - Specify an email address using the -email flag\n")
-		fmt.Fprintf(os.Stderr, " - Specify the path to an executable script using the -script flag\n")
-		fmt.Fprintf(os.Stderr, " - Specify the -stdout flag\n")
-		os.Exit(2)
-	}
-
-	if flags.watchlist == "-" {
-		watchlist, err := monitor.ReadWatchList(os.Stdin)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s: error reading watchlist from standard in: %s\n", programName, err)
+			fmt.Fprintf(os.Stderr, "%s: (stdin): %s\n", os.Args[0], err)
 			os.Exit(1)
 		}
-		config.WatchList = watchlist
 	} else {
-		watchlist, err := readWatchListFile(flags.watchlist)
+		file, err := os.Open(*watchlistFilename)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s: error reading watchlist from %q: %s\n", programName, flags.watchlist, err)
+			fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err)
+			os.Exit(1)
+		}
+		watchlist, err = readWatchlist(file)
+		file.Close()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err)
 			os.Exit(1)
 		}
-		config.WatchList = watchlist
 	}
 
-	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	os.Exit(cmd.Main(*stateDir, processEntry))
-	defer stop()
-
-	if err := monitor.Run(ctx, config); err != nil && !errors.Is(err, context.Canceled) {
-		fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err)
-		os.Exit(1)
-	}
 }

cmd/common.go

@@ -0,0 +1,354 @@
+// Copyright (C) 2016-2017 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/x509"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/user"
+	"path/filepath"
+	"sync"
+
+	"software.sslmate.com/src/certspotter"
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+const defaultLogList = "https://loglist.certspotter.org/monitor.json"
+
+var batchSize = flag.Int("batch_size", 1000, "Max number of entries to request at per call to get-entries (advanced)")
+var numWorkers = flag.Int("num_workers", 2, "Number of concurrent matchers (advanced)")
+var script = flag.String("script", "", "Script to execute when a matching certificate is found")
+var logsURL = flag.String("logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
+var noSave = flag.Bool("no_save", false, "Do not save a copy of matching certificates")
+var verbose = flag.Bool("verbose", false, "Be verbose")
+var startAtEnd = flag.Bool("start_at_end", false, "Start monitoring logs from the end rather than the beginning")
+var allTime = flag.Bool("all_time", false, "Scan certs from all time, not just since last scan")
+var state *State
+
+var printMutex sync.Mutex
+
+func homedir() string {
+	home := os.Getenv("HOME")
+	if home != "" {
+		return home
+	}
+	user, err := user.Current()
+	if err == nil {
+		return user.HomeDir
+	}
+	panic("Unable to determine home directory")
+}
+
+func DefaultStateDir(programName string) string {
+	return filepath.Join(homedir(), "."+programName)
+}
+
+func DefaultConfigDir(programName string) string {
+	return filepath.Join(homedir(), "."+programName)
+}
+
+func LogEntry(info *certspotter.EntryInfo) {
+	if !*noSave {
+		var alreadyPresent bool
+		var err error
+		alreadyPresent, info.Filename, err = state.SaveCert(info.IsPrecert, info.FullChain)
+		if err != nil {
+			log.Print(err)
+		}
+		if alreadyPresent {
+			return
+		}
+	}
+
+	if *script != "" {
+		if err := info.InvokeHookScript(*script); err != nil {
+			log.Print(err)
+		}
+	} else {
+		printMutex.Lock()
+		info.Write(os.Stdout)
+		fmt.Fprintf(os.Stdout, "\n")
+		printMutex.Unlock()
+	}
+}
+
+func loadLogList() ([]*loglist.Log, error) {
+	list, err := loglist.Load(*logsURL)
+	if err != nil {
+		return nil, fmt.Errorf("Error loading log list: %s", err)
+	}
+	return list.AllLogs(), nil
+}
+
+type logHandle struct {
+	scanner     *certspotter.Scanner
+	state       *LogState
+	tree        *certspotter.CollapsedMerkleTree
+	verifiedSTH *ct.SignedTreeHead
+}
+
+func makeLogHandle(logInfo *loglist.Log) (*logHandle, error) {
+	ctlog := new(logHandle)
+
+	logKey, err := x509.ParsePKIXPublicKey(logInfo.Key)
+	if err != nil {
+		return nil, fmt.Errorf("Bad public key: %s", err)
+	}
+	ctlog.scanner = certspotter.NewScanner(logInfo.URL, logInfo.LogID, logKey, &certspotter.ScannerOptions{
+		BatchSize:  *batchSize,
+		NumWorkers: *numWorkers,
+		Quiet:      !*verbose,
+	})
+
+	ctlog.state, err = state.OpenLogState(logInfo)
+	if err != nil {
+		return nil, fmt.Errorf("Error opening state directory: %s", err)
+	}
+	ctlog.tree, err = ctlog.state.GetTree()
+	if err != nil {
+		return nil, fmt.Errorf("Error loading tree: %s", err)
+	}
+	ctlog.verifiedSTH, err = ctlog.state.GetVerifiedSTH()
+	if err != nil {
+		return nil, fmt.Errorf("Error loading verified STH: %s", err)
+	}
+
+	if ctlog.tree == nil && ctlog.verifiedSTH == nil { // This branch can be removed eventually
+		legacySTH, err := state.GetLegacySTH(logInfo)
+		if err != nil {
+			return nil, fmt.Errorf("Error loading legacy STH: %s", err)
+		}
+		if legacySTH != nil {
+			log.Print(logInfo.URL, ": Initializing log state from legacy state directory")
+			ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(legacySTH)
+			if err != nil {
+				return nil, fmt.Errorf("Error reconstructing Merkle Tree for legacy STH: %s", err)
+			}
+			if err := ctlog.state.StoreTree(ctlog.tree); err != nil {
+				return nil, fmt.Errorf("Error storing tree: %s", err)
+			}
+			if err := ctlog.state.StoreVerifiedSTH(legacySTH); err != nil {
+				return nil, fmt.Errorf("Error storing verified STH: %s", err)
+			}
+			state.RemoveLegacySTH(logInfo)
+		}
+	}
+
+	return ctlog, nil
+}
+
+func (ctlog *logHandle) refresh() error {
+	if *verbose {
+		log.Print(ctlog.scanner.LogUri, ": Retrieving latest STH from log")
+	}
+	latestSTH, err := ctlog.scanner.GetSTH()
+	if err != nil {
+		return fmt.Errorf("Error retrieving STH from log: %s", err)
+	}
+	if ctlog.verifiedSTH == nil {
+		if *verbose {
+			log.Printf("%s: No existing STH is known; presuming latest STH (%d) is valid", ctlog.scanner.LogUri, latestSTH.TreeSize)
+		}
+		ctlog.verifiedSTH = latestSTH
+		if err := ctlog.state.StoreVerifiedSTH(ctlog.verifiedSTH); err != nil {
+			return fmt.Errorf("Error storing verified STH: %s", err)
+		}
+	} else {
+		if err := ctlog.state.StoreUnverifiedSTH(latestSTH); err != nil {
+			return fmt.Errorf("Error storing unverified STH: %s", err)
+		}
+	}
+	return nil
+}
+
+func (ctlog *logHandle) verifySTH(sth *ct.SignedTreeHead) error {
+	isValid, err := ctlog.scanner.CheckConsistency(ctlog.verifiedSTH, sth)
+	if err != nil {
+		return fmt.Errorf("Error fetching consistency proof: %s", err)
+	}
+	if !isValid {
+		return fmt.Errorf("Consistency proof between %d and %d is invalid", ctlog.verifiedSTH.TreeSize, sth.TreeSize)
+	}
+	return nil
+}
+
+func (ctlog *logHandle) audit() error {
+	sths, err := ctlog.state.GetUnverifiedSTHs()
+	if err != nil {
+		return fmt.Errorf("Error loading unverified STHs: %s", err)
+	}
+
+	for _, sth := range sths {
+		if *verbose {
+			log.Printf("%s: Verifying consistency of STH %d (%x) with previously-verified STH %d (%x)", ctlog.scanner.LogUri, sth.TreeSize, sth.SHA256RootHash, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash)
+		}
+		if err := ctlog.verifySTH(sth); err != nil {
+			log.Printf("%s: Unable to verify consistency of STH %d (%s) (if this error persists, it should be construed as misbehavior by the log): %s", ctlog.scanner.LogUri, sth.TreeSize, ctlog.state.UnverifiedSTHFilename(sth), err)
+			continue
+		}
+		if sth.TreeSize > ctlog.verifiedSTH.TreeSize {
+			if *verbose {
+				log.Printf("%s: STH %d (%x) is now the latest verified STH", ctlog.scanner.LogUri, sth.TreeSize, sth.SHA256RootHash)
+			}
+			ctlog.verifiedSTH = sth
+			if err := ctlog.state.StoreVerifiedSTH(ctlog.verifiedSTH); err != nil {
+				return fmt.Errorf("Error storing verified STH: %s", err)
+			}
+		}
+		if err := ctlog.state.RemoveUnverifiedSTH(sth); err != nil {
+			return fmt.Errorf("Error removing redundant STH: %s", err)
+		}
+	}
+
+	return nil
+}
+
+func (ctlog *logHandle) scan(processCallback certspotter.ProcessCallback) error {
+	startIndex := int64(ctlog.tree.GetSize())
+	endIndex := int64(ctlog.verifiedSTH.TreeSize)
+
+	if endIndex > startIndex {
+		tree := certspotter.CloneCollapsedMerkleTree(ctlog.tree)
+
+		if err := ctlog.scanner.Scan(startIndex, endIndex, processCallback, tree); err != nil {
+			return fmt.Errorf("Error scanning log (if this error persists, it should be construed as misbehavior by the log): %s", err)
+		}
+
+		rootHash := tree.CalculateRoot()
+		if !bytes.Equal(rootHash, ctlog.verifiedSTH.SHA256RootHash[:]) {
+			return fmt.Errorf("Log has misbehaved: log entries at tree size %d do not correspond to signed tree root", ctlog.verifiedSTH.TreeSize)
+		}
+
+		ctlog.tree = tree
+		if err := ctlog.state.StoreTree(ctlog.tree); err != nil {
+			return fmt.Errorf("Error storing tree: %s", err)
+		}
+	}
+
+	return nil
+}
+
+func processLog(logInfo *loglist.Log, processCallback certspotter.ProcessCallback) int {
+	ctlog, err := makeLogHandle(logInfo)
+	if err != nil {
+		log.Print(logInfo.URL, ": ", err)
+		return 1
+	}
+
+	if err := ctlog.refresh(); err != nil {
+		log.Print(logInfo.URL, ": ", err)
+		return 1
+	}
+
+	if err := ctlog.audit(); err != nil {
+		log.Print(logInfo.URL, ": ", err)
+		return 1
+	}
+
+	if *allTime {
+		ctlog.tree = certspotter.EmptyCollapsedMerkleTree()
+		if *verbose {
+			log.Printf("%s: Scanning all %d entries in the log because -all_time option specified", logInfo.URL, ctlog.verifiedSTH.TreeSize)
+		}
+	} else if ctlog.tree != nil {
+		if *verbose {
+			log.Printf("%s: Existing log; scanning %d new entries since previous scan", logInfo.URL, ctlog.verifiedSTH.TreeSize-ctlog.tree.GetSize())
+		}
+	} else if *startAtEnd {
+		ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(ctlog.verifiedSTH)
+		if err != nil {
+			log.Printf("%s: Error reconstructing Merkle Tree: %s", logInfo.URL, err)
+			return 1
+		}
+		if *verbose {
+			log.Printf("%s: New log; not scanning %d existing entries because -start_at_end option was specified", logInfo.URL, ctlog.verifiedSTH.TreeSize)
+		}
+	} else {
+		ctlog.tree = certspotter.EmptyCollapsedMerkleTree()
+		if *verbose {
+			log.Printf("%s: New log; scanning all %d entries in the log (use the -start_at_end option to scan new logs from the end rather than the beginning)", logInfo.URL, ctlog.verifiedSTH.TreeSize)
+		}
+	}
+	if err := ctlog.state.StoreTree(ctlog.tree); err != nil {
+		log.Printf("%s: Error storing tree: %s\n", logInfo.URL, err)
+		return 1
+	}
+
+	if err := ctlog.scan(processCallback); err != nil {
+		log.Print(logInfo.URL, ": ", err)
+		return 1
+	}
+
+	if *verbose {
+		log.Printf("%s: Final log size = %d, final root hash = %x", logInfo.URL, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash)
+	}
+
+	return 0
+}
+
+func Main(statePath string, processCallback certspotter.ProcessCallback) int {
+	var err error
+
+	logs, err := loadLogList()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err)
+		return 1
+	}
+
+	state, err = OpenState(statePath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err)
+		return 1
+	}
+	locked, err := state.Lock()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s: Error locking state directory: %s\n", os.Args[0], err)
+		return 1
+	}
+	if !locked {
+		var otherPidInfo string
+		if otherPid := state.LockingPid(); otherPid != 0 {
+			otherPidInfo = fmt.Sprintf(" (as process ID %d)", otherPid)
+		}
+		fmt.Fprintf(os.Stderr, "%s: Another instance of %s is already running%s; remove the file %s if this is not the case\n", os.Args[0], os.Args[0], otherPidInfo, state.LockFilename())
+		return 1
+	}
+
+	processLogResults := make(chan int)
+	for _, logInfo := range logs {
+		go func(logInfo *loglist.Log) {
+			processLogResults <- processLog(logInfo, processCallback)
+		}(logInfo)
+	}
+
+	exitCode := 0
+	for range logs {
+		exitCode |= <-processLogResults
+	}
+
+	if state.IsFirstRun() && exitCode == 0 {
+		if err := state.WriteOnceFile(); err != nil {
+			fmt.Fprintf(os.Stderr, "%s: Error writing once file: %s\n", os.Args[0], err)
+			exitCode |= 1
+		}
+	}
+
+	if err := state.Unlock(); err != nil {
+		fmt.Fprintf(os.Stderr, "%s: Error unlocking state directory: %s\n", os.Args[0], err)
+		exitCode |= 1
+	}
+
+	return exitCode
+}

cmd/ctparsewatch/.gitignore

@@ -0,0 +1 @@
+/ctparsewatch

cmd/ctparsewatch/main.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2016 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package main
+
+import (
+	"flag"
+	"os"
+
+	"software.sslmate.com/src/certspotter"
+	"software.sslmate.com/src/certspotter/cmd"
+	"software.sslmate.com/src/certspotter/ct"
+)
+
+func DefaultStateDir() string {
+	if envVar := os.Getenv("CTPARSEWATCH_STATE_DIR"); envVar != "" {
+		return envVar
+	} else {
+		return cmd.DefaultStateDir("ctparsewatch")
+	}
+}
+
+var stateDir = flag.String("state_dir", DefaultStateDir(), "Directory for storing state")
+
+func processEntry(scanner *certspotter.Scanner, entry *ct.LogEntry) {
+	info := certspotter.EntryInfo{
+		LogUri:    scanner.LogUri,
+		Entry:     entry,
+		IsPrecert: certspotter.IsPrecert(entry),
+		FullChain: certspotter.GetFullChain(entry),
+	}
+
+	info.CertInfo, info.ParseError = certspotter.MakeCertInfoFromLogEntry(entry)
+	if info.CertInfo != nil {
+		info.Identifiers, info.IdentifiersParseError = info.CertInfo.ParseIdentifiers()
+	}
+
+	if info.HasParseErrors() {
+		cmd.LogEntry(&info)
+	}
+}
+
+func main() {
+	flag.Parse()
+	os.Exit(cmd.Main(*stateDir, processEntry))
+}

cmd/helpers.go

@@ -0,0 +1,87 @@
+// Copyright (C) 2017 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cmd
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+
+	"software.sslmate.com/src/certspotter/ct"
+)
+
+func fileExists(path string) bool {
+	_, err := os.Lstat(path)
+	return err == nil
+}
+
+func writeFile(filename string, data []byte, perm os.FileMode) error {
+	tempname := filename + ".new"
+	if err := ioutil.WriteFile(tempname, data, perm); err != nil {
+		return err
+	}
+	if err := os.Rename(tempname, filename); err != nil {
+		os.Remove(tempname)
+		return err
+	}
+	return nil
+}
+
+func writeJSONFile(filename string, obj interface{}, perm os.FileMode) error {
+	tempname := filename + ".new"
+	f, err := os.OpenFile(tempname, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	if err := json.NewEncoder(f).Encode(obj); err != nil {
+		f.Close()
+		os.Remove(tempname)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tempname)
+		return err
+	}
+	if err := os.Rename(tempname, filename); err != nil {
+		os.Remove(tempname)
+		return err
+	}
+	return nil
+}
+
+func readJSONFile(filename string, obj interface{}) error {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+	if err = json.Unmarshal(bytes, obj); err != nil {
+		return err
+	}
+	return nil
+}
+
+func readSTHFile(filename string) (*ct.SignedTreeHead, error) {
+	sth := new(ct.SignedTreeHead)
+	if err := readJSONFile(filename, sth); err != nil {
+		return nil, err
+	}
+	return sth, nil
+}
+
+func sha256sum(data []byte) []byte {
+	sum := sha256.Sum256(data)
+	return sum[:]
+}
+
+func sha256hex(data []byte) string {
+	return hex.EncodeToString(sha256sum(data))
+}

cmd/log_state.go

@@ -0,0 +1,145 @@
+// Copyright (C) 2017 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cmd
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"software.sslmate.com/src/certspotter"
+	"software.sslmate.com/src/certspotter/ct"
+)
+
+type LogState struct {
+	path string
+}
+
+// generate a filename that uniquely identifies the STH (within the context of a particular log)
+func sthFilename(sth *ct.SignedTreeHead) string {
+	hasher := sha256.New()
+	switch sth.Version {
+	case ct.V1:
+		binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
+		binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash)
+	default:
+		panic(fmt.Sprintf("Unsupported STH version %d", sth.Version))
+	}
+	// For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic)
+	return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json"
+}
+
+func makeLogStateDir(logStatePath string) error {
+	if err := os.Mkdir(logStatePath, 0777); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("%s: %s", logStatePath, err)
+	}
+	for _, subdir := range []string{"unverified_sths"} {
+		path := filepath.Join(logStatePath, subdir)
+		if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) {
+			return fmt.Errorf("%s: %s", path, err)
+		}
+	}
+	return nil
+}
+
+func OpenLogState(logStatePath string) (*LogState, error) {
+	if err := makeLogStateDir(logStatePath); err != nil {
+		return nil, fmt.Errorf("Error creating log state directory: %s", err)
+	}
+	return &LogState{path: logStatePath}, nil
+}
+
+func (logState *LogState) VerifiedSTHFilename() string {
+	return filepath.Join(logState.path, "sth.json")
+}
+
+func (logState *LogState) GetVerifiedSTH() (*ct.SignedTreeHead, error) {
+	sth, err := readSTHFile(logState.VerifiedSTHFilename())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		} else {
+			return nil, err
+		}
+	}
+	return sth, nil
+}
+
+func (logState *LogState) StoreVerifiedSTH(sth *ct.SignedTreeHead) error {
+	return writeJSONFile(logState.VerifiedSTHFilename(), sth, 0666)
+}
+
+func (logState *LogState) GetUnverifiedSTHs() ([]*ct.SignedTreeHead, error) {
+	dir, err := os.Open(filepath.Join(logState.path, "unverified_sths"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return []*ct.SignedTreeHead{}, nil
+		} else {
+			return nil, err
+		}
+	}
+	filenames, err := dir.Readdirnames(0)
+	if err != nil {
+		return nil, err
+	}
+
+	sths := make([]*ct.SignedTreeHead, 0, len(filenames))
+	for _, filename := range filenames {
+		if !strings.HasPrefix(filename, ".") {
+			sth, _ := readSTHFile(filepath.Join(dir.Name(), filename))
+			if sth != nil {
+				sths = append(sths, sth)
+			}
+		}
+	}
+	return sths, nil
+}
+
+func (logState *LogState) UnverifiedSTHFilename(sth *ct.SignedTreeHead) string {
+	return filepath.Join(logState.path, "unverified_sths", sthFilename(sth))
+}
+
+func (logState *LogState) StoreUnverifiedSTH(sth *ct.SignedTreeHead) error {
+	filename := logState.UnverifiedSTHFilename(sth)
+	if fileExists(filename) {
+		return nil
+	}
+	return writeJSONFile(filename, sth, 0666)
+}
+
+func (logState *LogState) RemoveUnverifiedSTH(sth *ct.SignedTreeHead) error {
+	filename := logState.UnverifiedSTHFilename(sth)
+	err := os.Remove(filepath.Join(filename))
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+func (logState *LogState) GetTree() (*certspotter.CollapsedMerkleTree, error) {
+	tree := new(certspotter.CollapsedMerkleTree)
+	if err := readJSONFile(filepath.Join(logState.path, "tree.json"), tree); err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		} else {
+			return nil, err
+		}
+	}
+	return tree, nil
+}
+
+func (logState *LogState) StoreTree(tree *certspotter.CollapsedMerkleTree) error {
+	return writeJSONFile(filepath.Join(logState.path, "tree.json"), tree, 0666)
+}

cmd/state.go

@@ -0,0 +1,220 @@
+// Copyright (C) 2017 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
+)
+
+type State struct {
+	path string
+}
+
+func legacySTHFilename(logInfo *loglist.Log) string {
+	return strings.Replace(strings.Replace(logInfo.URL, "://", "_", 1), "/", "_", -1)
+}
+
+func readVersionFile(statePath string) (int, error) {
+	versionFilePath := filepath.Join(statePath, "version")
+	versionBytes, err := ioutil.ReadFile(versionFilePath)
+	if err == nil {
+		version, err := strconv.Atoi(string(bytes.TrimSpace(versionBytes)))
+		if err != nil {
+			return -1, fmt.Errorf("%s: contains invalid integer: %s", versionFilePath, err)
+		}
+		if version < 0 {
+			return -1, fmt.Errorf("%s: contains negative integer", versionFilePath)
+		}
+		return version, nil
+	} else if os.IsNotExist(err) {
+		if fileExists(filepath.Join(statePath, "sths")) {
+			// Original version of certspotter had no version file.
+			// Infer version 0 if "sths" directory is present.
+			return 0, nil
+		}
+		return -1, nil
+	} else {
+		return -1, fmt.Errorf("%s: %s", versionFilePath, err)
+	}
+}
+
+func writeVersionFile(statePath string) error {
+	version := 1
+	versionString := fmt.Sprintf("%d\n", version)
+	versionFilePath := filepath.Join(statePath, "version")
+	if err := ioutil.WriteFile(versionFilePath, []byte(versionString), 0666); err != nil {
+		return fmt.Errorf("%s: %s\n", versionFilePath, err)
+	}
+	return nil
+}
+
+func makeStateDir(statePath string) error {
+	if err := os.Mkdir(statePath, 0777); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("%s: %s", statePath, err)
+	}
+	for _, subdir := range []string{"certs", "logs"} {
+		path := filepath.Join(statePath, subdir)
+		if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) {
+			return fmt.Errorf("%s: %s", path, err)
+		}
+	}
+	return nil
+}
+
+func OpenState(statePath string) (*State, error) {
+	version, err := readVersionFile(statePath)
+	if err != nil {
+		return nil, fmt.Errorf("Error reading version file: %s", err)
+	}
+
+	if version < 1 {
+		if err := makeStateDir(statePath); err != nil {
+			return nil, fmt.Errorf("Error creating state directory: %s", err)
+		}
+		if version == 0 {
+			log.Printf("Migrating state directory (%s) to new layout...", statePath)
+			if err := os.Rename(filepath.Join(statePath, "sths"), filepath.Join(statePath, "legacy_sths")); err != nil {
+				return nil, fmt.Errorf("Error migrating STHs directory: %s", err)
+			}
+			for _, subdir := range []string{"evidence", "legacy_sths"} {
+				os.Remove(filepath.Join(statePath, subdir))
+			}
+			if err := ioutil.WriteFile(filepath.Join(statePath, "once"), []byte{}, 0666); err != nil {
+				return nil, fmt.Errorf("Error creating once file: %s", err)
+			}
+		}
+		if err := writeVersionFile(statePath); err != nil {
+			return nil, fmt.Errorf("Error writing version file: %s", err)
+		}
+	} else if version > 1 {
+		return nil, fmt.Errorf("%s was created by a newer version of Cert Spotter; please remove this directory or upgrade Cert Spotter", statePath)
+	}
+
+	return &State{path: statePath}, nil
+}
+
+func (state *State) IsFirstRun() bool {
+	return !fileExists(filepath.Join(state.path, "once"))
+}
+
+func (state *State) WriteOnceFile() error {
+	if err := ioutil.WriteFile(filepath.Join(state.path, "once"), []byte{}, 0666); err != nil {
+		return fmt.Errorf("Error writing once file: %s", err)
+	}
+	return nil
+}
+
+func (state *State) SaveCert(isPrecert bool, certs [][]byte) (bool, string, error) {
+	if len(certs) == 0 {
+		return false, "", fmt.Errorf("Cannot write an empty certificate chain")
+	}
+
+	fingerprint := sha256hex(certs[0])
+	prefixPath := filepath.Join(state.path, "certs", fingerprint[0:2])
+	var filenameSuffix string
+	if isPrecert {
+		filenameSuffix = ".precert.pem"
+	} else {
+		filenameSuffix = ".cert.pem"
+	}
+	if err := os.Mkdir(prefixPath, 0777); err != nil && !os.IsExist(err) {
+		return false, "", fmt.Errorf("Failed to create prefix directory %s: %s", prefixPath, err)
+	}
+	path := filepath.Join(prefixPath, fingerprint+filenameSuffix)
+	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+	if err != nil {
+		if os.IsExist(err) {
+			return true, path, nil
+		} else {
+			return false, path, fmt.Errorf("Failed to open %s for writing: %s", path, err)
+		}
+	}
+	for _, cert := range certs {
+		if err := pem.Encode(file, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil {
+			file.Close()
+			return false, path, fmt.Errorf("Error writing to %s: %s", path, err)
+		}
+	}
+	if err := file.Close(); err != nil {
+		return false, path, fmt.Errorf("Error writing to %s: %s", path, err)
+	}
+
+	return false, path, nil
+}
+
+func (state *State) OpenLogState(logInfo *loglist.Log) (*LogState, error) {
+	return OpenLogState(filepath.Join(state.path, "logs", base64.RawURLEncoding.EncodeToString(logInfo.LogID[:])))
+}
+
+func (state *State) GetLegacySTH(logInfo *loglist.Log) (*ct.SignedTreeHead, error) {
+	sth, err := readSTHFile(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo)))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		} else {
+			return nil, err
+		}
+	}
+	return sth, nil
+}
+func (state *State) RemoveLegacySTH(logInfo *loglist.Log) error {
+	err := os.Remove(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo)))
+	os.Remove(filepath.Join(state.path, "legacy_sths"))
+	return err
+}
+func (state *State) LockFilename() string {
+	return filepath.Join(state.path, "lock")
+}
+func (state *State) Lock() (bool, error) {
+	file, err := os.OpenFile(state.LockFilename(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+	if err != nil {
+		if os.IsExist(err) {
+			return false, nil
+		} else {
+			return false, err
+		}
+	}
+	if _, err := fmt.Fprintf(file, "%d\n", os.Getpid()); err != nil {
+		file.Close()
+		os.Remove(state.LockFilename())
+		return false, err
+	}
+	if err := file.Close(); err != nil {
+		os.Remove(state.LockFilename())
+		return false, err
+	}
+	return true, nil
+}
+func (state *State) Unlock() error {
+	return os.Remove(state.LockFilename())
+}
+func (state *State) LockingPid() int {
+	pidBytes, err := ioutil.ReadFile(state.LockFilename())
+	if err != nil {
+		return 0
+	}
+	pid, err := strconv.Atoi(string(bytes.TrimSpace(pidBytes)))
+	if err != nil {
+		return 0
+	}
+	return pid
+}

cmd/submitct/.gitignore

@@ -1 +0,0 @@
-/submitct

cmd/submitct/main.go

@@ -16,13 +16,12 @@ import (
 	"software.sslmate.com/src/certspotter/loglist"
 
 	"bytes"
-	"context"
 	"crypto/sha256"
 	"crypto/x509"
 	"encoding/pem"
 	"flag"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -122,7 +121,7 @@ type Log struct {
 
 func (ctlog *Log) SubmitChain(chain Chain) (*ct.SignedCertificateTimestamp, error) {
 	rawCerts := chain.GetRawCerts()
-	sct, err := ctlog.AddChain(context.Background(), rawCerts)
+	sct, err := ctlog.AddChain(rawCerts)
 	if err != nil {
 		return nil, err
 	}
@@ -146,31 +145,30 @@ func main() {
 	flag.Parse()
 	log.SetPrefix("submitct: ")
 
-	certsPem, err := io.ReadAll(os.Stdin)
+	certsPem, err := ioutil.ReadAll(os.Stdin)
 	if err != nil {
 		log.Fatalf("Error reading stdin: %s", err)
 	}
 
-	list, err := loglist.Load(context.Background(), *logsURL)
+	list, err := loglist.Load(*logsURL)
 	if err != nil {
 		log.Fatalf("Error loading log list: %s", err)
 	}
 
 	var logs []Log
 	for _, ctlog := range list.AllLogs() {
-		submissionURL := ctlog.GetSubmissionURL()
 		pubkey, err := x509.ParsePKIXPublicKey(ctlog.Key)
 		if err != nil {
-			log.Fatalf("%s: Failed to parse log public key: %s", submissionURL, err)
+			log.Fatalf("%s: Failed to parse log public key: %s", ctlog.URL, err)
 		}
 		verifier, err := ct.NewSignatureVerifier(pubkey)
 		if err != nil {
-			log.Fatalf("%s: Failed to create signature verifier for log: %s", submissionURL, err)
+			log.Fatalf("%s: Failed to create signature verifier for log: %s", ctlog.URL, err)
 		}
 		logs = append(logs, Log{
 			Log: ctlog,
 			SignatureVerifier: verifier,
-			LogClient: client.New(strings.TrimRight(submissionURL, "/")),
+			LogClient: client.New(strings.TrimRight(ctlog.URL, "/")),
 		})
 	}
 
@@ -213,11 +211,11 @@ func main() {
 			go func(fingerprint [32]byte, ctlog Log) {
 				sct, err := ctlog.SubmitChain(chain)
 				if err != nil {
-					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.GetSubmissionURL(), err)
+					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.URL, err)
 					atomic.AddUint32(&submitErrors, 1)
 				} else if *verbose {
 					timestamp := time.Unix(int64(sct.Timestamp)/1000, int64(sct.Timestamp%1000)*1000000)
-					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.GetSubmissionURL(), timestamp)
+					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.URL, timestamp)
 				}
 				wg.Done()
 			}(fingerprint, ctlog)

ct/README

@@ -1,4 +1,2 @@
-The code in this directory is based on Google's Certificiate Transparency Go library
+The code in this directory is from https://github.com/google/certificate-transparency/tree/master/go
-(originally at <https://github.com/google/certificate-transparency/tree/master/go>;
-now at <https://github.com/google/certificate-transparency-go>).
 See AUTHORS for the copyright holders, and LICENSE for the license.

ct/client/logclient.go

@@ -5,7 +5,6 @@ package client
 
 import (
 	"bytes"
-	"context"
 	"crypto/sha256"
 	"crypto/tls"
 	"encoding/base64"
@@ -13,49 +12,15 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	insecurerand "math/rand"
+	"io/ioutil"
 	"net/http"
 	"net/url"
-	"strconv"
 	"time"
 
+	"github.com/mreiferson/go-httpclient"
 	"software.sslmate.com/src/certspotter/ct"
 )
 
-const (
-	baseRetryDelay = 1 * time.Second
-	maxRetryDelay  = 120 * time.Second
-	maxRetries     = 10
-)
-
-func isRetryableStatusCode(code int) bool {
-	return code/100 == 5 || code == http.StatusTooManyRequests
-}
-
-func randomDuration(min, max time.Duration) time.Duration {
-	return min + time.Duration(insecurerand.Int63n(int64(max)-int64(min)+1))
-}
-
-func getRetryAfter(resp *http.Response) (time.Duration, bool) {
-	if resp == nil {
-		return 0, false
-	}
-	seconds, err := strconv.ParseUint(resp.Header.Get("Retry-After"), 10, 16)
-	if err != nil {
-		return 0, false
-	}
-	return time.Duration(seconds) * time.Second, true
-}
-
-func sleep(ctx context.Context, duration time.Duration) {
-	timer := time.NewTimer(duration)
-	defer timer.Stop()
-	select {
-	case <-ctx.Done():
-	case <-timer.C:
-	}
-}
-
 // URI paths for CT Log endpoints
 const (
 	GetSTHPath            = "/ct/v1/get-sth"
@@ -69,7 +34,6 @@ const (
 type LogClient struct {
 	uri        string       // the base URI of the log. e.g. http://ct.googleapis/pilot
 	httpClient *http.Client // used to interact with the log via HTTP
-	verifier   *ct.SignatureVerifier // if non-nil, used to verify STH signatures
 }
 
 //////////////////////////////////////////////////////////////////////////////////
@@ -77,7 +41,7 @@ type LogClient struct {
 // These represent the structures returned by the CT Log server.
 //////////////////////////////////////////////////////////////////////////////////
 
-// getSTHResponse represents the JSON response to the get-sth CT method
+// getSTHResponse respresents the JSON response to the get-sth CT method
 type getSTHResponse struct {
 	TreeSize          uint64 `json:"tree_size"`           // Number of certs in the current tree
 	Timestamp         uint64 `json:"timestamp"`           // Time that the tree was created
@@ -85,13 +49,13 @@ type getSTHResponse struct {
 	TreeHeadSignature []byte `json:"tree_head_signature"` // Log signature for this STH
 }
 
-// base64LeafEntry represents a Base64 encoded leaf entry
+// base64LeafEntry respresents a Base64 encoded leaf entry
 type base64LeafEntry struct {
 	LeafInput []byte `json:"leaf_input"`
 	ExtraData []byte `json:"extra_data"`
 }
 
-// getEntriesReponse represents the JSON response to the CT get-entries method
+// getEntriesReponse respresents the JSON response to the CT get-entries method
 type getEntriesResponse struct {
 	Entries []base64LeafEntry `json:"entries"` // the list of returned entries
 }
@@ -123,22 +87,15 @@ type addChainResponse struct {
 // |uri| is the base URI of the CT log instance to interact with, e.g.
 // http://ct.googleapis.com/pilot
 func New(uri string) *LogClient {
-	return NewWithVerifier(uri, nil)
-}
-
-func NewWithVerifier(uri string, verifier *ct.SignatureVerifier) *LogClient {
 	var c LogClient
 	c.uri = uri
-	c.verifier = verifier
+	transport := &httpclient.Transport{
-	transport := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
-		TLSHandshakeTimeout:   15 * time.Second,
+		ConnectTimeout:        10 * time.Second,
+		RequestTimeout:        60 * time.Second,
 		ResponseHeaderTimeout: 30 * time.Second,
 		MaxIdleConnsPerHost:   10,
 		DisableKeepAlives:     false,
-		MaxIdleConns:          100,
-		IdleConnTimeout:       15 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
 		TLSClientConfig: &tls.Config{
 			// We have to disable TLS certificate validation because because several logs
 			// (WoSign, StartCom, GDCA) use certificates that are not widely trusted.
@@ -150,109 +107,63 @@ func NewWithVerifier(uri string, verifier *ct.SignatureVerifier) *LogClient {
 			InsecureSkipVerify: true,
 		},
 	}
-	c.httpClient = &http.Client{Timeout: 60 * time.Second, Transport: transport}
+	c.httpClient = &http.Client{Transport: transport}
 	return &c
 }
 
-func (c *LogClient) fetchAndParse(ctx context.Context, uri string, respBody interface{}) error {
+// Makes a HTTP call to |uri|, and attempts to parse the response as a JSON
-	return c.doAndParse(ctx, "GET", uri, nil, respBody)
+// representation of the structure in |res|.
-}
+// Returns a non-nil |error| if there was a problem.
-
+func (c *LogClient) fetchAndParse(uri string, respBody interface{}) error {
-func (c *LogClient) postAndParse(ctx context.Context, uri string, body interface{}, respBody interface{}) error {
+	req, err := http.NewRequest("GET", uri, nil)
-	return c.doAndParse(ctx, "POST", uri, body, respBody)
-}
-
-func (c *LogClient) makeRequest(ctx context.Context, method string, uri string, body interface{}) (*http.Request, error) {
-	if body == nil {
-		return http.NewRequestWithContext(ctx, method, uri, nil)
-	} else {
-		bodyBytes, err := json.Marshal(body)
-		if err != nil {
-			return nil, err
-		}
-		req, err := http.NewRequestWithContext(ctx, method, uri, bytes.NewReader(bodyBytes))
-		if err != nil {
-			return nil, err
-		}
-		req.Header.Set("Content-Type", "application/json")
-		return req, nil
-	}
-}
-
-func (c *LogClient) doAndParse(ctx context.Context, method string, uri string, reqBody interface{}, respBody interface{}) error {
-	numRetries := 0
-retry:
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
-	req, err := c.makeRequest(ctx, method, uri, reqBody)
 	if err != nil {
-		return fmt.Errorf("%s %s: error creating request: %w", method, uri, err)
+		return fmt.Errorf("GET %s: Sending request failed: %s", uri, err)
 	}
-	req.Header.Set("User-Agent", "") // Don't send a User-Agent to make life harder for malicious logs
+	return c.doAndParse(req, respBody)
+}
+
+func (c *LogClient) postAndParse(uri string, body interface{}, respBody interface{}) error {
+	bodyReader, bodyWriter := io.Pipe()
+	go func() {
+		json.NewEncoder(bodyWriter).Encode(body)
+		bodyWriter.Close()
+	}()
+	req, err := http.NewRequest("POST", uri, bodyReader)
+	if err != nil {
+		return fmt.Errorf("POST %s: Sending request failed: %s", uri, err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	return c.doAndParse(req, respBody)
+}
+
+func (c *LogClient) doAndParse(req *http.Request, respBody interface{}) error {
+	//	req.Header.Set("Keep-Alive", "timeout=15, max=100")
 	resp, err := c.httpClient.Do(req)
-	if err != nil {
+	var respBodyBytes []byte
-		if c.shouldRetry(ctx, numRetries, nil) {
+	if resp != nil {
-			numRetries++
+		respBodyBytes, err = ioutil.ReadAll(resp.Body)
-			goto retry
+		resp.Body.Close()
+		if err != nil {
+			return fmt.Errorf("%s %s: Reading response failed: %s", req.Method, req.URL, err)
 		}
+	}
+	if err != nil {
 		return err
 	}
-	respBodyBytes, err := io.ReadAll(resp.Body)
-	resp.Body.Close()
-	if err != nil {
-		if c.shouldRetry(ctx, numRetries, nil) {
-			numRetries++
-			goto retry
-		}
-		return fmt.Errorf("%s %s: error reading response: %w", method, uri, err)
-	}
 	if resp.StatusCode/100 != 2 {
-		if c.shouldRetry(ctx, numRetries, resp) {
+		return fmt.Errorf("%s %s: %s (%s)", req.Method, req.URL, resp.Status, string(respBodyBytes))
-			numRetries++
-			goto retry
-		}
-		return fmt.Errorf("%s %s: %s (%s)", method, uri, resp.Status, string(respBodyBytes))
 	}
-	if err := json.Unmarshal(respBodyBytes, respBody); err != nil {
+	if err = json.Unmarshal(respBodyBytes, &respBody); err != nil {
-		return fmt.Errorf("%s %s: error parsing response JSON: %w", method, uri, err)
+		return fmt.Errorf("%s %s: Parsing response JSON failed: %s", req.Method, req.URL, err)
 	}
 	return nil
 }
 
-func (c *LogClient) shouldRetry(ctx context.Context, numRetries int, resp *http.Response) bool {
-	if numRetries == maxRetries {
-		return false
-	}
-
-	if resp != nil && !isRetryableStatusCode(resp.StatusCode) {
-		return false
-	}
-
-	var delay time.Duration
-	if retryAfter, hasRetryAfter := getRetryAfter(resp); hasRetryAfter {
-		delay = retryAfter
-	} else {
-		delay = baseRetryDelay * (1 << numRetries)
-		if delay > maxRetryDelay {
-			delay = maxRetryDelay
-		}
-		delay += randomDuration(0, delay/2)
-	}
-
-	if deadline, hasDeadline := ctx.Deadline(); hasDeadline && time.Now().Add(delay).After(deadline) {
-		return false
-	}
-
-	sleep(ctx, delay)
-	return true
-}
-
 // GetSTH retrieves the current STH from the log.
 // Returns a populated SignedTreeHead, or a non-nil error.
-func (c *LogClient) GetSTH(ctx context.Context) (sth *ct.SignedTreeHead, err error) {
+func (c *LogClient) GetSTH() (sth *ct.SignedTreeHead, err error) {
 	var resp getSTHResponse
-	if err = c.fetchAndParse(ctx, c.uri+GetSTHPath, &resp); err != nil {
+	if err = c.fetchAndParse(c.uri+GetSTHPath, &resp); err != nil {
 		return
 	}
 	sth = &ct.SignedTreeHead{
@@ -269,48 +180,15 @@ func (c *LogClient) GetSTH(ctx context.Context) (sth *ct.SignedTreeHead, err err
 	if err != nil {
 		return nil, err
 	}
+	// TODO(alcutter): Verify signature
 	sth.TreeHeadSignature = *ds
-	if c.verifier != nil {
-		if err := c.verifier.VerifySTHSignature(*sth); err != nil {
-			return nil, fmt.Errorf("STH returned by server has invalid signature: %w", err)
-		}
-	}
 	return
 }
 
-type GetEntriesItem struct {
-	LeafInput []byte `json:"leaf_input"`
-	ExtraData []byte `json:"extra_data"`
-}
-
-// Retrieve the entries in the sequence [start, end] from the CT log server.
-// If error is nil, at least one entry is returned, and no excess entries are returned.
-// Fewer entries than requested may be returned.
-func (c *LogClient) GetRawEntries(ctx context.Context, start, end uint64) ([]GetEntriesItem, error) {
-	if end < start {
-		panic("LogClient.GetRawEntries: end < start")
-	}
-	var response struct {
-		Entries []GetEntriesItem `json:"entries"`
-	}
-	uri := fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end)
-	err := c.fetchAndParse(ctx, uri, &response)
-	if err != nil {
-		return nil, err
-	}
-	if len(response.Entries) == 0 {
-		return nil, fmt.Errorf("GET %s: log server returned an empty get-entries response", uri)
-	}
-	if uint64(len(response.Entries)) > end-start+1 {
-		return nil, fmt.Errorf("GET %s: log server returned a get-entries response with extraneous entries", uri)
-	}
-	return response.Entries, nil
-}
-
 // GetEntries attempts to retrieve the entries in the sequence [|start|, |end|] from the CT
 // log server. (see section 4.6.)
 // Returns a slice of LeafInputs or a non-nil error.
-func (c *LogClient) GetEntries(ctx context.Context, start, end int64) ([]ct.LogEntry, error) {
+func (c *LogClient) GetEntries(start, end int64) ([]ct.LogEntry, error) {
 	if end < 0 {
 		return nil, errors.New("GetEntries: end should be >= 0")
 	}
@@ -318,7 +196,7 @@ func (c *LogClient) GetEntries(ctx context.Context, start, end int64) ([]ct.LogE
 		return nil, errors.New("GetEntries: start should be <= end")
 	}
 	var resp getEntriesResponse
-	err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end), &resp)
+	err := c.fetchAndParse(fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end), &resp)
 	if err != nil {
 		return nil, err
 	}
@@ -353,7 +231,7 @@ func (c *LogClient) GetEntries(ctx context.Context, start, end int64) ([]ct.LogE
 
 // GetConsistencyProof retrieves a Merkle Consistency Proof between two STHs (|first| and |second|)
 // from the log.  Returns a slice of MerkleTreeNodes (a ct.ConsistencyProof) or a non-nil error.
-func (c *LogClient) GetConsistencyProof(ctx context.Context, first, second int64) (ct.ConsistencyProof, error) {
+func (c *LogClient) GetConsistencyProof(first, second int64) (ct.ConsistencyProof, error) {
 	if second < 0 {
 		return nil, errors.New("GetConsistencyProof: second should be >= 0")
 	}
@@ -361,7 +239,7 @@ func (c *LogClient) GetConsistencyProof(ctx context.Context, first, second int64
 		return nil, errors.New("GetConsistencyProof: first should be <= second")
 	}
 	var resp getConsistencyProofResponse
-	err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?first=%d&second=%d", c.uri, GetSTHConsistencyPath, first, second), &resp)
+	err := c.fetchAndParse(fmt.Sprintf("%s%s?first=%d&second=%d", c.uri, GetSTHConsistencyPath, first, second), &resp)
 	if err != nil {
 		return nil, err
 	}
@@ -375,9 +253,9 @@ func (c *LogClient) GetConsistencyProof(ctx context.Context, first, second int64
 // GetAuditProof retrieves a Merkle Audit Proof (aka Inclusion Proof) for the given
 // |hash| based on the STH at |treeSize| from the log.  Returns a slice of MerkleTreeNodes
 // and the index of the leaf.
-func (c *LogClient) GetAuditProof(ctx context.Context, hash ct.MerkleTreeNode, treeSize uint64) (ct.AuditPath, uint64, error) {
+func (c *LogClient) GetAuditProof(hash ct.MerkleTreeNode, treeSize uint64) (ct.AuditPath, uint64, error) {
 	var resp getAuditProofResponse
-	err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?hash=%s&tree_size=%d", c.uri, GetProofByHashPath, url.QueryEscape(base64.StdEncoding.EncodeToString(hash)), treeSize), &resp)
+	err := c.fetchAndParse(fmt.Sprintf("%s%s?hash=%s&tree_size=%d", c.uri, GetProofByHashPath, url.QueryEscape(base64.StdEncoding.EncodeToString(hash)), treeSize), &resp)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -388,11 +266,11 @@ func (c *LogClient) GetAuditProof(ctx context.Context, hash ct.MerkleTreeNode, t
 	return path, resp.LeafIndex, nil
 }
 
-func (c *LogClient) AddChain(ctx context.Context, chain [][]byte) (*ct.SignedCertificateTimestamp, error) {
+func (c *LogClient) AddChain(chain [][]byte) (*ct.SignedCertificateTimestamp, error) {
 	req := addChainRequest{Chain: chain}
 
 	var resp addChainResponse
-	if err := c.postAndParse(ctx, c.uri+AddChainPath, &req, &resp); err != nil {
+	if err := c.postAndParse(c.uri+AddChainPath, &req, &resp); err != nil {
 		return nil, err
 	}
 

ct/types.go

@@ -6,7 +6,6 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"time"
 )
 
 const (
@@ -156,7 +155,7 @@ func (h HashAlgorithm) String() string {
 	}
 }
 
-// SignatureAlgorithm from the DigitallySigned struct
+// SignatureAlgorithm from the the DigitallySigned struct
 type SignatureAlgorithm byte
 
 // SignatureAlgorithm constants
@@ -260,11 +259,6 @@ func (s SHA256Hash) Base64String() string {
 	return base64.StdEncoding.EncodeToString(s[:])
 }
 
-// Returns the raw base64url representation of this SHA256Hash.
-func (s SHA256Hash) Base64URLString() string {
-	return base64.RawURLEncoding.EncodeToString(s[:])
-}
-
 // MarshalJSON implements the json.Marshaller interface for SHA256Hash.
 func (s SHA256Hash) MarshalJSON() ([]byte, error) {
 	return []byte(`"` + s.Base64String() + `"`), nil
@@ -290,10 +284,6 @@ type SignedTreeHead struct {
 	LogID             SHA256Hash      `json:"log_id"`              // The SHA256 hash of the log's public key
 }
 
-func (sth *SignedTreeHead) TimestampTime() time.Time {
-	return time.Unix(int64(sth.Timestamp/1000), int64(sth.Timestamp%1000)*1_000_000).UTC()
-}
-
 // SignedCertificateTimestamp represents the structure returned by the
 // add-chain and add-pre-chain methods after base64 decoding. (see RFC sections
 // 3.2 ,4.1 and 4.2)
@@ -301,7 +291,7 @@ type SignedCertificateTimestamp struct {
 	SCTVersion Version          `json:"sct_version"`    // The version of the protocol to which the SCT conforms
 	LogID      SHA256Hash       `json:"id"` // the SHA-256 hash of the log's public key, calculated over
 	// the DER encoding of the key represented as SubjectPublicKeyInfo.
-	Timestamp  uint64           `json:"timestamp"`          // Timestamp (in ms since unix epoch) at which the SCT was issued
+	Timestamp  uint64           `json:"timestamp"`          // Timestamp (in ms since unix epoc) at which the SCT was issued
 	Extensions CTExtensions     `json:"extensions"`    // For future extensions to the protocol
 	Signature  DigitallySigned  `json:"signature"` // The Log's signature for this SCT
 }
@@ -324,7 +314,7 @@ type TimestampedEntry struct {
 	Extensions   CTExtensions
 }
 
-// MerkleTreeLeaf represents the deserialized structure of the hash input for the
+// MerkleTreeLeaf represents the deserialized sructure of the hash input for the
 // leaves of a log's Merkle tree. See RFC section 3.4
 type MerkleTreeLeaf struct {
 	Version          Version          // the version of the protocol to which the MerkleTreeLeaf corresponds

go.mod

@@ -1,10 +0,0 @@
-module software.sslmate.com/src/certspotter
-
-go 1.21
-
-require (
-	golang.org/x/net v0.17.0
-	golang.org/x/sync v0.4.0
-)
-
-require golang.org/x/text v0.13.0 // indirect

go.sum

@@ -1,6 +0,0 @@
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
-golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

helpers.go

@@ -10,16 +10,105 @@
 package certspotter
 
 import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"math/big"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
 
 	"software.sslmate.com/src/certspotter/ct"
 )
 
+func ReadSTHFile(path string) (*ct.SignedTreeHead, error) {
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	var sth ct.SignedTreeHead
+	if err := json.Unmarshal(content, &sth); err != nil {
+		return nil, err
+	}
+
+	return &sth, nil
+}
+
+func WriteSTHFile(path string, sth *ct.SignedTreeHead) error {
+	sthJson, err := json.MarshalIndent(sth, "", "\t")
+	if err != nil {
+		return err
+	}
+	sthJson = append(sthJson, byte('\n'))
+	return ioutil.WriteFile(path, sthJson, 0666)
+}
+
+func WriteProofFile(path string, proof ct.ConsistencyProof) error {
+	proofJson, err := json.MarshalIndent(proof, "", "\t")
+	if err != nil {
+		return err
+	}
+	proofJson = append(proofJson, byte('\n'))
+	return ioutil.WriteFile(path, proofJson, 0666)
+}
+
 func IsPrecert(entry *ct.LogEntry) bool {
 	return entry.Leaf.TimestampedEntry.EntryType == ct.PrecertLogEntryType
 }
 
+func GetFullChain(entry *ct.LogEntry) [][]byte {
+	certs := make([][]byte, 0, len(entry.Chain)+1)
+
+	if entry.Leaf.TimestampedEntry.EntryType == ct.X509LogEntryType {
+		certs = append(certs, entry.Leaf.TimestampedEntry.X509Entry)
+	}
+	for _, cert := range entry.Chain {
+		certs = append(certs, cert)
+	}
+
+	return certs
+}
+
+func formatSerialNumber(serial *big.Int) string {
+	if serial != nil {
+		return fmt.Sprintf("%x", serial)
+	} else {
+		return ""
+	}
+}
+
+func sha256sum(data []byte) []byte {
+	sum := sha256.Sum256(data)
+	return sum[:]
+}
+
+func sha256hex(data []byte) string {
+	return hex.EncodeToString(sha256sum(data))
+}
+
+type EntryInfo struct {
+	LogUri                string
+	Entry                 *ct.LogEntry
+	IsPrecert             bool
+	FullChain             [][]byte // first entry is logged X509 cert or pre-cert
+	CertInfo              *CertInfo
+	ParseError            error // set iff CertInfo is nil
+	Identifiers           *Identifiers
+	IdentifiersParseError error
+	Filename              string
+	Bygone                bool
+}
+
 type CertInfo struct {
 	TBS *TBSCertificate
 
@@ -35,7 +124,6 @@ type CertInfo struct {
 	ValidityParseError     error
 	IsCA                   *bool
 	IsCAParseError         error
-	IsPreCert              bool
 }
 
 func MakeCertInfoFromTBS(tbs *TBSCertificate) *CertInfo {
@@ -47,7 +135,6 @@ func MakeCertInfoFromTBS(tbs *TBSCertificate) *CertInfo {
 	info.SerialNumber, info.SerialNumberParseError = tbs.ParseSerialNumber()
 	info.Validity, info.ValidityParseError = tbs.ParseValidity()
 	info.IsCA, info.IsCAParseError = tbs.ParseBasicConstraints()
-	info.IsPreCert = len(tbs.GetExtension(oidExtensionCTPoison)) > 0
 
 	return info
 }
@@ -81,6 +168,202 @@ func MakeCertInfoFromLogEntry(entry *ct.LogEntry) (*CertInfo, error) {
 	}
 }
 
+func (info *CertInfo) NotBefore() *time.Time {
+	if info.ValidityParseError == nil {
+		return &info.Validity.NotBefore
+	} else {
+		return nil
+	}
+}
+
+func (info *CertInfo) NotAfter() *time.Time {
+	if info.ValidityParseError == nil {
+		return &info.Validity.NotAfter
+	} else {
+		return nil
+	}
+}
+
+func (info *CertInfo) PubkeyHash() string {
+	return sha256hex(info.TBS.GetRawPublicKey())
+}
+
+func (info *CertInfo) PubkeyHashBytes() []byte {
+	return sha256sum(info.TBS.GetRawPublicKey())
+}
+
+func (info *CertInfo) Environ() []string {
+	env := make([]string, 0, 10)
+
+	env = append(env, "PUBKEY_HASH="+info.PubkeyHash())
+
+	if info.SerialNumberParseError != nil {
+		env = append(env, "SERIAL_PARSE_ERROR="+info.SerialNumberParseError.Error())
+	} else {
+		env = append(env, "SERIAL="+formatSerialNumber(info.SerialNumber))
+	}
+
+	if info.ValidityParseError != nil {
+		env = append(env, "VALIDITY_PARSE_ERROR="+info.ValidityParseError.Error())
+	} else {
+		env = append(env, "NOT_BEFORE="+info.Validity.NotBefore.String())
+		env = append(env, "NOT_BEFORE_UNIXTIME="+strconv.FormatInt(info.Validity.NotBefore.Unix(), 10))
+		env = append(env, "NOT_AFTER="+info.Validity.NotAfter.String())
+		env = append(env, "NOT_AFTER_UNIXTIME="+strconv.FormatInt(info.Validity.NotAfter.Unix(), 10))
+	}
+
+	if info.SubjectParseError != nil {
+		env = append(env, "SUBJECT_PARSE_ERROR="+info.SubjectParseError.Error())
+	} else {
+		env = append(env, "SUBJECT_DN="+info.Subject.String())
+	}
+
+	if info.IssuerParseError != nil {
+		env = append(env, "ISSUER_PARSE_ERROR="+info.IssuerParseError.Error())
+	} else {
+		env = append(env, "ISSUER_DN="+info.Issuer.String())
+	}
+
+	// TODO: include SANs in environment
+
+	return env
+}
+
+func (info *EntryInfo) HasParseErrors() bool {
+	return info.ParseError != nil ||
+		info.IdentifiersParseError != nil ||
+		info.CertInfo.SubjectParseError != nil ||
+		info.CertInfo.IssuerParseError != nil ||
+		info.CertInfo.SANsParseError != nil ||
+		info.CertInfo.SerialNumberParseError != nil ||
+		info.CertInfo.ValidityParseError != nil ||
+		info.CertInfo.IsCAParseError != nil
+}
+
+func (info *EntryInfo) Fingerprint() string {
+	if len(info.FullChain) > 0 {
+		return sha256hex(info.FullChain[0])
+	} else {
+		return ""
+	}
+}
+
+func (info *EntryInfo) FingerprintBytes() []byte {
+	if len(info.FullChain) > 0 {
+		return sha256sum(info.FullChain[0])
+	} else {
+		return []byte{}
+	}
+}
+
+func (info *EntryInfo) typeString() string {
+	if info.IsPrecert {
+		return "precert"
+	} else {
+		return "cert"
+	}
+}
+
+func (info *EntryInfo) typeFriendlyString() string {
+	if info.IsPrecert {
+		return "Pre-certificate"
+	} else {
+		return "Certificate"
+	}
+}
+
+func yesnoString(value bool) string {
+	if value {
+		return "yes"
+	} else {
+		return "no"
+	}
+}
+
+func (info *EntryInfo) Environ() []string {
+	env := []string{
+		"FINGERPRINT=" + info.Fingerprint(),
+		"CERT_TYPE=" + info.typeString(),
+		"CERT_PARSEABLE=" + yesnoString(info.ParseError == nil),
+		"LOG_URI=" + info.LogUri,
+		"ENTRY_INDEX=" + strconv.FormatInt(info.Entry.Index, 10),
+	}
+
+	if info.Filename != "" {
+		env = append(env, "CERT_FILENAME="+info.Filename)
+	}
+	if info.ParseError != nil {
+		env = append(env, "PARSE_ERROR="+info.ParseError.Error())
+	} else if info.CertInfo != nil {
+		certEnv := info.CertInfo.Environ()
+		env = append(env, certEnv...)
+	}
+	if info.IdentifiersParseError != nil {
+		env = append(env, "IDENTIFIERS_PARSE_ERROR="+info.IdentifiersParseError.Error())
+	} else if info.Identifiers != nil {
+		env = append(env, "DNS_NAMES="+info.Identifiers.dnsNamesString(","))
+		env = append(env, "IP_ADDRESSES="+info.Identifiers.ipAddrsString(","))
+	}
+
+	return env
+}
+
+func writeField(out io.Writer, name string, value interface{}, err error) {
+	if err == nil {
+		fmt.Fprintf(out, "\t%13s = %s\n", name, value)
+	} else {
+		fmt.Fprintf(out, "\t%13s = *** UNKNOWN (%s) ***\n", name, err)
+	}
+}
+
+func (info *EntryInfo) Write(out io.Writer) {
+	fingerprint := info.Fingerprint()
+	fmt.Fprintf(out, "%s:\n", fingerprint)
+	if info.IdentifiersParseError != nil {
+		writeField(out, "Identifiers", nil, info.IdentifiersParseError)
+	} else if info.Identifiers != nil {
+		for _, dnsName := range info.Identifiers.DNSNames {
+			writeField(out, "DNS Name", dnsName, nil)
+		}
+		for _, ipaddr := range info.Identifiers.IPAddrs {
+			writeField(out, "IP Address", ipaddr, nil)
+		}
+	}
+	if info.ParseError != nil {
+		writeField(out, "Parse Error", "*** "+info.ParseError.Error()+" ***", nil)
+	} else if info.CertInfo != nil {
+		writeField(out, "Pubkey", info.CertInfo.PubkeyHash(), nil)
+		writeField(out, "Issuer", info.CertInfo.Issuer, info.CertInfo.IssuerParseError)
+		writeField(out, "Not Before", info.CertInfo.NotBefore(), info.CertInfo.ValidityParseError)
+		writeField(out, "Not After", info.CertInfo.NotAfter(), info.CertInfo.ValidityParseError)
+		if info.Bygone {
+			writeField(out, "BygoneSSL", "True", info.CertInfo.ValidityParseError)
+		}
+	}
+	writeField(out, "Log Entry", fmt.Sprintf("%d @ %s (%s)", info.Entry.Index, info.LogUri, info.typeFriendlyString()), nil)
+	writeField(out, "crt.sh", "https://crt.sh/?sha256="+fingerprint, nil)
+	if info.Filename != "" {
+		writeField(out, "Filename", info.Filename, nil)
+	}
+}
+
+func (info *EntryInfo) InvokeHookScript(command string) error {
+	cmd := exec.Command(command)
+	cmd.Env = os.Environ()
+	infoEnv := info.Environ()
+	cmd.Env = append(cmd.Env, infoEnv...)
+	stderrBuffer := bytes.Buffer{}
+	cmd.Stderr = &stderrBuffer
+	if err := cmd.Run(); err != nil {
+		if _, isExitError := err.(*exec.ExitError); isExitError {
+			return fmt.Errorf("Script failed: %s: %s", command, strings.TrimSpace(stderrBuffer.String()))
+		} else {
+			return fmt.Errorf("Failed to execute script: %s: %s", command, err)
+		}
+	}
+	return nil
+}
+
 func MatchesWildcard(dnsName string, pattern string) bool {
 	for len(pattern) > 0 {
 		if pattern[0] == '*' {

loglist/helpers.go

@@ -13,16 +13,12 @@ import (
 	"time"
 )
 
-// Return all tiled and non-tiled logs from all operators
 func (list *List) AllLogs() []*Log {
 	logs := []*Log{}
 	for operator := range list.Operators {
 		for log := range list.Operators[operator].Logs {
 			logs = append(logs, &list.Operators[operator].Logs[log])
 		}
-		for log := range list.Operators[operator].TiledLogs {
-			logs = append(logs, &list.Operators[operator].TiledLogs[log])
-		}
 	}
 	return logs
 }

loglist/load.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2020, 2023 Opsmate, Inc.
+// Copyright (C) 2020 Opsmate, Inc.
 //
 // This Source Code Form is subject to the terms of the Mozilla
 // Public License, v. 2.0. If a copy of the MPL was not distributed
@@ -10,96 +10,39 @@
 package loglist
 
 import (
-	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"io"
 	"net/http"
-	"os"
+	"io/ioutil"
 	"strings"
-	"time"
 )
 
-var UserAgent = "certspotter"
+func Load(urlOrFile string) (*List, error) {
-
-type ModificationToken struct {
-	etag     string
-	modified time.Time
-}
-
-var ErrNotModified = errors.New("loglist has not been modified")
-
-func newModificationToken(response *http.Response) *ModificationToken {
-	token := &ModificationToken{
-		etag: response.Header.Get("ETag"),
-	}
-	if t, err := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified")); err == nil {
-		token.modified = t
-	}
-	return token
-}
-
-func (token *ModificationToken) setRequestHeaders(request *http.Request) {
-	if token.etag != "" {
-		request.Header.Set("If-None-Match", token.etag)
-	} else if !token.modified.IsZero() {
-		request.Header.Set("If-Modified-Since", token.modified.Format(http.TimeFormat))
-	}
-}
-
-func Load(ctx context.Context, urlOrFile string) (*List, error) {
-	list, _, err := LoadIfModified(ctx, urlOrFile, nil)
-	return list, err
-}
-
-func LoadIfModified(ctx context.Context, urlOrFile string, token *ModificationToken) (*List, *ModificationToken, error) {
 	if strings.HasPrefix(urlOrFile, "https://") {
-		return FetchIfModified(ctx, urlOrFile, token)
+		return Fetch(urlOrFile)
 	} else {
-		list, err := ReadFile(urlOrFile)
+		return ReadFile(urlOrFile)
-		return list, nil, err
 	}
 }
 
-func Fetch(ctx context.Context, url string) (*List, error) {
+func Fetch(url string) (*List, error) {
-	list, _, err := FetchIfModified(ctx, url, nil)
+	response, err := http.Get(url)
-	return list, err
-}
-
-func FetchIfModified(ctx context.Context, url string, token *ModificationToken) (*List, *ModificationToken, error) {
-	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
-	request.Header.Set("User-Agent", UserAgent)
+	content, err := ioutil.ReadAll(response.Body)
-	if token != nil {
-		token.setRequestHeaders(request)
-	}
-	response, err := http.DefaultClient.Do(request)
-	if err != nil {
-		return nil, nil, err
-	}
-	content, err := io.ReadAll(response.Body)
 	response.Body.Close()
 	if err != nil {
-		return nil, nil, err
+		return nil, err
-	}
-	if token != nil && response.StatusCode == http.StatusNotModified {
-		return nil, nil, ErrNotModified
 	}
 	if response.StatusCode != 200 {
-		return nil, nil, fmt.Errorf("%s: %s", url, response.Status)
+		return nil, fmt.Errorf("%s: %s", url, response.Status)
 	}
-	list, err := Unmarshal(content)
+	return Unmarshal(content)
-	if err != nil {
-		return nil, nil, fmt.Errorf("error parsing %s: %w", url, err)
-	}
-	return list, newModificationToken(response), err
 }
 
 func ReadFile(filename string) (*List, error) {
-	content, err := os.ReadFile(filename)
+	content, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return nil, err
 	}

loglist/schema.go

@@ -16,25 +16,21 @@ import (
 )
 
 type List struct {
-	Version          string     `json:"version"`
+	Version   string     `json:"version"`
-	LogListTimestamp time.Time  `json:"log_list_timestamp"` // Only present in v3 of schema
+	Operators []Operator `json:"operators"`
-	Operators        []Operator `json:"operators"`
 }
 
 type Operator struct {
-	Name      string   `json:"name"`
+	Name  string   `json:"name"`
-	Email     []string `json:"email"`
+	Email []string `json:"email"`
-	Logs      []Log    `json:"logs"`
+	Logs  []Log    `json:"logs"`
-	TiledLogs []Log    `json:"tiled_logs"`
 }
 
 type Log struct {
 	Key              []byte        `json:"key"`
 	LogID            ct.SHA256Hash `json:"log_id"`
 	MMD              int           `json:"mmd"`
-	URL              string        `json:"url,omitempty"`            // only for rfc6962 logs
+	URL              string        `json:"url"`
-	SubmissionURL    string        `json:"submission_url,omitempty"` // only for static-ct-api logs
-	MonitoringURL    string        `json:"monitoring_url,omitempty"` // only for static-ct-api logs
 	Description      string        `json:"description"`
 	State            State         `json:"state"`
 	DNS              string        `json:"dns"`
@@ -43,31 +39,6 @@ type Log struct {
 		StartInclusive time.Time `json:"start_inclusive"`
 		EndExclusive   time.Time `json:"end_exclusive"`
 	} `json:"temporal_interval"`
-
-	// TODO: add previous_operators
-}
-
-func (log *Log) IsRFC6962() bool     { return log.URL != "" }
-func (log *Log) IsStaticCTAPI() bool { return log.SubmissionURL != "" && log.MonitoringURL != "" }
-
-// Return URL prefix for submission using the RFC6962 protocol
-func (log *Log) GetSubmissionURL() string {
-	if log.SubmissionURL != "" {
-		return log.SubmissionURL
-	} else {
-		return log.URL
-	}
-}
-
-// Return URL prefix for monitoring.
-// Since the protocol understood by the URL might be either RFC6962 or static-ct-api, this URL is
-// only useful for informational purposes.
-func (log *Log) GetMonitoringURL() string {
-	if log.MonitoringURL != "" {
-		return log.MonitoringURL
-	} else {
-		return log.URL
-	}
 }
 
 type State struct {

loglist/validate.go

@@ -26,12 +26,7 @@ func (list *List) Validate() error {
 func (operator *Operator) Validate() error {
 	for i := range operator.Logs {
 		if err := operator.Logs[i].Validate(); err != nil {
-			return fmt.Errorf("problem with %dth non-tiled log (%s): %w", i, operator.Logs[i].LogIDString(), err)
+			return fmt.Errorf("problem with %dth log (%s): %w", i, operator.Logs[i].LogIDString(), err)
-		}
-	}
-	for i := range operator.TiledLogs {
-		if err := operator.TiledLogs[i].Validate(); err != nil {
-			return fmt.Errorf("problem with %dth tiled log (%s): %w", i, operator.TiledLogs[i].LogIDString(), err)
 		}
 	}
 	return nil
@@ -42,12 +37,5 @@ func (log *Log) Validate() error {
 	if log.LogID != realLogID {
 		return fmt.Errorf("log ID does not match log key")
 	}
-
-	if !log.IsRFC6962() && !log.IsStaticCTAPI() {
-		return fmt.Errorf("URL(s) not provided")
-	} else if log.IsRFC6962() && log.IsStaticCTAPI() {
-		return fmt.Errorf("inconsistent URLs provided")
-	}
-
 	return nil
 }

man/.gitignore

@@ -1 +0,0 @@
-*.8

man/Makefile

@@ -1,9 +0,0 @@
-all: certspotter-script.8 certspotter.8
-
-%.8: %.md
-	lowdown -s -Tman \
-		-M title:$(basename $(notdir $@)) \
-		-M section:$(subst .,,$(suffix $@)) \
-		-M date:$(if $(SOURCE_DATE_EPOCH),$(shell date -I -u -d "@$(SOURCE_DATE_EPOCH)"),$(shell date -I -u)) \
-		-o $@ $<
-

man/certspotter-script.md

@@ -1,258 +0,0 @@
-# NAME
-
-**certspotter-script** - Certificate Transparency Log Monitor (hook script)
-
-# DESCRIPTION
-
-**certspotter-script** is *any* program that is executed by **certspotter(8)**
-when it needs to notify you about an event, such as detecting a certificate for
-a domain on your watch list.
-
-Scripts are placed in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` directory
-(`~/.certspotter/hooks.d` by default), or specified on the command line
-using the `-script` argument.
-
-# ENVIRONMENT
-
-## Event information
-
-The following environment variables are set for all types of events:
-
-`EVENT`
-
-:   One of the following values, indicating the type of event:
-
-      * `discovered_cert` - certspotter has discovered a certificate for a
-      domain on your watch list.
-
-      * `malformed_cert` - certspotter can't determine if a certificate
-      matches your watch list because the certificate or the log entry
-      is malformed.
-
-      * `error` - a problem is preventing certspotter from monitoring all
-      logs.
-
-    Additional event types may be defined in the future, so your script should
-    be able to handle unknown values.
-
-`SUMMARY`
-
-:   A short human-readable string describing the event.  This is the same string
-    used in the subject line of emails sent by certspotter.
-
-
-## Discovered certificate information
-
-The following environment variables are set for `discovered_cert` events:
-
-`WATCH_ITEM`
-
-:    The item from your watch list which matches this certificate.
-     (If more than one item matches, the first one is used.)
-
-`LOG_URI`
-
-:    The URI of the log containing the certificate.
-
-`ENTRY_INDEX`
-
-:    The index of the log entry containing the certificate.
-
-`TBS_SHA256`
-
-:    The hex-encoded SHA-256 digest of the TBSCertificate, as defined in RFC 6962 Section 3.2.
-     Certificates and their corresponding precertificates have the same `TBS_SHA256` value.
-
-`CERT_SHA256`
-
-:    The hex-encoded SHA-256 digest (sometimes called fingerprint) of the certificate.
-     The digest is computed over the ASN.1 DER encoding. 
-
-`PUBKEY_SHA256`
-
-:    The hex-encoded SHA-256 digest of the certificate's Subject Public Key Info.
-
-`CERT_FILENAME`
-
-:    Path to a file containing the PEM-encoded certificate chain.  Not set if `-no_save` was used.
-
-`JSON_FILENAME`
-
-:    Path to a JSON file containing additional information about the certificate.  See below for the format of the JSON file.
-     Not set if `-no_save` was used.
-
-`TEXT_FILENAME`
-
-:    Path to a text file containing information about the certificate.  This file contains the same text that
-     certspotter uses in emails.  You should not attempt to parse this file because its format may change in the future.
-     Not set if `-no_save` was used.
-
-`NOT_BEFORE`, `NOT_BEFORE_UNIXTIME`, `NOT_BEFORE_RFC3339`
-
-:    The not before time of the certificate, in a human-readable format, seconds since the UNIX epoch, and RFC3339, respectively.  These variables may be unset if there was a parse error, in which case `VALIDITY_PARSE_ERROR` is set.
-
-`NOT_AFTER`, `NOT_AFTER_UNIXTIME`, `NOT_AFTER_RFC3339`
-
-:    The not after (expiration) time of the certificate, in a human-readable format, seconds since the UNIX epoch, and RFC3339, respectively.  These variables may be unset if there was a parse error, in which case `VALIDITY_PARSE_ERROR` is set.
-
-`VALIDITY_PARSE_ERROR`
-
-:    Error parsing not before and not after, if any.  If this variable is set, then the `NOT_BEFORE` and `NOT_AFTER` family of variables are unset.
-
-`SUBJECT_DN`
-
-:    The distinguished name of the certificate's subject.  This variable may be unset if there was a parse error, in which case `SUBJECT_PARSE_ERROR` is set.
-
-`SUBJECT_PARSE_ERROR`
-
-:    Error parsing the subject, if any.  If this variable is set, then `SUBJECT_DN` is unset.
-
-`ISSUER_DN`
-
-:    The distinguished name of the certificate's issuer.  This variable may be unset if there was a parse error, in which case `ISSUER_PARSE_ERROR` is set.
-
-`ISSUER_PARSE_ERROR`
-
-:    Error parsing the issuer, if any.  If this variable is set, then `ISSUER_DN` is unset.
-
-`SERIAL`
-
-:    The hex-encoded serial number of the certificate.  Prefixed with a minus (-) sign if negative.  This variable may be unset if there was a parse error, in which case `SERIAL_PARSE_ERROR` is set.
-
-`SERIAL_PARSE_ERROR`
-
-:    Error parsing the serial number, if any.  If this variable is set, then `SERIAL` is unset.
-
-## Malformed certificate information
-
-The following environment variables are set for `malformed_cert` events:
-
-`LOG_URI`
-
-:    The URI of the log containing the malformed certificate.
-
-`ENTRY_INDEX`
-
-:    The index of the log entry containing the malformed certificate.
-
-`LEAF_HASH`
-
-:    The base64-encoded Merkle hash of the leaf containing the malformed certificate.
-
-`PARSE_ERROR`
-
-:    A human-readable string describing why the certificate is malformed.
-
-`ENTRY_FILENAME`
-
-:    Path to a file containing the JSON log entry.  The file contains a JSON object with two fields, `leaf_input` and `extra_data`, as described in RFC 6962 Section 4.6.
-
-`TEXT_FILENAME`
-
-:    Path to a text file containing a description of the malformed certificate.  This file contains the same text that certspotter uses in emails.
-
-## Error information
-
-The following environment variables are set for `error` events:
-
-`TEXT_FILENAME`
-
-:    Path to a text file containing a description of the error.  This file contains the same text that certspotter uses in emails.
-
-# JSON FILE FORMAT
-
-Unless `-no_save` is used, certspotter saves a JSON file for every discovered certificate
-under `$CERTSPOTTER_STATE_DIR`, and puts the path to the file in `$JSON_FILENAME`.  Your
-script can read the JSON file, such as with the jq(1) command, to get additional information
-about the certificate which isn't appropriate for environment variables.
-
-The JSON file contains an object with the following fields:
-
-`tbs_sha256`
-
-:    A string containing the hex-encoded SHA-256 digest of the TBSCertificate, as defined in RFC 6962 Section 3.2.
-     Certificates and their corresponding precertificates have the same `tbs_sha256` value.
-
-`pubkey_sha256`
-
-:    A string containing the hex-encoded SHA-256 digest of the certificate's Subject Public Key Info.
-
-`dns_names`
-
-:    An array of strings containing the DNS names for which the
-     certificate is valid, taken from both the DNS subject alternative names
-     (SANs) and the subject common name (CN). Internationalized domain names
-     are encoded in Punycode.
-
-`ip_addresses`
-
-:    An array of strings containing the IP addresses for which the certificate is valid,
-     taken from both the IP subject alternative names (SANs) and the subject common name (CN).
-
-`not_before`
-
-:    A string containing the not before time of the certificate in RFC3339 format.
-     Null if there was an error parsing the certificate's validity.
-
-`not_after`
-
-:    A string containing the not after (expiration) time of the certificate in RFC3339 format.
-     Null if there was an error parsing the certificate's validity.
-
-Additional fields will be added in the future based on user feedback. Please open
-an issue at <https://github.com/SSLMate/certspotter> if you have a use case for another field.
-
-# EXAMPLES
-
-Example environment variables for a `discovered_cert` event:
-
-```
-CERT_FILENAME=/home/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.pem
-CERT_SHA256=3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a
-ENTRY_INDEX=6464843
-EVENT=discovered_cert
-ISSUER_DN=C=GB, ST=Greater Manchester, L=Salford, O=Sectigo Limited, CN=Sectigo RSA Domain Validation Secure Server CA
-JSON_FILENAME=/usr2/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.v1.json
-LOG_URI=https://ct.cloudflare.com/logs/nimbus2024/
-NOT_AFTER='2024-01-26 03:47:26 +0000 UTC'
-NOT_AFTER_RFC3339=2024-01-26T03:47:26Z
-NOT_AFTER_UNIXTIME=1706240846
-NOT_BEFORE='2023-01-31 03:47:26 +0000 UTC'
-NOT_BEFORE_RFC3339=2023-01-31T03:47:26Z
-NOT_BEFORE_UNIXTIME=1675136846
-PUBKEY_SHA256=33ac1d9b9e56005ccac045eac2398b3e9dd6b3f5b66ae6260f2d478c7c0d82c8
-SERIAL=c170fbf3bf27481e5c351a4db6f2dc5f
-SUBJECT_DN=CN=sslmate.com
-SUMMARY='certificate discovered for .sslmate.com'
-TBS_SHA256=2388ee81c6f45cffc73e68a35fa8921e839e20acc9a98e8e6dcaea07cbfbdef8
-TEXT_FILENAME=/usr2/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.txt
-WATCH_ITEM=.sslmate.com
-```
-
-Example JSON file for a discovered certificate:
-
-```
-{
-  "dns_names": [
-    "sslmate.com",
-    "www.sslmate.com"
-  ],
-  "ip_addresses": [],
-  "not_after": "2024-01-26T03:47:26Z",
-  "not_before": "2023-01-31T03:47:26Z",
-  "pubkey_sha256": "33ac1d9b9e56005ccac045eac2398b3e9dd6b3f5b66ae6260f2d478c7c0d82c8",
-  "tbs_sha256": "2388ee81c6f45cffc73e68a35fa8921e839e20acc9a98e8e6dcaea07cbfbdef8"
-}
-```
-
-# SEE ALSO
-
-certspotter(8)
-
-# COPYRIGHT
-
-Copyright (c) 2016-2023 Opsmate, Inc.
-
-# BUGS
-
-Report bugs to <https://github.com/SSLMate/certspotter>.

man/certspotter.md

@@ -1,242 +0,0 @@
-# NAME
-
-**certspotter** - Certificate Transparency Log Monitor
-
-# SYNOPSIS
-
-**certspotter** [`-start_at_end`] [`-watchlist` *FILENAME*] [`-email` *ADDRESS*] `...`
-
-# DESCRIPTION
-
-**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that
-alerts you when a SSL/TLS certificate is issued for one of your domains.
-Cert Spotter is easier to use than other open source CT monitors, since it does not require
-a database. It's also more robust, since it uses a special certificate parser
-that ensures it won't miss certificates.
-
-Cert Spotter is also available as a hosted service by SSLMate,
-<https://sslmate.com/certspotter>.
-
-You can use Cert Spotter to detect:
-
- * Certificates issued to attackers who have compromised your DNS and
-   are redirecting your visitors to their malicious site.
- * Certificates issued to attackers who have taken over an abandoned
-   sub-domain in order to serve malware under your name.
- * Certificates issued to attackers who have compromised a certificate
-   authority and want to impersonate your site.
- * Certificates issued in violation of your corporate policy
-   or outside of your centralized certificate procurement process.
-
-# OPTIONS
-
--batch_size *NUMBER*
-
-:   Maximum number of entries to request per call to get-entries.
-    You should not generally need to change this. Defaults to 1000.
-
--email *ADDRESS*
-
-:   Email address to contact when a matching certificate is discovered, or
-    an error occurs.  You can specify this option more than once to email
-    multiple addresses.  Your system must have a working sendmail(1) command.
-
-    Regardless of the `-email` option, certspotter also emails any address listed
-    in `$CERTSPOTTER_CONFIG_DIR/email_recipients` file
-    (`~/.certspotter/email_recipients` by default).  (One address per line,
-    blank lines are ignored.)  This file is read only at startup, so you
-    must restart certspotter if you change it.
-
--healthcheck *INTERVAL*
-
-:   Perform a health check at the given interval (default: "24h") as described
-    below.  *INTERVAL* must be a decimal number followed by "h" for hours or
-    "m" for minutes.
-
--logs *ADDRESS*
-
-:   Filename or HTTPS URL of a v2 or v3 JSON log list containing logs to monitor.
-    The schema for this file can be found at <https://www.gstatic.com/ct/log_list/v3/log_list_schema.json>.
-    Defaults to <https://loglist.certspotter.org/monitor.json>, which includes
-    the union of active logs recognized by Chrome and Apple.  certspotter periodically
-    reloads the log list in case it has changed.
-
--no\_save
-
-:   Do not save a copy of matching certificates. Note that enabling this option
-    will cause you to receive duplicate notifications, since certspotter will
-    have no way of knowing if you've been previously notified about a certificate.
-
--script *COMMAND*
-
-:   Command to execute when a matching certificate is found or an error occurs. See
-    certspotter-script(8) for information about the interface to scripts.
-
-    Regardless of the `-script` option, certspotter also executes any executable
-    file in the `$CERTSPOTTER_CONFIG_DIR/hooks.d` directory
-    (`~/.certspotter/hooks.d` by default).
-
--start\_at\_end
-
-:   Start monitoring logs from the end rather than the beginning.
-
-    **WARNING**: monitoring from the beginning guarantees detection of all
-    certificates, but requires downloading hundreds of millions of
-    certificates, which takes days.
-
--state\_dir *PATH*
-
-:   Directory for storing state. Defaults to `$CERTSPOTTER_STATE_DIR`, which is
-    "~/.certspotter" by default.
-
--stdout
-
-:   Write matching certificates and errors to stdout.
-
--verbose
-
-:   Be verbose.
-
--version
-
-:   Print version and exit.
-
--watchlist *PATH*
-
-:   File containing DNS names to monitor, one per line.  To monitor an entire
-    domain namespace (including the domain itself and all sub-domains) prefix
-    the domain name with a dot (e.g. ".example.com").  To monitor a single DNS
-    name only, do not prefix the name with a dot.
-    
-    Defaults to `$CERTSPOTTER_CONFIG_DIR/watchlist`, which is
-    "~/.certspotter/watchlist" by default.
-    Specify `-` to read the watch list from stdin.
-    
-    certspotter reads the watch list only when starting up, so you must restart
-    certspotter if you change it.
-
-# NOTIFICATIONS
-
-When certspotter detects a certificate matching your watchlist, or encounters
-an error that is preventing it from discovering certificates, it notifies you
-as follows:
-
-* Emails any address specified by the `-email` command line flag.
-
-* Emails any address listed in the `$CERTSPOTTER_CONFIG_DIR/email_recipients`
-  file (`~/.certspotter/email_recipients` by default).  (One address per line,
-  blank lines are ignored.)  This file is read only at startup, so you
-  must restart certspotter if you change it.
-
-* Executes the script specified by the `-script` command line flag.
-
-* Executes every executable file in the `$CERTSPOTTER_CONFIG_DIR/hooks.d`
- directory (`~/.certspotter/hooks.d` by default).
-
-* Writes the notification to standard out if the `-stdout` flag was specified.
-
-Sending email requires a working sendmail(1) command.  For details about
-the script interface, see certspotter-script(8).
-
-# OPERATION
-
-certspotter continuously monitors all browser-recognized Certificate
-Transparency logs looking for certificates which are valid for any domain
-on your watch list. When certspotter detects a matching certificate, it
-emails you, executes a script, and/or writes a report to standard out,
-as described above.
-
-certspotter also saves a copy of matching certificates in
-`$CERTSPOTTER_STATE_DIR/certs` ("~/.certspotter/certs" by default)
-unless you specify the `-no_save` option.
-
-When certspotter has not previously monitored a log, it can either start
-monitoring the log from the beginning, or seek to the end of the log and
-start monitoring from there. Monitoring from the beginning guarantees
-detection of all certificates, but requires downloading hundreds of
-millions of certificates, which takes days. The default behavior is to
-monitor from the beginning. To start monitoring new logs from the end,
-specify the `-start_at_end` option.
-
-If certspotter has previously monitored a log, it resumes monitoring
-the log from the previous position.  This means that if you add
-a domain to your watch list, certspotter will not detect any certificates
-that were logged prior to the addition.  To detect such certificates,
-you must delete `$CERTSPOTTER_STATE_DIR/logs`, which will cause certspotter
-to restart monitoring from the very beginning of each log (provided
-`-start_at_end` is not specified).  This will cause certspotter to download
-hundreds of millions of certificates, which takes days.  To find preexisting
-certificates, it's faster to use the Cert Spotter service
-<https://sslmate.com/certspotter>, SSLMate's Certificate Transparency Search
-API <https://sslmate.com/ct_search_api>, or a CT search engine such as
-<https://crt.sh>.
-
-# ERROR HANDLING
-
-When certspotter encounters a problem with the local system (e.g. failure
-to write a file or execute a script), it prints a message to stderr and
-exits with a non-zero status.
-
-When certspotter encounters a problem monitoring a log, it prints a message
-to stderr and continues running.  It will try monitoring the log again later;
-most log errors are transient.
-
-Every 24 hours (unless overridden by `-healthcheck`), certspotter performs the
-following health checks:
-
- * Ensure that the log list has been successfully retrieved at least once
-   since the previous health check.
- * Ensure that every log has been successfully contacted at least once
-   since the previous health check.
- * Ensure that certspotter is not falling behind monitoring any logs.
-
-If any health check fails, certspotter notifies you by email, script, and/or
-standard out, as described above.
-
-Health check failures should be rare, and you should take them seriously because it means
-certspotter might not detect all certificates.  It might also be an indication
-of CT log misbehavior.  Consult certspotter's stderr output for details, and if
-you need help, file an issue at <https://github.com/SSLMate/certspotter>.
-
-# EXIT STATUS
-
-certspotter exits 0 when it receives `SIGTERM` or `SIGINT`,
-and non-zero when a serious error occurs.
-
-# ENVIRONMENT
-
-`CERTSPOTTER_STATE_DIR`
-
-:   Directory for storing state. Overridden by `-state_dir`. Defaults to
-    `~/.certspotter`.
-
-`CERTSPOTTER_CONFIG_DIR`
-
-:   Directory from which any configuration, such as the watch list, is read.
-    Defaults to `~/.certspotter`.
-
-`EMAIL`
-
-:   Email address from which to send emails. If not set, certspotter lets sendmail pick
-    the address.
-
-`HTTPS_PROXY`
-
-:   URL of proxy server for making HTTPS requests.  `http://`, `https://`, and
-    `socks5://` URLs are supported.  By default, no proxy server is used.
-
-`SENDMAIL_PATH`
-
-:   Path to the sendmail binary used for sending emails. Defaults to `/usr/sbin/sendmail`.
-
-# SEE ALSO
-
-certspotter-script(8)
-
-# COPYRIGHT
-
-Copyright (c) 2016-2023 Opsmate, Inc.
-
-# BUGS
-
-Report bugs to <https://github.com/SSLMate/certspotter>.

merkletree/collapsed_tree.go

@@ -1,182 +0,0 @@
-// Copyright (C) 2022 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package merkletree
-
-import (
-	"encoding/json"
-	"fmt"
-	"math/bits"
-	"slices"
-)
-
-// CollapsedTree is an efficient representation of a Merkle (sub)tree that permits appending
-// nodes and calculating the root hash.
-type CollapsedTree struct {
-	offset uint64
-	nodes  []Hash
-	size   uint64
-}
-
-func calculateNumNodes(size uint64) int {
-	return bits.OnesCount64(size)
-}
-
-// TODO: phase out this function
-func EmptyCollapsedTree() *CollapsedTree {
-	return &CollapsedTree{nodes: []Hash{}, size: 0}
-}
-
-// TODO: phase out this function
-func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) {
-	tree := new(CollapsedTree)
-	if err := tree.Init(nodes, size); err != nil {
-		return nil, err
-	}
-	return tree, nil
-}
-
-func (tree CollapsedTree) Equal(other CollapsedTree) bool {
-	return tree.offset == other.offset && tree.size == other.size && slices.Equal(tree.nodes, other.nodes)
-}
-
-func (tree CollapsedTree) Clone() CollapsedTree {
-	return CollapsedTree{
-		offset: tree.offset,
-		nodes:  slices.Clone(tree.nodes),
-		size:   tree.size,
-	}
-}
-
-// Add a new leaf hash to the end of the tree.
-// Returns an error if and only if the new tree would be too large for the subtree offset.
-// Always returns a nil error if tree.Offset() == 0.
-func (tree *CollapsedTree) Add(hash Hash) error {
-	if tree.offset > 0 {
-		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
-		if tree.size+1 > maxSize {
-			return fmt.Errorf("subtree at offset %d is already at maximum size %d", tree.offset, maxSize)
-		}
-	}
-	tree.nodes = append(tree.nodes, hash)
-	tree.size++
-	tree.collapse()
-	return nil
-}
-
-func (tree *CollapsedTree) Append(other CollapsedTree) error {
-	if tree.offset+tree.size != other.offset {
-		return fmt.Errorf("subtree at offset %d cannot be appended to subtree ending at offset %d", other.offset, tree.offset+tree.size)
-	}
-	if tree.offset > 0 {
-		newSize := tree.size + other.size
-		maxSize := uint64(1) << bits.TrailingZeros64(tree.offset)
-		if newSize > maxSize {
-			return fmt.Errorf("size of new subtree (%d) would exceed maximum size %d for a subtree at offset %d", newSize, maxSize, tree.offset)
-		}
-	}
-	if tree.size > 0 {
-		maxSize := uint64(1) << bits.TrailingZeros64(tree.size)
-		if other.size > maxSize {
-			return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize)
-		}
-	}
-
-	tree.nodes = append(tree.nodes, other.nodes...)
-	tree.size += other.size
-	tree.collapse()
-	return nil
-}
-
-func (tree *CollapsedTree) collapse() {
-	numNodes := calculateNumNodes(tree.size)
-	for len(tree.nodes) > numNodes {
-		left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1]
-		tree.nodes = tree.nodes[:len(tree.nodes)-2]
-		tree.nodes = append(tree.nodes, HashChildren(left, right))
-	}
-}
-
-func (tree CollapsedTree) CalculateRoot() Hash {
-	if len(tree.nodes) == 0 {
-		return HashNothing()
-	}
-	i := len(tree.nodes) - 1
-	hash := tree.nodes[i]
-	for i > 0 {
-		i -= 1
-		hash = HashChildren(tree.nodes[i], hash)
-	}
-	return hash
-}
-
-// Return the subtree offset (0 if this represents an entire tree)
-func (tree CollapsedTree) Offset() uint64 {
-	return tree.offset
-}
-
-// Return a non-nil slice containing the nodes.  The slice
-// must not be modified.
-func (tree CollapsedTree) Nodes() []Hash {
-	if tree.nodes == nil {
-		return []Hash{}
-	} else {
-		return tree.nodes
-	}
-}
-
-// Return the number of leaf nodes in the tree.
-func (tree CollapsedTree) Size() uint64 {
-	return tree.size
-}
-
-type collapsedTreeMessage struct {
-	Offset uint64 `json:"offset,omitempty"`
-	Nodes  []Hash `json:"nodes"` // never nil
-	Size   uint64 `json:"size"`
-}
-
-func (tree CollapsedTree) MarshalJSON() ([]byte, error) {
-	return json.Marshal(collapsedTreeMessage{
-		Offset: tree.offset,
-		Nodes:  tree.Nodes(),
-		Size:   tree.size,
-	})
-}
-
-func (tree *CollapsedTree) UnmarshalJSON(b []byte) error {
-	var rawTree collapsedTreeMessage
-	if err := json.Unmarshal(b, &rawTree); err != nil {
-		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
-	}
-	if err := tree.InitSubtree(rawTree.Offset, rawTree.Nodes, rawTree.Size); err != nil {
-		return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err)
-	}
-	return nil
-}
-
-func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error {
-	if len(nodes) != calculateNumNodes(size) {
-		return fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes))
-	}
-	tree.size = size
-	tree.nodes = nodes
-	return nil
-}
-
-func (tree *CollapsedTree) InitSubtree(offset uint64, nodes []Hash, size uint64) error {
-	if offset > 0 {
-		maxSize := uint64(1) << bits.TrailingZeros64(offset)
-		if size > maxSize {
-			return fmt.Errorf("subtree size (%d) is too large for offset %d (maximum size is %d)", size, offset, maxSize)
-		}
-	}
-	tree.offset = offset
-	return tree.Init(nodes, size)
-}

merkletree/fragment.go

@@ -1,121 +0,0 @@
-// Copyright (C) 2024 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package merkletree
-
-import (
-	"encoding/json"
-	"fmt"
-	"slices"
-)
-
-// FragmentedCollapsedTree represents a sequence of non-overlapping subtrees
-type FragmentedCollapsedTree struct {
-	subtrees []CollapsedTree // sorted by offset
-}
-
-func (tree *FragmentedCollapsedTree) AddHash(position uint64, hash Hash) error {
-	return tree.Add(CollapsedTree{
-		offset: position,
-		nodes:  []Hash{hash},
-		size:   1,
-	})
-}
-
-func (tree *FragmentedCollapsedTree) Add(subtree CollapsedTree) error {
-	if subtree.size == 0 {
-		return nil
-	}
-	i := len(tree.subtrees)
-	for i > 0 && tree.subtrees[i-1].offset > subtree.offset {
-		i--
-	}
-	if i > 0 && tree.subtrees[i-1].offset+tree.subtrees[i-1].size > subtree.offset {
-		return fmt.Errorf("overlaps with subtree ending at %d", tree.subtrees[i-1].offset+tree.subtrees[i-1].size)
-	}
-	if i < len(tree.subtrees) && subtree.offset+subtree.size > tree.subtrees[i].offset {
-		return fmt.Errorf("overlaps with subtree starting at %d", tree.subtrees[i].offset)
-	}
-	if i == 0 || tree.subtrees[i-1].Append(subtree) != nil {
-		tree.subtrees = slices.Insert(tree.subtrees, i, subtree)
-		i++
-	}
-	for i < len(tree.subtrees) && tree.subtrees[i-1].Append(tree.subtrees[i]) == nil {
-		tree.subtrees = slices.Delete(tree.subtrees, i, i+1)
-	}
-	return nil
-}
-
-func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error {
-	for _, subtree := range other.subtrees {
-		if err := tree.Add(subtree); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (tree FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) {
-	var prevEnd uint64
-	for i := range tree.subtrees {
-		if prevEnd != tree.subtrees[i].offset {
-			if !yield(prevEnd, tree.subtrees[i].offset) {
-				return
-			}
-		}
-		prevEnd = tree.subtrees[i].offset + tree.subtrees[i].size
-	}
-	yield(prevEnd, 0)
-}
-
-func (tree FragmentedCollapsedTree) NumSubtrees() int {
-	return len(tree.subtrees)
-}
-
-func (tree FragmentedCollapsedTree) Subtree(i int) CollapsedTree {
-	return tree.subtrees[i]
-}
-
-func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree {
-	if tree.subtrees == nil {
-		return []CollapsedTree{}
-	} else {
-		return tree.subtrees
-	}
-}
-
-// Return true iff the tree contains at least the first n nodes (without any gaps)
-func (tree FragmentedCollapsedTree) ContainsFirstN(n uint64) bool {
-	return len(tree.subtrees) >= 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size >= n
-}
-
-func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error {
-	for i := 1; i < len(subtrees); i++ {
-		if subtrees[i-1].offset+subtrees[i-1].size > subtrees[i].offset {
-			return fmt.Errorf("subtrees %d and %d overlap", i-1, i)
-		}
-	}
-	tree.subtrees = subtrees
-	return nil
-}
-
-func (tree FragmentedCollapsedTree) MarshalJSON() ([]byte, error) {
-	return json.Marshal(tree.Subtrees())
-}
-
-func (tree *FragmentedCollapsedTree) UnmarshalJSON(b []byte) error {
-	var subtrees []CollapsedTree
-	if err := json.Unmarshal(b, &subtrees); err != nil {
-		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
-	}
-	if err := tree.Init(subtrees); err != nil {
-		return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err)
-	}
-	return nil
-}

merkletree/hash.go

@@ -1,72 +0,0 @@
-// Copyright (C) 2022 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package merkletree
-
-import (
-	"crypto/sha256"
-	"encoding/base64"
-	"encoding/json"
-	"fmt"
-)
-
-const HashLen = 32
-
-type Hash [HashLen]byte
-
-func (h Hash) Base64String() string {
-	return base64.StdEncoding.EncodeToString(h[:])
-}
-
-func (h Hash) MarshalJSON() ([]byte, error) {
-	return json.Marshal(h[:])
-}
-
-func (h Hash) MarshalBinary() ([]byte, error) {
-	return h[:], nil
-}
-
-func (h *Hash) UnmarshalJSON(b []byte) error {
-	var hashBytes []byte
-	if err := json.Unmarshal(b, &hashBytes); err != nil {
-		return err
-	}
-	return h.UnmarshalBinary(hashBytes)
-}
-
-func (h *Hash) UnmarshalBinary(hashBytes []byte) error {
-	if len(hashBytes) != HashLen {
-		return fmt.Errorf("Merkle Tree hash has wrong length (should be %d bytes long, not %d)", HashLen, len(hashBytes))
-	}
-	copy(h[:], hashBytes)
-	return nil
-}
-
-func HashNothing() Hash {
-	return sha256.Sum256(nil)
-}
-
-func HashLeaf(leafBytes []byte) Hash {
-	var hash Hash
-	hasher := sha256.New()
-	hasher.Write([]byte{0x00})
-	hasher.Write(leafBytes)
-	hasher.Sum(hash[:0])
-	return hash
-}
-
-func HashChildren(left Hash, right Hash) Hash {
-	var hash Hash
-	hasher := sha256.New()
-	hasher.Write([]byte{0x01})
-	hasher.Write(left[:])
-	hasher.Write(right[:])
-	hasher.Sum(hash[:0])
-	return hash
-}

monitor/config.go

@@ -1,23 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"time"
-)
-
-type Config struct {
-	LogListSource       string
-	State               StateProvider
-	StartAtEnd          bool
-	WatchList           WatchList
-	Verbose             bool
-	HealthCheckInterval time.Duration
-}

monitor/daemon.go

@@ -1,168 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"golang.org/x/sync/errgroup"
-	"log"
-	insecurerand "math/rand"
-	"software.sslmate.com/src/certspotter/loglist"
-	"time"
-)
-
-const (
-	reloadLogListIntervalMin = 30 * time.Minute
-	reloadLogListIntervalMax = 90 * time.Minute
-)
-
-func randomDuration(min, max time.Duration) time.Duration {
-	return min + time.Duration(insecurerand.Int63n(int64(max-min+1)))
-}
-
-func reloadLogListInterval() time.Duration {
-	return randomDuration(reloadLogListIntervalMin, reloadLogListIntervalMax)
-}
-
-type task struct {
-	log  *loglist.Log
-	stop context.CancelFunc
-}
-
-type daemon struct {
-	config         *Config
-	taskgroup      *errgroup.Group
-	tasks          map[LogID]task
-	logsLoadedAt   time.Time
-	logListToken   *loglist.ModificationToken
-	logListError   string
-	logListErrorAt time.Time
-}
-
-func (daemon *daemon) healthCheck(ctx context.Context) error {
-	if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval {
-		info := &StaleLogListInfo{
-			Source:        daemon.config.LogListSource,
-			LastSuccess:   daemon.logsLoadedAt,
-			LastError:     daemon.logListError,
-			LastErrorTime: daemon.logListErrorAt,
-		}
-		if err := daemon.config.State.NotifyHealthCheckFailure(ctx, nil, info); err != nil {
-			return fmt.Errorf("error notifying about stale log list: %w", err)
-		}
-	}
-
-	for _, task := range daemon.tasks {
-		if err := healthCheckLog(ctx, daemon.config, task.log); err != nil {
-			return fmt.Errorf("error checking health of log %q: %w", task.log.URL, err)
-		}
-	}
-	return nil
-}
-
-func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task {
-	ctx, cancel := context.WithCancel(ctx)
-	daemon.taskgroup.Go(func() error {
-		defer cancel()
-		err := monitorLogContinously(ctx, daemon.config, ctlog)
-		if daemon.config.Verbose {
-			log.Printf("task for log %s stopped with error %s", ctlog.URL, err)
-		}
-		if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
-			return nil
-		} else {
-			return fmt.Errorf("error while monitoring %s: %w", ctlog.URL, err)
-		}
-	})
-	return task{log: ctlog, stop: cancel}
-}
-
-func (daemon *daemon) loadLogList(ctx context.Context) error {
-	newLogList, newToken, err := getLogList(ctx, daemon.config.LogListSource, daemon.logListToken)
-	if errors.Is(err, loglist.ErrNotModified) {
-		return nil
-	} else if err != nil {
-		return err
-	}
-
-	if daemon.config.Verbose {
-		log.Printf("fetched %d logs from %q", len(newLogList), daemon.config.LogListSource)
-	}
-
-	for logID, task := range daemon.tasks {
-		if _, exists := newLogList[logID]; exists {
-			continue
-		}
-		if daemon.config.Verbose {
-			log.Printf("stopping task for log %s", logID.Base64String())
-		}
-		task.stop()
-		delete(daemon.tasks, logID)
-	}
-	for logID, ctlog := range newLogList {
-		if _, isRunning := daemon.tasks[logID]; isRunning {
-			continue
-		}
-		if daemon.config.Verbose {
-			log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.URL)
-		}
-		daemon.tasks[logID] = daemon.startTask(ctx, ctlog)
-	}
-	daemon.logsLoadedAt = time.Now()
-	daemon.logListToken = newToken
-	return nil
-}
-
-func (daemon *daemon) run(ctx context.Context) error {
-	if err := daemon.config.State.Prepare(ctx); err != nil {
-		return fmt.Errorf("error preparing state: %w", err)
-	}
-
-	if err := daemon.loadLogList(ctx); err != nil {
-		return fmt.Errorf("error loading log list: %w", err)
-	}
-
-	reloadLogListTicker := time.NewTicker(reloadLogListInterval())
-	defer reloadLogListTicker.Stop()
-
-	healthCheckTicker := time.NewTicker(daemon.config.HealthCheckInterval)
-	defer healthCheckTicker.Stop()
-
-	for ctx.Err() == nil {
-		select {
-		case <-ctx.Done():
-		case <-reloadLogListTicker.C:
-			if err := daemon.loadLogList(ctx); err != nil {
-				daemon.logListError = err.Error()
-				daemon.logListErrorAt = time.Now()
-				recordError(ctx, daemon.config, nil, fmt.Errorf("error reloading log list (will try again later): %w", err))
-			}
-			reloadLogListTicker.Reset(reloadLogListInterval())
-		case <-healthCheckTicker.C:
-			if err := daemon.healthCheck(ctx); err != nil {
-				return err
-			}
-		}
-	}
-	return ctx.Err()
-}
-
-func Run(ctx context.Context, config *Config) error {
-	group, ctx := errgroup.WithContext(ctx)
-	daemon := &daemon{
-		config:    config,
-		taskgroup: group,
-		tasks:     make(map[LogID]task),
-	}
-	group.Go(func() error { return daemon.run(ctx) })
-	return group.Wait()
-}

monitor/discoveredcert.go

@@ -1,176 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"bytes"
-	"encoding/hex"
-	"encoding/pem"
-	"fmt"
-	"strings"
-	"time"
-
-	"software.sslmate.com/src/certspotter"
-	"software.sslmate.com/src/certspotter/ct"
-)
-
-type DiscoveredCert struct {
-	WatchItem    WatchItem
-	LogEntry     *LogEntry
-	Info         *certspotter.CertInfo
-	Chain        []ct.ASN1Cert // first entry is the leaf certificate or precertificate
-	TBSSHA256    [32]byte      // computed over Info.TBS.Raw
-	SHA256       [32]byte      // computed over Chain[0]
-	PubkeySHA256 [32]byte      // computed over Info.TBS.PublicKey.FullBytes
-	Identifiers  *certspotter.Identifiers
-}
-
-type certPaths struct {
-	certPath string
-	jsonPath string
-	textPath string
-}
-
-func (cert *DiscoveredCert) pemChain() []byte {
-	var buffer bytes.Buffer
-	for _, certBytes := range cert.Chain {
-		if err := pem.Encode(&buffer, &pem.Block{
-			Type:  "CERTIFICATE",
-			Bytes: certBytes,
-		}); err != nil {
-			panic(fmt.Errorf("encoding certificate as PEM failed unexpectedly: %w", err))
-		}
-	}
-	return buffer.Bytes()
-}
-
-func (cert *DiscoveredCert) json() any {
-	object := map[string]any{
-		"tbs_sha256":    hex.EncodeToString(cert.TBSSHA256[:]),
-		"pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]),
-		"dns_names":     cert.Identifiers.DNSNames,
-		"ip_addresses":  cert.Identifiers.IPAddrs,
-	}
-
-	if cert.Info.ValidityParseError == nil {
-		object["not_before"] = cert.Info.Validity.NotBefore
-		object["not_after"] = cert.Info.Validity.NotAfter
-	} else {
-		object["not_before"] = nil
-		object["not_after"] = nil
-	}
-
-	return object
-}
-
-func writeCertFiles(cert *DiscoveredCert, paths *certPaths) error {
-	if err := writeFile(paths.certPath, cert.pemChain(), 0666); err != nil {
-		return err
-	}
-	if err := writeJSONFile(paths.jsonPath, cert.json(), 0666); err != nil {
-		return err
-	}
-	if err := writeTextFile(paths.textPath, certNotificationText(cert, paths), 0666); err != nil {
-		return err
-	}
-	return nil
-}
-
-func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string {
-	env := []string{
-		"EVENT=discovered_cert",
-		"SUMMARY=" + certNotificationSummary(cert),
-		"CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented
-		"LOG_URI=" + cert.LogEntry.Log.URL,
-		"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
-		"WATCH_ITEM=" + cert.WatchItem.String(),
-		"TBS_SHA256=" + hex.EncodeToString(cert.TBSSHA256[:]),
-		"CERT_SHA256=" + hex.EncodeToString(cert.SHA256[:]),
-		"FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented
-		"PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]),
-		"PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented
-	}
-
-	if paths != nil {
-		env = append(env, "CERT_FILENAME="+paths.certPath)
-		env = append(env, "JSON_FILENAME="+paths.jsonPath)
-		env = append(env, "TEXT_FILENAME="+paths.textPath)
-	}
-
-	if cert.Info.ValidityParseError == nil {
-		env = append(env, "NOT_BEFORE="+cert.Info.Validity.NotBefore.String())
-		env = append(env, "NOT_BEFORE_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotBefore.Unix()))
-		env = append(env, "NOT_BEFORE_RFC3339="+cert.Info.Validity.NotBefore.Format(time.RFC3339))
-		env = append(env, "NOT_AFTER="+cert.Info.Validity.NotAfter.String())
-		env = append(env, "NOT_AFTER_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotAfter.Unix()))
-		env = append(env, "NOT_AFTER_RFC3339="+cert.Info.Validity.NotAfter.Format(time.RFC3339))
-	} else {
-		env = append(env, "VALIDITY_PARSE_ERROR="+cert.Info.ValidityParseError.Error())
-	}
-
-	if cert.Info.SubjectParseError == nil {
-		env = append(env, "SUBJECT_DN="+cert.Info.Subject.String())
-	} else {
-		env = append(env, "SUBJECT_PARSE_ERROR="+cert.Info.SubjectParseError.Error())
-	}
-
-	if cert.Info.IssuerParseError == nil {
-		env = append(env, "ISSUER_DN="+cert.Info.Issuer.String())
-	} else {
-		env = append(env, "ISSUER_PARSE_ERROR="+cert.Info.IssuerParseError.Error())
-	}
-
-	if cert.Info.SerialNumberParseError == nil {
-		env = append(env, "SERIAL="+fmt.Sprintf("%x", cert.Info.SerialNumber))
-	} else {
-		env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error())
-	}
-
-	return env
-}
-
-func certNotificationText(cert *DiscoveredCert, paths *certPaths) string {
-	// TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration)
-
-	text := new(strings.Builder)
-	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
-
-	fmt.Fprintf(text, "%x:\n", cert.SHA256)
-	for _, dnsName := range cert.Identifiers.DNSNames {
-		writeField("DNS Name", dnsName)
-	}
-	for _, ipaddr := range cert.Identifiers.IPAddrs {
-		writeField("IP Address", ipaddr)
-	}
-	writeField("Pubkey", hex.EncodeToString(cert.PubkeySHA256[:]))
-	if cert.Info.IssuerParseError == nil {
-		writeField("Issuer", cert.Info.Issuer)
-	} else {
-		writeField("Issuer", fmt.Sprintf("[unable to parse: %s]", cert.Info.IssuerParseError))
-	}
-	if cert.Info.ValidityParseError == nil {
-		writeField("Not Before", cert.Info.Validity.NotBefore)
-		writeField("Not After", cert.Info.Validity.NotAfter)
-	} else {
-		writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
-		writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
-	}
-	writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL))
-	writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:]))
-	if paths != nil {
-		writeField("Filename", paths.certPath)
-	}
-
-	return text.String()
-}
-
-func certNotificationSummary(cert *DiscoveredCert) string {
-	return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem)
-}

monitor/errors.go

@@ -1,28 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"context"
-	"log"
-
-	"software.sslmate.com/src/certspotter/loglist"
-)
-
-func recordError(ctx context.Context, config *Config, ctlog *loglist.Log, errToRecord error) {
-	if err := config.State.NotifyError(ctx, ctlog, errToRecord); err != nil {
-		log.Print("unable to notify about error: ", err)
-		if ctlog == nil {
-			log.Print(errToRecord)
-		} else {
-			log.Print(ctlog.URL, ": ", errToRecord)
-		}
-	}
-}

monitor/fileutils.go

@@ -1,56 +0,0 @@
-// Copyright (C) 2017, 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"crypto/rand"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
-	"os"
-)
-
-func randomFileSuffix() string {
-	var randomBytes [12]byte
-	if _, err := rand.Read(randomBytes[:]); err != nil {
-		panic(err)
-	}
-	return hex.EncodeToString(randomBytes[:])
-}
-
-func writeFile(filename string, data []byte, perm os.FileMode) error {
-	tempname := filename + ".tmp." + randomFileSuffix()
-	if err := os.WriteFile(tempname, data, perm); err != nil {
-		return fmt.Errorf("error writing %s: %w", filename, err)
-	}
-	if err := os.Rename(tempname, filename); err != nil {
-		os.Remove(tempname)
-		return fmt.Errorf("error writing %s: %w", filename, err)
-	}
-	return nil
-}
-
-func writeTextFile(filename string, fileText string, perm os.FileMode) error {
-	return writeFile(filename, []byte(fileText), perm)
-}
-
-func writeJSONFile(filename string, data any, perm os.FileMode) error {
-	fileBytes, err := json.Marshal(data)
-	if err != nil {
-		return err
-	}
-	fileBytes = append(fileBytes, '\n')
-	return writeFile(filename, fileBytes, perm)
-}
-
-func fileExists(filename string) bool {
-	_, err := os.Lstat(filename)
-	return err == nil
-}

monitor/fsstate.go

@@ -1,239 +0,0 @@
-// Copyright (C) 2024 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"context"
-	"encoding/hex"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/fs"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/loglist"
-)
-
-type FilesystemState struct {
-	StateDir  string
-	SaveCerts bool
-	Script    string
-	ScriptDir string
-	Email     []string
-	Stdout    bool
-}
-
-func (s *FilesystemState) logStateDir(logID LogID) string {
-	return filepath.Join(s.StateDir, "logs", logID.Base64URLString())
-}
-
-func (s *FilesystemState) Prepare(ctx context.Context) error {
-	return prepareStateDir(s.StateDir)
-}
-
-func (s *FilesystemState) PrepareLog(ctx context.Context, logID LogID) error {
-	var (
-		stateDirPath        = s.logStateDir(logID)
-		sthsDirPath         = filepath.Join(stateDirPath, "unverified_sths")
-		malformedDirPath    = filepath.Join(stateDirPath, "malformed_entries")
-		healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
-	)
-	for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} {
-		if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
-			return err
-		}
-	}
-	return nil
-}
-
-func (s *FilesystemState) LoadLogState(ctx context.Context, logID LogID) (*LogState, error) {
-	filePath := filepath.Join(s.logStateDir(logID), "state.json")
-	fileBytes, err := os.ReadFile(filePath)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil, nil
-	} else if err != nil {
-		return nil, err
-	}
-	state := new(LogState)
-	if err := json.Unmarshal(fileBytes, state); err != nil {
-		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
-	}
-	return state, nil
-}
-
-func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state *LogState) error {
-	filePath := filepath.Join(s.logStateDir(logID), "state.json")
-	return writeJSONFile(filePath, state, 0666)
-}
-
-func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
-	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
-	return storeSTHInDir(sthsDirPath, sth)
-}
-
-func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*ct.SignedTreeHead, error) {
-	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
-	return loadSTHsFromDir(sthsDirPath)
-}
-
-func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
-	sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
-	return removeSTHFromDir(sthsDirPath, sth)
-}
-
-func (s *FilesystemState) NotifyCert(ctx context.Context, cert *DiscoveredCert) error {
-	var notifiedPath string
-	var paths *certPaths
-	if s.SaveCerts {
-		hexFingerprint := hex.EncodeToString(cert.SHA256[:])
-		prefixPath := filepath.Join(s.StateDir, "certs", hexFingerprint[0:2])
-		var (
-			notifiedFilename      = "." + hexFingerprint + ".notified"
-			certFilename          = hexFingerprint + ".pem"
-			jsonFilename          = hexFingerprint + ".v1.json"
-			textFilename          = hexFingerprint + ".txt"
-			legacyCertFilename    = hexFingerprint + ".cert.pem"
-			legacyPrecertFilename = hexFingerprint + ".precert.pem"
-		)
-
-		for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} {
-			if fileExists(filepath.Join(prefixPath, filename)) {
-				return nil
-			}
-		}
-
-		if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
-			return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
-		}
-
-		notifiedPath = filepath.Join(prefixPath, notifiedFilename)
-		paths = &certPaths{
-			certPath: filepath.Join(prefixPath, certFilename),
-			jsonPath: filepath.Join(prefixPath, jsonFilename),
-			textPath: filepath.Join(prefixPath, textFilename),
-		}
-		if err := writeCertFiles(cert, paths); err != nil {
-			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
-		}
-	} else {
-		// TODO-4: save cert to temporary files, and defer their unlinking
-	}
-
-	if err := s.notify(ctx, &notification{
-		summary: certNotificationSummary(cert),
-		environ: certNotificationEnviron(cert, paths),
-		text:    certNotificationText(cert, paths),
-	}); err != nil {
-		return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
-	}
-
-	if notifiedPath != "" {
-		if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
-			return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
-		}
-	}
-
-	return nil
-}
-
-func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError error) error {
-	var (
-		dirPath   = filepath.Join(s.logStateDir(entry.Log.LogID), "malformed_entries")
-		entryPath = filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index))
-		textPath  = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index))
-	)
-
-	summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.URL)
-
-	entryJSON := struct {
-		LeafInput []byte `json:"leaf_input"`
-		ExtraData []byte `json:"extra_data"`
-	}{
-		LeafInput: entry.LeafInput,
-		ExtraData: entry.ExtraData,
-	}
-
-	text := new(strings.Builder)
-	writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
-	fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
-	writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.URL))
-	writeField("Leaf Hash", entry.LeafHash.Base64String())
-	writeField("Error", parseError.Error())
-
-	if err := writeJSONFile(entryPath, entryJSON, 0666); err != nil {
-		return fmt.Errorf("error saving JSON file: %w", err)
-	}
-	if err := writeTextFile(textPath, text.String(), 0666); err != nil {
-		return fmt.Errorf("error saving texT file: %w", err)
-	}
-
-	environ := []string{
-		"EVENT=malformed_cert",
-		"SUMMARY=" + summary,
-		"LOG_URI=" + entry.Log.URL,
-		"ENTRY_INDEX=" + fmt.Sprint(entry.Index),
-		"LEAF_HASH=" + entry.LeafHash.Base64String(),
-		"PARSE_ERROR=" + parseError.Error(),
-		"ENTRY_FILENAME=" + entryPath,
-		"TEXT_FILENAME=" + textPath,
-		"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
-	}
-
-	if err := s.notify(ctx, &notification{
-		environ: environ,
-		summary: summary,
-		text:    text.String(),
-	}); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *FilesystemState) healthCheckDir(ctlog *loglist.Log) string {
-	if ctlog == nil {
-		return filepath.Join(s.StateDir, "healthchecks")
-	} else {
-		return filepath.Join(s.logStateDir(ctlog.LogID), "healthchecks")
-	}
-}
-
-func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info HealthCheckFailure) error {
-	textPath := filepath.Join(s.healthCheckDir(ctlog), healthCheckFilename())
-	environ := []string{
-		"EVENT=error",
-		"SUMMARY=" + info.Summary(),
-		"TEXT_FILENAME=" + textPath,
-	}
-	text := info.Text()
-	if err := writeTextFile(textPath, text, 0666); err != nil {
-		return fmt.Errorf("error saving text file: %w", err)
-	}
-	if err := s.notify(ctx, &notification{
-		environ: environ,
-		summary: info.Summary(),
-		text:    text,
-	}); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, err error) error {
-	if ctlog == nil {
-		log.Print(err)
-	} else {
-		log.Print(ctlog.URL, ":", err)
-	}
-	return nil
-}

monitor/healthcheck.go

@@ -1,138 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"time"
-
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/loglist"
-)
-
-func healthCheckFilename() string {
-	return time.Now().UTC().Format(time.RFC3339) + ".txt"
-}
-
-func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error {
-	state, err := config.State.LoadLogState(ctx, ctlog.LogID)
-	if err != nil {
-		return fmt.Errorf("error loading log state: %w", err)
-	} else if state == nil {
-		return nil
-	}
-
-	if time.Since(state.LastSuccess) < config.HealthCheckInterval {
-		return nil
-	}
-
-	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
-	if err != nil {
-		return fmt.Errorf("error loading STHs: %w", err)
-	}
-
-	if len(sths) == 0 {
-		info := &StaleSTHInfo{
-			Log:         ctlog,
-			LastSuccess: state.LastSuccess,
-			LatestSTH:   state.VerifiedSTH,
-		}
-		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
-			return fmt.Errorf("error notifying about stale STH: %w", err)
-		}
-	} else {
-		info := &BacklogInfo{
-			Log:       ctlog,
-			LatestSTH: sths[len(sths)-1],
-			Position:  state.DownloadPosition.Size(),
-		}
-		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
-			return fmt.Errorf("error notifying about backlog: %w", err)
-		}
-	}
-
-	return nil
-}
-
-type HealthCheckFailure interface {
-	Summary() string
-	Text() string
-}
-
-type StaleSTHInfo struct {
-	Log         *loglist.Log
-	LastSuccess time.Time
-	LatestSTH   *ct.SignedTreeHead // may be nil
-}
-
-type BacklogInfo struct {
-	Log       *loglist.Log
-	LatestSTH *ct.SignedTreeHead
-	Position  uint64
-}
-
-type StaleLogListInfo struct {
-	Source        string
-	LastSuccess   time.Time
-	LastError     string
-	LastErrorTime time.Time
-}
-
-func (e *BacklogInfo) Backlog() uint64 {
-	return e.LatestSTH.TreeSize - e.Position
-}
-
-func (e *StaleSTHInfo) Summary() string {
-	return fmt.Sprintf("Unable to contact %s since %s", e.Log.URL, e.LastSuccess)
-}
-func (e *BacklogInfo) Summary() string {
-	return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.URL)
-}
-func (e *StaleLogListInfo) Summary() string {
-	return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess)
-}
-
-func (e *StaleSTHInfo) Text() string {
-	text := new(strings.Builder)
-	fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess)
-	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "For details, see certspotter's stderr output.\n")
-	fmt.Fprintf(text, "\n")
-	if e.LatestSTH != nil {
-		fmt.Fprintf(text, "Latest known log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.TimestampTime())
-	} else {
-		fmt.Fprintf(text, "Latest known log size = none\n")
-	}
-	return text.String()
-}
-func (e *BacklogInfo) Text() string {
-	text := new(strings.Builder)
-	fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL)
-	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n")
-	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.TimestampTime())
-	fmt.Fprintf(text, "Current position = %d\n", e.Position)
-	fmt.Fprintf(text, "         Backlog = %d\n", e.Backlog())
-	return text.String()
-}
-func (e *StaleLogListInfo) Text() string {
-	text := new(strings.Builder)
-	fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess)
-	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "Last error (at %s): %s\n", e.LastErrorTime, e.LastError)
-	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "Consequentially, certspotter may not be monitoring all logs, and might fail to detect certificates.\n")
-	return text.String()
-}
-
-// TODO-3: make the errors more actionable

monitor/loglist.go

@@ -1,38 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"context"
-	"fmt"
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/loglist"
-)
-
-type LogID = ct.SHA256Hash
-
-func getLogList(ctx context.Context, source string, token *loglist.ModificationToken) (map[LogID]*loglist.Log, *loglist.ModificationToken, error) {
-	list, newToken, err := loglist.LoadIfModified(ctx, source, token)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	logs := make(map[LogID]*loglist.Log)
-	for operatorIndex := range list.Operators {
-		for logIndex := range list.Operators[operatorIndex].Logs {
-			log := &list.Operators[operatorIndex].Logs[logIndex]
-			if _, exists := logs[log.LogID]; exists {
-				return nil, nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String())
-			}
-			logs[log.LogID] = log
-		}
-	}
-	return logs, newToken, nil
-}

monitor/mailutils.go

@@ -1,34 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"crypto/rand"
-	"encoding/hex"
-	"os"
-)
-
-const mailDateFormat = "Mon, 2 Jan 2006 15:04:05 -0700"
-
-func generateMessageID() string {
-	var randomBytes [16]byte
-	if _, err := rand.Read(randomBytes[:]); err != nil {
-		panic(err)
-	}
-	return hex.EncodeToString(randomBytes[:]) + "@selfhosted.certspotter.org"
-}
-
-func sendmailPath() string {
-	if envVar := os.Getenv("SENDMAIL_PATH"); envVar != "" {
-		return envVar
-	} else {
-		return "/usr/sbin/sendmail"
-	}
-}

monitor/monitor.go

@@ -1,285 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"context"
-	"crypto/x509"
-	"errors"
-	"fmt"
-	"log"
-	"strings"
-	"time"
-
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/ct/client"
-	"software.sslmate.com/src/certspotter/loglist"
-	"software.sslmate.com/src/certspotter/merkletree"
-)
-
-const (
-	maxGetEntriesSize  = 1000
-	monitorLogInterval = 5 * time.Minute
-)
-
-func isFatalLogError(err error) bool {
-	return errors.Is(err, context.Canceled)
-}
-
-func newLogClient(ctlog *loglist.Log) (*client.LogClient, error) {
-	logKey, err := x509.ParsePKIXPublicKey(ctlog.Key)
-	if err != nil {
-		return nil, fmt.Errorf("error parsing log key: %w", err)
-	}
-	verifier, err := ct.NewSignatureVerifier(logKey)
-	if err != nil {
-		return nil, fmt.Errorf("error with log key: %w", err)
-	}
-	return client.NewWithVerifier(strings.TrimRight(ctlog.URL, "/"), verifier), nil
-}
-
-func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) error {
-	logClient, err := newLogClient(ctlog)
-	if err != nil {
-		return err
-	}
-
-	ticker := time.NewTicker(monitorLogInterval)
-	defer ticker.Stop()
-
-	for ctx.Err() == nil {
-		if err := monitorLog(ctx, config, ctlog, logClient); err != nil {
-			return err
-		}
-		select {
-		case <-ctx.Done():
-		case <-ticker.C:
-		}
-	}
-	return ctx.Err()
-}
-
-func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClient *client.LogClient) (returnedErr error) {
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	if err := config.State.PrepareLog(ctx, ctlog.LogID); err != nil {
-		return fmt.Errorf("error preparing state: %w", err)
-	}
-
-	startTime := time.Now()
-	latestSTH, err := logClient.GetSTH(ctx)
-	if isFatalLogError(err) {
-		return err
-	} else if err != nil {
-		recordError(ctx, config, ctlog, fmt.Errorf("error fetching latest STH: %w", err))
-		return nil
-	}
-	latestSTH.LogID = ctlog.LogID
-	if err := config.State.StoreSTH(ctx, ctlog.LogID, latestSTH); err != nil {
-		return fmt.Errorf("error storing latest STH: %w", err)
-	}
-
-	state, err := config.State.LoadLogState(ctx, ctlog.LogID)
-	if err != nil {
-		return fmt.Errorf("error loading log state: %w", err)
-	}
-	if state == nil {
-		if config.StartAtEnd {
-			tree, err := reconstructTree(ctx, logClient, latestSTH)
-			if isFatalLogError(err) {
-				return err
-			} else if err != nil {
-				recordError(ctx, config, ctlog, fmt.Errorf("error reconstructing tree of size %d: %w", latestSTH.TreeSize, err))
-				return nil
-			}
-			state = &LogState{
-				DownloadPosition: tree,
-				VerifiedPosition: tree,
-				VerifiedSTH:      latestSTH,
-				LastSuccess:      startTime.UTC(),
-			}
-		} else {
-			state = &LogState{
-				DownloadPosition: merkletree.EmptyCollapsedTree(),
-				VerifiedPosition: merkletree.EmptyCollapsedTree(),
-				VerifiedSTH:      nil,
-				LastSuccess:      startTime.UTC(),
-			}
-		}
-		if config.Verbose {
-			log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size())
-		}
-		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
-			return fmt.Errorf("error storing log state: %w", err)
-		}
-	}
-
-	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
-	if err != nil {
-		return fmt.Errorf("error loading STHs: %w", err)
-	}
-
-	for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() {
-		// TODO-4: audit sths[0] against state.VerifiedSTH
-		if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
-			return fmt.Errorf("error removing STH: %w", err)
-		}
-		sths = sths[1:]
-	}
-
-	defer func() {
-		if config.Verbose {
-			log.Printf("saving state in defer for %s", ctlog.URL)
-		}
-		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil && returnedErr == nil {
-			returnedErr = fmt.Errorf("error storing log state: %w", err)
-		}
-	}()
-
-	if len(sths) == 0 {
-		state.LastSuccess = startTime.UTC()
-		return nil
-	}
-
-	var (
-		downloadBegin = state.DownloadPosition.Size()
-		downloadEnd   = sths[len(sths)-1].TreeSize
-		entries       = make(chan client.GetEntriesItem, maxGetEntriesSize)
-		downloadErr   error
-	)
-	if config.Verbose {
-		log.Printf("downloading entries from %s in range [%d, %d)", ctlog.URL, downloadBegin, downloadEnd)
-	}
-	go func() {
-		defer close(entries)
-		downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd)
-	}()
-	for rawEntry := range entries {
-		entry := &LogEntry{
-			Log:       ctlog,
-			Index:     state.DownloadPosition.Size(),
-			LeafInput: rawEntry.LeafInput,
-			ExtraData: rawEntry.ExtraData,
-			LeafHash:  merkletree.HashLeaf(rawEntry.LeafInput),
-		}
-		if err := processLogEntry(ctx, config, entry); err != nil {
-			return fmt.Errorf("error processing entry %d: %w", entry.Index, err)
-		}
-
-		state.DownloadPosition.Add(entry.LeafHash)
-		rootHash := state.DownloadPosition.CalculateRoot()
-		shouldSaveState := state.DownloadPosition.Size()%10000 == 0
-
-		for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize {
-			if merkletree.Hash(sths[0].SHA256RootHash) != rootHash {
-				recordError(ctx, config, ctlog, fmt.Errorf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
-
-				state.DownloadPosition = state.VerifiedPosition
-				if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
-					return fmt.Errorf("error storing log state: %w", err)
-				}
-				return nil
-			}
-
-			state.VerifiedPosition = state.DownloadPosition
-			state.VerifiedSTH = sths[0]
-			shouldSaveState = true
-			if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
-				return fmt.Errorf("error removing verified STH: %w", err)
-			}
-
-			sths = sths[1:]
-		}
-
-		if shouldSaveState {
-			if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
-				return fmt.Errorf("error storing state file: %w", err)
-			}
-		}
-	}
-
-	if isFatalLogError(downloadErr) {
-		return downloadErr
-	} else if downloadErr != nil {
-		recordError(ctx, config, ctlog, fmt.Errorf("error downloading entries: %w", downloadErr))
-		return nil
-	}
-
-	if config.Verbose {
-		log.Printf("finished downloading entries from %s", ctlog.URL)
-	}
-
-	state.LastSuccess = startTime.UTC()
-	return nil
-}
-
-func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error {
-	for begin < end && ctx.Err() == nil {
-		size := end - begin
-		if size > maxGetEntriesSize {
-			size = maxGetEntriesSize
-		}
-		entries, err := logClient.GetRawEntries(ctx, begin, begin+size-1)
-		if err != nil {
-			return err
-		}
-		for _, entry := range entries {
-			if ctx.Err() != nil {
-				return ctx.Err()
-			}
-			select {
-			case <-ctx.Done():
-				return ctx.Err()
-			case entriesChan <- entry:
-			}
-		}
-		begin += uint64(len(entries))
-	}
-	return ctx.Err()
-}
-
-func reconstructTree(ctx context.Context, logClient *client.LogClient, sth *ct.SignedTreeHead) (*merkletree.CollapsedTree, error) {
-	if sth.TreeSize == 0 {
-		return merkletree.EmptyCollapsedTree(), nil
-	}
-	entries, err := logClient.GetRawEntries(ctx, sth.TreeSize-1, sth.TreeSize-1)
-	if err != nil {
-		return nil, err
-	}
-	leafHash := merkletree.HashLeaf(entries[0].LeafInput)
-
-	var tree *merkletree.CollapsedTree
-	if sth.TreeSize > 1 {
-		// XXX: if leafHash is in the tree in more than one place, this might not return the proof that we need ... get-entry-and-proof avoids this problem but not all logs support it
-		auditPath, _, err := logClient.GetAuditProof(ctx, leafHash[:], sth.TreeSize)
-		if err != nil {
-			return nil, err
-		}
-		hashes := make([]merkletree.Hash, len(auditPath))
-		for i := range hashes {
-			copy(hashes[i][:], auditPath[len(auditPath)-i-1])
-		}
-		tree, err = merkletree.NewCollapsedTree(hashes, sth.TreeSize-1)
-		if err != nil {
-			return nil, fmt.Errorf("log returned invalid audit proof for %x to %d: %w", leafHash, sth.TreeSize, err)
-		}
-	} else {
-		tree = merkletree.EmptyCollapsedTree()
-	}
-
-	tree.Add(leafHash)
-	rootHash := tree.CalculateRoot()
-	if rootHash != merkletree.Hash(sth.SHA256RootHash) {
-		return nil, fmt.Errorf("calculated root hash (%x) does not match signed tree head (%x) at size %d", rootHash, sth.SHA256RootHash, sth.TreeSize)
-	}
-
-	return tree, nil
-}

monitor/notify.go

@@ -1,156 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"sync"
-	"time"
-)
-
-var stdoutMu sync.Mutex
-
-type notification struct {
-	environ []string
-	summary string
-	text    string
-}
-
-func (s *FilesystemState) notify(ctx context.Context, notif *notification) error {
-	if s.Stdout {
-		writeToStdout(notif)
-	}
-
-	if len(s.Email) > 0 {
-		if err := sendEmail(ctx, s.Email, notif); err != nil {
-			return err
-		}
-	}
-
-	if s.Script != "" {
-		if err := execScript(ctx, s.Script, notif); err != nil {
-			return err
-		}
-	}
-
-	if s.ScriptDir != "" {
-		if err := execScriptDir(ctx, s.ScriptDir, notif); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func writeToStdout(notif *notification) {
-	stdoutMu.Lock()
-	defer stdoutMu.Unlock()
-	os.Stdout.WriteString(notif.text + "\n")
-}
-
-func sendEmail(ctx context.Context, to []string, notif *notification) error {
-	stdin := new(bytes.Buffer)
-	stderr := new(bytes.Buffer)
-
-	from := os.Getenv("EMAIL")
-
-	if from != "" {
-		fmt.Fprintf(stdin, "From: %s\n", from)
-	}
-	fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", "))
-	fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.summary)
-	fmt.Fprintf(stdin, "Date: %s\n", time.Now().Format(mailDateFormat))
-	fmt.Fprintf(stdin, "Message-ID: <%s>\n", generateMessageID())
-	fmt.Fprintf(stdin, "Mime-Version: 1.0\n")
-	fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n")
-	fmt.Fprintf(stdin, "X-Mailer: certspotter\n")
-	fmt.Fprintf(stdin, "\n")
-	fmt.Fprint(stdin, notif.text)
-
-	args := []string{"-i"}
-	if from != "" {
-		args = append(args, "-f", from)
-	}
-	args = append(args, "--")
-	args = append(args, to...)
-
-	sendmail := exec.CommandContext(ctx, sendmailPath(), args...)
-	sendmail.Stdin = stdin
-	sendmail.Stderr = stderr
-
-	if err := sendmail.Run(); err == nil {
-		return nil
-	} else if ctx.Err() != nil {
-		return ctx.Err()
-	} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
-		return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
-	} else {
-		return fmt.Errorf("error sending email to %v: %w", to, err)
-	}
-}
-
-func execScript(ctx context.Context, scriptName string, notif *notification) error {
-	stderr := new(bytes.Buffer)
-
-	cmd := exec.CommandContext(ctx, scriptName)
-	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, notif.environ...)
-	cmd.Stderr = stderr
-
-	if err := cmd.Run(); err == nil {
-		return nil
-	} else if ctx.Err() != nil {
-		return ctx.Err()
-	} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
-		return fmt.Errorf("script %q exited with code %d and error %q", scriptName, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
-	} else if isExitError {
-		return fmt.Errorf("script %q terminated by signal with error %q", scriptName, strings.TrimSpace(stderr.String()))
-	} else {
-		return fmt.Errorf("error executing script: %w", err)
-	}
-}
-
-func execScriptDir(ctx context.Context, dirPath string, notif *notification) error {
-	dirents, err := os.ReadDir(dirPath)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("error executing scripts in directory %q: %w", dirPath, err)
-	}
-	for _, dirent := range dirents {
-		if strings.HasPrefix(dirent.Name(), ".") {
-			continue
-		}
-		scriptPath := filepath.Join(dirPath, dirent.Name())
-		info, err := os.Stat(scriptPath)
-		if errors.Is(err, fs.ErrNotExist) {
-			continue
-		} else if err != nil {
-			return fmt.Errorf("error executing %q in directory %q: %w", dirent.Name(), dirPath, err)
-		} else if info.Mode().IsRegular() && isExecutable(info.Mode()) {
-			if err := execScript(ctx, scriptPath, notif); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func isExecutable(mode os.FileMode) bool {
-	return mode&0111 != 0
-}

monitor/process.go

@@ -1,118 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"fmt"
-	"software.sslmate.com/src/certspotter"
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/loglist"
-	"software.sslmate.com/src/certspotter/merkletree"
-)
-
-type LogEntry struct {
-	Log       *loglist.Log
-	Index     uint64
-	LeafInput []byte
-	ExtraData []byte
-	LeafHash  merkletree.Hash
-}
-
-func processLogEntry(ctx context.Context, config *Config, entry *LogEntry) error {
-	leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput))
-	if err != nil {
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err))
-	}
-	switch leaf.TimestampedEntry.EntryType {
-	case ct.X509LogEntryType:
-		return processX509LogEntry(ctx, config, entry, leaf.TimestampedEntry.X509Entry)
-	case ct.PrecertLogEntryType:
-		return processPrecertLogEntry(ctx, config, entry, leaf.TimestampedEntry.PrecertEntry)
-	default:
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType))
-	}
-}
-
-func processX509LogEntry(ctx context.Context, config *Config, entry *LogEntry, cert ct.ASN1Cert) error {
-	certInfo, err := certspotter.MakeCertInfoFromRawCert(cert)
-	if err != nil {
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err))
-	}
-
-	chain, err := ct.UnmarshalX509ChainArray(entry.ExtraData)
-	if err != nil {
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for X.509 entry: %w", err))
-	}
-	chain = append([]ct.ASN1Cert{cert}, chain...)
-
-	if precertTBS, err := certspotter.ReconstructPrecertTBS(certInfo.TBS); err == nil {
-		certInfo.TBS = precertTBS
-	} else {
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error reconstructing precertificate TBSCertificate: %w", err))
-	}
-
-	return processCertificate(ctx, config, entry, certInfo, chain)
-}
-
-func processPrecertLogEntry(ctx context.Context, config *Config, entry *LogEntry, precert ct.PreCert) error {
-	certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate)
-	if err != nil {
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err))
-	}
-
-	chain, err := ct.UnmarshalPrecertChainArray(entry.ExtraData)
-	if err != nil {
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for precert entry: %w", err))
-	}
-
-	if _, err := certspotter.ValidatePrecert(chain[0], precert.TBSCertificate); err != nil {
-		return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err))
-	}
-
-	return processCertificate(ctx, config, entry, certInfo, chain)
-}
-
-func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
-	identifiers, err := certInfo.ParseIdentifiers()
-	if err != nil {
-		return processMalformedLogEntry(ctx, config, entry, err)
-	}
-	matched, watchItem := config.WatchList.Matches(identifiers)
-	if !matched {
-		return nil
-	}
-
-	cert := &DiscoveredCert{
-		WatchItem:    watchItem,
-		LogEntry:     entry,
-		Info:         certInfo,
-		Chain:        chain,
-		TBSSHA256:    sha256.Sum256(certInfo.TBS.Raw),
-		SHA256:       sha256.Sum256(chain[0]),
-		PubkeySHA256: sha256.Sum256(certInfo.TBS.PublicKey.FullBytes),
-		Identifiers:  identifiers,
-	}
-
-	if err := config.State.NotifyCert(ctx, cert); err != nil {
-		return fmt.Errorf("error notifying about certificate %x: %w", cert.SHA256, err)
-	}
-
-	return nil
-}
-
-func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error {
-	if err := config.State.NotifyMalformedEntry(ctx, entry, parseError); err != nil {
-		return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
-	}
-	return nil
-}

monitor/state.go

@@ -1,68 +0,0 @@
-// Copyright (C) 2024 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"context"
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/loglist"
-	"software.sslmate.com/src/certspotter/merkletree"
-	"time"
-)
-
-type LogState struct {
-	DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
-	VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
-	VerifiedSTH      *ct.SignedTreeHead        `json:"verified_sth"`
-	LastSuccess      time.Time                 `json:"last_success"`
-}
-
-type StateProvider interface {
-	// Initialize the state.  Called before any other method in this interface.
-	// Idempotent: returns nil if the state is already initialized.
-	Prepare(context.Context) error
-
-	// Initialize the state for the given log.  Called before any other method
-	// with the log ID.  Idempotent: returns nil if log state already initialized.
-	PrepareLog(context.Context, LogID) error
-
-	// Store log state for retrieval by LoadLogState.
-	StoreLogState(context.Context, LogID, *LogState) error
-
-	// Load log state that was previously stored with StoreLogState.
-	// Returns nil, nil if StoreLogState has not been called yet for this log.
-	LoadLogState(context.Context, LogID) (*LogState, error)
-
-	// Store STH for retrieval by LoadSTHs.  If an STH with the same
-	// timestamp and root hash is already stored, this STH can be ignored.
-	StoreSTH(context.Context, LogID, *ct.SignedTreeHead) error
-
-	// Load all STHs for this log previously stored with StoreSTH.
-	// The returned slice must be sorted by tree size.
-	LoadSTHs(context.Context, LogID) ([]*ct.SignedTreeHead, error)
-
-	// Remove an STH so it is no longer returned by LoadSTHs.
-	RemoveSTH(context.Context, LogID, *ct.SignedTreeHead) error
-
-	// Called when a certificate matching the watch list is discovered.
-	NotifyCert(context.Context, *DiscoveredCert) error
-
-	// Called when certspotter fails to parse a log entry.
-	NotifyMalformedEntry(context.Context, *LogEntry, error) error
-
-	// Called when a health check fails.  The log is nil if the
-	// feailure is not associated with a log.
-	NotifyHealthCheckFailure(context.Context, *loglist.Log, HealthCheckFailure) error
-
-	// Called when a non-fatal error occurs.  The log is nil if the error is
-	// not associated with a log.  Note that most errors are transient, and
-	// certspotter will retry the failed operation later.
-	NotifyError(context.Context, *loglist.Log, error) error
-}

monitor/statedir.go

@@ -1,155 +0,0 @@
-// Copyright (C) 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"software.sslmate.com/src/certspotter/ct"
-	"software.sslmate.com/src/certspotter/merkletree"
-	"strconv"
-	"strings"
-	"time"
-)
-
-func readVersion(stateDir string) (int, error) {
-	path := filepath.Join(stateDir, "version")
-
-	fileBytes, err := os.ReadFile(path)
-	if errors.Is(err, fs.ErrNotExist) {
-		if fileExists(filepath.Join(stateDir, "evidence")) {
-			return 0, nil
-		} else {
-			return -1, nil
-		}
-	} else if err != nil {
-		return -1, err
-	}
-
-	version, err := strconv.Atoi(strings.TrimSpace(string(fileBytes)))
-	if err != nil {
-		return -1, fmt.Errorf("version file %q is malformed: %w", path, err)
-	}
-
-	return version, nil
-}
-
-func writeVersion(stateDir string) error {
-	return writeFile(filepath.Join(stateDir, "version"), []byte{'2', '\n'}, 0666)
-}
-
-func migrateLogStateDirV1(dir string) error {
-	var sth ct.SignedTreeHead
-	var tree merkletree.CollapsedTree
-
-	sthPath := filepath.Join(dir, "sth.json")
-	sthData, err := os.ReadFile(sthPath)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil
-	} else if err != nil {
-		return err
-	}
-
-	treePath := filepath.Join(dir, "tree.json")
-	treeData, err := os.ReadFile(treePath)
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil
-	} else if err != nil {
-		return err
-	}
-
-	if err := json.Unmarshal(sthData, &sth); err != nil {
-		return fmt.Errorf("error unmarshaling %s: %w", sthPath, err)
-	}
-	if err := json.Unmarshal(treeData, &tree); err != nil {
-		return fmt.Errorf("error unmarshaling %s: %w", treePath, err)
-	}
-
-	stateFile := LogState{
-		DownloadPosition: &tree,
-		VerifiedPosition: &tree,
-		VerifiedSTH:      &sth,
-		LastSuccess:      time.Now().UTC(),
-	}
-	if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil {
-		return err
-	}
-
-	if err := os.Remove(sthPath); err != nil {
-		return err
-	}
-	if err := os.Remove(treePath); err != nil {
-		return err
-	}
-	return nil
-}
-
-func migrateStateDirV1(stateDir string) error {
-	if lockfile := filepath.Join(stateDir, "lock"); fileExists(lockfile) {
-		return fmt.Errorf("directory is locked by another instance of certspotter; remove %s if this is not the case", lockfile)
-	}
-
-	if logDirs, err := os.ReadDir(filepath.Join(stateDir, "logs")); err == nil {
-		for _, logDir := range logDirs {
-			if strings.HasPrefix(logDir.Name(), ".") || !logDir.IsDir() {
-				continue
-			}
-			if err := migrateLogStateDirV1(filepath.Join(stateDir, "logs", logDir.Name())); err != nil {
-				return fmt.Errorf("error migrating log state: %w", err)
-			}
-		}
-	} else if !errors.Is(err, fs.ErrNotExist) {
-		return err
-	}
-
-	if err := writeVersion(stateDir); err != nil {
-		return err
-	}
-
-	if err := os.Remove(filepath.Join(stateDir, "once")); err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return err
-	}
-
-	return nil
-}
-
-func prepareStateDir(stateDir string) error {
-	if err := os.Mkdir(stateDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
-		return err
-	}
-
-	if version, err := readVersion(stateDir); err != nil {
-		return err
-	} else if version == -1 {
-		if err := writeVersion(stateDir); err != nil {
-			return err
-		}
-	} else if version == 0 {
-		return fmt.Errorf("%s was created by a very old version of certspotter; run any version of certspotter after 0.2 and before 0.15.0 to upgrade this directory, or remove it to start from scratch", stateDir)
-	} else if version == 1 {
-		if err := migrateStateDirV1(stateDir); err != nil {
-			return err
-		}
-	} else if version > 2 {
-		return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir)
-	}
-
-	for _, subdir := range []string{"certs", "logs", "healthchecks"} {
-		if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) {
-			return err
-		}
-	}
-
-	return nil
-}

monitor/sthdir.go

@@ -1,93 +0,0 @@
-// Copyright (C) 2017, 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"cmp"
-	"crypto/sha256"
-	"encoding/base64"
-	"encoding/binary"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"slices"
-	"software.sslmate.com/src/certspotter/ct"
-	"strconv"
-	"strings"
-)
-
-func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) {
-	entries, err := os.ReadDir(dirPath)
-	if errors.Is(err, fs.ErrNotExist) {
-		return []*ct.SignedTreeHead{}, nil
-	} else if err != nil {
-		return nil, err
-	}
-	sths := make([]*ct.SignedTreeHead, 0, len(entries))
-	for _, entry := range entries {
-		filename := entry.Name()
-		if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") {
-			continue
-		}
-		sth, err := readSTHFile(filepath.Join(dirPath, filename))
-		if err != nil {
-			return nil, err
-		}
-		sths = append(sths, sth)
-	}
-	slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) int { return cmp.Compare(a.TreeSize, b.TreeSize) })
-	return sths, nil
-}
-
-func readSTHFile(filePath string) (*ct.SignedTreeHead, error) {
-	fileBytes, err := os.ReadFile(filePath)
-	if err != nil {
-		return nil, err
-	}
-	sth := new(ct.SignedTreeHead)
-	if err := json.Unmarshal(fileBytes, sth); err != nil {
-		return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
-	}
-	return sth, nil
-}
-
-func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error {
-	filePath := filepath.Join(dirPath, sthFilename(sth))
-	if fileExists(filePath) {
-		return nil
-	}
-	return writeJSONFile(filePath, sth, 0666)
-}
-
-func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error {
-	filePath := filepath.Join(dirPath, sthFilename(sth))
-	err := os.Remove(filePath)
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return err
-	}
-	return nil
-}
-
-// generate a filename that uniquely identifies the STH (within the context of a particular log)
-func sthFilename(sth *ct.SignedTreeHead) string {
-	hasher := sha256.New()
-	switch sth.Version {
-	case ct.V1:
-		binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
-		binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash)
-	default:
-		panic(fmt.Errorf("sthFilename: invalid STH version %d", sth.Version))
-	}
-	// For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic)
-	return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json"
-}

monitor/watchlist.go

@@ -1,135 +0,0 @@
-// Copyright (C) 2016, 2023 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package monitor
-
-import (
-	"bufio"
-	"fmt"
-	"golang.org/x/net/idna"
-	"io"
-	"software.sslmate.com/src/certspotter"
-	"strings"
-)
-
-type WatchItem struct {
-	domain       []string
-	acceptSuffix bool
-}
-
-type WatchList []WatchItem
-
-func ParseWatchItem(str string) (WatchItem, error) {
-	fields := strings.Fields(str)
-	if len(fields) == 0 {
-		return WatchItem{}, fmt.Errorf("empty domain")
-	}
-	domain := fields[0]
-
-	for _, field := range fields[1:] {
-		switch {
-		case strings.HasPrefix(field, "valid_at:"):
-			// Ignore for backwards compatibility
-		default:
-			return WatchItem{}, fmt.Errorf("unknown parameter %q", field)
-		}
-	}
-
-	if domain == "." {
-		// "." as in root zone -> matches everything
-		return WatchItem{
-			domain:       []string{},
-			acceptSuffix: true,
-		}, nil
-	}
-
-	acceptSuffix := false
-	if strings.HasPrefix(domain, ".") {
-		acceptSuffix = true
-		domain = domain[1:]
-	}
-
-	asciiDomain, err := idna.ToASCII(strings.ToLower(strings.TrimRight(domain, ".")))
-	if err != nil {
-		return WatchItem{}, fmt.Errorf("invalid domain %q (%w)", domain, err)
-	}
-	return WatchItem{
-		domain:       strings.Split(asciiDomain, "."),
-		acceptSuffix: acceptSuffix,
-	}, nil
-}
-
-func ReadWatchList(reader io.Reader) (WatchList, error) {
-	items := make(WatchList, 0, 50)
-	scanner := bufio.NewScanner(reader)
-	lineNo := 0
-	for scanner.Scan() {
-		line := scanner.Text()
-		lineNo++
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		item, err := ParseWatchItem(line)
-		if err != nil {
-			return nil, fmt.Errorf("%w on line %d", err, lineNo)
-		}
-		items = append(items, item)
-	}
-	return items, scanner.Err()
-}
-
-func (item WatchItem) String() string {
-	if item.acceptSuffix {
-		return "." + strings.Join(item.domain, ".")
-	} else {
-		return strings.Join(item.domain, ".")
-	}
-}
-
-func (item WatchItem) matchesDNSName(dnsName []string) bool {
-	watchDomain := item.domain
-	for len(dnsName) > 0 && len(watchDomain) > 0 {
-		certLabel := dnsName[len(dnsName)-1]
-		watchLabel := watchDomain[len(watchDomain)-1]
-
-		if !dnsLabelMatches(certLabel, watchLabel) {
-			return false
-		}
-
-		dnsName = dnsName[:len(dnsName)-1]
-		watchDomain = watchDomain[:len(watchDomain)-1]
-	}
-	return len(watchDomain) == 0 && (item.acceptSuffix || len(dnsName) == 0)
-}
-
-func dnsLabelMatches(certLabel string, watchLabel string) bool {
-	// For fail-safe behavior, if a label was unparsable, it matches everything.
-	// Similarly, redacted labels match everything, since the label _might_ be
-	// for a name we're interested in.
-
-	return certLabel == "*" ||
-		certLabel == "?" ||
-		certLabel == certspotter.UnparsableDNSLabelPlaceholder ||
-		certspotter.MatchesWildcard(watchLabel, certLabel)
-}
-
-func (list WatchList) Matches(identifiers *certspotter.Identifiers) (bool, WatchItem) {
-	dnsNames := make([][]string, len(identifiers.DNSNames))
-	for i, dnsName := range identifiers.DNSNames {
-		dnsNames[i] = strings.Split(dnsName, ".")
-	}
-	for _, item := range list {
-		for _, dnsName := range dnsNames {
-			if item.matchesDNSName(dnsName) {
-				return true, item
-			}
-		}
-	}
-	return false, WatchItem{}
-}

scanner.go

@@ -0,0 +1,327 @@
+// Copyright (C) 2016 Opsmate, Inc.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License, v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
+// See the Mozilla Public License for details.
+//
+// This file contains code from https://github.com/google/certificate-transparency/tree/master/go
+// See ct/AUTHORS and ct/LICENSE for copyright and license information.
+
+package certspotter
+
+import (
+	//	"container/list"
+	"bytes"
+	"crypto"
+	"errors"
+	"fmt"
+	"log"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/ct/client"
+)
+
+type ProcessCallback func(*Scanner, *ct.LogEntry)
+
+const (
+	FETCH_RETRIES    = 10
+	FETCH_RETRY_WAIT = 1
+)
+
+// ScannerOptions holds configuration options for the Scanner
+type ScannerOptions struct {
+	// Number of entries to request in one batch from the Log
+	BatchSize int
+
+	// Number of concurrent proecssors to run
+	NumWorkers int
+
+	// Don't print any status messages to stdout
+	Quiet bool
+}
+
+// Creates a new ScannerOptions struct with sensible defaults
+func DefaultScannerOptions() *ScannerOptions {
+	return &ScannerOptions{
+		BatchSize:  1000,
+		NumWorkers: 1,
+		Quiet:      false,
+	}
+}
+
+// Scanner is a tool to scan all the entries in a CT Log.
+type Scanner struct {
+	// Base URI of CT log
+	LogUri string
+
+	// Public key of the log
+	publicKey crypto.PublicKey
+	LogId     ct.SHA256Hash
+
+	// Client used to talk to the CT log instance
+	logClient *client.LogClient
+
+	// Configuration options for this Scanner instance
+	opts ScannerOptions
+}
+
+// fetchRange represents a range of certs to fetch from a CT log
+type fetchRange struct {
+	start int64
+	end   int64
+}
+
+// Worker function to process certs.
+// Accepts ct.LogEntries over the |entries| channel, and invokes processCert on them.
+// Returns true over the |done| channel when the |entries| channel is closed.
+func (s *Scanner) processerJob(id int, certsProcessed *int64, entries <-chan ct.LogEntry, processCert ProcessCallback, wg *sync.WaitGroup) {
+	for entry := range entries {
+		atomic.AddInt64(certsProcessed, 1)
+		processCert(s, &entry)
+	}
+	wg.Done()
+}
+
+func (s *Scanner) fetch(r fetchRange, entries chan<- ct.LogEntry, tree *CollapsedMerkleTree) error {
+	success := false
+	retries := FETCH_RETRIES
+	retryWait := FETCH_RETRY_WAIT
+	for !success {
+		s.Log(fmt.Sprintf("Fetching entries %d to %d", r.start, r.end))
+		logEntries, err := s.logClient.GetEntries(r.start, r.end)
+		if err != nil {
+			if retries == 0 {
+				s.Warn(fmt.Sprintf("Problem fetching entries %d to %d from log: %s", r.start, r.end, err.Error()))
+				return err
+			} else {
+				s.Log(fmt.Sprintf("Problem fetching entries %d to %d from log (will retry): %s", r.start, r.end, err.Error()))
+				time.Sleep(time.Duration(retryWait) * time.Second)
+				retries--
+				retryWait *= 2
+				continue
+			}
+		}
+		retries = FETCH_RETRIES
+		retryWait = FETCH_RETRY_WAIT
+		for _, logEntry := range logEntries {
+			if tree != nil {
+				tree.Add(hashLeaf(logEntry.LeafBytes))
+			}
+			logEntry.Index = r.start
+			entries <- logEntry
+			r.start++
+		}
+		if r.start > r.end {
+			// Only complete if we actually got all the leaves we were
+			// expecting -- Logs MAY return fewer than the number of
+			// leaves requested.
+			success = true
+		}
+	}
+	return nil
+}
+
+// Worker function for fetcher jobs.
+// Accepts cert ranges to fetch over the |ranges| channel, and if the fetch is
+// successful sends the individual LeafInputs out into the
+// |entries| channel for the processors to chew on.
+// Will retry failed attempts to retrieve ranges indefinitely.
+// Sends true over the |done| channel when the |ranges| channel is closed.
+/* disabled becuase error handling is broken
+func (s *Scanner) fetcherJob(id int, ranges <-chan fetchRange, entries chan<- ct.LogEntry, wg *sync.WaitGroup) {
+	for r := range ranges {
+		s.fetch(r, entries, nil)
+	}
+	wg.Done()
+}
+*/
+
+// Returns the smaller of |a| and |b|
+func min(a int64, b int64) int64 {
+	if a < b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// Returns the larger of |a| and |b|
+func max(a int64, b int64) int64 {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// Pretty prints the passed in number of |seconds| into a more human readable
+// string.
+func humanTime(seconds int) string {
+	nanos := time.Duration(seconds) * time.Second
+	hours := int(nanos / (time.Hour))
+	nanos %= time.Hour
+	minutes := int(nanos / time.Minute)
+	nanos %= time.Minute
+	seconds = int(nanos / time.Second)
+	s := ""
+	if hours > 0 {
+		s += fmt.Sprintf("%d hours ", hours)
+	}
+	if minutes > 0 {
+		s += fmt.Sprintf("%d minutes ", minutes)
+	}
+	if seconds > 0 {
+		s += fmt.Sprintf("%d seconds ", seconds)
+	}
+	return s
+}
+
+func (s Scanner) Log(msg string) {
+	if !s.opts.Quiet {
+		log.Print(s.LogUri, ": ", msg)
+	}
+}
+
+func (s Scanner) Warn(msg string) {
+	log.Print(s.LogUri, ": ", msg)
+}
+
+func (s *Scanner) GetSTH() (*ct.SignedTreeHead, error) {
+	latestSth, err := s.logClient.GetSTH()
+	if err != nil {
+		return nil, err
+	}
+	if s.publicKey != nil {
+		verifier, err := ct.NewSignatureVerifier(s.publicKey)
+		if err != nil {
+			return nil, err
+		}
+		if err := verifier.VerifySTHSignature(*latestSth); err != nil {
+			return nil, errors.New("STH signature is invalid: " + err.Error())
+		}
+	}
+	latestSth.LogID = s.LogId
+	return latestSth, nil
+}
+
+func (s *Scanner) CheckConsistency(first *ct.SignedTreeHead, second *ct.SignedTreeHead) (bool, error) {
+	if first.TreeSize == 0 || second.TreeSize == 0 {
+		// RFC 6962 doesn't define how to generate a consistency proof in this case,
+		// and it doesn't matter anyways since the tree is empty.  The DigiCert logs
+		// return a 400 error if we ask for such a proof.
+		return true, nil
+	} else if first.TreeSize < second.TreeSize {
+		proof, err := s.logClient.GetConsistencyProof(int64(first.TreeSize), int64(second.TreeSize))
+		if err != nil {
+			return false, err
+		}
+		return VerifyConsistencyProof(proof, first, second), nil
+	} else if first.TreeSize > second.TreeSize {
+		proof, err := s.logClient.GetConsistencyProof(int64(second.TreeSize), int64(first.TreeSize))
+		if err != nil {
+			return false, err
+		}
+		return VerifyConsistencyProof(proof, second, first), nil
+	} else {
+		// There is no need to ask the server for a consistency proof if the trees
+		// are the same size, and the DigiCert log returns a 400 error if we try.
+		return bytes.Equal(first.SHA256RootHash[:], second.SHA256RootHash[:]), nil
+	}
+}
+
+func (s *Scanner) MakeCollapsedMerkleTree(sth *ct.SignedTreeHead) (*CollapsedMerkleTree, error) {
+	if sth.TreeSize == 0 {
+		return &CollapsedMerkleTree{}, nil
+	}
+
+	entries, err := s.logClient.GetEntries(int64(sth.TreeSize-1), int64(sth.TreeSize-1))
+	if err != nil {
+		return nil, err
+	}
+	if len(entries) == 0 {
+		return nil, fmt.Errorf("Log did not return entry %d", sth.TreeSize-1)
+	}
+	leafHash := hashLeaf(entries[0].LeafBytes)
+
+	var tree *CollapsedMerkleTree
+	if sth.TreeSize > 1 {
+		auditPath, _, err := s.logClient.GetAuditProof(leafHash, sth.TreeSize)
+		if err != nil {
+			return nil, err
+		}
+		reverseHashes(auditPath)
+		tree, err = NewCollapsedMerkleTree(auditPath, sth.TreeSize-1)
+		if err != nil {
+			return nil, fmt.Errorf("Error returned bad audit proof for %x to %d", leafHash, sth.TreeSize)
+		}
+	} else {
+		tree = EmptyCollapsedMerkleTree()
+	}
+
+	tree.Add(leafHash)
+	if !bytes.Equal(tree.CalculateRoot(), sth.SHA256RootHash[:]) {
+		return nil, fmt.Errorf("Calculated root hash does not match signed tree head at size %d", sth.TreeSize)
+	}
+
+	return tree, nil
+}
+
+func (s *Scanner) Scan(startIndex int64, endIndex int64, processCert ProcessCallback, tree *CollapsedMerkleTree) error {
+	s.Log("Starting scan...")
+
+	certsProcessed := new(int64)
+	startTime := time.Now()
+	/* TODO: only launch ticker goroutine if in verbose mode; kill the goroutine when the scanner finishes
+	ticker := time.NewTicker(time.Second)
+	go func() {
+		for range ticker.C {
+			throughput := float64(s.certsProcessed) / time.Since(startTime).Seconds()
+			remainingCerts := int64(endIndex) - int64(startIndex) - s.certsProcessed
+			remainingSeconds := int(float64(remainingCerts) / throughput)
+			remainingString := humanTime(remainingSeconds)
+			s.Log(fmt.Sprintf("Processed: %d certs (to index %d). Throughput: %3.2f ETA: %s", s.certsProcessed,
+				startIndex+int64(s.certsProcessed), throughput, remainingString))
+		}
+	}()
+	*/
+
+	// Start processor workers
+	jobs := make(chan ct.LogEntry, 100)
+	var processorWG sync.WaitGroup
+	for w := 0; w < s.opts.NumWorkers; w++ {
+		processorWG.Add(1)
+		go s.processerJob(w, certsProcessed, jobs, processCert, &processorWG)
+	}
+
+	for start := startIndex; start < int64(endIndex); {
+		end := min(start+int64(s.opts.BatchSize), int64(endIndex)) - 1
+		if err := s.fetch(fetchRange{start, end}, jobs, tree); err != nil {
+			return err
+		}
+		start = end + 1
+	}
+	close(jobs)
+	processorWG.Wait()
+	s.Log(fmt.Sprintf("Completed %d certs in %s", *certsProcessed, humanTime(int(time.Since(startTime).Seconds()))))
+
+	return nil
+}
+
+// Creates a new Scanner instance using |client| to talk to the log, and taking
+// configuration options from |opts|.
+func NewScanner(logUri string, logId ct.SHA256Hash, publicKey crypto.PublicKey, opts *ScannerOptions) *Scanner {
+	var scanner Scanner
+	scanner.LogUri = logUri
+	scanner.LogId = logId
+	scanner.publicKey = publicKey
+	scanner.logClient = client.New(strings.TrimRight(logUri, "/"))
+	scanner.opts = *opts
+	return &scanner
+}

x509.go

@@ -320,7 +320,7 @@ func (tbs *TBSCertificate) ParseSubjectAltNames() ([]SubjectAltName, error) {
 
 	for _, sanExt := range tbs.GetExtension(oidExtensionSubjectAltName) {
 		var err error
-		sans, err = ParseSANExtension(sans, sanExt.Value)
+		sans, err = parseSANExtension(sans, sanExt.Value)
 		if err != nil {
 			return nil, err
 		}
@@ -377,7 +377,7 @@ func (cert *Certificate) ParseSignatureValue() ([]byte, error) {
 	return signatureValue.RightAlign(), nil
 }
 
-func ParseSANExtension(sans []SubjectAltName, value []byte) ([]SubjectAltName, error) {
+func parseSANExtension(sans []SubjectAltName, value []byte) ([]SubjectAltName, error) {
 	var seq asn1.RawValue
 	if rest, err := asn1.Unmarshal(value, &seq); err != nil {
 		return nil, errors.New("failed to parse subjectAltName extension: " + err.Error())