Tandem
/
certspotter
mirror of https://github.com/SSLMate/certspotter.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

static-ct-api support, parallel downloading

This commit is contained in:
Andrew Ayer 2025-05-04 20:32:52 -04:00
parent 84bd080553
commit b856d7f163
21 changed files with 571 additions and 1875 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
ct/AUTHORS
View File

@ -1,24 +0,0 @@
# This is the official list of benchmark authors for copyright purposes.
# This file is distinct from the CONTRIBUTORS files.
# See the latter for an explanation.
#
# Names should be added to this file as:
# Name or Organization <email address>
# The email address is not required for organizations.
#
# Please keep the list sorted.
Comodo CA Limited
Ed Maste <emaste@freebsd.org>
Fiaz Hossain <fiaz.hossain@salesforce.com>
Google Inc.
Jeff Trawick <trawick@gmail.com>
Katriel Cohn-Gordon <katriel.cohn-gordon@cybersecurity.ox.ac.uk>
Mark Schloesser <ms@mwcollect.org>
NORDUnet A/S
Nicholas Galbreath <nickg@client9.com>
Oliver Weidner <Oliver.Weidner@gmail.com>
Ruslan Kovalov <ruslan.kovalyov@gmail.com>
Venafi, Inc.
Vladimir Rutsky <vladimir@rutsky.org>
Ximin Luo <infinity0@gmx.com>

202
ct/LICENSE
View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

4
ct/README
View File

@ -1,4 +0,0 @@
The code in this directory is based on Google's Certificiate Transparency Go library
(originally at <https://github.com/google/certificate-transparency/tree/master/go>;
now at <https://github.com/google/certificate-transparency-go>).
See AUTHORS for the copyright holders, and LICENSE for the license.

416
ct/client/logclient.go
View File

@ -1,416 +0,0 @@
// Package client is a CT log client implementation and contains types and code
// for interacting with RFC6962-compliant CT Log instances.
// See http://tools.ietf.org/html/rfc6962 for details
package client
import (
"bytes"
"context"
"crypto/sha256"
"crypto/tls"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
insecurerand "math/rand"
"net/http"
"net/url"
"strconv"
"time"
"software.sslmate.com/src/certspotter/ct"
)
const (
baseRetryDelay = 1 * time.Second
maxRetryDelay = 120 * time.Second
maxRetries = 10
)
func isRetryableStatusCode(code int) bool {
return code/100 == 5 || code == http.StatusTooManyRequests
}
func randomDuration(min, max time.Duration) time.Duration {
return min + time.Duration(insecurerand.Int63n(int64(max)-int64(min)+1))
}
func getRetryAfter(resp *http.Response) (time.Duration, bool) {
if resp == nil {
return 0, false
}
seconds, err := strconv.ParseUint(resp.Header.Get("Retry-After"), 10, 16)
if err != nil {
return 0, false
}
return time.Duration(seconds) * time.Second, true
}
func sleep(ctx context.Context, duration time.Duration) {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
}
}
// URI paths for CT Log endpoints
const (
GetSTHPath = "/ct/v1/get-sth"
GetEntriesPath = "/ct/v1/get-entries"
GetSTHConsistencyPath = "/ct/v1/get-sth-consistency"
GetProofByHashPath = "/ct/v1/get-proof-by-hash"
AddChainPath = "/ct/v1/add-chain"
)
// LogClient represents a client for a given CT Log instance
type LogClient struct {
uri string // the base URI of the log. e.g. http://ct.googleapis/pilot
httpClient *http.Client // used to interact with the log via HTTP
verifier *ct.SignatureVerifier // if non-nil, used to verify STH signatures
}
//////////////////////////////////////////////////////////////////////////////////
// JSON structures follow.
// These represent the structures returned by the CT Log server.
//////////////////////////////////////////////////////////////////////////////////
// getSTHResponse represents the JSON response to the get-sth CT method
type getSTHResponse struct {
TreeSize uint64 `json:"tree_size"` // Number of certs in the current tree
Timestamp uint64 `json:"timestamp"` // Time that the tree was created
SHA256RootHash []byte `json:"sha256_root_hash"` // Root hash of the tree
TreeHeadSignature []byte `json:"tree_head_signature"` // Log signature for this STH
}
// base64LeafEntry represents a Base64 encoded leaf entry
type base64LeafEntry struct {
LeafInput []byte `json:"leaf_input"`
ExtraData []byte `json:"extra_data"`
}
// getEntriesReponse represents the JSON response to the CT get-entries method
type getEntriesResponse struct {
Entries []base64LeafEntry `json:"entries"` // the list of returned entries
}
// getConsistencyProofResponse represents the JSON response to the CT get-consistency-proof method
type getConsistencyProofResponse struct {
Consistency [][]byte `json:"consistency"`
}
// getAuditProofResponse represents the JSON response to the CT get-proof-by-hash method
type getAuditProofResponse struct {
LeafIndex uint64 `json:"leaf_index"`
AuditPath [][]byte `json:"audit_path"`
}
type addChainRequest struct {
Chain [][]byte `json:"chain"`
}
type addChainResponse struct {
SCTVersion uint8 `json:"sct_version"`
ID []byte `json:"id"`
Timestamp uint64 `json:"timestamp"`
Extensions []byte `json:"extensions"`
Signature []byte `json:"signature"`
}
// New constructs a new LogClient instance.
// |uri| is the base URI of the CT log instance to interact with, e.g.
// http://ct.googleapis.com/pilot
func New(uri string) *LogClient {
return NewWithVerifier(uri, nil)
}
func NewWithVerifier(uri string, verifier *ct.SignatureVerifier) *LogClient {
var c LogClient
c.uri = uri
c.verifier = verifier
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSHandshakeTimeout: 15 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
MaxIdleConnsPerHost: 10,
DisableKeepAlives: false,
MaxIdleConns: 100,
IdleConnTimeout: 15 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{
// We have to disable TLS certificate validation because because several logs
// (WoSign, StartCom, GDCA) use certificates that are not widely trusted.
// Since we verify that every response we receive from the log is signed
// by the log's CT public key (either directly, or indirectly via the Merkle Tree),
// TLS certificate validation is not actually necessary. (We don't want to ship
// our own trust store because that adds undesired complexity and would require
// updating should a log ever change to a different CA.)
InsecureSkipVerify: true,
},
}
c.httpClient = &http.Client{Timeout: 60 * time.Second, Transport: transport}
return &c
}
func (c *LogClient) fetchAndParse(ctx context.Context, uri string, respBody interface{}) error {
return c.doAndParse(ctx, "GET", uri, nil, respBody)
}
func (c *LogClient) postAndParse(ctx context.Context, uri string, body interface{}, respBody interface{}) error {
return c.doAndParse(ctx, "POST", uri, body, respBody)
}
func (c *LogClient) makeRequest(ctx context.Context, method string, uri string, body interface{}) (*http.Request, error) {
if body == nil {
return http.NewRequestWithContext(ctx, method, uri, nil)
} else {
bodyBytes, err := json.Marshal(body)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, method, uri, bytes.NewReader(bodyBytes))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
return req, nil
}
}
func (c *LogClient) doAndParse(ctx context.Context, method string, uri string, reqBody interface{}, respBody interface{}) error {
numRetries := 0
retry:
if ctx.Err() != nil {
return ctx.Err()
}
req, err := c.makeRequest(ctx, method, uri, reqBody)
if err != nil {
return fmt.Errorf("%s %s: error creating request: %w", method, uri, err)
}
req.Header.Set("User-Agent", "") // Don't send a User-Agent to make life harder for malicious logs
resp, err := c.httpClient.Do(req)
if err != nil {
if c.shouldRetry(ctx, numRetries, nil) {
numRetries++
goto retry
}
return err
}
respBodyBytes, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
if c.shouldRetry(ctx, numRetries, nil) {
numRetries++
goto retry
}
return fmt.Errorf("%s %s: error reading response: %w", method, uri, err)
}
if resp.StatusCode/100 != 2 {
if c.shouldRetry(ctx, numRetries, resp) {
numRetries++
goto retry
}
return fmt.Errorf("%s %s: %s (%s)", method, uri, resp.Status, string(respBodyBytes))
}
if err := json.Unmarshal(respBodyBytes, respBody); err != nil {
return fmt.Errorf("%s %s: error parsing response JSON: %w", method, uri, err)
}
return nil
}
func (c *LogClient) shouldRetry(ctx context.Context, numRetries int, resp *http.Response) bool {
if numRetries == maxRetries {
return false
}
if resp != nil && !isRetryableStatusCode(resp.StatusCode) {
return false
}
var delay time.Duration
if retryAfter, hasRetryAfter := getRetryAfter(resp); hasRetryAfter {
delay = retryAfter
} else {
delay = baseRetryDelay * (1 << numRetries)
if delay > maxRetryDelay {
delay = maxRetryDelay
}
delay += randomDuration(0, delay/2)
}
if deadline, hasDeadline := ctx.Deadline(); hasDeadline && time.Now().Add(delay).After(deadline) {
return false
}
sleep(ctx, delay)
return true
}
// GetSTH retrieves the current STH from the log.
// Returns a populated SignedTreeHead, or a non-nil error.
func (c *LogClient) GetSTH(ctx context.Context) (sth *ct.SignedTreeHead, err error) {
var resp getSTHResponse
if err = c.fetchAndParse(ctx, c.uri+GetSTHPath, &resp); err != nil {
return
}
sth = &ct.SignedTreeHead{
TreeSize: resp.TreeSize,
Timestamp: resp.Timestamp,
}
if len(resp.SHA256RootHash) != sha256.Size {
return nil, fmt.Errorf("STH returned by server has invalid sha256_root_hash (expected length %d got %d)", sha256.Size, len(resp.SHA256RootHash))
}
copy(sth.SHA256RootHash[:], resp.SHA256RootHash)
ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.TreeHeadSignature))
if err != nil {
return nil, err
}
sth.TreeHeadSignature = *ds
if c.verifier != nil {
if err := c.verifier.VerifySTHSignature(*sth); err != nil {
return nil, fmt.Errorf("STH returned by server has invalid signature: %w", err)
}
}
return
}
type GetEntriesItem struct {
LeafInput []byte `json:"leaf_input"`
ExtraData []byte `json:"extra_data"`
}
// Retrieve the entries in the sequence [start, end] from the CT log server.
// If error is nil, at least one entry is returned, and no excess entries are returned.
// Fewer entries than requested may be returned.
func (c *LogClient) GetRawEntries(ctx context.Context, start, end uint64) ([]GetEntriesItem, error) {
if end < start {
panic("LogClient.GetRawEntries: end < start")
}
var response struct {
Entries []GetEntriesItem `json:"entries"`
}
uri := fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end)
err := c.fetchAndParse(ctx, uri, &response)
if err != nil {
return nil, err
}
if len(response.Entries) == 0 {
return nil, fmt.Errorf("GET %s: log server returned an empty get-entries response", uri)
}
if uint64(len(response.Entries)) > end-start+1 {
return nil, fmt.Errorf("GET %s: log server returned a get-entries response with extraneous entries", uri)
}
return response.Entries, nil
}
// GetEntries attempts to retrieve the entries in the sequence [|start|, |end|] from the CT
// log server. (see section 4.6.)
// Returns a slice of LeafInputs or a non-nil error.
func (c *LogClient) GetEntries(ctx context.Context, start, end int64) ([]ct.LogEntry, error) {
if end < 0 {
return nil, errors.New("GetEntries: end should be >= 0")
}
if end < start {
return nil, errors.New("GetEntries: start should be <= end")
}
var resp getEntriesResponse
err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end), &resp)
if err != nil {
return nil, err
}
entries := make([]ct.LogEntry, len(resp.Entries))
for index, entry := range resp.Entries {
leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewBuffer(entry.LeafInput))
if err != nil {
return nil, fmt.Errorf("Reading Merkle Tree Leaf at index %d failed: %s", start+int64(index), err)
}
entries[index].LeafBytes = entry.LeafInput
entries[index].Leaf = *leaf
var chain []ct.ASN1Cert
switch leaf.TimestampedEntry.EntryType {
case ct.X509LogEntryType:
chain, err = ct.UnmarshalX509ChainArray(entry.ExtraData)
case ct.PrecertLogEntryType:
chain, err = ct.UnmarshalPrecertChainArray(entry.ExtraData)
default:
return nil, fmt.Errorf("Unknown entry type at index %d: %v", start+int64(index), leaf.TimestampedEntry.EntryType)
}
if err != nil {
return nil, fmt.Errorf("Parsing entry of type %d at index %d failed: %s", leaf.TimestampedEntry.EntryType, start+int64(index), err)
}
entries[index].Chain = chain
entries[index].Index = start + int64(index)
}
return entries, nil
}
// GetConsistencyProof retrieves a Merkle Consistency Proof between two STHs (|first| and |second|)
// from the log. Returns a slice of MerkleTreeNodes (a ct.ConsistencyProof) or a non-nil error.
func (c *LogClient) GetConsistencyProof(ctx context.Context, first, second int64) (ct.ConsistencyProof, error) {
if second < 0 {
return nil, errors.New("GetConsistencyProof: second should be >= 0")
}
if second < first {
return nil, errors.New("GetConsistencyProof: first should be <= second")
}
var resp getConsistencyProofResponse
err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?first=%d&second=%d", c.uri, GetSTHConsistencyPath, first, second), &resp)
if err != nil {
return nil, err
}
nodes := make([]ct.MerkleTreeNode, len(resp.Consistency))
for index, nodeBytes := range resp.Consistency {
nodes[index] = nodeBytes
}
return nodes, nil
}
// GetAuditProof retrieves a Merkle Audit Proof (aka Inclusion Proof) for the given
// |hash| based on the STH at |treeSize| from the log. Returns a slice of MerkleTreeNodes
// and the index of the leaf.
func (c *LogClient) GetAuditProof(ctx context.Context, hash ct.MerkleTreeNode, treeSize uint64) (ct.AuditPath, uint64, error) {
var resp getAuditProofResponse
err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?hash=%s&tree_size=%d", c.uri, GetProofByHashPath, url.QueryEscape(base64.StdEncoding.EncodeToString(hash)), treeSize), &resp)
if err != nil {
return nil, 0, err
}
path := make([]ct.MerkleTreeNode, len(resp.AuditPath))
for index, nodeBytes := range resp.AuditPath {
path[index] = nodeBytes
}
return path, resp.LeafIndex, nil
}
func (c *LogClient) AddChain(ctx context.Context, chain [][]byte) (*ct.SignedCertificateTimestamp, error) {
req := addChainRequest{Chain: chain}
var resp addChainResponse
if err := c.postAndParse(ctx, c.uri+AddChainPath, &req, &resp); err != nil {
return nil, err
}
sct := &ct.SignedCertificateTimestamp{
SCTVersion: ct.Version(resp.SCTVersion),
Timestamp: resp.Timestamp,
Extensions: resp.Extensions,
}
if len(resp.ID) != sha256.Size {
return nil, fmt.Errorf("SCT returned by server has invalid id (expected length %d got %d)", sha256.Size, len(resp.ID))
}
copy(sct.LogID[:], resp.ID)
ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.Signature))
if err != nil {
return nil, err
}
sct.Signature = *ds
return sct, nil
}

462
ct/serialization.go
View File

@ -1,462 +0,0 @@
package ct
import (
"bytes"
"container/list"
"crypto"
"encoding/binary"
"errors"
"fmt"
"io"
)
// Variable size structure prefix-header byte lengths
const (
CertificateLengthBytes = 3
PreCertificateLengthBytes = 3
ExtensionsLengthBytes = 2
CertificateChainLengthBytes = 3
SignatureLengthBytes = 2
)
// Max lengths
const (
MaxCertificateLength = (1 << 24) - 1
MaxExtensionsLength = (1 << 16) - 1
)
func writeUint(w io.Writer, value uint64, numBytes int) error {
buf := make([]uint8, numBytes)
for i := 0; i < numBytes; i++ {
buf[numBytes-i-1] = uint8(value & 0xff)
value >>= 8
}
if value != 0 {
return errors.New("numBytes was insufficiently large to represent value")
}
if _, err := w.Write(buf); err != nil {
return err
}
return nil
}
func writeVarBytes(w io.Writer, value []byte, numLenBytes int) error {
if err := writeUint(w, uint64(len(value)), numLenBytes); err != nil {
return err
}
if _, err := w.Write(value); err != nil {
return err
}
return nil
}
func readUint(r io.Reader, numBytes int) (uint64, error) {
var l uint64
for i := 0; i < numBytes; i++ {
l <<= 8
var t uint8
if err := binary.Read(r, binary.BigEndian, &t); err != nil {
return 0, err
}
l |= uint64(t)
}
return l, nil
}
// Reads a variable length array of bytes from |r|. |numLenBytes| specifies the
// number of (BigEndian) prefix-bytes which contain the length of the actual
// array data bytes that follow.
// Allocates an array to hold the contents and returns a slice view into it if
// the read was successful, or an error otherwise.
func readVarBytes(r io.Reader, numLenBytes int) ([]byte, error) {
switch {
case numLenBytes > 8:
return nil, fmt.Errorf("numLenBytes too large (%d)", numLenBytes)
case numLenBytes == 0:
return nil, errors.New("numLenBytes should be > 0")
}
l, err := readUint(r, numLenBytes)
if err != nil {
return nil, err
}
data := make([]byte, l)
if n, err := io.ReadFull(r, data); err != nil {
if err == io.EOF || err == io.ErrUnexpectedEOF {
return nil, fmt.Errorf("short read: expected %d but got %d", l, n)
}
return nil, err
}
return data, nil
}
// Reads a list of ASN1Cert types from |r|
func readASN1CertList(r io.Reader, totalLenBytes int, elementLenBytes int) ([]ASN1Cert, error) {
listBytes, err := readVarBytes(r, totalLenBytes)
if err != nil {
return []ASN1Cert{}, err
}
list := list.New()
listReader := bytes.NewReader(listBytes)
var entry []byte
for err == nil {
entry, err = readVarBytes(listReader, elementLenBytes)
if err != nil {
if err != io.EOF {
return []ASN1Cert{}, err
}
} else {
list.PushBack(entry)
}
}
ret := make([]ASN1Cert, list.Len())
i := 0
for e := list.Front(); e != nil; e = e.Next() {
ret[i] = e.Value.([]byte)
i++
}
return ret, nil
}
// ReadTimestampedEntryInto parses the byte-stream representation of a
// TimestampedEntry from |r| and populates the struct |t| with the data. See
// RFC section 3.4 for details on the format.
// Returns a non-nil error if there was a problem.
func ReadTimestampedEntryInto(r io.Reader, t *TimestampedEntry) error {
var err error
if err = binary.Read(r, binary.BigEndian, &t.Timestamp); err != nil {
return err
}
if err = binary.Read(r, binary.BigEndian, &t.EntryType); err != nil {
return err
}
switch t.EntryType {
case X509LogEntryType:
if t.X509Entry, err = readVarBytes(r, CertificateLengthBytes); err != nil {
return err
}
case PrecertLogEntryType:
if err := binary.Read(r, binary.BigEndian, &t.PrecertEntry.IssuerKeyHash); err != nil {
return err
}
if t.PrecertEntry.TBSCertificate, err = readVarBytes(r, PreCertificateLengthBytes); err != nil {
return err
}
default:
return fmt.Errorf("unknown EntryType: %d", t.EntryType)
}
t.Extensions, err = readVarBytes(r, ExtensionsLengthBytes)
return nil
}
// ReadMerkleTreeLeaf parses the byte-stream representation of a MerkleTreeLeaf
// and returns a pointer to a new MerkleTreeLeaf structure containing the
// parsed data.
// See RFC section 3.4 for details on the format.
// Returns a pointer to a new MerkleTreeLeaf or non-nil error if there was a
// problem
func ReadMerkleTreeLeaf(r io.Reader) (*MerkleTreeLeaf, error) {
var m MerkleTreeLeaf
if err := binary.Read(r, binary.BigEndian, &m.Version); err != nil {
return nil, err
}
if m.Version != V1 {
return nil, fmt.Errorf("unknown Version %d", m.Version)
}
if err := binary.Read(r, binary.BigEndian, &m.LeafType); err != nil {
return nil, err
}
if m.LeafType != TimestampedEntryLeafType {
return nil, fmt.Errorf("unknown LeafType %d", m.LeafType)
}
if err := ReadTimestampedEntryInto(r, &m.TimestampedEntry); err != nil {
return nil, err
}
return &m, nil
}
// UnmarshalX509ChainArray unmarshalls the contents of the "chain:" entry in a
// GetEntries response in the case where the entry refers to an X509 leaf.
func UnmarshalX509ChainArray(b []byte) ([]ASN1Cert, error) {
return readASN1CertList(bytes.NewReader(b), CertificateChainLengthBytes, CertificateLengthBytes)
}
// UnmarshalPrecertChainArray unmarshalls the contents of the "chain:" entry in
// a GetEntries response in the case where the entry refers to a Precertificate
// leaf.
func UnmarshalPrecertChainArray(b []byte) ([]ASN1Cert, error) {
var chain []ASN1Cert
reader := bytes.NewReader(b)
// read the pre-cert entry:
precert, err := readVarBytes(reader, CertificateLengthBytes)
if err != nil {
return chain, err
}
chain = append(chain, precert)
// and then read and return the chain up to the root:
remainingChain, err := readASN1CertList(reader, CertificateChainLengthBytes, CertificateLengthBytes)
if err != nil {
return chain, err
}
chain = append(chain, remainingChain...)
return chain, nil
}
// UnmarshalDigitallySigned reconstructs a DigitallySigned structure from a Reader
func UnmarshalDigitallySigned(r io.Reader) (*DigitallySigned, error) {
var h byte
if err := binary.Read(r, binary.BigEndian, &h); err != nil {
return nil, fmt.Errorf("failed to read HashAlgorithm: %v", err)
}
var s byte
if err := binary.Read(r, binary.BigEndian, &s); err != nil {
return nil, fmt.Errorf("failed to read SignatureAlgorithm: %v", err)
}
sig, err := readVarBytes(r, SignatureLengthBytes)
if err != nil {
return nil, fmt.Errorf("failed to read Signature bytes: %v", err)
}
return &DigitallySigned{
HashAlgorithm: HashAlgorithm(h),
SignatureAlgorithm: SignatureAlgorithm(s),
Signature: sig,
}, nil
}
// MarshalDigitallySigned marshalls a DigitallySigned structure into a byte array
func MarshalDigitallySigned(ds DigitallySigned) ([]byte, error) {
var b bytes.Buffer
if err := b.WriteByte(byte(ds.HashAlgorithm)); err != nil {
return nil, fmt.Errorf("failed to write HashAlgorithm: %v", err)
}
if err := b.WriteByte(byte(ds.SignatureAlgorithm)); err != nil {
return nil, fmt.Errorf("failed to write SignatureAlgorithm: %v", err)
}
if err := writeVarBytes(&b, ds.Signature, SignatureLengthBytes); err != nil {
return nil, fmt.Errorf("failed to write HashAlgorithm: %v", err)
}
return b.Bytes(), nil
}
func checkCertificateFormat(cert ASN1Cert) error {
if len(cert) == 0 {
return errors.New("certificate is zero length")
}
if len(cert) > MaxCertificateLength {
return errors.New("certificate too large")
}
return nil
}
func checkExtensionsFormat(ext CTExtensions) error {
if len(ext) > MaxExtensionsLength {
return errors.New("extensions too large")
}
return nil
}
func serializeV1CertSCTSignatureInput(timestamp uint64, cert ASN1Cert, ext CTExtensions) ([]byte, error) {
if err := checkCertificateFormat(cert); err != nil {
return nil, err
}
if err := checkExtensionsFormat(ext); err != nil {
return nil, err
}
var buf bytes.Buffer
if err := binary.Write(&buf, binary.BigEndian, V1); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, CertificateTimestampSignatureType); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, timestamp); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, X509LogEntryType); err != nil {
return nil, err
}
if err := writeVarBytes(&buf, cert, CertificateLengthBytes); err != nil {
return nil, err
}
if err := writeVarBytes(&buf, ext, ExtensionsLengthBytes); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func serializeV1PrecertSCTSignatureInput(timestamp uint64, issuerKeyHash [issuerKeyHashLength]byte, tbs []byte, ext CTExtensions) ([]byte, error) {
if err := checkCertificateFormat(tbs); err != nil {
return nil, err
}
if err := checkExtensionsFormat(ext); err != nil {
return nil, err
}
var buf bytes.Buffer
if err := binary.Write(&buf, binary.BigEndian, V1); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, CertificateTimestampSignatureType); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, timestamp); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, PrecertLogEntryType); err != nil {
return nil, err
}
if _, err := buf.Write(issuerKeyHash[:]); err != nil {
return nil, err
}
if err := writeVarBytes(&buf, tbs, CertificateLengthBytes); err != nil {
return nil, err
}
if err := writeVarBytes(&buf, ext, ExtensionsLengthBytes); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func serializeV1SCTSignatureInput(sct SignedCertificateTimestamp, entry LogEntry) ([]byte, error) {
if sct.SCTVersion != V1 {
return nil, fmt.Errorf("unsupported SCT version, expected V1, but got %s", sct.SCTVersion)
}
if entry.Leaf.LeafType != TimestampedEntryLeafType {
return nil, fmt.Errorf("Unsupported leaf type %s", entry.Leaf.LeafType)
}
switch entry.Leaf.TimestampedEntry.EntryType {
case X509LogEntryType:
return serializeV1CertSCTSignatureInput(sct.Timestamp, entry.Leaf.TimestampedEntry.X509Entry, entry.Leaf.TimestampedEntry.Extensions)
case PrecertLogEntryType:
return serializeV1PrecertSCTSignatureInput(sct.Timestamp, entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash,
entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate,
entry.Leaf.TimestampedEntry.Extensions)
default:
return nil, fmt.Errorf("unknown TimestampedEntryLeafType %s", entry.Leaf.TimestampedEntry.EntryType)
}
}
// SerializeSCTSignatureInput serializes the passed in sct and log entry into
// the correct format for signing.
func SerializeSCTSignatureInput(sct SignedCertificateTimestamp, entry LogEntry) ([]byte, error) {
switch sct.SCTVersion {
case V1:
return serializeV1SCTSignatureInput(sct, entry)
default:
return nil, fmt.Errorf("unknown SCT version %d", sct.SCTVersion)
}
}
func serializeV1SCT(sct SignedCertificateTimestamp) ([]byte, error) {
if err := checkExtensionsFormat(sct.Extensions); err != nil {
return nil, err
}
var buf bytes.Buffer
if err := binary.Write(&buf, binary.BigEndian, V1); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, sct.LogID); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, sct.Timestamp); err != nil {
return nil, err
}
if err := writeVarBytes(&buf, sct.Extensions, ExtensionsLengthBytes); err != nil {
return nil, err
}
sig, err := MarshalDigitallySigned(sct.Signature)
if err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, sig); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// SerializeSCT serializes the passed in sct into the format specified
// by RFC6962 section 3.2
func SerializeSCT(sct SignedCertificateTimestamp) ([]byte, error) {
switch sct.SCTVersion {
case V1:
return serializeV1SCT(sct)
default:
return nil, fmt.Errorf("unknown SCT version %d", sct.SCTVersion)
}
}
func deserializeSCTV1(r io.Reader, sct *SignedCertificateTimestamp) error {
if err := binary.Read(r, binary.BigEndian, &sct.LogID); err != nil {
return err
}
if err := binary.Read(r, binary.BigEndian, &sct.Timestamp); err != nil {
return err
}
ext, err := readVarBytes(r, ExtensionsLengthBytes)
if err != nil {
return err
}
sct.Extensions = ext
ds, err := UnmarshalDigitallySigned(r)
if err != nil {
return err
}
sct.Signature = *ds
return nil
}
func DeserializeSCT(r io.Reader) (*SignedCertificateTimestamp, error) {
var sct SignedCertificateTimestamp
if err := binary.Read(r, binary.BigEndian, &sct.SCTVersion); err != nil {
return nil, err
}
switch sct.SCTVersion {
case V1:
return &sct, deserializeSCTV1(r, &sct)
default:
return nil, fmt.Errorf("unknown SCT version %d", sct.SCTVersion)
}
}
func serializeV1STHSignatureInput(sth SignedTreeHead) ([]byte, error) {
if sth.Version != V1 {
return nil, fmt.Errorf("invalid STH version %d", sth.Version)
}
if sth.TreeSize < 0 {
return nil, fmt.Errorf("invalid tree size %d", sth.TreeSize)
}
if len(sth.SHA256RootHash) != crypto.SHA256.Size() {
return nil, fmt.Errorf("invalid TreeHash length, got %d expected %d", len(sth.SHA256RootHash), crypto.SHA256.Size())
}
var buf bytes.Buffer
if err := binary.Write(&buf, binary.BigEndian, V1); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, TreeHashSignatureType); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, sth.Timestamp); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, sth.TreeSize); err != nil {
return nil, err
}
if err := binary.Write(&buf, binary.BigEndian, sth.SHA256RootHash); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// SerializeSTHSignatureInput serializes the passed in sth into the correct
// format for signing.
func SerializeSTHSignatureInput(sth SignedTreeHead) ([]byte, error) {
switch sth.Version {
case V1:
return serializeV1STHSignatureInput(sth)
default:
return nil, fmt.Errorf("unsupported STH version %d", sth.Version)
}
}

109
ct/signatures.go
View File

@ -1,109 +0,0 @@
package ct
import (
"crypto"
"crypto/ecdsa"
"crypto/rsa"
"crypto/sha256"
"crypto/x509"
"encoding/asn1"
"encoding/pem"
"errors"
"fmt"
"math/big"
)
// PublicKeyFromPEM parses a PEM formatted block and returns the public key contained within and any remaining unread bytes, or an error.
func PublicKeyFromPEM(b []byte) (crypto.PublicKey, SHA256Hash, []byte, error) {
p, rest := pem.Decode(b)
if p == nil {
return nil, [sha256.Size]byte{}, rest, fmt.Errorf("no PEM block found in %s", string(b))
}
k, err := x509.ParsePKIXPublicKey(p.Bytes)
return k, sha256.Sum256(p.Bytes), rest, err
}
// SignatureVerifier can verify signatures on SCTs and STHs
type SignatureVerifier struct {
pubKey crypto.PublicKey
}
// NewSignatureVerifier creates a new SignatureVerifier using the passed in PublicKey.
func NewSignatureVerifier(pk crypto.PublicKey) (*SignatureVerifier, error) {
switch pkType := pk.(type) {
case *rsa.PublicKey:
case *ecdsa.PublicKey:
default:
return nil, fmt.Errorf("Unsupported public key type %v", pkType)
}
return &SignatureVerifier{
pubKey: pk,
}, nil
}
// verifySignature verifies that the passed in signature over data was created by our PublicKey.
// Currently, only SHA256 is supported as a HashAlgorithm, and only ECDSA and RSA signatures are supported.
func (s SignatureVerifier) verifySignature(data []byte, sig DigitallySigned) error {
if sig.HashAlgorithm != SHA256 {
return fmt.Errorf("unsupported HashAlgorithm in signature: %v", sig.HashAlgorithm)
}
hasherType := crypto.SHA256
hasher := hasherType.New()
if _, err := hasher.Write(data); err != nil {
return fmt.Errorf("failed to write to hasher: %v", err)
}
hash := hasher.Sum([]byte{})
switch sig.SignatureAlgorithm {
case RSA:
rsaKey, ok := s.pubKey.(*rsa.PublicKey)
if !ok {
return fmt.Errorf("cannot verify RSA signature with %T key", s.pubKey)
}
if err := rsa.VerifyPKCS1v15(rsaKey, hasherType, hash, sig.Signature); err != nil {
return fmt.Errorf("failed to verify rsa signature: %v", err)
}
case ECDSA:
ecdsaKey, ok := s.pubKey.(*ecdsa.PublicKey)
if !ok {
return fmt.Errorf("cannot verify ECDSA signature with %T key", s.pubKey)
}
var ecdsaSig struct {
R, S *big.Int
}
rest, err := asn1.Unmarshal(sig.Signature, &ecdsaSig)
if err != nil {
return fmt.Errorf("failed to unmarshal ECDSA signature: %v", err)
}
if len(rest) != 0 {
return fmt.Errorf("Garbage following signature %v", rest)
}
if !ecdsa.Verify(ecdsaKey, hash, ecdsaSig.R, ecdsaSig.S) {
return errors.New("failed to verify ecdsa signature")
}
default:
return fmt.Errorf("unsupported signature type %v", sig.SignatureAlgorithm)
}
return nil
}
// VerifySCTSignature verifies that the SCT's signature is valid for the given LogEntry
func (s SignatureVerifier) VerifySCTSignature(sct SignedCertificateTimestamp, entry LogEntry) error {
sctData, err := SerializeSCTSignatureInput(sct, entry)
if err != nil {
return err
}
return s.verifySignature(sctData, sct.Signature)
}
// VerifySTHSignature verifies that the STH's signature is valid.
func (s SignatureVerifier) VerifySTHSignature(sth SignedTreeHead) error {
sthData, err := SerializeSTHSignatureInput(sth)
if err != nil {
return err
}
return s.verifySignature(sthData, sth.TreeHeadSignature)
}

333
ct/types.go
View File

@ -1,333 +0,0 @@
package ct
import (
"bytes"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
"time"
)
const (
issuerKeyHashLength = 32
)
///////////////////////////////////////////////////////////////////////////////
// The following structures represent those outlined in the RFC6962 document:
///////////////////////////////////////////////////////////////////////////////
// LogEntryType represents the LogEntryType enum from section 3.1 of the RFC:
// enum { x509_entry(0), precert_entry(1), (65535) } LogEntryType;
type LogEntryType uint16
func (e LogEntryType) String() string {
switch e {
case X509LogEntryType:
return "X509LogEntryType"
case PrecertLogEntryType:
return "PrecertLogEntryType"
}
panic(fmt.Sprintf("No string defined for LogEntryType constant value %d", e))
}
// LogEntryType constants, see section 3.1 of RFC6962.
const (
X509LogEntryType LogEntryType = 0
PrecertLogEntryType LogEntryType = 1
)
// MerkleLeafType represents the MerkleLeafType enum from section 3.4 of the
// RFC: enum { timestamped_entry(0), (255) } MerkleLeafType;
type MerkleLeafType uint8
func (m MerkleLeafType) String() string {
switch m {
case TimestampedEntryLeafType:
return "TimestampedEntryLeafType"
default:
return fmt.Sprintf("UnknownLeafType(%d)", m)
}
}
// MerkleLeafType constants, see section 3.4 of the RFC.
const (
TimestampedEntryLeafType MerkleLeafType = 0 // Entry type for an SCT
)
// Version represents the Version enum from section 3.2 of the RFC:
// enum { v1(0), (255) } Version;
type Version uint8
func (v Version) String() string {
switch v {
case V1:
return "V1"
default:
return fmt.Sprintf("UnknownVersion(%d)", v)
}
}
// CT Version constants, see section 3.2 of the RFC.
const (
V1 Version = 0
)
// SignatureType differentiates STH signatures from SCT signatures, see RFC
// section 3.2
type SignatureType uint8
func (st SignatureType) String() string {
switch st {
case CertificateTimestampSignatureType:
return "CertificateTimestamp"
case TreeHashSignatureType:
return "TreeHash"
default:
return fmt.Sprintf("UnknownSignatureType(%d)", st)
}
}
// SignatureType constants, see RFC section 3.2
const (
CertificateTimestampSignatureType SignatureType = 0
TreeHashSignatureType SignatureType = 1
)
// ASN1Cert type for holding the raw DER bytes of an ASN.1 Certificate
// (section 3.1)
type ASN1Cert []byte
// PreCert represents a Precertificate (section 3.2)
type PreCert struct {
IssuerKeyHash [issuerKeyHashLength]byte
TBSCertificate []byte
}
// CTExtensions is a representation of the raw bytes of any CtExtension
// structure (see section 3.2)
type CTExtensions []byte
// MerkleTreeNode represents an internal node in the CT tree
type MerkleTreeNode []byte
// ConsistencyProof represents a CT consistency proof (see sections 2.1.2 and
// 4.4)
type ConsistencyProof []MerkleTreeNode
// AuditPath represents a CT inclusion proof (see sections 2.1.1 and 4.5)
type AuditPath []MerkleTreeNode
// LeafInput represents a serialized MerkleTreeLeaf structure
type LeafInput []byte
// HashAlgorithm from the DigitallySigned struct
type HashAlgorithm byte
// HashAlgorithm constants
const (
None HashAlgorithm = 0
MD5 HashAlgorithm = 1
SHA1 HashAlgorithm = 2
SHA224 HashAlgorithm = 3
SHA256 HashAlgorithm = 4
SHA384 HashAlgorithm = 5
SHA512 HashAlgorithm = 6
)
func (h HashAlgorithm) String() string {
switch h {
case None:
return "None"
case MD5:
return "MD5"
case SHA1:
return "SHA1"
case SHA224:
return "SHA224"
case SHA256:
return "SHA256"
case SHA384:
return "SHA384"
case SHA512:
return "SHA512"
default:
return fmt.Sprintf("UNKNOWN(%d)", h)
}
}
// SignatureAlgorithm from the DigitallySigned struct
type SignatureAlgorithm byte
// SignatureAlgorithm constants
const (
Anonymous SignatureAlgorithm = 0
RSA SignatureAlgorithm = 1
DSA SignatureAlgorithm = 2
ECDSA SignatureAlgorithm = 3
)
func (s SignatureAlgorithm) String() string {
switch s {
case Anonymous:
return "Anonymous"
case RSA:
return "RSA"
case DSA:
return "DSA"
case ECDSA:
return "ECDSA"
default:
return fmt.Sprintf("UNKNOWN(%d)", s)
}
}
// DigitallySigned represents an RFC5246 DigitallySigned structure
type DigitallySigned struct {
HashAlgorithm HashAlgorithm
SignatureAlgorithm SignatureAlgorithm
Signature []byte
}
// FromBase64String populates the DigitallySigned structure from the base64 data passed in.
// Returns an error if the base64 data is invalid.
func (d *DigitallySigned) FromBase64String(b64 string) error {
raw, err := base64.StdEncoding.DecodeString(b64)
if err != nil {
return fmt.Errorf("failed to unbase64 DigitallySigned: %v", err)
}
ds, err := UnmarshalDigitallySigned(bytes.NewReader(raw))
if err != nil {
return fmt.Errorf("failed to unmarshal DigitallySigned: %v", err)
}
*d = *ds
return nil
}
// Base64String returns the base64 representation of the DigitallySigned struct.
func (d DigitallySigned) Base64String() (string, error) {
b, err := MarshalDigitallySigned(d)
if err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(b), nil
}
// MarshalJSON implements the json.Marshaller interface.
func (d DigitallySigned) MarshalJSON() ([]byte, error) {
b64, err := d.Base64String()
if err != nil {
return []byte{}, err
}
return []byte(`"` + b64 + `"`), nil
}
// UnmarshalJSON implements the json.Unmarshaler interface.
func (d *DigitallySigned) UnmarshalJSON(b []byte) error {
var content string
if err := json.Unmarshal(b, &content); err != nil {
return fmt.Errorf("failed to unmarshal DigitallySigned: %v", err)
}
return d.FromBase64String(content)
}
// LogEntry represents the contents of an entry in a CT log, see section 3.1.
type LogEntry struct {
Index int64
Leaf MerkleTreeLeaf
Chain []ASN1Cert
LeafBytes []byte
}
// SHA256Hash represents the output from the SHA256 hash function.
type SHA256Hash [sha256.Size]byte
// FromBase64String populates the SHA256 struct with the contents of the base64 data passed in.
func (s *SHA256Hash) FromBase64String(b64 string) error {
bs, err := base64.StdEncoding.DecodeString(b64)
if err != nil {
return fmt.Errorf("failed to unbase64 LogID: %v", err)
}
if len(bs) != sha256.Size {
return fmt.Errorf("invalid SHA256 length, expected 32 but got %d", len(bs))
}
copy(s[:], bs)
return nil
}
// Base64String returns the base64 representation of this SHA256Hash.
func (s SHA256Hash) Base64String() string {
return base64.StdEncoding.EncodeToString(s[:])
}
// Returns the raw base64url representation of this SHA256Hash.
func (s SHA256Hash) Base64URLString() string {
return base64.RawURLEncoding.EncodeToString(s[:])
}
// MarshalJSON implements the json.Marshaller interface for SHA256Hash.
func (s SHA256Hash) MarshalJSON() ([]byte, error) {
return []byte(`"` + s.Base64String() + `"`), nil
}
// UnmarshalJSON implements the json.Unmarshaller interface.
func (s *SHA256Hash) UnmarshalJSON(b []byte) error {
var content string
if err := json.Unmarshal(b, &content); err != nil {
return fmt.Errorf("failed to unmarshal SHA256Hash: %v", err)
}
return s.FromBase64String(content)
}
// SignedTreeHead represents the structure returned by the get-sth CT method
// after base64 decoding. See sections 3.5 and 4.3 in the RFC)
type SignedTreeHead struct {
Version Version `json:"sth_version"` // The version of the protocol to which the STH conforms
TreeSize uint64 `json:"tree_size"` // The number of entries in the new tree
Timestamp uint64 `json:"timestamp"` // The time at which the STH was created
SHA256RootHash SHA256Hash `json:"sha256_root_hash"` // The root hash of the log's Merkle tree
TreeHeadSignature DigitallySigned `json:"tree_head_signature"` // The Log's signature for this STH (see RFC section 3.5)
LogID SHA256Hash `json:"log_id"` // The SHA256 hash of the log's public key
}
func (sth *SignedTreeHead) TimestampTime() time.Time {
return time.Unix(int64(sth.Timestamp/1000), int64(sth.Timestamp%1000)*1_000_000).UTC()
}
// SignedCertificateTimestamp represents the structure returned by the
// add-chain and add-pre-chain methods after base64 decoding. (see RFC sections
// 3.2 ,4.1 and 4.2)
type SignedCertificateTimestamp struct {
SCTVersion Version `json:"sct_version"` // The version of the protocol to which the SCT conforms
LogID SHA256Hash `json:"id"` // the SHA-256 hash of the log's public key, calculated over
// the DER encoding of the key represented as SubjectPublicKeyInfo.
Timestamp uint64 `json:"timestamp"` // Timestamp (in ms since unix epoch) at which the SCT was issued
Extensions CTExtensions `json:"extensions"` // For future extensions to the protocol
Signature DigitallySigned `json:"signature"` // The Log's signature for this SCT
}
func (s SignedCertificateTimestamp) String() string {
return fmt.Sprintf("{Version:%d LogId:%s Timestamp:%d Extensions:'%s' Signature:%v}", s.SCTVersion,
base64.StdEncoding.EncodeToString(s.LogID[:]),
s.Timestamp,
s.Extensions,
s.Signature)
}
// TimestampedEntry is part of the MerkleTreeLeaf structure.
// See RFC section 3.4
type TimestampedEntry struct {
Timestamp uint64
EntryType LogEntryType
X509Entry ASN1Cert
PrecertEntry PreCert
Extensions CTExtensions
}
// MerkleTreeLeaf represents the deserialized structure of the hash input for the
// leaves of a log's Merkle tree. See RFC section 3.4
type MerkleTreeLeaf struct {
Version Version // the version of the protocol to which the MerkleTreeLeaf corresponds
LeafType MerkleLeafType // The type of the leaf input, currently only TimestampedEntry can exist
TimestampedEntry TimestampedEntry // The entry data itself
}

20
helpers.go
View File

@ -10,16 +10,9 @@
package certspotter
import (
"fmt"
"math/big"
"software.sslmate.com/src/certspotter/ct"
)
func IsPrecert(entry *ct.LogEntry) bool {
return entry.Leaf.TimestampedEntry.EntryType == ct.PrecertLogEntryType
}
type CertInfo struct {
TBS *TBSCertificate
@ -68,19 +61,6 @@ func MakeCertInfoFromRawCert(certBytes []byte) (*CertInfo, error) {
return MakeCertInfoFromRawTBS(cert.GetRawTBSCertificate())
}
func MakeCertInfoFromLogEntry(entry *ct.LogEntry) (*CertInfo, error) {
switch entry.Leaf.TimestampedEntry.EntryType {
case ct.X509LogEntryType:
return MakeCertInfoFromRawCert(entry.Leaf.TimestampedEntry.X509Entry)
case ct.PrecertLogEntryType:
return MakeCertInfoFromRawTBS(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
default:
return nil, fmt.Errorf("MakeCertInfoFromCTEntry: unknown CT entry type (neither X509 nor precert)")
}
}
func MatchesWildcard(dnsName string, pattern string) bool {
for len(pattern) > 0 {
if pattern[0] == '*' {

7
loglist/schema.go
View File

@ -12,7 +12,7 @@ package loglist
import (
"time"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/cttypes"
)
type List struct {
@ -30,7 +30,7 @@ type Operator struct {
type Log struct {
Key []byte `json:"key"`
LogID ct.SHA256Hash `json:"log_id"`
LogID cttypes.LogID `json:"log_id"`
MMD int `json:"mmd"`
URL string `json:"url,omitempty"` // only for rfc6962 logs
SubmissionURL string `json:"submission_url,omitempty"` // only for static-ct-api logs
@ -44,6 +44,9 @@ type Log struct {
EndExclusive time.Time `json:"end_exclusive"`
} `json:"temporal_interval"`
DownloadWorkers int `json:"certspotter_download_workers,omitempty"`
DownloadJobSize int `json:"certspotter_download_job_size,omitempty"`
// TODO: add previous_operators
}

4
man/certspotter-script.md
View File

@ -123,6 +123,10 @@ The following environment variables are set for `discovered_cert` events:
: Error parsing the serial number, if any. If this variable is set, then `SERIAL` is unset.
`CHAIN_ERROR`
: Error building or verifying the certificate chain, if any. If this variable is set, then the certificate chain in `CERT_FILENAME` may be incomplete or invalid.
## Malformed certificate information
The following environment variables are set for `malformed_cert` events:

8
monitor/daemon.go
View File

@ -63,7 +63,7 @@ func (daemon *daemon) healthCheck(ctx context.Context) error {
for _, task := range daemon.tasks {
if err := healthCheckLog(ctx, daemon.config, task.log); err != nil {
return fmt.Errorf("error checking health of log %q: %w", task.log.URL, err)
return fmt.Errorf("error checking health of log %q: %w", task.log.GetMonitoringURL(), err)
}
}
return nil
@ -75,12 +75,12 @@ func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task {
defer cancel()
err := monitorLogContinously(ctx, daemon.config, ctlog)
if daemon.config.Verbose {
log.Printf("task for log %s stopped with error %s", ctlog.URL, err)
log.Printf("task for log %s stopped with error %s", ctlog.GetMonitoringURL(), err)
}
if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
return nil
} else {
return fmt.Errorf("error while monitoring %s: %w", ctlog.URL, err)
return fmt.Errorf("error while monitoring %s: %w", ctlog.GetMonitoringURL(), err)
}
})
return task{log: ctlog, stop: cancel}
@ -113,7 +113,7 @@ func (daemon *daemon) loadLogList(ctx context.Context) error {
continue
}
if daemon.config.Verbose {
log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.URL)
log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.GetMonitoringURL())
}
daemon.tasks[logID] = daemon.startTask(ctx, ctlog)
}

25
monitor/discoveredcert.go
View File

@ -18,17 +18,18 @@ import (
"time"
"software.sslmate.com/src/certspotter"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/cttypes"
)
type DiscoveredCert struct {
WatchItem WatchItem
LogEntry *LogEntry
Info *certspotter.CertInfo
Chain []ct.ASN1Cert // first entry is the leaf certificate or precertificate
TBSSHA256 [32]byte // computed over Info.TBS.Raw
SHA256 [32]byte // computed over Chain[0]
PubkeySHA256 [32]byte // computed over Info.TBS.PublicKey.FullBytes
Chain []cttypes.ASN1Cert // first entry is the leaf certificate or precertificate
ChainError error // any error generating or validating Chain; if non-nil, Chain may be partial or incorrect
TBSSHA256 [32]byte // computed over Info.TBS.Raw
SHA256 [32]byte // computed over Chain[0]
PubkeySHA256 [32]byte // computed over Info.TBS.PublicKey.FullBytes
Identifiers *certspotter.Identifiers
}
@ -40,6 +41,9 @@ type certPaths struct {
func (cert *DiscoveredCert) pemChain() []byte {
var buffer bytes.Buffer
if cert.ChainError != nil {
fmt.Fprintln(&buffer, "Warning: this chain may be incomplete or invalid: %s", cert.ChainError)
}
for _, certBytes := range cert.Chain {
if err := pem.Encode(&buffer, &pem.Block{
Type: "CERTIFICATE",
@ -88,7 +92,7 @@ func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string {
"EVENT=discovered_cert",
"SUMMARY=" + certNotificationSummary(cert),
"CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented
"LOG_URI=" + cert.LogEntry.Log.URL,
"LOG_URI=" + cert.LogEntry.Log.GetMonitoringURL(),
"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
"WATCH_ITEM=" + cert.WatchItem.String(),
"TBS_SHA256=" + hex.EncodeToString(cert.TBSSHA256[:]),
@ -133,6 +137,10 @@ func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string {
env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error())
}
if cert.ChainError != nil {
env = append(env, "CHAIN_ERROR="+cert.ChainError.Error())
}
return env
}
@ -162,8 +170,11 @@ func certNotificationText(cert *DiscoveredCert, paths *certPaths) string {
writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
}
writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL))
writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.GetMonitoringURL()))
writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:]))
if cert.ChainError != nil {
writeField("Error Building Chain", cert.ChainError.Error())
}
if paths != nil {
writeField("Filename", paths.certPath)
}

2
monitor/errors.go
View File

@ -22,7 +22,7 @@ func recordError(ctx context.Context, config *Config, ctlog *loglist.Log, errToR
if ctlog == nil {
log.Print(errToRecord)
} else {
log.Print(ctlog.URL, ": ", errToRecord)
log.Print(ctlog.GetMonitoringURL(), ": ", errToRecord)
}
}
}

32
monitor/fsstate.go
View File

@ -21,8 +21,9 @@ import (
"path/filepath"
"strings"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/cttypes"
"software.sslmate.com/src/certspotter/loglist"
"software.sslmate.com/src/certspotter/merkletree"
)
type FilesystemState struct {
@ -77,17 +78,17 @@ func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state
return writeJSONFile(filePath, state, 0666)
}
func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) error {
sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
return storeSTHInDir(sthsDirPath, sth)
}
func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*ct.SignedTreeHead, error) {
func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*cttypes.SignedTreeHead, error) {
sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
return loadSTHsFromDir(sthsDirPath)
}
func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) error {
sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
return removeSTHFromDir(sthsDirPath, sth)
}
@ -154,24 +155,17 @@ func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEn
textPath = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index))
)
summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.URL)
entryJSON := struct {
LeafInput []byte `json:"leaf_input"`
ExtraData []byte `json:"extra_data"`
}{
LeafInput: entry.LeafInput,
ExtraData: entry.ExtraData,
}
summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.GetMonitoringURL())
leafHash := merkletree.HashLeaf(entry.LeafInput())
text := new(strings.Builder)
writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.URL))
writeField("Leaf Hash", entry.LeafHash.Base64String())
writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.GetMonitoringURL()))
writeField("Leaf Hash", leafHash.Base64String())
writeField("Error", parseError.Error())
if err := writeJSONFile(entryPath, entryJSON, 0666); err != nil {
if err := writeJSONFile(entryPath, entry.Entry, 0666); err != nil {
return fmt.Errorf("error saving JSON file: %w", err)
}
if err := writeTextFile(textPath, text.String(), 0666); err != nil {
@ -181,9 +175,9 @@ func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEn
environ := []string{
"EVENT=malformed_cert",
"SUMMARY=" + summary,
"LOG_URI=" + entry.Log.URL,
"LOG_URI=" + entry.Log.GetMonitoringURL(),
"ENTRY_INDEX=" + fmt.Sprint(entry.Index),
"LEAF_HASH=" + entry.LeafHash.Base64String(),
"LEAF_HASH=" + leafHash.Base64String(),
"PARSE_ERROR=" + parseError.Error(),
"ENTRY_FILENAME=" + entryPath,
"TEXT_FILENAME=" + textPath,
@ -233,7 +227,7 @@ func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, e
if ctlog == nil {
log.Print(err)
} else {
log.Print(ctlog.URL, ":", err)
log.Print(ctlog.GetMonitoringURL(), ":", err)
}
return nil
}

24
monitor/healthcheck.go
View File

@ -15,7 +15,7 @@ import (
"strings"
"time"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/cttypes"
"software.sslmate.com/src/certspotter/loglist"
)
@ -31,7 +31,7 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err
return nil
}
if time.Since(state.LastSuccess) < config.HealthCheckInterval {
if state.VerifiedSTH != nil && time.Since(state.VerifiedSTH.TimestampTime()) < config.HealthCheckInterval {
return nil
}
@ -42,9 +42,8 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err
if len(sths) == 0 {
info := &StaleSTHInfo{
Log: ctlog,
LastSuccess: state.LastSuccess,
LatestSTH: state.VerifiedSTH,
Log: ctlog,
LatestSTH: state.VerifiedSTH,
}
if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
return fmt.Errorf("error notifying about stale STH: %w", err)
@ -69,14 +68,13 @@ type HealthCheckFailure interface {
}
type StaleSTHInfo struct {
Log *loglist.Log
LastSuccess time.Time
LatestSTH *ct.SignedTreeHead // may be nil
Log *loglist.Log
LatestSTH *cttypes.SignedTreeHead // may be nil
}
type BacklogInfo struct {
Log *loglist.Log
LatestSTH *ct.SignedTreeHead
LatestSTH *cttypes.SignedTreeHead
Position uint64
}
@ -92,10 +90,10 @@ func (e *BacklogInfo) Backlog() uint64 {
}
func (e *StaleSTHInfo) Summary() string {
return fmt.Sprintf("Unable to contact %s since %s", e.Log.URL, e.LastSuccess)
return fmt.Sprintf("%s is out-of-date", e.Log.GetMonitoringURL())
}
func (e *BacklogInfo) Summary() string {
return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.URL)
return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.GetMonitoringURL())
}
func (e *StaleLogListInfo) Summary() string {
return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess)
@ -103,7 +101,7 @@ func (e *StaleLogListInfo) Summary() string {
func (e *StaleSTHInfo) Text() string {
text := new(strings.Builder)
fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess)
fmt.Fprintf(text, "certspotter has been unable to get up-to-date information about %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.GetMonitoringURL())
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "For details, see certspotter's stderr output.\n")
fmt.Fprintf(text, "\n")
@ -116,7 +114,7 @@ func (e *StaleSTHInfo) Text() string {
}
func (e *BacklogInfo) Text() string {
text := new(strings.Builder)
fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL)
fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.GetMonitoringURL())
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n")
fmt.Fprintf(text, "\n")

11
monitor/loglist.go
View File

@ -12,11 +12,11 @@ package monitor
import (
"context"
"fmt"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/cttypes"
"software.sslmate.com/src/certspotter/loglist"
)
type LogID = ct.SHA256Hash
type LogID = cttypes.LogID
func getLogList(ctx context.Context, source string, token *loglist.ModificationToken) (map[LogID]*loglist.Log, *loglist.ModificationToken, error) {
list, newToken, err := loglist.LoadIfModified(ctx, source, token)
@ -33,6 +33,13 @@ func getLogList(ctx context.Context, source string, token *loglist.ModificationT
}
logs[log.LogID] = log
}
for logIndex := range list.Operators[operatorIndex].TiledLogs {
log := &list.Operators[operatorIndex].TiledLogs[logIndex]
if _, exists := logs[log.LogID]; exists {
return nil, nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String())
}
logs[log.LogID] = log
}
}
return logs, newToken, nil
}

614
monitor/monitor.go
View File

@ -1,4 +1,4 @@
// Copyright (C) 2023 Opsmate, Inc.
// Copyright (C) 2025 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
@ -11,275 +11,503 @@ package monitor
import (
"context"
"crypto/x509"
"errors"
"fmt"
"golang.org/x/sync/errgroup"
"log"
"strings"
mathrand "math/rand/v2"
"net/url"
"slices"
"time"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/ct/client"
"software.sslmate.com/src/certspotter/ctclient"
"software.sslmate.com/src/certspotter/ctcrypto"
"software.sslmate.com/src/certspotter/cttypes"
"software.sslmate.com/src/certspotter/loglist"
"software.sslmate.com/src/certspotter/merkletree"
"software.sslmate.com/src/certspotter/sequencer"
)
const (
maxGetEntriesSize = 1000
monitorLogInterval = 5 * time.Minute
getSTHInterval = 5 * time.Minute
)
func isFatalLogError(err error) bool {
return errors.Is(err, context.Canceled)
func downloadJobSize(ctlog *loglist.Log) uint64 {
if ctlog.IsStaticCTAPI() {
return ctclient.StaticTileWidth
} else if ctlog.DownloadJobSize != 0 {
return uint64(ctlog.DownloadJobSize)
} else {
return 256
}
}
func newLogClient(ctlog *loglist.Log) (*client.LogClient, error) {
logKey, err := x509.ParsePKIXPublicKey(ctlog.Key)
if err != nil {
return nil, fmt.Errorf("error parsing log key: %w", err)
func downloadWorkers(ctlog *loglist.Log) int {
if ctlog.DownloadWorkers != 0 {
return ctlog.DownloadWorkers
} else {
return 4
}
verifier, err := ct.NewSignatureVerifier(logKey)
if err != nil {
return nil, fmt.Errorf("error with log key: %w", err)
}
return client.NewWithVerifier(strings.TrimRight(ctlog.URL, "/"), verifier), nil
}
func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) error {
logClient, err := newLogClient(ctlog)
if err != nil {
return err
}
type verifyEntriesError struct {
sth *cttypes.SignedTreeHead
entriesRootHash merkletree.Hash
}
ticker := time.NewTicker(monitorLogInterval)
defer ticker.Stop()
func (e *verifyEntriesError) Error() string {
return fmt.Sprintf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", e.sth.TreeSize, e.sth.RootHash, e.entriesRootHash)
}
func withRetry(ctx context.Context, maxRetries int, f func() error) error {
const minSleep = 1 * time.Second
const maxSleep = 10 * time.Minute
numRetries := 0
for ctx.Err() == nil {
if err := monitorLog(ctx, config, ctlog, logClient); err != nil {
err := f()
if err == nil || errors.Is(err, context.Canceled) {
return err
}
select {
case <-ctx.Done():
case <-ticker.C:
if maxRetries != -1 && numRetries >= maxRetries {
return fmt.Errorf("%w (retried %d times)", err, numRetries)
}
upperBound := min(minSleep*(1<<numRetries)*2, maxSleep)
lowerBound := max(upperBound/2, minSleep)
sleepTime := lowerBound + mathrand.N(upperBound-lowerBound)
if err := sleep(ctx, sleepTime); err != nil {
return err
}
numRetries++
}
return ctx.Err()
}
func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClient *client.LogClient) (returnedErr error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
func getEntriesFull(ctx context.Context, client ctclient.Log, startInclusive, endInclusive uint64) ([]ctclient.Entry, error) {
allEntries := make([]ctclient.Entry, 0, endInclusive-startInclusive+1)
for startInclusive <= endInclusive {
entries, err := client.GetEntries(ctx, startInclusive, endInclusive)
if err != nil {
return nil, err
}
allEntries = append(allEntries, entries...)
startInclusive += uint64(len(entries))
}
return allEntries, nil
}
func getAndVerifySTH(ctx context.Context, ctlog *loglist.Log, client ctclient.Log) (*cttypes.SignedTreeHead, string, error) {
sth, url, err := client.GetSTH(ctx)
if err != nil {
return nil, "", fmt.Errorf("error getting STH: %w", err)
}
if err := ctcrypto.PublicKey(ctlog.Key).Verify(ctcrypto.SignatureInputForSTH(sth), sth.Signature); err != nil {
return nil, "", fmt.Errorf("STH has invalid signature: %w", err)
}
if now := time.Now(); sth.TimestampTime().After(now) {
return nil, "", fmt.Errorf("STH timestamp %s is after current time %s (either log is misbehaving or your system clock is incorrect)", sth.TimestampTime(), now)
}
return sth, url, nil
}
type logClient struct {
log *loglist.Log
client ctclient.Log
}
func (client *logClient) GetSTH(ctx context.Context) (sth *cttypes.SignedTreeHead, url string, err error) {
err = withRetry(ctx, -1, func() error {
sth, url, err = getAndVerifySTH(ctx, client.log, client.client)
return err
})
return
}
func (client *logClient) GetRoots(ctx context.Context) (roots [][]byte, err error) {
err = withRetry(ctx, -1, func() error {
roots, err = client.client.GetRoots(ctx)
return err
})
return
}
func (client *logClient) GetEntries(ctx context.Context, startInclusive, endInclusive uint64) (entries []ctclient.Entry, err error) {
err = withRetry(ctx, -1, func() error {
entries, err = client.client.GetEntries(ctx, startInclusive, endInclusive)
return err
})
return
}
func (client *logClient) ReconstructTree(ctx context.Context, sth *cttypes.SignedTreeHead) (tree *merkletree.CollapsedTree, err error) {
err = withRetry(ctx, -1, func() error {
tree, err = client.client.ReconstructTree(ctx, sth)
return err
})
return
}
type issuerGetter struct {
logGetter ctclient.IssuerGetter
}
func (ig *issuerGetter) GetIssuer(ctx context.Context, fingerprint *[32]byte) (issuer []byte, err error) {
// TODO-2 check cache
err = withRetry(ctx, 7, func() error {
issuer, err = ig.logGetter.GetIssuer(ctx, fingerprint)
return err
})
if err == nil {
// TODO-2 insert into cache
}
return
}
func newLogClient(ctlog *loglist.Log) (ctclient.Log, ctclient.IssuerGetter, error) {
switch {
case ctlog.IsRFC6962():
logURL, err := url.Parse(ctlog.URL)
if err != nil {
return nil, nil, fmt.Errorf("log has invalid URL: %w", err)
}
return &logClient{
log: ctlog,
client: &ctclient.RFC6962Log{URL: logURL},
}, nil, nil
case ctlog.IsStaticCTAPI():
submissionURL, err := url.Parse(ctlog.SubmissionURL)
if err != nil {
return nil, nil, fmt.Errorf("log has invalid submission URL: %w", err)
}
monitoringURL, err := url.Parse(ctlog.MonitoringURL)
if err != nil {
return nil, nil, fmt.Errorf("log has invalid monitoring URL: %w", err)
}
client := &ctclient.StaticLog{
SubmissionURL: submissionURL,
MonitoringURL: monitoringURL,
ID: ctlog.LogID,
}
return &logClient{
log: ctlog,
client: client,
}, &issuerGetter{
logGetter: client,
}, nil
default:
return nil, nil, fmt.Errorf("log uses unknown protocol")
}
}
func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) (returnedErr error) {
client, issuerGetter, err := newLogClient(ctlog)
if err != nil {
return err
}
if err := config.State.PrepareLog(ctx, ctlog.LogID); err != nil {
return fmt.Errorf("error preparing state: %w", err)
}
startTime := time.Now()
latestSTH, err := logClient.GetSTH(ctx)
if isFatalLogError(err) {
return err
} else if err != nil {
recordError(ctx, config, ctlog, fmt.Errorf("error fetching latest STH: %w", err))
return nil
}
latestSTH.LogID = ctlog.LogID
if err := config.State.StoreSTH(ctx, ctlog.LogID, latestSTH); err != nil {
return fmt.Errorf("error storing latest STH: %w", err)
}
state, err := config.State.LoadLogState(ctx, ctlog.LogID)
if err != nil {
return fmt.Errorf("error loading log state: %w", err)
}
if state == nil {
if config.StartAtEnd {
tree, err := reconstructTree(ctx, logClient, latestSTH)
if isFatalLogError(err) {
sth, _, err := client.GetSTH(ctx)
if err != nil {
return err
}
tree, err := client.ReconstructTree(ctx, sth)
if err != nil {
return err
} else if err != nil {
recordError(ctx, config, ctlog, fmt.Errorf("error reconstructing tree of size %d: %w", latestSTH.TreeSize, err))
return nil
}
state = &LogState{
DownloadPosition: tree,
VerifiedPosition: tree,
VerifiedSTH: latestSTH,
LastSuccess: startTime.UTC(),
VerifiedSTH: sth,
}
} else {
state = &LogState{
DownloadPosition: merkletree.EmptyCollapsedTree(),
VerifiedPosition: merkletree.EmptyCollapsedTree(),
VerifiedSTH: nil,
LastSuccess: startTime.UTC(),
}
}
if config.Verbose {
log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size())
log.Printf("brand new log %s (starting from %d)", ctlog.GetMonitoringURL(), state.DownloadPosition.Size())
}
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
return fmt.Errorf("error storing log state: %w", err)
}
}
sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
if err != nil {
return fmt.Errorf("error loading STHs: %w", err)
}
for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() {
// TODO-4: audit sths[0] against state.VerifiedSTH
if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
return fmt.Errorf("error removing STH: %w", err)
}
sths = sths[1:]
}
defer func() {
if config.Verbose {
log.Printf("saving state in defer for %s", ctlog.URL)
log.Printf("saving state in defer for %s", ctlog.GetMonitoringURL())
}
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil && returnedErr == nil {
storeCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
if err := config.State.StoreLogState(storeCtx, ctlog.LogID, state); err != nil && returnedErr == nil {
returnedErr = fmt.Errorf("error storing log state: %w", err)
}
}()
if len(sths) == 0 {
state.LastSuccess = startTime.UTC()
return nil
}
retry:
position := state.DownloadPosition.Size()
var (
downloadBegin = state.DownloadPosition.Size()
downloadEnd = sths[len(sths)-1].TreeSize
entries = make(chan client.GetEntriesItem, maxGetEntriesSize)
downloadErr error
)
if config.Verbose {
log.Printf("downloading entries from %s in range [%d, %d)", ctlog.URL, downloadBegin, downloadEnd)
// generateBatchesWorker ==> downloadWorker ==> processWorker ==> saveStateWorker
batches := make(chan *batch, downloadWorkers(ctlog))
processedBatches := sequencer.New[batch](0, uint64(downloadWorkers(ctlog))*10)
group, gctx := errgroup.WithContext(ctx)
group.Go(func() error { return getSTHWorker(gctx, config, ctlog, client) })
group.Go(func() error { return generateBatchesWorker(gctx, config, ctlog, position, batches) })
for range downloadWorkers(ctlog) {
downloadedBatches := make(chan *batch, 1)
group.Go(func() error { return downloadWorker(gctx, config, ctlog, client, batches, downloadedBatches) })
group.Go(func() error {
return processWorker(gctx, config, ctlog, issuerGetter, downloadedBatches, processedBatches)
})
}
go func() {
defer close(entries)
downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd)
}()
for rawEntry := range entries {
entry := &LogEntry{
Log: ctlog,
Index: state.DownloadPosition.Size(),
LeafInput: rawEntry.LeafInput,
ExtraData: rawEntry.ExtraData,
LeafHash: merkletree.HashLeaf(rawEntry.LeafInput),
group.Go(func() error { return saveStateWorker(gctx, config, ctlog, state, processedBatches) })
err = group.Wait()
if verifyErr := (*verifyEntriesError)(nil); errors.As(err, &verifyErr) {
recordError(ctx, config, ctlog, verifyErr)
state.rewindDownloadPosition()
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
return fmt.Errorf("error storing log state: %w", err)
}
if err := processLogEntry(ctx, config, entry); err != nil {
return fmt.Errorf("error processing entry %d: %w", entry.Index, err)
}
state.DownloadPosition.Add(entry.LeafHash)
rootHash := state.DownloadPosition.CalculateRoot()
shouldSaveState := state.DownloadPosition.Size()%10000 == 0
for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize {
if merkletree.Hash(sths[0].SHA256RootHash) != rootHash {
recordError(ctx, config, ctlog, fmt.Errorf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
state.DownloadPosition = state.VerifiedPosition
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
return fmt.Errorf("error storing log state: %w", err)
}
return nil
}
state.VerifiedPosition = state.DownloadPosition
state.VerifiedSTH = sths[0]
shouldSaveState = true
if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
return fmt.Errorf("error removing verified STH: %w", err)
}
sths = sths[1:]
}
if shouldSaveState {
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
return fmt.Errorf("error storing state file: %w", err)
}
if err := sleep(ctx, 5*time.Minute); err != nil {
return err
}
goto retry
}
if isFatalLogError(downloadErr) {
return downloadErr
} else if downloadErr != nil {
recordError(ctx, config, ctlog, fmt.Errorf("error downloading entries: %w", downloadErr))
return nil
}
if config.Verbose {
log.Printf("finished downloading entries from %s", ctlog.URL)
}
state.LastSuccess = startTime.UTC()
return nil
return err
}
func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error {
for begin < end && ctx.Err() == nil {
size := end - begin
if size > maxGetEntriesSize {
size = maxGetEntriesSize
}
entries, err := logClient.GetRawEntries(ctx, begin, begin+size-1)
func getSTHWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log) error {
for ctx.Err() == nil {
sth, _, err := client.GetSTH(ctx)
if err != nil {
return err
}
for _, entry := range entries {
if ctx.Err() != nil {
return ctx.Err()
}
select {
case <-ctx.Done():
return ctx.Err()
case entriesChan <- entry:
}
if err := config.State.StoreSTH(ctx, ctlog.LogID, sth); err != nil {
return fmt.Errorf("error storing STH: %w", err)
}
if err := sleep(ctx, getSTHInterval); err != nil {
return err
}
begin += uint64(len(entries))
}
return ctx.Err()
}
func reconstructTree(ctx context.Context, logClient *client.LogClient, sth *ct.SignedTreeHead) (*merkletree.CollapsedTree, error) {
if sth.TreeSize == 0 {
return merkletree.EmptyCollapsedTree(), nil
}
entries, err := logClient.GetRawEntries(ctx, sth.TreeSize-1, sth.TreeSize-1)
if err != nil {
return nil, err
}
leafHash := merkletree.HashLeaf(entries[0].LeafInput)
var tree *merkletree.CollapsedTree
if sth.TreeSize > 1 {
// XXX: if leafHash is in the tree in more than one place, this might not return the proof that we need ... get-entry-and-proof avoids this problem but not all logs support it
auditPath, _, err := logClient.GetAuditProof(ctx, leafHash[:], sth.TreeSize)
if err != nil {
return nil, err
}
hashes := make([]merkletree.Hash, len(auditPath))
for i := range hashes {
copy(hashes[i][:], auditPath[len(auditPath)-i-1])
}
tree, err = merkletree.NewCollapsedTree(hashes, sth.TreeSize-1)
if err != nil {
return nil, fmt.Errorf("log returned invalid audit proof for %x to %d: %w", leafHash, sth.TreeSize, err)
}
} else {
tree = merkletree.EmptyCollapsedTree()
}
tree.Add(leafHash)
rootHash := tree.CalculateRoot()
if rootHash != merkletree.Hash(sth.SHA256RootHash) {
return nil, fmt.Errorf("calculated root hash (%x) does not match signed tree head (%x) at size %d", rootHash, sth.SHA256RootHash, sth.TreeSize)
}
return tree, nil
type batch struct {
number uint64
begin, end uint64
sths []*cttypes.SignedTreeHead // STHs with sizes in range [begin,end], sorted by TreeSize
entries []ctclient.Entry // in range [begin,end)
}
func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.Log, position uint64, batches chan<- *batch) error {
ticker := time.NewTicker(15 * time.Second)
var number uint64
for ctx.Err() == nil {
sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
if err != nil {
return fmt.Errorf("error loading STHs: %w", err)
}
for len(sths) > 0 && sths[0].TreeSize < position {
// TODO-4: audit sths[0] against log's verified STH
if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
return fmt.Errorf("error removing STH: %w", err)
}
sths = sths[1:]
}
position, number, err = generateBatches(ctx, ctlog, position, number, sths, batches)
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
return ctx.Err()
}
// return the earliest STH timestamp within the right-most tile
func tileEarliestTimestamp(sths []*cttypes.SignedTreeHead) time.Time {
largestSTH, sths := sths[len(sths)-1], sths[:len(sths)-1]
tileNumber := largestSTH.TreeSize / ctclient.StaticTileWidth
earliest := largestSTH.TimestampTime()
for _, sth := range slices.Backward(sths) {
if sth.TreeSize/ctclient.StaticTileWidth != tileNumber {
break
}
if timestamp := sth.TimestampTime(); timestamp.Before(earliest) {
earliest = timestamp
}
}
return earliest
}
func generateBatches(ctx context.Context, ctlog *loglist.Log, position uint64, number uint64, sths []*cttypes.SignedTreeHead, batches chan<- *batch) (uint64, uint64, error) {
downloadJobSize := downloadJobSize(ctlog)
if len(sths) == 0 {
return position, number, nil
}
largestSTH := sths[len(sths)-1]
treeSize := largestSTH.TreeSize
if ctlog.IsStaticCTAPI() && time.Since(tileEarliestTimestamp(sths)) < 5*time.Minute {
// Round down to the tile boundary to avoid downloading a partial tile that was recently discovered
// In a future invocation of this function, either enough time will have passed that this code path will be skipped, or the log will have grown and treeSize will be rounded to a larger tile boundary
treeSize -= treeSize % ctclient.StaticTileWidth
}
for {
batch := &batch{
number: number,
begin: position,
end: min(treeSize, (position/downloadJobSize+1)*downloadJobSize),
}
for len(sths) > 0 && sths[0].TreeSize <= batch.end {
batch.sths = append(batch.sths, sths[0])
sths = sths[1:]
}
select {
case <-ctx.Done():
return position, number, ctx.Err()
default:
}
select {
case <-ctx.Done():
return position, number, ctx.Err()
case batches <- batch:
}
number++
if position == batch.end {
break
}
position = batch.end
}
return position, number, nil
}
func downloadWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log, batchesIn <-chan *batch, batchesOut chan<- *batch) error {
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
var batch *batch
select {
case <-ctx.Done():
return ctx.Err()
case batch = <-batchesIn:
}
entries, err := getEntriesFull(ctx, client, batch.begin, batch.end-1)
if err != nil {
return err
}
batch.entries = entries
select {
case <-ctx.Done():
return ctx.Err()
default:
}
select {
case <-ctx.Done():
return ctx.Err()
case batchesOut <- batch:
}
}
return nil
}
func processWorker(ctx context.Context, config *Config, ctlog *loglist.Log, issuerGetter ctclient.IssuerGetter, batchesIn <-chan *batch, batchesOut *sequencer.Channel[batch]) error {
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
var batch *batch
select {
case <-ctx.Done():
return ctx.Err()
case batch = <-batchesIn:
}
for offset, entry := range batch.entries {
index := batch.begin + uint64(offset)
if err := processLogEntry(ctx, config, issuerGetter, &LogEntry{
Entry: entry,
Index: index,
Log: ctlog,
}); err != nil {
return fmt.Errorf("error processing entry %d: %w", index, err)
}
}
if err := batchesOut.Add(ctx, batch.number, batch); err != nil {
return err
}
}
}
func saveStateWorker(ctx context.Context, config *Config, ctlog *loglist.Log, state *LogState, batchesIn *sequencer.Channel[batch]) error {
for {
batch, err := batchesIn.Next(ctx)
if err != nil {
return err
}
if batch.begin != state.DownloadPosition.Size() {
panic(fmt.Errorf("saveStateWorker: expected batch to start at %d but got %d instead", state.DownloadPosition.Size(), batch.begin))
}
rootHash := state.DownloadPosition.CalculateRoot()
for {
for len(batch.sths) > 0 && batch.sths[0].TreeSize == state.DownloadPosition.Size() {
sth := batch.sths[0]
batch.sths = batch.sths[1:]
if sth.RootHash != rootHash {
return &verifyEntriesError{
sth: sth,
entriesRootHash: rootHash,
}
}
state.advanceVerifiedPosition()
state.VerifiedSTH = sth
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
return fmt.Errorf("error storing log state: %w", err)
}
// don't remove the STH until state has been durably stored
if err := config.State.RemoveSTH(ctx, ctlog.LogID, sth); err != nil {
return fmt.Errorf("error removing verified STH: %w", err)
}
}
if len(batch.entries) == 0 {
break
}
entry := batch.entries[0]
batch.entries = batch.entries[1:]
leafHash := merkletree.HashLeaf(entry.LeafInput())
state.DownloadPosition.Add(leafHash)
rootHash = state.DownloadPosition.CalculateRoot()
}
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
return fmt.Errorf("error storing log state: %w", err)
}
}
}
func sleep(ctx context.Context, duration time.Duration) error {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}

91
monitor/process.go
View File

@ -1,4 +1,4 @@
// Copyright (C) 2023 Opsmate, Inc.
// Copyright (C) 2025 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
@ -10,79 +10,94 @@
package monitor
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"software.sslmate.com/src/certspotter"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/ctclient"
"software.sslmate.com/src/certspotter/cttypes"
"software.sslmate.com/src/certspotter/loglist"
"software.sslmate.com/src/certspotter/merkletree"
)
type LogEntry struct {
Log *loglist.Log
Index uint64
LeafInput []byte
ExtraData []byte
LeafHash merkletree.Hash
ctclient.Entry
Index uint64
Log *loglist.Log
}
func processLogEntry(ctx context.Context, config *Config, entry *LogEntry) error {
leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput))
func processLogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry) error {
leaf, err := cttypes.ParseLeafInput(entry.LeafInput())
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err))
}
switch leaf.TimestampedEntry.EntryType {
case ct.X509LogEntryType:
return processX509LogEntry(ctx, config, entry, leaf.TimestampedEntry.X509Entry)
case ct.PrecertLogEntryType:
return processPrecertLogEntry(ctx, config, entry, leaf.TimestampedEntry.PrecertEntry)
case cttypes.X509EntryType:
return processX509LogEntry(ctx, config, issuerGetter, entry, leaf.TimestampedEntry.SignedEntryASN1Cert)
case cttypes.PrecertEntryType:
return processPrecertLogEntry(ctx, config, issuerGetter, entry, leaf.TimestampedEntry.SignedEntryPreCert)
default:
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType))
}
return nil
}
func processX509LogEntry(ctx context.Context, config *Config, entry *LogEntry, cert ct.ASN1Cert) error {
certInfo, err := certspotter.MakeCertInfoFromRawCert(cert)
func processX509LogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry, cert *cttypes.ASN1Cert) error {
certInfo, err := certspotter.MakeCertInfoFromRawCert(*cert)
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err))
}
chain, err := ct.UnmarshalX509ChainArray(entry.ExtraData)
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for X.509 entry: %w", err))
}
chain = append([]ct.ASN1Cert{cert}, chain...)
if precertTBS, err := certspotter.ReconstructPrecertTBS(certInfo.TBS); err == nil {
certInfo.TBS = precertTBS
} else {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error reconstructing precertificate TBSCertificate: %w", err))
}
return processCertificate(ctx, config, entry, certInfo, chain)
getChain := func(ctx context.Context) ([]cttypes.ASN1Cert, error) {
var (
chain = []cttypes.ASN1Cert{*cert}
errs = []error{}
)
if issuers, err := entry.GetChain(ctx, issuerGetter); err == nil {
chain = append(chain, issuers...)
} else {
errs = append(errs, err)
}
return chain, errors.Join(errs...)
}
return processCertificate(ctx, config, entry, certInfo, getChain)
}
func processPrecertLogEntry(ctx context.Context, config *Config, entry *LogEntry, precert ct.PreCert) error {
func processPrecertLogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry, precert *cttypes.PreCert) error {
certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate)
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err))
}
chain, err := ct.UnmarshalPrecertChainArray(entry.ExtraData)
precertBytes, err := entry.Precertificate()
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for precert entry: %w", err))
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error getting precert entry's precertificate: %w", err))
}
if _, err := certspotter.ValidatePrecert(chain[0], precert.TBSCertificate); err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err))
getChain := func(ctx context.Context) ([]cttypes.ASN1Cert, error) {
var (
chain = []cttypes.ASN1Cert{precertBytes}
errs = []error{}
)
if issuers, err := entry.GetChain(ctx, issuerGetter); err == nil {
chain = append(chain, issuers...)
} else {
errs = append(errs, err)
}
if _, err := certspotter.ValidatePrecert(precertBytes, precert.TBSCertificate); err != nil {
errs = append(errs, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err))
}
return chain, errors.Join(errs...)
}
return processCertificate(ctx, config, entry, certInfo, chain)
return processCertificate(ctx, config, entry, certInfo, getChain)
}
func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, getChain func(context.Context) ([]cttypes.ASN1Cert, error)) error {
identifiers, err := certInfo.ParseIdentifiers()
if err != nil {
return processMalformedLogEntry(ctx, config, entry, err)
@ -92,11 +107,17 @@ func processCertificate(ctx context.Context, config *Config, entry *LogEntry, ce
return nil
}
chain, chainErr := getChain(ctx)
if errors.Is(chainErr, context.Canceled) {
return chainErr
}
cert := &DiscoveredCert{
WatchItem: watchItem,
LogEntry: entry,
Info: certInfo,
Chain: chain,
ChainError: chainErr,
TBSSHA256: sha256.Sum256(certInfo.TBS.Raw),
SHA256: sha256.Sum256(chain[0]),
PubkeySHA256: sha256.Sum256(certInfo.TBS.PublicKey.FullBytes),
@ -112,7 +133,7 @@ func processCertificate(ctx context.Context, config *Config, entry *LogEntry, ce
func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error {
if err := config.State.NotifyMalformedEntry(ctx, entry, parseError); err != nil {
return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.GetMonitoringURL(), parseError, err)
}
return nil
}

22
monitor/state.go
View File

@ -11,17 +11,25 @@ package monitor
import (
"context"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/cttypes"
"software.sslmate.com/src/certspotter/loglist"
"software.sslmate.com/src/certspotter/merkletree"
"time"
)
type LogState struct {
DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
VerifiedSTH *ct.SignedTreeHead `json:"verified_sth"`
LastSuccess time.Time `json:"last_success"`
VerifiedSTH *cttypes.SignedTreeHead `json:"verified_sth"`
}
func (state *LogState) rewindDownloadPosition() {
position := state.VerifiedPosition.Clone()
state.DownloadPosition = &position
}
func (state *LogState) advanceVerifiedPosition() {
position := state.DownloadPosition.Clone()
state.VerifiedPosition = &position
}
// Methods are safe to call concurrently.
@ -43,14 +51,14 @@ type StateProvider interface {
// Store STH for retrieval by LoadSTHs. If an STH with the same
// timestamp and root hash is already stored, this STH can be ignored.
StoreSTH(context.Context, LogID, *ct.SignedTreeHead) error
StoreSTH(context.Context, LogID, *cttypes.SignedTreeHead) error
// Load all STHs for this log previously stored with StoreSTH.
// The returned slice must be sorted by tree size.
LoadSTHs(context.Context, LogID) ([]*ct.SignedTreeHead, error)
LoadSTHs(context.Context, LogID) ([]*cttypes.SignedTreeHead, error)
// Remove an STH so it is no longer returned by LoadSTHs.
RemoveSTH(context.Context, LogID, *ct.SignedTreeHead) error
RemoveSTH(context.Context, LogID, *cttypes.SignedTreeHead) error
// Called when a certificate matching the watch list is discovered.
NotifyCert(context.Context, *DiscoveredCert) error

6
monitor/statedir.go
View File

@ -16,11 +16,10 @@ import (
"io/fs"
"os"
"path/filepath"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/cttypes"
"software.sslmate.com/src/certspotter/merkletree"
"strconv"
"strings"
"time"
)
func readVersion(stateDir string) (int, error) {
@ -50,7 +49,7 @@ func writeVersion(stateDir string) error {
}
func migrateLogStateDirV1(dir string) error {
var sth ct.SignedTreeHead
var sth cttypes.SignedTreeHead
var tree merkletree.CollapsedTree
sthPath := filepath.Join(dir, "sth.json")
@ -80,7 +79,6 @@ func migrateLogStateDirV1(dir string) error {
DownloadPosition: &tree,
VerifiedPosition: &tree,
VerifiedSTH: &sth,
LastSuccess: time.Now().UTC(),
}
if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil {
return err

30
monitor/sthdir.go
View File

@ -21,19 +21,19 @@ import (
"os"
"path/filepath"
"slices"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/cttypes"
"strconv"
"strings"
)
func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) {
func loadSTHsFromDir(dirPath string) ([]*cttypes.SignedTreeHead, error) {
entries, err := os.ReadDir(dirPath)
if errors.Is(err, fs.ErrNotExist) {
return []*ct.SignedTreeHead{}, nil
return []*cttypes.SignedTreeHead{}, nil
} else if err != nil {
return nil, err
}
sths := make([]*ct.SignedTreeHead, 0, len(entries))
sths := make([]*cttypes.SignedTreeHead, 0, len(entries))
for _, entry := range entries {
filename := entry.Name()
if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") {
@ -45,23 +45,23 @@ func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) {
}
sths = append(sths, sth)
}
slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) int { return cmp.Compare(a.TreeSize, b.TreeSize) })
slices.SortFunc(sths, func(a, b *cttypes.SignedTreeHead) int { return cmp.Compare(a.TreeSize, b.TreeSize) })
return sths, nil
}
func readSTHFile(filePath string) (*ct.SignedTreeHead, error) {
func readSTHFile(filePath string) (*cttypes.SignedTreeHead, error) {
fileBytes, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
sth := new(ct.SignedTreeHead)
sth := new(cttypes.SignedTreeHead)
if err := json.Unmarshal(fileBytes, sth); err != nil {
return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
}
return sth, nil
}
func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error {
func storeSTHInDir(dirPath string, sth *cttypes.SignedTreeHead) error {
filePath := filepath.Join(dirPath, sthFilename(sth))
if fileExists(filePath) {
return nil
@ -69,7 +69,7 @@ func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error {
return writeJSONFile(filePath, sth, 0666)
}
func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error {
func removeSTHFromDir(dirPath string, sth *cttypes.SignedTreeHead) error {
filePath := filepath.Join(dirPath, sthFilename(sth))
err := os.Remove(filePath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
@ -79,15 +79,9 @@ func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error {
}
// generate a filename that uniquely identifies the STH (within the context of a particular log)
func sthFilename(sth *ct.SignedTreeHead) string {
func sthFilename(sth *cttypes.SignedTreeHead) string {
hasher := sha256.New()
switch sth.Version {
case ct.V1:
binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash)
default:
panic(fmt.Errorf("sthFilename: invalid STH version %d", sth.Version))
}
// For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic)
binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
hasher.Write(sth.RootHash[:])
return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json"
}