crypto/x509: read Darwin trust settings for root CAs

Darwin separately stores bits indicating whether a root certificate
should be trusted; this changes Go to read and use those when
initializing SystemCertPool.

Unfortunately, the trust API is very slow. To avoid a delay of up to
0.5s in initializing the system cert pool, we assume that
the trust settings found in kSecTrustSettingsDomainSystem will always
indicate trust. (That is, all root certs Apple distributes are trusted.)
This is not guaranteed by the API but is true in practice.

In the non-cgo codepath, we do not have that benefit, so we must check
the trust status of every certificate. This causes about 0.5s of delay
in initializing the SystemCertPool.

On OS X 10.11 and older, the "security" command requires a certificate
to be provided in a file and not on stdin, so the non-cgo codepath
creates temporary files for each certificate, further slowing initialization.

Updates #18141.

Change-Id: If681c514047afe5e1a68de6c9d40ceabbce54755
Reviewed-on: https://go-review.googlesource.com/33721
Run-TryBot: Quentin Smith <quentin@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>

src/crypto/x509/root_cgo_darwin.go

@@ -73,10 +73,11 @@ int useOldCode() {
 //
 // On success it returns 0 and fills pemRoots with a CFDataRef that contains the extracted root
 // certificates of the system. On failure, the function returns -1.
+// Additionally, it fills untrustedPemRoots with certs that must be removed from pemRoots.
 //
-// Note: The CFDataRef returned in pemRoots must be released (using CFRelease) after
-// we've consumed its content.
-int FetchPEMRoots(CFDataRef *pemRoots) {
+// Note: The CFDataRef returned in pemRoots and untrustedPemRoots must
+// be released (using CFRelease) after we've consumed its content.
+int FetchPEMRoots(CFDataRef *pemRoots, CFDataRef *untrustedPemRoots) {
 	if (useOldCode()) {
 		return FetchPEMRoots_MountainLion(pemRoots);
 	}
@@ -93,23 +94,69 @@ int FetchPEMRoots(CFDataRef *pemRoots) {
 		return -1;
 	}
 
+	// kSecTrustSettingsResult is defined as CFSTR("kSecTrustSettingsResult"),
+	// but the Go linker's internal linking mode can't handle CFSTR relocations.
+	// Create our own dynamic string instead and release it below.
+	CFStringRef policy = CFStringCreateWithCString(NULL, "kSecTrustSettingsResult", kCFStringEncodingUTF8);
+
 	CFMutableDataRef combinedData = CFDataCreateMutable(kCFAllocatorDefault, 0);
+	CFMutableDataRef combinedUntrustedData = CFDataCreateMutable(kCFAllocatorDefault, 0);
 	for (int i = 0; i < numDomains; i++) {
 		CFArrayRef certs = NULL;
-		// Only get certificates from domain that are trusted
 		OSStatus err = SecTrustSettingsCopyCertificates(domains[i], &certs);
 		if (err != noErr) {
 			continue;
 		}
 
-		int numCerts = CFArrayGetCount(certs);
+		CFIndex numCerts = CFArrayGetCount(certs);
 		for (int j = 0; j < numCerts; j++) {
 			CFDataRef data = NULL;
 			CFErrorRef errRef = NULL;
+			CFArrayRef trustSettings = NULL;
 			SecCertificateRef cert = (SecCertificateRef)CFArrayGetValueAtIndex(certs, j);
 			if (cert == NULL) {
 				continue;
 			}
+			// We only want trusted certs.
+			int untrusted = 0;
+			if (i != 0) {
+				// Certs found in the system domain are always trusted. If the user
+				// configures "Never Trust" on such a cert, it will also be found in the
+				// admin or user domain, causing it to be added to untrustedPemRoots. The
+				// Go code will then clean this up.
+
+				// Trust may be stored in any of the domains. According to Apple's
+				// SecTrustServer.c, "user trust settings overrule admin trust settings",
+				// so take the last trust settings array we find.
+				// Skip the system domain since it is always trusted.
+				for (int k = 1; k < numDomains; k++) {
+					CFArrayRef domainTrustSettings = NULL;
+					err = SecTrustSettingsCopyTrustSettings(cert, domains[k], &domainTrustSettings);
+					if (err == errSecSuccess && domainTrustSettings != NULL) {
+						if (trustSettings) {
+							CFRelease(trustSettings);
+						}
+						trustSettings = domainTrustSettings;
+					}
+				}
+				if (trustSettings == NULL) {
+					// "this certificate must be verified to a known trusted certificate"; aka not a root.
+					continue;
+				}
+				for (CFIndex k = 0; k < CFArrayGetCount(trustSettings); k++) {
+					CFNumberRef cfNum;
+					CFDictionaryRef tSetting = (CFDictionaryRef)CFArrayGetValueAtIndex(trustSettings, k);
+					if (CFDictionaryGetValueIfPresent(tSetting, policy, (const void**)&cfNum)){
+						SInt32 result = 0;
+						CFNumberGetValue(cfNum, kCFNumberSInt32Type, &result);
+						// TODO: The rest of the dictionary specifies conditions for evaluation.
+						if (result == kSecTrustSettingsResultDeny) {
+							untrusted = 1;
+						}
+					}
+				}
+				CFRelease(trustSettings);
+			}
 			// We only want to add Root CAs, so make sure Subject and Issuer Name match
 			CFDataRef subjectName = SecCertificateCopyNormalizedSubjectContent(cert, &errRef);
 			if (errRef != NULL) {
@@ -138,13 +185,16 @@ int FetchPEMRoots(CFDataRef *pemRoots) {
 			}
 
 			if (data != NULL) {
-				CFDataAppendBytes(combinedData, CFDataGetBytePtr(data), CFDataGetLength(data));
+				CFMutableDataRef appendTo = untrusted ? combinedUntrustedData : combinedData;
+				CFDataAppendBytes(appendTo, CFDataGetBytePtr(data), CFDataGetLength(data));
 				CFRelease(data);
 			}
 		}
 		CFRelease(certs);
 	}
+	CFRelease(policy);
 	*pemRoots = combinedData;
+	*untrustedPemRoots = combinedUntrustedData;
 	return 0;
 }
 */
@@ -158,7 +208,8 @@ func loadSystemRoots() (*CertPool, error) {
 	roots := NewCertPool()
 
 	var data C.CFDataRef = nil
-	err := C.FetchPEMRoots(&data)
+	var untrustedData C.CFDataRef = nil
+	err := C.FetchPEMRoots(&data, &untrustedData)
 	if err == -1 {
 		// TODO: better error message
 		return nil, errors.New("crypto/x509: failed to load darwin system roots with cgo")
@@ -167,5 +218,19 @@ func loadSystemRoots() (*CertPool, error) {
 	defer C.CFRelease(C.CFTypeRef(data))
 	buf := C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(data)), C.int(C.CFDataGetLength(data)))
 	roots.AppendCertsFromPEM(buf)
-	return roots, nil
+	if untrustedData == nil {
+		return roots, nil
+	}
+	defer C.CFRelease(C.CFTypeRef(untrustedData))
+	buf = C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(untrustedData)), C.int(C.CFDataGetLength(untrustedData)))
+	untrustedRoots := NewCertPool()
+	untrustedRoots.AppendCertsFromPEM(buf)
+
+	trustedRoots := NewCertPool()
+	for _, c := range roots.certs {
+		if !untrustedRoots.contains(c) {
+			trustedRoots.AddCert(c)
+		}
+	}
+	return trustedRoots, nil
 }

src/crypto/x509/root_darwin.go

@@ -6,12 +6,27 @@
 
 package x509
 
-import "os/exec"
+import (
+	"bytes"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"sync"
+	"syscall"
+)
 
 func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) {
 	return nil, nil
 }
 
+// This code is only used when compiling without cgo.
+// It is here, instead of root_nocgo_darwin.go, so that tests can check it
+// even if the tests are run with cgo enabled.
+// The linker will not include these unused functions in binaries built with cgo enabled.
+
 func execSecurityRoots() (*CertPool, error) {
 	cmd := exec.Command("/usr/bin/security", "find-certificate", "-a", "-p", "/System/Library/Keychains/SystemRootCertificates.keychain")
 	data, err := cmd.Output()
@@ -19,7 +34,100 @@ func execSecurityRoots() (*CertPool, error) {
 		return nil, err
 	}
 
-	roots := NewCertPool()
-	roots.AppendCertsFromPEM(data)
+	var (
+		mu    sync.Mutex
+		roots = NewCertPool()
+	)
+	add := func(cert *Certificate) {
+		mu.Lock()
+		defer mu.Unlock()
+		roots.AddCert(cert)
+	}
+	blockCh := make(chan *pem.Block)
+	var wg sync.WaitGroup
+	for i := 0; i < 4; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for block := range blockCh {
+				verifyCertWithSystem(block, add)
+			}
+		}()
+	}
+	for len(data) > 0 {
+		var block *pem.Block
+		block, data = pem.Decode(data)
+		if block == nil {
+			break
+		}
+		if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
+			continue
+		}
+		blockCh <- block
+	}
+	close(blockCh)
+	wg.Wait()
 	return roots, nil
 }
+
+func verifyCertWithSystem(block *pem.Block, add func(*Certificate)) {
+	data := pem.EncodeToMemory(block)
+	var cmd *exec.Cmd
+	if needsTmpFiles() {
+		f, err := ioutil.TempFile("", "cert")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "can't create temporary file for cert: %v", err)
+			return
+		}
+		defer os.Remove(f.Name())
+		if _, err := f.Write(data); err != nil {
+			fmt.Fprintf(os.Stderr, "can't write temporary file for cert: %v", err)
+			return
+		}
+		if err := f.Close(); err != nil {
+			fmt.Fprintf(os.Stderr, "can't write temporary file for cert: %v", err)
+			return
+		}
+		cmd = exec.Command("/usr/bin/security", "verify-cert", "-c", f.Name(), "-l")
+	} else {
+		cmd = exec.Command("/usr/bin/security", "verify-cert", "-c", "/dev/stdin", "-l")
+		cmd.Stdin = bytes.NewReader(data)
+	}
+	if cmd.Run() == nil {
+		// Non-zero exit means untrusted
+		cert, err := ParseCertificate(block.Bytes)
+		if err != nil {
+			return
+		}
+
+		add(cert)
+	}
+}
+
+var versionCache struct {
+	sync.Once
+	major int
+}
+
+// needsTmpFiles reports whether the OS is <= 10.11 (which requires real
+// files as arguments to the security command).
+func needsTmpFiles() bool {
+	versionCache.Do(func() {
+		release, err := syscall.Sysctl("kern.osrelease")
+		if err != nil {
+			return
+		}
+		for i, c := range release {
+			if c == '.' {
+				release = release[:i]
+				break
+			}
+		}
+		major, err := strconv.Atoi(release)
+		if err != nil {
+			return
+		}
+		versionCache.major = major
+	})
+	return versionCache.major <= 15
+}

src/crypto/x509/root_darwin_test.go

@@ -29,6 +29,7 @@ func TestSystemRoots(t *testing.T) {
 		// On Mavericks, there are 212 bundled certs; require only
 		// 150 here, since this is just a sanity check, and the
 		// exact number will vary over time.
+		t.Logf("got %d roots", len(tt.certs))
 		if want, have := 150, len(tt.certs); have < want {
 			t.Fatalf("want at least %d system roots, have %d", want, have)
 		}