crypto/x509: new home for root fetchers; build chains using Windows API

This moves the various CA root fetchers from crypto/tls into crypto/x509.

The move was brought about by issue 2997. Windows doesn't ship with all
its root certificates, but will instead download them as-needed when using
CryptoAPI for certificate verification.

This CL changes crypto/x509 to verify a certificate using the system root
CAs when VerifyOptions.RootCAs == nil. On Windows, this verification is
now implemented using Windows's CryptoAPI. All other root fetchers are
unchanged, and still use Go's own verification code.

The CL also fixes the hostname matching logic in crypto/tls/tls.go, in
order to be able to test whether hostname mismatches are honored by the
Windows verification code.

The move to crypto/x509 also allows other packages to use the OS-provided
root certificates, instead of hiding them inside the crypto/tls package.

Fixes #2997.

R=agl, golang-dev, alex.brainman, rsc, mikkel
CC=golang-dev
https://golang.org/cl/5700087

common.go

@@ -198,14 +198,6 @@ func (c *Config) time() time.Time {
 	return t()
 }
 
-func (c *Config) rootCAs() *x509.CertPool {
-	s := c.RootCAs
-	if s == nil {
-		s = defaultRoots()
-	}
-	return s
-}
-
 func (c *Config) cipherSuites() []uint16 {
 	s := c.CipherSuites
 	if s == nil {
@@ -311,28 +303,16 @@ func defaultConfig() *Config {
 	return &emptyConfig
 }
 
-var once sync.Once
-
-func defaultRoots() *x509.CertPool {
-	once.Do(initDefaults)
-	return varDefaultRoots
-}
-
-func defaultCipherSuites() []uint16 {
-	once.Do(initDefaults)
-	return varDefaultCipherSuites
-}
-
-func initDefaults() {
-	initDefaultRoots()
-	initDefaultCipherSuites()
-}
-
 var (
-	varDefaultRoots        *x509.CertPool
+	once                   sync.Once
 	varDefaultCipherSuites []uint16
 )
 
+func defaultCipherSuites() []uint16 {
+	once.Do(initDefaultCipherSuites)
+	return varDefaultCipherSuites
+}
+
 func initDefaultCipherSuites() {
 	varDefaultCipherSuites = make([]uint16, len(cipherSuites))
 	for i, suite := range cipherSuites {

handshake_client.go

@@ -102,7 +102,7 @@ func (c *Conn) clientHandshake() error {
 
 	if !c.config.InsecureSkipVerify {
 		opts := x509.VerifyOptions{
-			Roots:         c.config.rootCAs(),
+			Roots:         c.config.RootCAs,
 			CurrentTime:   c.config.time(),
 			DNSName:       c.config.ServerName,
 			Intermediates: x509.NewCertPool(),

root_darwin.go

@@ -1,79 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package tls
-
-/*
-#cgo CFLAGS: -mmacosx-version-min=10.6 -D__MAC_OS_X_VERSION_MAX_ALLOWED=1060
-#cgo LDFLAGS: -framework CoreFoundation -framework Security
-
-#include <CoreFoundation/CoreFoundation.h>
-#include <Security/Security.h>
-
-// FetchPEMRoots fetches the system's list of trusted X.509 root certificates.
-//
-// On success it returns 0 and fills pemRoots with a CFDataRef that contains the extracted root
-// certificates of the system. On failure, the function returns -1.
-//
-// Note: The CFDataRef returned in pemRoots must be released (using CFRelease) after
-// we've consumed its content.
-int FetchPEMRoots(CFDataRef *pemRoots) {
-	if (pemRoots == NULL) {
-		return -1;
-	}
-
-	CFArrayRef certs = NULL;
-	OSStatus err = SecTrustCopyAnchorCertificates(&certs);
-	if (err != noErr) {
-		return -1;
-	}
-
-	CFMutableDataRef combinedData = CFDataCreateMutable(kCFAllocatorDefault, 0);
-	int i, ncerts = CFArrayGetCount(certs);
-	for (i = 0; i < ncerts; i++) {
-		CFDataRef data = NULL;
-		SecCertificateRef cert = (SecCertificateRef)CFArrayGetValueAtIndex(certs, i);
-		if (cert == NULL) {
-			continue;
-		}
-
-		// Note: SecKeychainItemExport is deprecated as of 10.7 in favor of SecItemExport.
-		// Once we support weak imports via cgo we should prefer that, and fall back to this
-		// for older systems.
-		err = SecKeychainItemExport(cert, kSecFormatX509Cert, kSecItemPemArmour, NULL, &data);
-		if (err != noErr) {
-			continue;
-		}
-
-		if (data != NULL) {
-			CFDataAppendBytes(combinedData, CFDataGetBytePtr(data), CFDataGetLength(data));
-			CFRelease(data);
-		}
-	}
-
-	CFRelease(certs);
-
-	*pemRoots = combinedData;
-	return 0;
-}
-*/
-import "C"
-import (
-	"crypto/x509"
-	"unsafe"
-)
-
-func initDefaultRoots() {
-	roots := x509.NewCertPool()
-
-	var data C.CFDataRef = nil
-	err := C.FetchPEMRoots(&data)
-	if err != -1 {
-		defer C.CFRelease(C.CFTypeRef(data))
-		buf := C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(data)), C.int(C.CFDataGetLength(data)))
-		roots.AppendCertsFromPEM(buf)
-	}
-
-	varDefaultRoots = roots
-}

root_stub.go

@@ -1,10 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build plan9 darwin,!cgo
-
-package tls
-
-func initDefaultRoots() {
-}

root_test.go

@@ -5,25 +5,25 @@
 package tls
 
 import (
+	"crypto/x509"
+	"runtime"
 	"testing"
 )
 
 var tlsServers = []string{
-	"google.com:443",
+	"google.com",
-	"github.com:443",
+	"github.com",
-	"twitter.com:443",
+	"twitter.com",
 }
 
 func TestOSCertBundles(t *testing.T) {
-	defaultRoots()
-
 	if testing.Short() {
 		t.Logf("skipping certificate tests in short mode")
 		return
 	}
 
 	for _, addr := range tlsServers {
-		conn, err := Dial("tcp", addr, nil)
+		conn, err := Dial("tcp", addr+":443", &Config{ServerName: addr})
 		if err != nil {
 			t.Errorf("unable to verify %v: %v", addr, err)
 			continue
@@ -34,3 +34,28 @@ func TestOSCertBundles(t *testing.T) {
 		}
 	}
 }
+
+func TestCertHostnameVerifyWindows(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		return
+	}
+
+	if testing.Short() {
+		t.Logf("skipping certificate tests in short mode")
+		return
+	}
+
+	for _, addr := range tlsServers {
+		cfg := &Config{ServerName: "example.com"}
+		conn, err := Dial("tcp", addr+":443", cfg)
+		if err == nil {
+			conn.Close()
+			t.Errorf("should fail to verify for example.com: %v", addr, err)
+			continue
+		}
+		_, ok := err.(x509.HostnameError)
+		if !ok {
+			t.Errorf("error type mismatch, got: %v", err)
+		}
+	}
+}

root_unix.go

@@ -1,33 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build freebsd linux openbsd netbsd
-
-package tls
-
-import (
-	"crypto/x509"
-	"io/ioutil"
-)
-
-// Possible certificate files; stop after finding one.
-var certFiles = []string{
-	"/etc/ssl/certs/ca-certificates.crt",     // Linux etc
-	"/etc/pki/tls/certs/ca-bundle.crt",       // Fedora/RHEL
-	"/etc/ssl/ca-bundle.pem",                 // OpenSUSE
-	"/etc/ssl/cert.pem",                      // OpenBSD
-	"/usr/local/share/certs/ca-root-nss.crt", // FreeBSD
-}
-
-func initDefaultRoots() {
-	roots := x509.NewCertPool()
-	for _, file := range certFiles {
-		data, err := ioutil.ReadFile(file)
-		if err == nil {
-			roots.AppendCertsFromPEM(data)
-			break
-		}
-	}
-	varDefaultRoots = roots
-}

root_windows.go

@@ -1,47 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package tls
-
-import (
-	"crypto/x509"
-	"syscall"
-	"unsafe"
-)
-
-func loadStore(roots *x509.CertPool, name string) {
-	store, err := syscall.CertOpenSystemStore(syscall.InvalidHandle, syscall.StringToUTF16Ptr(name))
-	if err != nil {
-		return
-	}
-	defer syscall.CertCloseStore(store, 0)
-
-	var cert *syscall.CertContext
-	for {
-		cert, err = syscall.CertEnumCertificatesInStore(store, cert)
-		if err != nil {
-			return
-		}
-
-		buf := (*[1 << 20]byte)(unsafe.Pointer(cert.EncodedCert))[:]
-		// ParseCertificate requires its own copy of certificate data to keep.
-		buf2 := make([]byte, cert.Length)
-		copy(buf2, buf)
-		if c, err := x509.ParseCertificate(buf2); err == nil {
-			roots.AddCert(c)
-		}
-	}
-}
-
-func initDefaultRoots() {
-	roots := x509.NewCertPool()
-
-	// Roots
-	loadStore(roots, "ROOT")
-
-	// Intermediates
-	loadStore(roots, "CA")
-
-	varDefaultRoots = roots
-}

tls.go

@@ -97,7 +97,9 @@ func Dial(network, addr string, config *Config) (*Conn, error) {
 	if config == nil {
 		config = defaultConfig()
 	}
-	if config.ServerName != "" {
+	// If no ServerName is set, infer the ServerName
+	// from the hostname we're connecting to.
+	if config.ServerName == "" {
 		// Make a copy to avoid polluting argument or default.
 		c := *config
 		c.ServerName = hostname