Add |EC_METHOD| method for verifying public key order.

In some cases it would be good to restrict the input range of scalars
given to |EC_METHOD::mul| to be [0, order-1]. This is a first step
towards that goal.

Change-Id: I58a25db06f6c7a68a0ac1fe79794b04f7a173b23
Reviewed-on: https://boringssl-review.googlesource.com/6562
Reviewed-by: Adam Langley <agl@google.com>

crypto/ec/ec_key.c

@@ -302,10 +302,8 @@ int EC_KEY_check_key(const EC_KEY *eckey) {
   }
 
   ctx = BN_CTX_new();
-  point = EC_POINT_new(eckey->group);
 
-  if (ctx == NULL ||
-      point == NULL) {
+  if (ctx == NULL) {
     goto err;
   }
 
@@ -314,15 +312,11 @@ int EC_KEY_check_key(const EC_KEY *eckey) {
     OPENSSL_PUT_ERROR(EC, EC_R_POINT_IS_NOT_ON_CURVE);
     goto err;
   }
-  /* testing whether pub_key * order is the point at infinity */
   /* TODO(fork): can this be skipped if the cofactor is one or if we're about
    * to check the private key, below? */
-  const BIGNUM *order = EC_GROUP_get0_order(eckey->group);
-  if (!EC_POINT_mul(eckey->group, point, NULL, eckey->pub_key, order, ctx)) {
-    OPENSSL_PUT_ERROR(EC, ERR_R_EC_LIB);
-    goto err;
-  }
-  if (!EC_POINT_is_at_infinity(eckey->group, point)) {
+  if (eckey->group->meth->check_pub_key_order != NULL &&
+      !eckey->group->meth->check_pub_key_order(eckey->group, eckey->pub_key,
+                                               ctx)) {
     OPENSSL_PUT_ERROR(EC, EC_R_WRONG_ORDER);
     goto err;
   }
@@ -330,11 +324,13 @@ int EC_KEY_check_key(const EC_KEY *eckey) {
    * check if generator * priv_key == pub_key
    */
   if (eckey->priv_key) {
-    if (BN_cmp(eckey->priv_key, order) >= 0) {
+    if (BN_cmp(eckey->priv_key, EC_GROUP_get0_order(eckey->group)) >= 0) {
       OPENSSL_PUT_ERROR(EC, EC_R_WRONG_ORDER);
       goto err;
     }
-    if (!EC_POINT_mul(eckey->group, point, eckey->priv_key, NULL, NULL, ctx)) {
+    point = EC_POINT_new(eckey->group);
+    if (point == NULL ||
+        !EC_POINT_mul(eckey->group, point, eckey->priv_key, NULL, NULL, ctx)) {
       OPENSSL_PUT_ERROR(EC, ERR_R_EC_LIB);
       goto err;
     }

crypto/ec/ec_montgomery.c

@@ -74,23 +74,6 @@
 #include "internal.h"
 
 
-const EC_METHOD *EC_GFp_mont_method(void) {
-  static const EC_METHOD ret = {ec_GFp_mont_group_init,
-                                ec_GFp_mont_group_finish,
-                                ec_GFp_mont_group_clear_finish,
-                                ec_GFp_mont_group_copy,
-                                ec_GFp_mont_group_set_curve,
-                                ec_GFp_simple_point_get_affine_coordinates,
-                                ec_wNAF_mul /* XXX: Not constant time. */,
-                                ec_GFp_mont_field_mul,
-                                ec_GFp_mont_field_sqr,
-                                ec_GFp_mont_field_encode,
-                                ec_GFp_mont_field_decode,
-                                ec_GFp_mont_field_set_to_one};
-
-  return &ret;
-}
-
 int ec_GFp_mont_group_init(EC_GROUP *group) {
   int ok;
 
@@ -255,3 +238,43 @@ int ec_GFp_mont_field_set_to_one(const EC_GROUP *group, BIGNUM *r,
   }
   return 1;
 }
+
+static int ec_GFp_mont_check_pub_key_order(const EC_GROUP *group,
+                                           const EC_POINT* pub_key,
+                                           BN_CTX *ctx) {
+  EC_POINT *point = EC_POINT_new(group);
+  int ret = 0;
+
+  if (point == NULL ||
+      !ec_wNAF_mul(group, point, NULL, pub_key, EC_GROUP_get0_order(group),
+                   ctx) ||
+      !EC_POINT_is_at_infinity(group, point)) {
+    goto err;
+  }
+
+  ret = 1;
+
+err:
+  EC_POINT_free(point);
+  return ret;
+}
+
+const EC_METHOD *EC_GFp_mont_method(void) {
+  static const EC_METHOD ret = {
+    ec_GFp_mont_group_init,
+    ec_GFp_mont_group_finish,
+    ec_GFp_mont_group_clear_finish,
+    ec_GFp_mont_group_copy,
+    ec_GFp_mont_group_set_curve,
+    ec_GFp_simple_point_get_affine_coordinates,
+    ec_wNAF_mul /* XXX: Not constant time. */,
+    ec_GFp_mont_check_pub_key_order,
+    ec_GFp_mont_field_mul,
+    ec_GFp_mont_field_sqr,
+    ec_GFp_mont_field_encode,
+    ec_GFp_mont_field_decode,
+    ec_GFp_mont_field_set_to_one,
+  };
+
+  return &ret;
+}

crypto/ec/internal.h

@@ -103,6 +103,15 @@ struct ec_method_st {
   int (*mul)(const EC_GROUP *group, EC_POINT *r, const BIGNUM *g_scalar,
              const EC_POINT *p, const BIGNUM *p_scalar, BN_CTX *ctx);
 
+  /* |check_pub_key_order| checks that the public key is in the proper subgroup
+   * by checking that |pub_key*group->order| is the point at infinity. This may
+   * be NULL for |EC_METHOD|s specialized for prime-order curves (i.e. with
+   * cofactor one), as this check is not necessary for such curves (See section
+   * A.3 of the NSA's "Suite B Implementer's Guide to FIPS 186-3
+   * (ECDSA)"). */
+  int (*check_pub_key_order)(const EC_GROUP *group, const EC_POINT *pub_key,
+                             BN_CTX *ctx);
+
   /* internal functions */
 
   /* 'field_mul' and 'field_sqr' can be used by 'add' and 'dbl' so that the

crypto/ec/p224-64.c

@@ -1292,6 +1292,7 @@ const EC_METHOD *EC_GFp_nistp224_method(void) {
                                 ec_GFp_nistp224_group_set_curve,
                                 ec_GFp_nistp224_point_get_affine_coordinates,
                                 ec_GFp_nistp224_points_mul,
+                                0 /* check_pub_key_order */,
                                 ec_GFp_simple_field_mul,
                                 ec_GFp_simple_field_sqr,
                                 0 /* field_encode */,

crypto/ec/p256-64.c

@@ -1867,6 +1867,7 @@ const EC_METHOD *EC_GFp_nistp256_method(void) {
       ec_GFp_simple_group_copy, ec_GFp_nistp256_group_set_curve,
       ec_GFp_nistp256_point_get_affine_coordinates,
       ec_GFp_nistp256_points_mul,
+      0 /* check_pub_key_order */,
       ec_GFp_simple_field_mul, ec_GFp_simple_field_sqr,
       0 /* field_encode */, 0 /* field_decode */, 0 /* field_set_to_one */
   };

crypto/ec/p256-x86_64.c

@@ -581,6 +581,7 @@ const EC_METHOD *EC_GFp_nistz256_method(void) {
       ec_GFp_mont_group_set_curve,
       ecp_nistz256_get_affine,
       ecp_nistz256_points_mul,
+      0 /* check_pub_key_order */,
       ec_GFp_mont_field_mul,
       ec_GFp_mont_field_sqr,
       ec_GFp_mont_field_encode,