APPS/pkeyutl: add missing high-level check for -verifyrecover being u…

…sable only with RSA
siemens · Nov 8, 2024 · 44a712f · 44a712f
1 parent 9849d2b
commit 44a712f
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 4 deletions.
diff --git a/apps/pkeyutl.c b/apps/pkeyutl.c
@@ -43,14 +43,19 @@ static int do_raw_keyop(int pkey_op, EVP_MD_CTX *mctx,
                         int filesize, unsigned char *sig, int siglen,
                         unsigned char **out, size_t *poutlen);
 
-static int only_rawin(const EVP_PKEY *pkey)
+static int is_EdDSA(const EVP_PKEY *pkey)
 {
     if (pkey == NULL)
         return 0;
     return EVP_PKEY_is_a(pkey, "ED25519")
         || EVP_PKEY_is_a(pkey, "ED448");
 }
 
+static int only_rawin(const EVP_PKEY *pkey)
+{
+    return is_EdDSA(pkey);
+}
+
 typedef enum OPTION_choice {
     OPT_COMMON,
     OPT_ENGINE, OPT_ENGINE_IMPL, OPT_IN, OPT_OUT,
@@ -309,10 +314,15 @@ int pkeyutl_main(int argc, char **argv)
     }
 
     pkey = get_pkey(kdfalg, inkey, keyform, key_type, passinarg, pkey_op, e);
+
+    if (pkey_op == EVP_PKEY_OP_VERIFYRECOVER && !EVP_PKEY_is_a(pkey, "RSA")) {
+        BIO_printf(bio_err, "%s: -verifyrecover can be used only with RSA\n", prog);
+        goto end;
+    }
+
     if (pkey_op == EVP_PKEY_OP_SIGN || pkey_op == EVP_PKEY_OP_VERIFY) {
         if (only_rawin(pkey)) {
-            if ((EVP_PKEY_is_a(pkey, "ED25519") || EVP_PKEY_is_a(pkey, "ED448"))
-                && digestname != NULL) {
+            if (is_EdDSA(pkey) && digestname != NULL) {
                 BIO_printf(bio_err,
                            "%s: -digest (prehash) is not supported with EdDSA\n", prog);
                 EVP_PKEY_free(pkey);

diff --git a/test/recipes/20-test_pkeyutl.t b/test/recipes/20-test_pkeyutl.t
@@ -54,7 +54,7 @@ SKIP: {
 }
 
 SKIP: {
-    skip "Skipping tests that require ECX", 6
+    skip "Skipping tests that require ECX", 7
         if disabled("ecx");
 
     # Ed25519
@@ -68,6 +68,9 @@ SKIP: {
                   '-inkey', srctop_file('test', 'certs', 'server-ed25519-cert.pem'),
                   '-sigfile', 'Ed25519.sig']))),
                   "Verify an Ed25519 signature against a piece of data");
+    ok(!run(app(([ 'openssl', 'pkeyutl', '-verifyrecover', '-in', 'Ed25519.sig',
+                   '-inkey', srctop_file('test', 'certs', 'server-ed25519-key.pem')]))),
+       "Cannot use -verifyrecover with EdDSA");
 
     # Ed448
     ok(run(app(([ 'openssl', 'pkeyutl', '-sign', '-in',