Consolidate ML-KEM and ML-DSA codecs

These previously duplicated some code and structures, now shared.

Reviewed-by: Tim Hudson <[email protected]>
Reviewed-by: Tomas Mraz <[email protected]>
Reviewed-by: Shane Lontis <[email protected]>
(Merged from openssl#26764)

providers/implementations/encode_decode/build.info

@@ -26,3 +26,7 @@ ENDIF
 IF[{- !$disabled{'ml-kem'} -}]
   SOURCE[$DECODER_GOAL]=ml_kem_codecs.c
 ENDIF
+
+IF[{- !$disabled{'ml-dsa'} || !$disabled{'ml-kem'} -}]
+  SOURCE[$DECODER_GOAL]=ml_common_codecs.c
+ENDIF

providers/implementations/encode_decode/ml_common_codecs.c

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <string.h>
+#include <openssl/crypto.h>
+#include <openssl/err.h>
+#include <openssl/proverr.h>
+#include "ml_common_codecs.h"
+
+static int pref_cmp(const void *va, const void *vb)
+{
+    const ML_COMMON_PKCS8_FMT_PREF *a = va;
+    const ML_COMMON_PKCS8_FMT_PREF *b = vb;
+
+    /*
+     * Zeros sort last, otherwise the sort is in increasing order.
+     *
+     * The preferences are small enough to ensure the comparison is transitive
+     * as required by qsort(3).  When overflow or underflow is possible, the
+     * correct transitive comparison would be: (b < a) - (a < b).
+     */
+    if (a->pref > 0 && b->pref > 0)
+        return a->pref - b->pref;
+    /* A preference of 0 is "larger" than (sorts after) any nonzero value. */
+    return b->pref - a->pref;
+}
+
+ML_COMMON_PKCS8_FMT_PREF *
+ossl_ml_common_pkcs8_fmt_order(const char *algorithm_name,
+                               const ML_COMMON_PKCS8_FMT *p8fmt,
+                               const char *direction, const char *formats)
+{
+    ML_COMMON_PKCS8_FMT_PREF *ret;
+    int i, count = 0;
+    const char *fmt = formats, *end;
+    const char *sep = "\t ,";
+
+    /* Reserve an extra terminal slot with fmt == NULL */
+    if ((ret = OPENSSL_zalloc((NUM_PKCS8_FORMATS + 1) * sizeof(*ret))) == NULL)
+        return NULL;
+
+    /* Entries that match a format will get a non-zero preference. */
+    for (i = 0; i < NUM_PKCS8_FORMATS; ++i) {
+        ret[i].fmt = &p8fmt[i];
+        ret[i].pref = 0;
+    }
+
+    /* Default to compile-time table order when none specified. */
+    if (formats == NULL)
+        return ret;
+
+    /*
+     * Formats are case-insensitive, separated by spaces, tabs or commas.
+     * Duplicate formats are allowed, the first occurence determines the order.
+     */
+    do {
+        if (*(fmt += strspn(fmt, sep)) == '\0')
+            break;
+        end = fmt + strcspn(fmt, sep);
+        for (i = 0; i < NUM_PKCS8_FORMATS; ++i) {
+            /* Skip slots already selected or with a different name. */
+            if (ret[i].pref > 0
+                || OPENSSL_strncasecmp(ret[i].fmt->p8_name,
+                                       fmt, (end - fmt)) != 0)
+                continue;
+            /* First time match */
+            ret[i].pref = ++count;
+            break;
+        }
+        fmt = end;
+    } while (count < NUM_PKCS8_FORMATS);
+
+    /* No formats matched, raise an error */
+    if (count == 0) {
+        OPENSSL_free(ret);
+        ERR_raise_data(ERR_LIB_PROV, PROV_R_ML_DSA_NO_FORMAT,
+                       "no %s private key %s formats are enabled",
+                       algorithm_name, direction);
+        return NULL;
+    }
+    /* Sort by preference, with 0's last */
+    qsort(ret, NUM_PKCS8_FORMATS, sizeof(*ret), pref_cmp);
+    /* Terminate the list at first unselected entry, perhaps reserved slot. */
+    ret[count].fmt = NULL;
+    return ret;
+}

providers/implementations/encode_decode/ml_common_codecs.h

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#ifndef PROV_ML_COMMON_CODECS_H
+# define PROV_ML_COMMON_CODECS_H
+# pragma once
+
+#  include <openssl/e_os2.h>
+#  include "crypto/ml_dsa.h"
+#  include "prov/provider_ctx.h"
+
+ /*-
+  * The DER ASN.1 encoding of ML-DSA and ML-KEM public keys prepends 22 bytes
+  * to the encoded public key:
+  *
+  * - 4 byte outer sequence tag and length
+  * -  2 byte algorithm sequence tag and length
+  * -    2 byte algorithm OID tag and length
+  * -      9 byte algorithm OID (from NIST CSOR OID arc)
+  * -  4 byte bit string tag and length
+  * -    1 bitstring lead byte
+  */
+#  define ML_COMMON_SPKI_OVERHEAD   22
+typedef struct {
+    const uint8_t asn1_prefix[ML_COMMON_SPKI_OVERHEAD];
+} ML_COMMON_SPKI_FMT;
+
+/*-
+* For each parameter set we support a few PKCS#8 input formats, three
+ * corresponding to the "either or both" variants of:
+ *
+ *  ML-DSA-PrivateKey ::= CHOICE {
+ *    seed [0] IMPLICIT OCTET STRING (SIZE (32)),
+ *    expandedKey OCTET STRING (SIZE (2560 | 4032 | 4896)),
+ *    both SEQUENCE {
+ *      seed OCTET STRING (SIZE (32)),
+ *      expandedKey OCTET STRING (SIZE (2560 | 4032 | 4896)) } }
+ *
+ *  ML-KEM-PrivateKey ::= CHOICE {
+ *    seed [0] IMPLICIT OCTET STRING (SIZE (64)),
+ *    expandedKey OCTET STRING (SIZE (1632 | 2400 | 3168)),
+ *    both SEQUENCE {
+ *      seed OCTET STRING (SIZE (64)),
+ *      expandedKey OCTET STRING SIZE ((1632 | 2400 | 3168)) } }
+ *
+ * one more for a historical OQS encoding:
+ *
+ * - OQS private + public key: OCTET STRING
+ *   (The public key is ignored, just as with PKCS#8 v2.)
+ *
+ * and two more that are the minimal IETF non-ASN.1 seed encoding:
+ *
+ * - Bare seed (just the 32 or 64 bytes)
+ * - Bare priv (just the key bytes)
+ *
+ * A length of zero means that particular field is absent.
+ *
+ * The p8_shift is 0 when the top-level tag+length occupy four bytes, 2 when
+ * they occupy two by†es, and 4 when no tag is used at all.
+ */
+#define NUM_PKCS8_FORMATS   6
+
+typedef struct {
+    const char *p8_name;    /* Format name */
+    size_t p8_bytes;        /* Total P8 encoding length */
+    int    p8_shift;        /* 4 - (top-level tag + len) */
+    uint32_t p8_magic;      /* The tag + len value */
+    uint16_t seed_magic;    /* Interior tag + len for the seed */
+    size_t seed_offset;     /* Seed offset from start */
+    size_t seed_length;     /* Seed bytes */
+    uint32_t priv_magic;    /* Interior tag + len for the key */
+    size_t priv_offset;     /* Key offset from start */
+    size_t priv_length;     /* Key bytes */
+    size_t pub_offset;      /* Pubkey offset */
+    size_t pub_length;      /* Pubkey bytes */
+} ML_COMMON_PKCS8_FMT;
+
+typedef struct {
+    const ML_COMMON_SPKI_FMT *spkifmt;
+    const ML_COMMON_PKCS8_FMT *p8fmt;
+} ML_COMMON_CODEC;
+
+typedef struct {
+    const ML_COMMON_PKCS8_FMT *fmt;
+    int pref;
+} ML_COMMON_PKCS8_FMT_PREF;
+
+ML_COMMON_PKCS8_FMT_PREF *
+ossl_ml_common_pkcs8_fmt_order(const char *algorithm_name,
+                               const ML_COMMON_PKCS8_FMT *p8fmt,
+                               const char *direction, const char *formats);
+#endif