diff --git a/src/libp11-int.h b/src/libp11-int.h
index c3d7d899..fd84f97b 100644
--- a/src/libp11-int.h
+++ b/src/libp11-int.h
@@ -355,9 +355,15 @@ extern int pkcs11_private_decrypt(
 /* Retrieve PKCS11_KEY from an RSA key */
 extern PKCS11_OBJECT_private *pkcs11_get_ex_data_rsa(const RSA *rsa);
 
+/* Set PKCS11_KEY for an RSA key */
+void pkcs11_set_ex_data_rsa(RSA *rsa, PKCS11_OBJECT_private *key);
+
 /* Retrieve PKCS11_KEY from an EC_KEY */
 extern PKCS11_OBJECT_private *pkcs11_get_ex_data_ec(const EC_KEY *ec);
 
+/* Set PKCS11_KEY for an EC_KEY */
+extern void pkcs11_set_ex_data_ec(EC_KEY *ec, PKCS11_OBJECT_private *key);
+
 /* Free the global RSA_METHOD */
 extern void pkcs11_rsa_method_free(void);
 
diff --git a/src/p11_ec.c b/src/p11_ec.c
index e8027f0d..88a4cbdf 100644
--- a/src/p11_ec.c
+++ b/src/p11_ec.c
@@ -351,7 +351,7 @@ PKCS11_OBJECT_private *pkcs11_get_ex_data_ec(const EC_KEY *ec)
 #endif
 }
 
-static void pkcs11_set_ex_data_ec(EC_KEY *ec, PKCS11_OBJECT_private *key)
+void pkcs11_set_ex_data_ec(EC_KEY *ec, PKCS11_OBJECT_private *key)
 {
 #if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
 	EC_KEY_set_ex_data(ec, ec_ex_index, key);
diff --git a/src/p11_key.c b/src/p11_key.c
index dbef9207..e49c1c9c 100644
--- a/src/p11_key.c
+++ b/src/p11_key.c
@@ -481,7 +481,10 @@ EVP_PKEY *pkcs11_get_key(PKCS11_OBJECT_private *key0, CK_OBJECT_CLASS object_cla
 			EVP_PKEY_free(ret);
 			goto err;
 		}
-		pkcs11_object_ref(key);
+		if (key->object_class == CKO_PRIVATE_KEY)
+			pkcs11_object_ref(key);
+		else /* Public key -> detach PKCS11_OBJECT */
+			pkcs11_set_ex_data_rsa(rsa, NULL);
 		break;
 	case EVP_PKEY_EC:
 #if OPENSSL_VERSION_NUMBER < 0x30000000L || defined(LIBRESSL_VERSION_NUMBER)
@@ -498,9 +501,17 @@ EVP_PKEY *pkcs11_get_key(PKCS11_OBJECT_private *key0, CK_OBJECT_CLASS object_cla
 			EVP_PKEY_free(ret);
 			goto err;
 		}
-		pkcs11_object_ref(key);
+		if (key->object_class == CKO_PRIVATE_KEY)
+			pkcs11_object_ref(key);
+		else /* Public key -> detach PKCS11_OBJECT */
+			pkcs11_set_ex_data_ec(ec_key, NULL);
 #else
 		ret = EVP_PKEY_dup(key->evp_key);
+		if (key->object_class != CKO_PRIVATE_KEY) {
+			/* Public key -> detach and free PKCS11_OBJECT */
+			pkcs11_set_ex_data_ec((EC_KEY *)EVP_PKEY_get0_EC_KEY(ret), NULL);
+			pkcs11_object_free(key);
+		}
 #endif
 		break;
 	default:
diff --git a/src/p11_rsa.c b/src/p11_rsa.c
index 0558e674..359e9de9 100644
--- a/src/p11_rsa.c
+++ b/src/p11_rsa.c
@@ -280,7 +280,7 @@ PKCS11_OBJECT_private *pkcs11_get_ex_data_rsa(const RSA *rsa)
 	return RSA_get_ex_data(rsa, rsa_ex_index);
 }
 
-static void pkcs11_set_ex_data_rsa(RSA *rsa, PKCS11_OBJECT_private *key)
+void pkcs11_set_ex_data_rsa(RSA *rsa, PKCS11_OBJECT_private *key)
 {
 	RSA_set_ex_data(rsa, rsa_ex_index, key);
 }