diff --git a/test/fake_rsaprov.c b/test/fake_rsaprov.c
index 501da0b53be39b..33a98c10e94cf0 100644
--- a/test/fake_rsaprov.c
+++ b/test/fake_rsaprov.c
@@ -30,12 +30,18 @@ static int has_selection;
 static int imptypes_selection;
 static int exptypes_selection;
 static int query_id;
+static int key_deleted;
 
 struct fake_rsa_keydata {
     int selection;
     int status;
 };
 
+void fake_rsa_restore_store_state(void)
+{
+    key_deleted = 0;
+}
+
 static void *fake_rsa_keymgmt_new(void *provctx)
 {
     struct fake_rsa_keydata *key;
@@ -524,6 +530,7 @@ static OSSL_FUNC_store_set_ctx_params_fn fake_rsa_st_set_ctx_params;
 static OSSL_FUNC_store_load_fn fake_rsa_st_load;
 static OSSL_FUNC_store_eof_fn fake_rsa_st_eof;
 static OSSL_FUNC_store_close_fn fake_rsa_st_close;
+static OSSL_FUNC_store_delete_fn fake_rsa_st_delete;
 
 static const char fake_rsa_scheme[] = "fake_rsa:";
 
@@ -570,6 +577,11 @@ static int fake_rsa_st_load(void *loaderctx,
 
     switch (*storectx) {
     case 0:
+        if (key_deleted == 1) {
+            *storectx = 1;
+            break;
+	}
+
         /* Construct a new key using our keymgmt functions */
         if (!TEST_ptr(key = fake_rsa_keymgmt_new(NULL)))
             break;
@@ -600,13 +612,21 @@ static int fake_rsa_st_load(void *loaderctx,
 
     TEST_info("fake_rsa_load called - rv: %d", rv);
 
-    if (rv == 0) {
+    if (rv == 0 && key_deleted == 0) {
         fake_rsa_keymgmt_free(key);
         *storectx = 2;
     }
     return rv;
 }
 
+static int fake_rsa_st_delete(void *loaderctx, const char *uri,
+                              const UI_METHOD *ui_method, void *ui_data,
+                              const OSSL_PARAM params[])
+{
+    key_deleted = 1;
+    return 1;
+}
+
 static int fake_rsa_st_eof(void *loaderctx)
 {
     unsigned char *storectx = loaderctx;
@@ -629,6 +649,7 @@ static const OSSL_DISPATCH fake_rsa_store_funcs[] = {
     { OSSL_FUNC_STORE_LOAD, (void (*)(void))fake_rsa_st_load },
     { OSSL_FUNC_STORE_EOF, (void (*)(void))fake_rsa_st_eof },
     { OSSL_FUNC_STORE_CLOSE, (void (*)(void))fake_rsa_st_close },
+    { OSSL_FUNC_STORE_DELETE, (void (*)(void))fake_rsa_st_delete },
     OSSL_DISPATCH_END,
 };
 
diff --git a/test/fake_rsaprov.h b/test/fake_rsaprov.h
index 190c46a285c0ae..53056fa59f6947 100644
--- a/test/fake_rsaprov.h
+++ b/test/fake_rsaprov.h
@@ -13,3 +13,4 @@
 OSSL_PROVIDER *fake_rsa_start(OSSL_LIB_CTX *libctx);
 void fake_rsa_finish(OSSL_PROVIDER *p);
 OSSL_PARAM *fake_rsa_key_params(int priv);
+void fake_rsa_restore_store_state(void);
diff --git a/test/provider_pkey_test.c b/test/provider_pkey_test.c
index 3b190baa5e6559..a0928e3b803a6b 100644
--- a/test/provider_pkey_test.c
+++ b/test/provider_pkey_test.c
@@ -18,6 +18,7 @@
 #include "fake_rsaprov.h"
 
 static OSSL_LIB_CTX *libctx = NULL;
+extern int key_deleted; /* From fake_rsaprov.c */
 
 /* Fetch SIGNATURE method using a libctx and propq */
 static int fetch_sig(OSSL_LIB_CTX *ctx, const char *alg, const char *propq,
@@ -288,6 +289,76 @@ static int test_pkey_store(int idx)
     return ret;
 }
 
+static int test_pkey_delete(void)
+{
+    OSSL_PROVIDER *deflt = NULL;
+    OSSL_PROVIDER *fake_rsa = NULL;
+    int ret = 0;
+    EVP_PKEY *pkey = NULL;
+    OSSL_STORE_LOADER *loader = NULL;
+    OSSL_STORE_CTX *ctx = NULL;
+    OSSL_STORE_INFO *info;
+    const char *propq = "?provider=fake-rsa";
+
+    /* It's important to load the default provider first for this test */
+    if (!TEST_ptr(deflt = OSSL_PROVIDER_load(libctx, "default")))
+        goto end;
+
+    if (!TEST_ptr(fake_rsa = fake_rsa_start(libctx)))
+        goto end;
+
+    if (!TEST_ptr(loader = OSSL_STORE_LOADER_fetch(libctx, "fake_rsa",
+                                                   propq)))
+        goto end;
+
+    OSSL_STORE_LOADER_free(loader);
+
+    /* First iteration: load key, check it, delete it */
+    if (!TEST_ptr(ctx = OSSL_STORE_open_ex("fake_rsa:test", libctx, propq,
+                                           NULL, NULL, NULL, NULL, NULL)))
+        goto end;
+
+    while (!OSSL_STORE_eof(ctx)
+           && (info = OSSL_STORE_load(ctx)) != NULL
+           && pkey == NULL) {
+        if (OSSL_STORE_INFO_get_type(info) == OSSL_STORE_INFO_PKEY)
+            pkey = OSSL_STORE_INFO_get1_PKEY(info);
+        OSSL_STORE_INFO_free(info);
+        info = NULL;
+    }
+
+    if (!TEST_ptr(pkey) || !TEST_int_eq(EVP_PKEY_is_a(pkey, "RSA"), 1))
+        goto end;
+    EVP_PKEY_free(pkey);
+    pkey = NULL;
+
+    if (!TEST_int_eq(OSSL_STORE_delete("fake_rsa:test", libctx, propq,
+                                       NULL, NULL, NULL), 1))
+        goto end;
+    if (!TEST_int_eq(OSSL_STORE_close(ctx), 1))
+        goto end;
+
+    /* Second iteration: load key should fail */
+    if (!TEST_ptr(ctx = OSSL_STORE_open_ex("fake_rsa:test", libctx, propq,
+                                           NULL, NULL, NULL, NULL, NULL)))
+        goto end;
+
+    while (!OSSL_STORE_eof(ctx)) {
+           info = OSSL_STORE_load(ctx);
+	   if (!TEST_ptr_null(info))
+               goto end;
+    }
+
+    ret = 1;
+
+end:
+    fake_rsa_finish(fake_rsa);
+    OSSL_PROVIDER_unload(deflt);
+    OSSL_STORE_close(ctx);
+    fake_rsa_restore_store_state();
+    return ret;
+}
+
 int setup_tests(void)
 {
     libctx = OSSL_LIB_CTX_new();
@@ -298,6 +369,7 @@ int setup_tests(void)
     ADD_TEST(test_alternative_keygen_init);
     ADD_TEST(test_pkey_eq);
     ADD_ALL_TESTS(test_pkey_store, 2);
+    ADD_TEST(test_pkey_delete);
 
     return 1;
 }