diff --git a/drivers/crypto/fsl/fsl_rsa.c b/drivers/crypto/fsl/fsl_rsa.c
index c5c76e319f..0cb3c6b5f2 100644
--- a/drivers/crypto/fsl/fsl_rsa.c
+++ b/drivers/crypto/fsl/fsl_rsa.c
@@ -52,7 +52,6 @@ U_BOOT_DRIVER(fsl_rsa_mod_exp) = {
 	.name	= "fsl_rsa_mod_exp",
 	.id	= UCLASS_MOD_EXP,
 	.ops	= &fsl_mod_exp_ops,
-	.flags  = DM_FLAG_PRE_RELOC,
 };
 
 U_BOOT_DEVICE(fsl_rsa) = {
diff --git a/drivers/crypto/rsa_mod_exp/mod_exp_sw.c b/drivers/crypto/rsa_mod_exp/mod_exp_sw.c
index 46b9f1825c..c9b571a461 100644
--- a/drivers/crypto/rsa_mod_exp/mod_exp_sw.c
+++ b/drivers/crypto/rsa_mod_exp/mod_exp_sw.c
@@ -31,7 +31,6 @@ U_BOOT_DRIVER(mod_exp_sw) = {
 	.name	= "mod_exp_sw",
 	.id	= UCLASS_MOD_EXP,
 	.ops	= &mod_exp_ops_sw,
-	.flags	= DM_FLAG_PRE_RELOC,
 };
 
 U_BOOT_DEVICE(mod_exp_sw) = {