diff --git a/arch/riscv/cpu/andesv5/Kconfig b/arch/riscv/cpu/andesv5/Kconfig
index 82bb5a2a53..f311291aed 100644
--- a/arch/riscv/cpu/andesv5/Kconfig
+++ b/arch/riscv/cpu/andesv5/Kconfig
@@ -4,8 +4,9 @@ config RISCV_NDS
 	imply CPU
 	imply CPU_RISCV
 	imply RISCV_TIMER if (RISCV_SMODE || SPL_RISCV_SMODE)
+	imply ANDES_PLMT_TIMER
+	imply SPL_ANDES_PLMT_TIMER
 	imply ANDES_PLICSW if (RISCV_MMODE || SPL_RISCV_MMODE)
-	imply ANDES_PLMT_TIMER if (RISCV_MMODE || SPL_RISCV_MMODE)
 	imply V5L2_CACHE
 	imply SPL_CPU
 	imply SPL_OPENSBI
diff --git a/drivers/timer/Kconfig b/drivers/timer/Kconfig
index a98be9dfae..60519c3b53 100644
--- a/drivers/timer/Kconfig
+++ b/drivers/timer/Kconfig
@@ -59,7 +59,14 @@ config ALTERA_TIMER
 
 config ANDES_PLMT_TIMER
 	bool
-	depends on RISCV_MMODE || SPL_RISCV_MMODE
+	depends on RISCV_MMODE
+	help
+	  The Andes PLMT block holds memory-mapped mtime register
+	  associated with timer tick.
+
+config SPL_ANDES_PLMT_TIMER
+	bool
+	depends on SPL_RISCV_MMODE
 	help
 	  The Andes PLMT block holds memory-mapped mtime register
 	  associated with timer tick.
diff --git a/drivers/timer/Makefile b/drivers/timer/Makefile
index 1ef814970b..b93145e8d4 100644
--- a/drivers/timer/Makefile
+++ b/drivers/timer/Makefile
@@ -4,7 +4,7 @@
 
 obj-y += timer-uclass.o
 obj-$(CONFIG_ALTERA_TIMER)	+= altera_timer.o
-obj-$(CONFIG_ANDES_PLMT_TIMER) += andes_plmt_timer.o
+obj-$(CONFIG_$(SPL_)ANDES_PLMT_TIMER) += andes_plmt_timer.o
 obj-$(CONFIG_ARC_TIMER)	+= arc_timer.o
 obj-$(CONFIG_ARM_TWD_TIMER)	+= arm_twd_timer.o
 obj-$(CONFIG_AST_TIMER)	+= ast_timer.o