diff --git a/arch/arm/mach-uniphier/cache_uniphier.c b/arch/arm/mach-uniphier/cache_uniphier.c
index 4bf01bce3e..c1e9164489 100644
--- a/arch/arm/mach-uniphier/cache_uniphier.c
+++ b/arch/arm/mach-uniphier/cache_uniphier.c
@@ -122,23 +122,6 @@ void v7_outer_cache_disable(void)
 
 void enable_caches(void)
 {
-	uint32_t reg;
-
-	/*
-	 * UniPhier SoCs must use L2 cache for init stack pointer.
-	 * We disable L2 and L1 in this order.
-	 * If CONFIG_SYS_DCACHE_OFF is not defined,
-	 * caches are enabled again with a new page table.
-	 */
-
-	/* L2 disable */
-	v7_outer_cache_disable();
-
-	/* L1 disable */
-	reg = get_cr();
-	reg &= ~(CR_C | CR_M);
-	set_cr(reg);
-
 #ifndef CONFIG_SYS_DCACHE_OFF
 	dcache_enable();
 #endif
diff --git a/arch/arm/mach-uniphier/late_lowlevel_init.S b/arch/arm/mach-uniphier/late_lowlevel_init.S
index 22be2a21da..1363364c80 100644
--- a/arch/arm/mach-uniphier/late_lowlevel_init.S
+++ b/arch/arm/mach-uniphier/late_lowlevel_init.S
@@ -6,7 +6,12 @@
  */
 
 #include <linux/linkage.h>
+#include <mach/ssc-regs.h>
 
 ENTRY(lowlevel_init)
+	ldr	r1, = SSCC
+	ldr	r0, [r1]
+	bic	r0, r0, #SSCC_ON	@ L2 disable
+	str	r0, [r1]
 	mov	pc, lr
 ENDPROC(lowlevel_init)