diff --git a/drivers/net/zynq_gem.c b/drivers/net/zynq_gem.c
index 412daf7d58..da4b6fba9f 100644
--- a/drivers/net/zynq_gem.c
+++ b/drivers/net/zynq_gem.c
@@ -451,8 +451,12 @@ static int zynq_gem_init(struct udevice *dev)
 		nwconfig |= ZYNQ_GEM_NWCFG_SGMII_ENBL |
 			    ZYNQ_GEM_NWCFG_PCS_SEL;
 #ifdef CONFIG_ARM64
+	if (priv->phydev->phy_id != PHY_FIXED_ID)
 		writel(readl(&regs->pcscntrl) | ZYNQ_GEM_PCS_CTL_ANEG_ENBL,
 		       &regs->pcscntrl);
+	else
+		writel(readl(&regs->pcscntrl) & ~ZYNQ_GEM_PCS_CTL_ANEG_ENBL,
+		       &regs->pcscntrl);
 #endif
 	}