diff --git a/Makefile.arm64 b/Makefile.arm64
index 3d713b853..d7fc7a349 100644
--- a/Makefile.arm64
+++ b/Makefile.arm64
@@ -305,7 +305,7 @@ endif
 
 ifeq ($(CORE), VORTEXM4)
 CCOMMON_OPT += -march=armv8.4-a+sme
-FCOMMON_OPT += -march=armv8.4-a+sme
+FCOMMON_OPT += -march=armv8.4-a
 endif
 
 ifeq (1, $(filter 1,$(GCCVERSIONGTEQ9) $(ISCLANG)))