diff --git a/build_debian.sh b/build_debian.sh
index 9462e4a54f..b7a2884782 100755
--- a/build_debian.sh
+++ b/build_debian.sh
@@ -537,6 +537,9 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'docke
 # Install scapy
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'scapy==2.4.4'
 
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'PyYAML==5.4.1' --no-build-isolation
+
 ## Note: keep pip installed for maintainance purpose
 
 # Install GCC, needed for building/installing some Python packages
diff --git a/dockers/docker-config-engine-bullseye/Dockerfile.j2 b/dockers/docker-config-engine-bullseye/Dockerfile.j2
index 57631df95c..892b1de5bb 100644
--- a/dockers/docker-config-engine-bullseye/Dockerfile.j2
+++ b/dockers/docker-config-engine-bullseye/Dockerfile.j2
@@ -23,6 +23,10 @@ RUN apt-get install -y        \
 RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
+# Fix armhf build failure
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip3 install PyYAML==5.4.1 --no-build-isolation
+
 # Install python-redis
 RUN pip3 install redis==4.5.4
 
diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2
index ccadb6e439..39f14543b6 100644
--- a/dockers/docker-config-engine-buster/Dockerfile.j2
+++ b/dockers/docker-config-engine-buster/Dockerfile.j2
@@ -23,6 +23,10 @@ RUN apt-get install -y        \
 RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
+# Fix armhf build failure
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip3 install PyYAML==5.4.1 --no-build-isolation
+
 # Install python-redis
 RUN pip3 install redis==4.5.4
 
diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2
index 89b1240bb1..42230f998e 100644
--- a/sonic-slave-bullseye/Dockerfile.j2
+++ b/sonic-slave-bullseye/Dockerfile.j2
@@ -526,7 +526,8 @@ RUN pip3 uninstall -y enum34
 RUN pip3 install j2cli==0.3.10
 
 # For sonic-mgmt-framework
-RUN pip3 install "PyYAML==5.4.1"
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
 {%- if CROSS_BUILD_ENVIRON != "y" %}
 RUN pip3 install "lxml==4.9.1"
 {%- endif %}
diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
index 8c292e2105..8c1a3a629f 100644
--- a/sonic-slave-buster/Dockerfile.j2
+++ b/sonic-slave-buster/Dockerfile.j2
@@ -541,8 +541,9 @@ RUN pip3 install MarkupSafe==2.0.1
 RUN pip3 install Jinja2==3.0.3
 
 # For sonic-mgmt-framework
-RUN pip2 install "PyYAML==5.4.1"
-RUN pip3 install "PyYAML==5.4.1"
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip2 install "PyYAML==5.4.1" --no-build-isolation
+RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
 
 {%- if CROSS_BUILD_ENVIRON != "y" %}
 RUN pip2 install "lxml==4.9.1"