diff --git a/build_debian.sh b/build_debian.sh
index 13e39ac736..f71bdd9fd0 100755
--- a/build_debian.sh
+++ b/build_debian.sh
@@ -546,6 +546,9 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'docke
 # Install scapy
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'scapy==2.4.4'
 
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'PyYAML==5.4.1' --no-build-isolation
+
 ## Note: keep pip installed for maintainance purpose
 
 # Install GCC, needed for building/installing some Python packages
diff --git a/dockers/docker-config-engine-bullseye/Dockerfile.j2 b/dockers/docker-config-engine-bullseye/Dockerfile.j2
index f6804de903..700af660a9 100644
--- a/dockers/docker-config-engine-bullseye/Dockerfile.j2
+++ b/dockers/docker-config-engine-bullseye/Dockerfile.j2
@@ -23,6 +23,10 @@ RUN apt-get install -y        \
 RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
+# Fix armhf build failure
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip3 install PyYAML==5.4.1 --no-build-isolation
+
 # Install python-redis
 RUN pip3 install redis==4.5.4
 
diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2
index 084956eed5..cfa61bc0ba 100644
--- a/dockers/docker-config-engine-buster/Dockerfile.j2
+++ b/dockers/docker-config-engine-buster/Dockerfile.j2
@@ -23,6 +23,10 @@ RUN apt-get install -y        \
 RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
+# Fix armhf build failure
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip3 install PyYAML==5.4.1 --no-build-isolation
+
 # Install python-redis
 RUN pip3 install redis==4.5.4
 
diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2
index dacff48453..72783f3e6d 100644
--- a/sonic-slave-bullseye/Dockerfile.j2
+++ b/sonic-slave-bullseye/Dockerfile.j2
@@ -530,7 +530,8 @@ RUN pip3 uninstall -y enum34
 RUN pip3 install j2cli==0.3.10
 
 # For sonic-mgmt-framework
-RUN pip3 install "PyYAML==5.4.1"
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
 {%- if CROSS_BUILD_ENVIRON != "y" %}
 RUN pip3 install "lxml==4.9.1"
 {%- endif %}
diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
index 7e0bf6d06b..637ac02f85 100644
--- a/sonic-slave-buster/Dockerfile.j2
+++ b/sonic-slave-buster/Dockerfile.j2
@@ -541,8 +541,9 @@ RUN pip3 install MarkupSafe==2.0.1
 RUN pip3 install Jinja2==3.0.3
 
 # For sonic-mgmt-framework
-RUN pip2 install "PyYAML==5.4.1"
-RUN pip3 install "PyYAML==5.4.1"
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip2 install "PyYAML==5.4.1" --no-build-isolation
+RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
 
 {%- if CROSS_BUILD_ENVIRON != "y" %}
 RUN pip2 install "lxml==4.9.1"