diff --git a/openssh.spec b/openssh.spec
index a8c106d..da696e8 100644
--- a/openssh.spec
+++ b/openssh.spec
@@ -51,7 +51,7 @@
 
 # Do not forget to bump pam_ssh_agent_auth release if you rewind the main package release to 1
 %global openssh_ver 8.6p1
-%global openssh_rel 3
+%global openssh_rel 4
 %global pam_ssh_agent_ver 0.10.4
 %global pam_ssh_agent_rel 3
 
@@ -388,9 +388,10 @@ autoreconf
 popd
 
 %build
+%set_build_flags
 # the -fvisibility=hidden is needed for clean build of the pam_ssh_agent_auth
 # it is needed for lib(open)ssh build too since it is linked to the pam module too
-CFLAGS="$RPM_OPT_FLAGS -fvisibility=hidden"; export CFLAGS
+CFLAGS="$CFLAGS -fvisibility=hidden"; export CFLAGS
 %if %{pie}
 %ifarch s390 s390x sparc sparcv9 sparc64
 CFLAGS="$CFLAGS -fPIC"
@@ -659,6 +660,9 @@ test -f %{sysconfig_anaconda} && \
 %endif
 
 %changelog
+* Tue May 25 2021 Timm Bäder <tbaeder@redhat.com> - 8.6p1-4
+- Use %%set_build_flags to set all builds flags
+
 * Fri May 21 2021 Dmitry Belyavskiy <dbelyavs@redhat.com> - 8.6p1-3
 - Hostbased ssh authentication fails if session ID contains a '/' (#1963059)