[dockers][supervisor] Increase event buffer size for dependent-startup (#5247)

When stopping the swss, pmon or bgp containers, log messages like the following can be seen:

```
Aug 23 22:50:43.789760 sonic-dut INFO swss#supervisord 2020-08-23 22:50:10,061 ERRO pool dependent-startup event buffer overflowed, discarding event 34
Aug 23 22:50:43.789760 sonic-dut INFO swss#supervisord 2020-08-23 22:50:10,063 ERRO pool dependent-startup event buffer overflowed, discarding event 35
Aug 23 22:50:43.789760 sonic-dut INFO swss#supervisord 2020-08-23 22:50:10,064 ERRO pool dependent-startup event buffer overflowed, discarding event 36
Aug 23 22:50:43.789760 sonic-dut INFO swss#supervisord 2020-08-23 22:50:10,066 ERRO pool dependent-startup event buffer overflowed, discarding event 37
```

This is due to the number of programs in the container managed by supervisor, all generating events at the same time. The default event queue buffer size in supervisor is 10. This patch increases that value in all containers in order to eliminate these errors. As more programs are added to the containers, we may need to further adjust these values. I increased all buffer sizes to 25 except for containers with more programs or templated supervisor.conf files which allow for a variable number of programs. In these cases I increased the buffer size to 50. One final exception is the swss container, where the buffer fills up to ~50, so I increased this buffer to 100.

Resolves https://github.com/Azure/sonic-buildimage/issues/5241
This commit is contained in:
Joe LeVeque 2020-09-08 23:36:38 -07:00 committed by GitHub
parent d4fc8e5b22
commit 5b3b4804ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 24 additions and 0 deletions

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=50
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name dhcp_relay command=/usr/bin/supervisor-proc-exit-listener --container-name dhcp_relay

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=50
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name bgp command=/usr/bin/supervisor-proc-exit-listener --container-name bgp

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name lldp command=/usr/bin/supervisor-proc-exit-listener --container-name lldp

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name nat command=/usr/bin/supervisor-proc-exit-listener --container-name nat

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=100
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name swss command=/usr/bin/supervisor-proc-exit-listener --container-name swss

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=100
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name pmon command=/usr/bin/supervisor-proc-exit-listener --container-name pmon

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-script] [eventlistener:supervisor-proc-exit-script]
command=/usr/bin/supervisor-proc-exit-listener --container-name radv command=/usr/bin/supervisor-proc-exit-listener --container-name radv

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name sflow command=/usr/bin/supervisor-proc-exit-listener --container-name sflow

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=50
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name snmp command=/usr/bin/supervisor-proc-exit-listener --container-name snmp

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[program:rsyslogd] [program:rsyslogd]
command=/usr/sbin/rsyslogd -n -iNONE command=/usr/sbin/rsyslogd -n -iNONE

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name restapi command=/usr/bin/supervisor-proc-exit-listener --container-name restapi

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=50
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name telemetry command=/usr/bin/supervisor-proc-exit-listener --container-name telemetry

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=50
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name teamd command=/usr/bin/supervisor-proc-exit-listener --container-name teamd

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name syncd command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name syncd command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name syncd command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[program:rsyslogd] [program:rsyslogd]
command=/usr/sbin/rsyslogd -n -iNONE command=/usr/sbin/rsyslogd -n -iNONE

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name syncd command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name syncd command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name syncd command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name syncd command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name syncd command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=25
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name syncd command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

View File

@ -10,6 +10,7 @@ autorestart=unexpected
startretries=0 startretries=0
exitcodes=0,3 exitcodes=0,3
events=PROCESS_STATE events=PROCESS_STATE
buffer_size=50
[eventlistener:supervisor-proc-exit-listener] [eventlistener:supervisor-proc-exit-listener]
command=/usr/bin/supervisor-proc-exit-listener --container-name dhcp_relay command=/usr/bin/supervisor-proc-exit-listener --container-name dhcp_relay