Revert "revert [syslog] Add remote syslog configuration (cherry-pick to 202305) (#15897) (#16179)"

This reverts commit 164fa102c0.

files/image_config/rsyslog/rsyslog-config.sh

@@ -17,7 +17,10 @@ if [[ ($NUM_ASIC -gt 1) ]]; then
 else
     udp_server_ip=$(ip -j -4 addr list lo scope host | jq -r -M '.[0].addr_info[0].local')
 fi
+hostname=$(hostname)
 
-sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog.conf.j2 -a "{\"udp_server_ip\": \"$udp_server_ip\"}"  >/etc/rsyslog.conf
+sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog.conf.j2 \
+    -a "{\"udp_server_ip\": \"$udp_server_ip\", \"hostname\": \"$hostname\"}" \
+    > /etc/rsyslog.conf
 
 systemctl restart rsyslog

files/image_config/rsyslog/rsyslog.conf.j2

@@ -15,21 +15,14 @@
 
 $ModLoad imuxsock # provides support for local system logging
 
-{% if SYSLOG_CONFIG is defined %}
+{% set gconf = (SYSLOG_CONFIG | d({})).get('GLOBAL', {}) -%}
-{% if 'GLOBAL' in SYSLOG_CONFIG %}
+{% set rate_limit_interval = gconf.get('rate_limit_interval') %}
-{% if 'rate_limit_interval' in SYSLOG_CONFIG['GLOBAL']%}
+{% set rate_limit_burst = gconf.get('rate_limit_burst') %}
-{% set rate_limit_interval = SYSLOG_CONFIG['GLOBAL']['rate_limit_interval'] %}
-{% endif %}
-{% if 'rate_limit_burst' in SYSLOG_CONFIG['GLOBAL']%}
-{% set rate_limit_burst = SYSLOG_CONFIG['GLOBAL']['rate_limit_burst'] %}
-{% endif %}
-{% endif %}
-{% endif %}
 
-{% if rate_limit_interval is defined %}
+{% if rate_limit_interval is not none %}
 $SystemLogRateLimitInterval {{ rate_limit_interval }}
 {% endif %}
-{% if rate_limit_burst is defined %}
+{% if rate_limit_burst is not none %}
 $SystemLogRateLimitBurst {{ rate_limit_burst }}
 {% endif %}
 
@@ -49,6 +42,8 @@ $UDPServerRun 514
 ###########################
 #### GLOBAL DIRECTIVES ####
 ###########################
+{% set format = gconf.get('format', 'standard') -%}
+{% set fw_name = gconf.get('welf_firewall_name', hostname) -%}
 #
 # Use traditional timestamp format.
 # To enable high precision timestamps, comment out the following line.
@@ -59,6 +54,10 @@ $UDPServerRun 514
 $template SONiCFileFormat,"%timegenerated%.%timegenerated:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
 $ActionFileDefaultTemplate SONiCFileFormat
 
+template(name="WelfRemoteFormat" type="string" string="%TIMESTAMP% id=firewall time=\"%timereported\
+:::date-year%-%timereported:::date-month%-%timereported:::date-day% %timereported:::date-hour%:%timereported:::date-minute%:%timereported\
+:::date-second%\" fw=\"{{ fw_name }}\" pri=%syslogpriority% msg=\"%syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\"\n")
+
 #
 # Set the default permissions for all log files.
 #
@@ -91,25 +90,36 @@ $RepeatedMsgReduction on
 # Remote syslog logging
 #
 
-# The omfwd plug-in provides the core functionality of traditional message forwarding via UDP and plain TCP.
+# The omfwd plug-in provides the core functionality of traditional message
-# It is a built-in module that does not need to be loaded.
+# forwarding via UDP and plain TCP. It is a built-in module that does not need
+# to be loaded.
 
-{% if SYSLOG_SERVER is defined %}
+{% set servers = SYSLOG_SERVER | d({}) -%}
-{% for server, data in SYSLOG_SERVER.items() %}
+{% for server in servers %}
-{% set params_list = [] %}
+{% set conf = servers[server] | d({}) -%}
-{% if 'source' in data %}
+
-{% set dummy = params_list.append('address=' + '"' + data.source|string + '"') %}
+{% set source = conf.get('source') -%}
-{% endif %}
+{% set port = conf.get('port', 514) -%}
-{% if 'port' in data %}
+{% set proto = conf.get('protocol', 'udp') -%}
-{% set dummy = params_list.append('port=' + '"' + data.port|string + '"') %}
+{% set vrf = conf.get('vrf', 'default') -%}
-{% endif %}
+{% set severity = conf.get('severity', gconf.get('severity', 'notice')) -%}
-{% if 'vrf' in data and data['vrf'] != "default" %}
+{% set filter = conf.get('filter') -%}
-{% set dummy = params_list.append('device=' + '"' + data.vrf|string + '"') %}
+{% set regex = conf.get('filter_regex') -%}
-{% endif %}
+
-{% if params_list %}
+{% set fmodifier = '!' if filter == 'exclude' else '' %}
-*.* action(type="omfwd" target="{{ server }}" protocol="udp" {{ params_list|join(' ') }} template="SONiCFileFormat")
+{% set device = 'eth0' if vrf == 'default' else vrf -%}
-{% else %}
+{% set template = 'WelfRemoteFormat' if format == 'welf' else 'SONiCFileFormat' -%}
-*.* action(type="omfwd" target="{{ server }}" protocol="udp" template="SONiCFileFormat")
+
+{# Server extra options -#}
+{% set options = '' -%}
+
+{% if source -%}
+    {% set options = options ~ ' Address="' ~ source ~ '"'-%}
+{% endif -%}
+
+{% if filter %}
+:msg, {{ fmodifier }}ereregex, "{{ regex }}"
 {% endif %}
+*.{{ severity }}
+action(type="omfwd" Target="{{ server }}" Port="{{ port }}" Protocol="{{ proto }}" Device="{{ device }}" Template="{{ template }}"{{ options }})
 {% endfor %}
-{% endif %}

src/sonic-yang-models/doc/Configuration.md

@@ -51,7 +51,7 @@ Table of Contents
          * [MUX_LINKMGR](#mux_linkmgr)
          * [NEIGH](#neigh)
          * [NTP Global Configuration](#ntp-global-configuration)
-         * [NTP and SYSLOG servers](#ntp-and-syslog-servers)
+         * [NTP Servers](#ntp-servers)
          * [Peer Switch](#peer-switch)
          * [Policer](#policer)
          * [Port](#port)
@@ -60,7 +60,8 @@ Table of Contents
          * [Scheduler](#scheduler)
          * [Port QoS Map](#port-qos-map)
          * [Queue](#queue)
-         * [Syslog Rate Limit](#syslog-rate-limit)
+         * [Syslog Global Configuration](#syslog-global-configuration)
+         * [Syslog Servers](#syslog-servers)
          * [Sflow](#sflow)
          * [Restapi](#restapi)
          * [System Port](#system-port)
@@ -1517,7 +1518,7 @@ for that address.
 }
 ```
 
-### NTP and SYSLOG servers
+### NTP servers
 
 These information are configured in individual tables. Domain name or IP
 address of the server is used as object key. Currently there are no
@@ -1540,35 +1541,6 @@ attributes in those objects.
 }
 ```
 
-***Syslog server***
-```
-{
-    "SYSLOG_SERVER": {
-        "10.0.0.5": {},
-        "10.0.0.6": {},
-        "10.11.150.5": {}
-    },
-
-    "SYSLOG_SERVER" : {
-        "2.2.2.2": {
-            "source": "1.1.1.1",
-            "port": "514",
-            "vrf": "default"
-        },
-        "4.4.4.4": {
-            "source": "3.3.3.3",
-            "port": "514",
-            "vrf": "mgmt"
-        },
-        "2222::2222": {
-            "source": "1111::1111",
-            "port": "514",
-            "vrf": "Vrf-Data"
-        }
-    }
-}
-```
-
 ### Peer Switch
 
 Below is an exmaple of the peer switch table configuration.
@@ -1854,7 +1826,33 @@ key - name
 | collector_port | Destination L4 port of the Sflow collector                                              |             | 6343      |             |
 | collector_vrf  | Specify the Collector VRF. In this revision, it is either default VRF or Management VRF.|             |           |             |
 
-### Syslog Rate Limit
+### Syslog Global Configuration
+
+These configuration options are used to configure rsyslog utility and the way
+the system generates logs.
+
+***Configuration sample***
+```
+{
+    "SYSLOG_CONFIG": {
+        "GLOBAL": {
+            "rate_limit_interval": "5",
+            "rate_limit_burst": "100",
+            "format": "welf",
+            "welf_firewall_name": "bla",
+            "severity": "info"
+        }
+    }
+}
+```
+
+* `rate_limit_interval` - determines the amount of time that is being measured for rate limiting: `unsigned integer`
+* `rate_limit_burst` - defines the amount of messages, that have to occur in the time limit: `unsigned integer`
+* `format` - syslog log format: `{standard, welf}`
+* `welf_firewall_name` - WELF format firewall name: `string`
+* `severity` - global log severity: `{emerg, alert, crit, error, warning, notice, info, debug}`
+
+***Syslog Rate Limit***
 
 Host side configuration:
 
@@ -1886,6 +1884,50 @@ Container side configuration:
 }
 ```
 
+### Syslog servers
+
+These information are configured in individual tables. Domain name or IP
+address of the server is used as object key. Each server can be configurable.
+
+***Configuration sample***
+```
+{
+    "SYSLOG_SERVER": {
+        "10.0.0.5": {},
+        "10.0.0.6": {},
+        "10.11.150.5": {}
+    },
+
+    "SYSLOG_SERVER" : {
+        "4.4.4.4": {
+            "source": "3.3.3.3",
+            "port": "514",
+            "vrf": "mgmt"
+        },
+        "2222::2222": {
+            "source": "1111::1111",
+            "port": "514",
+            "vrf": "Vrf-Data"
+        },
+        "somehostname": {
+            "filter": "include",
+            "filter_regex": "ololo",
+            "port": "514",
+            "protocol": "tcp",
+            "severity": "notice",
+            "vrf": "default"
+        }
+    }
+}
+```
+
+* `filter` - determines if syslog will include or exclude messages specified by regex: `{include, exclude}`
+* `filter_regex` - filter messages by this regex: `string`
+* `port` - network port to use to connect to remote server: `integer: 1..65535`
+* `protocol` - network protocol to use to connect to remote server: `{tcp, udp}`
+* `severity` - per-server log severity, overrifes global one: `{emerg, alert, crit, error, warning, notice, info, debug}`
+
+
 ### System Port
 Every port on the system requires a global representation, known as a System Port,
 and is listed in this table.

src/sonic-yang-models/tests/files/sample_config_db.json

@@ -499,12 +499,23 @@
                 "source": "1111::1111",
                 "port": "514",
                 "vrf": "Vrf_blue"
+            },
+            "somehostname": {
+                "filter": "include",
+                "filter_regex": "ololo",
+                "port": "514",
+                "protocol": "tcp",
+                "severity": "notice",
+                "vrf": "default"
             }
         },
         "SYSLOG_CONFIG" : {
             "GLOBAL": {
                 "rate_limit_interval": "5",
-                "rate_limit_burst": "100"
+                "rate_limit_burst": "100",
+                "format": "welf",
+                "welf_firewall_name": "bla",
+                "severity": "info"
             }
         },
         "SYSLOG_CONFIG_FEATURE" : {

src/sonic-yang-models/tests/yang_model_tests/tests/syslog.json

@@ -29,10 +29,6 @@
         "desc": "Load syslog server table with empty address as syslog server.",
         "eStrKey": "InvalidValue"
     },
-    "SYSLOG_SERVER_INVALID_IPADDR_TEST": {
-        "desc": "Load syslog server table with invalid ipv4 address as syslog server.",
-        "eStrKey": "InvalidValue"
-    },
     "SYSLOG_SERVER_INVALID_IPV6_ADDR_TEST": {
         "desc": "Load syslog server table with invalid ipv6 address as syslog server.",
         "eStrKey": "InvalidValue"
@@ -62,5 +58,60 @@
     "SYSLOG_CONFIG_FEATURE_INVALID_BURST": {
         "desc": "Configure invalid rate_limit_burst in SYSLOG_CONFIG_FEATURE.",
         "eStrKey": "InvalidValue"
+    },
+    "SYSLOG_SERVER_HOSTNAME": {
+        "desc": "Load syslog server table with hostname"
+    },
+    "SYSLOG_SERVER_HOSTNAME_INVALID": {
+        "desc": "Load syslog server table with invalid hostname",
+        "eStrKey": "InvalidValue"
+    },
+    "SYSLOG_SERVER_FILTER_TYPE": {
+        "desc": "Valid filter type for syslog server"
+    },
+    "SYSLOG_SERVER_FILTER_TYPE_INVALID": {
+        "desc": "Invalid filter type for syslog server",
+        "eStrKey": "InvalidValue"
+    },
+    "SYSLOG_SERVER_FILTER_REGEX": {
+        "desc": "Valid filter regex"
+    },
+    "SYSLOG_SERVER_PROTOCOL": {
+        "desc": "Valid syslog server protocol"
+    },
+    "SYSLOG_SERVER_PROTOCOL_INVALID": {
+        "desc": "Invalid syslog server protocol",
+        "eStrKey": "InvalidValue"
+    },
+    "SYSLOG_SERVER_SEVERITY": {
+        "desc": "Syslog server valid severity"
+    },
+    "SYSLOG_SERVER_SEVERITY_INVALID": {
+        "desc": "Syslog server invalid severity",
+        "eStrKey": "InvalidValue"
+    },
+    "SYSLOG_CONFIG_GLOBAL_VALID": {
+        "desc": "Global syslog configuration"
+    },
+    "SYSLOG_CONFIG_FORMAT": {
+        "desc": "Syslog format type"
+    },
+    "SYSLOG_CONFIG_FORMAT_INVALID": {
+        "desc": "Invalid syslog format",
+        "eStrKey": "InvalidValue"
+    },
+    "SYSLOG_CONFIG_FORMAT_WELF_FW_NAME": {
+        "desc": "Syslog format WELF firewall name"
+    },
+    "SYSLOG_CONFIG_FORMAT_WELF_FW_NAME_INVALID": {
+        "desc": "Syslog format WELF invalid firewall name",
+        "eStrKey": "Must"
+    },
+    "SYSLOG_CONFIG_SEVERITY": {
+        "desc": "Global syslog severity"
+    },
+    "SYSLOG_CONFIG_SEVERITY_INVALID": {
+        "desc": "Global invalid syslog severity",
+        "eStrKey": "InvalidValue"
     }
 }

src/sonic-yang-models/tests/yang_model_tests/tests_config/syslog.json

@@ -136,17 +136,6 @@
             }
         }
     },
-    "SYSLOG_SERVER_INVALID_IPADDR_TEST" : {
-        "sonic-syslog:sonic-syslog": {
-            "sonic-syslog:SYSLOG_SERVER": {
-                "SYSLOG_SERVER_LIST": [
-                    {
-                        "server_address": "1111.22.33.1"
-                    }
-                ]
-            }
-        }
-    },
     "SYSLOG_SERVER_INVALID_IPV6_ADDR_TEST" : {
         "sonic-syslog:sonic-syslog": {
             "sonic-syslog:SYSLOG_SERVER": {
@@ -209,6 +198,64 @@
             }
         }
     },
+    "SYSLOG_SERVER_HOSTNAME" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_SERVER": {
+                "SYSLOG_SERVER_LIST": [
+                    {
+                        "server_address": "thebestswitch"
+                    }
+                ]
+            }
+        }
+    },
+    "SYSLOG_SERVER_HOSTNAME_INVALID" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_SERVER": {
+                "SYSLOG_SERVER_LIST": [
+                    {
+                        "server_address": "-"
+                    }
+                ]
+            }
+        }
+    },
+    "SYSLOG_SERVER_FILTER_TYPE" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_SERVER": {
+                "SYSLOG_SERVER_LIST": [
+                    {
+                        "server_address": "1.2.3.4",
+                        "filter": "exclude"
+                    }
+                ]
+            }
+        }
+    },
+    "SYSLOG_SERVER_FILTER_TYPE_INVALID" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_SERVER": {
+                "SYSLOG_SERVER_LIST": [
+                    {
+                        "server_address": "1.2.3.4",
+                        "filter": "excludalol"
+                    }
+                ]
+            }
+        }
+    },
+    "SYSLOG_SERVER_FILTER_REGEX" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_SERVER": {
+                "SYSLOG_SERVER_LIST": [
+                    {
+                        "server_address": "1.2.3.4",
+                        "filter_regex": "^expeliarmus.*"
+                    }
+                ]
+            }
+        }
+    },
     "SYSLOG_CONFIG_FEATURE_INVALID_SERVICE_NAME": {
         "sonic-syslog:sonic-syslog": {
             "sonic-syslog:SYSLOG_CONFIG_FEATURE": {
@@ -222,6 +269,18 @@
             }
         }
     },
+    "SYSLOG_SERVER_PROTOCOL" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_SERVER": {
+                "SYSLOG_SERVER_LIST": [
+                    {
+                        "server_address": "1.2.3.4",
+                        "protocol": "tcp"
+                    }
+                ]
+            }
+        }
+    },
     "SYSLOG_CONFIG_FEATURE_INVALID_INTERVAL": {
         "sonic-syslog:sonic-syslog": {
             "sonic-syslog:SYSLOG_CONFIG_FEATURE": {
@@ -244,6 +303,30 @@
             }
         }
     },
+    "SYSLOG_SERVER_PROTOCOL_INVALID" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_SERVER": {
+                "SYSLOG_SERVER_LIST": [
+                    {
+                        "server_address": "1.2.3.4",
+                        "protocol": "order66"
+                    }
+                ]
+            }
+        }
+    },
+    "SYSLOG_SERVER_SEVERITY" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_SERVER": {
+                "SYSLOG_SERVER_LIST": [
+                    {
+                        "server_address": "1.2.3.4",
+                        "severity": "info"
+                    }
+                ]
+            }
+        }
+    },
     "SYSLOG_CONFIG_FEATURE_INVALID_BURST": {
         "sonic-syslog:sonic-syslog": {
             "sonic-syslog:SYSLOG_CONFIG_FEATURE": {
@@ -265,5 +348,84 @@
                 ]
             }
         }
+    },
+    "SYSLOG_SERVER_SEVERITY_INVALID" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_SERVER": {
+                "SYSLOG_SERVER_LIST": [
+                    {
+                        "server_address": "1.2.3.4",
+                        "severity": "information"
+                    }
+                ]
+            }
+        }
+    },
+    "SYSLOG_CONFIG_GLOBAL_VALID" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_CONFIG": {
+                "GLOBAL": {
+                    "format": "welf",
+                    "welf_firewall_name": "welf-fw-name",
+                    "severity": "debug"
+                }
+            }
+        }
+    },
+    "SYSLOG_CONFIG_FORMAT" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_CONFIG": {
+                "GLOBAL": {
+                    "format": "standard"
+                }
+            }
+        }
+    },
+    "SYSLOG_CONFIG_FORMAT_INVALID" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_CONFIG": {
+                "GLOBAL": {
+                    "format": "nonstandard"
+                }
+            }
+        }
+    },
+    "SYSLOG_CONFIG_FORMAT_WELF_FW_NAME" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_CONFIG": {
+                "GLOBAL": {
+                    "format": "welf",
+                    "welf_firewall_name": "welf-switch"
+                }
+            }
+        }
+    },
+    "SYSLOG_CONFIG_FORMAT_WELF_FW_NAME_INVALID" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_CONFIG": {
+                "GLOBAL": {
+                    "format": "standard",
+                    "welf_firewall_name": "welf-switch"
+                }
+            }
+        }
+    },
+    "SYSLOG_CONFIG_SEVERITY" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_CONFIG": {
+                "GLOBAL": {
+                    "severity": "crit"
+                }
+            }
+        }
+    },
+    "SYSLOG_CONFIG_SEVERITY_INVALID" : {
+        "sonic-syslog:sonic-syslog": {
+            "sonic-syslog:SYSLOG_CONFIG": {
+                "GLOBAL": {
+                    "severity": "critical"
+                }
+            }
+        }
     }
 }

src/sonic-yang-models/yang-models/sonic-syslog.yang

@@ -49,6 +49,44 @@ module sonic-syslog {
         }
     }
 
+    typedef log-format {
+        description "Represents syslog log format";
+        type enumeration {
+            enum welf;
+            enum standard;
+        }
+    }
+
+    typedef rsyslog-protocol {
+        description "The protocol to send logs to remote server";
+        type enumeration {
+            enum tcp;
+            enum udp;
+        }
+    }
+
+    typedef syslog-filter-type {
+        description "The filter type";
+        type enumeration {
+            enum include;
+            enum exclude;
+        }
+    }
+
+    typedef rsyslog-severity {
+        description "The protocol to send logs to remote server";
+        type enumeration {
+            enum none;
+            enum debug;
+            enum info;
+            enum notice;
+            enum warn;
+            enum error;
+            enum crit;
+        }
+    }
+
+
     container sonic-syslog {
 
         container SYSLOG_SERVER {
@@ -61,7 +99,7 @@ module sonic-syslog {
 
                 leaf server_address {
                     description "Syslog server IP address";
-                    type inet:ip-address;
+                    type inet:host;
                 }
 
                 leaf source {
@@ -88,6 +126,26 @@ module sonic-syslog {
                     or (/mvrf:sonic-mgmt_vrf/mvrf:MGMT_VRF_CONFIG/mvrf:vrf_global/mvrf:mgmtVrfEnabled = 'true')";
                 }
 
+                leaf filter {
+                    description "Syslog filter type";
+                    type syslog-filter-type;
+                }
+
+                leaf filter_regex {
+                    description "Filter regex";
+                    type string;
+                }
+
+                leaf protocol {
+                    description "The protocol to send logs to remote server";
+                    type rsyslog-protocol;
+                }
+
+                leaf severity {
+                    description "Limit the severity to send logs to remote server";
+                    type rsyslog-severity;
+                }
+
             }
             /* end of list SYSLOG_SERVER_LIST */
         }
@@ -105,6 +163,24 @@ module sonic-syslog {
                 leaf rate_limit_burst {
                     type syslog-rate-limit-burst;
                 }
+
+                leaf format {
+                    description "Log format";
+                    type log-format;
+                    default standard;
+                }
+
+                leaf welf_firewall_name {
+                    description "WELF format Firewall name";
+                    type string;
+                    must "(../format != 'standard')";
+                }
+
+                leaf severity {
+                    type rsyslog-severity;
+                    default notice;
+                }
+
             }
             /* end of list SYSLOG_CONFIG_LIST */
         }