[hostcfgd] Synchronize all feature statuses once upon start (#4714)

- Ensure all features (services) are in the configured state when hostcfgd starts
- Better functionalization of code
- Also replace calls to deprecated `has_key()` method in `tacacs_server_handler()` and `tacacs_global_handler()` with `in` keyword.

This PR depends on https://github.com/Azure/sonic-utilities/pull/944, otherwise `config load_minigraph` will fail when trying to restart disabled services.

files/image_config/hostcfgd/hostcfgd

@@ -40,6 +40,40 @@ def obfuscate(data):
     else:
         return data
 
+
+def update_feature_status(feature_name, status):
+    if status == "enabled":
+        start_cmds = []
+        start_cmds.append("sudo systemctl unmask {}.service".format(feature_name))
+        start_cmds.append("sudo systemctl enable {}.service".format(feature_name))
+        start_cmds.append("sudo systemctl start {}.service".format(feature_name))
+        for cmd in start_cmds:
+            syslog.syslog(syslog.LOG_INFO, "Running cmd: '{}'".format(cmd))
+            try:
+                subprocess.check_call(cmd, shell=True)
+            except subprocess.CalledProcessError as err:
+                syslog.syslog(syslog.LOG_ERR, "'{}' failed. RC: {}, output: {}"
+                              .format(err.cmd, err.returncode, err.output))
+                continue
+        syslog.syslog(syslog.LOG_INFO, "Feature '{}' is enabled and started".format(feature_name))
+    elif status == "disabled":
+        stop_cmds = []
+        stop_cmds.append("sudo systemctl stop {}.service".format(feature_name))
+        stop_cmds.append("sudo systemctl disable {}.service".format(feature_name))
+        stop_cmds.append("sudo systemctl mask {}.service".format(feature_name))
+        for cmd in stop_cmds:
+            syslog.syslog(syslog.LOG_INFO, "Running cmd: '{}'".format(cmd))
+            try:
+                subprocess.check_call(cmd, shell=True)
+            except subprocess.CalledProcessError as err:
+                syslog.syslog(syslog.LOG_ERR, "'{}' failed. RC: {}, output: {}"
+                              .format(err.cmd, err.returncode, err.output))
+                continue
+        syslog.syslog(syslog.LOG_INFO, "Feature '{}' is stopped and disabled".format(feature_name))
+    else:
+        syslog.syslog(syslog.LOG_ERR, "Unexpected status value '{}' for feature '{}'".format(status, feature_name))
+
+
 class Iptables(object):
     def __init__(self):
         '''
@@ -114,7 +148,7 @@ class Iptables(object):
             try:
                 subprocess.check_call(cmd, shell=True)
             except subprocess.CalledProcessError as err:
-                syslog.syslog(syslog.LOG_ERR, "{} - failed: return code - {}, output:\n{}"
+                syslog.syslog(syslog.LOG_ERR, "'{}' failed. RC: {}, output: {}"
                               .format(err.cmd, err.returncode, err.output))
 
 class AaaCfg(object):
@@ -234,26 +268,40 @@ class HostConfigDaemon:
         self.iptables = Iptables()
         self.iptables.load(lpbk_table)
 
+    def update_all_feature_statuses(self):
+        feature_table = self.config_db.get_table('FEATURE')
+        for feature_name in feature_table.keys():
+            if not feature_name:
+                syslog.syslog(syslog.LOG_WARNING, "Feature is None")
+                continue
+
+            status = feature_table[feature_name]['status']
+            if not status:
+                syslog.syslog(syslog.LOG_WARNING, "Status of feature '{}' is None".format(feature_name))
+                continue
+
+            update_feature_status(feature_name, status)
+
     def aaa_handler(self, key, data):
         self.aaacfg.aaa_update(key, data)
 
     def tacacs_server_handler(self, key, data):
         self.aaacfg.tacacs_server_update(key, data)
         log_data = copy.deepcopy(data)
-        if log_data.has_key('passkey'):
+        if 'passkey' in log_data:
             log_data['passkey'] = obfuscate(log_data['passkey'])
         syslog.syslog(syslog.LOG_INFO, 'value of {} changed to {}'.format(key, log_data))
 
     def tacacs_global_handler(self, key, data):
         self.aaacfg.tacacs_global_update(key, data)
         log_data = copy.deepcopy(data)
-        if log_data.has_key('passkey'):
+        if 'passkey' in log_data:
             log_data['passkey'] = obfuscate(log_data['passkey'])
         syslog.syslog(syslog.LOG_INFO, 'value of {} changed to {}'.format(key, log_data))
 
     def lpbk_handler(self, key, data):
         key = ConfigDBConnector.deserialize_key(key)
-        #Check if delete operation by fetch existing keys
+        # Check if delete operation by fetch existing keys
         keys = self.config_db.get_keys('LOOPBACK_INTERFACE')
         if key in keys:
             add = True
@@ -263,48 +311,23 @@ class HostConfigDaemon:
         self.iptables.iptables_handler(key, data, add)
 
     def feature_status_handler(self, key, data):
-        status_data = self.config_db.get_table('FEATURE')
+        feature_name = key
-        for key in status_data.keys():
+        feature_table = self.config_db.get_table('FEATURE')
-            if not key:
+        if feature_name not in feature_table.keys():
-                syslog.syslog(syslog.LOG_WARNING, "FEATURE key is missing")
+            syslog.syslog(syslog.LOG_WARNING, "Feature '{}' not in FEATURE table".format(feature_name))
-                continue
+            return
 
-            status = status_data[key]['status']
+        status = feature_table[feature_name]['status']
-            if not status:
+        if not status:
-                syslog.syslog(syslog.LOG_WARNING, "status is missing for {}".format(key))
+            syslog.syslog(syslog.LOG_WARNING, "Status of feature '{}' is None".format(feature_name))
-                continue
+            return
-            if status == "enabled":
+
-                start_cmds=[]
+        update_feature_status(feature_name, status)
-                start_cmds.append("sudo systemctl unmask {}.service".format(key))
-                start_cmds.append("sudo systemctl enable {}.service".format(key))
-                start_cmds.append("sudo systemctl start {}.service".format(key))
-                for cmd in start_cmds:
-                    syslog.syslog(syslog.LOG_INFO, "Running cmd - {}".format(cmd))
-                    try:
-                        subprocess.check_call(cmd, shell=True)
-                    except subprocess.CalledProcessError as err:
-                        syslog.syslog(syslog.LOG_ERR, "{} - failed: return code - {}, output:\n{}"
-                                  .format(err.cmd, err.returncode, err.output))
-                        continue
-                syslog.syslog(syslog.LOG_INFO, "Feature '{}' is enabled and started".format(key))
-            elif status == "disabled":
-                stop_cmds=[]
-                stop_cmds.append("sudo systemctl stop {}.service".format(key))
-                stop_cmds.append("sudo systemctl disable {}.service".format(key))
-                stop_cmds.append("sudo systemctl mask {}.service".format(key))
-                for cmd in stop_cmds:
-                    syslog.syslog(syslog.LOG_INFO, "Running cmd - {}".format(cmd))
-                    try:
-                        subprocess.check_call(cmd, shell=True)
-                    except subprocess.CalledProcessError as err:
-                        syslog.syslog(syslog.LOG_ERR, "{} - failed: return code - {}, output:\n{}"
-                                  .format(err.cmd, err.returncode, err.output))
-                        continue
-                syslog.syslog(syslog.LOG_INFO, "Feature '{}' is stopped and disabled".format(key))
-            else:
-                syslog.syslog(syslog.LOG_ERR, "Unexpected status value '{}' for '{}'".format(status, key))
 
     def start(self):
+        # Update all feature statuses once upon starting
+        self.update_all_feature_statuses()
+
         self.config_db.subscribe('AAA', lambda table, key, data: self.aaa_handler(key, data))
         self.config_db.subscribe('TACPLUS_SERVER', lambda table, key, data: self.tacacs_server_handler(key, data))
         self.config_db.subscribe('TACPLUS', lambda table, key, data: self.tacacs_global_handler(key, data))