[cfggen] Extend Template Argument to Support Batch Mode (#4941)

Calls to cfggen take considerable time. With batch mode, we will have the ability
to reduce number of calls from services.

Example of the batch mode command:
sonic-cfggen -t template-1.j2 -t template-2.j2,config-db -t template-3.j2,config-db -t template-4.j2,file1 -t template-5.j2,file2 --write-to-db.

template-1.j2 will be rendered to stdout since it is missing the dest part. stdout is default
config-db is a special keyword that will inject the rendered template into internal data structure. The internal data structure gets written to redis-db with --write-to-db switch. In the case the user would like to write to a file named config-db, it could be given as /config-db or ./config-db

signed-off-by: Tamer Ahmed <tamer.ahmed@microsoft.com>

src/sonic-config-engine/sonic-cfggen

@@ -35,6 +35,7 @@ import yaml
 import jinja2
 import netaddr
 import json
+import contextlib
 from functools import partial
 from minigraph import minigraph_encoder
 from minigraph import parse_xml
@@ -204,6 +205,45 @@ def sort_data(data):
             data[table] = OrderedDict(natsorted(data[table].items()))
     return data
 
+@contextlib.contextmanager
+def smart_open(filename=None, mode=None):
+    """
+    Provide contextual file descriptor of filename if it is not a file descriptor
+    """
+    smart_file = open(filename, mode) if not isinstance(filename, file) else filename
+    try:
+        yield smart_file
+    finally:
+        if not isinstance(filename, file):
+            smart_file.close()
+
+def _process_json(args, data):
+    """
+    Process JSON file and update switch configuration data
+    """
+    for json_file in args.json:
+        with open(json_file, 'r') as stream:
+            deep_update(data, FormatConverter.to_deserialized(json.load(stream)))
+
+def _get_jinja2_env(paths):
+    """
+    Retreive Jinj2 env used to render configuration templates
+    """
+    loader = jinja2.FileSystemLoader(paths)
+    redis_bcc = RedisBytecodeCache(SonicV2Connector(host='127.0.0.1'))
+    env = jinja2.Environment(loader=loader, trim_blocks=True, bytecode_cache=redis_bcc)
+    env.filters['sort_by_port_index'] = sort_by_port_index
+    env.filters['ipv4'] = is_ipv4
+    env.filters['ipv6'] = is_ipv6
+    env.filters['unique_name'] = unique_name
+    env.filters['pfx_filter'] = pfx_filter
+    env.filters['ip_network'] = ip_network
+    for attr in ['ip', 'network', 'prefixlen', 'netmask', 'broadcast']:
+        env.filters[attr] = partial(prefix_attr, attr)
+    # Pass the is_multi_npu function as global
+    env.globals['multi_asic'] = is_multi_npu
+
+    return env
 
 def main():
     parser=argparse.ArgumentParser(description="Render configuration file from minigraph data and jinja2 template.")
@@ -221,14 +261,15 @@ def main():
     parser.add_argument("-H", "--platform-info", help="read platform and hardware info", action='store_true')
     parser.add_argument("-s", "--redis-unix-sock-file", help="unix sock file for redis connection")
     group = parser.add_mutually_exclusive_group()
-    group.add_argument("-t", "--template", help="render the data with the template file")
+    group.add_argument("-t", "--template", help="render the data with the template file", action="append", default=[],
+                       type=lambda opt_value: tuple(opt_value.split(',')) if ',' in opt_value else (opt_value, sys.stdout))
     parser.add_argument("-T", "--template_dir", help="search base for the template files", action='store')
     group.add_argument("-v", "--var", help="print the value of a variable, support jinja2 expression")
     group.add_argument("--var-json", help="print the value of a variable, in json format")
     group.add_argument("-w", "--write-to-db", help="write config into configdb", action='store_true')
-    group.add_argument("--print-data", help="print all data", action='store_true')
     group.add_argument("--preset", help="generate sample configuration from a preset template", choices=get_available_config())
     group = parser.add_mutually_exclusive_group()
+    group.add_argument("--print-data", help="print all data", action='store_true')
     group.add_argument("-K", "--key", help="Lookup for a specific key")
     args = parser.parse_args()
 
@@ -267,9 +308,7 @@ def main():
         if  brkout_table is not None:
             deep_update(data, {'BREAKOUT_CFG': brkout_table})
 
-    for json_file in args.json:
-        with open(json_file, 'r') as stream:
-            deep_update(data, FormatConverter.to_deserialized(json.load(stream)))
+    _process_json(args, data)
 
     if args.minigraph != None:
         minigraph = args.minigraph
@@ -297,7 +336,7 @@ def main():
 
     if args.from_db:
         if args.namespace is None:
-            configdb = ConfigDBConnector(**db_kwargs)
+            configdb = ConfigDBConnector(use_unix_socket_path=True, **db_kwargs)
         else:
             configdb = ConfigDBConnector(use_unix_socket_path=True, namespace=args.namespace, **db_kwargs)
 
@@ -328,28 +367,23 @@ def main():
             hardware_data['DEVICE_METADATA']['localhost'].update(asic_id=asic_id)
         deep_update(data, hardware_data)
 
-    if args.template is not None:
-        template_file = os.path.abspath(args.template)
-        paths  = ['/', '/usr/share/sonic/templates', os.path.dirname(template_file)]
-        if args.template_dir is not None:
-            template_dir = os.path.abspath(args.template_dir)
-            paths.append(template_dir)
-        loader = jinja2.FileSystemLoader(paths)
+    paths = ['/', '/usr/share/sonic/templates']
+    if args.template_dir:
+        paths.append(os.path.abspath(args.template_dir))
 
-        redis_bcc = RedisBytecodeCache(SonicV2Connector(host='127.0.0.1'))
-        env = jinja2.Environment(loader=loader, trim_blocks=True, bytecode_cache=redis_bcc)
-        env.filters['sort_by_port_index'] = sort_by_port_index
-        env.filters['ipv4'] = is_ipv4
-        env.filters['ipv6'] = is_ipv6
-        env.filters['unique_name'] = unique_name
-        env.filters['pfx_filter'] = pfx_filter
-        env.filters['ip_network'] = ip_network
-        for attr in ['ip', 'network', 'prefixlen', 'netmask', 'broadcast']:
-            env.filters[attr] = partial(prefix_attr, attr)
-        # Pass the is_multi_npu function as global
-        env.globals['multi_asic'] = is_multi_npu
-        template = env.get_template(template_file)
-        print(template.render(sort_data(data)))
+    if args.template:
+        for template_file, _ in args.template:
+            paths.append(os.path.dirname(os.path.abspath(template_file)))
+        env = _get_jinja2_env(paths)
+        sorted_data = sort_data(data)
+        for template_file, dest_file in args.template:
+            template = env.get_template(os.path.basename(template_file))
+            template_data = template.render(sorted_data)
+            if dest_file == "config-db":
+                deep_update(data, FormatConverter.to_deserialized(json.loads(template_data)))
+            else:
+                with smart_open(dest_file, 'w') as df:
+                    print(template_data, file=df)
 
     if args.var != None:
         template = jinja2.Template('{{' + args.var + '}}')
@@ -363,7 +397,7 @@ def main():
 
     if args.write_to_db:
         if args.namespace is None:
-            configdb = ConfigDBConnector(**db_kwargs)
+            configdb = ConfigDBConnector(use_unix_socket_path=True, **db_kwargs)
         else:
             configdb = ConfigDBConnector(use_unix_socket_path=True, namespace=args.namespace, **db_kwargs)
 

src/sonic-config-engine/tests/sample-template-1.json.j2

@@ -0,0 +1,4 @@
+{
+    "jk1_1": "{{ key1_1 }}",
+    "jk1_2": "{{ key1_2 }}"
+}

src/sonic-config-engine/tests/sample-template-2.json.j2

@@ -0,0 +1,4 @@
+{
+    "jk2_1": "{{ key2_1 }}",
+    "jk2_2": "{{ key2_2 }}"
+}

src/sonic-config-engine/tests/test2.j2

@@ -0,0 +1 @@
+{{ key1 }}

src/sonic-config-engine/tests/test_cfggen.py

@@ -1,4 +1,5 @@
 from unittest import TestCase
+import json
 import subprocess
 import os
 
@@ -107,6 +108,30 @@ class TestCfgGen(TestCase):
         output = self.run_script(argument)
         self.assertEqual(output.strip(), 'value1\nvalue2')
 
+    def test_template_batch_mode(self):
+        argument = '-y ' + os.path.join(self.test_dir, 'test.yml')
+        argument += ' -a \'{"key1":"value"}\''
+        argument += ' -t ' + os.path.join(self.test_dir, 'test.j2') + ',' + os.path.join(self.test_dir, 'test.txt')
+        argument += ' -t ' + os.path.join(self.test_dir, 'test2.j2') + ',' + os.path.join(self.test_dir, 'test2.txt')
+        output = self.run_script(argument)
+        assert(os.path.exists(os.path.join(self.test_dir, 'test.txt')))
+        assert(os.path.exists(os.path.join(self.test_dir, 'test2.txt')))
+        with open(os.path.join(self.test_dir, 'test.txt')) as tf:
+            self.assertEqual(tf.read().strip(), 'value1\nvalue2')
+        with open(os.path.join(self.test_dir, 'test2.txt')) as tf:
+            self.assertEqual(tf.read().strip(), 'value')
+
+    def test_template_json_batch_mode(self):
+        data = {"key1_1":"value1_1", "key1_2":"value1_2", "key2_1":"value2_1", "key2_2":"value2_2"}
+        argument = " -a '{0}'".format(repr(data).replace('\'', '"'))
+        argument += ' -t ' + os.path.join(self.test_dir, 'sample-template-1.json.j2') + ",config-db"
+        argument += ' -t ' + os.path.join(self.test_dir, 'sample-template-2.json.j2') + ",config-db"
+        argument += ' --print-data'
+        output = self.run_script(argument)
+        output_data = json.loads(output)
+        for key, value in data.items():
+            self.assertEqual(output_data[key.replace("key", "jk")], value)
+
     # FIXME: This test depends heavily on the ordering of the interfaces and
     # it is not at all intuitive what that ordering should be. Could make it
     # more robust by adding better parsing logic.