From f9edf6e5ccb0f2c8f67a7c052ea205752957dfc6 Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Wed, 12 Aug 2020 15:13:06 -0700 Subject: [PATCH] [cfggen] Extend Template Argument to Support Batch Mode (#4941) Calls to cfggen take considerable time. With batch mode, we will have the ability to reduce number of calls from services. Example of the batch mode command: sonic-cfggen -t template-1.j2 -t template-2.j2,config-db -t template-3.j2,config-db -t template-4.j2,file1 -t template-5.j2,file2 --write-to-db. template-1.j2 will be rendered to stdout since it is missing the dest part. stdout is default config-db is a special keyword that will inject the rendered template into internal data structure. The internal data structure gets written to redis-db with --write-to-db switch. In the case the user would like to write to a file named config-db, it could be given as /config-db or ./config-db signed-off-by: Tamer Ahmed --- src/sonic-config-engine/sonic-cfggen | 90 +++++++++++++------ .../tests/sample-template-1.json.j2 | 4 + .../tests/sample-template-2.json.j2 | 4 + src/sonic-config-engine/tests/test2.j2 | 1 + src/sonic-config-engine/tests/test_cfggen.py | 25 ++++++ 5 files changed, 96 insertions(+), 28 deletions(-) create mode 100644 src/sonic-config-engine/tests/sample-template-1.json.j2 create mode 100644 src/sonic-config-engine/tests/sample-template-2.json.j2 create mode 100644 src/sonic-config-engine/tests/test2.j2 diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index b062fbda6b..9a65343a99 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -35,6 +35,7 @@ import yaml import jinja2 import netaddr import json +import contextlib from functools import partial from minigraph import minigraph_encoder from minigraph import parse_xml @@ -204,6 +205,45 @@ def sort_data(data): data[table] = OrderedDict(natsorted(data[table].items())) return data +@contextlib.contextmanager +def smart_open(filename=None, mode=None): + """ + Provide contextual file descriptor of filename if it is not a file descriptor + """ + smart_file = open(filename, mode) if not isinstance(filename, file) else filename + try: + yield smart_file + finally: + if not isinstance(filename, file): + smart_file.close() + +def _process_json(args, data): + """ + Process JSON file and update switch configuration data + """ + for json_file in args.json: + with open(json_file, 'r') as stream: + deep_update(data, FormatConverter.to_deserialized(json.load(stream))) + +def _get_jinja2_env(paths): + """ + Retreive Jinj2 env used to render configuration templates + """ + loader = jinja2.FileSystemLoader(paths) + redis_bcc = RedisBytecodeCache(SonicV2Connector(host='127.0.0.1')) + env = jinja2.Environment(loader=loader, trim_blocks=True, bytecode_cache=redis_bcc) + env.filters['sort_by_port_index'] = sort_by_port_index + env.filters['ipv4'] = is_ipv4 + env.filters['ipv6'] = is_ipv6 + env.filters['unique_name'] = unique_name + env.filters['pfx_filter'] = pfx_filter + env.filters['ip_network'] = ip_network + for attr in ['ip', 'network', 'prefixlen', 'netmask', 'broadcast']: + env.filters[attr] = partial(prefix_attr, attr) + # Pass the is_multi_npu function as global + env.globals['multi_asic'] = is_multi_npu + + return env def main(): parser=argparse.ArgumentParser(description="Render configuration file from minigraph data and jinja2 template.") @@ -221,14 +261,15 @@ def main(): parser.add_argument("-H", "--platform-info", help="read platform and hardware info", action='store_true') parser.add_argument("-s", "--redis-unix-sock-file", help="unix sock file for redis connection") group = parser.add_mutually_exclusive_group() - group.add_argument("-t", "--template", help="render the data with the template file") + group.add_argument("-t", "--template", help="render the data with the template file", action="append", default=[], + type=lambda opt_value: tuple(opt_value.split(',')) if ',' in opt_value else (opt_value, sys.stdout)) parser.add_argument("-T", "--template_dir", help="search base for the template files", action='store') group.add_argument("-v", "--var", help="print the value of a variable, support jinja2 expression") group.add_argument("--var-json", help="print the value of a variable, in json format") group.add_argument("-w", "--write-to-db", help="write config into configdb", action='store_true') - group.add_argument("--print-data", help="print all data", action='store_true') group.add_argument("--preset", help="generate sample configuration from a preset template", choices=get_available_config()) group = parser.add_mutually_exclusive_group() + group.add_argument("--print-data", help="print all data", action='store_true') group.add_argument("-K", "--key", help="Lookup for a specific key") args = parser.parse_args() @@ -267,9 +308,7 @@ def main(): if brkout_table is not None: deep_update(data, {'BREAKOUT_CFG': brkout_table}) - for json_file in args.json: - with open(json_file, 'r') as stream: - deep_update(data, FormatConverter.to_deserialized(json.load(stream))) + _process_json(args, data) if args.minigraph != None: minigraph = args.minigraph @@ -297,7 +336,7 @@ def main(): if args.from_db: if args.namespace is None: - configdb = ConfigDBConnector(**db_kwargs) + configdb = ConfigDBConnector(use_unix_socket_path=True, **db_kwargs) else: configdb = ConfigDBConnector(use_unix_socket_path=True, namespace=args.namespace, **db_kwargs) @@ -328,28 +367,23 @@ def main(): hardware_data['DEVICE_METADATA']['localhost'].update(asic_id=asic_id) deep_update(data, hardware_data) - if args.template is not None: - template_file = os.path.abspath(args.template) - paths = ['/', '/usr/share/sonic/templates', os.path.dirname(template_file)] - if args.template_dir is not None: - template_dir = os.path.abspath(args.template_dir) - paths.append(template_dir) - loader = jinja2.FileSystemLoader(paths) + paths = ['/', '/usr/share/sonic/templates'] + if args.template_dir: + paths.append(os.path.abspath(args.template_dir)) - redis_bcc = RedisBytecodeCache(SonicV2Connector(host='127.0.0.1')) - env = jinja2.Environment(loader=loader, trim_blocks=True, bytecode_cache=redis_bcc) - env.filters['sort_by_port_index'] = sort_by_port_index - env.filters['ipv4'] = is_ipv4 - env.filters['ipv6'] = is_ipv6 - env.filters['unique_name'] = unique_name - env.filters['pfx_filter'] = pfx_filter - env.filters['ip_network'] = ip_network - for attr in ['ip', 'network', 'prefixlen', 'netmask', 'broadcast']: - env.filters[attr] = partial(prefix_attr, attr) - # Pass the is_multi_npu function as global - env.globals['multi_asic'] = is_multi_npu - template = env.get_template(template_file) - print(template.render(sort_data(data))) + if args.template: + for template_file, _ in args.template: + paths.append(os.path.dirname(os.path.abspath(template_file))) + env = _get_jinja2_env(paths) + sorted_data = sort_data(data) + for template_file, dest_file in args.template: + template = env.get_template(os.path.basename(template_file)) + template_data = template.render(sorted_data) + if dest_file == "config-db": + deep_update(data, FormatConverter.to_deserialized(json.loads(template_data))) + else: + with smart_open(dest_file, 'w') as df: + print(template_data, file=df) if args.var != None: template = jinja2.Template('{{' + args.var + '}}') @@ -363,7 +397,7 @@ def main(): if args.write_to_db: if args.namespace is None: - configdb = ConfigDBConnector(**db_kwargs) + configdb = ConfigDBConnector(use_unix_socket_path=True, **db_kwargs) else: configdb = ConfigDBConnector(use_unix_socket_path=True, namespace=args.namespace, **db_kwargs) diff --git a/src/sonic-config-engine/tests/sample-template-1.json.j2 b/src/sonic-config-engine/tests/sample-template-1.json.j2 new file mode 100644 index 0000000000..96051b502f --- /dev/null +++ b/src/sonic-config-engine/tests/sample-template-1.json.j2 @@ -0,0 +1,4 @@ +{ + "jk1_1": "{{ key1_1 }}", + "jk1_2": "{{ key1_2 }}" +} diff --git a/src/sonic-config-engine/tests/sample-template-2.json.j2 b/src/sonic-config-engine/tests/sample-template-2.json.j2 new file mode 100644 index 0000000000..3de91ab36e --- /dev/null +++ b/src/sonic-config-engine/tests/sample-template-2.json.j2 @@ -0,0 +1,4 @@ +{ + "jk2_1": "{{ key2_1 }}", + "jk2_2": "{{ key2_2 }}" +} diff --git a/src/sonic-config-engine/tests/test2.j2 b/src/sonic-config-engine/tests/test2.j2 new file mode 100644 index 0000000000..df36b83cf3 --- /dev/null +++ b/src/sonic-config-engine/tests/test2.j2 @@ -0,0 +1 @@ +{{ key1 }} diff --git a/src/sonic-config-engine/tests/test_cfggen.py b/src/sonic-config-engine/tests/test_cfggen.py index 449ba8d01d..2c0368c9f1 100644 --- a/src/sonic-config-engine/tests/test_cfggen.py +++ b/src/sonic-config-engine/tests/test_cfggen.py @@ -1,4 +1,5 @@ from unittest import TestCase +import json import subprocess import os @@ -107,6 +108,30 @@ class TestCfgGen(TestCase): output = self.run_script(argument) self.assertEqual(output.strip(), 'value1\nvalue2') + def test_template_batch_mode(self): + argument = '-y ' + os.path.join(self.test_dir, 'test.yml') + argument += ' -a \'{"key1":"value"}\'' + argument += ' -t ' + os.path.join(self.test_dir, 'test.j2') + ',' + os.path.join(self.test_dir, 'test.txt') + argument += ' -t ' + os.path.join(self.test_dir, 'test2.j2') + ',' + os.path.join(self.test_dir, 'test2.txt') + output = self.run_script(argument) + assert(os.path.exists(os.path.join(self.test_dir, 'test.txt'))) + assert(os.path.exists(os.path.join(self.test_dir, 'test2.txt'))) + with open(os.path.join(self.test_dir, 'test.txt')) as tf: + self.assertEqual(tf.read().strip(), 'value1\nvalue2') + with open(os.path.join(self.test_dir, 'test2.txt')) as tf: + self.assertEqual(tf.read().strip(), 'value') + + def test_template_json_batch_mode(self): + data = {"key1_1":"value1_1", "key1_2":"value1_2", "key2_1":"value2_1", "key2_2":"value2_2"} + argument = " -a '{0}'".format(repr(data).replace('\'', '"')) + argument += ' -t ' + os.path.join(self.test_dir, 'sample-template-1.json.j2') + ",config-db" + argument += ' -t ' + os.path.join(self.test_dir, 'sample-template-2.json.j2') + ",config-db" + argument += ' --print-data' + output = self.run_script(argument) + output_data = json.loads(output) + for key, value in data.items(): + self.assertEqual(output_data[key.replace("key", "jk")], value) + # FIXME: This test depends heavily on the ordering of the interfaces and # it is not at all intuitive what that ordering should be. Could make it # more robust by adding better parsing logic.