From bc58c12ed87ae76ec766f41eae4ffe2071613c60 Mon Sep 17 00:00:00 2001 From: Vivek Date: Mon, 15 May 2023 06:47:24 -0400 Subject: [PATCH] [Mellanox] Add patch commit-id mapping to description (#15052) - Why I did it Add the commit-id patch map in the commit message. - How I did it By parsing the patch DB from hw-mgmt Signed-off-by: Vivek Reddy Karri --- platform/mellanox/integration-scripts.mk | 16 +- .../hwmgmt_kernel_patches.py | 146 ++++++++++++++++++ .../tests/data/Patch_Status_Table.txt | 57 +++++++ .../tests/test_hwmgmtaction.py | 34 ++++ 4 files changed, 249 insertions(+), 4 deletions(-) create mode 100644 platform/mellanox/integration-scripts/tests/data/Patch_Status_Table.txt diff --git a/platform/mellanox/integration-scripts.mk b/platform/mellanox/integration-scripts.mk index 9268880842..5c5091098c 100644 --- a/platform/mellanox/integration-scripts.mk +++ b/platform/mellanox/integration-scripts.mk @@ -29,6 +29,8 @@ KCFG_LIST = $(TEMP_HW_MGMT_DIR)/kconfig HWMGMT_NONUP_LIST = $(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hwmgmt_nonup_patches HWMGMT_USER_OUTFILE = $(BUILD_WORKDIR)/integrate-mlnx-hw-mgmt_user.out TMPFILE_OUT := $(shell mktemp) +SB_COM_MSG := $(shell mktemp -t sb_commit_msg_file_XXXXX.log) +SLK_COM_MSG := $(shell mktemp -t slk_commit_msg_file_XXXXX.log) SB_HEAD = $(shell git rev-parse --short HEAD) SLK_HEAD = $(shell cd src/sonic-linux-kernel; git rev-parse --short HEAD) @@ -66,7 +68,9 @@ endif # Pre-processing before runing hw_mgmt script integration-scripts/hwmgmt_kernel_patches.py pre \ --config_inclusion $(KCFG_LIST) \ - --build_root $(BUILD_WORKDIR) $(LOG_SIMPLE) + --build_root $(BUILD_WORKDIR) \ + --kernel_version $(KERNEL_VERSION) \ + --hw_mgmt_ver ${MLNX_HW_MANAGEMENT_VERSION} $(LOG_SIMPLE) $(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hw-mgmt/recipes-kernel/linux/deploy_kernel_patches.py \ --dst_accepted_folder $(PTCH_DIR) \ @@ -80,10 +84,14 @@ endif integration-scripts/hwmgmt_kernel_patches.py post \ --patches $(PTCH_DIR) \ --non_up_patches $(NON_UP_PTCH_DIR) \ + --kernel_version $(KERNEL_VERSION) \ + --hw_mgmt_ver ${MLNX_HW_MANAGEMENT_VERSION} \ --config_inclusion $(KCFG_LIST) \ --series $(PTCH_LIST) \ --current_non_up_patches $(HWMGMT_NONUP_LIST) \ - --build_root $(BUILD_WORKDIR) $(LOG_SIMPLE) + --build_root $(BUILD_WORKDIR) \ + --sb_msg $(SB_COM_MSG) \ + --slk_msg $(SLK_COM_MSG) $(LOG_SIMPLE) # Commit the changes in linux kernel and and log the diff pushd $(BUILD_WORKDIR)/src/sonic-linux-kernel @@ -102,7 +110,7 @@ endif git diff --no-color --staged --stat --output=${TMPFILE_OUT} cat ${TMPFILE_OUT} | tee -a ${HWMGMT_USER_OUTFILE} - git diff --staged --quiet || git commit -m "Intgerate HW-MGMT ${MLNX_HW_MANAGEMENT_VERSION} Changes"; + git diff --staged --quiet || git commit -m "$$(cat $(SLK_COM_MSG))"; popd # Commit the changes in buildimage and log the diff @@ -127,7 +135,7 @@ endif git diff --no-color --staged --stat --output=${TMPFILE_OUT} -- $(PLATFORM_PATH) cat ${TMPFILE_OUT} | tee -a ${HWMGMT_USER_OUTFILE} - git diff --staged --quiet || git commit -m "Intgerate HW-MGMT ${MLNX_HW_MANAGEMENT_VERSION} Changes"; + git diff --staged --quiet || git commit -m "$$(cat $(SB_COM_MSG))"; popd popd $(LOG_SIMPLE) diff --git a/platform/mellanox/integration-scripts/hwmgmt_kernel_patches.py b/platform/mellanox/integration-scripts/hwmgmt_kernel_patches.py index bd7c1ac13e..0e4bfa84b4 100755 --- a/platform/mellanox/integration-scripts/hwmgmt_kernel_patches.py +++ b/platform/mellanox/integration-scripts/hwmgmt_kernel_patches.py @@ -9,11 +9,107 @@ import difflib from helper import * +COMMIT_TITLE = "Intgerate HW-MGMT {} Changes" + +PATCH_TABLE_LOC = "platform/mellanox/hw-management/hw-mgmt/recipes-kernel/linux/" +PATCH_TABLE_NAME = "Patch_Status_Table.txt" +PATCH_TABLE_DELIMITER = "----------------------" +PATCH_NAME = "patch name" +COMMIT_ID = "Upstream commit id" + +def trim_array_str(str_list): + ret = [elem.strip() for elem in str_list] + return ret + +def get_line_elements(line): + columns_raw = line.split("|") + if len(columns_raw) < 3: + return False + # remove empty firsta and last elem + columns_raw = columns_raw[1:-1] + columns = trim_array_str(columns_raw) + return columns + +def load_patch_table(path, k_version): + patch_table_filename = os.path.join(path, PATCH_TABLE_NAME) + + major, minor, subversion = k_version.split(".") + k_ver = "{}.{}".format(major, minor) + + print("Loading patch table {} kver:{}".format(patch_table_filename, k_ver)) + + if not os.path.isfile(patch_table_filename): + print("-> ERR: file {} not found".format(patch_table_filename)) + return None + + # opening the file + patch_table_file = open(patch_table_filename, "r") + # reading the data from the file + patch_table_data = patch_table_file.read() + # splitting the file data into lines + patch_table_lines = patch_table_data.splitlines() + patch_table_file.close() + + # Extract patch table for specified kernel version + kversion_line = "Kernel-{}".format(k_ver) + table_ofset = 0 + for table_ofset, line in enumerate(patch_table_lines): + if line == kversion_line: + break + + # if kernel version not found + if table_ofset >= len(patch_table_lines)-5: + print ("Err: kernel version {} not found in {}".format(k_ver, patch_table_filename)) + return None + + table = [] + delimiter_count = 0 + column_names = None + for idx, line in enumerate(patch_table_lines[table_ofset:]): + if PATCH_TABLE_DELIMITER in line: + delimiter_count += 1 + if delimiter_count >= 3: + print ("Err: too much leading delimers line #{}: {}".format(table_ofset + idx, line)) + return None + elif table: + break + continue + + # line without delimiter but header still not found + if delimiter_count > 0: + if not column_names: + column_names = get_line_elements(line) + if not column_names: + print ("Err: parsing table header line #{}: {}".format(table_ofset + idx, line)) + return None + delimiter_count = 0 + continue + elif column_names: + line_arr = get_line_elements(line) + if len(line_arr) != len(column_names): + print ("Err: patch table wrong format linex #{}: {}".format(table_ofset + idx, line)) + return None + else: + table_line = dict(zip(column_names, line_arr)) + table.append(table_line) + return table + +def build_commit_description(changes): + if not changes: + return "" + content = "\n" + content = content + " ## Patch List\n" + for key, value in changes.items(): + content = content + f"* {key} : {value}\n" + return content + class Data: # list of new upstream patches new_up = list() # list of new non-upstream patches new_non_up = list() + # old upstream patches + old_up_patches = list() # current series file raw data old_series = list() # current non-upstream patch list @@ -275,6 +371,39 @@ class PostProcess(HwMgmtAction): FileHandler.write_lines(os.path.join(self.args.build_root, SLK_KCONFIG_EXCLUDE), new_lines, True) print("-> INFO: updated kconfig-exclusion: \n{}".format("".join(FileHandler.read_raw(os.path.join(self.args.build_root, SLK_KCONFIG_EXCLUDE))))) + def list_patches(self): + old_up_patches = [] + for i in range(Data.i_mlnx_start, Data.i_mlnx_end): + old_up_patches.append(Data.old_series[i].strip()) + old_non_up_patches = [ptch.strip() for ptch in Data.old_non_up] + return old_up_patches, old_non_up_patches + + def parse_id(self, id_): + if id_: + id_ = "https://github.com/gregkh/linux/commit/" + id_ + return id_ + + def create_commit_msg(self, table): + title = COMMIT_TITLE.format(self.args.hw_mgmt_ver) + changes_slk, changes_sb = {}, {} + old_up_patches, old_non_up_patches = self.list_patches() + for patch in table: + id_ = self.parse_id(patch.get(COMMIT_ID, "")) + patch_ = patch.get(PATCH_NAME) + if patch_ in Data.new_up and patch_ not in old_up_patches: + changes_slk[patch_] = id_ + print(f"-> INFO: Patch: {patch_}, Commit: {id_}, added to linux-kernel description") + elif patch_ in Data.new_non_up and patch_ not in old_non_up_patches: + changes_sb[patch_] = id_ + print(f"-> INFO: Patch: {patch_}, Commit: {id_}, added to buildimage description") + else: + print(f"-> INFO: Patch: {patch_}, Commit: {id_}, is not added") + slk_commit_msg = title + "\n" + build_commit_description(changes_slk) + sb_commit_msg = title + "\n" + build_commit_description(changes_sb) + print(f"-> INFO: SLK Commit Message: \n {slk_commit_msg}") + print(f"-> INFO: SB Commit Message: \n {sb_commit_msg}") + return sb_commit_msg, slk_commit_msg + def perform(self): """ Read the data output from the deploy_kernel_patches.py script and move to appropriate locations """ @@ -299,6 +428,18 @@ class PostProcess(HwMgmtAction): self.construct_series_with_non_up() self.write_series_diff() + path = os.path.join(self.args.build_root, PATCH_TABLE_LOC) + patch_table = load_patch_table(path, self.args.kernel_version) + + sb_msg, slk_msg = self.create_commit_msg(patch_table) + + if self.args.sb_msg and sb_msg: + with open(self.args.sb_msg, 'w') as f: + f.write(sb_msg) + + if self.args.slk_msg: + with open(self.args.slk_msg, 'w') as f: + f.write(slk_msg) def create_parser(): # Create argument parser @@ -314,6 +455,10 @@ def create_parser(): parser.add_argument("--series", type=str) parser.add_argument("--current_non_up_patches", type=str) parser.add_argument("--build_root", type=str) + parser.add_argument("--hw_mgmt_ver", type=str, required=True) + parser.add_argument("--kernel_version", type=str, required=True) + parser.add_argument("--sb_msg", type=str, required=False, default="") + parser.add_argument("--slk_msg", type=str, required=False, default="") parser.add_argument("--is_test", action="store_true") return parser @@ -322,3 +467,4 @@ if __name__ == '__main__': parser = create_parser() action = HwMgmtAction.get(parser.parse_args()) action.perform() + diff --git a/platform/mellanox/integration-scripts/tests/data/Patch_Status_Table.txt b/platform/mellanox/integration-scripts/tests/data/Patch_Status_Table.txt new file mode 100644 index 0000000000..8c37b515bf --- /dev/null +++ b/platform/mellanox/integration-scripts/tests/data/Patch_Status_Table.txt @@ -0,0 +1,57 @@ +Kernel-5.10 +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +| patch name | Upstream commit id | status | subversion | Notes | +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +|0001-i2c-mlxcpld-Update-module-license.patch | f069291bd5fc | Feature upstream | | | +|0002-i2c-mlxcpld-Decrease-polling-time-for-performance-im.patch | cb9744178f33 | Feature upstream | | | +|0003-i2c-mlxcpld-Add-support-for-I2C-bus-frequency-settin.patch | 66b0c2846ba8 | Feature upstream | | | +|0004-i2c-mux-mlxcpld-Update-module-license.patch | 337bc68c294d | Feature upstream | | | +|0005-i2c-mux-mlxcpld-Move-header-file-out-of-x86-realm.patch | 98d29c410475 | Feature upstream | | | +|0006-i2c-mux-mlxcpld-Convert-driver-to-platform-driver.patch | 84af1b168c50 | Feature upstream | | | +|0007-i2c-mux-mlxcpld-Prepare-mux-selection-infrastructure.patch | 81566938083a | Feature upstream | | | +|0008-i2c-mux-mlxcpld-Get-rid-of-adapter-numbers-enforceme.patch | cae5216387d1 | Feature upstream | | | +|0009-i2c-mux-mlxcpld-Extend-driver-to-support-word-addres.patch | c52a1c5f5db5 | Feature upstream | | | +|0010-i2c-mux-mlxcpld-Extend-supported-mux-number.patch | 699c0506543e | Feature upstream | | | +|0011-i2c-mux-mlxcpld-Add-callback-to-notify-mux-creation-.patch | a39bd92e92b9 | Feature upstream | | | +|0012-hwmon-mlxreg-fan-Add-support-for-fan-drawers-capabil.patch | f7bf7eb2d734 | Feature upstream | | | +|0013-hwmon-pmbus-shrink-code-and-remove-pmbus_do_remove.patch | 3bce071a301f | Feature upstream; os[sonic,opt] | | | +|0014-thermal-drivers-core-Use-a-char-pointer-for-the-cool.patch | 584837618100 | Bugfix upstream | 5.10.121 | | +|0015-mlxsw-core-Remove-critical-trip-points-from-thermal-.patch | d567fd6e82fa | Feature upstream | | | +|0016-net-don-t-include-ethtool.h-from-netdevice.h.patch | cc69837fcaf4 | Feature upstream | | | +|0017-mlxsw-reg-Extend-MTMP-register-with-new-threshold-fi.patch | 314dbb19f95b | Feature upstream | | | +|0018-mlxsw-thermal-Add-function-for-reading-module-temper.patch | e57977b34ab5 | Feature upstream | | | +|0019-mlxsw-thermal-Read-module-temperature-thresholds-usi.patch | 72a64c2fe9d8 | Feature upstream | | | +|0020-mlxsw-thermal-Fix-null-dereference-of-NULL-temperatu.patch | f3b5a8907543 | Feature upstream | | | +|0021-mlxsw-reg-Add-bank-number-to-MCIA-register.patch | d51ea60e01f9 | Feature upstream | | | +|0022-mlxsw-reg-Document-possible-MCIA-status-values.patch | cecefb3a6eeb | Feature upstream | | | +|0023-ethtool-Allow-network-drivers-to-dump-arbitrary-EEPR.patch | c781ff12a2f3 | Feature upstream | | | +|0024-net-ethtool-Export-helpers-for-getting-EEPROM-info.patch | 95dfc7effd88 | Feature upstream | | | +|0025-ethtool-Add-fallback-to-get_module_eeprom-from-netli.patch | 96d971e307cc | Feature upstream | | | +|0026-mlxsw-core-Add-support-for-module-EEPROM-read-by-pag.patch | 1e27b9e40803 | Feature upstream | | | +|0027-ethtool-Decrease-size-of-module-EEPROM-get-policy-ar.patch | f5fe211d13af | Feature upstream | | | +|0028-ethtool-Use-kernel-data-types-for-internal-EEPROM-st.patch | b8c48be23c2d | Feature upstream | | | +|0029-ethtool-Validate-module-EEPROM-length-as-part-of-pol.patch | 0dc7dd02ba7a | Feature upstream | | | +|0030-ethtool-Validate-module-EEPROM-offset-as-part-of-pol.patch | 88f9a87afeee | Feature upstream | | | +|0031-mlxsw-core_env-Read-module-temperature-thresholds-us.patch | befc2048088a | Feature upstream | | | +|0032-mlxsw-core_env-Avoid-unnecessary-memcpy-s.patch | 911bd1b1f08f | Feature upstream | | | +|0033-mlxsw-core-Set-thermal-zone-polling-delay-argument-t.patch | 2fd8d84ce309 | Bugfix upstream | 5.10.46 | | +|0034-mlxsw-Verify-the-accessed-index-doesn-t-exceed-the-a.patch | 0c1acde1d3d0 | Bugfix upstream | 5.10.83 | | +|0035-hwmon-pmbus-Increase-maximum-number-of-phases-per-pa.patch | e4db7719d037 | Feature upstream | | | +|0036-hwmon-pmbus-Add-support-for-MPS-Multi-phase-mp2888-c.patch | 0c1acde1d3d0 | Feature upstream; os[sonic,opt] | | | +|0037-dt-bindings-Add-MP2888-voltage-regulator-device.patch | 9abfb52b5028 | Feature upstream | | | +|0038-ethtool-wire-in-generic-SFP-module-access.patch | c97a31f66ebc | Feature upstream | | | +|0039-ethtool-Fix-NULL-pointer-dereference-during-module-E.patch | 51c96a561f24 | Feature upstream | | | +|0040-phy-sfp-add-netlink-SFP-support-to-generic-SFP-code.patch | d740513f05a2 | Feature upstream | | | +|0041-net-ethtool-clear-heap-allocations-for-ethtool-funct.patch | 80ec82e3d2c1 | Bugfix upstream | 5.10.47 | | +|0042-ethtool-support-FEC-settings-over-netlink.patch | 1e5d1f69d9fb | Feature upstream | | | +|0043-platform-mellanox-mlxreg-io-Fix-argument-base-in-kst.patch | 452dcfab9954 | Bugfix upstream | 5.10.75 | | +|0044-platform-mellanox-mlxreg-io-Fix-read-access-of-n-byt.patch | 5fd56f11838d | Bugfix upstream | 5.10.75 | | +|0045-i2c-mlxcpld-Fix-criteria-for-frequency-setting.patch | 52f57396c75a | Feature upstream | | | +|0046-i2c-mlxcpld-Reduce-polling-time-for-performance-impr.patch | 669b2e4aa1a8 | Feature upstream | | (5.16) | +|0047-i2c-mlxcpld-Allow-flexible-polling-time-setting-for-.patch | 712d6617d0a2 | Feature upstream | | (5.16) | +|0048-hwmon-pmbus-mp2975-Add-missed-POUT-attribute-for-pag.patch | 2292e2f685cd | Bugfix upstream | 5.10.71 | | +|0049-leds-mlxreg-Provide-conversion-for-hardware-LED-colo.patch | | Rejected; take[ALL] | |Need to check patch apply. Can break patch apply| +|0050-leds-mlxreg-Skip-setting-LED-color-during-initializa.patch | | Downstream | | | +|0051-leds-mlxreg-Allow-multi-instantiation-of-same-name-L.patch | | Downstream | | Modular SN4800 | +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + diff --git a/platform/mellanox/integration-scripts/tests/test_hwmgmtaction.py b/platform/mellanox/integration-scripts/tests/test_hwmgmtaction.py index 6dd71ee0e2..e95d10a44e 100644 --- a/platform/mellanox/integration-scripts/tests/test_hwmgmtaction.py +++ b/platform/mellanox/integration-scripts/tests/test_hwmgmtaction.py @@ -34,6 +34,27 @@ NEW_UP_LIST = """\ 0107-mlxsw-reg-Extend-MCION-register-with-new-slot-number.patch """ +TEST_SLK_COMMIT = """\ +Intgerate HW-MGMT 7.0030.0937 Changes + ## Patch List +* 0002-i2c-mlxcpld-Decrease-polling-time-for-performance-im.patch : https://github.com/gregkh/linux/commit/cb9744178f33 +* 0003-i2c-mlxcpld-Add-support-for-I2C-bus-frequency-settin.patch : https://github.com/gregkh/linux/commit/66b0c2846ba8 +* 0005-i2c-mux-mlxcpld-Move-header-file-out-of-x86-realm.patch : https://github.com/gregkh/linux/commit/98d29c410475 +* 0006-i2c-mux-mlxcpld-Convert-driver-to-platform-driver.patch : https://github.com/gregkh/linux/commit/84af1b168c50 +* 0007-i2c-mux-mlxcpld-Prepare-mux-selection-infrastructure.patch : https://github.com/gregkh/linux/commit/81566938083a +* 0008-i2c-mux-mlxcpld-Get-rid-of-adapter-numbers-enforceme.patch : https://github.com/gregkh/linux/commit/cae5216387d1 +* 0009-i2c-mux-mlxcpld-Extend-driver-to-support-word-addres.patch : https://github.com/gregkh/linux/commit/c52a1c5f5db5 +* 0010-i2c-mux-mlxcpld-Extend-supported-mux-number.patch : https://github.com/gregkh/linux/commit/699c0506543e +* 0011-i2c-mux-mlxcpld-Add-callback-to-notify-mux-creation-.patch : https://github.com/gregkh/linux/commit/a39bd92e92b9 + +""" + + +TEST_SB_COMMIT = """\ +Intgerate HW-MGMT 7.0030.0937 Changes + +""" + REL_INPUTS_DIR = "platform/mellanox/integration-scripts/tests/data/" MOCK_INPUTS_DIR = "/sonic/" + REL_INPUTS_DIR MOCK_WRITE_FILE = MOCK_INPUTS_DIR + "test_writer_file.out" @@ -54,6 +75,10 @@ def mock_hwmgmt_args(): "--config_inclusion", MOCK_INPUTS_DIR+"/new_kconfig", "--series", MOCK_INPUTS_DIR+"/new_series", "--current_non_up_patches", MOCK_INPUTS_DIR+"/hwmgmt_nonup_patches", + "--kernel_version", "5.10.140", + "--hw_mgmt_ver", "7.0030.0937", + "--sb_msg", "/tmp/sb_msg.log", + "--slk_msg", "/tmp/slk_msg.log", "--build_root", "/sonic", "--is_test"]): parser = create_parser() @@ -142,3 +167,12 @@ class TestHwMgmtPostAction(TestCase): self.action.construct_series_with_non_up() self.action.write_series_diff() assert check_file_content(MOCK_INPUTS_DIR+"expected_data/series.patch") + + def test_commit_msg(self): + table = load_patch_table(MOCK_INPUTS_DIR, "5.10.140") + sb, slk = self.action.create_commit_msg(table) + print(slk) + print(TEST_SLK_COMMIT) + assert slk.split() == TEST_SLK_COMMIT.split() + assert sb.split() == TEST_SB_COMMIT.split() +