matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Mellanox] Enhance the processing of Kconfig in the hw-mgmt integration (#16752) (#17009)

Browse Source
This commit is contained in:
mssonicbld 2023-10-26 00:46:54 +08:00 committed by GitHub
parent 7982c3a2a8
commit da90d5624d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 755 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
platform/mellanox/integration-scripts.mk
View File

@ -25,7 +25,6 @@ TEMP_HW_MGMT_DIR = /tmp/hw_mgmt
PTCH_DIR = $(TEMP_HW_MGMT_DIR)/patch_dir/
NON_UP_PTCH_DIR = $(TEMP_HW_MGMT_DIR)/non_up_patch_dir/
PTCH_LIST = $(TEMP_HW_MGMT_DIR)/series
KCFG_LIST = $(TEMP_HW_MGMT_DIR)/kconfig
HWMGMT_NONUP_LIST = $(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hwmgmt_nonup_patches
HWMGMT_USER_OUTFILE = $(BUILD_WORKDIR)/integrate-mlnx-hw-mgmt_user.out
SDK_USER_OUTFILE = $(BUILD_WORKDIR)/integrate-mlnx-sdk_user.out
@ -36,11 +35,33 @@ SLK_COM_MSG := $(shell mktemp -t slk_commit_msg_file_XXXXX.log)
SB_HEAD = $(shell git rev-parse --short HEAD)
SLK_HEAD = $(shell cd src/sonic-linux-kernel; git rev-parse --short HEAD)
# kconfig related variables
KCFG_BASE_TMPDIR = $(TEMP_HW_MGMT_DIR)/linux_kconfig/
KCFG_BASE = $(KCFG_BASE_TMPDIR)/amd64.config
KCFG_LIST = $(TEMP_HW_MGMT_DIR)/kconfig_amd64
KCFG_DOWN_LIST = $(TEMP_HW_MGMT_DIR)/kconfig_downstream_amd64
KCFG_BASE_ARM = $(KCFG_BASE_TMPDIR)/arm64.config
KCFG_LIST_ARM = $(TEMP_HW_MGMT_DIR)/kconfig_arm64
KCFG_DOWN_LIST_ARM = $(TEMP_HW_MGMT_DIR)/kconfig_downstream_arm64
integrate-mlnx-hw-mgmt:
$(FLUSH_LOG)
rm -rf $(TEMP_HW_MGMT_DIR) $(TMPFILE_OUT)
mkdir -p $(PTCH_DIR) $(NON_UP_PTCH_DIR)
touch $(PTCH_LIST) $(KCFG_LIST)
mkdir -p $(PTCH_DIR) $(NON_UP_PTCH_DIR) $(KCFG_BASE_TMPDIR)
touch $(PTCH_LIST) $(KCFG_LIST) $(KCFG_DOWN_LIST) $(KCFG_LIST_ARM) $(KCFG_DOWN_LIST_ARM)
# Fetch the vanilla .config files
pushd $(KCFG_BASE_TMPDIR) $(LOG_SIMPLE)
rm -rf linux/; mkdir linux
# Note: gregkh is the stable linux mirror
git clone --depth 1 --branch v$(KERNEL_VERSION) https://github.com/gregkh/linux.git linux $(LOG_SIMPLE)
pushd linux
rm -rf .config; make ARCH=x86_64 defconfig; cp -f .config $(KCFG_BASE) $(LOG_SIMPLE)
rm -rf .config; make ARCH=arm64 defconfig; cp -f .config $(KCFG_BASE_ARM) $(LOG_SIMPLE)
popd
popd $(LOG_SIMPLE)
# clean up existing untracked files
pushd $(BUILD_WORKDIR); git clean -f -- platform/mellanox/
@ -69,17 +90,32 @@ endif
# Pre-processing before runing hw_mgmt script
integration-scripts/hwmgmt_kernel_patches.py pre \
--config_inclusion $(KCFG_LIST) \
--config_base_amd $(KCFG_BASE) \
--config_base_arm $(KCFG_BASE_ARM) \
--config_inc_amd $(KCFG_LIST) \
--config_inc_arm $(KCFG_LIST_ARM) \
--build_root $(BUILD_WORKDIR) \
--kernel_version $(KERNEL_VERSION) \
--hw_mgmt_ver ${MLNX_HW_MANAGEMENT_VERSION} $(LOG_SIMPLE)
--hw_mgmt_ver ${MLNX_HW_MANAGEMENT_VERSION} $(LOG_SIMPLE)
$(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hw-mgmt/recipes-kernel/linux/deploy_kernel_patches.py \
--dst_accepted_folder $(PTCH_DIR) \
--dst_candidate_folder $(NON_UP_PTCH_DIR) \
--series_file $(PTCH_LIST) \
--config_file $(KCFG_LIST) \
--config_file $(KCFG_LIST_ARM) \
--config_file_downstream $(KCFG_DOWN_LIST_ARM) \
--kernel_version $(KERNEL_VERSION) \
--arch arm64 \
--os_type sonic $(LOG_SIMPLE)
$(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hw-mgmt/recipes-kernel/linux/deploy_kernel_patches.py \
--dst_accepted_folder $(PTCH_DIR) \
--dst_candidate_folder $(NON_UP_PTCH_DIR) \
--series_file $(PTCH_LIST) \
--config_file $(KCFG_LIST) \
--config_file_downstream $(KCFG_DOWN_LIST) \
--kernel_version $(KERNEL_VERSION) \
--arch amd64 \
--os_type sonic $(LOG_SIMPLE)
# Post-processing
@ -88,7 +124,12 @@ endif
--non_up_patches $(NON_UP_PTCH_DIR) \
--kernel_version $(KERNEL_VERSION) \
--hw_mgmt_ver ${MLNX_HW_MANAGEMENT_VERSION} \
--config_inclusion $(KCFG_LIST) \
--config_base_amd $(KCFG_BASE) \
--config_base_arm $(KCFG_BASE_ARM) \
--config_inc_amd $(KCFG_LIST) \
--config_inc_arm $(KCFG_LIST_ARM) \
--config_inc_down_amd $(KCFG_DOWN_LIST) \
--config_inc_down_arm $(KCFG_DOWN_LIST_ARM) \
--series $(PTCH_LIST) \
--current_non_up_patches $(HWMGMT_NONUP_LIST) \
--build_root $(BUILD_WORKDIR) \
@ -121,8 +162,11 @@ endif
git add -- $(PLATFORM_PATH)/non-upstream-patches/
git add -- $(PLATFORM_PATH)/hw-management.mk
echo -en '\n###-> Non Upstream series.patch changes <-###\n' >> ${HWMGMT_USER_OUTFILE}
git diff --no-color --staged -- $(PLATFORM_PATH)/non-upstream-patches/series.patch >> ${HWMGMT_USER_OUTFILE}
echo -en '\n###-> Non Upstream external-changes.patch changes <-###\n' >> ${HWMGMT_USER_OUTFILE}
git diff --no-color --staged -- $(PLATFORM_PATH)/non-upstream-patches/external-changes.patch >> ${HWMGMT_USER_OUTFILE}
echo -en '\n###-> Non Upstream kconfig-inclusions.patch changes <-###\n' >> ${HWMGMT_USER_OUTFILE}
git diff --no-color --staged -- $(PLATFORM_PATH)/non-upstream-patches/kconfig-inclusions.patch >> ${HWMGMT_USER_OUTFILE}
echo -en '\n###-> Non Upstream patch list file <-###\n' >> ${HWMGMT_USER_OUTFILE}
git diff --no-color --staged -- $($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hwmgmt_nonup_patches >> ${HWMGMT_USER_OUTFILE}
@ -142,6 +186,8 @@ endif
popd $(LOG_SIMPLE)
rm -rf $(TEMP_HW_MGMT_DIR)
integrate-mlnx-sdk:
$(FLUSH_LOG)
rm -rf $(MLNX_SDK_VERSION).zip sx_kernel-$(MLNX_SDK_VERSION)-$(MLNX_SDK_ISSU_VERSION).tar.gz

106
platform/mellanox/integration-scripts/helper.py
View File

@ -17,10 +17,12 @@
import os
import glob
import re
from collections import OrderedDict
MARK_ID = "###->"
MLNX_KFG_MARKER = "mellanox"
MLNX_KFG_MARKER = "mellanox_amd64"
MLNX_NOARCH_MARKER = "mellanox_common"
MLNX_ARM_KFG_SECTION = "mellanox-arm64"
SDK_MARKER = "mellanox_sdk"
HW_MGMT_MARKER = "mellanox_hw_mgmt"
SLK_PATCH_LOC = "src/sonic-linux-kernel/patch/"
@ -29,7 +31,7 @@ SLK_KCONFIG_EXCLUDE = SLK_PATCH_LOC + "kconfig-exclusions"
SLK_SERIES = SLK_PATCH_LOC + "series"
NON_UP_PATCH_DIR = "platform/mellanox/non-upstream-patches/"
NON_UP_PATCH_LOC = NON_UP_PATCH_DIR + "patches"
NON_UP_PATCH_DIFF = NON_UP_PATCH_DIR + "series.patch"
NON_UP_DIFF = NON_UP_PATCH_DIR + "external-changes.patch"
KCFG_HDR_RE = "\[(.*)\]"
KERNEL_BACKPORTS = "kernel_backports"
# kconfig_inclusion headers to consider
@ -100,83 +102,35 @@ class FileHandler:
return (i_start, i_end)
@staticmethod
def read_kconfig_inclusion(path, marker=MLNX_KFG_MARKER):
lines = FileHandler.read_strip(path)
if not marker:
return lines
i_start, i_end = FileHandler.find_marker_indices(lines, marker)
if i_start < 0 or i_end >= len(lines):
print("-> WARNING No Marker Found")
return []
def read_kconfig(path) -> dict:
# read the .config file generated during kernel compilation
lines = FileHandler.read_strip_minimal(path)
config_data = OrderedDict()
for line in lines:
if line.strip().startswith("#"):
continue
tokens = line.strip().split('=')
if len(tokens) == 2:
key = tokens[0].strip()
value = tokens[1].strip()
config_data[key] = value
return config_data
return lines[i_start+1:i_end]
@staticmethod
def insert_lines(lines: list, start: int, end: int, new_data: list) -> list:
return lines[0:start+1] + new_data + lines[end:]
@staticmethod
def write_lines_marker(path, writable_opts: list, marker=None):
# if marker is none, just write the opts into the file,
# otherwise write the data only b/w the marker
curr_data = FileHandler.read_raw(path)
i_start, i_end = FileHandler.find_marker_indices(curr_data, marker)
newline_writ_opts = [opt + "\n" for opt in writable_opts]
if i_start < 0 or i_end >= len(curr_data):
print("-> WARNING No Marker Found, writing data at the end of file")
curr_data.extend(["\n"])
curr_data.extend(newline_writ_opts)
else:
curr_data = curr_data[0:i_start+1] + newline_writ_opts + curr_data[i_end:]
print("-> INFO Written the following opts: \n{}".format("".join(FileHandler.read_raw(path))))
FileHandler.write_lines(path, curr_data, True)
def insert_kcfg_data(lines: list, start: int, end: int, new_data: OrderedDict) -> dict:
# inserts data into the lines, escape every lines
new_data_lines = ["{}={}\n".format(cfg, val) for (cfg, val) in new_data.items()]
return FileHandler.insert_lines(lines, start, end, new_data_lines)
@staticmethod
def read_kconfig_parser(path) -> dict:
# kconfig_inclusion output formatted to {"no_parent", "common":[,], "amd64": [,], "arm64": [,]}
lines = FileHandler.read_strip_minimal(path)
ret = dict({"no_parent":[]})
curr_hdr = ""
for line in lines:
match = re.search(KCFG_HDR_RE, line)
if match:
curr_hdr = match.group(1)
ret[curr_hdr] = []
elif curr_hdr in ret:
ret[curr_hdr].append(line)
else:
ret["no_parent"].append(line)
return ret
class KCFG:
@staticmethod
def parse_opt_str(opt: str) -> tuple:
if not opt.startswith("CONFIG"):
print("-> DEBUG: Malformed kconfig opt, {}".format(opt))
return ()
tmp = opt.split("=")
if len(tmp) != 2:
print("-> DEBUG: Malformed kconfig opt, {}".format(opt))
return ()
return (tmp[0], tmp[1])
@staticmethod
def parse_opts_strs(kcfg_sec: list) -> list(tuple()):
opts = [] # list of tuples (CONFIG_*, "m|y|n")
for kcfg in kcfg_sec:
tmp = KCFG.parse_opt_str(kcfg)
if tmp:
opts.append(tmp)
return opts
@staticmethod
def get_writable_opts(opts):
lines = []
for opt in opts:
lines.append("{}={}".format(opt[0], opt[1]))
return lines
def insert_kcfg_excl_data(lines: list, start: int, end: int, new_data: OrderedDict) -> dict:
# inserts data into the lines, escape every lines
new_data_lines = ["{}\n".format(cfg) for (cfg, val) in new_data.items()]
return FileHandler.insert_lines(lines, start, end, new_data_lines)
class Action():

200
platform/mellanox/integration-scripts/hwmgmt_helper.py Normal file
View File

@ -0,0 +1,200 @@
#!/usr/bin/env python
#
# Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
# Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import io
import sys
import argparse
import shutil
import copy
import difflib
import configparser
from helper import *
################################################################################
#### KConfig Processing ####
################################################################################
class KCFGData:
x86_base = OrderedDict()
x86_updated = OrderedDict()
arm_base = OrderedDict()
arm_updated = OrderedDict()
x86_incl = OrderedDict()
arm_incl = OrderedDict()
x86_excl = OrderedDict()
arm_excl = OrderedDict()
x86_down = OrderedDict()
arm_down = OrderedDict()
noarch_incl = OrderedDict()
noarch_excl = OrderedDict()
noarch_down = OrderedDict()
class KConfigTask():
def __init__(self, args):
self.args = args
def read_data(self):
KCFGData.x86_base = FileHandler.read_kconfig(self.args.config_base_amd)
KCFGData.x86_updated = FileHandler.read_kconfig(self.args.config_inc_amd)
if os.path.isfile(self.args.config_inc_down_amd):
print(" -> Downstream Config for x86 found..")
KCFGData.x86_down = FileHandler.read_kconfig(self.args.config_inc_down_amd)
KCFGData.arm_base = FileHandler.read_kconfig(self.args.config_base_arm)
KCFGData.arm_updated = FileHandler.read_kconfig(self.args.config_inc_arm)
if os.path.isfile(self.args.config_inc_down_arm):
print(" -> Downstream Config for arm64 found..")
KCFGData.arm_down = FileHandler.read_kconfig(self.args.config_inc_down_arm)
return
def parse_inc_exc(self, base: OrderedDict, updated: OrderedDict):
# parse the updates/deletions in the Kconfig
add, remove = OrderedDict(), copy.deepcopy(base)
for (key, val) in updated.items():
if val != base.get(key, "empty"):
add[key] = val
# items remaining in remove are the ones to be excluded
if key in remove:
del remove[key]
return add, remove
def parse_noarch_inc_exc(self):
# Filter the common inc/excl out from the arch specific inc/excl
x86_incl_base = copy.deepcopy(KCFGData.x86_incl)
for (key, val) in x86_incl_base.items():
if key in KCFGData.arm_incl and val == KCFGData.arm_incl[key]:
print("-> INFO: NoArch KConfig Inclusion {}:{} found and moving to common marker".format(key, val))
del KCFGData.arm_incl[key]
del KCFGData.x86_incl[key]
KCFGData.noarch_incl[key] = val
x86_excl_base = copy.deepcopy(KCFGData.x86_excl)
for (key, val) in x86_excl_base.items():
if key in KCFGData.arm_excl:
print("-> INFO: NoArch KConfig Exclusion {} found and moving to common marker".format(key))
del KCFGData.arm_excl[key]
del KCFGData.x86_excl[key]
KCFGData.noarch_excl[key] = val
if not (KCFGData.x86_down or KCFGData.arm_down):
return
# Filter the common inc config from the downstream kconfig
x86_down_base = copy.deepcopy(KCFGData.x86_down)
for (key, val) in x86_down_base.items():
if key in KCFGData.arm_down:
print("-> INFO: NoArch KConfig Downstream Inclusion {} found and moving to common marker".format(key))
del KCFGData.arm_down[key]
del KCFGData.x86_down[key]
KCFGData.noarch_down[key] = val
def insert_arm64_section(self, raw_lines: list, arm_data: OrderedDict, is_exclusion=False, section=MLNX_ARM_KFG_SECTION) -> list:
# For arm64, config is not added under markers, but it is added under the section [mellanox-arm64]
# This design decision is taken because of the possibility that there might be conflicting options
# present between two different arm64 platforms
try:
# comment_prefixes needed to also read comments under a section
configParser = configparser.ConfigParser(allow_no_value=True, strict=False, comment_prefixes='////')
configParser.optionxform = str
configParser.read_string("".join(raw_lines))
if not configParser.has_section(MLNX_ARM_KFG_SECTION):
configParser.add_section(MLNX_ARM_KFG_SECTION)
for (key, val) in arm_data.items():
if not is_exclusion:
configParser.set(MLNX_ARM_KFG_SECTION, key, val)
else:
configParser.set(MLNX_ARM_KFG_SECTION, key)
str_io = io.StringIO()
configParser.write(str_io, space_around_delimiters=False)
return str_io.getvalue().splitlines(True)
except Exception as e:
print("-> FATAL: Exception {} found while adding opts under arm".format(str(e)))
raise e
return raw_lines
def get_kconfig_inc(self) -> list:
kcfg_inc_raw = FileHandler.read_raw(os.path.join(self.args.build_root, SLK_KCONFIG))
# Insert common config
noarch_start, noarch_end = FileHandler.find_marker_indices(kcfg_inc_raw, MLNX_NOARCH_MARKER)
kcfg_inc_raw = FileHandler.insert_kcfg_data(kcfg_inc_raw, noarch_start, noarch_end, KCFGData.noarch_incl)
# Insert x86 config
x86_start, x86_end = FileHandler.find_marker_indices(kcfg_inc_raw, MLNX_KFG_MARKER)
kcfg_inc_raw = FileHandler.insert_kcfg_data(kcfg_inc_raw, x86_start, x86_end, KCFGData.x86_incl)
# Insert arm config
kcfg_inc_raw = self.insert_arm64_section(kcfg_inc_raw, KCFGData.arm_incl)
print("\n -> INFO: kconfig-inclusion file is generated \n {}".format("".join(kcfg_inc_raw)))
return kcfg_inc_raw
def get_downstream_kconfig_inc(self, new_kcfg_upstream) -> list:
kcfg_final = copy.deepcopy(new_kcfg_upstream)
# insert common Kconfig
noarch_start, noarch_end = FileHandler.find_marker_indices(kcfg_final, MLNX_NOARCH_MARKER)
noarch_final = OrderedDict(list(KCFGData.noarch_incl.items()) + list(KCFGData.noarch_down.items()))
kcfg_final = FileHandler.insert_kcfg_data(kcfg_final, noarch_start, noarch_end, noarch_final)
# insert x86 Kconfig
x86_start, x86_end = FileHandler.find_marker_indices(kcfg_final, MLNX_KFG_MARKER)
x86_final = OrderedDict(list(KCFGData.x86_incl.items()) + list(KCFGData.x86_down.items()))
kcfg_final = FileHandler.insert_kcfg_data(kcfg_final, x86_start, x86_end, x86_final)
# insert arm Kconfig
arm_final = OrderedDict(list(KCFGData.arm_incl.items()) + list(KCFGData.arm_down.items()))
kcfg_final = self.insert_arm64_section(kcfg_final, arm_final)
# generate diff
diff = difflib.unified_diff(new_kcfg_upstream, kcfg_final, fromfile='a/patch/kconfig-inclusions', tofile="b/patch/kconfig-inclusions", lineterm="\n")
lines = []
for line in diff:
lines.append(line)
print("\n -> INFO: kconfig-inclusion.patch file is generated \n{}".format("".join(lines)))
return lines
def get_kconfig_excl(self) -> list:
# noarch_excl
kcfg_excl_raw = FileHandler.read_raw(os.path.join(self.args.build_root, SLK_KCONFIG_EXCLUDE))
# insert common Kconfig
noarch_start, noarch_end = FileHandler.find_marker_indices(kcfg_excl_raw, MLNX_NOARCH_MARKER)
kcfg_excl_raw = FileHandler.insert_kcfg_excl_data(kcfg_excl_raw, noarch_start, noarch_end, KCFGData.noarch_excl)
# insert x86 Kconfig
x86_start, x86_end = FileHandler.find_marker_indices(kcfg_excl_raw, MLNX_KFG_MARKER)
kcfg_excl_raw = FileHandler.insert_kcfg_excl_data(kcfg_excl_raw, x86_start, x86_end, KCFGData.x86_excl)
# insert arm Kconfig
kcfg_excl_raw = self.insert_arm64_section(kcfg_excl_raw, KCFGData.arm_excl, True)
print("\n -> INFO: kconfig-exclusion file is generated \n{}".format("".join(kcfg_excl_raw)))
return kcfg_excl_raw
def perform(self):
self.read_data()
KCFGData.x86_incl, KCFGData.x86_excl = self.parse_inc_exc(KCFGData.x86_base, KCFGData.x86_updated)
KCFGData.arm_incl, KCFGData.arm_excl = self.parse_inc_exc(KCFGData.arm_base, KCFGData.arm_updated)
self.parse_noarch_inc_exc()
# Get the updated kconfig-inclusions
kcfg_inc_upstream = self.get_kconfig_inc()
FileHandler.write_lines(os.path.join(self.args.build_root, SLK_KCONFIG), kcfg_inc_upstream, True)
# Get the updated kconfig-exclusions
kcfg_excl_upstream = self.get_kconfig_excl()
FileHandler.write_lines(os.path.join(self.args.build_root, SLK_KCONFIG_EXCLUDE), kcfg_excl_upstream, True)
# return the kconfig-inclusions diff
return self.get_downstream_kconfig_inc(kcfg_inc_upstream)

167
platform/mellanox/integration-scripts/hwmgmt_kernel_patches.py
View File

@ -16,14 +16,7 @@
# limitations under the License.
#
import os
import sys
import shutil
import argparse
import copy
import difflib
from helper import *
from hwmgmt_helper import *
COMMIT_TITLE = "Intgerate HW-MGMT {} Changes"
@ -127,8 +120,6 @@ class Data:
old_non_up = list()
# New series file written by hw_mgmt integration script
new_series = list()
# List of new opts written by hw_mgmt integration script
updated_kcfg = list(tuple())
# index of the mlnx_hw_mgmt patches start marker in old_series
i_mlnx_start = -1
# index of the mlnx_hw_mgmt patches end marker in old_series
@ -137,15 +128,10 @@ class Data:
up_slk_series = list()
# SLK series file content updated with non-upstream patches, used to generate diff
agg_slk_series = list()
# Diff to be written into the series.patch file
agg_slk_series_diff = list()
# current kcfg opts
current_kcfg = list(tuple())
# current raw kconfig exclude data
kcfg_exclude = list()
# kernel version
k_ver = ""
class HwMgmtAction(Action):
@staticmethod
@ -162,25 +148,28 @@ class HwMgmtAction(Action):
return action
def return_false(self, str_):
print(str_)
return False
def check(self):
if not self.args.config_inclusion:
print("-> ERR: config_inclusion is missing")
return False
if not self.args.kernel_version:
return self.return_false("-> ERR: Kernel Version is missing")
if not self.args.build_root:
print("-> ERR: build_root is missing")
return False
if not os.path.isfile(self.args.config_inclusion):
print("-> ERR: config_inclusion {} doesn't exist".format(self.args.config_inclusion))
return False
return self.return_false("-> ERR: build_root is missing")
if not os.path.exists(self.args.build_root):
print("-> ERR: Build Root {} doesn't exist".format(self.args.build_root))
return False
return self.return_false("-> ERR: Build Root {} doesn't exist".format(self.args.build_root))
if not os.path.isfile(self.args.config_base_amd):
return self.return_false("-> ERR: config_base {} doesn't exist".format(self.args.config_base_amd))
if not os.path.isfile(self.args.config_base_arm):
return self.return_false("-> ERR: config_base_arm {} doesn't exist".format(self.args.config_base_arm))
if not os.path.isfile(self.args.config_inc_amd):
return self.return_false("-> ERR: config_inclusion {} doesn't exist".format(self.args.config_inc_amd))
if not os.path.isfile(self.args.config_inc_arm):
return self.return_false("-> ERR: config_inclusion {} doesn't exist".format(self.args.config_inc_arm))
return True
class PreProcess(HwMgmtAction):
def __init__(self, args):
super().__init__(args)
@ -189,36 +178,28 @@ class PreProcess(HwMgmtAction):
return super(PreProcess, self).check()
def perform(self):
""" Move MLNX Kconfig to the loc pointed by config_inclusion """
kcfg_sec = FileHandler.read_kconfig_inclusion(os.path.join(self.args.build_root, SLK_KCONFIG))
writable_opts = KCFG.get_writable_opts(KCFG.parse_opts_strs(kcfg_sec))
FileHandler.write_lines(self.args.config_inclusion, writable_opts)
print("-> OPTS written to temp config_inclusion file: \n{}".format(FileHandler.read_strip(self.args.config_inclusion, True)))
""" Move Base Kconfig to the loc pointed by config_inclusion """
shutil.copy2(self.args.config_base_amd, self.args.config_inc_amd)
shutil.copy2(self.args.config_base_arm, self.args.config_inc_arm)
print("-> Kconfig amd64/arm64 copied to the relevant directory")
class PostProcess(HwMgmtAction):
def __init__(self, args):
super().__init__(args)
self.kcfg_handler = KConfigTask(self.args)
def check(self):
if not super(PostProcess, self).check():
return False
if not (self.args.patches and os.path.exists(self.args.patches)):
print("-> ERR: upstream patch directory is missing ")
return False
return self.return_false("-> ERR: upstream patch directory is missing ")
if not (self.args.non_up_patches and os.path.exists(self.args.non_up_patches)):
print("-> ERR: non upstream patch directory is missing")
return False
return self.return_false("-> ERR: non upstream patch directory is missing")
if not (self.args.series and os.path.isfile(self.args.series)):
print("-> ERR: series file doesn't exist {}".format(self.args.series))
return False
return self.return_false("-> ERR: series file doesn't exist {}".format(self.args.series))
if not (self.args.current_non_up_patches and os.path.exists(self.args.current_non_up_patches)):
print("-> ERR: current non_up_patches doesn't exist {}".format(self.args.current_non_up_patches))
return False
return self.return_false("-> ERR: current non_up_patches doesn't exist {}".format(self.args.current_non_up_patches))
return True
def read_data(self):
@ -227,19 +208,6 @@ class PostProcess(HwMgmtAction):
Data.old_series = FileHandler.read_raw(os.path.join(self.args.build_root, SLK_SERIES))
Data.old_non_up = FileHandler.read_strip_minimal(self.args.current_non_up_patches)
# Read the new kcfg
new_cfg = FileHandler.read_kconfig_inclusion(self.args.config_inclusion, None)
Data.updated_kcfg = KCFG.parse_opts_strs(new_cfg)
# entire current config, [common] + [amd64]
all_kcfg = FileHandler.read_kconfig_parser(os.path.join(self.args.build_root, SLK_KCONFIG))
Data.current_kcfg = []
for hdr in HDRS:
Data.current_kcfg.extend(all_kcfg.get(hdr, []))
Data.current_kcfg = KCFG.parse_opts_strs(Data.current_kcfg)
Data.kcfg_exclude = FileHandler.read_raw(os.path.join(self.args.build_root, SLK_KCONFIG_EXCLUDE))
new_up = set(FileHandler.read_dir(self.args.patches, "*.patch"))
new_non_up = set(FileHandler.read_dir(self.args.non_up_patches, "*.patch"))
@ -332,58 +300,16 @@ class PostProcess(HwMgmtAction):
FileHandler.write_lines(self.args.current_non_up_patches, lines)
print("\n -> POST: series file updated with non-upstream patches \n{}".format("".join(Data.agg_slk_series)))
def write_series_diff(self):
def get_series_diff(self):
diff = difflib.unified_diff(Data.up_slk_series, Data.agg_slk_series, fromfile='a/patch/series', tofile="b/patch/series", lineterm="\n")
lines = []
for line in diff:
lines.append(line)
print("\n -> POST: final series.diff \n{}".format("".join(lines)))
FileHandler.write_lines(os.path.join(self.args.build_root, NON_UP_PATCH_DIFF), lines, True)
return lines
def check_kconfig_conflicts(self):
# current config under mellanox marker
old_mlnx_kcfg = FileHandler.read_kconfig_inclusion(os.path.join(self.args.build_root, SLK_KCONFIG))
old_mlnx_kcfg = KCFG.parse_opts_strs(old_mlnx_kcfg)
print("-> INFO: [common] + [amd64] Kconfig: \n{}".format("\n".join(KCFG.get_writable_opts(Data.current_kcfg))))
print("-> INFO: current mellanox marker Kconfig: \n{}".format("\n".join(KCFG.get_writable_opts(old_mlnx_kcfg))))
# Filter the mellanox config from current config
conflict_prone = set(Data.current_kcfg)
for kcfg in old_mlnx_kcfg:
if kcfg in conflict_prone:
conflict_prone.remove(kcfg)
print("-> INFO: conflict prone Kconfig: \n{}".format("\n".join(KCFG.get_writable_opts(list(conflict_prone)))))
print("-> INFO: updated kconfig for mellanox marker: \n{}".format("\n".join(KCFG.get_writable_opts(Data.updated_kcfg))))
# check for conflicts
has_conflict = False
for (cfg, val) in Data.updated_kcfg:
for (cfg_o, val_o) in conflict_prone:
if cfg == cfg_o and val != val_o:
print("-> ERR Conflict seen on the following kconfig: {}, old_opt: {}, new_opt: {}".format(cfg, val_o, val))
has_conflict = True
return has_conflict
def handle_exclusions(self):
new_lines = []
curr_hdr = ""
for line_raw in Data.kcfg_exclude:
line = line_raw.strip()
should_exclude = False
if line:
match = re.search(KCFG_HDR_RE, line)
if match:
curr_hdr = match.group(1)
else:
for (kcfg, _) in Data.updated_kcfg:
if kcfg == line and curr_hdr in HDRS:
should_exclude = True
if not should_exclude:
new_lines.append(line_raw)
FileHandler.write_lines(os.path.join(self.args.build_root, SLK_KCONFIG_EXCLUDE), new_lines, True)
print("-> INFO: updated kconfig-exclusion: \n{}".format("".join(FileHandler.read_raw(os.path.join(self.args.build_root, SLK_KCONFIG_EXCLUDE)))))
def get_merged_diff(self, series_diff: list, kcfg_diff: list) -> list:
return series_diff + ["\n"] + kcfg_diff
def list_patches(self):
old_up_patches = []
@ -428,6 +354,7 @@ class PostProcess(HwMgmtAction):
title = COMMIT_TITLE.format(self.args.hw_mgmt_ver)
changes_slk, changes_sb = {}, {}
old_up_patches, old_non_up_patches = self.list_patches()
print(old_up_patches)
for patch in table:
patch_ = patch.get(PATCH_NAME)
id_ = self._fetch_description(patch_, patch.get(COMMIT_ID, ""))
@ -448,18 +375,9 @@ class PostProcess(HwMgmtAction):
def perform(self):
""" Read the data output from the deploy_kernel_patches.py script
and move to appropriate locations """
# Handle Patches related logic
self.read_data()
self.find_mlnx_hw_mgmt_markers()
# Find and report conflicts in new kconfig
if self.check_kconfig_conflicts():
print("-> FATAL Conflicts in kconfig-inclusion detected, exiting...")
sys.exit(1)
else:
# Write the new kcfg to the new file
path = os.path.join(self.args.build_root, SLK_KCONFIG)
FileHandler.write_lines_marker(path, KCFG.get_writable_opts(Data.updated_kcfg), MLNX_KFG_MARKER)
self.handle_exclusions()
# Handle Upstream patches
self.rm_old_up_mlnx()
self.mv_new_up_mlnx()
self.write_final_slk_series()
@ -467,7 +385,11 @@ class PostProcess(HwMgmtAction):
self.rm_old_non_up_mlnx()
self.mv_new_non_up_mlnx()
self.construct_series_with_non_up()
self.write_series_diff()
series_diff = self.get_series_diff()
# handle kconfig and get any diff
kcfg_diff = self.kcfg_handler.perform()
final_diff = self.get_merged_diff(series_diff, kcfg_diff)
FileHandler.write_lines(os.path.join(self.args.build_root, NON_UP_DIFF), final_diff, True)
path = os.path.join(self.args.build_root, PATCH_TABLE_LOC)
patch_table = load_patch_table(path, Data.k_ver)
@ -481,6 +403,8 @@ class PostProcess(HwMgmtAction):
if self.args.slk_msg:
with open(self.args.slk_msg, 'w') as f:
f.write(slk_msg)
def create_parser():
# Create argument parser
@ -492,7 +416,12 @@ def create_parser():
# Optional arguments
parser.add_argument("--patches", type=str)
parser.add_argument("--non_up_patches", type=str)
parser.add_argument("--config_inclusion", type=str)
parser.add_argument("--config_base_amd", type=str, required=True)
parser.add_argument("--config_base_arm", type=str, required=True)
parser.add_argument("--config_inc_amd", type=str, required=True)
parser.add_argument("--config_inc_arm", type=str, required=True)
parser.add_argument("--config_inc_down_amd", type=str)
parser.add_argument("--config_inc_down_arm", type=str)
parser.add_argument("--series", type=str)
parser.add_argument("--current_non_up_patches", type=str)
parser.add_argument("--build_root", type=str)

20
platform/mellanox/integration-scripts/tests/data/expected_data/series.patch → platform/mellanox/integration-scripts/tests/data/expected_data/external-changes.patch
View File

@ -11,4 +11,22 @@
0188-i2c-mux-Add-register-map-based-mux-driver.patch
###-> mellanox_hw_mgmt-end
# Cisco patches for 5.10 kernel
--- a/patch/kconfig-inclusions
+++ b/patch/kconfig-inclusions
@@ -41,4 +41,15 @@
CONFIG_HWMON=y
CONFIG_OF=y
CONFIG_THERMAL_NETLINK=y
+CONFIG_SENSORS_MP2891=m
+CONFIG_MMC_SDHCI_OF_DWCMSHC=m
+CONFIG_VFIO_PLATFORM=m
+CONFIG_SENSORS_ARM_SCMI=m
+CONFIG_MLXBF_GIGE=m
+CONFIG_I2C_MLXBF=m
+CONFIG_GPIO_MLXBF3=m
+CONFIG_MLXBF_TMFIFO=m
+CONFIG_MLXBF_BOOTCTL=m
+CONFIG_MLXBF_PMC=m
+CONFIG_MLXBF_PTM=m

9
platform/mellanox/integration-scripts/tests/data/expected_data/kconfig-exclusions
View File

@ -6,12 +6,18 @@ CONFIG_CGROUP_NET_CLASSID
CONFIG_NET_CLS_CGROUP
CONFIG_NETFILTER_XT_MATCH_CGROUP
CONFIG_CGROUP_NET_PRIO
###-> mellanox_common-start
###-> mellanox_common-end
[amd64]
# Unset X86_PAT according to Broadcom's requirement
CONFIG_SENSORS_UCD9200
CONFIG_X86_PAT
CONFIG_MLXSW_PCI
CONFIG_THERMAL_STATISTICS
###-> mellanox_amd64-start
CONFIG_I2C_DESIGNWARE_BAYTRAIL
###-> mellanox_amd64-end
[arm64]
@ -21,3 +27,6 @@ CONFIG_THERMAL_STATISTICS
CONFIG_ARCH_VIRT
CONFIG_ARCH_BCM
CONFIG_ARCH_EXYNOS
[mellanox-arm64]

42
platform/mellanox/integration-scripts/tests/data/expected_data/kconfig-inclusions
View File

@ -1,11 +1,9 @@
# Description: Add kconfig options that has to be included in the kernel
# Note: Markers should start with "###->"
# Header should end with .*-start eg: any_description-start
# Footer must end with .*-end eg: any_description-end
# Markers should not be nested
[common]
CONFIG_LOG_BUF_SHIFT=20
###-> mellanox_common-start
CONFIG_THERMAL=y
CONFIG_THERMAL_OF=y
###-> mellanox_common-end
[amd64]
# For Inventec d7032
@ -14,19 +12,13 @@ CONFIG_GPIO_ICH=m
CONFIG_SENSORS_MAX31790=m
# For optoe
CONFIG_EEPROM_OPTOE=m
###-> mellanox-start
CONFIG_SPI_PXA2XX=m
CONFIG_THERMAL_NETLINK=y
CONFIG_SENSORS_XDPE152=m
CONFIG_SENSORS_DRIVETEMP=m
CONFIG_SENSORS_IIO_HWMON=m
CONFIG_SENSORS_LM25066=m
CONFIG_SENSORS_UCD9000=m
CONFIG_SENSORS_UCD9200=m
CONFIG_THERMAL_OF=y
###-> mellanox-end
###-> mellanox_amd64-start
CONFIG_PINCTRL=y
CONFIG_DW_DMAC_PCI=y
CONFIG_TI_ADS1015=m
CONFIG_I2C_DESIGNWARE_CORE=m
CONFIG_I2C_DESIGNWARE_PCI=m
###-> mellanox_amd64-end
# For Cisco 8000
CONFIG_PHYLIB=m
CONFIG_GPIOLIB=y
@ -34,7 +26,19 @@ CONFIG_OF_GPIO=y
CONFIG_OF=y
CONFIG_OF_MDIO=m
[arm64]
[armhf]
CONFIG_EEPROM_SFF_8436=m
CONFIG_EEPROM_OPTOE=m
CONFIG_I2C_MUX_GPIO=m
[mellanox-arm64]
CONFIG_MELLANOX_PLATFORM=y
CONFIG_THERMAL_WRITABLE_TRIPS=y
CONFIG_PMBUS=m
CONFIG_SENSORS_PMBUS=m
CONFIG_HWMON=y
CONFIG_OF=y
CONFIG_THERMAL_NETLINK=y

6
platform/mellanox/integration-scripts/tests/data/kconfig-exclusions
View File

@ -6,6 +6,8 @@ CONFIG_CGROUP_NET_CLASSID
CONFIG_NET_CLS_CGROUP
CONFIG_NETFILTER_XT_MATCH_CGROUP
CONFIG_CGROUP_NET_PRIO
###-> mellanox_common-start
###-> mellanox_common-end
[amd64]
# Unset X86_PAT according to Broadcom's requirement
@ -13,6 +15,8 @@ CONFIG_SENSORS_UCD9200
CONFIG_X86_PAT
CONFIG_MLXSW_PCI
CONFIG_THERMAL_STATISTICS
###-> mellanox_amd64-start
###-> mellanox_amd64-end
[arm64]
@ -22,3 +26,5 @@ CONFIG_THERMAL_STATISTICS
CONFIG_ARCH_VIRT
CONFIG_ARCH_BCM
CONFIG_ARCH_EXYNOS
[mellanox-arm64]

16
platform/mellanox/integration-scripts/tests/data/kconfig-inclusions
View File

@ -1,11 +1,7 @@
# Description: Add kconfig options that has to be included in the kernel
# Note: Markers should start with "###->"
# Header should end with .*-start eg: any_description-start
# Footer must end with .*-end eg: any_description-end
# Markers should not be nested
[common]
CONFIG_LOG_BUF_SHIFT=20
###-> mellanox_common-start
###-> mellanox_common-end
[amd64]
# For Inventec d7032
@ -15,13 +11,13 @@ CONFIG_SENSORS_MAX31790=m
# For optoe
CONFIG_EEPROM_OPTOE=m
###-> mellanox-start
###-> mellanox_amd64-start
CONFIG_OF=y
CONFIG_THERMAL_OF=y
CONFIG_CPU_THERMAL=y
CONFIG_PINCTRL_CANNONLAKE=m
CONFIG_PINCTRL_DENVERTON=m
###-> mellanox-end
###-> mellanox_amd64-end
# For Cisco 8000
CONFIG_PHYLIB=m
@ -30,7 +26,11 @@ CONFIG_OF_GPIO=y
CONFIG_OF=y
CONFIG_OF_MDIO=m
[arm64]
[armhf]
CONFIG_EEPROM_SFF_8436=m
CONFIG_EEPROM_OPTOE=m
CONFIG_I2C_MUX_GPIO=m
[mellanox-arm64]

36
platform/mellanox/integration-scripts/tests/data/kconfig/arm64.config Normal file
View File

@ -0,0 +1,36 @@
#
# Automatically generated file; DO NOT EDIT.
# Linux/arm64 5.10.179 Kernel Configuration
#
CONFIG_NET_VENDOR_MELLANOX=y
# CONFIG_MELLANOX_PLATFORM is not set
#
# ACPI drivers
#
# CONFIG_SENSORS_ACPI_POWER is not set
CONFIG_THERMAL=n
# CONFIG_THERMAL_NETLINK is not set
# CONFIG_THERMAL_STATISTICS is not set
CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=0
CONFIG_THERMAL_HWMON=y
CONFIG_THERMAL_OF=n
# CONFIG_THERMAL_WRITABLE_TRIPS is not set
CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y
CONFIG_THERMAL_GOV_STEP_WISE=y
CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
CONFIG_CPU_THERMAL=y
#
# Sensors
#
CONFIG_SENSORS_XDPE152=m
CONFIG_SENSORS_DRIVETEMP=m
CONFIG_SENSORS_IIO_HWMON=m
CONFIG_SENSORS_LM25066=m
CONFIG_SENSORS_UCD9000=m
CONFIG_SENSORS_UCD9200=m
CONFIG_FUSE_FS=m

45
platform/mellanox/integration-scripts/tests/data/kconfig/new_arm.config Normal file
View File

@ -0,0 +1,45 @@
#
# Automatically generated file; DO NOT EDIT.
# Linux/arm64 5.10.179 Kernel Configuration
#
CONFIG_NET_VENDOR_MELLANOX=y
# CONFIG_MELLANOX_PLATFORM is not set
#
# ACPI drivers
#
# CONFIG_SENSORS_ACPI_POWER is not set
CONFIG_THERMAL=y
# CONFIG_THERMAL_NETLINK is not set
# CONFIG_THERMAL_STATISTICS is not set
CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=0
CONFIG_THERMAL_HWMON=y
CONFIG_THERMAL_OF=y
# CONFIG_THERMAL_WRITABLE_TRIPS is not set
CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y
CONFIG_THERMAL_GOV_STEP_WISE=y
CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
CONFIG_CPU_THERMAL=y
#
# Sensors
#
CONFIG_SENSORS_XDPE152=m
CONFIG_SENSORS_DRIVETEMP=m
CONFIG_SENSORS_IIO_HWMON=m
CONFIG_SENSORS_LM25066=m
CONFIG_SENSORS_UCD9000=m
CONFIG_SENSORS_UCD9200=m
CONFIG_FUSE_FS=m
# New hw-mgmt 7.0020.5048 arm64 kernel config flags
CONFIG_MELLANOX_PLATFORM=y
CONFIG_THERMAL_WRITABLE_TRIPS=y
CONFIG_PMBUS=m
CONFIG_SENSORS_PMBUS=m
CONFIG_HWMON=y
CONFIG_OF=y
CONFIG_THERMAL_NETLINK=y

14
platform/mellanox/integration-scripts/tests/data/kconfig/new_arm_down.config Normal file
View File

@ -0,0 +1,14 @@
# New hw-mgmt 7.0020.5048 kernel downstream config flags
CONFIG_SENSORS_MP2891=m
CONFIG_MMC_SDHCI_OF_DWCMSHC=m
CONFIG_VFIO_PLATFORM=m
CONFIG_SENSORS_ARM_SCMI=m
CONFIG_MLXBF_GIGE=m
CONFIG_I2C_MLXBF=m
CONFIG_GPIO_MLXBF3=m
CONFIG_MLXBF_TMFIFO=m
CONFIG_MLXBF_BOOTCTL=m
CONFIG_MLXBF_PMC=m
CONFIG_MLXBF_PTM=m

41
platform/mellanox/integration-scripts/tests/data/kconfig/new_x86.config Normal file
View File

@ -0,0 +1,41 @@
#
# Automatically generated file; DO NOT EDIT.
# Linux/x86_c64 5.10.179 Kernel Configuration
#
CONFIG_NET_VENDOR_MELLANOX=y
# CONFIG_MELLANOX_PLATFORM is not set
#
# ACPI drivers
#
CONFIG_THERMAL=y
CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=0
CONFIG_THERMAL_HWMON=y
CONFIG_THERMAL_OF=y
CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y
CONFIG_THERMAL_GOV_STEP_WISE=y
CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
CONFIG_CPU_THERMAL=y
#
# Sensors
#
CONFIG_SENSORS_XDPE152=m
CONFIG_SENSORS_DRIVETEMP=m
CONFIG_SENSORS_IIO_HWMON=m
CONFIG_SENSORS_LM25066=m
CONFIG_SENSORS_UCD9000=m
CONFIG_SENSORS_UCD9200=m
CONFIG_I2C_MUX_PCA954x=m
CONFIG_I2C_DESIGNWARE_PLATFORM=m
# New hw-mgmt 7.0020.5048 kernel config flags
CONFIG_PINCTRL=y
CONFIG_DW_DMAC_PCI=y
CONFIG_TI_ADS1015=m
CONFIG_I2C_DESIGNWARE_CORE=m
CONFIG_I2C_DESIGNWARE_PCI=m

0
platform/mellanox/integration-scripts/tests/data/kconfig/new_x86_down.config Normal file
View File

34
platform/mellanox/integration-scripts/tests/data/kconfig/x86.config Normal file
View File

@ -0,0 +1,34 @@
#
# Automatically generated file; DO NOT EDIT.
# Linux/x86_c64 5.10.179 Kernel Configuration
#
CONFIG_NET_VENDOR_MELLANOX=y
# CONFIG_MELLANOX_PLATFORM is not set
#
# ACPI drivers
#
CONFIG_THERMAL=n
CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=0
CONFIG_THERMAL_HWMON=y
CONFIG_THERMAL_OF=n
CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y
CONFIG_THERMAL_GOV_STEP_WISE=y
CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
CONFIG_CPU_THERMAL=y
#
# Sensors
#
CONFIG_SENSORS_XDPE152=m
CONFIG_SENSORS_DRIVETEMP=m
CONFIG_SENSORS_IIO_HWMON=m
CONFIG_SENSORS_LM25066=m
CONFIG_SENSORS_UCD9000=m
CONFIG_SENSORS_UCD9200=m
CONFIG_I2C_MUX_PCA954x=m
CONFIG_I2C_DESIGNWARE_PLATFORM=m
CONFIG_I2C_DESIGNWARE_BAYTRAIL=y

12
platform/mellanox/integration-scripts/tests/data/new_kconfig
View File

@ -1,12 +0,0 @@
# New hw-mgmt 7.0020.5048 kernel config flags
CONFIG_SPI_PXA2XX=m
CONFIG_THERMAL_NETLINK=y
CONFIG_SENSORS_XDPE152=m
CONFIG_SENSORS_DRIVETEMP=m
CONFIG_SENSORS_IIO_HWMON=m
CONFIG_SENSORS_LM25066=m
CONFIG_SENSORS_UCD9000=m
CONFIG_SENSORS_UCD9200=m
CONFIG_THERMAL_OF=y

118
platform/mellanox/integration-scripts/tests/test_filehandler.py
View File

@ -46,11 +46,11 @@ CONFIG_LOG_BUF_SHIFT=20
[amd64]
CONFIG_SENSORS_DPS1900=m
###-> mellanox-start
###-> mellanox_amd64-start
CONFIG_OF=y
CONFIG_THERMAL_OF=y
CONFIG_DW_DMAC_PCI=y
###-> mellanox-end
###-> mellanox_amd64-end
[armhf]
CONFIG_EEPROM_SFF_8436=m
@ -58,17 +58,6 @@ CONFIG_EEPROM_OPTOE=m
CONFIG_I2C_MUX_GPIO=y
"""
UPDATED_MLNX_KCFG = """\
CONFIG_OF=y
CONFIG_THERMAL_OF=y
CONFIG_DW_DMAC_PCI=y
CONFIG_I2C_I801=m
CONFIG_PINCTRL=y
CONFIG_PINCTRL_INTEL=m
CONFIG_I2C_MUX_PCA954x=m
CONFIG_SPI_PXA2XX=m
"""
FINAL_MOCK_SLK_KCFG = """\
CONFIG_RANDOM=rrr
@ -78,7 +67,7 @@ CONFIG_LOG_BUF_SHIFT=20
[amd64]
CONFIG_SENSORS_DPS1900=m
###-> mellanox-start
###-> mellanox_amd64-start
CONFIG_OF=y
CONFIG_THERMAL_OF=y
CONFIG_DW_DMAC_PCI=y
@ -87,7 +76,7 @@ CONFIG_PINCTRL=y
CONFIG_PINCTRL_INTEL=m
CONFIG_I2C_MUX_PCA954x=m
CONFIG_SPI_PXA2XX=m
###-> mellanox-end
###-> mellanox_amd64-end
[armhf]
CONFIG_EEPROM_SFF_8436=m
@ -95,6 +84,44 @@ CONFIG_EEPROM_OPTOE=m
CONFIG_I2C_MUX_GPIO=y
"""
MOCK_SLK_EXCL = """\
[common]
CONFIG_CGROUP_NET_CLASSID
CONFIG_NET_CLS_CGROUP
CONFIG_NETFILTER_XT_MATCH_CGROUP
CONFIG_CGROUP_NET_PRIO
[amd64]
# Unset X86_PAT according to Broadcom's requirement
CONFIG_X86_PAT
CONFIG_MLXSW_PCI
###-> mellanox_amd64-start
###-> mellanox_amd64-end
[arm64]
CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU
"""
FINAL_MOCK_SLK_EXCL = """\
[common]
CONFIG_CGROUP_NET_CLASSID
CONFIG_NET_CLS_CGROUP
CONFIG_NETFILTER_XT_MATCH_CGROUP
CONFIG_CGROUP_NET_PRIO
[amd64]
# Unset X86_PAT according to Broadcom's requirement
CONFIG_X86_PAT
CONFIG_MLXSW_PCI
###-> mellanox_amd64-start
CONFIG_OF
CONFIG_THERMAL_OF
###-> mellanox_amd64-end
[arm64]
CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU
"""
LINES_WRITE = []
LINES_READ = []
@ -137,40 +164,39 @@ class TestFilehandler:
print(i_start, i_end)
assert i_start == -1
assert i_end == len(lines)
@mock.patch('helper.FileHandler.read_raw', side_effect=read_raw_mock)
def test_read_kconfig_parser(self, mock_read_raw):
def test_insert_kcfg(self, mock_read_raw):
global LINES_READ
LINES_READ = MOCK_SLK_KCFG.split("\n")
all_cfg = FileHandler.read_kconfig_parser("")
print(all_cfg)
assert all_cfg['no_parent'] == ['CONFIG_RANDOM=rrr']
assert all_cfg['common'] == ['CONFIG_LOG_BUF_SHIFT=20']
assert all_cfg['amd64'] == ['CONFIG_SENSORS_DPS1900=m', 'CONFIG_OF=y', 'CONFIG_THERMAL_OF=y', 'CONFIG_DW_DMAC_PCI=y']
assert all_cfg['armhf'] == ['CONFIG_EEPROM_SFF_8436=m', 'CONFIG_EEPROM_OPTOE=m', 'CONFIG_I2C_MUX_GPIO=y']
@mock.patch('helper.FileHandler.write_lines', side_effect=writer_mock)
@mock.patch('helper.FileHandler.read_raw', side_effect=read_raw_mock)
def test_write_lines_marker(self, mock_read_raw, mock_write_lines_marker):
global LINES_READ
global LINES_WRITE
LINES_READ = MOCK_SLK_KCFG.splitlines(True)
LINES_WRITE = FINAL_MOCK_SLK_KCFG.splitlines(True)
kcfg_inc_raw = FileHandler.read_raw("")
new_opts = OrderedDict({
"CONFIG_OF" : "y",
"CONFIG_THERMAL_OF" : "y",
"CONFIG_DW_DMAC_PCI" : "y",
"CONFIG_I2C_I801" : "m",
"CONFIG_PINCTRL" : "y",
"CONFIG_PINCTRL_INTEL" : "m",
"CONFIG_I2C_MUX_PCA954x" : "m",
"CONFIG_SPI_PXA2XX" : "m"
})
x86_start, x86_end = FileHandler.find_marker_indices(kcfg_inc_raw, MLNX_KFG_MARKER)
assert "###-> mellanox_amd64-start" in kcfg_inc_raw[x86_start]
assert "###-> mellanox_amd64-end" in kcfg_inc_raw[x86_end]
final_kcfg = FileHandler.insert_kcfg_data(kcfg_inc_raw, x86_start, x86_end, new_opts)
assert final_kcfg == FINAL_MOCK_SLK_KCFG.splitlines(True)
list_opts = KCFG.parse_opts_strs(UPDATED_MLNX_KCFG.split("\n"))
writable_opts = KCFG.get_writable_opts(list_opts)
FileHandler.write_lines_marker("", writable_opts, marker="mellanox")
@mock.patch('helper.FileHandler.write_lines', side_effect=writer_mock)
@mock.patch('helper.FileHandler.read_raw', side_effect=read_raw_mock)
def test_read_kconfig_inclusion(self, mock_read_raw, mock_write_lines_marker):
def test_insert_kcfg_excl(self, mock_read_raw):
global LINES_READ
LINES_READ = FINAL_MOCK_SLK_KCFG.splitlines(True)
opts = FileHandler.read_kconfig_inclusion("")
global LINES_WRITE
LINES_WRITE = UPDATED_MLNX_KCFG.splitlines()
writable_opts = KCFG.get_writable_opts(KCFG.parse_opts_strs(opts))
print(writable_opts)
FileHandler.write_lines("", writable_opts)
LINES_READ = MOCK_SLK_EXCL.splitlines(True)
kcfg_inc_raw = FileHandler.read_raw("")
new_opts = OrderedDict({
"CONFIG_OF" : "y",
"CONFIG_THERMAL_OF" : "y"
})
x86_start, x86_end = FileHandler.find_marker_indices(kcfg_inc_raw, MLNX_KFG_MARKER)
assert "###-> mellanox_amd64-start" in kcfg_inc_raw[x86_start]
assert "###-> mellanox_amd64-end" in kcfg_inc_raw[x86_end]
final_kcfg = FileHandler.insert_kcfg_excl_data(kcfg_inc_raw, x86_start, x86_end, new_opts)
assert final_kcfg == FINAL_MOCK_SLK_EXCL.splitlines(True)

155
platform/mellanox/integration-scripts/tests/test_hwmgmtaction.py
View File

@ -20,8 +20,10 @@ import shutil
from unittest import mock, TestCase
from pyfakefs.fake_filesystem_unittest import Patcher
sys.path.append('../')
from hwmgmt_helper import *
from hwmgmt_kernel_patches import *
NEW_NONUP_LIST = """ \
0168-TMP-mlxsw-minimal-Ignore-error-reading-SPAD-register.patch
0169-TMP-mlxsw-i2c-Prevent-transaction-execution-for-spec.patch
@ -76,6 +78,7 @@ Intgerate HW-MGMT 7.0030.0937 Changes
REL_INPUTS_DIR = "platform/mellanox/integration-scripts/tests/data/"
MOCK_INPUTS_DIR = "/sonic/" + REL_INPUTS_DIR
MOCK_WRITE_FILE = MOCK_INPUTS_DIR + "test_writer_file.out"
MOCK_KCFG_DIR = MOCK_INPUTS_DIR + "/kconfig"
def write_lines_mock(path, lines, raw=False):
# Create the dir if it doesn't exist already
@ -90,7 +93,12 @@ def mock_hwmgmt_args():
with mock.patch("sys.argv", ["hwmgmt_kernel_patches.py", "post",
"--patches", "/tmp",
"--non_up_patches", "/tmp",
"--config_inclusion", MOCK_INPUTS_DIR+"/new_kconfig",
"--config_inc_amd", MOCK_KCFG_DIR+"/new_x86.config",
"--config_inc_arm", MOCK_KCFG_DIR+"/new_arm.config",
"--config_base_amd", MOCK_KCFG_DIR+"/x86.config",
"--config_base_arm", MOCK_KCFG_DIR+"/arm64.config",
"--config_inc_down_amd", MOCK_KCFG_DIR+"/new_x86_down.config",
"--config_inc_down_arm", MOCK_KCFG_DIR+"/new_arm_down.config",
"--series", MOCK_INPUTS_DIR+"/new_series",
"--current_non_up_patches", MOCK_INPUTS_DIR+"/hwmgmt_nonup_patches",
"--kernel_version", "5.10.140",
@ -113,74 +121,56 @@ def check_file_content(path):
@mock.patch('helper.SLK_PATCH_LOC', REL_INPUTS_DIR)
@mock.patch('helper.SLK_SERIES', REL_INPUTS_DIR+"series")
@mock.patch('helper.SLK_KCONFIG', REL_INPUTS_DIR+"kconfig-inclusion")
@mock.patch('hwmgmt_helper.SLK_KCONFIG', REL_INPUTS_DIR+"kconfig-inclusions")
@mock.patch('hwmgmt_helper.SLK_KCONFIG_EXCLUDE', REL_INPUTS_DIR+"kconfig-exclusions")
class TestHwMgmtPostAction(TestCase):
def setUp(self):
self.action = HwMgmtAction.get(mock_hwmgmt_args())
self.action.read_data()
self.kcfgaction = KConfigTask(mock_hwmgmt_args())
self.kcfgaction.read_data()
# Populate the new_up, new_non_up list
Data.new_up = NEW_UP_LIST.splitlines()
Data.new_non_up = NEW_NONUP_LIST.splitlines()
Data.old_series = FileHandler.read_raw(MOCK_INPUTS_DIR+"/series")
all_kcfg = FileHandler.read_kconfig_parser(MOCK_INPUTS_DIR+"/kconfig-inclusions")
Data.current_kcfg = []
for hdr in HDRS:
Data.current_kcfg.extend(all_kcfg.get(hdr, []))
Data.current_kcfg = KCFG.parse_opts_strs(Data.current_kcfg)
Data.kcfg_exclude = FileHandler.read_raw(MOCK_INPUTS_DIR+"/kconfig-exclusions")
def tearDown(self):
KCFGData.x86_incl.clear()
KCFGData.arm_incl.clear()
KCFGData.x86_excl.clear()
KCFGData.arm_excl.clear()
KCFGData.x86_down.clear()
KCFGData.arm_down.clear()
KCFGData.noarch_incl.clear()
KCFGData.noarch_excl.clear()
KCFGData.noarch_down.clear()
def test_find_mlnx_hw_mgmt_markers(self):
self.action.find_mlnx_hw_mgmt_markers()
print(Data.i_mlnx_start, Data.i_mlnx_end)
assert Data.old_series[Data.i_mlnx_start].strip() == "###-> mellanox_hw_mgmt-start"
assert Data.old_series[Data.i_mlnx_end].strip() == "###-> mellanox_hw_mgmt-end"
def test_check_kconfig_conflicts(self):
# Add a line to create conflict
print(Data.current_kcfg)
Data.updated_kcfg.append(["CONFIG_EEPROM_OPTOE", "n"])
self.action.find_mlnx_hw_mgmt_markers()
assert self.action.check_kconfig_conflicts() == True
# Add a duplicate option
Data.updated_kcfg.pop(-1)
Data.updated_kcfg.append(["CONFIG_EEPROM_OPTOE", "m"])
assert self.action.check_kconfig_conflicts() == False
# Check with no conflicts or duplicates
Data.updated_kcfg.pop(-1)
assert self.action.check_kconfig_conflicts() == False
@mock.patch('helper.FileHandler.write_lines', side_effect=write_lines_mock)
def test_write_final_slk_series(self, mock_write_lines):
self.action.find_mlnx_hw_mgmt_markers()
assert not self.action.check_kconfig_conflicts()
self.action.write_final_slk_series()
assert check_file_content(MOCK_INPUTS_DIR+"expected_data/series")
def test_write_kconfig_inclusion(self):
self.action.find_mlnx_hw_mgmt_markers()
assert not self.action.check_kconfig_conflicts()
print(Data.updated_kcfg)
shutil.copy(MOCK_INPUTS_DIR+"/kconfig-inclusions", MOCK_WRITE_FILE)
FileHandler.write_lines_marker(MOCK_WRITE_FILE, KCFG.get_writable_opts(Data.updated_kcfg), MLNX_KFG_MARKER)
assert check_file_content(MOCK_INPUTS_DIR+"expected_data/kconfig-inclusions")
@mock.patch('helper.FileHandler.write_lines', side_effect=write_lines_mock)
def test_handle_exclusions(self, mock_write_lines):
self.action.find_mlnx_hw_mgmt_markers()
self.action.handle_exclusions()
assert check_file_content(MOCK_INPUTS_DIR+"expected_data/kconfig-exclusions")
@mock.patch('helper.FileHandler.write_lines', side_effect=write_lines_mock)
def test_write_series_diff(self, mock_write_lines):
def test_write_final_diff(self, mock_write_lines):
self.action.find_mlnx_hw_mgmt_markers()
self.action.write_final_slk_series()
self.action.construct_series_with_non_up()
self.action.write_series_diff()
assert check_file_content(MOCK_INPUTS_DIR+"expected_data/series.patch")
series_diff = self.action.get_series_diff()
kcfg_diff = self._get_kcfg_incl_diff()
final_diff = self.action.get_merged_diff(series_diff, kcfg_diff)
print("".join(final_diff))
FileHandler.write_lines(os.path.join(self.action.args.build_root, NON_UP_DIFF), final_diff, True)
assert check_file_content(MOCK_INPUTS_DIR+"expected_data/external-changes.patch")
def test_commit_msg(self):
self.action.find_mlnx_hw_mgmt_markers()
root_dir = "/sonic/" + PATCH_TABLE_LOC + PATCHWORK_LOC.format("5.10")
content = "patchwork_link: https://patchwork.ozlabs.org/project/linux-i2c/patch/20230215195322.21955-1-vadimp@nvidia.com/\n"
file = "0188-i2c-mux-Add-register-map-based-mux-driver.patch.txt"
@ -193,3 +183,84 @@ class TestHwMgmtPostAction(TestCase):
assert slk.split() == TEST_SLK_COMMIT.split()
assert sb.split() == TEST_SB_COMMIT.split()
def _parse_inc_excl(self):
KCFGData.x86_incl, KCFGData.x86_excl = self.kcfgaction.parse_inc_exc(KCFGData.x86_base, KCFGData.x86_updated)
KCFGData.arm_incl, KCFGData.arm_excl = self.kcfgaction.parse_inc_exc(KCFGData.arm_base, KCFGData.arm_updated)
def _parse_noarch_inc_excl(self):
self._parse_inc_excl()
self.kcfgaction.parse_noarch_inc_exc()
def _get_kcfg_incl_raw(self):
self._parse_noarch_inc_excl()
return self.kcfgaction.get_kconfig_inc()
def _get_kcfg_incl_diff(self):
kcfg_raw = self._get_kcfg_incl_raw()
return self.kcfgaction.get_downstream_kconfig_inc(kcfg_raw)
def test_parse_inc_excl(self):
self._parse_inc_excl()
test_x86_incl = OrderedDict({
"CONFIG_THERMAL" : "y",
"CONFIG_THERMAL_OF" : "y",
"CONFIG_PINCTRL" : "y",
"CONFIG_DW_DMAC_PCI" : "y",
"CONFIG_TI_ADS1015" : "m",
"CONFIG_I2C_DESIGNWARE_CORE" : "m",
"CONFIG_I2C_DESIGNWARE_PCI" : "m"
})
test_arm_incl = OrderedDict({
"CONFIG_THERMAL" : "y",
"CONFIG_THERMAL_OF" : "y",
"CONFIG_MELLANOX_PLATFORM" : "y",
"CONFIG_THERMAL_WRITABLE_TRIPS" : "y",
"CONFIG_PMBUS" : "m",
"CONFIG_SENSORS_PMBUS" : "m",
"CONFIG_HWMON" : "y",
"CONFIG_OF" : "y",
"CONFIG_THERMAL_NETLINK" : "y"
})
test_x86_excl = OrderedDict({"CONFIG_I2C_DESIGNWARE_BAYTRAIL" : "y"})
assert KCFGData.x86_incl == test_x86_incl
assert KCFGData.x86_excl == test_x86_excl
assert KCFGData.arm_incl == test_arm_incl
def test_parse_inc_excl_noarch(self):
self._parse_noarch_inc_excl()
test_x86_incl = OrderedDict({
"CONFIG_PINCTRL" : "y",
"CONFIG_DW_DMAC_PCI" : "y",
"CONFIG_TI_ADS1015" : "m",
"CONFIG_I2C_DESIGNWARE_CORE" : "m",
"CONFIG_I2C_DESIGNWARE_PCI" : "m"
})
test_arm_incl = OrderedDict({
"CONFIG_MELLANOX_PLATFORM" : "y",
"CONFIG_THERMAL_WRITABLE_TRIPS" : "y",
"CONFIG_PMBUS" : "m",
"CONFIG_SENSORS_PMBUS" : "m",
"CONFIG_HWMON" : "y",
"CONFIG_OF" : "y",
"CONFIG_THERMAL_NETLINK" : "y"
})
test_noarch_incl = OrderedDict({
"CONFIG_THERMAL" : "y",
"CONFIG_THERMAL_OF" : "y"
})
assert KCFGData.x86_incl == test_x86_incl
assert KCFGData.noarch_incl == test_noarch_incl
assert KCFGData.arm_incl == test_arm_incl
@mock.patch('helper.FileHandler.write_lines', side_effect=write_lines_mock)
def test_kcfg_incl_file(self, mock_write_lines_mock):
kcfg_raw = self._get_kcfg_incl_raw()
FileHandler.write_lines("", kcfg_raw, True)
assert check_file_content(MOCK_INPUTS_DIR+"expected_data/kconfig-inclusions")
@mock.patch('helper.FileHandler.write_lines', side_effect=write_lines_mock)
def test_kcfg_excl(self, mock_write_lines_mock):
self._parse_noarch_inc_excl()
kcfg_excl = self.kcfgaction.get_kconfig_excl()
FileHandler.write_lines("", kcfg_excl, True)
assert check_file_content(MOCK_INPUTS_DIR+"expected_data/kconfig-exclusions")

4
platform/mellanox/non-upstream-patches/README.md
View File

@ -11,10 +11,10 @@ EXTERNAL_KERNEL_PATCH_LOC/
├── 0002-mlx5-Implement-get_module_eeprom_by_page.patch.patch
├── 0005-mlx5-Add-support-for-DSFP-module-EEPROM-dumps.patch
├── .............
├──── series.patch
├──── external-changes.patch
```
1. It should contain a file named series.patch. This should contain a diff that is applied on the sonic-linux-kernel/patch/series file. The diff should include all the non-upstream patches.
1. It should contain a file named external-changes.patch. This should contain a diff that is applied on the sonic-linux-kernel/patch/* files. The diff should include all the non-upstream patches.
2. All the patches should be present in the patches folder
3. Developers should make sure patches apply cleanly over the existing patches present in the src/sonic-linux-kernel .

0
platform/mellanox/non-upstream-patches/series.patch → platform/mellanox/non-upstream-patches/external-changes.patch
View File