matthew/sonic-buildimage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[dash-api]: Add dash-api and related protobuf library (#14515)

Browse Source 
Why I did it
For the DASH scenario, the APP_DB will be optimized by protobuf message for less memory consumption.

How I did it
Download the Debian package of protobuf 3.21.12 and create a corresponding rule for building it.
Add a submodule of sonic-dash-api and generated its Debian package which includes C++ library and Python library

How to verify it
Check artifacts of Azp that the protobuf-related and dash-api deb packages should be generated.

Signed-off-by: Ze Gan <ganze718@gmail.com>
This commit is contained in:
Ze Gan 2023-07-06 00:59:35 +08:00 committed by GitHub
parent de65640633
commit 2f8994999b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 262 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
.azure-pipelines/azure-pipelines-build-ubuntu-2004.yml Normal file
View File

@ -0,0 +1,55 @@
# C/C++ with GCC
# Build your C/C++ project with GCC using make.
# Add steps that publish test results, save build artifacts, deploy, and more:
# https://docs.microsoft.com/azure/devops/pipelines/apps/c-cpp/gcc
pr: none
trigger:
batch: true
branches:
include:
- master
stages:
- stage: Build
jobs:
- job:
displayName: "amd64/ubuntu-20.04"
pool:
vmImage: 'ubuntu-20.04'
steps:
- checkout: self
submodules: true
- script: |
sudo apt-get update
sudo apt-get install -y make wget libtool m4 autoconf dh-exec libdebhelper-perl=13.6ubuntu1~bpo20.04.1 debhelper=13.6ubuntu1~bpo20.04.1 \
cmake pkg-config python3-pip python cmake libgtest-dev libgmock-dev libyang-dev \
debhelper-compat dh-elpa dh-sequence-python3 python3-all \
libpython3-all-dev python3-six xmlto unzip rake-compiler gem2deb pkg-php-tools \
ant default-jdk maven-repo-helper libguava-java
wget http://ftp.us.debian.org/debian/pool/main/libg/libgoogle-gson-java/libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
sudo dpkg -i libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
mkdir -p /tmp/artifacts
displayName: "Install dependencies"
- script: |
SONIC_CONFIG_MAKE_JOBS=$(nproc) CONFIGURED_ARCH=amd64 DEST=/tmp/artifacts make -f ../rules/protobuf.mk -f protobuf/Makefile
workingDirectory: src
displayName: "Build protobuf"
- script: |
sudo dpkg -i protobuf-compiler_3.21.12-3_amd64.deb libprotoc32_3.21.12-3_amd64.deb \
libprotobuf32_3.21.12-3_amd64.deb libprotobuf-dev_3.21.12-3_amd64.deb \
libprotobuf-lite32_3.21.12-3_amd64.deb
workingDirectory: /tmp/artifacts
displayName: "Install protobuf"
- script: |
dpkg-buildpackage -rfakeroot -b -us -uc
workingDirectory: src/sonic-dash-api
displayName: "Build sonic-dash-api"
- script: |
cp *.deb /tmp/artifacts
workingDirectory: src
- publish: /tmp/artifacts
artifact: sonic-buildimage.amd64.ubuntu20_04
displayName: "Archive sonic-buildimage debian packages for ubuntu20.04"

3
.gitmodules vendored
View File

@ -115,3 +115,6 @@
[submodule "src/dhcpmon"]
path = src/dhcpmon
url = https://github.com/sonic-net/sonic-dhcpmon.git
[submodule "src/sonic-dash-api/sonic-dash-api"]
path = src/sonic-dash-api/sonic-dash-api
url = https://github.com/sonic-net/sonic-dash-api.git

9
rules/protobuf.dep Normal file
View File

@ -0,0 +1,9 @@
SPATH := $($(PROTOBUF)_SRC_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/protobuf.mk rules/protobuf.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
DEP_FILES += $(shell git ls-files $(SPATH))
$(PROTOBUF)_CACHE_MODE := GIT_CONTENT_SHA
$(PROTOBUF)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
$(PROTOBUF)_DEP_FILES := $(DEP_FILES)

36
rules/protobuf.mk Normal file
View File

@ -0,0 +1,36 @@
# protobuf package
PROTOBUF_VERSION = 3.21.12
PROTOBUF_VERSION_FULL = $(PROTOBUF_VERSION)-3
export PROTOBUF_VERSION
export PROTOBUF_VERSION_FULL
PROTOBUF = libprotobuf32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
$(PROTOBUF)_SRC_PATH = $(SRC_PATH)/protobuf
SONIC_MAKE_DEBS += $(PROTOBUF)
PROTOBUF_DEV = libprotobuf-dev_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
$(PROTOBUF_DEV)_DEPENDS = $(PROTOBUF) $(PROTOBUF_LITE)
$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_DEV)))
PROTOBUF_LITE = libprotobuf-lite32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_LITE)))
PROTOC = libprotoc_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
$(PROTOC)_RDEPENDS = $(PROTOBUF) $(PROTOBUF_LITE)
$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOC)))
PROTOC32 = libprotoc32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
$(PROTOC32)_RDEPENDS = $(PROTOBUF) $(PROTOBUF_LITE)
$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOC32)))
PROTOBUF_COMPILER = protobuf-compiler_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
$(PROTOBUF_COMPILER)_DEPENDS = $(PROTOC32)
$(PROTOBUF_COMPILER)_RDEPENDS = $(PROTOC)
$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_COMPILER)))
PYTHON3_PROTOBUF = python3-protobuf_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
$(PYTHON3_PROTOBUF)_DEPENDS = $(PROTOBUF_DEV) $(PROTOBUF)
$(PYTHON3_PROTOBUF)_RDEPENDS = $(PROTOBUF)
$(eval $(call add_derived_package,$(PROTOBUF),$(PYTHON3_PROTOBUF)))

12
rules/sonic-dash-api.dep Normal file
View File

@ -0,0 +1,12 @@
SPATH := $($(LIB_SONIC_DASH_API)_SRC_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-dash-api.mk rules/sonic-dash-api.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
$(LIB_SONIC_DASH_API)_CACHE_MODE := GIT_CONTENT_SHA
$(LIB_SONIC_DASH_API)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
$(LIB_SONIC_DASH_API)_DEP_FILES := $(DEP_FILES)
$(LIB_SONIC_DASH_API)_SMDEP_FILES := $(SMDEP_FILES)
$(LIB_SONIC_DASH_API)_SMDEP_PATHS := $(SPATH)

20
rules/sonic-dash-api.mk Normal file
View File

@ -0,0 +1,20 @@
# libdashsai package
LIB_SONIC_DASH_API_VERSION = 1.0.0
LIB_SONIC_DASH_API = libdashapi_$(LIB_SONIC_DASH_API_VERSION)_$(CONFIGURED_ARCH).deb
$(LIB_SONIC_DASH_API)_SRC_PATH = $(SRC_PATH)/sonic-dash-api
$(LIB_SONIC_DASH_API)_DEPENDS += $(PROTOBUF) $(PROTOBUF_LITE) $(PROTOBUF_DEV) $(PROTOBUF_COMPILER)
$(LIB_SONIC_DASH_API)_RDEPENDS += $(LIBSWSSCOMMON) $(LIBNL3) $(LIBNL_GENL3) $(LIBNL_ROUTE3) $(PROTOBUF) $(PROTOBUF_LITE)
SONIC_DPKG_DEBS += $(LIB_SONIC_DASH_API)
LIB_SONIC_DASH_API_DBG = libdashapi-dbgsym_$(LIB_SONIC_DASH_API_VERSION)_$(CONFIGURED_ARCH).deb
$(eval $(call add_derived_package,$(LIB_SONIC_DASH_API),$(LIB_SONIC_DASH_API_DBG)))
# The .c, .cpp, .h & .hpp files under src/{$DBG_SRC_ARCHIVE list}
# are archived into debug one image to facilitate debugging.
#
DBG_SRC_ARCHIVE += sonic-dash-api

6
rules/swss.mk
View File

@ -4,11 +4,13 @@ SWSS = swss_1.0.0_$(CONFIGURED_ARCH).deb
$(SWSS)_SRC_PATH = $(SRC_PATH)/sonic-swss
$(SWSS)_DEPENDS += $(LIBSAIREDIS_DEV) $(LIBSAIMETADATA_DEV) $(LIBTEAM_DEV) \
$(LIBTEAMDCTL) $(LIBTEAM_UTILS) $(LIBSWSSCOMMON_DEV) \
$(LIBSAIVS) $(LIBSAIVS_DEV)
$(LIBSAIVS) $(LIBSAIVS_DEV) \
$(PROTOBUF) $(PROTOBUF_LITE) $(PROTOBUF_DEV) $(LIB_SONIC_DASH_API)
$(SWSS)_UNINSTALLS = $(LIBSAIVS_DEV)
$(SWSS)_RDEPENDS += $(LIBSAIREDIS) $(LIBSAIMETADATA) $(LIBTEAM) \
$(LIBTEAMDCTL) $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON)
$(LIBTEAMDCTL) $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON) \
$(PROTOBUF) $(PROTOBUF_LITE) $(PYTHON3_PROTOBUF) $(LIB_SONIC_DASH_API)
SONIC_DPKG_DEBS += $(SWSS)
SWSS_DBG = swss-dbg_1.0.0_$(CONFIGURED_ARCH).deb

10
sonic-slave-bullseye/Dockerfile.j2
View File

@ -344,8 +344,6 @@ RUN apt-get update && apt-get install -y \
libnetfilter-conntrack-dev \
libnftnl-dev \
# For SAI3.7
protobuf-compiler \
libprotobuf-dev \
xxd \
# For DHCP Monitor tool
libexplain-dev \
@ -408,7 +406,13 @@ RUN apt-get update && apt-get install -y \
pkg-config \
# For audisp-tacplus
libauparse-dev \
auditd
auditd \
# For protobuf
dh-elpa \
xmlto \
rake-compiler \
default-jdk \
libgoogle-gson-java
{%- if CROSS_BUILD_ENVIRON == "y" %}
# Arm vs. amd64 versions conflict - remove amd64 packages

33
src/protobuf/Makefile Normal file
View File

@ -0,0 +1,33 @@
SHELL = /bin/bash
.ONESHELL:
.SHELLFLAGS += -e
MAIN_TARGET = libprotobuf32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
DERIVED_TARGETS = protobuf-compiler_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
libprotobuf-dev_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
libprotobuf-lite32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
libprotoc32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
python3-protobuf_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
# Remove any stale files
rm -rf protobuf-$(PROTOBUF_VERSION)
wget -O protobuf_$(PROTOBUF_VERSION).orig.tar.gz http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION).orig.tar.gz
wget -O protobuf_$(PROTOBUF_VERSION_FULL).dsc http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION_FULL).dsc
wget -O protobuf_$(PROTOBUF_VERSION_FULL).debian.tar.xz http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION_FULL).debian.tar.xz
dpkg-source -x protobuf_$(PROTOBUF_VERSION_FULL).dsc
pushd protobuf-$(PROTOBUF_VERSION)
ifeq ($(CROSS_BUILD_ENVIRON), y)
dpkg-buildpackage -us -uc -b -a$(CONFIGURED_ARCH) -Pcross,nocheck -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
else
dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
endif
popd
mv $(DERIVED_TARGETS) $* $(DEST)/
$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET)

2
src/sonic-dash-api/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
build
debian/sonic-dash-api

47
src/sonic-dash-api/Makefile Normal file
View File

@ -0,0 +1,47 @@
.ONESHELL:
SHELL = /bin/bash
.SHELLFLAGS += -e
RM := rm -rf
CP := cp -rf
MKDIR := mkdir
MV := mv
LIBDASHAPI := libdashapi.so
BUILD_DIR := build
PYPKG_DIR := pypkg
DESTDIR :=
DASH_API_PROTO_DIR := sonic-dash-api/proto
INSTALLED_HEADER_DIR := $(DESTDIR)/usr/include/dash_api
INSTALLED_LIB_DIR := $(DESTDIR)/usr/lib
INSTALLED_PYTHON_DIR := $(DESTDIR)/usr/lib/python3/dist-packages/dash_api
all: compile_cpp_proto dashapi.so compile_py_proto
compile_cpp_proto:
$(MKDIR) -p $(BUILD_DIR)
protoc -I=$(DASH_API_PROTO_DIR) --cpp_out=$(BUILD_DIR) $(DASH_API_PROTO_DIR)/*.proto
dashapi.so: compile_cpp_proto
g++ -std=c++14 -fPIC -shared -o $(BUILD_DIR)/$(LIBDASHAPI) $(wildcard $(BUILD_DIR)/*.pb.cc) -lprotobuf
compile_py_proto:
protoc -I=$(DASH_API_PROTO_DIR) --python_out=$(PYPKG_DIR) $(DASH_API_PROTO_DIR)/*.proto
clean:
$(RM) $(BUILD_DIR)
$(RM) $(PYPKG_DIR)/*_pb2.py
install:
$(MKDIR) -p $(INSTALLED_HEADER_DIR)
$(CP) $(BUILD_DIR)/*.pb.h $(INSTALLED_HEADER_DIR)
$(MKDIR) -p $(INSTALLED_LIB_DIR)
$(CP) $(BUILD_DIR)/$(LIBDASHAPI) $(INSTALLED_LIB_DIR)
$(MKDIR) -p $(INSTALLED_PYTHON_DIR)
$(CP) $(PYPKG_DIR)/* $(INSTALLED_PYTHON_DIR)
uninstall:
$(RM) $(INSTALLED_HEADER_DIR)
$(RM) $(INSTALLED_LIB_DIR)/$(LIBDASHAPI)
$(RM) $(INSTALLED_PYTHON_DIR)
.PHONY: uninstall clean

6
src/sonic-dash-api/debian/changelog Normal file
View File

@ -0,0 +1,6 @@
sonic (1.0.0) stable; urgency=medium
* Initial release.
-- Ze Gan <zegan@microsoft.com> Wed, 14 Jun 2023 12:00:00 -0800

1
src/sonic-dash-api/debian/compat Normal file
View File

@ -0,0 +1 @@
11

13
src/sonic-dash-api/debian/control Normal file
View File

@ -0,0 +1,13 @@
Source: sonic
Maintainer: Ze Gan <zegan@microsoft.com>
Section: net
Priority: optional
Build-Depends: dh-exec (>=0.3), debhelper (>= 12), autotools-dev,
Standards-Version: 1.0.0
Package: libdashapi
Architecture: any
Build-Depends: protobuf-compiler (>=3.21.12), libprotobuf-dev (>=3.21.12)
Depends: libprotobuf32 (>=3.21.12), libprotobuf-lite32 (>=3.21.12)
Section: libs
Description: DASH API definition for the SONiC project.

8
src/sonic-dash-api/debian/rules Executable file
View File

@ -0,0 +1,8 @@
#!/usr/bin/make -f
#export DH_VERBOSE = 1
%:
dh $@
override_dh_auto_build:
make all

5
src/sonic-dash-api/pypkg/__init__.py Normal file
View File

@ -0,0 +1,5 @@
import sys
import os
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))

1
src/sonic-dash-api/sonic-dash-api Submodule

@ -0,0 +1 @@
Subproject commit 30415cd41ef2ae62841417b4a6a9113129cd718d