[dash-api]: Add dash-api and related protobuf library (#14515)

Why I did it
For the DASH scenario, the APP_DB will be optimized by protobuf message for less memory consumption.

How I did it
Download the Debian package of protobuf 3.21.12 and create a corresponding rule for building it.
Add a submodule of sonic-dash-api and generated its Debian package which includes C++ library and Python library

How to verify it
Check artifacts of Azp that the protobuf-related and dash-api deb packages should be generated.

Signed-off-by: Ze Gan <ganze718@gmail.com>

.azure-pipelines/azure-pipelines-build-ubuntu-2004.yml

@@ -0,0 +1,55 @@
+# C/C++ with GCC
+# Build your C/C++ project with GCC using make.
+# Add steps that publish test results, save build artifacts, deploy, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/apps/c-cpp/gcc
+pr: none
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+
+stages:
+- stage: Build
+
+  jobs:
+  - job:
+    displayName: "amd64/ubuntu-20.04"
+    pool:
+      vmImage: 'ubuntu-20.04'
+
+    steps:
+    - checkout: self
+      submodules: true
+    - script: |
+        sudo apt-get update
+        sudo apt-get install -y make wget libtool m4 autoconf dh-exec libdebhelper-perl=13.6ubuntu1~bpo20.04.1 debhelper=13.6ubuntu1~bpo20.04.1 \
+                         cmake pkg-config python3-pip python cmake libgtest-dev libgmock-dev libyang-dev \
+                         debhelper-compat dh-elpa dh-sequence-python3 python3-all \
+                         libpython3-all-dev python3-six xmlto unzip rake-compiler gem2deb pkg-php-tools \
+                         ant default-jdk maven-repo-helper libguava-java
+        wget http://ftp.us.debian.org/debian/pool/main/libg/libgoogle-gson-java/libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
+        sudo dpkg -i libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
+        mkdir -p /tmp/artifacts
+      displayName: "Install dependencies"
+    - script: |
+        SONIC_CONFIG_MAKE_JOBS=$(nproc) CONFIGURED_ARCH=amd64 DEST=/tmp/artifacts make -f ../rules/protobuf.mk -f protobuf/Makefile
+      workingDirectory: src
+      displayName: "Build protobuf"
+    - script: |
+        sudo dpkg -i protobuf-compiler_3.21.12-3_amd64.deb libprotoc32_3.21.12-3_amd64.deb \
+                      libprotobuf32_3.21.12-3_amd64.deb libprotobuf-dev_3.21.12-3_amd64.deb \
+                      libprotobuf-lite32_3.21.12-3_amd64.deb
+      workingDirectory: /tmp/artifacts
+      displayName: "Install protobuf"
+    - script: |
+        dpkg-buildpackage -rfakeroot -b -us -uc
+      workingDirectory: src/sonic-dash-api
+      displayName: "Build sonic-dash-api"
+    - script: |
+        cp *.deb /tmp/artifacts
+      workingDirectory: src
+    - publish: /tmp/artifacts
+      artifact: sonic-buildimage.amd64.ubuntu20_04
+      displayName: "Archive sonic-buildimage debian packages for ubuntu20.04"

.gitmodules

@@ -115,3 +115,6 @@
 [submodule "src/dhcpmon"]
 	path = src/dhcpmon
 	url = https://github.com/sonic-net/sonic-dhcpmon.git
+[submodule "src/sonic-dash-api/sonic-dash-api"]
+	path = src/sonic-dash-api/sonic-dash-api
+	url = https://github.com/sonic-net/sonic-dash-api.git

rules/protobuf.dep

@@ -0,0 +1,9 @@
+
+SPATH       := $($(PROTOBUF)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/protobuf.mk rules/protobuf.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(SPATH))
+
+$(PROTOBUF)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(PROTOBUF)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(PROTOBUF)_DEP_FILES   := $(DEP_FILES)

rules/protobuf.mk

@@ -0,0 +1,36 @@
+# protobuf package
+
+PROTOBUF_VERSION = 3.21.12
+PROTOBUF_VERSION_FULL = $(PROTOBUF_VERSION)-3
+
+export PROTOBUF_VERSION
+export PROTOBUF_VERSION_FULL
+
+PROTOBUF = libprotobuf32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
+$(PROTOBUF)_SRC_PATH = $(SRC_PATH)/protobuf
+SONIC_MAKE_DEBS += $(PROTOBUF)
+
+PROTOBUF_DEV = libprotobuf-dev_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
+$(PROTOBUF_DEV)_DEPENDS = $(PROTOBUF) $(PROTOBUF_LITE)
+$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_DEV)))
+
+PROTOBUF_LITE = libprotobuf-lite32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
+$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_LITE)))
+
+PROTOC = libprotoc_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
+$(PROTOC)_RDEPENDS = $(PROTOBUF) $(PROTOBUF_LITE)
+$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOC)))
+
+PROTOC32 = libprotoc32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
+$(PROTOC32)_RDEPENDS = $(PROTOBUF) $(PROTOBUF_LITE)
+$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOC32)))
+
+PROTOBUF_COMPILER = protobuf-compiler_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
+$(PROTOBUF_COMPILER)_DEPENDS = $(PROTOC32)
+$(PROTOBUF_COMPILER)_RDEPENDS = $(PROTOC)
+$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_COMPILER)))
+
+PYTHON3_PROTOBUF = python3-protobuf_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
+$(PYTHON3_PROTOBUF)_DEPENDS = $(PROTOBUF_DEV) $(PROTOBUF)
+$(PYTHON3_PROTOBUF)_RDEPENDS = $(PROTOBUF)
+$(eval $(call add_derived_package,$(PROTOBUF),$(PYTHON3_PROTOBUF)))

rules/sonic-dash-api.dep

@@ -0,0 +1,12 @@
+
+SPATH       := $($(LIB_SONIC_DASH_API)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-dash-api.mk rules/sonic-dash-api.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(LIB_SONIC_DASH_API)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(LIB_SONIC_DASH_API)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(LIB_SONIC_DASH_API)_DEP_FILES   := $(DEP_FILES)
+$(LIB_SONIC_DASH_API)_SMDEP_FILES := $(SMDEP_FILES)
+$(LIB_SONIC_DASH_API)_SMDEP_PATHS := $(SPATH)
+

rules/sonic-dash-api.mk

@@ -0,0 +1,20 @@
+# libdashsai package
+
+LIB_SONIC_DASH_API_VERSION = 1.0.0
+
+LIB_SONIC_DASH_API = libdashapi_$(LIB_SONIC_DASH_API_VERSION)_$(CONFIGURED_ARCH).deb
+$(LIB_SONIC_DASH_API)_SRC_PATH = $(SRC_PATH)/sonic-dash-api
+
+$(LIB_SONIC_DASH_API)_DEPENDS += $(PROTOBUF) $(PROTOBUF_LITE) $(PROTOBUF_DEV) $(PROTOBUF_COMPILER)
+$(LIB_SONIC_DASH_API)_RDEPENDS += $(LIBSWSSCOMMON) $(LIBNL3) $(LIBNL_GENL3) $(LIBNL_ROUTE3) $(PROTOBUF) $(PROTOBUF_LITE) 
+
+SONIC_DPKG_DEBS += $(LIB_SONIC_DASH_API)
+
+LIB_SONIC_DASH_API_DBG = libdashapi-dbgsym_$(LIB_SONIC_DASH_API_VERSION)_$(CONFIGURED_ARCH).deb
+$(eval $(call add_derived_package,$(LIB_SONIC_DASH_API),$(LIB_SONIC_DASH_API_DBG)))
+
+# The .c, .cpp, .h & .hpp files under src/{$DBG_SRC_ARCHIVE list}
+# are archived into debug one image to facilitate debugging.
+#
+DBG_SRC_ARCHIVE += sonic-dash-api
+

rules/swss.mk

@@ -4,11 +4,13 @@ SWSS = swss_1.0.0_$(CONFIGURED_ARCH).deb
 $(SWSS)_SRC_PATH = $(SRC_PATH)/sonic-swss
 $(SWSS)_DEPENDS += $(LIBSAIREDIS_DEV) $(LIBSAIMETADATA_DEV) $(LIBTEAM_DEV) \
     $(LIBTEAMDCTL) $(LIBTEAM_UTILS) $(LIBSWSSCOMMON_DEV) \
-    $(LIBSAIVS) $(LIBSAIVS_DEV)
+    $(LIBSAIVS) $(LIBSAIVS_DEV) \
+    $(PROTOBUF) $(PROTOBUF_LITE) $(PROTOBUF_DEV) $(LIB_SONIC_DASH_API)
 $(SWSS)_UNINSTALLS = $(LIBSAIVS_DEV)
 
 $(SWSS)_RDEPENDS += $(LIBSAIREDIS) $(LIBSAIMETADATA) $(LIBTEAM) \
-    $(LIBTEAMDCTL) $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON)
+    $(LIBTEAMDCTL) $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON) \
+    $(PROTOBUF) $(PROTOBUF_LITE) $(PYTHON3_PROTOBUF) $(LIB_SONIC_DASH_API)
 SONIC_DPKG_DEBS += $(SWSS)
 
 SWSS_DBG = swss-dbg_1.0.0_$(CONFIGURED_ARCH).deb

sonic-slave-bullseye/Dockerfile.j2

@@ -344,8 +344,6 @@ RUN apt-get update && apt-get install -y \
         libnetfilter-conntrack-dev \
         libnftnl-dev \
 # For SAI3.7
-        protobuf-compiler \
-        libprotobuf-dev \
         xxd \
 # For DHCP Monitor tool
         libexplain-dev \
@@ -408,7 +406,13 @@ RUN apt-get update && apt-get install -y \
         pkg-config \
 # For audisp-tacplus
         libauparse-dev \
-        auditd
+        auditd \
+# For protobuf
+        dh-elpa \
+        xmlto \
+        rake-compiler \
+        default-jdk \
+        libgoogle-gson-java
 
 {%- if CROSS_BUILD_ENVIRON == "y" %}
 # Arm vs. amd64 versions conflict - remove amd64 packages

src/protobuf/Makefile

@@ -0,0 +1,33 @@
+SHELL = /bin/bash
+.ONESHELL:
+.SHELLFLAGS += -e
+
+MAIN_TARGET = libprotobuf32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
+DERIVED_TARGETS = protobuf-compiler_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
+					libprotobuf-dev_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
+					libprotobuf-lite32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
+					libprotoc32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
+					python3-protobuf_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+	# Remove any stale files
+	rm -rf protobuf-$(PROTOBUF_VERSION)
+
+	wget -O protobuf_$(PROTOBUF_VERSION).orig.tar.gz http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION).orig.tar.gz
+	wget -O protobuf_$(PROTOBUF_VERSION_FULL).dsc http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION_FULL).dsc
+	wget -O protobuf_$(PROTOBUF_VERSION_FULL).debian.tar.xz http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION_FULL).debian.tar.xz
+	dpkg-source -x protobuf_$(PROTOBUF_VERSION_FULL).dsc
+
+	pushd protobuf-$(PROTOBUF_VERSION)
+
+ifeq ($(CROSS_BUILD_ENVIRON), y)
+	dpkg-buildpackage -us -uc -b -a$(CONFIGURED_ARCH) -Pcross,nocheck -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
+else
+	dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
+endif
+
+	popd
+
+	mv $(DERIVED_TARGETS) $* $(DEST)/
+
+$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET)

src/sonic-dash-api/.gitignore

@@ -0,0 +1,2 @@
+build
+debian/sonic-dash-api

src/sonic-dash-api/Makefile

@@ -0,0 +1,47 @@
+.ONESHELL:
+SHELL = /bin/bash
+.SHELLFLAGS += -e
+
+RM := rm -rf
+CP := cp -rf
+MKDIR := mkdir
+MV := mv
+LIBDASHAPI := libdashapi.so
+BUILD_DIR := build
+PYPKG_DIR := pypkg
+DESTDIR := 
+DASH_API_PROTO_DIR := sonic-dash-api/proto
+INSTALLED_HEADER_DIR := $(DESTDIR)/usr/include/dash_api
+INSTALLED_LIB_DIR := $(DESTDIR)/usr/lib
+INSTALLED_PYTHON_DIR := $(DESTDIR)/usr/lib/python3/dist-packages/dash_api
+
+all: compile_cpp_proto dashapi.so compile_py_proto
+
+compile_cpp_proto:
+	$(MKDIR) -p $(BUILD_DIR)
+	protoc -I=$(DASH_API_PROTO_DIR) --cpp_out=$(BUILD_DIR) $(DASH_API_PROTO_DIR)/*.proto
+
+dashapi.so: compile_cpp_proto
+	g++ -std=c++14 -fPIC -shared -o $(BUILD_DIR)/$(LIBDASHAPI) $(wildcard $(BUILD_DIR)/*.pb.cc) -lprotobuf
+
+compile_py_proto:
+	protoc -I=$(DASH_API_PROTO_DIR) --python_out=$(PYPKG_DIR) $(DASH_API_PROTO_DIR)/*.proto
+
+clean:
+	$(RM) $(BUILD_DIR)
+	$(RM) $(PYPKG_DIR)/*_pb2.py
+
+install:
+	$(MKDIR) -p $(INSTALLED_HEADER_DIR)
+	$(CP) $(BUILD_DIR)/*.pb.h $(INSTALLED_HEADER_DIR)
+	$(MKDIR) -p $(INSTALLED_LIB_DIR)
+	$(CP) $(BUILD_DIR)/$(LIBDASHAPI) $(INSTALLED_LIB_DIR)
+	$(MKDIR) -p $(INSTALLED_PYTHON_DIR)
+	$(CP) $(PYPKG_DIR)/* $(INSTALLED_PYTHON_DIR)
+
+uninstall:
+	$(RM) $(INSTALLED_HEADER_DIR)
+	$(RM) $(INSTALLED_LIB_DIR)/$(LIBDASHAPI)
+	$(RM) $(INSTALLED_PYTHON_DIR)
+
+.PHONY: uninstall clean

src/sonic-dash-api/debian/changelog

@@ -0,0 +1,6 @@
+sonic (1.0.0) stable; urgency=medium
+
+  * Initial release.
+
+ -- Ze Gan <zegan@microsoft.com>  Wed, 14 Jun 2023 12:00:00 -0800
+

src/sonic-dash-api/debian/compat

@@ -0,0 +1 @@
+11

src/sonic-dash-api/debian/control

@@ -0,0 +1,13 @@
+Source: sonic
+Maintainer: Ze Gan <zegan@microsoft.com>
+Section: net
+Priority: optional
+Build-Depends: dh-exec (>=0.3), debhelper (>= 12), autotools-dev,
+Standards-Version: 1.0.0
+
+Package: libdashapi
+Architecture: any
+Build-Depends: protobuf-compiler (>=3.21.12), libprotobuf-dev (>=3.21.12)
+Depends: libprotobuf32 (>=3.21.12), libprotobuf-lite32 (>=3.21.12)
+Section: libs
+Description: DASH API definition for the SONiC project.

src/sonic-dash-api/debian/rules

@@ -0,0 +1,8 @@
+#!/usr/bin/make -f
+#export DH_VERBOSE = 1
+
+%:
+	dh $@
+
+override_dh_auto_build:
+	make all

src/sonic-dash-api/pypkg/__init__.py

@@ -0,0 +1,5 @@
+import sys
+import os
+
+
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))

src/sonic-dash-api/sonic-dash-api

@@ -0,0 +1 @@
+Subproject commit 30415cd41ef2ae62841417b4a6a9113129cd718d