Add AZP agent necessary packages to sonic-mgmt-docker (#14291)

Why I did it
Add AZP agent necessary packages to sonic-mgmt-docker
Remove Python 201811 venv
Update some packages in order to meet internal security requirements

How I did it
Update sonic-mgmt-docker file

How to verify it
sonic-mgmt-docker can run: bash, apt update, apt install and ping.
start.sh is under /azp with exec permission.
env-201811 venv is removed.
jinja2 is upgrade to 2.10.1

dockers/docker-sonic-mgmt/Dockerfile.j2

@@ -3,18 +3,23 @@ FROM {{ prefix }}ubuntu:20.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y build-essential \
+RUN apt-get update && apt-get install -y apt-transport-https \
+                                         apt-utils \
+                                         build-essential \
+                                         ca-certificates \
                                          cmake \
                                          curl \
                                          default-jre \
                                          git \
-                                         inetutils-ping \
                                          iproute2 \
+                                         iputils-ping \
                                          isc-dhcp-client \
                                          libffi-dev \
                                          libssl-dev \
                                          libxml2 \
                                          libxslt1-dev \
+                                         lsb-release \
+                                         jq \
                                          openssh-server \
                                          psmisc \
                                          python \
@@ -56,7 +61,7 @@ RUN pip install allure-pytest==2.8.22 \
                 ipython==5.4.1 \
                 ixnetwork-restpy==1.0.64 \
                 ixnetwork-open-traffic-generator==0.0.79 \
-                jinja2==2.7.2 \
+                jinja2==2.10.1 \
                 jsonpatch \
                 lazy-object-proxy==1.6.0 \
                 lxml \
@@ -172,10 +177,6 @@ WORKDIR /var/$user
 # Add az symlink for backwards compatibility
 RUN mkdir bin && ln -s /usr/bin/az bin/az
 
-# Install Virtual Environments
-RUN python -m virtualenv --system-site-packages env-201811
-RUN env-201811/bin/pip install cryptography==3.3.2 ansible==2.0.0.2
-
 RUN python3 -m venv env-python3
 
 # Activating a virtualenv. The virtualenv automatically works for RUN, ENV and CMD.
@@ -204,7 +205,7 @@ RUN python3 -m pip install aiohttp \
                            ipython==5.4.1 \
                            ixnetwork-restpy==1.0.64 \
                            ixnetwork-open-traffic-generator==0.0.79 \
-                           jinja2==2.7.2 \
+                           jinja2==2.10.1 \
                            jsonpatch \
                            lxml \
                            markupsafe==2.0.1 \
@@ -250,3 +251,11 @@ RUN python3 -m pip install aiohttp \
 
 # Deactivating a virtualenv
 ENV PATH="$BACKUP_OF_PATH"
+
+USER root
+WORKDIR /azp
+COPY ./start.sh .
+RUN chmod +x start.sh
+
+USER $user
+WORKDIR /var/$user

dockers/docker-sonic-mgmt/start.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+set -e
+
+if [ -z "$AZP_URL" ]; then
+  echo 1>&2 "error: missing AZP_URL environment variable"
+  exit 1
+fi
+
+if [ -z "$AZP_TOKEN_FILE" ]; then
+  if [ -z "$AZP_TOKEN" ]; then
+    echo 1>&2 "error: missing AZP_TOKEN environment variable"
+    exit 1
+  fi
+
+  AZP_TOKEN_FILE=/azp/.token
+  echo -n $AZP_TOKEN > "$AZP_TOKEN_FILE"
+fi
+
+unset AZP_TOKEN
+
+if [ -n "$AZP_WORK" ]; then
+  mkdir -p "$AZP_WORK"
+fi
+
+export AGENT_ALLOW_RUNASROOT="1"
+
+cleanup() {
+  if [ -e config.sh ]; then
+    print_header "Cleanup. Removing Azure Pipelines agent..."
+
+    # If the agent has some running jobs, the configuration removal process will fail.
+    # So, give it some time to finish the job.
+    while true; do
+      ./config.sh remove --unattended --auth PAT --token $(cat "$AZP_TOKEN_FILE") && break
+
+      echo "Retrying in 30 seconds..."
+      sleep 30
+    done
+  fi
+}
+
+print_header() {
+  lightcyan='\033[1;36m'
+  nocolor='\033[0m'
+  echo -e "${lightcyan}$1${nocolor}"
+}
+
+# Let the agent ignore the token env variables
+export VSO_AGENT_IGNORE=AZP_TOKEN,AZP_TOKEN_FILE
+
+print_header "1. Determining matching Azure Pipelines agent..."
+
+AZP_AGENT_PACKAGES=$(curl -LsS \
+    -u user:$(cat "$AZP_TOKEN_FILE") \
+    -H 'Accept:application/json;' \
+    "$AZP_URL/_apis/distributedtask/packages/agent?platform=$TARGETARCH&top=1")
+
+AZP_AGENT_PACKAGE_LATEST_URL=$(echo "$AZP_AGENT_PACKAGES" | jq -r '.value[0].downloadUrl')
+
+if [ -z "$AZP_AGENT_PACKAGE_LATEST_URL" -o "$AZP_AGENT_PACKAGE_LATEST_URL" == "null" ]; then
+  echo 1>&2 "error: could not determine a matching Azure Pipelines agent"
+  echo 1>&2 "check that account '$AZP_URL' is correct and the token is valid for that account"
+  exit 1
+fi
+
+print_header "2. Downloading and extracting Azure Pipelines agent..."
+
+curl -LsS $AZP_AGENT_PACKAGE_LATEST_URL | tar -xz & wait $!
+
+source ./env.sh
+
+trap 'cleanup; exit 0' EXIT
+trap 'cleanup; exit 130' INT
+trap 'cleanup; exit 143' TERM
+
+print_header "3. Configuring Azure Pipelines agent..."
+
+./config.sh --unattended \
+  --agent "${AZP_AGENT_NAME:-$(hostname)}" \
+  --url "$AZP_URL" \
+  --auth PAT \
+  --token $(cat "$AZP_TOKEN_FILE") \
+  --pool "${AZP_POOL:-Default}" \
+  --work "${AZP_WORK:-_work}" \
+  --replace \
+  --acceptTeeEula & wait $!
+
+print_header "4. Running Azure Pipelines agent..."
+
+trap 'cleanup; exit 0' EXIT
+trap 'cleanup; exit 130' INT
+trap 'cleanup; exit 143' TERM
+
+chmod +x ./run.sh
+
+# To be aware of TERM and INT signals call run.sh
+# Running it with the --once flag at the end will shut down the agent after the build is executed
+./run.sh "$@" & wait $!
+