docs: better docs, mv doxygen to sphinx
This patch refactors the VPP sphinx docs
in order to make it easier to consume
for external readers as well as VPP developers.
It also makes sphinx the single source
of documentation, which simplifies maintenance
and operation.
Most important updates are:
- reformat the existing documentation as rst
- split RELEASE.md and move it into separate rst files
- remove section 'events'
- remove section 'archive'
- remove section 'related projects'
- remove section 'feature by release'
- remove section 'Various links'
- make (Configuration reference, CLI docs,
developer docs) top level items in the list
- move 'Use Cases' as part of 'About VPP'
- move 'Troubleshooting' as part of 'Getting Started'
- move test framework docs into 'Developer Documentation'
- add a 'Contributing' section for gerrit,
docs and other contributer related infos
- deprecate doxygen and test-docs targets
- redirect the "make doxygen" target to "make docs"
Type: refactor
Change-Id: I552a5645d5b7964d547f99b1336e2ac24e7c209f
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>
diff --git a/docs/Makefile b/docs/Makefile
index 4993864..ca4a3ac 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -5,33 +5,46 @@
OS_ID = darwin
endif
+# These should be passed in by the root Makefile
+WS_ROOT ?= $(CURDIR)/..
+BR ?= $(WS_ROOT)/build-root
+DOCS_DIR ?= $(WS_ROOT)/docs
+
+VENV_DIR ?= $(DOCS_DIR)/venv
+SPHINX_SCRIPTS_DIR ?= $(WS_ROOT)/docs/scripts
+
# Work out the OS if we haven't already
OS_ID ?= $(shell grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+OS_VERSION ?= $(shell grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+PIP_VERSION ?= $(shell grep 'PIP_VERSION=' ${WS_ROOT}/test/Makefile | cut -d'=' -f2)
+PIP_TOOLS_VERSION ?= $(shell grep 'PIP_TOOLS_VERSION=' ${WS_ROOT}/test/Makefile | cut -d'=' -f2)
+
+PYTHON ?= "python3"
DOC_DEB_DEPENDS = enchant
DOC_RPM_DEPENDS = enchant
# You can set these variables from the command line.
-SPHINXOPTS =
-SPHINXBUILD = sphinx-build
-SPHINXPROJ = fdio-vpp
-SOURCEDIR = .
-BUILDDIR = _build
+SPHINXOPTS = --keep-going -n -W
+SPHINXBUILD = sphinx-build
+SPHINXPROJ = fdio-vpp
+SOURCEDIR = .
+BUILDDIR = ${BR}/docs
+BUILDDIR_SRC = ${BUILDDIR}/src
+BUILDDIR_OUT = ${BUILDDIR}/html
+SCRIPTS_DIR = _scripts
+
# Put it first so that "make" without argument is like "make help".
+.PHONY: help
help:
- @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+ @( \
+ . ${VENV_DIR}/bin/activate; \
+ $(SPHINXBUILD) --help ;\
+ )
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-# Generate dynamic content
- @python3 ./includes_renderer.py
- @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-spell:
+.PHONY: checkdeps
+checkdeps:
@echo "Checking whether dependencies for Docs are installed..."
ifeq ($(OS_ID),ubuntu)
@set -e; inst=; \
@@ -45,4 +58,45 @@
else ifneq ("$(wildcard /etc/redhat-release)","")
@sudo yum install $(CONFIRM) $(DOC_RPM_DEPENDS)
endif
- @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" -W -b spelling $(O)
+
+.PHONY: spell
+spell: clean checkdeps venv ${BUILDDIR_SRC}
+ @( \
+ . ${VENV_DIR}/bin/activate; \
+ make -C ${SCRIPTS_DIR} generate && \
+ $(SPHINXBUILD) -b spelling $(SPHINXOPTS) $(BUILDDIR_SRC) $(BUILDDIR_OUT); \
+ )
+
+.PHONY: venv
+venv:
+ @( \
+ if [ ! -d ${VENV_DIR} ]; then \
+ ${PYTHON} -m venv ${VENV_DIR}; \
+ . ${VENV_DIR}/bin/activate; \
+ ${PYTHON} -m pip install pip==${PIP_VERSION}; \
+ ${PYTHON} -m pip install pip-tools==${PIP_TOOLS_VERSION}; \
+ ${PYTHON} -m pip install -r ${WS_ROOT}/test/requirements-3.txt; \
+ fi; \
+ )
+
+${BUILDDIR_SRC}:
+ @mkdir -p ${BUILDDIR_SRC}
+ @cp -r $(SOURCEDIR) ${BUILDDIR_SRC}
+ @cd ${BUILDDIR_SRC} && find . -type l -exec cp --remove-destination -L ${DOCS_DIR}/{} {} \;
+
+.PHONY: docs
+docs: clean venv ${BUILDDIR_SRC}
+ @( \
+ . ${VENV_DIR}/bin/activate; \
+ make -C ${SCRIPTS_DIR} generate && \
+ $(SPHINXBUILD) $(SPHINXOPTS) -b html $(BUILDDIR_SRC) $(BUILDDIR_OUT); \
+ )
+
+.PHONY: clean
+clean:
+ @rm -rf $(BUILDDIR)
+ @make -C ${SCRIPTS_DIR} clean
+
+.PHONY: build
+build: docs
+
diff --git a/docs/README b/docs/README
index d9a7091..7e82709 100644
--- a/docs/README
+++ b/docs/README
@@ -23,7 +23,7 @@
Install the virtual environment
----------------------------------------------
-$ python -m pip install --user virtualenv
+$ python -m pip install --user virtualenv
$ python -m virtualenv env
$ source env/bin/activate
$ pip install -r docs/etc/requirements.txt
diff --git a/docs/_images/htmlBuild.png b/docs/_images/htmlBuild.png
deleted file mode 100644
index c95a643..0000000
--- a/docs/_images/htmlBuild.png
+++ /dev/null
Binary files differ
diff --git a/docs/_images/libmemif_architecture.png b/docs/_images/libmemif_architecture.png
new file mode 120000
index 0000000..77cbc4f
--- /dev/null
+++ b/docs/_images/libmemif_architecture.png
@@ -0,0 +1 @@
+../../extras/libmemif/docs/architecture.png
\ No newline at end of file
diff --git a/docs/_scripts/Makefile b/docs/_scripts/Makefile
new file mode 100644
index 0000000..dbd88bf
--- /dev/null
+++ b/docs/_scripts/Makefile
@@ -0,0 +1,212 @@
+# Copyright (c) 2021 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Build the documentation
+#
+
+# Default target
+.PHONY: all
+all: siphon
+
+# These should be passed in by the root Makefile
+WS_ROOT ?= $(CURDIR)/../..
+BR ?= $(WS_ROOT)/build-root
+
+# Tag used in github repository path.
+# Change this when genearting for a release
+VPP_TAG ?= master
+
+REPOSITORY_URL ?= https://github.com/FDio/vpp/blob/$(VPP_TAG)/
+
+# Doxygen configuration and our utility scripts
+SCRIPTS_DIR ?= $(WS_ROOT)/docs/_scripts
+
+# docs root directory
+DOCS_DIR ?= ${BR}/docs/src
+
+FEATURE_LIST_FILE = ${DOCS_DIR}/aboutvpp/featurelist.md
+
+# Siphoned fragements are processed into here
+DOCS_GENERATED_DIR ?= $(DOCS_DIR)/_generated
+
+# Siphoned fragments end up in here
+SIPHON_INPUT_DIR ?= $(DOCS_GENERATED_DIR)/fragments
+
+DYNAMIC_RENDER_DIR ?= ${DOCS_GENERATED_DIR}/includes
+
+# Primary source directories
+SIPHON_SRC ?= $(WS_ROOT)/src
+SIPHON_SRC_DIRECTORIES = \
+ $(shell find $(SIPHON_SRC) -name '*.md' -print | xargs dirname \
+ | sort | uniq) \
+ $(SIPHON_SRC)/vppinfra \
+ $(SIPHON_SRC)/svm \
+ $(SIPHON_SRC)/vlib \
+ $(SIPHON_SRC)/vlibapi \
+ $(SIPHON_SRC)/vlibmemory \
+ $(SIPHON_SRC)/vnet \
+ $(SIPHON_SRC)/vpp \
+ $(SIPHON_SRC)/vpp-api \
+ $(SIPHON_SRC)/examples
+
+# Input directories and files
+SIPHON_INPUT ?= \
+ $(wildcard $(WS_ROOT)/*.md) \
+ $(wildcard $(SCRIPTS_DIR)/*.md) \
+ $(SIPHON_SRC_DIRECTORIES) \
+ $(SIPHON_SRC)/plugins \
+ extras
+
+# Strip leading workspace path from input names
+SIPHON_INPUT := $(subst $(WS_ROOT)/,,$(SIPHON_INPUT))
+
+# Files to exclude, from pre-Doxygen steps, eg because they're
+# selectively compiled.
+# Examples would be to exclude non-DPDK related sources when
+# there's a DPDK equivalent that conflicts.
+# These must be left-anchored paths for the regexp below to work.
+SIPHON_EXCLUDE ?= \
+ $(SIPHON_SRC)/vpp-api/lua
+
+# Generate a regexp for filenames to exclude
+SIPHON_EXCLUDE_REGEXP = ($(subst .,\.,$(shell echo '$(strip $(SIPHON_EXCLUDE))' | sed -e 's/ /|/g')))
+
+# Include all the normal source directories in the include file path
+SIPHON_INCLUDE_PATH = $(SIPHON_SRC_DIRECTORIES)
+
+# Find API header directories and include them in the header path.
+# This is only useful if VPP and plugins are already built; nothing
+# here depends on those targets. We don't build documentation for these
+# header files, they're just added to the INCLUDE search path for Doxygen.
+_vpp_br = $(shell find "$(BR)" -maxdepth 1 -type d \
+ '(' -name build-vpp_debug-native -o -name build-vpp-native ')' -print \
+ | sed -e 's@^$(WS_ROOT)/*@@' -e 1q)
+ifneq ($(strip $(_vpp_br)),)
+SIPHON_INCLUDE_PATH += \
+ $(_vpp_br)/vlib-api \
+ $(_vpp_br)/vpp
+# Also include any plugin directories that exist
+SIPHON_INCLUDE_PATH += \
+ $(shell find $(WS_ROOT)/$(_vpp_br)/plugins -maxdepth 1 -type d | sed -e 's@^$(WS_ROOT)/*@@')
+endif
+
+# Discover if we have CPP available
+_cpp = $(shell which cpp)
+ifneq ($(strip $(_cpp)),)
+# Add whatever directories CPP normally includes to the header path
+SIPHON_INCLUDE_PATH += $(shell set -e; $(_cpp) -v </dev/null 2>&1 | awk 'f&&/^ /{print $$1} /^\#include/{f=1}')
+endif
+
+# All the siphon types we know about
+SIPHONS ?= clicmd syscfg
+
+SIPHON_FILES = $(addprefix $(SIPHON_INPUT_DIR)/,$(addsuffix .siphon,$(SIPHONS)))
+SIPHON_DOCS = $(addprefix $(DOCS_GENERATED_DIR)/,$(addsuffix .rst,$(SIPHONS)))
+
+BUILT_ON = $(shell date '+%d %B %Y')
+VPP_VERSION = $(shell ${WS_ROOT}/src/scripts/version)
+
+.PHONY: featurelist
+featurelist:
+ @( \
+ cd $(WS_ROOT) && \
+ find . -name FEATURE.yaml | \
+ ./src/scripts/fts.py \
+ --markdown \
+ --repolink $(REPOSITORY_URL) > \
+ $(FEATURE_LIST_FILE) ; \
+ )
+
+
+.PHONY: includes-render
+includes-render:
+ @mkdir -p "$(DYNAMIC_RENDER_DIR)"
+ @python3 $(SCRIPTS_DIR)/includes_renderer.py ${WS_ROOT} ${DYNAMIC_RENDER_DIR}
+
+.PHONY: template-index
+template-index:
+ @sed -ie "s/__VPP_VERSION__/${VPP_VERSION}/g" ${DOCS_DIR}/index.rst
+ @sed -ie "s/__BUILT_ON__/${BUILT_ON}/g" ${DOCS_DIR}/index.rst
+
+.NOTPARALLEL: $(SIPHON_FILES)
+$(SIPHON_FILES): $(SCRIPTS_DIR)/siphon-generate \
+ $(addprefix,$(WSROOT),$(SIPHON_INPUT)) \
+ $(wildcard $(SCRIPTS_DIR)/siphon/*.py)
+ @echo "Validating source tree..."
+ @set -e; for input in $(SIPHON_INPUT); do \
+ if [ ! -e "$(WS_ROOT)/$$input" ]; then \
+ echo "ERROR: Input path '$$input' does not exist." >&2; \
+ exit 1; \
+ fi; \
+ done
+ @rm -rf "$(SIPHON_INPUT_DIR)" "$(DOCS_GENERATED_DIR)"
+ @mkdir -p "$(SIPHON_INPUT_DIR)" "$(DOCS_GENERATED_DIR)"
+ @touch $(SIPHON_INPUT_DIR)/files
+ @echo "Collating source file list for siphoning..."
+ @for input in $(SIPHON_INPUT); do \
+ cd "$(WS_ROOT)"; \
+ find "$$input" -type f \
+ \( -name '*.[ch]' -or -name '*.dox' \) -print \
+ | grep -v -E '^src/examples/' \
+ | grep -v -E '^$(SIPHON_EXCLUDE_REGEXP)' \
+ >> $(SIPHON_INPUT_DIR)/files; \
+ done
+ @echo "Generating siphons..."
+ @set -e; \
+ cd "$(WS_ROOT)"; \
+ $(SCRIPTS_DIR)/siphon-generate \
+ --output="$(SIPHON_INPUT_DIR)" \
+ "@$(SIPHON_INPUT_DIR)/files"
+
+# Evaluate this to build a siphon doc output target for each desired
+# output type:
+# $(eval $(call siphon-process,file_extension,output_type_name))
+define siphon-process
+$(DOCS_GENERATED_DIR)/%.$(1): $(SIPHON_INPUT_DIR)/%.siphon \
+ $(SCRIPTS_DIR)/siphon-process \
+ $(wildcard $(SCRIPTS_DIR)/siphon/*.py) \
+ $(wildcard $(SCRIPTS_DIR)/siphon_templates/$(2)/*/*.$(1))
+ @echo "Processing siphon for $(2) from $$(notdir $$<)..."
+ @set -e; \
+ cd "$(WS_ROOT)"; \
+ mkdir -p $(DOCS_GENERATED_DIR)/$$(basename $$(notdir $$<)).$(1).dir; \
+ $(SCRIPTS_DIR)/siphon-process \
+ --type=$$(basename $$(notdir $$<)) \
+ --format=$(2) \
+ --repolink=$(REPOSITORY_URL)/ \
+ --outdir=$(DOCS_GENERATED_DIR)/$$(basename $$(notdir $$<)).$(1).dir \
+ --output="$$@" \
+ "$$<"
+endef
+
+# Process the .siphon source fragments and render them into siphon flavored
+# markdown documentation
+.DELETE_ON_ERROR: $(SIPHON_DOCS)
+$(eval $(call siphon-process,rst,markdown))
+
+# This target can be used just to generate the siphoned things
+.PHONY: siphon
+siphon: $(SIPHON_DOCS)
+ @cp $(DOCS_GENERATED_DIR)/clicmd.rst $(DOCS_DIR)/cli-reference/index.rst
+ @cp -r $(DOCS_GENERATED_DIR)/clicmd.rst.dir $(DOCS_DIR)/cli-reference/clis
+
+.PHONY: generate
+generate: siphon includes-render template-index featurelist
+
+.PHONY: clean
+clean:
+ @rm -rf $(BR)/.siphon.dep
+ @rm -rf $(SCRIPTS_DIR)/siphon/__pycache__
+
diff --git a/docs/_scripts/filter_api.py b/docs/_scripts/filter_api.py
new file mode 100755
index 0000000..4848814
--- /dev/null
+++ b/docs/_scripts/filter_api.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Filter for vpe.api to make it Doxygenish.
+
+import re
+import sys
+
+if len(sys.argv) < 2:
+ sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0]))
+ sys.exit(1)
+
+patterns = [
+ # Search for "define" blocks and treat them as structs
+ (re.compile(r"^.*(manual_.[^\s]+\s+)?define\s+(?P<name>[^\s]+)"),
+ r"typedef struct vl_api_\g<name>_t"),
+
+ # For every "brief" statement at the start of a comment block, add an
+ # xref with whatever is on the same line. This gives us an index page
+ # with all the API methods in one place.
+ # XXX Commented out for now; works but duplicates the brief text in the
+ # struct documentation
+ # (re.compile(r"/\*\*\s*(?P<b>[\\@]brief)\s+(?P<c>.+)(\*/)$"),
+ # r'/** @xrefitem api "" "VPP API" \g<c> \g<b> \g<c>'), # capture inline comment close
+ # (re.compile(r"/\*\*\s*(?P<b>[\\@]brief)\s+(?P<c>.+)$"),
+ # r'/** @xrefitem api "" "VPP API" \g<c> \g<b> \g<c>'),
+
+ # Since structs don't have params, replace @param with @tparam
+ ( re.compile("[\\@]param\\b"), "@tparam"),
+]
+
+with open(sys.argv[1]) as fd:
+ for line in fd:
+ str = line[:-1] # strip \n
+ for p in patterns:
+ str = p[0].sub(p[1], str)
+ sys.stdout.write(str+"\n")
diff --git a/docs/_scripts/filter_c.py b/docs/_scripts/filter_c.py
new file mode 100755
index 0000000..897f9f6
--- /dev/null
+++ b/docs/_scripts/filter_c.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Filter for .c files to make various preprocessor tricks Doxygenish
+
+import os, sys, re
+
+if len(sys.argv) < 2:
+ sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0]))
+ sys.exit(1)
+
+replace_patterns = [
+ # Search for VLIB_CLI_COMMAND, extract its parameters and add a docblock for it
+ ( re.compile("(?P<m>VLIB_CLI_COMMAND)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>) */ vlib_cli_command_t \g<name>"),
+ ( re.compile("(?P<m>VLIB_CLI_COMMAND)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<qual>[^)]*)[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>) */ \g<qual> vlib_cli_command_t \g<name>"),
+
+ # Search for VLIB_REGISTER_NODE, extract its parameters and add a docblock for it
+ ( re.compile("(?P<m>VLIB_REGISTER_NODE)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>) */ vlib_node_registration_t \g<name>"),
+ ( re.compile("(?P<m>VLIB_REGISTER_NODE)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<qual>[^)]*)[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>) */ \g<qual> vlib_node_registration_t \g<name>"),
+
+ # Search for VLIB_INIT_FUNCTION, extract its parameter and add a docblock for it
+ ( re.compile("(?P<m>VLIB_INIT_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"),
+ r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ vlib_init_function_t * _vlib_init_function_\g<name>"),
+ ( re.compile("(?P<m>VLIB_DECLARE_INIT_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"),
+ r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ vlib_init_function_t * _vlib_init_function_\g<name>"),
+
+ # Search for VLIB_LOOP_ENTER_FUNCTION, extract the parameters and add a docblock for it
+ ( re.compile("(?P<m>VLIB_MAIN_LOOP_ENTER_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"),
+ r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ _vlib_main_loop_enter_\g<name>"),
+ ( re.compile("(?P<m>VLIB_MAIN_LOOP_EXIT_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"),
+ r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ _vlib_main_loop_exit_\g<name>"),
+
+ # Search for VLIB_CONFIG_FUNCTION, extract the parameters and add a docblock for it
+ ( re.compile("(?P<m>VLIB_CONFIG_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<n>\"[^\"]+\")(,[^)]*)?[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>, \g<n>) */ vlib_config_function_runtime_t _vlib_config_function_\g<name>"),
+ ( re.compile("(?P<m>VLIB_EARLY_CONFIG_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<n>\"[^\"]+\")(,[^)]*)?[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>, \g<n>) */ vlib_config_function_runtime_t _vlib_config_function_\g<name>"),
+
+ # Search for "format_thing" and "unformat_thing" when used as a function pointer and add parens
+ ( re.compile("(?P<pre>(^|,)\s*)(?P<name>(un)?format_[a-zA-Z0-9_]+)(?P<post>\s*(,|$))"),
+ r"\g<pre>\g<name>()\g<post>" ),
+
+ # Search for CLIB_PAD_FROM_TO(...); and replace with padding
+ # #define CLIB_PAD_FROM_TO(from,to) u8 pad_##from[(to) - (from)]
+ ( re.compile("(?P<m>CLIB_PAD_FROM_TO)\s*[(](?P<from>[^,]+),\s*(?P<to>[^)]+)[)]"),
+ r"/** Padding. */ u8 pad_\g<from>[(\g<to>) - (\g<from>)]" ),
+
+]
+
+
+filename = sys.argv[1]
+cwd = os.getcwd()
+if filename[0:len(cwd)] == cwd:
+ filename = filename[len(cwd):]
+ if filename[0] == "/":
+ filename = filename[1:]
+
+with open(filename) as fd:
+ line_num = 0
+
+ for line in fd:
+ line_num += 1
+ str = line[:-1] # filter \n
+
+ # Look for search/replace patterns
+ for p in replace_patterns:
+ str = p[0].sub(p[1], str)
+
+ sys.stdout.write(str+"\n")
+
+# All done
diff --git a/docs/_scripts/filter_h.py b/docs/_scripts/filter_h.py
new file mode 100755
index 0000000..0891fa7
--- /dev/null
+++ b/docs/_scripts/filter_h.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Filter for .c files to make various preprocessor tricks Doxygenish
+
+import os
+import re
+import sys
+
+if len(sys.argv) < 2:
+ sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0]))
+ sys.exit(1)
+
+replace_patterns = [
+ # Search for CLIB_PAD_FROM_TO(...); and replace with padding
+ # #define CLIB_PAD_FROM_TO(from,to) u8 pad_##from[(to) - (from)]
+ (re.compile(r"(?P<m>CLIB_PAD_FROM_TO)\s*[(](?P<from>[^,]+),"
+ r"\s*(?P<to>[^)]+)[)]"),
+ r"/** Padding. */ u8 pad_\g<from>[(\g<to>) - (\g<from>)]"),
+
+]
+
+
+filename = sys.argv[1]
+cwd = os.getcwd()
+if filename[0:len(cwd)] == cwd:
+ filename = filename[len(cwd):]
+ if filename[0] == "/":
+ filename = filename[1:]
+
+with open(filename) as fd:
+ line_num = 0
+
+ for line in fd:
+ line_num += 1
+ str = line[:-1] # filter \n
+
+ # Look for search/replace patterns
+ for p in replace_patterns:
+ str = p[0].sub(p[1], str)
+
+ sys.stdout.write(str+"\n")
+
+# All done
diff --git a/docs/_scripts/includes_renderer.py b/docs/_scripts/includes_renderer.py
new file mode 100644
index 0000000..6bd501d
--- /dev/null
+++ b/docs/_scripts/includes_renderer.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020. Vinci Consulting Corp. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import inspect
+import os
+import re
+import sys
+
+
+class ContentRenderer:
+ def __init__(self, ws_root, output_dir):
+ self.ws_root = ws_root
+ self.output_dir = output_dir
+
+ def plugin_dir(self):
+ return os.path.join(self.ws_root, "src/plugins")
+
+ def render(self):
+ raise NotImplementedError
+
+
+class PluginRenderer(ContentRenderer):
+
+ def _render_entry(self, output_file, entry):
+ description = "<no-description-found>"
+ # we use glob because a plugin can (ioam for now)
+ # define the plugin definition in
+ # a further subdirectory.
+ path = os.path.join(self.plugin_dir(), entry.name, '**')
+ for f in glob.iglob(path, recursive=True):
+ if not f.endswith('.c'):
+ continue
+ with open(f, "r", encoding="utf-8") as src:
+ for match in self.regex.finditer(src.read()):
+ description = "%s" % (match.group(1))
+
+ output_file.write(f"* {entry.name} - {description}\n")
+
+ def render(self):
+ pattern = r'VLIB_PLUGIN_REGISTER\s?\(\)\s*=\s*{.*\.description\s?=\s?"([^"]*)".*};' # noqa: 501
+ self.regex = re.compile(pattern, re.MULTILINE | re.DOTALL)
+ fname = os.path.join(self.output_dir, "plugin_list.inc")
+ with open(fname, "w") as output_file:
+ with os.scandir(self.plugin_dir()) as pdir:
+ for entry in sorted(pdir, key=lambda entry: entry.name):
+ if not entry.name.startswith('.') and entry.is_dir():
+ self._render_entry(output_file, entry)
+
+
+renderers = [PluginRenderer]
+
+
+def main():
+ if len(sys.argv) != 3:
+ print("You need to pass WS_ROOT and OUTPUT_DIR")
+ exit(1)
+
+ print("rendering dynamic includes...")
+ for renderer in renderers:
+ renderer(*sys.argv[1:]).render()
+ print("done.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docs/scripts/prepare-for-site.sh b/docs/_scripts/prepare-for-site.sh
similarity index 100%
rename from docs/scripts/prepare-for-site.sh
rename to docs/_scripts/prepare-for-site.sh
diff --git a/docs/scripts/publish-docs.sh b/docs/_scripts/publish-docs.sh
similarity index 97%
rename from docs/scripts/publish-docs.sh
rename to docs/_scripts/publish-docs.sh
index 8c3c810..026ab2f 100755
--- a/docs/scripts/publish-docs.sh
+++ b/docs/_scripts/publish-docs.sh
@@ -72,7 +72,7 @@
# Create the feature list
pushd ..
-source ./sphinx_venv/bin/activate
+source ./docs/venv/bin/activate
find . -name FEATURE.yaml | ./src/scripts/fts.py --markdown > site/content/vppProject/vppfeatures/features.md
deactivate
popd
diff --git a/docs/_scripts/siphon-generate b/docs/_scripts/siphon-generate
new file mode 100755
index 0000000..9b69c52
--- /dev/null
+++ b/docs/_scripts/siphon-generate
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Looks for preprocessor macros with struct initializers and siphons them
+# off into another file for later parsing; ostensibly to generate
+# documentation from struct initializer data.
+
+import argparse
+import logging
+import os
+
+import siphon
+
+DEFAULT_LOGFILE = None
+DEFAULT_LOGLEVEL = "info"
+DEFAULT_OUTPUT = "build-root/docs/siphons"
+DEFAULT_PREFIX = os.getcwd()
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--log-file", default=DEFAULT_LOGFILE,
+ help="Log file [%s]" % DEFAULT_LOGFILE)
+ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
+ choices=["debug", "info", "warning", "error", "critical"],
+ help="Logging level [%s]" % DEFAULT_LOGLEVEL)
+
+ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT,
+ help="Output directory for .siphon files [%s]" %
+ DEFAULT_OUTPUT)
+ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX,
+ help="Prefix to strip from input pathnames [%s]" %
+ DEFAULT_PREFIX)
+ap.add_argument("input", nargs='+', metavar="input_file",
+ help="Input C source files")
+args = ap.parse_args()
+
+logging.basicConfig(filename=args.log_file,
+ level=getattr(logging, args.log_level.upper(), None))
+log = logging.getLogger("siphon_generate")
+
+
+generate = siphon.generate.Generate(output_directory=args.output,
+ input_prefix=args.input_prefix)
+
+# Pre-process file names in case they indicate a file with
+# a list of files
+files = []
+for filename in args.input:
+ if filename.startswith('@'):
+ with open(filename[1:], 'r') as fp:
+ lines = fp.readlines()
+ for line in lines:
+ file = line.strip()
+ if file not in files:
+ files.append(file)
+ lines = None
+ else:
+ if filename not in files:
+ files.append(filename)
+
+# Iterate all the input files we've been given
+for filename in files:
+ generate.parse(filename)
+
+# Write the extracted data
+generate.deliver()
+
+# All done
diff --git a/docs/_scripts/siphon-process b/docs/_scripts/siphon-process
new file mode 100755
index 0000000..cbee1e9
--- /dev/null
+++ b/docs/_scripts/siphon-process
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Filter for .siphon files that are generated by other filters.
+# The idea is to siphon off certain initializers so that we can better
+# auto-document the contents of that initializer.
+
+import argparse
+import logging
+import os
+import sys
+
+import siphon
+
+DEFAULT_LOGFILE = None
+DEFAULT_LOGLEVEL = "info"
+DEFAULT_SIPHON = "clicmd"
+DEFAULT_FORMAT = "markdown"
+DEFAULT_OUTPUT = None
+DEFAULT_TEMPLATES = os.path.dirname(__file__) + "/siphon_templates"
+DEFAULT_OUTPUT_DIR = os.path.dirname(__file__) + "/siphon_docs"
+DEFAULT_REPO_LINK = "https://github.com/FDio/vpp/blob/master/"
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--log-file", default=DEFAULT_LOGFILE,
+ help="Log file [%s]" % DEFAULT_LOGFILE)
+ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
+ choices=["debug", "info", "warning", "error", "critical"],
+ help="Logging level [%s]" % DEFAULT_LOGLEVEL)
+
+ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON,
+ choices=siphon.process.siphons.keys(),
+ help="Siphon type to process [%s]" % DEFAULT_SIPHON)
+ap.add_argument("--format", '-f', default=DEFAULT_FORMAT,
+ choices=siphon.process.formats.keys(),
+ help="Output format to generate [%s]" % DEFAULT_FORMAT)
+ap.add_argument("--output", '-o', metavar="file", default=DEFAULT_OUTPUT,
+ help="Output file (uses stdout if not defined) [%s]" %
+ DEFAULT_OUTPUT)
+ap.add_argument("--templates", metavar="directory", default=DEFAULT_TEMPLATES,
+ help="Path to render templates directory [%s]" %
+ DEFAULT_TEMPLATES)
+ap.add_argument("--outdir", metavar="directory", default=DEFAULT_OUTPUT_DIR,
+ help="Path to output rendered parts [%s]" %
+ DEFAULT_OUTPUT_DIR)
+ap.add_argument("--repolink", metavar="repolink", default=DEFAULT_REPO_LINK,
+ help="Link to public repository [%s]" %
+ DEFAULT_REPO_LINK)
+ap.add_argument("input", nargs='+', metavar="input_file",
+ help="Input .siphon files")
+args = ap.parse_args()
+
+logging.basicConfig(filename=args.log_file,
+ level=getattr(logging, args.log_level.upper(), None))
+log = logging.getLogger("siphon_process")
+
+# Determine where to send the generated output
+if args.output is None:
+ out = sys.stdout
+else:
+ out = open(args.output, "w+")
+
+# Get our processor
+klass = siphon.process.siphons[args.type]
+processor = klass(
+ template_directory=args.templates,
+ format=args.format,
+ outdir=args.outdir,
+ repository_link=args.repolink
+)
+
+# Load the input files
+processor.load_json(args.input)
+
+# Process the data
+processor.process(out=out)
+
+# All done
diff --git a/docs/_scripts/siphon/__init__.py b/docs/_scripts/siphon/__init__.py
new file mode 100644
index 0000000..f641731
--- /dev/null
+++ b/docs/_scripts/siphon/__init__.py
@@ -0,0 +1,24 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Siphon classes
+
+from . import generate
+from . import generate_clicmd
+from . import generate_syscfg
+
+from . import parsers
+from . import process
+from . import process_clicmd
+from . import process_syscfg
diff --git a/docs/_scripts/siphon/generate.py b/docs/_scripts/siphon/generate.py
new file mode 100644
index 0000000..2ae5a1b
--- /dev/null
+++ b/docs/_scripts/siphon/generate.py
@@ -0,0 +1,306 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generate .siphon source fragments for later processing
+
+import json
+import logging
+import os
+import re
+
+"""List of (regexp, siphon_name) tuples for matching the start of C
+ initializer blocks in source files. Each siphon class registers
+ themselves on this list."""
+siphon_patterns = []
+
+class Generate(object):
+ """Matches a siphon comment block start"""
+ siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$")
+
+ """Matches a siphon comment block stop"""
+ siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$")
+
+ """Siphon block directive delimiter"""
+ siphon_block_delimiter = "%%"
+
+ """Matches a siphon block directive such as
+ '%clicmd:group_label Debug CLI%'"""
+ siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \
+ (siphon_block_delimiter, siphon_block_delimiter))
+
+ """Matches the start of an initializer block"""
+ siphon_initializer = re.compile("\s*=")
+
+ """Collated output for each siphon"""
+ output = None
+
+ """Directory prefix to strip from input filenames to keep things tidy."""
+ input_prefix = None
+
+ """List of known siphons"""
+ known_siphons = None
+
+ """Logging handler"""
+ log = None
+
+
+ def __init__(self, output_directory, input_prefix):
+ super(Generate, self).__init__()
+ self.log = logging.getLogger("siphon.generate")
+
+ # Build a list of known siphons
+ self.known_siphons = []
+ for item in siphon_patterns:
+ siphon = item[1]
+ if siphon not in self.known_siphons:
+ self.known_siphons.append(siphon)
+
+ # Setup information for siphons we know about
+ self.output = {}
+ for siphon in self.known_siphons:
+ self.output[siphon] = {
+ "file": "%s/%s.siphon" % (output_directory, siphon),
+ "global": {},
+ "items": [],
+ }
+
+ self.input_prefix = input_prefix
+
+
+ """
+ count open and close braces in str
+ return (0, index) when braces were found and count becomes 0.
+ index indicates the position at which the last closing brace was
+ found.
+ return (-1, -1) if a closing brace is found before any opening one.
+ return (count, -1) if not all opening braces are closed, count is the
+ current depth
+ """
+ def count_braces(self, str, count=0, found=False):
+ for index in range(0, len(str)):
+ if str[index] == '{':
+ count += 1;
+ found = True
+ elif str[index] == '}':
+ if count == 0:
+ # means we never found an open brace
+ return (-1, -1)
+ count -= 1;
+
+ if count == 0 and found:
+ return (count, index)
+
+ return (count, -1)
+
+ def parse(self, filename):
+ # Strip the current directory off the start of the
+ # filename for brevity
+ if filename[0:len(self.input_prefix)] == self.input_prefix:
+ filename = filename[len(self.input_prefix):]
+ if filename[0] == "/":
+ filename = filename[1:]
+
+ # Work out the abbreviated directory name
+ directory = os.path.dirname(filename)
+ if directory[0:2] == "./":
+ directory = directory[2:]
+ elif directory[0:len(self.input_prefix)] == self.input_prefix:
+ directory = directory[len(self.input_prefix):]
+ if directory[0] == "/":
+ directory = directory[1:]
+
+ # Open the file and explore its contents...
+ self.log.info("Siphoning from %s." % filename)
+ directives = {}
+ with open(filename) as fd:
+ siphon = None
+ close_siphon = None
+ siphon_block = ""
+ in_block = False
+ line_num = 0
+ siphon_line = 0
+
+ for line in fd:
+ line_num += 1
+ str = line[:-1] # filter \n
+
+ """See if there is a block directive and if so extract it"""
+ def process_block_directive(str, directives):
+ m = self.siphon_block_directive.search(str)
+ if m is not None:
+ k = m.group(2)
+ v = m.group(3).strip()
+ directives[k] = v
+ # Return only the parts we did not match
+ return str[0:m.start(1)] + str[m.end(4):]
+
+ return str
+
+ def process_block_prefix(str):
+ if str.startswith(" * "):
+ str = str[3:]
+ elif str == " *":
+ str = ""
+ return str
+
+ if not in_block:
+ # See if the line contains the start of a siphon doc block
+ m = self.siphon_block_start.search(str)
+ if m is not None:
+ in_block = True
+ t = m.group(1)
+
+ # Now check if the block closes on the same line
+ m = self.siphon_block_stop.search(t)
+ if m is not None:
+ t = m.group(1)
+ in_block = False
+
+ # Check for directives
+ t = process_block_directive(t, directives)
+
+ # Filter for normal comment prefixes
+ t = process_block_prefix(t)
+
+ # Add what is left
+ siphon_block += t
+
+ # Skip to next line
+ continue
+
+ else:
+ # Check to see if we have an end block marker
+ m = self.siphon_block_stop.search(str)
+ if m is not None:
+ in_block = False
+ t = m.group(1)
+ else:
+ t = str
+
+ # Check for directives
+ t = process_block_directive(t, directives)
+
+ # Filter for normal comment prefixes
+ t = process_block_prefix(t)
+
+ # Add what is left
+ siphon_block += t + "\n"
+
+ # Skip to next line
+ continue
+
+
+ if siphon is None:
+ # Look for blocks we need to siphon
+ for p in siphon_patterns:
+ if p[0].match(str):
+ siphon = [ p[1], str + "\n", 0 ]
+ siphon_line = line_num
+
+ # see if we have an initializer
+ m = self.siphon_initializer.search(str)
+ if m is not None:
+ # count the braces on this line
+ (count, index) = \
+ self.count_braces(str[m.start():])
+ siphon[2] = count
+ # TODO - it's possible we have the
+ # initializer all on the first line
+ # we should check for it, but also
+ # account for the possibility that
+ # the open brace is on the next line
+ #if count == 0:
+ # # braces balanced
+ # close_siphon = siphon
+ # siphon = None
+ else:
+ # no initializer: close the siphon right now
+ close_siphon = siphon
+ siphon = None
+ else:
+ # See if we should end the siphon here - do we have
+ # balanced braces?
+ (count, index) = self.count_braces(str,
+ count=siphon[2], found=True)
+ if count == 0:
+ # braces balanced - add the substring and
+ # close the siphon
+ siphon[1] += str[:index+1] + ";\n"
+ close_siphon = siphon
+ siphon = None
+ else:
+ # add the whole string, move on
+ siphon[2] = count
+ siphon[1] += str + "\n"
+
+ if close_siphon is not None:
+ # Write the siphoned contents to the right place
+ siphon_name = close_siphon[0]
+
+ # Copy directives for the file
+ details = {}
+ for key in directives:
+ if ":" in key:
+ (sn, label) = key.split(":")
+ if sn == siphon_name:
+ details[label] = directives[key]
+ else:
+ details[key] = directives[key]
+
+ # Copy details for this block
+ details['file'] = filename
+ details['directory'] = directory
+ details['line_start'] = siphon_line
+ details['line_end'] = line_num
+ details['siphon_block'] = siphon_block.strip()
+ details["block"] = close_siphon[1]
+
+ # Store the item
+ self.output[siphon_name]['items'].append(details)
+
+ # All done
+ close_siphon = None
+ siphon_block = ""
+
+ # Update globals
+ for key in directives.keys():
+ if ':' not in key:
+ continue
+
+ if filename.endswith("/dir.dox"):
+ # very special! use the parent directory name
+ l = directory
+ else:
+ l = filename
+
+ (sn, label) = key.split(":")
+
+ if sn not in self.output:
+ self.output[sn] = {}
+ if 'global' not in self.output[sn]:
+ self.output[sn]['global'] = {}
+ if l not in self.output[sn]['global']:
+ self.output[sn]['global'][l] = {}
+
+ self.output[sn]['global'][l][label] = directives[key]
+
+ def deliver(self):
+ # Write out the data
+ for siphon in self.output.keys():
+ self.log.info("Saving siphon data %s." % siphon)
+ s = self.output[siphon]
+ with open(s['file'], "a") as fp:
+ json.dump(s, fp,
+ separators=(',', ': '), indent=4, sort_keys=True)
+
diff --git a/docs/_scripts/siphon/generate_clicmd.py b/docs/_scripts/siphon/generate_clicmd.py
new file mode 100644
index 0000000..6d24aaf
--- /dev/null
+++ b/docs/_scripts/siphon/generate_clicmd.py
@@ -0,0 +1,24 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from . import generate
+
+# Register our regexp
+generate.siphon_patterns.append((
+ re.compile("(?P<m>VLIB_CLI_COMMAND)\s*"
+ "[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"),
+ "clicmd"
+))
diff --git a/docs/_scripts/siphon/generate_syscfg.py b/docs/_scripts/siphon/generate_syscfg.py
new file mode 100644
index 0000000..52c802e
--- /dev/null
+++ b/docs/_scripts/siphon/generate_syscfg.py
@@ -0,0 +1,24 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from . import generate
+
+# Register our regexp
+generate.siphon_patterns.append((
+ re.compile("(?P<m>VLIB_CONFIG_FUNCTION)\s*"
+ '[(](?P<fn>[a-zA-Z0-9_]+)\s*,\s*"(?P<name>[^"]*)"[)]'),
+ "syscfg"
+))
diff --git a/docs/_scripts/siphon/parsers.py b/docs/_scripts/siphon/parsers.py
new file mode 100644
index 0000000..162205d
--- /dev/null
+++ b/docs/_scripts/siphon/parsers.py
@@ -0,0 +1,150 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import html
+import pyparsing as pp
+
+# Some useful primitives
+ident = pp.Word(pp.alphas + "_", pp.alphas + pp.nums + "_")
+intNum = pp.Word(pp.nums)
+hexNum = pp.Literal("0x") + pp.Word(pp.hexnums)
+octalNum = pp.Literal("0") + pp.Word("01234567")
+integer = (hexNum | octalNum | intNum) + \
+ pp.Optional(pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L"))
+floatNum = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + pp.Optional(pp.Literal("f"))
+char = pp.Literal("'") + pp.Word(pp.printables, exact=1) + pp.Literal("'")
+arrayIndex = integer | ident
+
+lbracket = pp.Literal("(").suppress()
+rbracket = pp.Literal(")").suppress()
+lbrace = pp.Literal("{").suppress()
+rbrace = pp.Literal("}").suppress()
+comma = pp.Literal(",").suppress()
+equals = pp.Literal("=").suppress()
+dot = pp.Literal(".").suppress()
+semicolon = pp.Literal(";").suppress()
+
+# initializer := { [member = ] (variable | expression | { initializer } ) }
+typeName = ident
+varName = ident
+typeSpec = pp.Optional("unsigned") + \
+ pp.oneOf("int long short float double char u8 i8 void") + \
+ pp.Optional(pp.Word("*"), default="")
+typeCast = pp.Combine( "(" + ( typeSpec | typeName ) + ")" ).suppress()
+
+string = pp.Combine(pp.OneOrMore(pp.QuotedString(quoteChar='"',
+ escChar='\\', multiline=True)), adjacent=False)
+literal = pp.Optional(typeCast) + (integer | floatNum | char | string)
+var = pp.Combine(pp.Optional(typeCast) + varName +
+ pp.Optional("[" + arrayIndex + "]"))
+
+# This could be more complete, but suffices for our uses
+expr = (literal | var)
+
+"""Parse and render a block of text into a Python dictionary."""
+class Parser(object):
+ """Compiled PyParsing BNF"""
+ _parser = None
+
+ def __init__(self):
+ super(Parser, self).__init__()
+ self._parser = self.BNF()
+
+ def BNF(self):
+ raise NotImplementedError
+
+ def item(self, item):
+ raise NotImplementedError
+
+ def parse(self, input):
+ item = self._parser.parseString(input).asList()
+ return self.item(item)
+
+
+"""Parser for function-like macros - without the closing semi-colon."""
+class ParserFunctionMacro(Parser):
+ def BNF(self):
+ # VLIB_CONFIG_FUNCTION (unix_config, "unix")
+ macroName = ident
+ params = pp.Group(pp.ZeroOrMore(expr + comma) + expr)
+ macroParams = lbracket + params + rbracket
+
+ return macroName + macroParams
+
+ def item(self, item):
+ r = {
+ "macro": item[0],
+ "name": item[1][1],
+ "function": item[1][0],
+ }
+
+ return r
+
+
+"""Parser for function-like macros with a closing semi-colon."""
+class ParseFunctionMacroStmt(ParserFunctionMacro):
+ def BNF(self):
+ # VLIB_CONFIG_FUNCTION (unix_config, "unix");
+ function_macro = super(ParseFunctionMacroStmt, self).BNF()
+ mi = function_macro + semicolon
+ mi.ignore(pp.cppStyleComment)
+
+ return mi
+
+
+"""
+Parser for our struct initializers which are composed from a
+function-like macro, equals sign, and then a normal C struct initializer
+block.
+"""
+class MacroInitializer(ParserFunctionMacro):
+ def BNF(self):
+ # VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = {
+ # .path = "show sr tunnel",
+ # .short_help = "show sr tunnel [name <sr-tunnel-name>]",
+ # .function = show_sr_tunnel_fn,
+ # };
+ cs = pp.Forward()
+
+
+ member = pp.Combine(dot + varName + pp.Optional("[" + arrayIndex + "]"),
+ adjacent=False)
+ value = (expr | cs)
+
+ entry = pp.Group(pp.Optional(member + equals, default="") + value)
+ entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | \
+ (pp.ZeroOrMore(entry + comma))
+
+ cs << (lbrace + entries + rbrace)
+
+ macroName = ident
+ params = pp.Group(pp.ZeroOrMore(expr + comma) + expr)
+ macroParams = lbracket + params + rbracket
+
+ function_macro = super(MacroInitializer, self).BNF()
+ mi = function_macro + equals + pp.Group(cs) + semicolon
+ mi.ignore(pp.cppStyleComment)
+
+ return mi
+
+ def item(self, item):
+ r = {
+ "macro": item[0],
+ "name": item[1][0],
+ "params": item[2],
+ "value": {},
+ }
+
+ for param in item[2]:
+ r["value"][param[0]] = html.escape(param[1])
+
+ return r
diff --git a/docs/_scripts/siphon/process.py b/docs/_scripts/siphon/process.py
new file mode 100644
index 0000000..e3a7015
--- /dev/null
+++ b/docs/_scripts/siphon/process.py
@@ -0,0 +1,407 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generation template class
+
+import html.parser
+import json
+import logging
+import os
+import sys
+import re
+
+import jinja2
+
+# Classes register themselves in this dictionary
+"""Mapping of known processors to their classes"""
+siphons = {}
+
+"""Mapping of known output formats to their classes"""
+formats = {}
+
+
+class Siphon(object):
+ """Generate rendered output for siphoned data."""
+
+ # Set by subclasses
+ """Our siphon name"""
+ name = None
+
+ # Set by subclasses
+ """Name of an identifier used by this siphon"""
+ identifier = None
+
+ # Set by subclasses
+ """The pyparsing object to use to parse with"""
+ _parser = None
+
+ """The input data"""
+ _cmds = None
+
+ """Group key to (directory,file) mapping"""
+ _group = None
+
+ """Logging handler"""
+ log = None
+
+ """Directory to look for siphon rendering templates"""
+ template_directory = None
+
+ """Directory to output parts in"""
+ outdir = None
+
+ """Template environment, if we're using templates"""
+ _tplenv = None
+
+ def __init__(self, template_directory, format, outdir, repository_link):
+ super(Siphon, self).__init__()
+ self.log = logging.getLogger("siphon.process.%s" % self.name)
+
+ # Get our output format details
+ fmt_klass = formats[format]
+ fmt = fmt_klass()
+ self._format = fmt
+
+ # Sort out the template search path
+ def _tpldir(name):
+ return os.sep.join((template_directory, fmt.name, name))
+
+ self.template_directory = template_directory
+ searchpath = [
+ _tpldir(self.name),
+ _tpldir("default"),
+ ]
+ self.outdir = outdir
+ loader = jinja2.FileSystemLoader(searchpath=searchpath)
+ self._tplenv = jinja2.Environment(
+ loader=loader,
+ trim_blocks=True,
+ autoescape=False,
+ keep_trailing_newline=True)
+
+ # Convenience, get a reference to the internal escape and
+ # unescape methods in html.parser. These then become
+ # available to templates to use, if needed.
+ self._h = html.parser.HTMLParser()
+ self.escape = html.escape
+ self.unescape = html.unescape
+
+ # TODO: customize release
+ self.repository_link = repository_link
+
+ # Output renderers
+
+ """Returns an object to be used as the sorting key in the item index."""
+ def index_sort_key(self, group):
+ return group
+
+ """Returns a string to use as the header at the top of the item index."""
+ def index_header(self):
+ return self.template("index_header")
+
+ """Returns the string fragment to use for each section in the item
+ index."""
+ def index_section(self, group):
+ return self.template("index_section", group=group)
+
+ """Returns the string fragment to use for each entry in the item index."""
+ def index_entry(self, meta, item):
+ return self.template("index_entry", meta=meta, item=item)
+
+ """Returns an object, typically a string, to be used as the sorting key
+ for items within a section."""
+ def item_sort_key(self, item):
+ return item['name']
+
+ """Returns a key for grouping items together."""
+ def group_key(self, directory, file, macro, name):
+ _global = self._cmds['_global']
+
+ if file in _global and 'group_label' in _global[file]:
+ self._group[file] = (directory, file)
+ return file
+
+ self._group[directory] = (directory, None)
+ return directory
+
+ """Returns a key for identifying items within a grouping."""
+ def item_key(self, directory, file, macro, name):
+ return name
+
+ """Returns a string to use as the header when rendering the item."""
+ def item_header(self, group):
+ return self.template("item_header", group=group)
+
+ """Returns a string to use as the body when rendering the item."""
+ def item_format(self, meta, item):
+ return self.template("item_format", meta=meta, item=item)
+
+ """Returns a string to use as the label for the page reference."""
+ def page_label(self, group):
+ return "_".join((
+ self.name,
+ self.sanitize_label(group)
+ ))
+
+ """Returns a title to use for a page."""
+ def page_title(self, group):
+ _global = self._cmds['_global']
+ (directory, file) = self._group[group]
+
+ if file and file in _global and 'group_label' in _global[file]:
+ return _global[file]['group_label']
+
+ if directory in _global and 'group_label' in _global[directory]:
+ return _global[directory]['group_label']
+
+ return directory
+
+ """Returns a string to use as the label for the section reference."""
+ def item_label(self, group, item):
+ return "__".join((
+ self.name,
+ item
+ ))
+
+ """Label sanitizer; for creating Doxygen references"""
+ def sanitize_label(self, value):
+ return value.replace(" ", "_") \
+ .replace("/", "_") \
+ .replace(".", "_")
+
+ """Template processor"""
+ def template(self, name, **kwargs):
+ tpl = self._tplenv.get_template(name + self._format.extension)
+ return tpl.render(
+ this=self,
+ **kwargs)
+
+ # Processing methods
+
+ """Parse the input file into a more usable dictionary structure."""
+ def load_json(self, files):
+ self._cmds = {}
+ self._group = {}
+
+ line_num = 0
+ line_start = 0
+ for filename in files:
+ filename = os.path.relpath(filename)
+ self.log.info("Parsing items in file \"%s\"." % filename)
+ data = None
+ with open(filename, "r") as fd:
+ data = json.load(fd)
+
+ self._cmds['_global'] = data['global']
+
+ # iterate the items loaded and regroup it
+ for item in data["items"]:
+ try:
+ o = self._parser.parse(item['block'])
+ except Exception:
+ self.log.error("Exception parsing item: %s\n%s"
+ % (json.dumps(item, separators=(',', ': '),
+ indent=4),
+ item['block']))
+ raise
+
+ # Augment the item with metadata
+ o["meta"] = {}
+ for key in item:
+ if key == 'block':
+ continue
+ o['meta'][key] = item[key]
+
+ # Load some interesting fields
+ directory = item['directory']
+ file = item['file']
+ macro = o["macro"]
+ name = o["name"]
+
+ # Generate keys to group items by
+ group_key = self.group_key(directory, file, macro, name)
+ item_key = self.item_key(directory, file, macro, name)
+
+ if group_key not in self._cmds:
+ self._cmds[group_key] = {}
+
+ self._cmds[group_key][item_key] = o
+
+ """Iterate over the input data, calling render methods to generate the
+ output."""
+ def process(self, out=None):
+
+ if out is None:
+ out = sys.stdout
+
+ # Accumulated body contents
+ contents = ""
+
+ # Write the header for this siphon type
+ out.write(self.index_header())
+
+ # Sort key helper for the index
+ def group_sort_key(group):
+ return self.index_sort_key(group)
+
+ # Iterate the dictionary and process it
+ for group in sorted(self._cmds.keys(), key=group_sort_key):
+ if group.startswith('_'):
+ continue
+
+ self.log.info("Processing items in group \"%s\" (%s)." %
+ (group, group_sort_key(group)))
+
+ # Generate the section index entry (write it now)
+ out.write(self.index_section(group))
+
+ # Generate the item header (save for later)
+ contents += self.item_header(group)
+
+ def item_sort_key(key):
+ return self.item_sort_key(self._cmds[group][key])
+
+ for key in sorted(self._cmds[group].keys(), key=item_sort_key):
+ self.log.debug("--- Processing key \"%s\" (%s)." %
+ (key, item_sort_key(key)))
+
+ o = self._cmds[group][key]
+ meta = {
+ "directory": o['meta']['directory'],
+ "file": o['meta']['file'],
+ "macro": o['macro'],
+ "name": o['name'],
+ "key": key,
+ "label": self.item_label(group, key),
+ }
+
+ # Generate the index entry for the item (write it now)
+ out.write(self.index_entry(meta, o))
+
+ # Generate the item itself (save for later)
+ contents += self.item_format(meta, o)
+
+ page_name = self.separate_page_names(group)
+ if page_name != "":
+ path = os.path.join(self.outdir, page_name)
+ with open(path, "w+") as page:
+ page.write(contents)
+ contents = ""
+
+ # Deliver the accumulated body output
+ out.write(contents)
+
+ def do_cliexstart(self, matchobj):
+ title = matchobj.group(1)
+ title = ' '.join(title.splitlines())
+ content = matchobj.group(2)
+ content = re.sub(r"\n", r"\n ", content)
+ return "\n\n.. code-block:: console\n\n %s\n %s\n\n" % (title, content)
+
+ def do_clistart(self, matchobj):
+ content = matchobj.group(1)
+ content = re.sub(r"\n", r"\n ", content)
+ return "\n\n.. code-block:: console\n\n %s\n\n" % content
+
+ def do_cliexcmd(self, matchobj):
+ content = matchobj.group(1)
+ content = ' '.join(content.splitlines())
+ return "\n\n.. code-block:: console\n\n %s\n\n" % content
+
+ def process_list(self, matchobj):
+ content = matchobj.group(1)
+ content = self.reindent(content, 2)
+ return "@@@@%s\nBBBB" % content
+
+ def process_special(self, s):
+ # ----------- markers to remove
+ s = re.sub(r"@cliexpar\s*", r"", s)
+ s = re.sub(r"@parblock\s*", r"", s)
+ s = re.sub(r"@endparblock\s*", r"", s)
+ s = re.sub(r"<br>", "", s)
+ # ----------- emphasis
+ # <b><em>
+ s = re.sub(r"<b><em>\s*", "``", s)
+ s = re.sub(r"\s*</b></em>", "``", s)
+ s = re.sub(r"\s*</em></b>", "``", s)
+ # <b>
+ s = re.sub(r"<b>\s*", "**", s)
+ s = re.sub(r"\s*</b>", "**", s)
+ # <code>
+ s = re.sub(r"<code>\s*", "``", s)
+ s = re.sub(r"\s*</code>", "``", s)
+ # <em>
+ s = re.sub(r"'?<em>\s*", r"``", s)
+ s = re.sub(r"\s*</em>'?", r"``", s)
+ # @c <something>
+ s = re.sub(r"@c\s(\S+)", r"``\1``", s)
+ # ----------- todos
+ s = re.sub(r"@todo[^\n]*", "", s)
+ s = re.sub(r"@TODO[^\n]*", "", s)
+ # ----------- code blocks
+ s = re.sub(r"@cliexcmd{(.+?)}", self.do_cliexcmd, s, flags=re.DOTALL)
+ s = re.sub(r"@cliexstart{(.+?)}(.+?)@cliexend", self.do_cliexstart, s, flags=re.DOTALL)
+ s = re.sub(r"@clistart(.+?)@cliend", self.do_clistart, s, flags=re.DOTALL)
+ # ----------- lists
+ s = re.sub(r"^\s*-", r"\n@@@@", s, flags=re.MULTILINE)
+ s = re.sub(r"@@@@(.*?)\n\n+", self.process_list, s, flags=re.DOTALL)
+ s = re.sub(r"BBBB@@@@", r"-", s)
+ s = re.sub(r"@@@@", r"-", s)
+ s = re.sub(r"BBBB", r"\n\n", s)
+ # ----------- Cleanup remains
+ s = re.sub(r"@cliexend\s*", r"", s)
+ return s
+
+ def separate_page_names(self, group):
+ return ""
+
+ # This push the given textblock <indent> spaces right
+ def reindent(self, s, indent):
+ ind = " " * indent
+ s = re.sub(r"\n", "\n" + ind, s)
+ return s
+
+ # This aligns the given textblock left (no indent)
+ def noindent(self, s):
+ s = re.sub(r"\n[ \f\v\t]*", "\n", s)
+ return s
+
+class Format(object):
+ """Output format class"""
+
+ """Name of this output format"""
+ name = None
+
+ """Expected file extension of templates that build this format"""
+ extension = None
+
+
+class FormatMarkdown(Format):
+ """Markdown output format"""
+ name = "markdown"
+ extension = ".md"
+
+
+# Register 'markdown'
+formats["markdown"] = FormatMarkdown
+
+
+class FormatItemlist(Format):
+ """Itemlist output format"""
+ name = "itemlist"
+ extension = ".itemlist"
+
+
+# Register 'itemlist'
+formats["itemlist"] = FormatItemlist
diff --git a/docs/_scripts/siphon/process_clicmd.py b/docs/_scripts/siphon/process_clicmd.py
new file mode 100644
index 0000000..bf27051
--- /dev/null
+++ b/docs/_scripts/siphon/process_clicmd.py
@@ -0,0 +1,73 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generate clicmd formatted output
+
+from . import process, parsers
+import os
+
+class SiphonCLICMD(process.Siphon):
+
+ name = "clicmd"
+ identifier = "VLIB_CLI_COMMAND"
+
+ def __init__(self, *args, **kwargs):
+ super(SiphonCLICMD, self).__init__(*args, **kwargs)
+ self._parser = parsers.MacroInitializer()
+
+ # Output renderers
+
+ def separate_page_names(self, group):
+ return self.page_label(group) + ".rst"
+
+ def index_sort_key(self, group):
+ _global = self._cmds['_global']
+ if group not in self._group:
+ return group
+ (directory, file) = self._group[group]
+
+ if file in _global and 'group_label' in _global[file]:
+ return _global[file]['group_label']
+
+ if directory in _global and 'group_label' in _global[directory]:
+ return _global[directory]['group_label']
+
+ return group
+
+ def item_sort_key(self, item):
+ return item['value']['path']
+
+ def item_label(self, group, item):
+ return "_".join((
+ self.name,
+ self.sanitize_label(self._cmds[group][item]['value']['path'])
+ ))
+
+ def page_title(self, group):
+ _global = self._cmds['_global']
+ (directory, file) = self._group[group]
+
+ if file and file in _global and 'group_label' in _global[file]:
+ return _global[file]['group_label']
+
+ if directory in _global and 'group_label' in _global[directory]:
+ return _global[directory]['group_label']
+
+ file_ext = os.path.basename(directory)
+ fname, ext = os.path.splitext(file_ext)
+ return "%s cli reference" % fname.capitalize()
+
+
+# Register our processor
+process.siphons["clicmd"] = SiphonCLICMD
diff --git a/docs/_scripts/siphon/process_syscfg.py b/docs/_scripts/siphon/process_syscfg.py
new file mode 100644
index 0000000..bccde2c
--- /dev/null
+++ b/docs/_scripts/siphon/process_syscfg.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generate syscfg formatted output
+
+from . import process, parsers
+
+
+class SiphonSYSCFG(process.Siphon):
+
+ name = "syscfg"
+ identifier = "VLIB_CONFIG_FUNCTION"
+
+ def __init__(self, *args, **kwargs):
+ super(SiphonSYSCFG, self).__init__(*args, **kwargs)
+ self._parser = parsers.ParseFunctionMacroStmt()
+
+
+# Register our processor
+process.siphons["syscfg"] = SiphonSYSCFG
diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md b/docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md
new file mode 100644
index 0000000..de517f1
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md
@@ -0,0 +1,15 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
\ No newline at end of file
diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/index_header.md b/docs/_scripts/siphon_templates/markdown/clicmd/index_header.md
new file mode 100644
index 0000000..86e5bb6
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/clicmd/index_header.md
@@ -0,0 +1,23 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+.. _cmdreference:
+
+Reference
+=========
+
+.. toctree::
+ :maxdepth: 2
+
diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/index_section.md b/docs/_scripts/siphon_templates/markdown/clicmd/index_section.md
new file mode 100644
index 0000000..0da934a
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/clicmd/index_section.md
@@ -0,0 +1,16 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+ clis/{{ this.page_label(group) }}
diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/item_format.md b/docs/_scripts/siphon_templates/markdown/clicmd/item_format.md
new file mode 100644
index 0000000..f1caa96
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/clicmd/item_format.md
@@ -0,0 +1,62 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{% set v = item['value'] %}
+{% if 'short_help' in v %}
+{% set str = v['short_help'] %}
+{% set str = this.unescape(str) %}
+
+{# Summary/usage #}
+{{ item['value']['path'] }}
+-------------------------------------------------------------------------
+
+.. code-block:: console
+
+ {{ this.reindent(str, 4) }}
+
+{% endif %}
+{% if 'long_help' in v %}
+{% set long_help = v['long_help'] %}
+{% set long_help = this.unescape(long_help) %}
+{# This is seldom used and will likely be deprecated #}
+{# Long help #}
+.. code-block:: console
+
+ {{ this.reindent(long_help, 4) }}
+
+{% endif %}
+{% if 'siphon_block' in item['meta'] %}
+{% set sb = item["meta"]["siphon_block"] %}
+{% set sb = this.process_special(sb) %}
+{% if sb %}
+{# Extracted from the code in /*? ... ?*/ blocks #}
+
+{# Description #}
+
+{{ sb }}
+{% endif %}
+{% endif %}
+{% if 'name' in meta or 'function' in v %}
+{# Gives some developer-useful linking #}
+
+{% if "name" in meta %}
+Declaration: ``{{ meta['name'] }}`` `{{ meta["file"] }} line {{ item["meta"]["line_start"] }} <{{ this.repository_link }}{{ meta["file"] }}#L{{ item["meta"]["line_start"] }}>`_
+{% endif %}
+{% if "function" in v %}
+
+Implementation: ``{{ v["function"] }}``
+{% endif %}
+{% endif %}
+
diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/item_header.md b/docs/_scripts/siphon_templates/markdown/clicmd/item_header.md
new file mode 100644
index 0000000..20e4803
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/clicmd/item_header.md
@@ -0,0 +1,20 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+.. _{{ this.page_label(group) }}:
+
+===============================================================
+{{ this.page_title(group) }}
+===============================================================
diff --git a/docs/_scripts/siphon_templates/markdown/default/index_entry.md b/docs/_scripts/siphon_templates/markdown/default/index_entry.md
new file mode 100644
index 0000000..479dcdb
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/default/index_entry.md
@@ -0,0 +1,16 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{{ "* [%s](@ref %s)" % (item["name"], meta["label"]) }}
diff --git a/docs/_scripts/siphon_templates/markdown/default/index_section.md b/docs/_scripts/siphon_templates/markdown/default/index_section.md
new file mode 100644
index 0000000..3c9d2b4
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/default/index_section.md
@@ -0,0 +1,18 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+@subpage {{ this.page_label(group) }}
+
diff --git a/docs/_scripts/siphon_templates/markdown/default/item_format.md b/docs/_scripts/siphon_templates/markdown/default/item_format.md
new file mode 100644
index 0000000..ed1b1bf
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/default/item_format.md
@@ -0,0 +1,16 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{{ raise NotImplementedError }}
diff --git a/docs/_scripts/siphon_templates/markdown/default/item_header.md b/docs/_scripts/siphon_templates/markdown/default/item_header.md
new file mode 100644
index 0000000..0c21e51
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/default/item_header.md
@@ -0,0 +1,18 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{{ "@page %s %s" % (this.page_label(group), this.page_title(group)) }}
+
diff --git a/docs/_scripts/siphon_templates/markdown/syscfg/index_header.md b/docs/_scripts/siphon_templates/markdown/syscfg/index_header.md
new file mode 100644
index 0000000..f5a9816
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/syscfg/index_header.md
@@ -0,0 +1,111 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+# Startup Configuration {{'{#'}}syscfg}
+
+The VPP network stack comes with several configuration options that can be
+provided either on the command line or in a configuration file.
+
+Specific applications built on the stack have been known to require a dozen
+arguments, depending on requirements. This section describes commonly-used
+options and parameters.
+
+You can find command-line argument parsers in the source code by searching for
+instances of the `VLIB_CONFIG_FUNCTION` macro. The invocation
+`VLIB_CONFIG_FUNCTION(foo_config, "foo")` will cause the function
+`foo_config` to receive all the options and values supplied in a parameter
+block named "`foo`", for example: `foo { arg1 arg2 arg3 ... }`.
+
+@todo Tell the nice people where this document lives so that the might
+help improve it!
+
+## Command-line arguments
+
+Parameters are grouped by a section name. When providing more than one
+parameter to a section all parameters for that section must be wrapped in
+curly braces.
+
+```
+/usr/bin/vpp unix { interactive cli-listen 127.0.0.1:5002 }
+```
+
+Which will produce output similar to this:
+
+ <startup diagnostic messages>
+ _______ _ _ _____ ___
+ __/ __/ _ \ (_)__ | | / / _ \/ _ \
+ _/ _// // / / / _ \ | |/ / ___/ ___/
+ /_/ /____(_)_/\___/ |___/_/ /_/
+
+ vpp# <start-typing>
+
+When providing only one such parameter the braces are optional. For example,
+the following command argument, `unix interactive` does not have braces:
+
+```
+/usr/bin/vpp unix interactive
+```
+
+The command line can be presented as a single string or as several; anything
+given on the command line is concatenated with spaces into a single string
+before parsing.
+
+VPP applications must be able to locate their own executable images. The
+simplest way to ensure this will work is to invoke a VPP application by giving
+its absolute path; for example: `/usr/bin/vpp <options>`. At startup, VPP
+applications parse through their own ELF-sections (primarily) to make lists
+of init, configuration, and exit handlers.
+
+When developing with VPP, in _gdb_ it's often sufficient to start an application
+like this at the `(gdb)` prompt:
+
+```
+run unix interactive
+```
+
+## Configuration file
+
+It is also possible to supply parameters in a startup configuration file the
+path of which is provided to the VPP application on its command line.
+
+The format of the configuration file is a simple text file with the same
+content as the command line but with the benefit of being able to use newlines
+to make the content easier to read. For example:
+
+```
+unix {
+ nodaemon
+ log /var/log/vpp/vpp.log
+ full-coredump
+ cli-listen localhost:5002
+}
+api-trace {
+ on
+}
+dpdk {
+ dev 0000:03:00.0
+}
+```
+
+VPP is then instructed to load this file with the `-c` option:
+
+```
+/usr/bin/vpp -c /etc/vpp/startup.conf
+```
+
+## Index of startup command sections
+
+[TOC]
+
diff --git a/docs/_scripts/siphon_templates/markdown/syscfg/item_format.md b/docs/_scripts/siphon_templates/markdown/syscfg/item_format.md
new file mode 100644
index 0000000..5313611
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/syscfg/item_format.md
@@ -0,0 +1,42 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{% set v = item['value'] %}
+{{ "@section %s %s" % (meta['label'], item['name']) }}
+{% if 'siphon_block' in item['meta'] %}
+{% set sb = item["meta"]["siphon_block"] %}
+{% if sb %}
+{# Extracted from the code in /*? ... ?*/ blocks #}
+
+### Description
+
+{{ sb }}
+{% endif %}
+{% endif %}
+{% if "name" in meta or "function" in item %}
+{# Gives some developer-useful linking #}
+
+### Declaration and implementation
+{% if "name" in meta %}
+
+{{ "Declaration: @ref %s (@ref %s line %d)" %
+ (meta['name'], meta["file"], item["meta"]["line_start"]) }}
+{% endif %}
+{% if "function" in item %}
+
+{{ "Implementation: @ref %s." % item["function"] }}
+{% endif %}
+{% endif %}
+
diff --git a/docs/about.rst b/docs/about.rst
deleted file mode 100644
index dc04d09..0000000
--- a/docs/about.rst
+++ /dev/null
@@ -1,9 +0,0 @@
-.. _about:
-
-=====
-About
-=====
-
-**VPP Version:** 21.10-rc0~204-g13e841847
-
-**Built on:** Thu Jul 22 23:44:06 GMT 2021
diff --git a/docs/whatisvpp/developer.rst b/docs/aboutvpp/developer.rst
similarity index 94%
rename from docs/whatisvpp/developer.rst
rename to docs/aboutvpp/developer.rst
index 5151e65..cc5783d 100644
--- a/docs/whatisvpp/developer.rst
+++ b/docs/aboutvpp/developer.rst
@@ -1,8 +1,8 @@
.. _developer-friendly:
-=======================
-Features for Developers
-=======================
+===================
+Additional features
+===================
This section describes a little about the VPP environment and some of the features
that can be used by developers.
diff --git a/docs/whatisvpp/extensible.rst b/docs/aboutvpp/extensible.rst
similarity index 89%
rename from docs/whatisvpp/extensible.rst
rename to docs/aboutvpp/extensible.rst
index 1df3b9f..adba873 100644
--- a/docs/whatisvpp/extensible.rst
+++ b/docs/aboutvpp/extensible.rst
@@ -4,7 +4,7 @@
The Packet Processing Graph
===========================
-At the core of the FD.io VPP design is the **Packet Procerssing Graph**
+At the core of the FD.io VPP design is the **Packet Processing Graph**
This makes the software:
@@ -21,8 +21,8 @@
.. figure:: /_images/VPP_custom_application_packet_processing_graph.280.jpg
:alt: Extensible, modular graph node architecture?
-
- Extensible and modular graph node architecture.
+
+ Extensible and modular graph node architecture.
At runtime, the FD.io VPP platform assembles a vector of packets from RX rings,
typically up to 256 packets in a single vector. The packet processing graph is
@@ -33,10 +33,10 @@
coupled. This makes it easy to introduce new graph nodes and rewire existing
graph nodes.
-Plugins are `shared libraries <https://en.wikipedia.org/wiki/Library_(computing)>`_
-and are loaded at runtime by VPP. VPP find plugins by searching the plugin path
-for libraries, and then dynamically loads each one in turn on startup.
-A plugin can introduce new graph nodes or rearrange the packet processing graph.
+Plugins are `shared libraries <https://en.wikipedia.org/wiki/Library_(computing)>`_
+and are loaded at runtime by VPP. VPP find plugins by searching the plugin path
+for libraries, and then dynamically loads each one in turn on startup.
+A plugin can introduce new graph nodes or rearrange the packet processing graph.
You can build a plugin completely independently of the FD.io VPP source tree,
which means you can treat it as an independent component.
diff --git a/docs/whatisvpp/hoststack.rst b/docs/aboutvpp/hoststack.rst
similarity index 89%
rename from docs/whatisvpp/hoststack.rst
rename to docs/aboutvpp/hoststack.rst
index 77e259a..b50c715 100644
--- a/docs/whatisvpp/hoststack.rst
+++ b/docs/aboutvpp/hoststack.rst
@@ -1,26 +1,26 @@
.. _hoststack:
-==============
-TCP Host Stack
-==============
+==========
+Host Stack
+==========
VPP’s host stack leverages VPP’s graph based forwarding model and vectorized packet
processing to ensure high throughput and scale transport protocol termination. It
exposes apis that apart from allowing for efficient user-space app consumption and
-generation of data, also enables highly efficient local inter-app communication.
-
-At a high level VPP’s host stack consists of 3 major components:
+generation of data, also enables highly efficient local inter-app communication.
+
+At a high level VPP’s host stack consists of 3 major components:
* A session layer that facilitates interaction between transport protocols and applications
* Pluggable transport protocols, including TCP, QUIC, TLS, UDP
* VCL (VPPComs library) a set of libraries meant to ease the consumability of the stack from application perspective
-
+
All of these components were custom built to fit within VPP’s architecture and to
leverage its speed. As a result, a significant amount of effort was invested into:
-* building a transport pluggable session layer that abstracts the interaction between applications and transports using a custom-built shared memory infrastructure. Notably, this also allows for transport protocols that are typically implemented in applications, like QUIC and TLS, to be implemented within VPP.
-* a clean slate TCP implementation that supports vectorized packet processing and follows VPP’s highly scalable threading model. The implementation is RFC compliant, supports a high number of high-speed TCP protocol features and it was validated using Defensic’s Codenomicon 1M+ tests suite.
-* VCL, a library that emulates traditional asynchronous communication functions in user-space, all while allowing for new patterns to be developed, if needed.
+* building a transport pluggable session layer that abstracts the interaction between applications and transports using a custom-built shared memory infrastructure. Notably, this also allows for transport protocols that are typically implemented in applications, like QUIC and TLS, to be implemented within VPP.
+* a clean slate TCP implementation that supports vectorized packet processing and follows VPP’s highly scalable threading model. The implementation is RFC compliant, supports a high number of high-speed TCP protocol features and it was validated using Defensic’s Codenomicon 1M+ tests suite.
+* VCL, a library that emulates traditional asynchronous communication functions in user-space, all while allowing for new patterns to be developed, if needed.
* implementing a high performance “cut-through” communication mode that enables applications attached to vpp to transparently exchange data over shared memory without incurring the extra cost of a traditional transport protocol. Testing has shown this to be much more efficient than traditional inter-container networking.
For developer features press next.
diff --git a/docs/whatisvpp/networkstack.rst b/docs/aboutvpp/networkstack.rst
similarity index 82%
rename from docs/whatisvpp/networkstack.rst
rename to docs/aboutvpp/networkstack.rst
index 20c4708..8141d4f 100644
--- a/docs/whatisvpp/networkstack.rst
+++ b/docs/aboutvpp/networkstack.rst
@@ -1,18 +1,18 @@
.. _network-stack:
-=============
-Network Stack
-=============
+======================
+Network Stack Features
+======================
This section describes a little about the FD.io network stack and describes some benefits:
* Layer 2 - 4 Network Stack
* Fast lookup tables for routes, bridge entries
- * Arbitrary n-tuple classifiers
+ * Arbitrary n-tuple classifiers
* Control Plane, Traffic Management and Overlays
-
+
* `Linux <https://en.wikipedia.org/wiki/Linux>`_ and `FreeBSD <https://en.wikipedia.org/wiki/FreeBSD>`_ support
* Support for standard Operating System Interfaces such as AF_Packet, Tun/Tap & Netmap.
@@ -23,17 +23,15 @@
* Para-virtualized interfaces; Vhost and Virtio
* Network Adapters over PCI passthrough
* Native container interfaces; MemIF
-
+
* Host Stack
* Universal Data Plane: one code base, for many use cases
-
+
* Discrete appliances; such as `Routers <https://en.wikipedia.org/wiki/Router_(computing)>`_ and `Switches <https://en.wikipedia.org/wiki/Network_switch>`_.
* `Cloud Infrastructure and Virtual Network Functions <https://en.wikipedia.org/wiki/Network_function_virtualization>`_
* `Cloud Native Infrastructure <https://www.cncf.io/>`_
- * The same binary package for all use cases.
+ * The same binary package for all use cases.
-* Out of the box production quality, with thanks to `CSIT <https://wiki.fd.io/view/CSIT#Start_Here>`_.
-
-For more information, please see :ref:`featuresbyrelease` for the complete list.
+* Out of the box production quality, with thanks to `CSIT <https://wiki.fd.io/view/CSIT#Start_Here>`_.
For more on the TCP Host Stack press next.
diff --git a/docs/whatisvpp/performance.rst b/docs/aboutvpp/performance.rst
similarity index 91%
rename from docs/whatisvpp/performance.rst
rename to docs/aboutvpp/performance.rst
index d7af24a..c818ff6 100644
--- a/docs/whatisvpp/performance.rst
+++ b/docs/aboutvpp/performance.rst
@@ -18,9 +18,9 @@
* The same optimized code-paths run execute on the host, and inside VMs and Linux containers
* Leverages best-of-breed open source driver technology: `DPDK <https://www.dpdk.org/>`_
-* Tested at scale; linear core scaling, tested with millions of flows and mac addresses
+* Tested at scale; linear core scaling, tested with millions of flows and mac addresses
-These features have been designed to take full advantage of common micro-processor optimization techniques, such as:
+These features have been designed to take full advantage of common micro-processor optimization techniques, such as:
* Reducing cache and TLS misses by processing packets in vectors
* Realizing `IPC <https://en.wikipedia.org/wiki/Instructions_per_cycle>`_ gains with vector instructions such as: SSE, AVX and NEON
@@ -28,8 +28,8 @@
* Cache-lined aligned buffers for cache and memory efficiency
-Continuous System Integration and Testing (CSIT)
-------------------------------------------------
+CSIT
+----
The Continuous System Integration and Testing (CSIT) project provides functional and performance
testing for FD.io VPP. This testing is focused on functional and performance regressions. The results
@@ -42,12 +42,12 @@
* `VPP Performance Dashboard <https://docs.fd.io/csit/master/trending/introduction/index.html>`_
-CSIT Packet Throughput examples
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Packet Throughput examples
+^^^^^^^^^^^^^^^^^^^^^^^^^^
Following are pointers to a few of the CSIT test reports. The test's titles read like this:
-<packet size>-<number of threads><number of cores>-<test>-<interface type>
+<packet size>-<number of threads><number of cores>-<test>-<interface type>
For example the test with the title 64b-2t1c-l2switching-base-i40e is the
test that does l2 switching using 64 byte packets, 2 threads, 1 core using an i40e
@@ -61,7 +61,7 @@
Trending Throughput Graphs
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^
These are some of the trending packet throughput graphs from the CSIT `trending dashboard <https://docs.fd.io/csit/master/trending/introduction/index.html>`_. **Please note that**, performance in the trending graphs will change on a nightly basis in line with the software development cycle:
diff --git a/docs/aboutvpp/releasenotes/index.rst b/docs/aboutvpp/releasenotes/index.rst
new file mode 100644
index 0000000..694096f
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/index.rst
@@ -0,0 +1,11 @@
+.. _release_notes:
+
+Release notes
+=============
+
+.. toctree::
+ :maxdepth: 2
+
+ v21.10
+ v21.06
+ past
diff --git a/docs/aboutvpp/releasenotes/past.rst b/docs/aboutvpp/releasenotes/past.rst
new file mode 100644
index 0000000..2ead5ec
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/past.rst
@@ -0,0 +1,39 @@
+.. _past_releases:
+
+Past releases
+=============
+
+.. toctree::
+ :maxdepth: 1
+
+ v21.01
+ v20.09
+ v20.05.1
+ v20.05
+ v20.01
+ v19.08.3
+ v19.08.2
+ v19.08.1
+ v19.08
+ v19.04.3
+ v19.04.2
+ v19.04.1
+ v19.04
+ v19.01.3
+ v19.01.2
+ v19.01.1
+ v19.01
+ v18.10
+ v18.07
+ v18.04
+ v18.01.2
+ v18.01.1
+ v18.01
+ v17.10
+ v17.07
+ v17.04
+ v17.01.1
+ v17.01
+ v16.09
+ v16.06
+
diff --git a/docs/aboutvpp/releasenotes/v16.06.rst b/docs/aboutvpp/releasenotes/v16.06.rst
new file mode 100644
index 0000000..400f2b3
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v16.06.rst
@@ -0,0 +1,52 @@
+Release notes for VPP 16.06
+===========================
+
+The FD.io Project, relentlessly focused on data IO speed and efficiency
+supporting the creation of high performance, flexible, and scalable
+software defined infrastructures, announces the availability of the
+community’s first software release (16.06).
+
+In the four months since launching, FD.io has brought together more than
+75 developers from 11 different companies including network operators,
+solution providers chip vendors, and network equipment vendors who are
+collaborating to enhance and innovate around the Vector Packet
+Processing (VPP) technology. The FD.io community has quickly formed to
+grow the number of projects from the initial VPP project to an
+additional 6 projects addressing a diverse set of requirements and
+usability across a variety of deployment environments.
+
+The 16.06 release brings unprecedented performance: 480Gbps/200mpps with
+8 million routes and 2k whitelist entries on standard high volume x86
+servers.
+
+Features
+--------
+
+In addition to the existing full suite of vswitch/vrouter features, the
+new 16.06 release adds:
+
+- Enhanced Switching and Routing:
+
+ - IPv6 Segment Routing multicast support.
+ - LISP xTR support.
+ - VXLAN over IPv6 underlay.
+ - Per interface whitelists.
+ - Shared adjacencies in FIB.
+
+- New and improved interface support:
+
+ - Jumbo frame support for vhost-user.
+ - Netmap interface support.
+ - AF_Packet interface support.
+
+- Expanded and improved programmability:
+
+ - Python API bindings.
+ - Enhanced JVPP Java API bindings.
+ - Debugging CLI.
+
+- Expanded Hardware and Software Support:
+
+ - Support for ARM 32 targets including Rasberry Pi single-board
+ computer.
+ - Support for DPDK 16.04.
diff --git a/docs/aboutvpp/releasenotes/v16.09.rst b/docs/aboutvpp/releasenotes/v16.09.rst
new file mode 100644
index 0000000..00179bc
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v16.09.rst
@@ -0,0 +1,252 @@
+Release notes for VPP 16.09
+===========================
+
+Features
+--------
+
+- `Integrated July 2016 DPDK release <http://www.dpdk.org/doc/guides/rel_notes/release_16_07.html>`__
+
+ - DPDK-vhost is depreciated pending a complete rework of the
+ original integration and addressing of rx performance deltas.
+ - Patches required for DPDK 16.07:
+
+ - Correctly setting the Packet Type in the IGB, IXGBE and i40e
+ drivers.
+ - Correctly setting checksum in the i40e driver.
+ - NXP DPAA2 PMD Driver.
+ - rte_delay (yield) functionality.
+
+- Add “in tree” plugins:
+
+ - IPv6 ILA.
+ - iOAM.
+ - Load Balancer.
+ - SNAT.
+
+- High-performance (line-rate) “neutron like” L4 port-filtering.
+
+- API refactoring - addressing some of the issues around JVPP bindings.
+
+ - Accommodating plugins
+ `(e.g. NSH_SFC) <https://wiki.fd.io/view/NSH_SFC>`__
+ - Binding for `python <https://wiki.fd.io/view/VPP/Python_API>`__
+
+- LISP
+
+ - L2 LISP overlays
+ - Multitenancy
+ - Multihoming
+ - RTR mode
+ - Map-resolver failover algorithm
+
+- Support 64-bit vector lengths, huge shared-memory segments.
+
+- Dynamic IP Feature ordering
+
+ - IP Features can now specify features they appear before and after
+
+- 16.09 Builds
+
+ - Ubuntu 14.04 LTS - Trusty Tahr
+ - Ubuntu 16.04 LTS - Xenial Xerus
+ - CentOS 7
+ - More information on `VPP wiki <https://wiki.fd.io/view/VPP/Installing_VPP_binaries_from_packages>`__
+
+- Performance, characterize and document performance for this release
+ `(more information on CSIT page) <https://wiki.fd.io/view/CSIT>`__
+
+ - IPv4 and IPv6 Scale - performance tests.
+
+ - Bidirectional 10k/100k/1M flows.
+ - 64B,570B, 1518B,9000B packet sizes.
+
+ - IPv6 iACL - performance
+
+ - DUT1 and DUT2 are configured with IPv6 routing, two static IPv6
+ /64 routes and IPv6 iAcl security whitelist ingress /64 filter
+ entries applied on links.
+ - TG traffic profile contains two L3 flow-groups (flow-group per
+ direction, 253 flows per flow-group) with all packets
+ containing Ethernet header, IPv6 header and generated payload.
+ MAC addresses are matching MAC addresses of the TG node
+ interfaces.
+
+ - L2XC VXLANoIPv4 - performance
+
+ - DUT1 and DUT2 are configured with L2 cross-connect. VXLAN
+ tunnels are configured between L2XCs on DUT1 and DUT2.
+ - TG traffic profile contains two L3 flow-groups (flow-group per
+ direction, 253 flows per flow-group) with all packets
+ containing Ethernet header, IPv4 header with IP protocol=61 and
+ generated payload. MAC addresses are matching MAC addresses of
+ the TG node interfaces.
+
+- Documentation
+
+ - Autogenerated CLI documentation.
+ - Using doxygen to automate API/Node documentation.
+ - `(available online) <https://docs.fd.io/vpp/16.09/>`__
+
+- Resolved all static analysis issues found by Coverity
+
+ - Beginning of 16.09 cycle: 505 issues.
+ - Release: 0 outstanding issues.
+
+Known issues
+------------
+
+Issues in fd.io are tracked in `JIRA <https://jira.fd.io>`__.
+
++-----------------------------------+-----------------------------------+
+| Issue | Description |
++===================================+===================================+
+| VPP-391 | vpp debug version assert appeared |
+| | in the process of start |
++-----------------------------------+-----------------------------------+
+| VPP-380 | Mapping algorithm compute wrong |
+| | ea-bits when IPv4 prefix |
+| | 0.0.0.0/0 |
++-----------------------------------+-----------------------------------+
+| VPP-371 | load_one_plugin:63: Loaded |
+| | plugin: message from vppctl |
++-----------------------------------+-----------------------------------+
+| VPP-367 | vpp packages need to depend on |
+| | specific versions of each other |
++-----------------------------------+-----------------------------------+
+| VPP-312 | IP6 FIB gets in indeterminate |
+| | state by duplicating commands |
++-----------------------------------+-----------------------------------+
+| VPP-224 | Lookup-in-vrf can not be set |
+| | correctly |
++-----------------------------------+-----------------------------------+
+| VPP-206 | Fix classify table delete |
++-----------------------------------+-----------------------------------+
+| VPP-203 | Fix binary API for reading vpp |
+| | node graph |
++-----------------------------------+-----------------------------------+
+| VPP-147 | Inconsistent behaviour when |
+| | adding L2 FIB filter entry |
++-----------------------------------+-----------------------------------+
+| VPP-99 | VPP doesn’t discard DHCPOFFER |
+| | message with wrong XID |
++-----------------------------------+-----------------------------------+
+
+Issues fixed
+------------
+
+Issues in fd.io are tracked in `JIRA <https://jira.fd.io>`__.
+
++-----------------------------------+-----------------------------------+
+| Issue | Description |
++===================================+===================================+
+| VPP-396 | Ubuntu systems Graphviz bug |
++-----------------------------------+-----------------------------------+
+| VPP-390 | vpp-lib rpm fails to include |
+| | \*.so symlinks, causing linking |
+| | problems with out of tree builds |
++-----------------------------------+-----------------------------------+
+| VPP-388 | IPSec output feature assumes |
+| | packets have been ethernet |
+| | rewritten |
++-----------------------------------+-----------------------------------+
+| VPP-385 | ARP for indirect adjacencies not |
+| | working correctly |
++-----------------------------------+-----------------------------------+
+| VPP-361 | Memory leak on delete of VXLAN |
+| | over IPv6 tunnel |
++-----------------------------------+-----------------------------------+
+| VPP-357 | VNI not set correctly when |
+| | removing LISP fwd entries |
++-----------------------------------+-----------------------------------+
+| VPP-349 | sw_interface_vhost_user_dump not |
+| | working |
++-----------------------------------+-----------------------------------+
+| VPP-345 | net/enic: bad L4 checksum ptype |
+| | set on ICMP packets |
++-----------------------------------+-----------------------------------+
+| VPP-340 | MAP-T wrong destination address |
++-----------------------------------+-----------------------------------+
+| VPP-330 | Use fifo to store LISP pending |
+| | map-requests |
++-----------------------------------+-----------------------------------+
+| VPP-326 | map_add_domain VAT command: |
+| | unable to configure domain with |
+| | mtu parameter |
++-----------------------------------+-----------------------------------+
+| VPP-318 | The map_add_domain VAT command |
+| | accepts invalid arguments |
++-----------------------------------+-----------------------------------+
+| VPP-315 | Fix “show vxlan-gpe” issue |
++-----------------------------------+-----------------------------------+
+| VPP-310 | Mapping algorithm compute wrong |
+| | ea-bits |
++-----------------------------------+-----------------------------------+
+| VPP-239 | LISP IP forwarding does not tag |
+| | packets that hit negative mapping |
+| | entries |
++-----------------------------------+-----------------------------------+
+| VPP-235 | Invalid help in VAT for |
+| | sw_interface_set_l2_bridge |
++-----------------------------------+-----------------------------------+
+| VPP-228 | Mapping algorithm sends packet to |
+| | wrong IPv6 address |
++-----------------------------------+-----------------------------------+
+| VPP-214 | vpp-api-test: |
+| | api_ipsec_sad_add_del_entry: |
+| | vector “ck” not initialized |
++-----------------------------------+-----------------------------------+
+| VPP-200 | VPP - TAP port create problem |
++-----------------------------------+-----------------------------------+
+| VPP-189 | Coverity Issues for 16.09 |
++-----------------------------------+-----------------------------------+
+| VPP-184 | u16 translating to char ,not |
+| | short |
++-----------------------------------+-----------------------------------+
+| VPP-179 | Adjacency share-count botch |
++-----------------------------------+-----------------------------------+
+| VPP-163 | “show ip6 interface” ignores |
+| | non-global addresses |
++-----------------------------------+-----------------------------------+
+| VPP-155 | Netmap: Inconsistency in |
+| | interface state between “show |
+| | hardware” and “show interface” |
++-----------------------------------+-----------------------------------+
+| VPP-145 | Dynamically compute IP feature |
+| | ordering based on constraints |
++-----------------------------------+-----------------------------------+
+| VPP-137 | VPP sends ARP with wrong |
+| | requested IP |
++-----------------------------------+-----------------------------------+
+| VPP-118 | JVpp: 0 length arrays not handled |
+| | properly in VPP responses |
++-----------------------------------+-----------------------------------+
+| VPP-112 | linux kernel info missing from |
+| | build log |
++-----------------------------------+-----------------------------------+
+| VPP-110 | vxlan encap node should never |
+| | touch a deleted tunnel |
++-----------------------------------+-----------------------------------+
+| VPP-107 | RPM build broken in master |
++-----------------------------------+-----------------------------------+
+| VPP-92 | segment routing is not properly |
+| | filling out the segment list |
++-----------------------------------+-----------------------------------+
+| VPP-91 | segment routing add/del tunnel |
+| | lookup doesn’t work |
++-----------------------------------+-----------------------------------+
+| VPP-84 | af_packet throws a fatal error on |
+| | EAGAIN |
++-----------------------------------+-----------------------------------+
+| VPP-74 | Clang compile fails due to |
+| | warning in vlib/unix/cli.c |
++-----------------------------------+-----------------------------------+
+| VPP-64 | Top level “make pkg-deb” fails if |
+| | CDPATH is set in user env. |
++-----------------------------------+-----------------------------------+
+| VPP-48 | Traceroute does not terminate |
+| | when VPP is the target |
++-----------------------------------+-----------------------------------+
+| VPP-23 | CLI pager does not gracefully |
+| | handle lines longer than the |
+| | terminal width |
++-----------------------------------+-----------------------------------+
diff --git a/docs/aboutvpp/releasenotes/v17.01.1.rst b/docs/aboutvpp/releasenotes/v17.01.1.rst
new file mode 100644
index 0000000..9453654
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v17.01.1.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 17.01.1
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1701>`__
diff --git a/docs/aboutvpp/releasenotes/v17.01.rst b/docs/aboutvpp/releasenotes/v17.01.rst
new file mode 100644
index 0000000..0f22fa7
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v17.01.rst
@@ -0,0 +1,99 @@
+Release notes for VPP 17.01
+===========================
+
+@note This release was for a while known as 16.12.
+
+Features
+--------
+
+- `Integrated November 2016 DPDK release <http://www.dpdk.org/doc/guides/rel_notes/release_16_11.html>`__
+
+- Complete rework of Forwarding Information Base (FIB)
+
+- Performance Improvements
+
+ - Improvements in DPDK input and output nodes
+ - Improvements in L2 path
+ - Improvmeents in IPv4 lookup node
+
+- Feature Arcs Improvements
+
+ - Consolidation of the code
+ - New feature arcs
+
+ - device-input
+ - interface-output
+
+- DPDK Cryptodev Support
+
+ - Software and Hardware Crypto Support
+
+- DPDK HQoS support
+
+- Simple Port Analyzer (SPAN)
+
+- Bidirectional Forwarding Detection
+
+ - Basic implementation
+
+- IPFIX Improvements
+
+- L2 GRE over IPSec tunnels
+
+- Link Layer Discovery Protocol (LLDP)
+
+- Vhost-user Improvements
+
+ - Performance Improvements
+ - Multiqueue
+ - Reconnect
+
+- LISP Enhancements
+
+ - Source/Dest control plane support
+ - L2 over LISP and GRE
+ - Map-Register/Map-Notify/RLOC-probing support
+ - L2 API improvements, overall code hardening
+
+- Plugins:
+
+ - New: ACL
+ - New: Flow per Packet
+ - Improved: SNAT
+
+ - Mutlithreading
+ - Flow export
+
+- Doxygen Enhancements
+
+- Luajit API bindings
+
+- API Refactoring
+
+ - file split
+ - message signatures
+
+- Python and Scapy based unit testing infrastructure
+
+ - Infrastructure
+ - Various tests
+
+- Packet Generator improvements
+
+- TUN/TAP jumbo frames support
+
+- Other various bug fixes and improvements
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1701>`__
diff --git a/docs/aboutvpp/releasenotes/v17.04.rst b/docs/aboutvpp/releasenotes/v17.04.rst
new file mode 100644
index 0000000..dd21462
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v17.04.rst
@@ -0,0 +1,146 @@
+Release notes for VPP 17.04
+===========================
+
+More than 500 commits since the 1701 release.
+
+Features
+--------
+
+- Infrastructure
+
+ - make test improvements
+ - vnet: add device-input threadplacement infra
+ - 64 bit per-thread counters
+ - process restart cli
+ - High performance timer wheels
+ - Plugin infrastructure improvements
+
+ - Support for .default_disabled, .version_required
+
+ - Added MAINTAINERS file
+
+- Host stack
+
+ - TCP stack (experimental)
+ - DHCPv4 / DHCPv6 relay multi-destination
+ - DHCPv4 option 82
+ - ND proxy
+ - Attached hosts
+ - Consolidated DHCPv4 and DHCPv6 implementation
+
+- Interfaces
+
+ - DPDK 17.02 (retire support for DPDK 16.07)
+ - Add memif - packet memory interface for intra-host communication
+ - vhost: support interrupt mode
+ - DPDK as plugin (retired vpp_lite)
+ - DPDPK input optimizations
+ - Loopback interface allocation scheme
+
+- Network features
+
+ - IP Multicast FIB
+
+ - Bridging
+
+ - Learning on local interfaces
+ - Flushing of MACs from the L2 FIB
+
+ - SNAT
+
+ - CGN (Deterministic and dynamic)
+ - CGN configurable port allocation algorithm
+ - ICMP support
+ - Tentant VRF id for SNAT outside addresses
+ - Session dump / User dump
+ - Port allocation per protocol
+
+ - Security groups
+
+ - Routed interface support
+ - L2+L3 unified processing node
+ - Improve fragment handling
+
+ - Segement routing v6
+
+ - SR policies with weighted SID lists
+ - Binding SID
+ - SR steering policies
+ - SR Local SIDs
+ - Framework to expand local SIDs w/plugins
+ - Documentation
+
+ - IOAM
+
+ - UDP Pinger w/path fault isolation
+ - IOAM as type 2 metadata in NSH
+ - IAOM raw IPFIX collector and analyzer
+ - Anycast active server selection
+ - Documentation
+ - SRv6 Local SID
+ - IP6 HBH header and SR header co-existence
+ - Active probe
+
+ - LISP
+
+ - Statistics collection
+ - Generalize encap for overlay transport (vxlan-gpe support)
+ - Improve data plane speed
+
+ - GPE
+
+ - CLI
+ - NSH added to encap/decap path
+ - Renamed LISP GPE API to GPE
+
+ - MPLS
+
+ - Performance improvements (quad loop)
+
+ - BFD
+
+ - Command line interface
+ - Echo function
+ - Remote demand mode
+ - SHA1 authentication
+
+ - IPsec
+
+ - IKEv2 initiator features
+
+ - VXLAN
+
+ - unify IP4/IP6 control plane handling
+
+API changes
+-----------
+
+- Python API: To avoid conflicts between VPP API messages names and the
+ Python API binding function names, VPP API methods are put in a
+ separate proxy object. https://gerrit.fd.io/r/#/c/5570/ The api
+ methods are now referenced as: vpp_handle = VPP(jsonfiles)
+ vpp_handle.connect(…) vpp = vpp_handle.api vpp.show_version()
+ vpp_handle.disconnect()
+
+ For backwards compatibility VPP API methods are left in the main name
+ space (VPP), but will be removed from 17.07.
+
+ - Python API: Change from cPython to CFFI.
+
+- create_loopback message to be replaced with create_loopback_instance
+ create_loopback will be removed from 17.07.
+ https://gerrit.fd.io/r/#/c/5572/
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1704>`__
diff --git a/docs/aboutvpp/releasenotes/v17.07.rst b/docs/aboutvpp/releasenotes/v17.07.rst
new file mode 100644
index 0000000..aaa4d5a
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v17.07.rst
@@ -0,0 +1,76 @@
+Release notes for VPP 17.07
+===========================
+
+More than 400 commits since the 1704 release.
+
+Features
+--------
+
+- Infrastructure
+
+ - make test; improved debuggability.
+ - TAB auto-completion on the CLI
+ - DPDK 17.05
+ - python 3 support in test infra
+
+- Host stack
+
+ - Improved Linux TCP stack compatibility using IWL test suite
+ (https://jira.fd.io/browse/VPP-720)
+ - Improved loss recovery (RFC5681, RFC6582, RF6675)
+ - Basic implementation of Eifel detection algorithm (RFC3522)
+ - Basic support for buffer chains
+ - Refactored session layer API
+ - Overall performance, scale and hardening
+
+- Interfaces
+
+ - memif: IP mode, jumbo frames, multi queue
+ - virtio-user support
+ - vhost-usr; adaptive (poll/interupt) support.
+
+- Network features
+
+ - MPLS Multicast FIB
+
+ - BFD FIB integration
+
+ - NAT64 support
+
+ - GRE over IPv6
+
+ - Segement routing MPLS
+
+ - IOAM configuration for SRv6 localsid
+
+ - LISP
+
+ - NSH support
+ - native forward static routes
+ - L2 ARP
+
+ - ACL multi-core suuport
+
+ - Flowprobe:
+
+ - Add flowstartns, flowendns and tcpcontrolbits
+ - Stateful flows and IPv6, L4 recording
+
+ - GTP-U support
+
+ - VXLAN GPE support for FIB2.0 and bypass.
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1707>`__
+
diff --git a/docs/aboutvpp/releasenotes/v17.10.rst b/docs/aboutvpp/releasenotes/v17.10.rst
new file mode 100644
index 0000000..d96f5ab
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v17.10.rst
@@ -0,0 +1,236 @@
+Release notes for VPP 17.10
+===========================
+
+More than 400 commits since the 1707 release.
+
+Features
+--------
+
+- Infrastructure
+
+ - DPDK 17.08
+ - IP reassembly
+ - Bounded-index extensible hash bucket-level LRU cache
+ - Templated timer wheel improvements
+
+- API
+
+ - C/C++ language binding
+ - API stats
+
+- Host stack
+
+ - VPP TCP stack scale/congestion improvements
+ - VPP Comms Library (VCL)
+ - Overall performance, scale and hardening
+
+- Network features
+
+ - IPSec rework - utilize new FIB
+
+ - VPLS and VPWS implementation
+
+ - NAT
+
+ - Renamed SNAT to NAT
+ - Performance / Scale
+ - Destination NAT44 with load-balancing
+ - In2out translation as an output feature on the outside
+ interface
+ - Fullback to 3-tuple key for non TCP/UDP/ICMP sessions
+
+ - Security Groups/ACLs
+
+ - “Replace” semantics for adding a new MacIP acl
+ - Test suite tests for MacIP ACLs
+
+ - ONE-LISP
+
+ - Map-server fallback support
+ - Preemptive re-fetch of active mappings that are about to expire
+ - ND termination
+
+ - PPPoE
+
+ - PPPoE Control Plane packet dispatch
+ - PPPoE decapsulation
+ - PPPoE encapsulation
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1710>`__
+
+
+API changes
+-----------
+
+======================================== ==================
+Message Name Result
+======================================== ==================
+bridge_domain_add_del definition changed
+bridge_domain_details definition changed
+connect_session definition changed
+connect_sock definition changed
+connect_sock_reply definition changed
+connect_uri_reply definition changed
+create_vhost_user_if definition changed
+dhcp_client_config definition changed
+ip4_arp_event definition changed
+ip6_fib_details definition changed
+ip6_nd_event definition changed
+ip_add_del_route definition changed
+ip_fib_details definition changed
+ip_table_add_del definition changed
+l2_macs_event only in image
+macip_acl_add_replace definition changed
+macip_acl_interface_list_details only in image
+macip_acl_interface_list_dump only in image
+modify_vhost_user_if definition changed
+mpls_fib_details definition changed
+mpls_route_add_del definition changed
+mpls_table_add_del definition changed
+mpls_tunnel_add_del definition changed
+nat44_add_del_address_range definition changed
+nat44_add_del_interface_addr definition changed
+nat44_add_del_lb_static_mapping definition changed
+nat44_add_del_static_mapping definition changed
+nat44_address_details only in image
+nat44_address_dump only in image
+nat44_interface_add_del_feature definition changed
+nat44_interface_add_del_output_feature definition changed
+nat44_interface_addr_details only in image
+nat44_interface_addr_dump only in image
+nat44_interface_details only in image
+nat44_interface_dump only in image
+nat44_interface_output_feature_details only in image
+nat44_interface_output_feature_dump only in image
+nat44_lb_static_mapping_details only in image
+nat44_lb_static_mapping_dump only in image
+nat44_static_mapping_details only in image
+nat44_static_mapping_dump only in image
+nat44_user_details only in image
+nat44_user_dump only in image
+nat44_user_session_details only in image
+nat44_user_session_dump only in image
+nat_control_ping definition changed
+nat_det_add_del_map definition changed
+nat_det_close_session_in definition changed
+nat_det_close_session_out definition changed
+nat_det_forward definition changed
+nat_det_get_timeouts definition changed
+nat_det_map_details only in image
+nat_det_map_dump only in image
+nat_det_reverse definition changed
+nat_det_session_details only in image
+nat_det_session_dump only in image
+nat_det_set_timeouts definition changed
+nat_ipfix_enable_disable definition changed
+nat_set_workers definition changed
+nat_show_config definition changed
+nat_worker_details only in image
+nat_worker_dump only in image
+one_add_del_ndp_entry definition changed
+one_enable_disable_petr_mode definition changed
+one_enable_disable_pitr_mode definition changed
+one_enable_disable_xtr_mode definition changed
+one_get_transport_protocol definition changed
+one_map_register_fallback_threshold definition changed
+one_map_register_set_ttl definition changed
+one_ndp_bd_get definition changed
+one_ndp_entries_get definition changed
+one_set_transport_protocol definition changed
+one_show_petr_mode definition changed
+one_show_pitr_mode definition changed
+one_show_xtr_mode definition changed
+p2p_ethernet_add definition changed
+pppoe_add_del_session definition changed
+pppoe_session_details only in image
+pppoe_session_dump only in image
+punt_socket_deregister definition changed
+punt_socket_register definition changed
+show_one_map_register_fallback_threshold definition changed
+show_one_map_register_ttl definition changed
+snat_interface_add_del_output_feature definition changed
+snat_interface_output_feature_details only in image
+snat_interface_output_feature_dump only in image
+sw_interface_event only in image
+sw_interface_set_flags definition changed
+sw_interface_span_dump definition changed
+sw_interface_span_enable_disable definition changed
+sw_interface_vhost_user_details definition changed
+tcp_configure_src_addresses definition changed
+vnet_per_interface_combined_counters only in image
+vnet_per_interface_simple_counters only in image
+want_interface_combined_stats definition changed
+want_interface_simple_stats definition changed
+want_ip4_fib_stats definition changed
+want_ip4_nbr_stats definition changed
+want_ip6_fib_stats definition changed
+want_ip6_nbr_stats definition changed
+want_l2_macs_events definition changed
+want_per_interface_combined_stats definition changed
+want_per_interface_simple_stats definition changed
+======================================== ==================
+
+Found 103 api message signature differences
+
+Patches that updated the API files
+----------------------------------
+* ``./src/plugins/pppoe/pppoe.api`` 62f9cdd8 Add PPPoE Plugin
+* ``./src/plugins/acl/acl.api`` c29940c5 ACL-plugin add “replace” semantics
+ for adding a new MacIP acl de9fbf43 MAC IP ACL interface list dump (as
+ an alternative to the get/reply)
+* ``./src/plugins/nat/nat.api`` 704018cf NAT: Destination NAT44 with
+ load-balancing (VPP-954) 2ba92e32 NAT: Rename snat plugin to nat
+ (VPP-955)
+* ``./src/vnet/interface.api`` 831fb59f Stats refactor d292ab1e No context in
+ SW interface event a07bd708 Dedicated SW Interface Event
+* ``./src/vnet/dhcp/dhcp.api`` 51822bf0 DHCP client option 61 “client_id”
+ 4729b1ec DHCP complete event sends mask length
+* ``./src/vnet/lldp/lldp.api`` 99a0e60e Add API support for LLDP
+ config/interface set
+* ``./src/vnet/lisp-cp/one.api`` d630713d LISP: add neighbor discovery and CP
+ protocol separation APIs 111a5cea LISP: Add APIs for enable/disable
+ xTR/P-ITR/P-ETR modes 7048ff1e LISP: Map-server fallback feature
+ 1e553a00 LISP: make TTL for map register messages configurable
+* ``./src/vnet/ethernet/p2p_ethernet.api`` 15ac81c1 P2P Ethernet
+* ``./src/vnet/mpls/mpls.api`` 2297af01 Add a name to the creation of an IP
+ and MPLS table 28ab9cc1 FIB table add/delete API only da78f957 L2 over
+ MPLS a0a908f1 FIB path weight incorrect in dump (VPP-922) 57b5860f FIB
+ path preference
+* ``./src/vnet/session/session.api`` 33e002b1 Fix session connect api message
+ handling.
+* ``./src/vnet/span/span.api`` 5b311202 SPAN/API:enable L2 dump 001fd406
+ SPAN:add l2 mirror
+* ``./src/vnet/devices/virtio/vhost_user.api`` 4ba75f54 vhost: Remove
+ operation mode in the API
+* ``./src/vnet/vxlan-gpe/vxlan_gpe.api`` 04ffd0ad VPP crash on creating vxlan
+ gpe interface. VPP-875
+* ``./src/vnet/tcp/tcp.api`` 3bbcfab1 TCP source address automation
+* ``./src/vnet/ip/ip.api`` 2297af01 Add a name to the creation of an IP and
+ MPLS table 28ab9cc1 FIB table add/delete API only 57b5860f FIB path
+ preference
+* ``./src/vnet/lisp-gpe/lisp_gpe.api`` af3d9771 Remove unused retval from
+ gpe_native_fwd_rpath type definition
+* ``./src/vnet/l2/l2.api`` 50570ece Update of free text tag patch for BD
+ 48304141 Support for bridge domain free text tag e531f4cb Increase
+ default MAC learn limit and check it in learn-update path 8d00fff8 Add
+ support for API client to receive L2 MAC events
+* ``./src/vpp/api/vpe.api`` 8a19f12a Allow individual stats API and introduce
+ stats.api 4802632d Punt socket: Fix coverity error for pathname length
+ mismatch between API and sun_path. f7a55ad7 PUNT socket: External
+ control plane processes connected via UNIX domain sockets. 75e2f2ac
+ API:fix arp/ND event messages - remove context 99a0e60e Add API support
+ for LLDP config/interface set
+* ``./src/vpp/stats/stats.api`` 831fb59f Stats refactor 8a19f12a Allow
+ individual stats API and introduce stats.api
diff --git a/docs/aboutvpp/releasenotes/v18.01.1.rst b/docs/aboutvpp/releasenotes/v18.01.1.rst
new file mode 100644
index 0000000..7f2cb6f
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v18.01.1.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 18.01.1
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1801>`__
diff --git a/docs/aboutvpp/releasenotes/v18.01.2.rst b/docs/aboutvpp/releasenotes/v18.01.2.rst
new file mode 100644
index 0000000..c984b36
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v18.01.2.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 18.01.2
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1801>`__
diff --git a/docs/aboutvpp/releasenotes/v18.01.rst b/docs/aboutvpp/releasenotes/v18.01.rst
new file mode 100644
index 0000000..e760de2
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v18.01.rst
@@ -0,0 +1,382 @@
+Release notes for VPP 18.01
+===========================
+
+More than 560 commits since the 17.10 release.
+
+Features
+--------
+
+- Infrastructure
+
+ - DPDK 17.11
+ - TCP Checksum Offload
+ - Arm64/Arm-v8 support
+ - SUSE packaging
+ - bihash_vec8_8 variant
+ - PCI rework to support VFIO
+ - chi-squared test calculator
+
+- SNAT / NAT
+
+ - One armed NAT
+ - Twice NAT44
+ - NAT hairpinning rework
+ - NAT64 multi-thread
+ - NAT64 IPFIX
+ - NAT64 Fragmentation
+ - NAT: DS-Lite
+ - Remove old SNAT API
+ - ACL-based NAT
+
+- VNET
+
+ - DNS name resolver
+ - BIER
+ - GENEVE Tunnel
+ - IPSec Openssl 1.1.0 api support
+ - FIB improvements
+ - tap v2
+
+- API
+
+ - VPP stats (Broadcast & Multicast support)
+ - SR MPLS
+ - VPP Object Model (VOM)
+
+- Host Stack
+
+ - VPP TCP Stack scale / congestion improvements
+ - Refactor UDP
+ - Namespace support
+ - Session rules table
+ - VPP Comms Library (VCL) improvements
+
+- ACL
+
+ - ACL stats
+
+- Plugins
+
+ - Kube-proxy
+ - L2 Emulation
+ - Memif
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1801>`__
+
+
+API changes
+-----------
+
+===================================== ==================
+Message Name Result
+===================================== ==================
+af_packet_set_l4_cksum_offload definition changed
+api_versions definition changed
+app_namespace_add_del definition changed
+application_attach definition changed
+bier_disp_entry_add_del definition changed
+bier_disp_entry_details only in image
+bier_disp_entry_dump only in image
+bier_disp_table_add_del definition changed
+bier_disp_table_details only in image
+bier_disp_table_dump only in image
+bier_imp_add definition changed
+bier_imp_del definition changed
+bier_imp_details only in image
+bier_imp_dump only in image
+bier_route_add_del definition changed
+bier_route_details only in image
+bier_route_dump only in image
+bier_table_add_del definition changed
+bier_table_details only in image
+bier_table_dump only in image
+bind_sock_reply definition changed
+connect_session_reply definition changed
+connect_sock definition changed
+connect_uri definition changed
+dhcp_proxy_details definition changed
+dhcp_proxy_set_vss definition changed
+dns_enable_disable definition changed
+dns_name_server_add_del definition changed
+dns_resolve_ip definition changed
+dns_resolve_name definition changed
+dslite_add_del_pool_addr_range definition changed
+dslite_set_aftr_addr definition changed
+geneve_add_del_tunnel definition changed
+geneve_tunnel_details only in image
+geneve_tunnel_dump only in image
+ip_add_del_route definition changed
+ip_container_proxy_add_del definition changed
+ip_mroute_add_del definition changed
+ip_neighbor_details definition changed
+ip_punt_police definition changed
+ip_punt_redirect definition changed
+ipsec_sa_details only in image
+ipsec_sa_dump only in image
+ipsec_sad_add_del_entry definition changed
+ipsec_tunnel_if_set_key definition changed
+ipsec_tunnel_if_set_sa definition changed
+kp_add_del_pod definition changed
+kp_add_del_vip definition changed
+kp_conf definition changed
+l2_emulation definition changed
+l2_fib_table_details definition changed
+l2fib_add_del definition changed
+memclnt_keepalive definition changed
+memfd_segment_create definition changed
+mpls_ip_bind_unbind definition changed
+mpls_route_add_del definition changed
+nat44_add_del_address_range definition changed
+nat44_add_del_identity_mapping definition changed
+nat44_add_del_interface_addr definition changed
+nat44_add_del_lb_static_mapping definition changed
+nat44_add_del_static_mapping definition changed
+nat44_address_details definition changed
+nat44_del_session definition changed
+nat44_forwarding_enable_disable definition changed
+nat44_forwarding_is_enabled definition changed
+nat44_identity_mapping_details only in image
+nat44_identity_mapping_dump only in image
+nat44_interface_addr_details definition changed
+nat44_lb_static_mapping_details definition changed
+nat44_static_mapping_details definition changed
+nat64_add_del_interface_addr definition changed
+nat_get_reass definition changed
+nat_reass_details only in image
+nat_reass_dump only in image
+nat_set_reass definition changed
+reset_vrf definition changed
+session_rule_add_del definition changed
+session_rules_details only in image
+session_rules_dump only in image
+snat_add_address_range definition changed
+snat_add_del_interface_addr definition changed
+snat_add_det_map definition changed
+snat_add_static_mapping definition changed
+snat_address_details only in file
+snat_address_dump only in file
+snat_control_ping definition changed
+snat_det_close_session_in definition changed
+snat_det_close_session_out definition changed
+snat_det_forward definition changed
+snat_det_get_timeouts definition changed
+snat_det_map_details only in file
+snat_det_map_dump only in file
+snat_det_reverse definition changed
+snat_det_session_details only in file
+snat_det_session_dump only in file
+snat_det_set_timeouts definition changed
+snat_interface_add_del_feature definition changed
+snat_interface_add_del_output_feature definition changed
+snat_interface_addr_details only in file
+snat_interface_addr_dump only in file
+snat_interface_details only in file
+snat_interface_dump only in file
+snat_interface_output_feature_details only in file
+snat_interface_output_feature_dump only in file
+snat_ipfix_enable_disable definition changed
+snat_set_workers definition changed
+snat_show_config definition changed
+snat_static_mapping_details only in file
+snat_static_mapping_dump only in file
+snat_user_details only in file
+snat_user_dump only in file
+snat_user_session_details only in file
+snat_user_session_dump only in file
+snat_worker_details only in file
+snat_worker_dump only in file
+sockclnt_create definition changed
+sockclnt_delete definition changed
+sr_localsids_details only in image
+sr_localsids_dump only in image
+sr_mpls_policy_add definition changed
+sr_mpls_policy_assign_endpoint_color definition changed
+sr_mpls_policy_del definition changed
+sr_mpls_policy_mod definition changed
+sr_mpls_steering_add_del definition changed
+sr_set_encap_source definition changed
+stn_add_del_rule definition changed
+stn_rule_details only in image
+stn_rules_dump only in image
+sw_interface_set_geneve_bypass definition changed
+sw_interface_set_lldp definition changed
+sw_interface_set_rx_mode definition changed
+sw_interface_tap_v2_details only in image
+sw_interface_tap_v2_dump only in image
+tap_create_v2 definition changed
+tap_delete_v2 definition changed
+udp_encap_add_del definition changed
+udp_encap_details only in image
+udp_encap_dump only in image
+vnet_ip4_mfib_counters only in image
+vnet_ip6_mfib_counters only in image
+want_ip4_mfib_stats definition changed
+want_ip6_mfib_stats definition changed
+===================================== ==================
+
+Found 142 api message signature differences
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``./src/examples/sample-plugin/sample/sample.api`` 0d056e5e vppapigen:
+ support per-file (major,minor,patch) version stamps
+* ``./src/vnet/interface.api b8d4481a`` Break up vpe.api ad8015be devices: Add
+ binary API for set interface rx-mode 0d056e5e vppapigen: support
+ per-file (major,minor,patch) version stamps
+* ``./src/vnet/feature/feature.api`` b8d4481a Break up vpe.api
+* ``./src/vnet/srv6/sr.api`` 1a5e301f SRv6 improvements to binary API 0d056e5e
+ vppapigen: support per-file (major,minor,patch) version stamps
+* ``./src/vnet/bier/bier.api`` be302d72 BIER coverity fix in route downlaod
+ ceb4d05b BIER disposition default route fa1da15c BIER: API documentation
+ fixes. 9128637e BIER in non-MPLS netowrks d792d9c0 BIER
+* ``./src/vnet/vxlan-gpe/vxlan_gpe.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/unix/tap.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/lldp/lldp.api`` 9a6fcef4 LLDP: Add Management Address TLV
+ 0d056e5e vppapigen: support per-file (major,minor,patch) version stamps
+* ``./src/vnet/dns/dns.api`` d2080159 Add reverse DNS (ip to name) resolution
+ 6545716c VPP-1027: DNS name resolver
+* ``./src/vnet/session/session.api`` dcf55ce2 vppcom: improve listener session
+ handling 6e8c6679 session: add app ns index to ns create api c97a7398
+ session: add rule tags 6c36f53f session: add api to dump rules 1c710451
+ session: rules tables ade70e45 session: return local transport endpoint
+ in connect reply cea194d8 session: add support for application
+ namespacing 0d056e5e vppapigen: support per-file (major,minor,patch)
+ version stamps
+* ``./src/vnet/devices/af_packet/af_packet.api`` 92b0275a af_packet: invalid
+ TCP/UDP offload checksum on RX node recalculation 0d056e5e vppapigen:
+ support per-file (major,minor,patch) version stamps
+* ``./src/vnet/devices/netmap/netmap.api`` 0d056e5e vppapigen: support
+ per-file (major,minor,patch) version stamps
+* ``./src/vnet/devices/tap/tapv2.api`` 73e7f427 tap_v2: include host-side
+ parameters in the dump binary API 2df39094 tapv2: multiple improvements
+ c99b4cd1 tap_v2: move code to vnet/devices/tap
+* ``./src/vnet/devices/virtio/vhost_user.api`` 0d056e5e vppapigen: support
+ per-file (major,minor,patch) version stamps
+* ``./src/vnet/lisp-gpe/lisp_gpe.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/srmpls/sr_mpls.api`` 42998828 SR-MPLS: binary API and automated
+ steering
+* ``./src/vnet/l2/l2.api`` b8d4481a Break up vpe.api 57938f63 l2fib: MAC: Fix
+ uint64 to u8 byte array 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/udp/udp.api`` 810086d8 UDP Encapsulation.
+* ``./src/vnet/policer/policer.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/bfd/bfd.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/geneve/geneve.api`` 556033a0 Add API versioning to GENEVE
+ tunnel implementation. b598f1d3 Initial GENEVE TUNNEL implementation and
+ tests.
+* ``./src/vnet/gre/gre.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/map/map.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/flow/flow.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/pg/pg.api`` b8d4481a Break up vpe.api
+* ``./src/vnet/dhcp/dhcp.api`` 70bfcaf4 Add Support of DHCP VSS Type 0 where
+ VPN-ID is ASCII 0d056e5e vppapigen: support per-file (major,minor,patch)
+ version stamps
+* ``./src/vnet/ipsec/ipsec.api`` ca514fda Allow IPsec interface to have SAs
+ reset 75d85609 Add API call to set keys on IPsec tunnel intf 0d056e5e
+ vppapigen: support per-file (major,minor,patch) version stamps 28029530
+ Add API support to dump IPsec SAs
+* ``./src/vnet/mpls/mpls.api`` c42fc05b Remove the unused ‘create VRF if
+ needed’ API parameters b8d4481a Break up vpe.api d792d9c0 BIER d0a59722
+ Revert “Enforce FIB table creation before use” f9342023 Enforce FIB
+ table creation before use 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/ethernet/p2p_ethernet.api`` 0d056e5e vppapigen: support
+ per-file (major,minor,patch) version stamps
+* ``./src/vnet/span/span.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/lisp-cp/lisp.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/lisp-cp/one.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/vxlan/vxlan.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/ipsec-gre/ipsec_gre.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/vnet/classify/classify.api`` b8d4481a Break up vpe.api 8527f12b add
+ classify session action set-sr-policy-index 0d056e5e vppapigen: support
+ per-file (major,minor,patch) version stamps
+* ``./src/vnet/ip/punt.api`` b8d4481a Break up vpe.api
+* ``./src/vnet/ip/ip.api`` c42fc05b Remove the unused ‘create VRF if needed’
+ API parameters b8d4481a Break up vpe.api af8dfbf6 Add sw_if_index to the
+ ip_neighbor_details_t response. d792d9c0 BIER 810086d8 UDP
+ Encapsulation. 595992c5 ip: add container proxy api 0164a06d Remove
+ unused ‘not_last’ parameter from ip_add_del_route d0a59722 Revert
+ “Enforce FIB table creation before use” 054c03ac Source Lookup
+ progammable via API f9342023 Enforce FIB table creation before use
+ d91c1dbd punt and drop features: - new IPv4 and IPv6 feature arcs on the
+ punt and drop nodes - new features: - redirect punted traffic to an
+ interface and nexthop - police punted traffic. 0d056e5e vppapigen:
+ support per-file (major,minor,patch) version stamps 6f631156 Distributed
+ Virtual Router Support
+* ``./src/vnet/cop/cop.api`` 0d056e5e vppapigen: support per-file (major,minor,patch) version stamps
+* ``./src/vnet/l2tp/l2tp.api`` 0d056e5e vppapigen: support per-file (major,minor,patch) version stamps
+* ``./src/vpp/oam/oam.api`` b8d4481a Break up vpe.api
+* ``./src/vpp/stats/stats.api`` ff233898 Stats for Multicast FIB 0d056e5e
+ vppapigen: support per-file (major,minor,patch) version stamps
+* ``./src/vpp/api/vpe.api`` b8d4481a Break up vpe.api d792d9c0 BIER 0d056e5e
+ vppapigen: support per-file (major,minor,patch) version stamps b598f1d3
+ Initial GENEVE TUNNEL implementation and tests.
+* ``./src/plugins/ioam/udp-ping/udp_ping.api`` 0d056e5e vppapigen: support
+ per-file (major,minor,patch) version stamps
+* ``./src/plugins/ioam/ip6/ioam_cache.api`` 0d056e5e vppapigen: support
+ per-file (major,minor,patch) version stamps
+* ``./src/plugins/ioam/lib-pot/pot.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/plugins/ioam/lib-trace/trace.api`` 0d056e5e vppapigen: support
+ per-file (major,minor,patch) version stamps
+* ``./src/plugins/ioam/export/ioam_export.api`` 0d056e5e vppapigen: support
+ per-file (major,minor,patch) version stamps
+* ``./src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api`` 0d056e5e
+ vppapigen: support per-file (major,minor,patch) version stamps
+* ``./src/plugins/ioam/lib-vxlan-gpe/ioam_vxlan_gpe.api`` 0d056e5e vppapigen:
+ support per-file (major,minor,patch) version stamps
+* ``./src/plugins/pppoe/pppoe.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/plugins/dpdk/api/dpdk.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/plugins/acl/acl.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/plugins/gtpu/gtpu.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/plugins/l2e/l2e.api`` 4ec38711 L2 emulation: remove usued
+ ip-table-id from API 55d03788 L2 Emulation
+* ``./src/plugins/flowprobe/flowprobe.api`` 0d056e5e vppapigen: support
+ per-file (major,minor,patch) version stamps
+* ``./src/plugins/nat/nat.api`` 7b929793 Translate matching packets using NAT
+ (VPP-1069) b932d26e NAT: Twice NAT44 (VPP-969) ab7a805f NAT44: identity
+ NAT (VPP-1073) c6fb36fc NAT: Remove old SNAT API (VPP-1070) 0938dcf1
+ NAT64 to use IPv4 address from interface (VPP-1051) efcd1e9e SNAT: IP
+ fragmentation (VPP-890) 8ebe6253 NAT: DS-Lite (VPP-1040) 5ba86f72 NAT:
+ delete session API/CLI (VPP-1041) 36ea2d6d One armed NAT (VPP-1035)
+ 0d056e5e vppapigen: support per-file (major,minor,patch) version stamps
+* ``./src/plugins/memif/memif.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/plugins/kubeproxy/kp.api`` c91f5024 Support kube-proxy data plane
+* ``./src/plugins/lb/lb.api`` 0d056e5e vppapigen: support per-file
+ (major,minor,patch) version stamps
+* ``./src/plugins/stn/stn.api`` 0906c5cf Plugin for IP-Address to Interface
+ Punting
+* ``./src/vlibmemory/memclnt.api 0d056e5e`` vppapigen: support per-file
+ (major,minor,patch) version stamps 59b2565c Repair vlib API socket
+ server
diff --git a/docs/aboutvpp/releasenotes/v18.04.rst b/docs/aboutvpp/releasenotes/v18.04.rst
new file mode 100644
index 0000000..603e1d8
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v18.04.rst
@@ -0,0 +1,2523 @@
+Release notes for VPP 18.04
+===========================
+
+More than 570 commits since the 18.01 release.
+
+Features
+--------
+
+Infrastructure
+~~~~~~~~~~~~~~
+
+- DPDK 18.02.1
+- ARM aarch64 integrated into CI
+
+VNET & Plugins
+~~~~~~~~~~~~~~
+
+- ERSPAN
+- L3DSR load balancing support
+- VPC bonding / LACP
+- IPv4/IPv6 packet reassembly
+- IPv6 link-local support
+- Asymmetrical static NAT
+- 464XLAT for NAT44
+- MAP-T CE support
+- Intel Adaptive Virtual Function native device driver plugin
+- Marvell device plugin
+- SRv6 static, dynamic and masquerading proxy plugins
+- MPLS Uniform mode
+- IGMP plugin
+- IPIP tunnel support (IPv4/IPv6 over IPv4/IPv6)
+- IPv6 Router Discovery mechanism
+
+VLIB
+~~~~
+
+- ARM-optimized library variations for key functions
+- Better handling of physmem on non-NUMA kernels
+
+Host stack
+~~~~~~~~~~
+
+- TLS support via OpenSSL or mbedtls software engines
+- Session layer can utilize both shm and memfd (secure) FIFO segments
+- STCP
+- VCL logging / tracing
+
+API framework
+~~~~~~~~~~~~~
+
+- New API definition compiler (vppapigen)
+- Memory (shm) and socket APIs refactored
+- API handlers refactored to make them transport (shared memory or
+ socket) agnostic
+- Improved support for bootstrapping of the shm API with memfd segments
+ over the socket API
+
+Packaging
+~~~~~~~~~
+
+- SELinux for RPM builds
+- Debuginfo RPMs
+- ARM aarch64 for Ubuntu
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1804>`__
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+- *Only in image*: indicates the API is new for this release.
+- *Only in file*: indicates the API has been removed in this release.
+
++--------------------------------------------------------+--------------------+
+| Message Name | Results |
++========================================================+====================+
+| accept_session | definition changed |
++--------------------------------------------------------+--------------------+
+| accept_session_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_add_replace | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_add_replace_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_del | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_details | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_interface_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_interface_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_interface_etype_whitelist_details | only in image |
++--------------------------------------------------------+--------------------+
+| acl_interface_etype_whitelist_dump | only in image |
++--------------------------------------------------------+--------------------+
+| acl_interface_list_details | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_interface_list_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_interface_set_acl_list | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_interface_set_acl_list_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_interface_set_etype_whitelist | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_plugin_control_ping | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_plugin_control_ping_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_plugin_get_version | definition changed |
++--------------------------------------------------------+--------------------+
+| acl_plugin_get_version_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| add_node_next | definition changed |
++--------------------------------------------------------+--------------------+
+| add_node_next_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| af_packet_create | definition changed |
++--------------------------------------------------------+--------------------+
+| af_packet_create_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| af_packet_delete | definition changed |
++--------------------------------------------------------+--------------------+
+| af_packet_delete_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| af_packet_set_l4_cksum_offload | definition changed |
++--------------------------------------------------------+--------------------+
+| af_packet_set_l4_cksum_offload_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| api_versions | definition changed |
++--------------------------------------------------------+--------------------+
+| api_versions_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| app_namespace_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| app_namespace_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| application_attach | definition changed |
++--------------------------------------------------------+--------------------+
+| application_attach_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| application_detach | definition changed |
++--------------------------------------------------------+--------------------+
+| application_detach_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| application_tls_cert_add | definition changed |
++--------------------------------------------------------+--------------------+
+| application_tls_key_add | definition changed |
++--------------------------------------------------------+--------------------+
+| bd_ip_mac_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| bd_ip_mac_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_auth_del_key | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_auth_del_key_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_auth_keys_details | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_auth_keys_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_auth_set_key | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_auth_set_key_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_add | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_add_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_auth_activate | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_auth_activate_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_auth_deactivate | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_auth_deactivate_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_del | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_del_echo_source | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_del_echo_source_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_mod | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_mod_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_session_details | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_session_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_session_set_flags | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_session_set_flags_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_set_echo_source | definition changed |
++--------------------------------------------------------+--------------------+
+| bfd_udp_set_echo_source_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_disp_entry_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_disp_entry_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_disp_entry_details | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_disp_entry_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_disp_table_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_disp_table_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_disp_table_details | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_disp_table_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_imp_add | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_imp_add_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_imp_del | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_imp_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_imp_details | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_imp_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_route_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_route_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_route_details | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_route_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_table_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_table_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_table_details | definition changed |
++--------------------------------------------------------+--------------------+
+| bier_table_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| bind_sock | definition changed |
++--------------------------------------------------------+--------------------+
+| bind_sock_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bind_uri | definition changed |
++--------------------------------------------------------+--------------------+
+| bind_uri_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bond_create | definition changed |
++--------------------------------------------------------+--------------------+
+| bond_delete | definition changed |
++--------------------------------------------------------+--------------------+
+| bond_detach_slave | definition changed |
++--------------------------------------------------------+--------------------+
+| bond_enslave | definition changed |
++--------------------------------------------------------+--------------------+
+| bridge_domain_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| bridge_domain_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bridge_domain_details | definition changed |
++--------------------------------------------------------+--------------------+
+| bridge_domain_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| bridge_domain_set_mac_age | definition changed |
++--------------------------------------------------------+--------------------+
+| bridge_domain_set_mac_age_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| bridge_flags | definition changed |
++--------------------------------------------------------+--------------------+
+| bridge_flags_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| cdp_enable | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_add_del_session | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_add_del_session_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_add_del_table | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_add_del_table_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_session_details | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_session_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_set_interface_ip_table | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_set_interface_ip_table_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_set_interface_l2_tables | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_set_interface_l2_tables_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_table_by_interface | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_table_by_interface_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_table_ids | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_table_ids_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_table_info | definition changed |
++--------------------------------------------------------+--------------------+
+| classify_table_info_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| cli | definition changed |
++--------------------------------------------------------+--------------------+
+| cli_inband | definition changed |
++--------------------------------------------------------+--------------------+
+| cli_inband_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| cli_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| collect_detailed_interface_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| connect_session | definition changed |
++--------------------------------------------------------+--------------------+
+| connect_session_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| connect_sock | definition changed |
++--------------------------------------------------------+--------------------+
+| connect_sock_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| connect_uri | definition changed |
++--------------------------------------------------------+--------------------+
+| connect_uri_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| control_ping | definition changed |
++--------------------------------------------------------+--------------------+
+| control_ping_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| cop_interface_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| cop_interface_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| cop_whitelist_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| cop_whitelist_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| create_loopback | definition changed |
++--------------------------------------------------------+--------------------+
+| create_loopback_instance | definition changed |
++--------------------------------------------------------+--------------------+
+| create_loopback_instance_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| create_loopback_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| create_subif | definition changed |
++--------------------------------------------------------+--------------------+
+| create_subif_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| create_vhost_user_if | definition changed |
++--------------------------------------------------------+--------------------+
+| create_vhost_user_if_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| create_vlan_subif | definition changed |
++--------------------------------------------------------+--------------------+
+| create_vlan_subif_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| delete_loopback | definition changed |
++--------------------------------------------------------+--------------------+
+| delete_loopback_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| delete_subif | definition changed |
++--------------------------------------------------------+--------------------+
+| delete_subif_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| delete_vhost_user_if | definition changed |
++--------------------------------------------------------+--------------------+
+| delete_vhost_user_if_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| dhcp_client_config | definition changed |
++--------------------------------------------------------+--------------------+
+| dhcp_client_config_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| dhcp_compl_event | definition changed |
++--------------------------------------------------------+--------------------+
+| dhcp_proxy_config | definition changed |
++--------------------------------------------------------+--------------------+
+| dhcp_proxy_config_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| dhcp_proxy_details | definition changed |
++--------------------------------------------------------+--------------------+
+| dhcp_proxy_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| dhcp_proxy_set_vss | definition changed |
++--------------------------------------------------------+--------------------+
+| dhcp_proxy_set_vss_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| disconnect_session | definition changed |
++--------------------------------------------------------+--------------------+
+| disconnect_session_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| dns_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| dns_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| dns_name_server_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| dns_name_server_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| dns_resolve_ip | definition changed |
++--------------------------------------------------------+--------------------+
+| dns_resolve_ip_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| dns_resolve_name | definition changed |
++--------------------------------------------------------+--------------------+
+| dns_resolve_name_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| dslite_add_del_pool_addr_range | definition changed |
++--------------------------------------------------------+--------------------+
+| dslite_add_del_pool_addr_range_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| dslite_get_aftr_addr | definition changed |
++--------------------------------------------------------+--------------------+
+| dslite_get_b4_addr | definition changed |
++--------------------------------------------------------+--------------------+
+| dslite_set_aftr_addr | definition changed |
++--------------------------------------------------------+--------------------+
+| dslite_set_aftr_addr_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| dslite_set_b4_addr | definition changed |
++--------------------------------------------------------+--------------------+
+| feature_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| feature_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| flow_classify_details | definition changed |
++--------------------------------------------------------+--------------------+
+| flow_classify_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| flow_classify_set_interface | definition changed |
++--------------------------------------------------------+--------------------+
+| flow_classify_set_interface_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| flowprobe_params | definition changed |
++--------------------------------------------------------+--------------------+
+| flowprobe_params_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| flowprobe_tx_interface_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| flowprobe_tx_interface_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gbp_contract_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| gbp_contract_details | only in image |
++--------------------------------------------------------+--------------------+
+| gbp_contract_dump | only in image |
++--------------------------------------------------------+--------------------+
+| gbp_endpoint_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| gbp_endpoint_details | only in image |
++--------------------------------------------------------+--------------------+
+| gbp_endpoint_dump | only in image |
++--------------------------------------------------------+--------------------+
+| geneve_add_del_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| geneve_add_del_tunnel_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| geneve_tunnel_details | definition changed |
++--------------------------------------------------------+--------------------+
+| geneve_tunnel_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| get_first_msg_id | definition changed |
++--------------------------------------------------------+--------------------+
+| get_first_msg_id_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| get_next_index | definition changed |
++--------------------------------------------------------+--------------------+
+| get_next_index_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| get_node_graph | definition changed |
++--------------------------------------------------------+--------------------+
+| get_node_graph_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| get_node_index | definition changed |
++--------------------------------------------------------+--------------------+
+| get_node_index_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_add_del_fwd_entry | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_add_del_fwd_entry_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_add_del_iface | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_add_del_iface_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_add_del_native_fwd_rpath | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_add_del_native_fwd_rpath_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_fwd_entries_get | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_fwd_entries_get_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_fwd_entry_path_details | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_fwd_entry_path_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_fwd_entry_vnis_get | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_fwd_entry_vnis_get_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_get_encap_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_get_encap_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_native_fwd_rpaths_get | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_native_fwd_rpaths_get_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_set_encap_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| gpe_set_encap_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gre_add_del_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| gre_add_del_tunnel_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gre_tunnel_details | definition changed |
++--------------------------------------------------------+--------------------+
+| gre_tunnel_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| gtpu_add_del_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| gtpu_add_del_tunnel_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| gtpu_tunnel_details | definition changed |
++--------------------------------------------------------+--------------------+
+| gtpu_tunnel_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| igmp_clear_interface | definition changed |
++--------------------------------------------------------+--------------------+
+| igmp_details | only in image |
++--------------------------------------------------------+--------------------+
+| igmp_dump | only in image |
++--------------------------------------------------------+--------------------+
+| igmp_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| igmp_event | only in image |
++--------------------------------------------------------+--------------------+
+| igmp_listen | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_initiate_del_child_sa | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_initiate_del_child_sa_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_initiate_del_ike_sa | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_initiate_del_ike_sa_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_initiate_rekey_child_sa | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_initiate_rekey_child_sa_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_initiate_sa_init | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_initiate_sa_init_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_profile_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_profile_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_profile_set_auth | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_profile_set_auth_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_profile_set_id | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_profile_set_id_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_profile_set_ts | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_profile_set_ts_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_set_esp_transforms | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_set_esp_transforms_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_set_ike_transforms | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_set_ike_transforms_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_set_local_key | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_set_local_key_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_set_responder | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_set_responder_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_set_sa_lifetime | definition changed |
++--------------------------------------------------------+--------------------+
+| ikev2_set_sa_lifetime_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| input_acl_set_interface | definition changed |
++--------------------------------------------------------+--------------------+
+| input_acl_set_interface_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| interface_name_renumber | definition changed |
++--------------------------------------------------------+--------------------+
+| interface_name_renumber_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ioam_cache_ip6_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| ioam_cache_ip6_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ioam_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| ioam_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ioam_enable | definition changed |
++--------------------------------------------------------+--------------------+
+| ioam_enable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ioam_export_ip6_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| ioam_export_ip6_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ip4_arp_event | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6_fib_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6_fib_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6_mfib_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6_mfib_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6_nd_address_autoconfig | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6_nd_event | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6_ra_event | only in image |
++--------------------------------------------------------+--------------------+
+| ip6nd_proxy_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6nd_proxy_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6nd_proxy_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6nd_proxy_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ip6nd_send_router_solicitation | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_add_del_route | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_add_del_route_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_address_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_address_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_container_proxy_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_container_proxy_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_fib_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_fib_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_mfib_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_mfib_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_mroute_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_mroute_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_neighbor_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_neighbor_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_neighbor_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_neighbor_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_punt_police | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_punt_police_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_punt_redirect | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_punt_redirect_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_reassembly_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_reassembly_get | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_reassembly_set | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_source_and_port_range_check_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_source_and_port_range_check_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_source_and_port_range_check_interface_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_source_and_port_range_check_interface_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_table_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ip_table_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ipfix_classify_stream_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ipfix_classify_stream_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ipfix_classify_table_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ipfix_classify_table_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ipfix_classify_table_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ipfix_classify_table_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ipfix_exporter_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ipfix_exporter_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ipip_6rd_add_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| ipip_6rd_del_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| ipip_add_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| ipip_del_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| ipip_tunnel_details | only in image |
++--------------------------------------------------------+--------------------+
+| ipip_tunnel_dump | only in image |
++--------------------------------------------------------+--------------------+
+| ipsec_gre_add_del_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_gre_add_del_tunnel_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_gre_tunnel_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_gre_tunnel_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_interface_add_del_spd | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_interface_add_del_spd_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_sa_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_sa_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_sa_set_key | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_sa_set_key_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_sad_add_del_entry | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_sad_add_del_entry_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_spd_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_spd_add_del_entry | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_spd_add_del_entry_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_spd_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_spd_details | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_spd_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_tunnel_if_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_tunnel_if_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_tunnel_if_set_key | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_tunnel_if_set_key_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_tunnel_if_set_sa | definition changed |
++--------------------------------------------------------+--------------------+
+| ipsec_tunnel_if_set_sa_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| kp_add_del_pod | definition changed |
++--------------------------------------------------------+--------------------+
+| kp_add_del_pod_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| kp_add_del_vip | definition changed |
++--------------------------------------------------------+--------------------+
+| kp_add_del_vip_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| kp_conf | definition changed |
++--------------------------------------------------------+--------------------+
+| kp_conf_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_emulation | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_emulation_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_fib_clear_table | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_fib_clear_table_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_fib_table_details | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_fib_table_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_flags | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_flags_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_interface_efp_filter | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_interface_efp_filter_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_interface_pbb_tag_rewrite | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_interface_pbb_tag_rewrite_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_interface_vlan_tag_rewrite | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_interface_vlan_tag_rewrite_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_macs_event | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_patch_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_patch_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_xconnect_details | definition changed |
++--------------------------------------------------------+--------------------+
+| l2_xconnect_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| l2fib_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| l2fib_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2fib_flush_all | definition changed |
++--------------------------------------------------------+--------------------+
+| l2fib_flush_all_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2fib_flush_bd | definition changed |
++--------------------------------------------------------+--------------------+
+| l2fib_flush_bd_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2fib_flush_int | definition changed |
++--------------------------------------------------------+--------------------+
+| l2fib_flush_int_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2tpv3_create_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| l2tpv3_create_tunnel_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2tpv3_interface_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| l2tpv3_interface_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2tpv3_set_lookup_key | definition changed |
++--------------------------------------------------------+--------------------+
+| l2tpv3_set_lookup_key_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| l2tpv3_set_tunnel_cookies | definition changed |
++--------------------------------------------------------+--------------------+
+| l2tpv3_set_tunnel_cookies_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lb_add_del_as | definition changed |
++--------------------------------------------------------+--------------------+
+| lb_add_del_as_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lb_add_del_vip | definition changed |
++--------------------------------------------------------+--------------------+
+| lb_add_del_vip_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lb_conf | definition changed |
++--------------------------------------------------------+--------------------+
+| lb_conf_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_adjacency | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_adjacency_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_local_eid | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_local_eid_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_locator | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_locator_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_locator_set | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_locator_set_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_map_request_itr_rlocs | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_map_request_itr_rlocs_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_map_resolver | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_map_resolver_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_map_server | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_map_server_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_remote_mapping | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_add_del_remote_mapping_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_adjacencies_get | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_adjacencies_get_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_eid_table_add_del_map | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_eid_table_add_del_map_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_eid_table_details | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_eid_table_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_eid_table_map_details | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_eid_table_map_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_eid_table_vni_details | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_eid_table_vni_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_get_map_request_itr_rlocs | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_get_map_request_itr_rlocs_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_locator_details | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_locator_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_locator_set_details | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_locator_set_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_map_register_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_map_register_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_map_request_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_map_request_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_map_resolver_details | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_map_resolver_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_map_server_details | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_map_server_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_pitr_set_locator_set | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_pitr_set_locator_set_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_rloc_probe_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_rloc_probe_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_use_petr | definition changed |
++--------------------------------------------------------+--------------------+
+| lisp_use_petr_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| lldp_config | definition changed |
++--------------------------------------------------------+--------------------+
+| lldp_config_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_add | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_add_replace | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_add_replace_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_add_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_del | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_details | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_interface_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_interface_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_interface_get | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_interface_get_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_interface_list_details | definition changed |
++--------------------------------------------------------+--------------------+
+| macip_acl_interface_list_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| map_add_del_rule | definition changed |
++--------------------------------------------------------+--------------------+
+| map_add_del_rule_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| map_add_domain | definition changed |
++--------------------------------------------------------+--------------------+
+| map_add_domain_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| map_another_segment | definition changed |
++--------------------------------------------------------+--------------------+
+| map_another_segment_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| map_del_domain | definition changed |
++--------------------------------------------------------+--------------------+
+| map_del_domain_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| map_domain_details | definition changed |
++--------------------------------------------------------+--------------------+
+| map_domain_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| map_rule_details | definition changed |
++--------------------------------------------------------+--------------------+
+| map_rule_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| map_summary_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| map_summary_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| memclnt_create | definition changed |
++--------------------------------------------------------+--------------------+
+| memclnt_create_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| memclnt_delete | definition changed |
++--------------------------------------------------------+--------------------+
+| memclnt_delete_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| memclnt_keepalive | definition changed |
++--------------------------------------------------------+--------------------+
+| memclnt_keepalive_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| memclnt_read_timeout | definition changed |
++--------------------------------------------------------+--------------------+
+| memclnt_rx_thread_suspend | definition changed |
++--------------------------------------------------------+--------------------+
+| memfd_segment_create | definition changed |
++--------------------------------------------------------+--------------------+
+| memfd_segment_create_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| memif_create | definition changed |
++--------------------------------------------------------+--------------------+
+| memif_create_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| memif_delete | definition changed |
++--------------------------------------------------------+--------------------+
+| memif_delete_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| memif_details | definition changed |
++--------------------------------------------------------+--------------------+
+| memif_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| memif_socket_filename_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| memif_socket_filename_details | only in image |
++--------------------------------------------------------+--------------------+
+| memif_socket_filename_dump | only in image |
++--------------------------------------------------------+--------------------+
+| mfib_signal_details | definition changed |
++--------------------------------------------------------+--------------------+
+| mfib_signal_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| modify_vhost_user_if | definition changed |
++--------------------------------------------------------+--------------------+
+| modify_vhost_user_if_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_fib_details | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_fib_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_ip_bind_unbind | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_ip_bind_unbind_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_route_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_route_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_table_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_table_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_tunnel_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_tunnel_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_tunnel_details | definition changed |
++--------------------------------------------------------+--------------------+
+| mpls_tunnel_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_add_del_address_range | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_add_del_address_range_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_add_del_identity_mapping | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_add_del_identity_mapping_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_add_del_interface_addr | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_add_del_interface_addr_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_add_del_lb_static_mapping | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_add_del_lb_static_mapping_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_add_del_static_mapping | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_add_del_static_mapping_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_address_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_address_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_del_session | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_del_session_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_forwarding_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_forwarding_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_forwarding_is_enabled | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_forwarding_is_enabled_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_identity_mapping_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_identity_mapping_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_interface_add_del_feature | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_interface_add_del_feature_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_interface_add_del_output_feature | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_interface_add_del_output_feature_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_interface_addr_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_interface_addr_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_interface_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_interface_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_interface_output_feature_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_interface_output_feature_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_lb_static_mapping_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_lb_static_mapping_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_static_mapping_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_static_mapping_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_user_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_user_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_user_session_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat44_user_session_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_add_del_interface_addr | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_add_del_interface_addr_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_add_del_interface | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_add_del_interface_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_add_del_pool_addr_range | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_add_del_pool_addr_range_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_add_del_prefix | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_add_del_prefix_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_add_del_static_bib | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_add_del_static_bib_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_bib_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_bib_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_get_timeouts | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_get_timeouts_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_interface_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_interface_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_pool_addr_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_pool_addr_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_prefix_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_prefix_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_set_timeouts | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_set_timeouts_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_st_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat64_st_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat66_add_del_interface | definition changed |
++--------------------------------------------------------+--------------------+
+| nat66_add_del_static_mapping | definition changed |
++--------------------------------------------------------+--------------------+
+| nat66_interface_details | only in image |
++--------------------------------------------------------+--------------------+
+| nat66_interface_dump | only in image |
++--------------------------------------------------------+--------------------+
+| nat66_static_mapping_details | only in image |
++--------------------------------------------------------+--------------------+
+| nat66_static_mapping_dump | only in image |
++--------------------------------------------------------+--------------------+
+| nat_control_ping | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_control_ping_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_add_del_map | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_add_del_map_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_close_session_in | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_close_session_in_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_close_session_out | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_close_session_out_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_forward | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_forward_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_get_timeouts | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_get_timeouts_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_map_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_map_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_reverse | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_reverse_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_session_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_session_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_set_timeouts | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_det_set_timeouts_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_get_reass | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_get_reass_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_ipfix_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_ipfix_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_reass_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_reass_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_set_reass | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_set_reass_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_set_workers | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_set_workers_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_show_config | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_show_config_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_worker_details | definition changed |
++--------------------------------------------------------+--------------------+
+| nat_worker_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| netmap_create | definition changed |
++--------------------------------------------------------+--------------------+
+| netmap_create_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| netmap_delete | definition changed |
++--------------------------------------------------------+--------------------+
+| netmap_delete_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| oam_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| oam_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| oam_event | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_adjacency | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_adjacency_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_l2_arp_entry | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_l2_arp_entry_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_local_eid | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_local_eid_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_locator | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_locator_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_locator_set | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_locator_set_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_map_request_itr_rlocs | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_map_request_itr_rlocs_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_map_resolver | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_map_resolver_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_map_server | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_map_server_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_ndp_entry | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_ndp_entry_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_remote_mapping | definition changed |
++--------------------------------------------------------+--------------------+
+| one_add_del_remote_mapping_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_adjacencies_get | definition changed |
++--------------------------------------------------------+--------------------+
+| one_adjacencies_get_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_eid_table_add_del_map | definition changed |
++--------------------------------------------------------+--------------------+
+| one_eid_table_add_del_map_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_eid_table_details | definition changed |
++--------------------------------------------------------+--------------------+
+| one_eid_table_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| one_eid_table_map_details | definition changed |
++--------------------------------------------------------+--------------------+
+| one_eid_table_map_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| one_eid_table_vni_details | definition changed |
++--------------------------------------------------------+--------------------+
+| one_eid_table_vni_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| one_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| one_enable_disable_petr_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| one_enable_disable_petr_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_enable_disable_pitr_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| one_enable_disable_pitr_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_enable_disable_xtr_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| one_enable_disable_xtr_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_get_map_request_itr_rlocs | definition changed |
++--------------------------------------------------------+--------------------+
+| one_get_map_request_itr_rlocs_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_get_transport_protocol | definition changed |
++--------------------------------------------------------+--------------------+
+| one_get_transport_protocol_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_l2_arp_bd_get | definition changed |
++--------------------------------------------------------+--------------------+
+| one_l2_arp_bd_get_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_l2_arp_entries_get | definition changed |
++--------------------------------------------------------+--------------------+
+| one_l2_arp_entries_get_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_locator_details | definition changed |
++--------------------------------------------------------+--------------------+
+| one_locator_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| one_locator_set_details | definition changed |
++--------------------------------------------------------+--------------------+
+| one_locator_set_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_register_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_register_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_register_fallback_threshold | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_register_fallback_threshold_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_register_set_ttl | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_register_set_ttl_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_request_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_request_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_resolver_details | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_resolver_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_server_details | definition changed |
++--------------------------------------------------------+--------------------+
+| one_map_server_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| one_ndp_bd_get | definition changed |
++--------------------------------------------------------+--------------------+
+| one_ndp_bd_get_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_ndp_entries_get | definition changed |
++--------------------------------------------------------+--------------------+
+| one_ndp_entries_get_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_nsh_set_locator_set | definition changed |
++--------------------------------------------------------+--------------------+
+| one_nsh_set_locator_set_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_pitr_set_locator_set | definition changed |
++--------------------------------------------------------+--------------------+
+| one_pitr_set_locator_set_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_rloc_probe_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| one_rloc_probe_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_set_transport_protocol | definition changed |
++--------------------------------------------------------+--------------------+
+| one_set_transport_protocol_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_show_petr_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| one_show_petr_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_show_pitr_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| one_show_pitr_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_show_xtr_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| one_show_xtr_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_stats_details | definition changed |
++--------------------------------------------------------+--------------------+
+| one_stats_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| one_stats_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| one_stats_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_stats_flush | definition changed |
++--------------------------------------------------------+--------------------+
+| one_stats_flush_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| one_use_petr | definition changed |
++--------------------------------------------------------+--------------------+
+| one_use_petr_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| output_acl_set_interface | definition changed |
++--------------------------------------------------------+--------------------+
+| p2p_ethernet_add | definition changed |
++--------------------------------------------------------+--------------------+
+| p2p_ethernet_add_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| p2p_ethernet_del | definition changed |
++--------------------------------------------------------+--------------------+
+| p2p_ethernet_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| pg_capture | definition changed |
++--------------------------------------------------------+--------------------+
+| pg_capture_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| pg_create_interface | definition changed |
++--------------------------------------------------------+--------------------+
+| pg_create_interface_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| pg_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| pg_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| policer_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| policer_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| policer_classify_details | definition changed |
++--------------------------------------------------------+--------------------+
+| policer_classify_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| policer_classify_set_interface | definition changed |
++--------------------------------------------------------+--------------------+
+| policer_classify_set_interface_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| policer_details | definition changed |
++--------------------------------------------------------+--------------------+
+| policer_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| pot_profile_activate | definition changed |
++--------------------------------------------------------+--------------------+
+| pot_profile_activate_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| pot_profile_add | definition changed |
++--------------------------------------------------------+--------------------+
+| pot_profile_add_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| pot_profile_del | definition changed |
++--------------------------------------------------------+--------------------+
+| pot_profile_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| pot_profile_show_config_details | definition changed |
++--------------------------------------------------------+--------------------+
+| pot_profile_show_config_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| pppoe_add_del_session | definition changed |
++--------------------------------------------------------+--------------------+
+| pppoe_add_del_session_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| pppoe_session_details | definition changed |
++--------------------------------------------------------+--------------------+
+| pppoe_session_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| proxy_arp_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| proxy_arp_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| proxy_arp_intfc_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| proxy_arp_intfc_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| punt | definition changed |
++--------------------------------------------------------+--------------------+
+| punt_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| punt_socket_deregister | definition changed |
++--------------------------------------------------------+--------------------+
+| punt_socket_deregister_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| punt_socket_register | definition changed |
++--------------------------------------------------------+--------------------+
+| punt_socket_register_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| qos_egress_map_delete | definition changed |
++--------------------------------------------------------+--------------------+
+| qos_egress_map_update | definition changed |
++--------------------------------------------------------+--------------------+
+| qos_mark_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| qos_record_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| reset_fib | definition changed |
++--------------------------------------------------------+--------------------+
+| reset_fib_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| reset_session | definition changed |
++--------------------------------------------------------+--------------------+
+| reset_session_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| rpc_call | definition changed |
++--------------------------------------------------------+--------------------+
+| rpc_call_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| rx_thread_exit | definition changed |
++--------------------------------------------------------+--------------------+
+| sctp_add_src_dst_connection | definition changed |
++--------------------------------------------------------+--------------------+
+| sctp_config | definition changed |
++--------------------------------------------------------+--------------------+
+| sctp_del_src_dst_connection | definition changed |
++--------------------------------------------------------+--------------------+
+| session_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| session_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| session_rule_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| session_rule_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| session_rules_details | definition changed |
++--------------------------------------------------------+--------------------+
+| session_rules_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| set_arp_neighbor_limit | definition changed |
++--------------------------------------------------------+--------------------+
+| set_arp_neighbor_limit_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| set_ip_flow_hash | definition changed |
++--------------------------------------------------------+--------------------+
+| set_ip_flow_hash_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| set_ipfix_classify_stream | definition changed |
++--------------------------------------------------------+--------------------+
+| set_ipfix_classify_stream_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| set_ipfix_exporter | definition changed |
++--------------------------------------------------------+--------------------+
+| set_ipfix_exporter_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_map_register_state | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_map_register_state_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_map_request_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_map_request_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_pitr | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_pitr_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_rloc_probe_state | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_rloc_probe_state_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_status | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_status_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_use_petr | definition changed |
++--------------------------------------------------------+--------------------+
+| show_lisp_use_petr_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_map_register_fallback_threshold | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_map_register_fallback_threshold_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_map_register_state | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_map_register_state_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_map_register_ttl | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_map_register_ttl_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_map_request_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_map_request_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_nsh_mapping | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_nsh_mapping_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_pitr | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_pitr_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_rloc_probe_state | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_rloc_probe_state_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_stats_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_stats_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_status | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_status_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_use_petr | definition changed |
++--------------------------------------------------------+--------------------+
+| show_one_use_petr_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| show_version | definition changed |
++--------------------------------------------------------+--------------------+
+| show_version_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sock_init_shm | definition changed |
++--------------------------------------------------------+--------------------+
+| sockclnt_create | definition changed |
++--------------------------------------------------------+--------------------+
+| sockclnt_create_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sockclnt_delete | definition changed |
++--------------------------------------------------------+--------------------+
+| sockclnt_delete_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_localsid_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_localsid_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_localsids_details | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_localsids_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_mpls_policy_add | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_mpls_policy_add_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_mpls_policy_assign_endpoint_color | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_mpls_policy_assign_endpoint_color_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_mpls_policy_del | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_mpls_policy_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_mpls_policy_mod | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_mpls_policy_mod_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_mpls_steering_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_mpls_steering_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_policy_add | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_policy_add_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_policy_del | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_policy_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_policy_mod | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_policy_mod_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_set_encap_source | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_set_encap_source_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_steering_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| sr_steering_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| stats_get_poller_delay | definition changed |
++--------------------------------------------------------+--------------------+
+| stn_add_del_rule | definition changed |
++--------------------------------------------------------+--------------------+
+| stn_add_del_rule_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| stn_rule_details | only in file |
++--------------------------------------------------------+--------------------+
+| stn_rules_details | only in image |
++--------------------------------------------------------+--------------------+
+| stn_rules_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_if_l2tpv3_tunnel_details | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_if_l2tpv3_tunnel_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_add_del_address | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_add_del_address_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_bond_details | only in image |
++--------------------------------------------------------+--------------------+
+| sw_interface_bond_dump | only in image |
++--------------------------------------------------------+--------------------+
+| sw_interface_clear_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_clear_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_details | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_event | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_get_mac_address | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_get_table | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_get_table_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_ip6_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_ip6_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_ip6_set_link_local_address | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_ip6_set_link_local_address_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_ip6nd_ra_config | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_ip6nd_ra_config_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_ip6nd_ra_prefix | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_ip6nd_ra_prefix_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_lacp_details | only in image |
++--------------------------------------------------------+--------------------+
+| sw_interface_lacp_dump | only in image |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_dpdk_hqos_pipe | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_dpdk_hqos_pipe_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_dpdk_hqos_subport | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_dpdk_hqos_subport_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_dpdk_hqos_tctbl | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_dpdk_hqos_tctbl_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_flags | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_flags_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_geneve_bypass | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_geneve_bypass_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_gtpu_bypass | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_gtpu_bypass_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_l2_bridge | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_l2_bridge_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_l2_xconnect | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_l2_xconnect_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_lldp | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_lldp_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_mac_address | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_mac_address_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_mpls_enable | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_mpls_enable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_mtu | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_mtu_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_rx_mode | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_rx_mode_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_table | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_table_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_unnumbered | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_unnumbered_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_vpath | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_vpath_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_vxlan_bypass | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_vxlan_bypass_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_vxlan_gpe_bypass | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_set_vxlan_gpe_bypass_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_slave_details | only in image |
++--------------------------------------------------------+--------------------+
+| sw_interface_slave_dump | only in image |
++--------------------------------------------------------+--------------------+
+| sw_interface_span_details | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_span_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_span_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_span_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_tag_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_tag_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_tap_details | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_tap_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_tap_v2_details | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_tap_v2_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_vhost_user_details | definition changed |
++--------------------------------------------------------+--------------------+
+| sw_interface_vhost_user_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| tap_connect | definition changed |
++--------------------------------------------------------+--------------------+
+| tap_connect_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| tap_create_v2 | definition changed |
++--------------------------------------------------------+--------------------+
+| tap_create_v2_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| tap_delete | definition changed |
++--------------------------------------------------------+--------------------+
+| tap_delete_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| tap_delete_v2 | definition changed |
++--------------------------------------------------------+--------------------+
+| tap_delete_v2_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| tap_modify | definition changed |
++--------------------------------------------------------+--------------------+
+| tap_modify_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| tcp_configure_src_addresses | definition changed |
++--------------------------------------------------------+--------------------+
+| tcp_configure_src_addresses_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| trace_plugin_msg_ids | definition changed |
++--------------------------------------------------------+--------------------+
+| trace_profile_add | definition changed |
++--------------------------------------------------------+--------------------+
+| trace_profile_add_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| trace_profile_del | definition changed |
++--------------------------------------------------------+--------------------+
+| trace_profile_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| trace_profile_show_config | definition changed |
++--------------------------------------------------------+--------------------+
+| trace_profile_show_config_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| udp_encap_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| udp_encap_add_del_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| udp_encap_details | definition changed |
++--------------------------------------------------------+--------------------+
+| udp_encap_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| udp_ping_add_del | definition changed |
++--------------------------------------------------------+--------------------+
+| udp_ping_add_del_reply | only in image |
++--------------------------------------------------------+--------------------+
+| udp_ping_add_del_req | only in file |
++--------------------------------------------------------+--------------------+
+| udp_ping_export | definition changed |
++--------------------------------------------------------+--------------------+
+| udp_ping_export_reply | only in image |
++--------------------------------------------------------+--------------------+
+| udp_ping_export_req | only in file |
++--------------------------------------------------------+--------------------+
+| unbind_sock | definition changed |
++--------------------------------------------------------+--------------------+
+| unbind_sock_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| unbind_uri | definition changed |
++--------------------------------------------------------+--------------------+
+| unbind_uri_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| unmap_segment | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_get_summary_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_get_summary_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_interface_combined_counters | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_interface_simple_counters | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_ip4_fib_counters | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_ip4_mfib_counters | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_ip4_nbr_counters | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_ip6_fib_counters | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_ip6_mfib_counters | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_ip6_nbr_counters | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_per_interface_combined_counters | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_per_interface_simple_counters | definition changed |
++--------------------------------------------------------+--------------------+
+| vnet_udp_encap_counters | only in image |
++--------------------------------------------------------+--------------------+
+| vxlan_add_del_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_add_del_tunnel_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_add_del_tunnel | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_add_del_tunnel_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_enable | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_enable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_export_enable_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_export_enable_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_transit_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_transit_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_transit_enable | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_transit_enable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_vni_disable | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_vni_disable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_vni_enable | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_ioam_vni_enable_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_tunnel_details | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_gpe_tunnel_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_tunnel_details | definition changed |
++--------------------------------------------------------+--------------------+
+| vxlan_tunnel_dump | definition changed |
++--------------------------------------------------------+--------------------+
+| want_bfd_events | definition changed |
++--------------------------------------------------------+--------------------+
+| want_bfd_events_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_igmp_events | definition changed |
++--------------------------------------------------------+--------------------+
+| want_interface_combined_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_interface_combined_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_interface_events | definition changed |
++--------------------------------------------------------+--------------------+
+| want_interface_events_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_interface_simple_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_interface_simple_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip4_arp_events | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip4_arp_events_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip4_fib_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip4_fib_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip4_mfib_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip4_mfib_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip4_nbr_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip4_nbr_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip6_fib_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip6_fib_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip6_mfib_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip6_mfib_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip6_nbr_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip6_nbr_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip6_nd_events | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip6_nd_events_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_ip6_ra_events | definition changed |
++--------------------------------------------------------+--------------------+
+| want_l2_macs_events | definition changed |
++--------------------------------------------------------+--------------------+
+| want_l2_macs_events_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_oam_events | definition changed |
++--------------------------------------------------------+--------------------+
+| want_oam_events_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_per_interface_combined_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_per_interface_combined_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_per_interface_simple_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_per_interface_simple_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_stats | definition changed |
++--------------------------------------------------------+--------------------+
+| want_stats_reply | definition changed |
++--------------------------------------------------------+--------------------+
+| want_udp_encap_stats | definition changed |
++--------------------------------------------------------+--------------------+
+
+Found 1036 api message signature differences
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``src/vpp/stats/stats.api``
+
+* `43b1f44 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=43b1f44>`_ UDP Encap counters
+* `ff92efe <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ff92efe>`_ stats: allow configuring poller delay
+* `51e5968 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=51e5968>`_ API: Add service definitions for events and singleton messages (second attempt)
+* `2de1f15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2de1f15>`_ Revert "API: Add service definitions for events and singleton messages."
+* `f7b7fa5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f7b7fa5>`_ API: Add service definitions for events and singleton messages.
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vpp/oam/oam.api``
+
+* `51e5968 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=51e5968>`_ API: Add service definitions for events and singleton messages (second attempt)
+* `2de1f15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2de1f15>`_ Revert "API: Add service definitions for events and singleton messages."
+* `f7b7fa5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f7b7fa5>`_ API: Add service definitions for events and singleton messages.
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vpp/api/vpe.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/interface.api``
+
+* `0cae3f7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0cae3f7>`_ Detailed Interface stats API takes sw_if_index
+* `6f4a6be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6f4a6be>`_ Interface Unicast, Multicast and Broadcast stats on the API
+* `c037423 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c037423>`_ IPv6 ND Router discovery control plane (VPP-1095)
+* `51e5968 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=51e5968>`_ API: Add service definitions for events and singleton messages (second attempt)
+* `2de1f15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2de1f15>`_ Revert "API: Add service definitions for events and singleton messages."
+* `f7b7fa5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f7b7fa5>`_ API: Add service definitions for events and singleton messages.
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/unix/tap.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/qos/qos.api``
+
+* `039cbfe <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=039cbfe>`_ QoS recording and marking
+
+``src/vnet/policer/policer.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/mpls/mpls.api``
+
+* `31ed744 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=31ed744>`_ MPLS Unifom mode
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/span/span.api``
+
+* `179ab36 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=179ab36>`_ SPAN: Add "is_l2" flag to DETAILS response messages.
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/vxlan-gpe/vxlan_gpe.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/l2tp/l2tp.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/lldp/lldp.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/srmpls/sr_mpls.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/cop/cop.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/feature/feature.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/ipsec-gre/ipsec_gre.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/geneve/geneve.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/lisp-gpe/lisp_gpe.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/map/map.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+* `e31d956 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e31d956>`_ MAP: Add RFC6052 mapping to MAP-T
+
+``src/vnet/lisp-cp/lisp.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/lisp-cp/one.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/tcp/tcp.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/dhcp/dhcp.api``
+
+* `51e5968 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=51e5968>`_ API: Add service definitions for events and singleton messages (second attempt)
+* `2de1f15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2de1f15>`_ Revert "API: Add service definitions for events and singleton messages."
+* `f7b7fa5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f7b7fa5>`_ API: Add service definitions for events and singleton messages.
+* `54c6dc4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=54c6dc4>`_ For DHCP client configuration control the setting of the broadcast flag in the DISCOVER message sent.
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/gre/gre.api``
+
+* `a43ccae <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a43ccae>`_ Optimize GRE Tunnel and add support for ERSPAN encap
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/flow/flow.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/devices/virtio/vhost_user.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/devices/af_packet/af_packet.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/devices/tap/tapv2.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+* `7866c45 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7866c45>`_ tapv2: add option to set host-side default gw
+
+``src/vnet/devices/netmap/netmap.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/dns/dns.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/bonding/bond.api``
+
+* `9cd2d7a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9cd2d7a>`_ bond: Add bonding driver and LACP protocol
+
+``src/vnet/session/session.api``
+
+* `8f89dd0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8f89dd0>`_ tls: enforce certificate verification
+* `371ca50 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=371ca50>`_ session: first approximation implementation of tls
+* `f8f516a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f8f516a>`_ session: support local sessions and deprecate redirects
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/ethernet/p2p_ethernet.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/ip/rd_cp.api``
+
+* `c037423 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c037423>`_ IPv6 ND Router discovery control plane (VPP-1095)
+
+``src/vnet/ip/punt.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/ip/ip.api``
+
+* `4c53313 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4c53313>`_ reassembly: feature/concurrency
+* `4b9669d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4b9669d>`_ IPv6 ND Router discovery data plane (VPP-1095)
+* `31ed744 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=31ed744>`_ MPLS Unifom mode
+* `51e5968 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=51e5968>`_ API: Add service definitions for events and singleton messages (second attempt)
+* `2de1f15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2de1f15>`_ Revert "API: Add service definitions for events and singleton messages."
+* `f7b7fa5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f7b7fa5>`_ API: Add service definitions for events and singleton messages.
+* `75e7d13 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75e7d13>`_ IPv4/6 reassembly
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+* `f068c3e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f068c3e>`_ DVR: run L3 output features
+
+``src/vnet/classify/classify.api``
+
+* `815d7d5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=815d7d5>`_ classifier-based ACL: refactor + add output ACL
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/ipip/ipip.api``
+
+* `298c695 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=298c695>`_ IPIP: Add IP{v4,v6} over IP{v4,v6} configured tunnel support.
+
+``src/vnet/udp/udp.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/bfd/bfd.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/srv6/sr.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/ipsec/ipsec.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/bier/bier.api``
+
+* `31ed744 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=31ed744>`_ MPLS Unifom mode
+* `f051072 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f051072>`_ BIER: fix support for longer bit-string lengths
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/sctp/sctp.api``
+
+* `c7fe4f3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c7fe4f3>`_ SCTP: API to configure some tunables
+* `465c087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=465c087>`_ SCTP: API to delete a sub-connection
+* `3c6a976 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3c6a976>`_ SCTP: API to add a sub-connection
+
+``src/vnet/l2/l2.api``
+
+* `e23c99e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e23c99e>`_ Improve l2_macs_events API to provide MAC move information
+* `51e5968 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=51e5968>`_ API: Add service definitions for events and singleton messages (second attempt)
+* `2de1f15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2de1f15>`_ Revert "API: Add service definitions for events and singleton messages."
+* `f7b7fa5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f7b7fa5>`_ API: Add service definitions for events and singleton messages.
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/vxlan/vxlan.api``
+
+* `31ed744 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=31ed744>`_ MPLS Unifom mode
+* `3d460bd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3d460bd>`_ VXLAN: Allow user to specify a custom vxlan tunnel instance id.
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/fib/fib_types.api``
+
+* `2303cb1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2303cb1>`_ FIB Interpose Source
+* `8145842 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8145842>`_ Common form of fib-path reproting in dumps
+* `31ed744 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=31ed744>`_ MPLS Unifom mode
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/vnet/pg/pg.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/examples/sample-plugin/sample/sample.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/lb/lb.api``
+
+* `647f609 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=647f609>`_ Add L3DSR feature in LB plugin
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/gtpu/gtpu.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/kubeproxy/kp.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/pppoe/pppoe.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/nat/nat.api``
+
+* `f2a23cc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f2a23cc>`_ NAT66 1:1 mapping (VPP-1108)
+* `9dba781 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9dba781>`_ NAT44: nat44_static_mapping_details protocol=0 if addr_only=0 (VPP-1158)
+* `bc39e34 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bc39e34>`_ NAT: add missing CLI and API documentation (VPP-1142)
+* `5f22499 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5f22499>`_ NAT44: add opaque string tag to static mapping APIs (VPP-1147)
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+* `e82488f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e82488f>`_ NAT44: asymmetrical static mapping rule (VPP-1135)
+* `240b5ef <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=240b5ef>`_ NAT44: asymmetrical load balancing static mapping rule (VPP-1132)
+* `c5c6a33 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c5c6a33>`_ Add basic support for DS-Lite CE (VPP-1059)
+
+``src/plugins/l2e/l2e.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/ioam/lib-pot/pot.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/ioam/ip6/ioam_cache.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/ioam/udp-ping/udp_ping.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+* `149a143 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=149a143>`_ fix udp_ping api naming error
+
+``src/plugins/ioam/export/ioam_export.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/ioam/lib-vxlan-gpe/ioam_vxlan_gpe.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/ioam/lib-trace/trace.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/igmp/igmp.api``
+
+* `7b867a8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7b867a8>`_ IGMP plugin
+
+``src/plugins/memif/memif.api``
+
+* `30349b0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=30349b0>`_ memif: Add new API calls to manage memif socket names.
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/lacp/lacp.api``
+
+* `9cd2d7a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9cd2d7a>`_ bond: Add bonding driver and LACP protocol
+
+``src/plugins/acl/acl.api``
+
+* `27fe75a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=27fe75a>`_ acl-plugin: add the support for dumping the ethertype whitelist (VPP-1163)
+* `c43b3f9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c43b3f9>`_ acl-plugin: add whitelisted ethertype mode (VPP-1163)
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/flowprobe/flowprobe.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/dpdk/api/dpdk.api``
+
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/gbp/gbp.api``
+
+* `bc27d1b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bc27d1b>`_ GBP plugin
+
+``src/plugins/stn/stn.api``
+
+* `62bab65 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=62bab65>`_ STN: Fix stn_rules_dump/details to follow API convention
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+
+``src/plugins/cdp/cdp.api``
+
+* `aaacfbc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=aaacfbc>`_ Move the vnet cdp protocol implementation to a plugin
+
+``src/vlibmemory/memclnt.api``
+
+* `51e5968 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=51e5968>`_ API: Add service definitions for events and singleton messages (second attempt)
+* `2de1f15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2de1f15>`_ Revert "API: Add service definitions for events and singleton messages."
+* `f7b7fa5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f7b7fa5>`_ API: Add service definitions for events and singleton messages.
+* `9d42087 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9d42087>`_ VPPAPIGEN: vppapigen replacement in Python PLY.
+* `90a6398 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=90a6398>`_ sock api: add infra for bootstrapping shm clients
+
+
diff --git a/docs/aboutvpp/releasenotes/v18.07.rst b/docs/aboutvpp/releasenotes/v18.07.rst
new file mode 100644
index 0000000..472f38b
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v18.07.rst
@@ -0,0 +1,306 @@
+Release notes for VPP 18.07
+===========================
+
+More than 533 commits since the 18.04 release.
+
+Features
+--------
+
+Infrastructure
+~~~~~~~~~~~~~~
+
+- DPDK 18.02.1
+
+ - Complete rework of the dpdk-input node
+ - Display rx/tx burst function name in “show hardware detail”
+ - Improve buffer alloc perfomance
+
+ - This is ~50% improvement in buffer alloc performance. For a 256
+ buffer allocation, it was ~10 clocks/buffer, now is < 5 clocks.
+
+ - Add per-numa page allocation info to ‘show memory’
+ - Vectorized bihash_{48,40,24,16}_8 key compare
+
+ - bihash_48_8 case:
+
+ - Scalar code: 6 clocks
+ - SSE4.2 code: 3 clocks
+ - AVX2 code: 2.27 clocks
+ - AVX512 code: 1.5 clocks
+
+ - Pollable Stats
+
+ - Stats are now available to a client in a shared memory segment
+ and in the form of a directory, allowing very high performance
+ polling of stats without directly querying VPP.
+
+VNET & Plugins
+~~~~~~~~~~~~~~
+
+- IGMP improvements
+
+ - Enable/Disable an interface for IGMP
+ - improve logging
+ - refactor common code
+ - no orphaned timers
+ - IGMP state changes in main thread only
+ - Large groups split over multiple state-change reports
+ - SSM range configuration API.
+ - more tests
+
+- IP: vectorized IP checksum
+- VXLAN : HW offload RX flow
+- Rework kube-proxy into LB plugin and add NATA66
+- NAT:
+
+ - Code refactor
+ - Syslog
+ - Multiple outside interfaces
+ - Endpoint dependent filtering and mapping
+
+- ACL:
+
+ - Tuple Merge algorithm cleanup and integration
+ - Processing pipeline optimizations
+ - Refactoring
+
+- Experimental AVF driver
+
+Host stack
+~~~~~~~~~~
+
+- Session: performance improvements, add support for connectionless
+ transports, datagram reception and transmission
+- TCP: congestion control improvements and overall fixes
+- UDP: datagram mode
+- TLS async support
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1807>`__
+
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+
+- *Only in image*: indicates the API is new for this release.
+
+- *Only in file*: indicates the API has been removed in this release.
+
+ ::
+
+ Message Name Result
+
+ abf_itf_attach_add_del definition changed abf_itf_attach_details only
+ in image abf_itf_attach_dump only in image abf_plugin_get_version
+ definition changed abf_policy_add_del definition changed
+ abf_policy_details only in image abf_policy_dump only in image
+ af_packet_details only in image af_packet_dump only in image
+ avf_create definition changed avf_delete definition changed
+ bind_sock_reply definition changed bind_uri_reply definition changed
+ dhcp6_client_enable_disable definition changed
+ dhcp6_clients_enable_disable definition changed dhcp6_duid_ll_set
+ definition changed dhcp6_pd_client_enable_disable definition changed
+ dhcp6_pd_reply_event only in image dhcp6_pd_send_client_message
+ definition changed dhcp6_reply_event only in image
+ dhcp6_send_client_message definition changed dhcp_client_config
+ definition changed dhcp_client_details only in image dhcp_client_dump
+ only in image dhcp_compl_event definition changed
+ dslite_address_details only in image dslite_address_dump only in
+ image gbp_endpoint_group_add_del definition changed
+ gbp_endpoint_group_details only in image gbp_endpoint_group_dump only
+ in image gbp_recirc_add_del definition changed gbp_recirc_details
+ only in image gbp_recirc_dump only in image gbp_subnet_add_del
+ definition changed gbp_subnet_details only in image gbp_subnet_dump
+ only in image hw_interface_set_mtu definition changed igmp_details
+ definition changed igmp_dump definition changed igmp_enable_disable
+ definition changed igmp_event definition changed
+ igmp_group_prefix_details only in image igmp_group_prefix_dump only
+ in image igmp_group_prefix_set definition changed igmp_listen
+ definition changed ikev2_profile_set_auth definition changed
+ ikev2_profile_set_id definition changed
+ ip6_add_del_address_using_prefix definition changed ip_mroute_add_del
+ definition changed ip_probe_neighbor definition changed
+ ip_scan_neighbor_enable_disable definition changed
+ ip_unnumbered_details only in image ip_unnumbered_dump only in image
+ ipip_6rd_add_tunnel definition changed ipip_add_tunnel definition
+ changed ipip_tunnel_details definition changed ipsec_sa_details
+ definition changed ipsec_sad_add_del_entry definition changed
+ ipsec_tunnel_if_add_del definition changed kp_add_del_pod definition
+ changed kp_add_del_vip definition changed kp_conf definition changed
+ lb_add_del_vip definition changed mactime_add_del definition changed
+ mactime_enable definition changed memclnt_create definition changed
+ memclnt_create_reply definition changed memfd_segment_create
+ definition changed nat44_add_del_lb_static_mapping definition changed
+ nat44_add_del_static_mapping definition changed nat44_del_session
+ definition changed nat44_lb_static_mapping_details definition changed
+ nat44_static_mapping_details definition changed
+ nat44_user_session_details definition changed pipe_create definition
+ changed pipe_delete definition changed pipe_details only in image
+ pipe_dump only in image pot_profile_activate definition changed
+ pot_profile_add definition changed pot_profile_del definition changed
+ proxy_arp_add_del definition changed proxy_arp_details only in image
+ proxy_arp_dump only in image proxy_arp_intfc_details only in image
+ proxy_arp_intfc_dump only in image sock_init_shm_reply definition
+ changed sockclnt_create definition changed sockclnt_create_reply
+ definition changed sr_localsid_add_del definition changed
+ sr_localsids_details definition changed sr_policies_details only in
+ image sr_policies_dump only in image sr_policy_add definition changed
+ sr_policy_del definition changed sr_policy_mod definition changed
+ sr_steering_pol_details only in image sr_steering_pol_dump only in
+ image sw_interface_details definition changed sw_interface_set_mtu
+ definition changed tap_create_v2 definition changed
+ vnet_bier_neighbor_counters only in image
+ vnet_get_summary_stats_reply definition changed vxlan_offload_rx
+ definition changed want_bier_neighbor_stats definition changed
+ want_dhcp6_pd_reply_events definition changed want_dhcp6_reply_events
+ definition changed Found 107 api message signature differences
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``src/plugins/ioam/lib-pot/pot.api``
+
+* `e9fcf23 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e9fcf23>`_ Fix some build warnings about "Old Style VLA"
+
+``src/plugins/gbp/gbp.api``
+
+* `25b0494 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=25b0494>`_ GBP V2
+
+``src/plugins/map/map.api``
+
+* `381e9a9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=381e9a9>`_ MAP: Move MAP-E/T to a plugin.
+
+``src/plugins/igmp/igmp.api``
+
+* `947ea62 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=947ea62>`_ IGMP improvements
+
+``src/plugins/lb/lb.api``
+
+* `d92a0b5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d92a0b5>`_ Rework kube-proxy into LB plugin
+
+``src/plugins/nat/nat.api``
+
+* `70a26ac <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=70a26ac>`_ NAT44: nat44_del_session and nat44_user_session_details API update (VPP-1271)
+* `ebdf190 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ebdf190>`_ NAT44: TCP connection close detection (VPP-1266)
+* `1e5c07d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1e5c07d>`_ Add special Twice-NAT feature (VPP-1221)
+* `16aa7f8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=16aa7f8>`_ DSLite: Implement new API call DSLITE_ADDRESS_DUMP.
+
+``src/plugins/avf/avf.api``
+
+* `258a189 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=258a189>`_ avf: api fix
+* `6c9b964 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6c9b964>`_ avf: binary API and configurable RX/TX queue size
+
+``src/plugins/mactime/mactime.api``
+
+* `7055e26 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7055e26>`_ Driver level time-based src mac filter
+
+``src/plugins/abf/abf.api``
+
+* `669d07d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=669d07d>`_ ACL based forwarding
+
+``src/vlibmemory/memclnt.api``
+
+* `dab732a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dab732a>`_ VPP-1335 vapi crash when memclnt_keepalive received
+* `7895872 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7895872>`_ Remove the historical memfd api segment bootstrap
+
+``src/vpp/stats/stats.api``
+
+* `a21a367 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a21a367>`_ VPP-1324 SIGSEGV vl_msg_api_handler_with_vm_node()
+* `586479a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=586479a>`_ BIER neighbor stats
+* `e906aac <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e906aac>`_ STATS: Separate socket for fd exchange.
+* `048a4e5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=048a4e5>`_ export counters in a memfd segment
+
+``src/vnet/interface.api``
+
+* `d723161 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d723161>`_ MTU: Software interface / Per-protocol MTU support
+* `fe7d4a2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fe7d4a2>`_ Revert "MTU: Setting of MTU on software interface (instead of hardware interface)"
+* `70083ee <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=70083ee>`_ MTU: Setting of MTU on software interface (instead of hardware interface)
+
+``src/vnet/ipfix-export/ipfix_export.api``
+
+* `a9855ef <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a9855ef>`_ Flow: Rename IPFIX exporter.
+
+``src/vnet/dhcp/dhcp6_pd_client_cp.api``
+
+* `81119e8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=81119e8>`_ Implement DHCPv6 PD client (VPP-718, VPP-1050)
+
+``src/vnet/dhcp/dhcp.api``
+
+* `dd3b8f7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dd3b8f7>`_ Implement DHCPv6 IA NA client (VPP-1094)
+* `d9778c2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d9778c2>`_ Update DHCPv6 DUID code and fix coverity warnings
+* `81119e8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=81119e8>`_ Implement DHCPv6 PD client (VPP-718, VPP-1050)
+* `daff178 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=daff178>`_ DHCP Client Dump
+
+``src/vnet/dhcp/dhcp6_ia_na_client_cp.api``
+
+* `dd3b8f7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dd3b8f7>`_ Implement DHCPv6 IA NA client (VPP-1094)
+
+``src/vnet/ip/ip.api``
+
+* `947ea62 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=947ea62>`_ IGMP improvements
+* `7eaaf74 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7eaaf74>`_ proxy_arp: remove unused is_add
+* `0053de6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0053de6>`_ ARP proxy dumps
+* `9e2f915 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9e2f915>`_ IP unnumbered dump
+* `7f358b3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7f358b3>`_ Periodic scan and probe of IP neighbors to maintain neighbor pools
+* `e821ab1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e821ab1>`_ IP mcast: allow unicast address as a next-hop
+* `c7b4304 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c7b4304>`_ Implement ip_probe_neighbor API
+
+``src/vnet/ip/ip_types.api``
+
+* `947ea62 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=947ea62>`_ IGMP improvements
+* `2c2feab <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2c2feab>`_ VPPAPIGEN: Add union and enum support and IP4/IP6 address type.
+
+``src/vnet/devices/af_packet/af_packet.api``
+
+* `04e0bb2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=04e0bb2>`_ af_packet: Add support for dump interfaces
+
+``src/vnet/devices/tap/tapv2.api``
+
+* `d600ffe <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d600ffe>`_ Update tapv2 documentation
+* `0b06111 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0b06111>`_ VPP-1305: Add support for tags
+
+``src/vnet/devices/pipe/pipe.api``
+
+* `ee8b973 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ee8b973>`_ VOM: support for pipes
+* `17ff3c1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=17ff3c1>`_ Pipes
+
+``src/vnet/ipip/ipip.api``
+
+* `d57f636 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d57f636>`_ VPP-1277: IPIP - Copy TOS/TC from inner packet to outer.
+
+``src/vnet/session/session.api``
+
+* `7fb0fe1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7fb0fe1>`_ udp/session: refactor to support dgram mode
+
+``src/vnet/ipsec/ipsec.api``
+
+* `4b089f2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4b089f2>`_ ipsec: support UDP encap/decap for NAT traversal
+* `e9fcf23 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e9fcf23>`_ Fix some build warnings about "Old Style VLA"
+* `8e1039a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8e1039a>`_ Allow an IPsec tunnel interface to be renumbered
+
+``src/vnet/vxlan/vxlan.api``
+
+* `af86a48 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=af86a48>`_ vxlan:offload RX flow
+
+``src/vnet/srv6/sr.api``
+
+* `3337bd2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3337bd2>`_ Fixed bugs in SRv6 API
+* `e9fcf23 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e9fcf23>`_ Fix some build warnings about "Old Style VLA"
+
+
diff --git a/docs/aboutvpp/releasenotes/v18.10.rst b/docs/aboutvpp/releasenotes/v18.10.rst
new file mode 100644
index 0000000..952d715
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v18.10.rst
@@ -0,0 +1,316 @@
+Release notes for VPP 18.10
+===========================
+
+More than 632 commits since the 18.07 release.
+
+Features
+--------
+
+Infrastructure
+~~~~~~~~~~~~~~
+
+- DPDK 18.08 integration
+- New Stats infrastructure (interface, error, node performance
+ counters)
+- Add configurable “Doug Lea malloc” support
+
+VNET & Plugins
+~~~~~~~~~~~~~~
+
+- Load balancing: support per-port VIP and all-port VIP
+- Port NSH plugin to VPP
+- NAT
+
+ - Configurable port range
+ - Virtual Fragmentation Reassembly for endpoint-dependent mode
+ - Client-IP based session affinity for load-balancing
+ - TCP MSS clamping
+ - Session timeout
+ - Bug-fixing and performance optimizations
+
+Host stack
+~~~~~~~~~~
+
+- Support for applications with multiple workers
+- Support for binds from multiple app workers to same ip:port
+- Switched to a message queue for io and control event notifications
+- Support for eventfd based notifications as alternative to
+ mutext-condvar pair
+- VCL refactor to support async event notifications and multiple
+ workers
+- TLS async support in client for HW accleration
+- Performance optimizations and bug-fixing
+- A number of binary APIs will be deprecated in favor of using the
+ event message queue. Details in the API section.
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1810>`__
+
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+
+- *Only in image*: indicates the API is new for this release.
+
+- *Only in file*: indicates the API has been removed in this release.
+
+ ::
+
+ Message Name Result
+
+ api_versions_reply definition changed
+ app_cut_through_registration_add definition changed
+ app_worker_add_del definition changed application_attach_reply
+ definition changed bd_ip_mac_details only in image bd_ip_mac_dump
+ only in image bfd_udp_get_echo_source definition changed
+ bier_imp_details definition changed bier_route_details definition
+ changed bind_sock definition changed bridge_domain_details definition
+ changed bridge_flags definition changed classify_add_del_session
+ definition changed classify_add_del_table definition changed
+ connect_sock definition changed create_vhost_user_if definition
+ changed get_first_msg_id_reply definition changed
+ gpe_add_del_fwd_entry_reply definition changed
+ gpe_fwd_entry_path_details definition changed ip6_fib_details
+ definition changed ip6nd_proxy_details definition changed
+ ip_add_del_route_reply definition changed ip_address_details
+ definition changed ip_details definition changed ip_fib_details
+ definition changed ip_mfib_details definition changed
+ ip_mroute_add_del_reply definition changed ip_neighbor_add_del_reply
+ definition changed ip_neighbor_details definition changed
+ ip_reassembly_get_reply definition changed ip_unnumbered_details
+ definition changed ipip_6rd_add_tunnel definition changed
+ ipip_add_tunnel definition changed ipsec_spds_details only in image
+ ipsec_spds_dump only in image l2_interface_efp_filter definition
+ changed lisp_eid_table_vni_details definition changed
+ map_another_segment definition changed mfib_signal_details definition
+ changed mpls_route_add_del_reply definition changed
+ mpls_tunnel_add_del definition changed mpls_tunnel_add_del_reply
+ definition changed mpls_tunnel_details definition changed
+ mpls_tunnel_dump definition changed one_eid_table_vni_details
+ definition changed qos_mark_enable_disable definition changed
+ qos_record_enable_disable definition changed reset_session_reply
+ definition changed rpc_call definition changed show_threads
+ definition changed sockclnt_create_reply definition changed
+ sockclnt_delete definition changed sockclnt_delete_reply definition
+ changed sw_interface_rx_placement_details only in image
+ sw_interface_rx_placement_dump only in image
+ sw_interface_set_ip_directed_broadcast definition changed
+ sw_interface_set_l2_bridge definition changed
+ sw_interface_set_rx_placement definition changed
+ sw_interface_set_vxlan_gbp_bypass definition changed udp_encap_add
+ definition changed udp_encap_add_del_reply only in file
+ udp_encap_add_reply only in image udp_encap_del definition changed
+ udp_encap_details definition changed unbind_sock definition changed
+ vxlan_gbp_tunnel_add_del definition changed vxlan_gbp_tunnel_details
+ only in image vxlan_gbp_tunnel_dump only in image Found 68 api
+ message signature differences
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``src/plugins/avf/avf.api``
+
+* `149d0e28 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=149d0e28>`_ avf: RSS support
+* `4e6014fc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4e6014fc>`_ avf: api fix
+
+``src/plugins/gbp/gbp.api``
+
+* `c0a93143 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0a93143>`_ GBP Endpoint Updates
+* `61b94c6b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=61b94c6b>`_ vxlan-gbp: Add support for vxlan gbp
+
+``src/plugins/igmp/igmp.api``
+
+* `bdc0e6b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdc0e6b7>`_ Trivial: Clean up some typos.
+
+``src/plugins/lb/lb.api``
+
+* `6a4375e0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6a4375e0>`_ LB: fix flush flow table issue
+* `49ca2601 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=49ca2601>`_ Add flush flag on del as command
+* `219cc90c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=219cc90c>`_ Support lb on both vip and per-port-vip case
+
+``src/plugins/nat/nat.api``
+
+* `bb4e0225 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bb4e0225>`_ NAT: TCP MSS clamping
+* `5d28c7af <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5d28c7af>`_ NAT: add support for configurable port range (VPP-1346)
+* `ea5b5be4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ea5b5be4>`_ NAT44: client-IP based session affinity for load-balancing (VPP-1297)
+* `878c646a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=878c646a>`_ NAT44: add support for session timeout (VPP-1272)
+* `69ce30d6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=69ce30d6>`_ NAT: update nat_show_config_reply API (VPP-1403)
+* `6bd197eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bd197eb>`_ Remove client_index field from replies in API
+* `c6c0d2a0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c6c0d2a0>`_ NAT44: LB NAT - local backends in multiple VRFs (VPP-1345)
+
+``src/plugins/vmxnet3/vmxnet3.api``
+
+* `df7f8e8c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df7f8e8c>`_ vmxnet3 device driver
+
+``src/plugins/nsh/nsh.api``
+
+* `d313f9e6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d313f9e6>`_ Port NSH plugin to VPP
+
+``src/plugins/nsim/nsim.api``
+
+* `9e3252b5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9e3252b5>`_ Network delay simulator plugin
+
+``src/plugins/svs/svs.api``
+
+* `d1e68ab7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d1e68ab7>`_ Source VRF Select
+
+``src/vlibmemory/memclnt.api``
+
+* `94495f2a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=94495f2a>`_ PAPI: Use UNIX domain sockets instead of shared memory
+* `6bd197eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bd197eb>`_ Remove client_index field from replies in API
+* `75282457 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75282457>`_ Fix "Old Style VLA" build warnings
+
+``src/vnet/interface.api``
+
+* `f0b42f48 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f0b42f48>`_ itf: dump interface rx-placement
+* `bdc0e6b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdc0e6b7>`_ Trivial: Clean up some typos.
+* `54f7c51f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=54f7c51f>`_ rx-placement: Add API call for interface rx-placement
+* `1855b8e4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1855b8e4>`_ IP directed broadcast
+
+``src/vnet/bfd/bfd.api``
+
+* `2d3c7b9c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2d3c7b9c>`_ BFD: add get echo source API (VPP-1367)
+
+``src/vnet/bier/bier.api``
+
+* `ef90ed08 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ef90ed08>`_ BIER API and load-balancing fixes
+* `6bd197eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bd197eb>`_ Remove client_index field from replies in API
+
+``src/vnet/classify/classify.api``
+
+* `34eb5d42 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=34eb5d42>`_ classify_add_del_session API: Use more descriptive docstring (VPP-1385)
+* `75282457 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75282457>`_ Fix "Old Style VLA" build warnings
+
+``src/vnet/devices/pipe/pipe.api``
+
+* `208c29aa <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=208c29aa>`_ VOM: support for pipes
+
+``src/vnet/devices/virtio/vhost_user.api``
+
+* `ee2e58f6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ee2e58f6>`_ vhost-user: Add disable feature support in api
+
+``src/vnet/ethernet/ethernet_types.api``
+
+* `de5b08fb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=de5b08fb>`_ Introduce a mac_address_t on the API and in VPP
+
+``src/vnet/ip/ip_types.api``
+
+* `d0df49f2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d0df49f2>`_ Use IP address types on UDP encap API
+
+``src/vnet/ip/ip.api``
+
+* `412ecd32 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=412ecd32>`_ Improve ip_mroute_add_del documentation
+* `14260393 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=14260393>`_ Add adjacency counters to the stats segment
+* `28c142e3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=28c142e3>`_ mroute routers in the stats segment
+* `008dbe10 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=008dbe10>`_ Route counters in the stats segment
+* `de5b08fb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=de5b08fb>`_ Introduce a mac_address_t on the API and in VPP
+* `6bd197eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bd197eb>`_ Remove client_index field from replies in API
+* `b11f903a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b11f903a>`_ Fix context field position in API definition
+
+``src/vnet/ipip/ipip.api``
+
+* `61502115 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=61502115>`_ IPIP and SIXRD tunnels create API needs table-IDs not fib-indexes
+
+``src/vnet/ipsec/ipsec.api``
+
+* `a9a0b2ce <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a9a0b2ce>`_ IPsec: add API for SPDs dump (VPP-1363)
+* `bdc0e6b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdc0e6b7>`_ Trivial: Clean up some typos.
+
+``src/vnet/l2/l2.api``
+
+* `0a4e0063 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0a4e0063>`_ Fix documentation about sw_interface_set_l2_bridge
+* `b474380f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b474380f>`_ L2 BD: introduce a BD interface on which to send UU packets
+* `bdc0e6b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdc0e6b7>`_ Trivial: Clean up some typos.
+* `5c7c49d1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5c7c49d1>`_ Fix documentation for SHG in bridge domain
+* `5d82d2f1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5d82d2f1>`_ l2: arp termination dump
+* `6b9b41c8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6b9b41c8>`_ L2 EFP: byteswap sw_if_index, enable flag can be u8 on .api
+
+``src/vnet/lisp-cp/lisp.api``
+
+* `bdc0e6b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdc0e6b7>`_ Trivial: Clean up some typos.
+* `6bd197eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bd197eb>`_ Remove client_index field from replies in API
+
+``src/vnet/lisp-cp/one.api``
+
+* `bdc0e6b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdc0e6b7>`_ Trivial: Clean up some typos.
+* `6bd197eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bd197eb>`_ Remove client_index field from replies in API
+
+``src/vnet/lisp-gpe/lisp_gpe.api``
+
+* `6bd197eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bd197eb>`_ Remove client_index field from replies in API
+* `b11f903a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b11f903a>`_ Fix context field position in API definition
+
+``src/vnet/mpls/mpls.api``
+
+* `f5fa5ae2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f5fa5ae2>`_ MPLS tunnel dump: use sw_if_index not tunnel_index
+* `6a30b5f9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6a30b5f9>`_ MPLS tunnel dump fix
+* `008dbe10 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=008dbe10>`_ Route counters in the stats segment
+* `7c922dc4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7c922dc4>`_ SR-MPLS: fixes and tests
+
+``src/vnet/qos/qos.api``
+
+* `bdc0e6b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdc0e6b7>`_ Trivial: Clean up some typos.
+* `ed234e7f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ed234e7f>`_ Enum type on the API for QoS sources
+
+``src/vnet/session/session.api``
+
+* `ab2f6dbf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ab2f6dbf>`_ session: support multiple worker binds
+* `134a996a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=134a996a>`_ vcl: add support for multi-worker apps
+* `1553197f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1553197f>`_ session: add support for multiple app workers
+* `6bd197eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bd197eb>`_ Remove client_index field from replies in API
+* `99368315 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=99368315>`_ vcl: support for eventfd mq signaling
+
+``src/vnet/span/span.api``
+
+* `bdc0e6b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdc0e6b7>`_ Trivial: Clean up some typos.
+
+``src/vnet/udp/udp.api``
+
+* `9c0a3c42 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9c0a3c42>`_ UDP-Encap: name counters for the stats segment
+* `d0df49f2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d0df49f2>`_ Use IP address types on UDP encap API
+
+``src/vnet/unix/tap.api``
+
+* `bdc0e6b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdc0e6b7>`_ Trivial: Clean up some typos.
+
+``src/vnet/vxlan-gbp/vxlan_gbp.api``
+
+* `79a05f54 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=79a05f54>`_ VXLAN-GBP: use common types on the API
+* `61b94c6b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=61b94c6b>`_ vxlan-gbp: Add support for vxlan gbp
+
+``src/vpp/api/vpe.api``
+
+* `5d64c786 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5d64c786>`_ thread: Add show threads api
+* `ec11b13a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ec11b13a>`_ Trivial: Cleanup some typos.
+
+``src/vpp/stats/stats.api``
+
+* `ec11b13a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ec11b13a>`_ Trivial: Cleanup some typos.
+
+Notice of future API deprecation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- bind_uri_reply
+- accept_session
+- accept_session_reply
+- disconnect_session_reply
+- reset_session
+- reset_session_reply
+- bind_sock_reply
+- connect_session_reply
diff --git a/docs/aboutvpp/releasenotes/v19.01.1.rst b/docs/aboutvpp/releasenotes/v19.01.1.rst
new file mode 100644
index 0000000..a3a53ab
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.01.1.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 19.01.1
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1901>`__
diff --git a/docs/aboutvpp/releasenotes/v19.01.2.rst b/docs/aboutvpp/releasenotes/v19.01.2.rst
new file mode 100644
index 0000000..5aef09c
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.01.2.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 19.01.2
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1901>`__
diff --git a/docs/aboutvpp/releasenotes/v19.01.3.rst b/docs/aboutvpp/releasenotes/v19.01.3.rst
new file mode 100644
index 0000000..f3d5a1c
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.01.3.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 19.01.3
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1901>`__
diff --git a/docs/aboutvpp/releasenotes/v19.01.rst b/docs/aboutvpp/releasenotes/v19.01.rst
new file mode 100644
index 0000000..b2311c8
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.01.rst
@@ -0,0 +1,413 @@
+Release notes for VPP 19.01
+===========================
+
+More than 649 commits since the 18.10 release.
+
+Features
+--------
+
+Infrastructure
+~~~~~~~~~~~~~~
+
+- NUMA-aware, growable physical memory allocator (pmalloc)
+- FIB: sticky load-balance
+- C11 safe string handling: provide and use “safe” C string handling
+ functions
+- vlib: allocate buffers on local numa, not on numa 1
+- vppinfra: autodetect default hugepage size
+- Move RPC traffic off the shared-memory API queue
+- IPv6: Make link-local configurable per-interface
+- IGMP: improve CLI debug output
+- IPSec: split ipsec nodes into ip4/ip6 nodes
+- IPSec: infra for selecting backends
+- vhost-user: cleanup and performance optimizations
+- ethernet-input, memif improvements and optimizations
+- DPDK: bump to DPDK 18.11
+- reassembly: harden reassembly code
+- stats: Deprecate old (event-based) stats framework
+- vlib: support Hyper-V/Azure VMBus
+- binary api clients: wait for vpp to start
+- graph dispatch trace: capture packet data and buffer metadata, output
+ in pcap format
+- improve feature arc order constraint specification
+
+VNET & Plugins
+~~~~~~~~~~~~~~
+
+- pktgen: correctly replay a mix of single and multi-buffer packets
+- add wireshark dissector to extras
+- avf: optimizations
+- acl-plugin: use L2 feature arc instead of L2 classifier
+- acl-plugin: performance enhancement
+- dpdk: allow interface name to be specified from startup.conf
+- dpdk: blacklist PCI devices by type
+- dpdk: switch to in-memory mode, deprecate use of socket-mem
+- vnet: store hw interface speed in kbps instead of using flags
+- vmxnet3: enable promiscuous mode & cli enhancements
+- gbp: Add support for flow hash profile & l3-out subnets
+- map: Add API support for setting parameters.
+- map: Convert from DPO to input feature
+- nat: improve expired sessions reuse in NAT44
+- nat: syslog - sessions logging
+- nsim: add packet loss simulation, docs
+- perfmon: x86_64 perf counter plugin
+- vnet: L2 feature arc infrastructure
+
+Host stack
+~~~~~~~~~~
+
+- TCP congestion control improvements
+- TCP Cubic congestion control algorithm
+- TCP fast path optimizations
+- Transport tx connection pacer. TCP uses it by default
+- Basic support for session flushing and TCP PSH segments
+- TCP/session api support for configuring custom local src ip/port
+- VCL/LDP basic support for multi-process applications
+- Overall code hardening, cleanup and bugfixing for tcp, session, vcl
+ and ldp
+
+PAPI & Test framework
+~~~~~~~~~~~~~~~~~~~~~
+
+- add specific API types for IP addresses, MAC address, interface index
+ etc.
+- add timeout support for socket transport
+- add support for format/unformat functions
+- generic API types format/unformat support for VAT and custom dump
+- python3 test adjustments
+- make test: create virtualenv under /test/
+- make test: print TEST= values for failed tests
+- add human-friendly annotations to log messages
+
+VOM
+~~~
+
+- Add support for redirect contracts in gbp
+- deprecate TAP add ip-punt redirect dump
+- vxlan-gbp support
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1901>`__
+
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+- *Only in image*: indicates the API is new for this release.
+- *Only in file*: indicates the API has been removed in this release.
+
+=============================================== ==================
+Message Name Results
+=============================================== ==================
+acl_plugin_get_conn_table_max_entries only in image
+acl_plugin_get_conn_table_max_entries_reply only in image
+app_worker_add_del definition changed
+app_worker_add_del_reply definition changed
+application_attach_reply definition changed
+bd_ip_mac_add_del definition changed
+bd_ip_mac_details definition changed
+bd_ip_mac_flush only in image
+bd_ip_mac_flush_reply only in image
+bond_create definition changed
+cli_inband definition changed
+cli_inband_reply definition changed
+gbp_bridge_domain_add only in image
+gbp_bridge_domain_add_reply only in image
+gbp_bridge_domain_del only in image
+gbp_bridge_domain_del_reply only in image
+gbp_bridge_domain_details only in image
+gbp_bridge_domain_dump only in image
+gbp_bridge_domain_dump_reply only in image
+gbp_endpoint_details definition changed
+gbp_endpoint_group_add only in image
+gbp_endpoint_group_add_del only in file
+gbp_endpoint_group_add_del_reply only in file
+gbp_endpoint_group_add_reply only in image
+gbp_endpoint_group_del only in image
+gbp_endpoint_group_del_reply only in image
+gbp_endpoint_learn_set_inactive_threshold only in image
+gbp_endpoint_learn_set_inactive_threshold_reply only in image
+gbp_ext_itf_add_del only in image
+gbp_ext_itf_add_del_reply only in image
+gbp_ext_itf_details only in image
+gbp_ext_itf_dump only in image
+gbp_route_domain_add only in image
+gbp_route_domain_add_reply only in image
+gbp_route_domain_del only in image
+gbp_route_domain_del_reply only in image
+gbp_route_domain_details only in image
+gbp_route_domain_dump only in image
+gbp_route_domain_dump_reply only in image
+gbp_vxlan_tunnel_add only in image
+gbp_vxlan_tunnel_add_reply only in image
+gbp_vxlan_tunnel_del only in image
+gbp_vxlan_tunnel_del_reply only in image
+gbp_vxlan_tunnel_details only in image
+gbp_vxlan_tunnel_dump only in image
+igmp_proxy_device_add_del only in image
+igmp_proxy_device_add_del_interface only in image
+igmp_proxy_device_add_del_interface_reply only in image
+igmp_proxy_device_add_del_reply only in image
+ip6_mfib_details definition changed
+ip_container_proxy_details only in image
+ip_container_proxy_dump only in image
+ip_mfib_details definition changed
+ip_punt_redirect definition changed
+ip_punt_redirect_details only in image
+ip_punt_redirect_dump only in image
+ip_source_check_interface_add_del only in image
+ip_source_check_interface_add_del_reply only in image
+ipip_6rd_add_tunnel_reply definition changed
+ipip_6rd_del_tunnel definition changed
+ipip_add_tunnel_reply definition changed
+ipip_del_tunnel definition changed
+ipip_tunnel_details definition changed
+ipip_tunnel_dump definition changed
+ipsec_backend_details only in image
+ipsec_backend_dump only in image
+ipsec_sa_details definition changed
+ipsec_select_backend only in image
+ipsec_select_backend_reply only in image
+ipsec_tunnel_if_add_del definition changed
+map_add_del_rule definition changed
+map_add_domain definition changed
+map_another_segment definition changed
+map_domain_details definition changed
+map_if_enable_disable only in image
+map_if_enable_disable_reply only in image
+map_param_add_del_pre_resolve only in image
+map_param_add_del_pre_resolve_reply only in image
+map_param_get only in image
+map_param_get_reply only in image
+map_param_set_fragmentation only in image
+map_param_set_fragmentation_reply only in image
+map_param_set_icmp6 only in image
+map_param_set_icmp6_reply only in image
+map_param_set_icmp only in image
+map_param_set_icmp_reply only in image
+map_param_set_reassembly only in image
+map_param_set_reassembly_reply only in image
+map_param_set_security_check only in image
+map_param_set_security_check_reply only in image
+map_param_set_tcp only in image
+map_param_set_tcp_reply only in image
+map_param_set_traffic_class only in image
+map_param_set_traffic_class_reply only in image
+map_rule_details definition changed
+memclnt_delete definition changed
+nat44_add_del_lb_static_mapping definition changed
+nat44_lb_static_mapping_add_del_local only in image
+nat44_lb_static_mapping_add_del_local_reply only in image
+nat44_lb_static_mapping_details definition changed
+nsim_configure definition changed
+punt only in file
+punt_details only in image
+punt_dump only in image
+punt_reply only in file
+punt_socket_deregister definition changed
+punt_socket_details only in image
+punt_socket_dump only in image
+punt_socket_register definition changed
+set_ip_flow_hash definition changed
+set_punt only in image
+set_punt_reply only in image
+show_version_reply definition changed
+stats_get_poller_delay only in file
+stats_get_poller_delay_reply only in file
+sw_interface_bond_details definition changed
+sw_interface_details definition changed
+sw_interface_ip6_set_link_local_address only in file
+sw_interface_ip6_set_link_local_address_reply only in file
+sw_interface_tap_v2_details definition changed
+syslog_get_filter only in image
+syslog_get_filter_reply only in image
+syslog_get_sender only in image
+syslog_get_sender_reply only in image
+syslog_set_filter only in image
+syslog_set_filter_reply only in image
+syslog_set_sender only in image
+syslog_set_sender_reply only in image
+tap_create_v2 definition changed
+unmap_segment definition changed
+vnet_bier_neighbor_counters only in file
+vnet_get_summary_stats only in file
+vnet_get_summary_stats_reply only in file
+vnet_interface_combined_counters only in file
+vnet_interface_simple_counters only in file
+vnet_ip4_fib_counters only in file
+vnet_ip4_mfib_counters only in file
+vnet_ip4_nbr_counters only in file
+vnet_ip6_fib_counters only in file
+vnet_ip6_mfib_counters only in file
+vnet_ip6_nbr_counters only in file
+vnet_per_interface_combined_counters only in file
+vnet_per_interface_simple_counters only in file
+vnet_udp_encap_counters only in file
+want_bier_neighbor_stats only in file
+want_bier_neighbor_stats_reply only in file
+want_interface_combined_stats only in file
+want_interface_combined_stats_reply only in file
+want_interface_simple_stats only in file
+want_interface_simple_stats_reply only in file
+want_ip4_fib_stats only in file
+want_ip4_fib_stats_reply only in file
+want_ip4_mfib_stats only in file
+want_ip4_mfib_stats_reply only in file
+want_ip4_nbr_stats only in file
+want_ip4_nbr_stats_reply only in file
+want_ip6_fib_stats only in file
+want_ip6_fib_stats_reply only in file
+want_ip6_mfib_stats only in file
+want_ip6_mfib_stats_reply only in file
+want_ip6_nbr_stats only in file
+want_ip6_nbr_stats_reply only in file
+want_per_interface_combined_stats only in file
+want_per_interface_combined_stats_reply only in file
+want_per_interface_simple_stats only in file
+want_per_interface_simple_stats_reply only in file
+want_stats only in file
+want_stats_reply only in file
+want_udp_encap_stats only in file
+want_udp_encap_stats_reply only in file
+=============================================== ==================
+
+Found 170 api message signature differences
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``src/vnet/interface_types.api``
+
+* `53fffa1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53fffa1>`_ API: Add support for type aliases
+
+``src/vnet/interface.api``
+
+* `f49ba0e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f49ba0e>`_ stats: Deprecate old stats framework
+* `53fffa1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53fffa1>`_ API: Add support for type aliases
+* `5100aa9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5100aa9>`_ vnet: store hw interface speed in kbps instead of using flags
+
+``src/vnet/syslog/syslog.api``
+
+* `b4515b4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b4515b4>`_ Add RFC5424 syslog protocol support (VPP-1139)
+
+``src/vnet/fib/fib_types.api``
+
+* `775f73c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=775f73c>`_ FIB: encode the label stack in the FIB path during table dump
+
+``src/vnet/ip/ip.api``
+
+* `7c03ed4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7c03ed4>`_ VOM: mroutes
+* `3460b01 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3460b01>`_ api: ip_source_check_interface_add_del api is added.
+* `609e121 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=609e121>`_ VPP-1507: Added binary api to dump configured ip_punt_redirect
+* `2af0e3a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2af0e3a>`_ flow-hash: Add symmetric flag for flow hashing
+* `47527b2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=47527b2>`_ IP-punt: add documentation to the API and fix IP address init
+* `5bb1eca <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5bb1eca>`_ IPv6: Make link-local configurable per-interface (VPP-1446)
+* `75b9f45 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75b9f45>`_ ip: add container proxy dump API (VPP-1364)
+
+``src/vnet/ip/ip_types.api``
+
+* `8c8acc0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8c8acc0>`_ API: Change ip4_address and ip6_address to use type alias.
+* `ffba3c3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ffba3c3>`_ MAP: Use explicit address/prefix types in API
+
+``src/vnet/ip/punt.api``
+
+* `e88865d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e88865d>`_ VPP-1506: dump local punts and registered punt sockets
+
+``src/vnet/ipsec/ipsec.api``
+
+* `4c422f9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4c422f9>`_ Add IPSec interface FIB index for TX packet
+* `b4a7a7d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b4a7a7d>`_ Add UDP encap flag
+* `b4d3053 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b4d3053>`_ ipsec: infra for selecting backends
+* `871bca9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=871bca9>`_ VPP-1450: binary api call for dumping SPD to interface registration
+
+``src/vnet/l2/l2.api``
+
+* `e26c81f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e26c81f>`_ L2 BD API to flush all IP-MAC entries in the specified BD
+* `8006c6a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8006c6a>`_ PAPI: Add MACAddress object wrapper for vl_api_mac_address_t
+* `93cc3ee <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=93cc3ee>`_ GBP Endpoint Learning
+* `4d5b917 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4d5b917>`_ BD ARP entry use common API types
+
+``src/vnet/vxlan-gbp/vxlan_gbp.api``
+
+* `93cc3ee <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=93cc3ee>`_ GBP Endpoint Learning
+
+``src/vnet/ipip/ipip.api``
+
+* `53fffa1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53fffa1>`_ API: Add support for type aliases
+
+``src/vnet/session/session.api``
+
+* `d85de68 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d85de68>`_ vcl: wait for segments with segment handle
+* `fa76a76 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fa76a76>`_ session: segment handle in accept/connect notifications
+* `c1f5a43 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c1f5a43>`_ session: cleanup use of api_client_index
+* `c0d532d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0d532d>`_ session: mark apis for deprecation
+
+``src/vnet/ethernet/ethernet_types.api``
+
+* `8006c6a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8006c6a>`_ PAPI: Add MACAddress object wrapper for vl_api_mac_address_t
+
+``src/vnet/bonding/bond.api``
+
+* `ad9d528 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ad9d528>`_ bonding: support custom interface IDs
+
+``src/vnet/devices/tap/tapv2.api``
+
+* `754f24b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=754f24b>`_ tapv2: add "tap_flags" field to the TAPv2 interface API
+
+``src/vlibmemory/memclnt.api``
+
+* `eaec2a6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=eaec2a6>`_ bapi: add options to have vpp cleanup client registration
+
+``src/vpp/api/vpe.api``
+
+* `f49ba0e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f49ba0e>`_ stats: Deprecate old stats framework
+* `413f4a5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=413f4a5>`_ API: Use string type instead of u8.
+
+``src/plugins/acl/acl.api``
+
+* `bb5d22d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bb5d22d>`_ New api in order to get max entries of connection table is added.
+
+``src/plugins/nsim/nsim.api``
+
+* `10c5ff1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=10c5ff1>`_ nsim: add packet loss simulation, docs
+
+``src/plugins/gbp/gbp.api``
+
+* `1c17e2e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1c17e2e>`_ GBP: add allowed ethertypes to contracts
+* `b6a4795 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b6a4795>`_ GBP: l3-out subnets
+* `33b81da <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=33b81da>`_ vom: Add support for redirect contracts in gbp
+* `13a08cc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=13a08cc>`_ GBP: redirect contracts
+* `c29c0af <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c29c0af>`_ GBP: Endpoints with VLAN tags and birdges that don't learn
+* `93cc3ee <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=93cc3ee>`_ GBP Endpoint Learning
+
+``src/plugins/nat/nat.api``
+
+* `b686508 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b686508>`_ NAT44: nat44_add_del_lb_static_mapping enhancements (VPP-1514)
+
+``src/plugins/map/map.api``
+
+* `fc7344f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fc7344f>`_ MAP: Convert from DPO to input feature.
+* `f34597f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f34597f>`_ MAP: Add API support for MAP input feature.
+* `5a2e278 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5a2e278>`_ MAP: Add API support for setting parameters.
+* `a173a7a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a173a7a>`_ MAP: Use bool type in map.api instead of u8.
+* `ffba3c3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ffba3c3>`_ MAP: Use explicit address/prefix types in API
+
+``src/plugins/igmp/igmp.api``
+
+* `97748ca <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=97748ca>`_ IGMP: proxy device
+
diff --git a/docs/aboutvpp/releasenotes/v19.04.1.rst b/docs/aboutvpp/releasenotes/v19.04.1.rst
new file mode 100644
index 0000000..f7c46df
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.04.1.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 19.04.1
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1904>`__
diff --git a/docs/aboutvpp/releasenotes/v19.04.2.rst b/docs/aboutvpp/releasenotes/v19.04.2.rst
new file mode 100644
index 0000000..9d15b52
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.04.2.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 19.04.2
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1904>`__
diff --git a/docs/aboutvpp/releasenotes/v19.04.3.rst b/docs/aboutvpp/releasenotes/v19.04.3.rst
new file mode 100644
index 0000000..c10d969
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.04.3.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 19.04.3
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1904>`__
diff --git a/docs/aboutvpp/releasenotes/v19.04.rst b/docs/aboutvpp/releasenotes/v19.04.rst
new file mode 100644
index 0000000..9c5c1b1
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.04.rst
@@ -0,0 +1,387 @@
+Release notes for VPP 19.04
+===========================
+
+More than 700 commits since the 19.01 release.
+
+Features
+--------
+
+Infrastructure
+~~~~~~~~~~~~~~
+
+- DPDK 19.02 integration
+- Buffer manager rework and improvements
+- Python3 migration (work in progress)
+
+ - vppapigen
+ - Python API wrappers
+ - Docs generation
+ - vpp_config
+ - “make test” python3 readiness and refactoring
+
+- Add “make test-gcov” target to main Makefile
+- Refactor multiarch code
+- vfctl script: bind VF to vfio-pci after VF is created
+- cmake cross-compilation support
+- CLI control of graph dispatch elogs
+- AppImage packaging (disabled by default)
+- Complete upstreaming of wireshark dissector
+- Remove JVPP which is now an FD.io project
+- Punt infra: manage dispatch of exception packets
+
+VNET & Plugins
+~~~~~~~~~~~~~~
+
+- BVI Interface
+- Deprecate TAP cli
+- Experimental TAP interface TCP segmentation offload
+- Vmxnet3 driver plugin
+- LACP passive mode
+- ACL plugin refactoring
+- RDMA (ibverb) driver plugin - MLX5 with multiqueue
+- IPSEC
+
+ - Intel IPSEC-MB engine plugin
+ - Tunnel fragmentation
+ - CLI improvements
+ - Performance improvements
+ - API modernisation and improvements
+ - New Tests and test refactoring
+
+- Crypto
+
+ - Introduce crypto infra
+ - crypto_ia32 plugin
+ - Add support for AEAD and AES-GCM
+ - Implement rfc4231 test cases
+ - Implement crypto tests per RFC2202
+
+- Perfmon improvements
+
+ - Python to C parser for intel CPUs
+ - 2-way parallel stat collection
+ - Collect data on selected thread(s)
+
+Host stack
+~~~~~~~~~~
+
+- Improve ldp/vls/vcl support for multi-process and multi-threaded
+ applications
+- Major refactor/cleanup of session layer
+- Refactor cut-through sessions to use a custom transport
+- Baseline QUIC transport support
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1904>`__
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+- *Only in image*: indicates the API is new for this release.
+- *Only in file*: indicates the API has been removed in this release.
+
+=============================================== ==================
+Message Name Result
+=============================================== ==================
+accept_session only in file
+accept_session_reply only in file
+bind_sock_reply definition changed
+bind_uri_reply definition changed
+bvi_create only in image
+bvi_create_reply only in image
+bvi_delete only in image
+bvi_delete_reply only in image
+connect_session only in file
+connect_session_reply only in file
+ct6_enable only in image
+ct6_enable_disable only in image
+gbp_contract_add_del_reply definition changed
+gbp_endpoint_group_del definition changed
+gbp_endpoint_learn_set_inactive_threshold only in file
+gbp_endpoint_learn_set_inactive_threshold_reply only in file
+ikev2_plugin_get_version only in image
+ikev2_plugin_get_version_reply only in image
+ip4_arp_event definition changed
+ip6_nd_event definition changed
+ip6_ra_event definition changed
+ip6nd_proxy_add_del definition changed
+ip6nd_proxy_details definition changed
+ip_container_proxy_add_del definition changed
+ip_neighbor_add_del definition changed
+ip_neighbor_details definition changed
+ip_probe_neighbor definition changed
+ip_source_and_port_range_check_add_del definition changed
+ipsec_backend_details definition changed
+ipsec_gre_add_del_tunnel only in file
+ipsec_gre_add_del_tunnel_reply only in file
+ipsec_gre_tunnel_add_del only in image
+ipsec_gre_tunnel_add_del_reply only in image
+ipsec_gre_tunnel_details definition changed
+ipsec_sa_details definition changed
+ipsec_sa_set_key definition changed
+ipsec_sad_add_del_entry only in file
+ipsec_sad_add_del_entry_reply only in file
+ipsec_sad_entry_add_del only in image
+ipsec_sad_entry_add_del_reply only in image
+ipsec_select_backend definition changed
+ipsec_spd_add_del_entry only in file
+ipsec_spd_add_del_entry_reply only in file
+ipsec_spd_details definition changed
+ipsec_spd_entry_add_del only in image
+ipsec_spd_entry_add_del_reply only in image
+ipsec_tunnel_if_add_del definition changed
+lb_conf definition changed
+map_add_domain definition changed
+map_domain_details definition changed
+nat_ha_flush only in image
+nat_ha_flush_reply only in image
+nat_ha_get_failover only in image
+nat_ha_get_failover_reply only in image
+nat_ha_get_listener only in image
+nat_ha_get_listener_reply only in image
+nat_ha_resync only in image
+nat_ha_resync_completed_event only in image
+nat_ha_resync_reply only in image
+nat_ha_set_failover only in image
+nat_ha_set_failover_reply only in image
+nat_ha_set_listener only in image
+nat_ha_set_listener_reply only in image
+reset_session only in file
+reset_session_reply only in file
+sw_interface_ip6nd_ra_prefix definition changed
+sw_interface_set_dpdk_hqos_pipe only in file
+sw_interface_set_dpdk_hqos_pipe_reply only in file
+sw_interface_set_dpdk_hqos_subport only in file
+sw_interface_set_dpdk_hqos_subport_reply only in file
+sw_interface_set_dpdk_hqos_tctbl only in file
+sw_interface_set_dpdk_hqos_tctbl_reply only in file
+sw_interface_tap_details only in file
+sw_interface_tap_dump only in file
+sw_interface_virtio_pci_details only in image
+sw_interface_virtio_pci_dump only in image
+tap_connect only in file
+tap_connect_reply only in file
+tap_delete only in file
+tap_delete_reply only in file
+tap_modify only in file
+tap_modify_reply only in file
+virtio_pci_create only in image
+virtio_pci_create_reply only in image
+virtio_pci_delete only in image
+virtio_pci_delete_reply only in image
+vmxnet3_create definition changed
+vmxnet3_details definition changed
+want_ip4_arp_events definition changed
+want_ip6_nd_events definition changed
+=============================================== ==================
+
+Found 90 api message signature differences
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``src/vlibmemory/memclnt.api``
+
+* `eaec2a6d9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=eaec2a6d9>`_ bapi: add options to have vpp cleanup client registration
+
+``src/vpp/api/vpe.api``
+
+* `1aaf0e343 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1aaf0e343>`_ deprecate tapcli
+* `f49ba0e81 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f49ba0e81>`_ stats: Deprecate old stats framework
+* `413f4a5b2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=413f4a5b2>`_ API: Use string type instead of u8.
+
+``src/vnet/interface.api``
+
+* `3b0d7e42f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3b0d7e42f>`_ Revert "API: Cleanup APIs interface.api"
+* `e63325e3c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e63325e3c>`_ API: Cleanup APIs interface.api
+* `bb2c7b580 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bb2c7b580>`_ Update documentation for src/vnet/interface.api sw_interface_dump
+* `f49ba0e81 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f49ba0e81>`_ stats: Deprecate old stats framework
+* `53fffa1db <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53fffa1db>`_ API: Add support for type aliases
+* `5100aa9cb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5100aa9cb>`_ vnet: store hw interface speed in kbps instead of using flags
+
+``src/vnet/interface_types.api``
+
+* `3b0d7e42f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3b0d7e42f>`_ Revert "API: Cleanup APIs interface.api"
+* `e63325e3c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e63325e3c>`_ API: Cleanup APIs interface.api
+* `53fffa1db <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53fffa1db>`_ API: Add support for type aliases
+
+``src/vnet/bonding/bond.api``
+
+* `ad9d52831 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ad9d52831>`_ bonding: support custom interface IDs
+
+``src/vnet/ipip/ipip.api``
+
+* `53fffa1db <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53fffa1db>`_ API: Add support for type aliases
+
+``src/vnet/ipsec-gre/ipsec_gre.api``
+
+* `e524d45ef <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e524d45ef>`_ IPSEC-GRE: fixes and API update to common types.
+
+``src/vnet/syslog/syslog.api``
+
+* `b4515b4be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b4515b4be>`_ Add RFC5424 syslog protocol support (VPP-1139)
+
+``src/vnet/devices/tap/tapv2.api``
+
+* `754f24b35 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=754f24b35>`_ tapv2: add "tap_flags" field to the TAPv2 interface API
+
+``src/vnet/devices/virtio/virtio.api``
+
+* `d6c15af33 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d6c15af33>`_ virtio: Native virtio driver
+
+``src/vnet/fib/fib_types.api``
+
+* `775f73c6b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=775f73c6b>`_ FIB: encode the label stack in the FIB path during table dump
+
+``src/vnet/ip/ip_types.api``
+
+* `8c8acc027 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8c8acc027>`_ API: Change ip4_address and ip6_address to use type alias.
+* `ffba3c377 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ffba3c377>`_ MAP: Use explicit address/prefix types in API
+
+``src/vnet/ip/ip.api``
+
+* `48ae19e90 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=48ae19e90>`_ API: Add python2.7 support for enum flags via aenum
+* `37029305c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=37029305c>`_ Use IP and MAC API types for neighbors
+* `7c03ed47d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7c03ed47d>`_ VOM: mroutes
+* `3460b014a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3460b014a>`_ api: ip_source_check_interface_add_del api is added.
+* `609e1210c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=609e1210c>`_ VPP-1507: Added binary api to dump configured ip_punt_redirect
+* `2af0e3a74 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2af0e3a74>`_ flow-hash: Add symmetric flag for flow hashing
+* `47527b24a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=47527b24a>`_ IP-punt: add documentation to the API and fix IP address init
+* `5bb1ecae8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5bb1ecae8>`_ IPv6: Make link-local configurable per-interface (VPP-1446)
+* `75b9f45a1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75b9f45a1>`_ ip: add container proxy dump API (VPP-1364)
+
+``src/vnet/ip/punt.api``
+
+* `e88865d7b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e88865d7b>`_ VPP-1506: dump local punts and registered punt sockets
+
+``src/vnet/vxlan-gbp/vxlan_gbp.api``
+
+* `4dd4cf4f9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4dd4cf4f9>`_ GBP: fixes for l3-out routing
+* `93cc3ee3b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=93cc3ee3b>`_ GBP Endpoint Learning
+
+``src/vnet/ethernet/ethernet_types.api``
+
+* `8006c6aa4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8006c6aa4>`_ PAPI: Add MACAddress object wrapper for vl_api_mac_address_t
+
+``src/vnet/ipsec/ipsec.api``
+
+* `1e3aa5e21 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1e3aa5e21>`_ ipsec: USE_EXTENDED_SEQ_NUM -> USE_ESN
+* `1ba5bc8d8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1ba5bc8d8>`_ ipsec: add ipv6 support for ipsec tunnel interface
+* `5d704aea5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5d704aea5>`_ updates now that flags are supported on the API
+* `53f526b68 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53f526b68>`_ TEST: IPSEC NAT-T with UDP header
+* `7c44d78ef <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7c44d78ef>`_ IKEv2 to plugin
+* `eba31eceb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=eba31eceb>`_ IPSEC: move SA counters into the stats segment
+* `8d7c50200 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8d7c50200>`_ IPSEC: no second lookup after tunnel encap
+* `a09c1ff5b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a09c1ff5b>`_ IPSEC: SPD counters in the stats sgement
+* `17dcec0b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=17dcec0b9>`_ IPSEC: API modernisation
+* `4c422f9a3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4c422f9a3>`_ Add IPSec interface FIB index for TX packet
+* `b4a7a7dcf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b4a7a7dcf>`_ Add UDP encap flag
+* `b4d305344 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b4d305344>`_ ipsec: infra for selecting backends
+* `871bca9aa <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=871bca9aa>`_ VPP-1450: binary api call for dumping SPD to interface registration
+
+``src/vnet/tcp/tcp.api``
+
+* `c5df8c71c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c5df8c71c>`_ host stack: update stale copyright
+
+``src/vnet/l2/l2.api``
+
+* `192b13f96 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=192b13f96>`_ BVI Interface
+* `5daf0c55c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5daf0c55c>`_ add default NONE flag for bd_flags
+* `e26c81fc8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e26c81fc8>`_ L2 BD API to flush all IP-MAC entries in the specified BD
+* `8006c6aa4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8006c6aa4>`_ PAPI: Add MACAddress object wrapper for vl_api_mac_address_t
+* `93cc3ee3b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=93cc3ee3b>`_ GBP Endpoint Learning
+* `4d5b917b1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4d5b917b1>`_ BD ARP entry use common API types
+
+``src/vnet/session/session.api``
+
+* `6442401c2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6442401c2>`_ session: remove deprecated binary apis
+* `d85de68ec <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d85de68ec>`_ vcl: wait for segments with segment handle
+* `fa76a76bf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fa76a76bf>`_ session: segment handle in accept/connect notifications
+* `c1f5a4336 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c1f5a4336>`_ session: cleanup use of api_client_index
+* `c0d532d17 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0d532d17>`_ session: mark apis for deprecation
+
+``src/vnet/udp/udp.api``
+
+* `c5df8c71c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c5df8c71c>`_ host stack: update stale copyright
+
+``src/plugins/cdp/cdp.api``
+
+* `76ef6094c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=76ef6094c>`_ tests: cdp plugin. Replace cdp enable cli command with API call.
+
+``src/plugins/nat/nat.api``
+
+* `8feeaff56 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8feeaff56>`_ Typos. A bunch of typos I've been collecting.
+* `34931eb47 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=34931eb47>`_ NAT44: active-passive HA (VPP-1571)
+* `b686508c4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b686508c4>`_ NAT44: nat44_add_del_lb_static_mapping enhancements (VPP-1514)
+
+``src/plugins/map/map.api``
+
+* `4dc5c7b90 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4dc5c7b90>`_ MAP: Add optional user-supplied 'tag' field in MAPs.
+* `fc7344f9b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fc7344f9b>`_ MAP: Convert from DPO to input feature.
+* `f34597fc8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f34597fc8>`_ MAP: Add API support for MAP input feature.
+* `5a2e278a0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5a2e278a0>`_ MAP: Add API support for setting parameters.
+* `a173a7a07 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a173a7a07>`_ MAP: Use bool type in map.api instead of u8.
+* `ffba3c377 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ffba3c377>`_ MAP: Use explicit address/prefix types in API
+
+``src/plugins/gbp/gbp.api``
+
+* `1aa35576e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1aa35576e>`_ GBP: Counters per-contract
+* `8ea109e40 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8ea109e40>`_ gbp: Add bd flags
+* `7bd343509 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7bd343509>`_ GBP: custom-dump functions
+* `fa0ac2c56 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fa0ac2c56>`_ GBP: contracts API fixed length of allowed ethertypes
+* `5d704aea5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5d704aea5>`_ updates now that flags are supported on the API
+* `4ba67723d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4ba67723d>`_ GBP: use sclass in the DP for policy
+* `8da9fc659 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8da9fc659>`_ GBP: learn from ARP and L2 packets
+* `32f6d8e0c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=32f6d8e0c>`_ GBP: per-group EP retention policy
+* `879d11c25 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=879d11c25>`_ GBP: Sclass to src-epg conversions
+* `1c17e2eca <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1c17e2eca>`_ GBP: add allowed ethertypes to contracts
+* `b6a479539 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b6a479539>`_ GBP: l3-out subnets
+* `33b81da54 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=33b81da54>`_ vom: Add support for redirect contracts in gbp
+* `13a08cc09 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=13a08cc09>`_ GBP: redirect contracts
+* `c29c0af40 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c29c0af40>`_ GBP: Endpoints with VLAN tags and birdges that don't learn
+* `93cc3ee3b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=93cc3ee3b>`_ GBP Endpoint Learning
+
+``src/plugins/acl/acl.api``
+
+* `bb5d22daf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bb5d22daf>`_ New api in order to get max entries of connection table is added.
+
+``src/plugins/vmxnet3/vmxnet3.api``
+
+* `ee8ba6877 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ee8ba6877>`_ vmxnet3: auto bind support
+* `854559d15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=854559d15>`_ vmxnet3: RSS support
+* `773291163 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=773291163>`_ vmxnet3: multiple TX queues support
+
+``src/plugins/nsim/nsim.api``
+
+* `10c5ff143 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=10c5ff143>`_ nsim: add packet loss simulation, docs
+
+``src/plugins/igmp/igmp.api``
+
+* `97748cae2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=97748cae2>`_ IGMP: proxy device
+
+``src/plugins/lb/lb.api``
+
+* `f7f13347b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f7f13347b>`_ tests: update test_lb.py to use api call lb_conf.
+
+``src/plugins/ct6/ct6.api``
+
+* `a55df1081 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a55df1081>`_ ipv6 connection tracking plugin
+
+``src/plugins/ikev2/ikev2.api``
+
+* `7c44d78ef <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7c44d78ef>`_ IKEv2 to plugin
diff --git a/docs/aboutvpp/releasenotes/v19.08.1.rst b/docs/aboutvpp/releasenotes/v19.08.1.rst
new file mode 100644
index 0000000..bd6eec8
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.08.1.rst
@@ -0,0 +1,75 @@
+Release notes for VPP 19.08.1
+=============================
+
+Exceptionally, this release has an API-changing fix introduced via
+https://gerrit.fd.io/r/#/c/vpp/+/21762/ - documented in VPP-1767. Given
+the exceptional nature of the change, also including the text here:
+
+Bug: https://gerrit.fd.io/r/c/vpp/+/21492
+
+Variable length strings were committed to VPP in 413f4a5b. The VPP
+server side of the API does not use a wire encoder/decoder. It maps a C
+struct directly onto on-the-wire API messages. The client side C
+language binding is the same, while other language bindings have their
+own encoder/decoders.
+
+Multiple strings alone or combined with other variable length types
+turned out to be error prone to manually implement on the VPP side, and
+not supported by VPP API (VAPI) very well at all.
+
+To avoid having to rewrite VAPI significantly, and to mitigate the risk
+and error prone server side support of multiple variable length fields,
+this patch extends strings to have a fixed size (on the wire) and a
+variable flavour, as well as adding detection in the API compiler to
+detect multiple variable length fields in a message (or type).
+
+Given that this change breaks the commitment to binary API
+compatibility, normally present in point builds, ALL 19.08 build
+artifacts are being deferred.
+
+This means the artifacts for the VPP 19.08.1 will be installed in the
+release repository (packagecloud.io/fdio/release), then ALL 19.08 build
+artifacts will be moved into the deferred repository
+(packagecloud.io/fdio/deferred). The 19.08 artifacts will always be
+available for archive purposes in the deferred repository.
+
+During the further testing by Networking-VPP team, they discovered
+another issue documented in VPP-1769 - which requires a CRC-affecting
+fix in https://gerrit.fd.io/r/#/c/vpp/+/22015/ - so the 19.08.1 will
+contain the fixes for both issues.
+
+These two changes have resulted in the following 20 messages changing
+their signatures:
+
+=============================== ==================
+Message Name Result
+=============================== ==================
+cli_inband definition changed
+cli_inband_reply definition changed
+connect_sock definition changed
+http_static_enable definition changed
+log_details definition changed
+map_add_domain definition changed
+map_domain_details definition changed
+nat44_add_del_identity_mapping definition changed
+nat44_add_del_lb_static_mapping definition changed
+nat44_add_del_static_mapping definition changed
+nat44_identity_mapping_details definition changed
+nat44_lb_static_mapping_details definition changed
+nat44_static_mapping_details definition changed
+nat_worker_details definition changed
+punt_reason_details definition changed
+punt_reason_dump definition changed
+show_version_reply definition changed
+sw_interface_details definition changed
+sw_interface_dump definition changed
+sw_interface_tag_add_del definition changed
+=============================== ==================
+
+Please accept our apologies for the inconvenience this caused.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1908>`__
+
diff --git a/docs/aboutvpp/releasenotes/v19.08.2.rst b/docs/aboutvpp/releasenotes/v19.08.2.rst
new file mode 100644
index 0000000..75eb3ff
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.08.2.rst
@@ -0,0 +1,187 @@
+Release notes for VPP 19.08.2
+=============================
+
+The 19.08.2 is an LTS release. It contains numerous fixes, as well as
+new features and API additions.
+
+Features
+--------
+
+- API trace tool
+
+ - Add text output (c395ff143)
+
+- Binary API Libraries
+
+ - Add API support for PP2 plugin to stable/1908 (1c3c9f039)
+
+- Build System
+
+ - Pass ‘no-pci’ to autgenerated config (c0552134e)
+ - Add env variable to pass extra cmake args (116e05f57)
+
+- Infrastructure Library
+
+ - Implement CLIB_PAUSE () for aarch64 platforms (a3c45242b)
+ - Create unformat function for data size parsing (cb19100c1)
+
+- Link Bonding
+
+ - Fix interface deletion (1517d5e72)
+ - Add GSO support (a06f68556)
+
+- Physical Memory Allocator
+
+ - Always lock pages (5b2eea6e0)
+
+- Plugins
+
+ - AVF Device driver
+
+ - Print queue id in packet trace (9e028d047)
+
+ - DPDK
+
+ - Ipsec tunnel support for ip6-in-ip4 (2dde5a478)
+ - QAT devices update, add c4xxx and xeon d15xx (f5d6c80ac)
+ - Add TSO support in DPDK plugin. (5564db853)
+
+ - Group Based Policy (GBP)
+
+ - Add extended SFC unit tests (30f7e4198)
+
+ - Host Stack Applications
+
+ - Make APP_OPTIONS_PREALLOC_FIFO_PAIRS configurable (47c6f36be)
+
+ - Internet Key Exchange (IKEv2) Protocol
+
+ - Add support for GCM cipher (2fa9f679c)
+
+ - QUIC protocol
+
+ - Add cli command for stats (88af6c3f4)
+ - Add Tx, Rx and packet drop counters (3a61a40dd)
+ - Create custom event logger (2f9ec5001)
+ - Make quic fifo size configurable via cli (7fc3d97b8)
+
+ - RDMA (ibverb) driver
+
+ - Add support for input feature arcs (cbae1e1c5)
+ - Add support for MAC changes (ffdfe308b)
+
+ - Http_static
+
+ - Add dynamic GET / POST method hooks (faf5195e3)
+
+- Python binding for the VPP API
+
+ - Let async calls return context (e6b29a9df)
+ - Introduce read_blocking (1c45b85df)
+
+- SVM Library
+
+ - Improve fifo segment verbose cli (d2bff0786)
+
+- Statistics Segment
+
+ - Add /if/<n>/<n>/state for lacp interface state (d5e8ed7be)
+
+- Test Infrastructure
+
+ - Support worker threads (51699e62c)
+ - Support setting random seed (fc000f0e1)
+ - Add cli_return_response to vpp_papi_provider (64d744350)
+ - Test tls case (87e1bcdd7)
+
+- VNET
+
+ - Classifier
+
+ - Use vector code even when data is not aligned (bebbd7f62)
+ - VPP packet tracer support (7c5a3536c)
+
+ - IPSec
+
+ - Add ‘detail’ option to ‘sh ipsec sa’ (56417fa94)
+ - Add insecure option for format of SA (591aa64e8)
+ - Support 4o6 and 6o4 for tunnel protect (2e6d73934)
+
+ - IPv4 and IPv6 LPM
+
+ - Allow addrs from the same prefix on intf (da900b25c)
+ - Punt rather than drop unkown IPv6 ICMP packets (fd2f6f89e)
+
+ - Session Layer
+
+ - Add explicit reset api (a267cba29)
+ - Improve cli (2ff21af39)
+ - Add session enable option in config file (b1ef5567b)
+ - Limit pacer bucket size (079895d95)
+ - Builtin app rx notifications regardless of state (8e4afc86d)
+ - Infra for transports to send buffers (57997c874)
+ - Reschedule asap when snd space constrained (89ab1762d)
+
+ - TCP
+
+ - Allow cc algos to set pacing rate (82df1eb90)
+ - Set cc_algo on connection alloc (7fe501a4b)
+ - Add option for always on event logging (e73bd8503)
+ - Track zero rwnd errors (a2c063712)
+ - Validate connections in output (ea584d137)
+ - Force zero window on full rx fifo (fbe948c81)
+ - Send rwnd update only if wnd is large enough (0ad8477ba)
+ - Enable gso in tcp hoststack (6f3621d77)
+ - Handle SACK reneging (9dba3dbf0)
+ - Use rate sample RTT in recovery if possible (6702641f5)
+ - Compute snd time for rate sample (69460ae11)
+ - Use sacks for timer based recovery (d4aa3d9f8)
+ - Custom checksum calculations for Ipv4/Ipv6 (3642782a2)
+ - Retry lost retransmits (7b135c639)
+ - Improve pacing after idle send periods (abdc7dfb5)
+ - Track lost rxt segments in byte tracker (6de46b40d)
+ - Validate the IP address while checking TCP connection
+ (6c1ce53b4)
+ - Improve lost rxt heuristic (04b4204d9)
+
+- VPP Comms Library
+
+ - Allow non-blocking connects (4767cf24f)
+ - Add api to set lcl ip (2c55610e2)
+
+- Vector Library
+
+ - Add flag to explicitelly mark nodes which can init per-node packet
+ trace (29dc11bde)
+ - Enhance the “show cli” debug CLI command (b5a0108ac)
+
+- Libmemif
+
+ - Introduce ‘memif_per_thread\_’ namespace (2736fc7fc)
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+- *Only in image*: indicates the API is new for this release.
+- *Only in file*: indicates the API has been removed in this release.
+
+================ =============
+Message Name Result
+================ =============
+app_attach only in image
+app_attach_reply only in image
+================ =============
+
+Found 2 api message signature differences
+
+Fixed issues
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1908>`__
diff --git a/docs/aboutvpp/releasenotes/v19.08.3.rst b/docs/aboutvpp/releasenotes/v19.08.3.rst
new file mode 100644
index 0000000..fe7ccc3
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.08.3.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 19.08.3
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1908>`__
diff --git a/docs/aboutvpp/releasenotes/v19.08.rst b/docs/aboutvpp/releasenotes/v19.08.rst
new file mode 100644
index 0000000..efbba1c
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v19.08.rst
@@ -0,0 +1,752 @@
+Release notes for VPP 19.08
+===========================
+
+More than 850 commits since the 19.04 release.
+
+Features
+--------
+
+Infrastructure
+~~~~~~~~~~~~~~
+
+- API
+
+ - API language: new types and limits support
+ - Python API - add support for defaults
+ - Export ip_types.api for out-of-tree plugins use
+ - Refactor ipip.api with explicit types
+
+- DPDK
+
+ - 19.05 integration
+ - Remove bonding code
+ - Rework extended stats
+
+- Debugging & Servicability
+
+ - debug CLI leak-checker
+ - vlib: add “memory-trace stats-segment”
+ - vppapitrace JSON/API trace converter
+ - ARP: add arp-disabled node
+ - igmp: Trace more data from input packets
+ - ip: Trace the packet from the punt node
+ - Python API debug introspection improvements
+ - Pin dependencies for make test infra
+ - FEATURE.yaml meta-data infrastructure
+ - tcp: add cc stats plotting tools
+ - Packet tracer support for thread handoffs
+
+- libmemif: support for multi-thread connection establishment
+- svm
+
+ - fifo ooo reads/writes with multiple chunks
+ - support addition/removal of chunks to fifos
+
+- vppinfra
+
+ - Mapped pcap file support
+ - More AVX2 and AVX512 inlines
+ - VLIB_INIT_FUNCTION sequencing rework
+ - refactor spinlocks and rwlocks
+ - add rbtree
+ - add doubly linked list
+
+- rdma: bump rdma-core to v25.0
+- stats
+
+ - Add the number of worker threads and per worker thread vector
+ rates
+ - Support multiple workers for error counters
+
+VNET & Plugins
+~~~~~~~~~~~~~~
+
+- New Plugins
+
+ - HTTP static page server with TLS support
+ - L3 cross connect
+
+- acl: implement stat-segment counters
+- arp: add feature arcs: arp-reply, arp-input, arp-proxy
+- avf: improved logging and added 2.5/5 Gbps speeds
+- bonding: NUMA-related improvements
+- crypto: add support for AES-CTR cipher
+- fib
+
+ - FIB Entry tracking
+ - Support the POP of a Pseudo Wire Control Word
+
+- gbp
+
+ - Anonymous l3-out subnets support
+ - ARP unicast forward in gbp bridge domain
+ - An Endpoint can change sclass
+ - Consider data-plane learnt source better than control-plane
+ - VRF scoped contracts
+
+- gso (experimental)
+
+ - Add support to pg interfaces
+ - Add support to vhost user
+ - Add support to native virtio
+ - Add support for tagged interfaces
+
+- punt: allow to specify packets by IP protocol Type
+- ip6-local: hop-by-hop protocol demux table
+- ipsec
+
+ - intel-ipsec-mb version 0.52
+ - AH encrypt rework
+ - handle UDP keepalives
+ - support GCM in ESP
+
+- virtio
+
+ - Refactor control queue support
+
+- dhcp-client: DSCP marking for transmitted packets
+- Idle resource usage improvements
+
+ - Allocate bihash virtual space on demand
+ - gre: don’t register gre input nodes unless a gre tunnel is created
+ - gtpu: don’t register udp ports unless a tunnel is created
+ - lacp: create lacp-process on demand
+ - lisp-cp: start lisp retry service on demand
+ - start the cdp period and dns resolver process on demand
+ - vat: unload unused vat plugins
+
+- nat: api cleanup & update
+- nsim: make available as an output feature
+- load-balance performance improvements
+- l2: Add support for arp unicast forwarding
+- mactime
+
+ - Mini-ACLs
+ - Per-MAC allow-with-quota feature
+
+- qos
+
+ - QoS dump APIs
+ - Store function
+
+- rdma: add support for promiscuous mode (l2-switching and xconnect)
+- sr: update the Segment Routing definition to be compliant with
+ current in IETF
+- udp-ping: disable due to conflict with mldv2
+- vxlan-gpe: improve encap performance
+- vom
+
+ - QoS support
+ - Bridge domain arp unicast forwarding flag
+ - Bridge domain unknown unicast flooding flag
+
+Host stack
+~~~~~~~~~~
+
+- session
+
+ - API to support manual svm fifo resizing
+ - Improved session output scheduler and close state machine
+ - Transport and session cleanup notifications for builtin apps
+ - Session migration notifications for builtin apps
+ - Support for no session layer lookup transports (quic and tls)
+ - Ability to retrieve local/remote endpoint in transport vft
+ - Cleanup segment manager and fifo segment
+ - Fix vpp to app msg generation on enqueue fail
+ - Improve event logging
+ - Moved test applications to hsa plugin
+
+- tcp
+
+ - Congestion control algorithm enhancements
+ - Delivery rate estimator
+ - ACK/retransmission refactor and pacing
+ - Add tcp-input sibling nodes without full 6-tuple lookup
+ - More RFC4898 connection statistics
+ - Allow custom output next node
+ - Allow custom congestion control algorithms
+
+- quic
+
+ - Multi-thread support
+ - Logs readability improvements
+ - Multistream support
+
+- tls
+
+ - Fix close with data and listen failures
+ - Handle TCP transport rests
+ - Support endpoint retrieval interface
+
+- vcl
+
+ - support quic streams and “connectable listeners”
+ - worker unregister api
+ - fix epoll with large events batch
+ - ldp: add option to eanble transparent TLS connections
+
+- udp:
+
+ - support close with data
+ - fixed session migration
+
+- sctp
+
+ - add option to enable/disable default to disable
+ - moved from vnet to plugins
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/1908>`__
+
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+- *Only in image*: indicates the API is new for this release.
+- *Only in file*: indicates the API has been removed in this release.
+
+============================================= ==================
+Message Name Result
+============================================= ==================
+abf_itf_attach_add_del definition changed
+abf_itf_attach_details definition changed
+abf_policy_add_del definition changed
+abf_policy_details definition changed
+acl_add_replace definition changed
+acl_details definition changed
+acl_stats_intf_counters_enable only in image
+acl_stats_intf_counters_enable_reply only in image
+api_versions_reply definition changed
+bd_ip_mac_add_del definition changed
+bd_ip_mac_details definition changed
+bier_disp_entry_add_del definition changed
+bier_disp_entry_details definition changed
+bier_imp_add definition changed
+bier_imp_details definition changed
+bier_route_add_del definition changed
+bier_route_details definition changed
+bier_route_dump definition changed
+bier_table_add_del definition changed
+bier_table_details definition changed
+bond_create definition changed
+bridge_domain_add_del definition changed
+bridge_domain_details definition changed
+bridge_flags definition changed
+connect_sock definition changed
+create_vhost_user_if definition changed
+ct6_enable only in file
+ct6_enable_disable only in image
+ct6_enable_disable_reply only in image
+ct6_enable_disable only in file
+dhcp6_pd_reply_event definition changed
+dhcp6_pd_send_client_message definition changed
+dhcp6_reply_event definition changed
+dhcp6_send_client_message definition changed
+dhcp_client_config definition changed
+dhcp_client_details definition changed
+dhcp_compl_event definition changed
+dhcp_proxy_details definition changed
+dslite_add_del_pool_addr_range definition changed
+dslite_address_details definition changed
+dslite_get_aftr_addr_reply definition changed
+dslite_get_b4_addr_reply definition changed
+dslite_set_aftr_addr definition changed
+dslite_set_b4_addr definition changed
+gbp_bridge_domain_add definition changed
+gbp_bridge_domain_details definition changed
+gbp_contract_add_del definition changed
+gbp_contract_details definition changed
+gbp_endpoint_add definition changed
+gbp_endpoint_details definition changed
+gbp_endpoint_group_add definition changed
+gbp_endpoint_group_details definition changed
+gbp_ext_itf_add_del definition changed
+gbp_ext_itf_details definition changed
+gbp_recirc_add_del definition changed
+gbp_recirc_details definition changed
+gbp_route_domain_add definition changed
+gbp_route_domain_details definition changed
+gbp_subnet_add_del definition changed
+gbp_subnet_details definition changed
+gbp_vxlan_tunnel_add definition changed
+gbp_vxlan_tunnel_details definition changed
+get_f64_endian_value only in image
+get_f64_endian_value_reply only in image
+get_f64_increment_by_one only in image
+get_f64_increment_by_one_reply only in image
+gpe_add_del_fwd_entry definition changed
+gpe_fwd_entries_get_reply definition changed
+gpe_fwd_entry_path_details definition changed
+gpe_native_fwd_rpaths_get_reply definition changed
+gre_add_del_tunnel only in file
+gre_add_del_tunnel_reply only in file
+gre_tunnel_add_del only in image
+gre_tunnel_add_del_reply only in image
+gre_tunnel_details definition changed
+gre_tunnel_dump definition changed
+http_static_enable only in image
+http_static_enable_reply only in image
+igmp_event definition changed
+igmp_group_prefix_details definition changed
+igmp_group_prefix_set definition changed
+igmp_listen definition changed
+ip6_fib_details only in file
+ip6_fib_dump only in file
+ip6_mfib_details only in file
+ip6_mfib_dump only in file
+ip6_ra_event definition changed
+ip_add_del_route only in file
+ip_add_del_route_reply only in file
+ip_address_details definition changed
+ip_container_proxy_add_del definition changed
+ip_container_proxy_details definition changed
+ip_fib_details only in file
+ip_fib_dump only in file
+ip_mfib_details only in file
+ip_mfib_dump only in file
+ip_mroute_add_del definition changed
+ip_mroute_details only in image
+ip_mroute_dump only in image
+ip_mtable_details only in image
+ip_mtable_dump only in image
+ip_neighbor_add_del definition changed
+ip_neighbor_details definition changed
+ip_probe_neighbor definition changed
+ip_punt_redirect definition changed
+ip_punt_redirect_details definition changed
+ip_reassembly_get_reply definition changed
+ip_reassembly_set definition changed
+ip_route_add_del only in image
+ip_route_add_del_reply only in image
+ip_route_details only in image
+ip_route_dump only in image
+ip_source_and_port_range_check_add_del definition changed
+ip_table_add_del definition changed
+ip_table_details only in image
+ip_table_dump only in image
+ipfix_flush only in image
+ipfix_flush_reply only in image
+ipip_6rd_add_tunnel definition changed
+ipip_add_tunnel definition changed
+ipip_tunnel_details definition changed
+ipsec_backend_details definition changed
+ipsec_gre_tunnel_add_del only in file
+ipsec_gre_tunnel_add_del_reply only in file
+ipsec_gre_tunnel_details only in file
+ipsec_gre_tunnel_dump only in file
+ipsec_sa_details definition changed
+ipsec_sa_set_key only in file
+ipsec_sa_set_key_reply only in file
+ipsec_sad_entry_add_del definition changed
+ipsec_select_backend definition changed
+ipsec_spd_details definition changed
+ipsec_spd_entry_add_del definition changed
+ipsec_tunnel_if_add_del definition changed
+ipsec_tunnel_if_set_key only in file
+ipsec_tunnel_if_set_key_reply only in file
+ipsec_tunnel_protect_del only in image
+ipsec_tunnel_protect_del_reply only in image
+ipsec_tunnel_protect_details only in image
+ipsec_tunnel_protect_dump only in image
+ipsec_tunnel_protect_update only in image
+ipsec_tunnel_protect_update_reply only in image
+l2_macs_event definition changed
+l3xc_del only in image
+l3xc_del_reply only in image
+l3xc_details only in image
+l3xc_dump only in image
+l3xc_plugin_get_version only in image
+l3xc_plugin_get_version_reply only in image
+l3xc_update only in image
+l3xc_update_reply only in image
+lb_add_del_as definition changed
+lb_add_del_vip definition changed
+lb_as_details only in image
+lb_as_dump only in image
+lb_flush_vip definition changed
+lb_vip_details only in image
+lb_vip_dump only in image
+lisp_add_del_locator_set definition changed
+lisp_add_del_remote_mapping definition changed
+lisp_adjacencies_get_reply definition changed
+log_details only in image
+log_dump only in image
+macip_acl_add definition changed
+macip_acl_add_replace definition changed
+macip_acl_details definition changed
+mactime_add_del_range definition changed
+map_add_domain definition changed
+map_domain_details definition changed
+mfib_signal_details definition changed
+modify_vhost_user_if definition changed
+mpls_fib_details only in file
+mpls_fib_dump only in file
+mpls_ip_bind_unbind definition changed
+mpls_route_add_del definition changed
+mpls_route_details only in image
+mpls_route_dump only in image
+mpls_table_add_del definition changed
+mpls_table_details only in image
+mpls_table_dump only in image
+mpls_tunnel_add_del definition changed
+mpls_tunnel_details definition changed
+nat44_add_del_address_range definition changed
+nat44_add_del_identity_mapping definition changed
+nat44_add_del_interface_addr definition changed
+nat44_add_del_lb_static_mapping definition changed
+nat44_add_del_static_mapping definition changed
+nat44_address_details definition changed
+nat44_del_session definition changed
+nat44_forwarding_enable_disable definition changed
+nat44_forwarding_is_enabled_reply definition changed
+nat44_identity_mapping_details definition changed
+nat44_interface_add_del_feature definition changed
+nat44_interface_add_del_output_feature definition changed
+nat44_interface_addr_details definition changed
+nat44_interface_details definition changed
+nat44_interface_output_feature_details definition changed
+nat44_lb_static_mapping_add_del_local definition changed
+nat44_lb_static_mapping_details definition changed
+nat44_static_mapping_details definition changed
+nat44_user_details definition changed
+nat44_user_session_details definition changed
+nat44_user_session_dump definition changed
+nat64_add_del_interface_addr definition changed
+nat64_add_del_interface definition changed
+nat64_add_del_pool_addr_range definition changed
+nat64_add_del_prefix definition changed
+nat64_add_del_static_bib definition changed
+nat64_bib_details definition changed
+nat64_interface_details definition changed
+nat64_pool_addr_details definition changed
+nat64_prefix_details definition changed
+nat64_st_details definition changed
+nat66_add_del_interface definition changed
+nat66_add_del_static_mapping definition changed
+nat66_interface_details definition changed
+nat66_static_mapping_details definition changed
+nat_det_add_del_map definition changed
+nat_det_close_session_in definition changed
+nat_det_close_session_out definition changed
+nat_det_forward definition changed
+nat_det_forward_reply definition changed
+nat_det_map_details definition changed
+nat_det_reverse definition changed
+nat_det_reverse_reply definition changed
+nat_det_session_details definition changed
+nat_det_session_dump definition changed
+nat_get_mss_clamping_reply definition changed
+nat_ipfix_enable_disable definition changed
+nat_reass_details definition changed
+nat_set_log_level only in image
+nat_set_log_level_reply only in image
+nat_set_mss_clamping definition changed
+nat_set_reass definition changed
+nat_show_config_reply definition changed
+nat_worker_details definition changed
+nsim_cross_connect_enable_disable only in image
+nsim_cross_connect_enable_disable_reply only in image
+nsim_enable_disable only in file
+nsim_enable_disable_reply only in file
+nsim_output_feature_enable_disable only in image
+nsim_output_feature_enable_disable_reply only in image
+oam_add_del only in file
+oam_add_del_reply only in file
+oam_event only in file
+one_add_del_locator_set definition changed
+one_add_del_remote_mapping definition changed
+one_adjacencies_get_reply definition changed
+one_l2_arp_entries_get_reply definition changed
+one_ndp_entries_get_reply definition changed
+p2p_ethernet_add definition changed
+p2p_ethernet_add_reply definition changed
+p2p_ethernet_del definition changed
+pg_create_interface definition changed
+proxy_arp_add_del definition changed
+proxy_arp_details definition changed
+punt_details only in file
+punt_dump only in file
+punt_reason_details only in image
+punt_reason_dump only in image
+punt_socket_deregister definition changed
+punt_socket_details definition changed
+punt_socket_dump definition changed
+punt_socket_register definition changed
+qos_egress_map_delete definition changed
+qos_egress_map_details only in image
+qos_egress_map_dump only in image
+qos_egress_map_update definition changed
+qos_mark_details only in image
+qos_mark_details_reply only in image
+qos_mark_dump only in image
+qos_mark_enable_disable definition changed
+qos_record_details only in image
+qos_record_dump only in image
+qos_record_enable_disable definition changed
+qos_store_details only in image
+qos_store_dump only in image
+qos_store_enable_disable only in image
+qos_store_enable_disable_reply only in image
+sctp_add_src_dst_connection only in file
+sctp_add_src_dst_connection_reply only in file
+sctp_config only in file
+sctp_config_reply only in file
+sctp_del_src_dst_connection only in file
+sctp_del_src_dst_connection_reply only in file
+set_punt definition changed
+show_threads_reply definition changed
+show_vpe_system_time only in image
+show_vpe_system_time_reply only in image
+sockclnt_create_reply definition changed
+sr_localsid_add_del definition changed
+sr_localsids_details definition changed
+sr_policies_details definition changed
+sr_policy_add definition changed
+sr_policy_del definition changed
+sr_policy_mod definition changed
+sr_steering_pol_details definition changed
+svs_details definition changed
+svs_enable_disable definition changed
+svs_route_add_del definition changed
+svs_table_add_del definition changed
+sw_interface_bond_details definition changed
+sw_interface_dump definition changed
+sw_interface_ip6_set_link_local_address only in image
+sw_interface_ip6_set_link_local_address_reply only in image
+sw_interface_ip6nd_ra_prefix definition changed
+sw_interface_set_l2_bridge definition changed
+sw_interface_tap_v2_details definition changed
+syslog_get_filter_reply definition changed
+syslog_set_filter definition changed
+tap_create_v2 definition changed
+udp_encap_add definition changed
+udp_encap_details definition changed
+virtio_pci_create definition changed
+vmxnet3_details definition changed
+vxlan_gbp_tunnel_add_del definition changed
+vxlan_gbp_tunnel_details definition changed
+want_oam_events only in file
+want_oam_events_reply only in file
+============================================= ==================
+
+Found 319 api message signature differences
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``src/vpp/api/vpe_types.api``
+
+* `a47a5f20a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a47a5f20a>`_ api papi: add alias for timestamp(datetime)/timedelta
+* `3cf9e67f5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3cf9e67f5>`_ api: add vl_api_version_t type
+
+``src/vpp/api/vpe.api``
+
+* `a47a5f20a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a47a5f20a>`_ api papi: add alias for timestamp(datetime)/timedelta
+* `888640a39 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=888640a39>`_ map gbp papi: match endianess of f64
+* `03f1af23b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=03f1af23b>`_ api: Implement log_dump/log_details
+* `c87b66c86 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c87b66c86>`_ ipsec: ipsec-tun protect
+* `9ac113815 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9ac113815>`_ API: Add support for limits to language.
+
+``src/examples/sample-plugin/sample/sample.api``
+
+* `78d91cf9a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=78d91cf9a>`_ sample-plugin: refactor .api to use explicit types
+
+``src/vnet/interface.api``
+
+* `0ad4a439d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0ad4a439d>`_ Fix vpp crash bug while deleting dhcp client
+* `9a29f795a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9a29f795a>`_ vpp_papi_provider.py: update defautmapping.
+* `b8591ac91 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b8591ac91>`_ API sw_interface_dump: Dump all if index is zero
+* `4a7240636 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4a7240636>`_ Make sw_interface_dump more compatible with 2.2.0
+* `6407ba56a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6407ba56a>`_ api: Add to interface crud - read by sw_if_index.
+
+``src/vnet/qos/qos.api``
+
+* `83832e7ce <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=83832e7ce>`_ qos: Store function
+* `5281a9029 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5281a9029>`_ qos: QoS dump APIs
+
+``src/vnet/bier/bier.api``
+
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+* `e6eefb6e3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e6eefb6e3>`_ Trivial Typo's in bier comments/docs.
+
+``src/vnet/ipfix-export/ipfix_export.api``
+
+* `21b83e96d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=21b83e96d>`_ api: implement ipfix_flush
+
+``src/vnet/session/session.api``
+
+* `8ac1d6d05 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8ac1d6d05>`_ session: Use parent_handle instead of transport_opts
+* `ba65ca496 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ba65ca496>`_ Add transport_opts to connect_sock bapi
+
+``src/vnet/gre/gre.api``
+
+* `814f15948 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=814f15948>`_ gre: update gre.api with explicit types
+* `d0aed2eb3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d0aed2eb3>`_ GRE: set gre_tunnel_type init value to zero in API
+* `5a8844bdb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5a8844bdb>`_ GRE: API update
+
+``src/vnet/pg/pg.api``
+
+* `22e9cfd76 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=22e9cfd76>`_ pg: add GSO support
+
+``src/vnet/l2/l2.api``
+
+* `bc764c8bc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bc764c8bc>`_ l2: BD ARP termination entry API update
+* `54bc5e40c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=54bc5e40c>`_ Update API description
+* `5e6f7348c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5e6f7348c>`_ l2: Add support for arp unicast forwarding
+
+``src/vnet/udp/udp.api``
+
+* `10dc2eabd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=10dc2eabd>`_ udp: fix copyright typo
+
+``src/vnet/devices/tap/tapv2.api``
+
+* `97d54ed43 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=97d54ed43>`_ tap: add support to configure tap interface host MTU size
+
+``src/vnet/devices/virtio/vhost_user.api``
+
+* `4208a4ce8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4208a4ce8>`_ devices interface tests: vhosst GSO support
+
+``src/vnet/devices/virtio/virtio.api``
+
+* `bbd6b746e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bbd6b746e>`_ virtio: Add gso support for native virtio driver
+* `43b512cac <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=43b512cac>`_ virtio: remove configurable queue size support
+
+``src/vnet/mfib/mfib_types.api``
+
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+
+``src/vnet/ipsec/ipsec.api``
+
+* `c87b66c86 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c87b66c86>`_ ipsec: ipsec-tun protect
+* `f2922422d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f2922422d>`_ ipsec: remove the set_key API
+* `80f6fd53f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=80f6fd53f>`_ IPSEC: Pass the algorithm salt (used in GCM) over the API
+
+``src/vnet/ethernet/p2p_ethernet.api``
+
+* `8edca1361 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8edca1361>`_ p2p ethernet: update p2p_ethernet.api with explicit types.
+
+``src/vnet/bonding/bond.api``
+
+* `751e3f382 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=751e3f382>`_ bonding: add support for numa-only in lacp mode
+
+``src/vnet/mpls/mpls.api``
+
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+
+``src/vnet/ipip/ipip.api``
+
+* `288e09362 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=288e09362>`_ ipip: refactor ipip.api with explicit types
+* `cbd0824d6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cbd0824d6>`_ IPIP tunnel: use address types on API
+
+``src/vnet/fib/fib_types.api``
+
+* `1dbcf30b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1dbcf30b7>`_ fib: Support the POP of a Psuedo Wire Control Word
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+
+``src/vnet/dhcp/dhcp.api``
+
+* `038e1dfbd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=038e1dfbd>`_ dhcp ip: DSCP settings for transmitted DHCP packets
+* `56bc738dc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=56bc738dc>`_ Fix VPP-1487 DHCP client does not support option 6-domain server
+
+``src/vnet/ip/punt.api``
+
+* `719beb709 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=719beb709>`_ ip ipsec: Remove IPSec SPI-0 punt reason
+* `b538dd868 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b538dd868>`_ Punt: specify packets by IP protocol Type
+* `50f0ac0f0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=50f0ac0f0>`_ Punt: socket register for exception dispatched/punted packets based on reason
+
+``src/vnet/ip/ip.api``
+
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+* `3a343d42d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3a343d42d>`_ reassembly: prevent long chain attack
+
+``src/vnet/ip/ip_types.api``
+
+* `515eed425 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=515eed425>`_ api: add prefix matcher typedef
+* `038e1dfbd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=038e1dfbd>`_ dhcp ip: DSCP settings for transmitted DHCP packets
+* `53c501512 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53c501512>`_ api: add DSCP definitions to ip_types.api
+* `ab05508e1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ab05508e1>`_ api: refactor format_vl_api_prefix_t return keys
+* `b538dd868 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b538dd868>`_ Punt: specify packets by IP protocol Type
+* `50f0ac0f0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=50f0ac0f0>`_ Punt: socket register for exception dispatched/punted packets based on reason
+
+``src/plugins/l3xc/l3xc.api``
+
+* `59fa121f8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=59fa121f8>`_ L3 cross connect
+
+``src/plugins/map/map.api``
+
+* `4d376f67a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4d376f67a>`_ map: Use vl_api_string macros.
+
+``src/plugins/http_static/http_static.api``
+
+* `68b24e2c9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=68b24e2c9>`_ plugins: http_static. Migrate to use api string type.
+* `22bc2c46e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=22bc2c46e>`_ Static http server
+
+``src/plugins/igmp/igmp.api``
+
+* `4ff09ae34 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4ff09ae34>`_ API: Python and Unix domain socket improvement
+
+``src/plugins/sctp/sctp.api``
+
+* `3ffe6cadf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3ffe6cadf>`_ sctp: move to plugins, disabled by default
+
+``src/plugins/lb/lb.api``
+
+* `3efcd0d7c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3efcd0d7c>`_ lb: vip and as dump/detail api's
+* `a0cb32cb9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a0cb32cb9>`_ lb: update api.c to use scaffolding from latest skel
+
+``src/plugins/lb/lb_types.api``
+
+* `3efcd0d7c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3efcd0d7c>`_ lb: vip and as dump/detail api's
+
+``src/plugins/mactime/mactime.api``
+
+* `7681b1c46 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7681b1c46>`_ mactime: add per-mac allow-with-quota feature
+* `0c6ac791d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0c6ac791d>`_ mactime: upstream new features
+
+``src/plugins/gbp/gbp.api``
+
+* `3918bdbcb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3918bdbcb>`_ gbp: update gbp-ext-itf API
+* `3c0d84c98 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3c0d84c98>`_ gbp: add anonymous l3-out subnets
+* `cfc7a107e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cfc7a107e>`_ gbp: add anonymous l3-out external interfaces
+* `160c923f9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=160c923f9>`_ gbp: VRF scoped contracts
+
+``src/plugins/acl/acl_types.api``
+
+* `bb2e5221a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bb2e5221a>`_ api acl: breakout acl_types.api for reuse by others
+
+``src/plugins/acl/acl.api``
+
+* `bb2e5221a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bb2e5221a>`_ api acl: breakout acl_types.api for reuse by others
+* `f995c7122 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f995c7122>`_ acl: implement counters
+
+``src/plugins/nat/nat.api``
+
+* `e6e09a4ac <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e6e09a4ac>`_ nat: elog rewrite for multi-worker support
+* `c1f93067e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c1f93067e>`_ Add default value for API Nat flags
+* `dd1e3e780 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dd1e3e780>`_ NAT: VPP-1531 api cleanup & update
+* `89fec713f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=89fec713f>`_ Revert "NAT: VPP-1531 api cleanup & update"
+* `bed1421b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bed1421b9>`_ NAT: VPP-1531 api cleanup & update
+
+``src/plugins/abf/abf.api``
+
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+
+``src/plugins/nsim/nsim.api``
+
+* `7c91007e1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7c91007e1>`_ Make the loss / delay sim available as an output feature
+
diff --git a/docs/aboutvpp/releasenotes/v20.01.rst b/docs/aboutvpp/releasenotes/v20.01.rst
new file mode 100644
index 0000000..a2fbebc
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v20.01.rst
@@ -0,0 +1,1384 @@
+Release notes for VPP 20.01
+===========================
+
+More than 1039 commits since the 19.08 release.
+
+Features
+--------
+
+- API trace tool
+
+ - Add text output (a2ac36c91)
+
+- Binary API Compiler for Python
+
+ - Raise ValueError when fieldname is python keyword (ff47fb645)
+
+- Binary API Libraries
+
+ - Add API support for marvell PP2 plugin (859b59133)
+ - Add bapi thread handle to api main structure. (8229580e8)
+ - Multiple connections per process (39d69112f)
+ - Multiple socket connections per single process (59cea1a9d)
+
+- Build System
+
+ - Add build types helpstring to cmake project (952a7b8b7)
+ - Add env variable to pass extra cmake args (297365403)
+ - Add yaml file linting to make checkstyle (6b0dd5502)
+ - Export vapi generation in vpp-dev (dc20371f8)
+ - Fix 3rd party CI systems. (86a9441c2)
+ - Pass ‘no-pci’ to autgenerated config (be7ef3b5c)
+
+- Crypto ipsecmb Plugin
+
+ - Bump to intel-ipsec-mb version 0.53 (d35fefe8b)
+ - Improve gcm performance using dedicated API. (76a36e83e)
+
+- Infrastructure Library
+
+ - Bihash walk cb typedef and continue/stop controls (f50bac1bb)
+ - Create unformat function for data size parsing (579b16506)
+ - Implement CLIB_PAUSE () for aarch64 platforms (18512b002)
+
+- libmemif
+
+ - Introduce ‘memif_per_thread\_’ namespace (17f2a7bbf)
+
+- Link Bonding
+
+ - Add/del secondary mac address callback (e83aa456b)
+ - Add /if/lacp/bond-sw-if-index/slave-sw-if-index/partner-state
+ (aa7257863)
+ - Add weight support for active-backup mode (a1876b84e)
+ - Fix interface deletion (cc3aac056)
+
+- Miscellaneous
+
+ - Add address sanitizer heap instrumentation (9fb6d40eb)
+ - Add CentOS 8 package support (c025329bb)
+ - Add gdb helpers for vlib buffers (2b65f9ca0)
+ - Add lcov scripts, README.md (8d74caa0a)
+ - Add “maxframe” and “rate” to packet-generator cli. (87d7bac5c)
+ - Add “show run summary” (ac78f8a90)
+ - Add vnet classify filter set support (f5667c305)
+ - Classifier-based packet trace filter (9137e5400)
+ - Improve pcap drop trace output (9382ad9b3)
+ - Update gitignore for /test/ext/.d (8161d73d7)
+
+- Physical Memory Allocator
+
+ - Always lock pages (801c7016a)
+
+- Plugins
+
+ - AVF Device driver
+ - Improve timeout handling (1a7bb281f)
+ - Print queue id in packet trace (c33eddebe)
+ - Buffer Metadata Modification Tracker
+ - Buffer metadata change tracker plugin (d7b306657)
+ - builtinurl
+ - New plugin (43765e2b4)
+ - DHCP
+ - Ipv6 prefix delegation improvements (d318a996b)
+ - Move to plugin (02bfd641b)
+ - DPDK
+ - Add devargs support (f2bde7ac5)
+ - Add function to add/del extra MAC addrs (153727743)
+ - Add TSO support in dpdk plugin. (de5ed58fd)
+ - Apply dual loop unrolling in DPDK TX (fe2523d1a)
+ - Bump DPDK version to 19.08 (b6103105f)
+ - Enable bnxt PMD (c3731ac89)
+ - Ipsec tunnel support for ip6-in-ip4 (5025d40a1)
+ - QAT devices update, add c4xxx and xeon d15xx (4d843b994)
+ - Group Based Policy (GBP)
+ - Add extended SFC unit tests (a3c8ca10e)
+ - Host Stack Applications
+ - Add ckpair & crypto engine in vpp_echo (7c40a3837)
+ - Add option for multiple ips (f98e59b86)
+ - Add periodic timing (ff6cdcca2)
+ - Improve for mq-intensive (b2fce36c8)
+ - Less verbose logging for many sessions (08f26641f)
+ - Make APP_OPTIONS_PREALLOC_FIFO_PAIRS configurable (7028a0157)
+ - http_static
+ - Add dynamic GET / POST method hooks (5554c56a6)
+ - Add “http static cache clear” CLI (e0fd9ed11)
+ - Add .json content (71a5da0c8)
+ - Internet Key Exchange (IKEv2) Protocol
+ - Add support for GCM cipher (de2dd6c35)
+ - IPv6 Segment Routing Mobile
+ - (57584d99d)
+ - Load Balancer
+ - Add APIs for set interface nat4 and nat6 (33538a150)
+ - NAT
+ - Handoff traffic matching for dynamic NAT (22bb417e9)
+ - Ping
+ - Move the echo responder into the ping plugin (f6c8f5090)
+ - QUIC protocol
+ - Add aggregated quicly stats (deaf97f45)
+ - Add cli command for stats (922f0b211)
+ - Add conn-timeout config option (2f566c23f)
+ - Add more detailed statistics (1802fcc5f)
+ - Add support for ckpair & crypto engine (dcbbf2833)
+ - Add support for unidirectional streams (c00f480ba)
+ - Add Tx, Rx and packet drop counters (ff1f6faaa)
+ - Create custom event logger (dd4d8ac29)
+ - Implement crypto contexts (d1b9e7068)
+ - Make quic fifo size configurable via cli (00078b991)
+ - Update quicly to v0.0.5 (72c159e64)
+ - Update quicly to v0.0.6-vpp (3afac8f81)
+ - Update quicly to v0.0.7-vpp (69885b72a)
+ - Update quicly to v0.0.8-vpp (ecb9d18c5)
+ - Update quicly to v0.0.9-vpp (84def7cb7)
+ - RDMA (ibverb) driver
+ - Add rdma API (812afe712)
+ - Add support for input feature arcs (74eba446b)
+ - Add support for MAC changes (0dcafcc50)
+ - API: prepare support for direct verb (d8c1ef925)
+ - Time-based MAC filter
+ - Add a “top” command to watch device stats (2c41a61d5)
+ - Add the “mactime.json” builtin URL (ef3c11ca9)
+ - vmxnet3 device driver
+ - Per interface gso support (2985e0af6)
+
+- Python binding for the VPP API
+
+ - Add a per-call \_timeout option (e2ccdf031)
+ - Add call stats (fd574087e)
+ - Add repr to packer types for troubleshooting (14b0b4791)
+ - Add wrapper to validate crc manifest (c046d709e)
+ - Enhance MACAddress() equality (6af62565e)
+ - Introduce read_blocking (0938547ea)
+ - Let async calls return context (2f6e0c600)
+ - Support default for type alias decaying to basetype (418ebb711)
+
+- Sphinx Documents
+
+ - Add spellcheck to ‘make docs’ sphinx docs (340c15c6e)
+
+- Statistics Segment
+
+ - Add /if/<n>/<n>/state for lacp interface state (0f09a828a)
+
+- SVM Library
+
+ - Improve fifo segment verbose cli (f8461bfb4)
+
+- Test Infrastructure
+
+ - Add cli_return_response to vpp_papi_provider (5932ce17e)
+ - Add test run time. (0c6293230)
+ - Support setting random seed (45a95dd78)
+ - Support worker threads (4ecbf105a)
+ - Test tls case (419d31f81)
+
+- Vector Library
+
+ - Add flag to explicitelly mark nodes which can init per-node packet
+ trace (7ca5aaac1)
+ - Add max-size configuration parameter for pmalloc (842506f3c)
+ - Add ‘wait’ cli command (bfd7d294d)
+ - Enhance the “show cli” debug CLI command (a1f5a956e)
+
+- VNET
+
+ - Classify
+
+ - Per-interface rx/tx pcap capture filters (d28437cdf)
+ - Use vector code even when data is not aligned (830493392)
+ - Vpp packet tracer support (87d24db65)
+
+ - Ethernet
+
+ - All dmac checks include secondary addrs (42bde459b)
+ - Dmac filter checks secondary mac addrs (d459bf344)
+
+ - FIB
+
+ - Adjacency creation notifications for dlegates (77cfc0171)
+ - Decouple source from priority and behaviour (3bab8f9c5)
+ - Table Replace (9db6ada77)
+
+ - FLOW
+
+ - Add ‘drop’ and ‘redirect-to-queue’ actions support (e8c9f4f1c)
+ - Add ethernet flow (4ff8d615c)
+ - Add GTP support (bf85a98fb)
+
+ - GRE
+
+ - Multi-point interfaces (5f8f61733)
+
+ - GSO
+
+ - Add protocol header parser (72e7312af)
+
+ - Interface Common
+
+ - Callback to manage extra MAC addresses (e0792fdff)
+ - Dump the interface device type (de312c2d5)
+
+ - IPIP
+
+ - Tunnel flags controlling copying data to/from payload/encap
+ (9534696b4)
+
+ - IPSec
+
+ - Add ‘detail’ option to ‘sh ipsec sa’ (670027a50)
+ - Add insecure option for format of SA (01d61e788)
+ - Bind an SA to a worker (f62a8c013)
+ - Remove dedicated IPSec tunnels (12989b538)
+ - Support 4o6 and 6o4 for tunnel protect (b325983a4)
+
+ - IPv4 LPM
+
+ - Add shallow virtual reassembly functionality (de34c35fc)
+ - Add tracing for ipv6 frag headers (0eb75d0e9)
+ - Allow addrs from the same prefix on intf (6c92f5bab)
+ - Apply dual loop unrolling in ip4_input (86b1871ba)
+ - Apply dual loop unrolling in ip4_rewrite (840f64b4b)
+
+ - IPv4 LPM
+
+ - Protocol Independent IP Neighbors (cbe25aab3)
+ - Punt rather than drop unkown IPv6 ICMP packets (1afe95272)
+ - Reassembly: trace ip headers over worker handoffs (8563cb389)
+
+ - Segment Routing (IPv6 and MPLS)
+
+ - Add “set sr encaps hop-limit” command (eeb5fb3a5)
+
+ - Session Layer
+
+ - Add certificate store (79f89537c)
+ - Add crypto context (de6caf481)
+ - Add explicit reset api (dfb3b8771)
+ - Add mq debug cli (cfdb10918)
+ - Add session enable option in config file (1292d19c7)
+ - Builtin app rx notifications regardless of state (5c29029ef)
+ - Ckpair store & crypto engine as mq params (45ec9f49b)
+ - Improve cli (5bb23ecd0)
+ - Increasing the Header lengthe size (93e060aee)
+ - Limit pacer bucket size (7c8f828ba)
+ - More show cli output (91f90d082)
+ - Reschedule asap when snd space constrained (dd97a48d9)
+ - Support registration of custom crypto engines (79ba25d40)
+ - Support for segments larger than 4GB (ef4f3e7fe)
+ - Add opaque data to show cli (d9035a409)
+ - Infra for transports to send buffers (2a7ea2ee9)
+ - Support pacer idle timeouts (11e9e3510)
+
+ - TAP Drivers
+
+ - Add check for vhost-net backend (39807d02c)
+ - Multiqueue support (7c6102b1a)
+
+ - TCP
+
+ - Add FEATURE.yaml (93e053ebe)
+ - Add no csum offload config option (f4ce6ba22)
+ - Add option for always on event logging (a436a4222)
+ - Allow cc algos to set pacing rate (d206724e7)
+ - Compute snd time for rate sample (7436b4367)
+ - Custom checksum calculations for Ipv4/Ipv6 (02833ff32)
+ - Enable gso in tcp hoststack (1146ff4bc)
+ - Enable TCP timewait port use (b092b77cf)
+ - Extend protocol configuration (9094b5c31)
+ - Force zero window on full rx fifo (182d21983)
+ - Handle sack reneging (558e3e095)
+ - Improve lost rxt heuristic (b3dce89a7)
+ - Improve pacing after idle send periods (c31dc31f8)
+ - Retry lost retransmits (be237bf02)
+ - Send rwnd update only if wnd is large enough (017dc4524)
+ - Set cc_algo on connection alloc (12f6936cd)
+ - Track lost rxt segments in byte tracker (46ec6e018)
+ - Track zero rwnd errors (a495a3ea1)
+ - Use rate sample rtt in recovery if possible (1dbda64b4)
+ - Use sacks for timer based recovery (36ebcfffb)
+ - Validate connections in output (78dae0088)
+ - Validate the IP address while checking TCP connection
+ (cf4c2102d)
+
+ - TLS and TLS engine plugins
+
+ - Add C API for TLS openssl to set engine (be4d1aa2c)
+ - Improve connection formating (0d74dd1f8)
+ - Picotls engine basic enabling for TLS (f83194c2f)
+
+- VPP Comms Library
+
+ - Add api to set lcl ip (ef7cbf6ad)
+ - Add config option for preferred tls engine (d747c3c36)
+ - Allow non-blocking connects (57c88938f)
+
+- VPP Object Model
+
+ - Get interface type from vpp device type (3f4be92ce)
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Issues fixed
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/2001>`__
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+- *Only in image*: indicates the API is new for this release.
+- *Only in file*: indicates the API has been removed in this release.
+
+================================================ ==================
+Message Name Result
+================================================ ==================
+abf_itf_attach_add_del definition changed
+abf_itf_attach_details definition changed
+abf_policy_add_del definition changed
+abf_policy_details definition changed
+af_packet_create definition changed
+af_packet_create_reply definition changed
+af_packet_delete definition changed
+af_packet_details definition changed
+af_packet_set_l4_cksum_offload definition changed
+api_versions_reply definition changed
+app_add_cert_key_pair only in image
+app_add_cert_key_pair_reply only in image
+app_attach only in image
+app_attach_reply only in image
+app_del_cert_key_pair only in image
+app_del_cert_key_pair_reply only in image
+avf_create_reply definition changed
+avf_delete definition changed
+bd_ip_mac_add_del definition changed
+bd_ip_mac_details definition changed
+bfd_udp_add definition changed
+bfd_udp_auth_activate definition changed
+bfd_udp_auth_deactivate definition changed
+bfd_udp_del definition changed
+bfd_udp_get_echo_source_reply definition changed
+bfd_udp_mod definition changed
+bfd_udp_session_details definition changed
+bfd_udp_session_set_flags definition changed
+bfd_udp_set_echo_source definition changed
+bier_disp_entry_add_del definition changed
+bier_disp_entry_details definition changed
+bier_disp_table_add_del definition changed
+bier_route_add_del definition changed
+bier_table_add_del definition changed
+bond_create definition changed
+bond_create_reply definition changed
+bond_delete definition changed
+bond_detach_slave definition changed
+bond_enslave definition changed
+builtinurl_enable only in image
+builtinurl_enable_reply only in image
+bvi_create definition changed
+cdp_enable_disable definition changed
+classify_add_del_session definition changed
+classify_add_del_table definition changed
+classify_set_interface_ip_table definition changed
+classify_set_interface_l2_tables definition changed
+classify_table_by_interface definition changed
+classify_table_by_interface_reply definition changed
+cli_inband definition changed
+cli_inband_reply definition changed
+collect_detailed_interface_stats definition changed
+connect_sock definition changed
+cop_interface_enable_disable definition changed
+cop_whitelist_enable_disable definition changed
+create_loopback definition changed
+create_loopback_instance definition changed
+create_loopback_instance_reply definition changed
+create_loopback_reply definition changed
+create_subif definition changed
+create_subif_reply definition changed
+create_vhost_user_if definition changed
+create_vhost_user_if_reply definition changed
+create_vlan_subif definition changed
+create_vlan_subif_reply definition changed
+ct6_enable_disable definition changed
+delete_loopback definition changed
+delete_subif definition changed
+delete_vhost_user_if definition changed
+dhcp6_client_enable_disable definition changed
+dhcp6_clients_enable_disable definition changed
+dhcp6_pd_client_enable_disable definition changed
+dhcp6_pd_reply_event definition changed
+dhcp6_pd_send_client_message definition changed
+dhcp6_reply_event definition changed
+dhcp6_send_client_message definition changed
+dhcp_client_config definition changed
+dhcp_client_details definition changed
+dhcp_compl_event definition changed
+dhcp_plugin_control_ping only in image
+dhcp_plugin_control_ping_reply only in image
+dhcp_plugin_get_version only in image
+dhcp_plugin_get_version_reply only in image
+dhcp_proxy_config definition changed
+dhcp_proxy_details definition changed
+dhcp_proxy_dump definition changed
+dhcp_proxy_set_vss definition changed
+dslite_add_del_pool_addr_range definition changed
+dslite_address_details definition changed
+dslite_get_aftr_addr_reply definition changed
+dslite_get_b4_addr_reply definition changed
+dslite_set_aftr_addr definition changed
+dslite_set_b4_addr definition changed
+feature_enable_disable definition changed
+feature_gso_enable_disable only in image
+feature_gso_enable_disable_reply only in image
+flow_classify_details definition changed
+flow_classify_dump definition changed
+flow_classify_set_interface definition changed
+flowprobe_params definition changed
+flowprobe_tx_interface_add_del definition changed
+gbp_bridge_domain_add definition changed
+gbp_bridge_domain_details definition changed
+gbp_contract_add_del definition changed
+gbp_contract_details definition changed
+gbp_endpoint_add definition changed
+gbp_endpoint_details definition changed
+gbp_endpoint_group_add definition changed
+gbp_endpoint_group_details definition changed
+gbp_ext_itf_add_del definition changed
+gbp_ext_itf_details definition changed
+gbp_recirc_add_del definition changed
+gbp_recirc_details definition changed
+gbp_route_domain_add definition changed
+gbp_route_domain_details definition changed
+gbp_subnet_add_del definition changed
+gbp_subnet_details definition changed
+gbp_vxlan_tunnel_add definition changed
+gbp_vxlan_tunnel_add_reply definition changed
+gbp_vxlan_tunnel_details definition changed
+geneve_add_del_tunnel definition changed
+geneve_add_del_tunnel_reply definition changed
+geneve_tunnel_details definition changed
+geneve_tunnel_dump definition changed
+get_first_msg_id definition changed
+gre_tunnel_add_del definition changed
+gre_tunnel_add_del_reply definition changed
+gre_tunnel_details definition changed
+gre_tunnel_dump definition changed
+gtpu_add_del_tunnel definition changed
+gtpu_add_del_tunnel_reply definition changed
+gtpu_tunnel_details definition changed
+gtpu_tunnel_dump definition changed
+http_static_enable definition changed
+hw_interface_set_mtu definition changed
+igmp_clear_interface definition changed
+igmp_details definition changed
+igmp_dump definition changed
+igmp_enable_disable definition changed
+igmp_event definition changed
+igmp_group_prefix_details definition changed
+igmp_group_prefix_set definition changed
+igmp_listen definition changed
+igmp_proxy_device_add_del definition changed
+igmp_proxy_device_add_del_interface definition changed
+ikev2_initiate_sa_init definition changed
+ikev2_profile_add_del definition changed
+ikev2_profile_set_auth definition changed
+ikev2_profile_set_id definition changed
+ikev2_profile_set_ts definition changed
+ikev2_set_esp_transforms definition changed
+ikev2_set_ike_transforms definition changed
+ikev2_set_responder definition changed
+ikev2_set_sa_lifetime definition changed
+input_acl_set_interface definition changed
+interface_name_renumber definition changed
+ioam_cache_ip6_enable_disable definition changed
+ioam_enable definition changed
+ioam_export_ip6_enable_disable definition changed
+ip4_arp_event only in file
+ip6_add_del_address_using_prefix definition changed
+ip6_nd_address_autoconfig definition changed
+ip6_nd_event only in file
+ip6_ra_event definition changed
+ip6nd_proxy_add_del definition changed
+ip6nd_proxy_details definition changed
+ip6nd_send_router_solicitation definition changed
+ip_address_details definition changed
+ip_address_dump definition changed
+ip_container_proxy_add_del definition changed
+ip_container_proxy_details definition changed
+ip_details definition changed
+ip_dump definition changed
+ip_mroute_add_del definition changed
+ip_mroute_details definition changed
+ip_mroute_dump definition changed
+ip_mtable_details definition changed
+ip_neighbor_add_del definition changed
+ip_neighbor_config only in image
+ip_neighbor_config_reply only in image
+ip_neighbor_details definition changed
+ip_neighbor_dump definition changed
+ip_neighbor_event only in image
+ip_probe_neighbor only in file
+ip_probe_neighbor_reply only in file
+ip_punt_police definition changed
+ip_punt_redirect definition changed
+ip_punt_redirect_details definition changed
+ip_punt_redirect_dump definition changed
+ip_reassembly_enable_disable definition changed
+ip_reassembly_get definition changed
+ip_reassembly_get_reply definition changed
+ip_reassembly_set definition changed
+ip_route_add_del definition changed
+ip_route_details definition changed
+ip_route_dump definition changed
+ip_scan_neighbor_enable_disable only in file
+ip_scan_neighbor_enable_disable_reply only in file
+ip_source_and_port_range_check_add_del definition changed
+ip_source_and_port_range_check_interface_add_del definition changed
+ip_source_check_interface_add_del definition changed
+ip_table_add_del definition changed
+ip_table_details definition changed
+ip_table_flush only in image
+ip_table_flush_reply only in image
+ip_table_replace_begin only in image
+ip_table_replace_begin_reply only in image
+ip_table_replace_end only in image
+ip_table_replace_end_reply only in image
+ip_unnumbered_details definition changed
+ip_unnumbered_dump definition changed
+ipfix_classify_table_add_del definition changed
+ipfix_classify_table_details definition changed
+ipfix_exporter_details definition changed
+ipip_6rd_add_tunnel definition changed
+ipip_6rd_add_tunnel_reply definition changed
+ipip_6rd_del_tunnel definition changed
+ipip_add_tunnel definition changed
+ipip_add_tunnel_reply definition changed
+ipip_del_tunnel definition changed
+ipip_tunnel_details definition changed
+ipip_tunnel_dump definition changed
+ipsec_spd_details definition changed
+ipsec_spd_entry_add_del definition changed
+ipsec_tunnel_if_add_del definition changed
+ipsec_tunnel_protect_del definition changed
+ipsec_tunnel_protect_details definition changed
+ipsec_tunnel_protect_dump definition changed
+ipsec_tunnel_protect_update definition changed
+l2_arp_term_event only in image
+l2_emulation definition changed
+l2tpv3_create_tunnel definition changed
+l2tpv3_create_tunnel_reply definition changed
+l2tpv3_interface_enable_disable definition changed
+l2tpv3_set_lookup_key definition changed
+l2tpv3_set_tunnel_cookies definition changed
+l3xc_del definition changed
+l3xc_details definition changed
+l3xc_dump definition changed
+l3xc_update definition changed
+lb_add_del_as definition changed
+lb_add_del_intf_nat4 only in image
+lb_add_del_intf_nat4_reply only in image
+lb_add_del_intf_nat6 only in image
+lb_add_del_intf_nat6_reply only in image
+lb_add_del_vip definition changed
+lb_as_details definition changed
+lb_as_dump definition changed
+lb_conf definition changed
+lb_flush_vip definition changed
+lb_vip_details definition changed
+lb_vip_dump definition changed
+log_details definition changed
+log_dump definition changed
+mactime_add_del_range definition changed
+mactime_details only in image
+mactime_dump only in image
+mactime_dump_reply only in image
+mactime_enable_disable definition changed
+map_add_del_rule definition changed
+map_add_domain definition changed
+map_domain_details definition changed
+map_if_enable_disable definition changed
+map_param_add_del_pre_resolve definition changed
+map_param_get_reply definition changed
+map_param_set_icmp definition changed
+map_param_set_reassembly only in file
+map_param_set_reassembly_reply only in file
+map_param_set_traffic_class definition changed
+map_rule_details definition changed
+mdata_enable_disable only in image
+mdata_enable_disable_reply only in image
+memclnt_create definition changed
+memclnt_delete definition changed
+memif_create definition changed
+memif_create_reply definition changed
+memif_delete definition changed
+memif_details definition changed
+memif_socket_filename_add_del definition changed
+memif_socket_filename_details definition changed
+mfib_signal_details definition changed
+modify_vhost_user_if definition changed
+mpls_ip_bind_unbind definition changed
+mpls_route_add_del definition changed
+mpls_route_details definition changed
+mpls_route_dump definition changed
+mpls_table_add_del definition changed
+mpls_table_details definition changed
+mpls_tunnel_add_del definition changed
+mpls_tunnel_add_del_reply definition changed
+mpls_tunnel_details definition changed
+mpls_tunnel_dump definition changed
+nat44_add_del_address_range definition changed
+nat44_add_del_identity_mapping definition changed
+nat44_add_del_interface_addr definition changed
+nat44_add_del_lb_static_mapping definition changed
+nat44_add_del_static_mapping definition changed
+nat44_address_details definition changed
+nat44_del_session definition changed
+nat44_identity_mapping_details definition changed
+nat44_interface_add_del_feature definition changed
+nat44_interface_add_del_output_feature definition changed
+nat44_interface_addr_details definition changed
+nat44_interface_details definition changed
+nat44_interface_output_feature_details definition changed
+nat44_lb_static_mapping_add_del_local definition changed
+nat44_lb_static_mapping_details definition changed
+nat44_static_mapping_details definition changed
+nat44_user_details definition changed
+nat44_user_session_details definition changed
+nat44_user_session_dump definition changed
+nat64_add_del_interface_addr definition changed
+nat64_add_del_interface definition changed
+nat64_add_del_pool_addr_range definition changed
+nat64_add_del_prefix definition changed
+nat64_add_del_static_bib definition changed
+nat64_bib_details definition changed
+nat64_interface_details definition changed
+nat64_pool_addr_details definition changed
+nat64_prefix_details definition changed
+nat64_st_details definition changed
+nat66_add_del_interface definition changed
+nat66_add_del_static_mapping definition changed
+nat66_interface_details definition changed
+nat66_static_mapping_details definition changed
+nat_det_add_del_map definition changed
+nat_det_close_session_in definition changed
+nat_det_close_session_out definition changed
+nat_det_forward definition changed
+nat_det_forward_reply definition changed
+nat_det_map_details definition changed
+nat_det_reverse definition changed
+nat_det_reverse_reply definition changed
+nat_det_session_details definition changed
+nat_det_session_dump definition changed
+nat_get_reass only in file
+nat_get_reass_reply only in file
+nat_ha_get_failover_reply definition changed
+nat_ha_get_listener_reply definition changed
+nat_ha_set_failover definition changed
+nat_ha_set_listener definition changed
+nat_reass_details only in file
+nat_reass_dump only in file
+nat_set_reass only in file
+nat_set_reass_reply only in file
+nat_worker_details definition changed
+nhrp_details only in image
+nhrp_dump only in image
+nhrp_entry_add_del only in image
+nhrp_entry_add_del_reply only in image
+nsh_add_del_entry definition changed
+nsh_add_del_map definition changed
+nsh_map_details definition changed
+nsim_cross_connect_enable_disable definition changed
+nsim_output_feature_enable_disable definition changed
+output_acl_set_interface definition changed
+p2p_ethernet_add definition changed
+p2p_ethernet_add_reply definition changed
+p2p_ethernet_del definition changed
+pipe_create definition changed
+pipe_create_reply definition changed
+pipe_delete definition changed
+pipe_details definition changed
+policer_classify_details definition changed
+policer_classify_dump definition changed
+policer_classify_set_interface definition changed
+pot_profile_activate definition changed
+pot_profile_add definition changed
+pot_profile_del definition changed
+pppoe_add_del_session definition changed
+pppoe_add_del_session_reply definition changed
+pppoe_session_details definition changed
+pppoe_session_dump definition changed
+proxy_arp_add_del definition changed
+proxy_arp_details definition changed
+proxy_arp_intfc_enable_disable definition changed
+punt_reason_details definition changed
+punt_reason_dump definition changed
+punt_socket_details definition changed
+punt_socket_register definition changed
+punt_socket_register_reply definition changed
+qos_mark_dump definition changed
+qos_mark_enable_disable definition changed
+qos_record_details definition changed
+qos_record_enable_disable definition changed
+qos_store_details definition changed
+qos_store_enable_disable definition changed
+rdma_create only in image
+rdma_create_reply only in image
+rdma_delete only in image
+rdma_delete_reply only in image
+reset_fib only in file
+reset_fib_reply only in file
+set_arp_neighbor_limit only in file
+set_arp_neighbor_limit_reply only in file
+set_ip_flow_hash definition changed
+set_ipfix_exporter definition changed
+set_punt definition changed
+show_version_reply definition changed
+show_vpe_system_time_reply definition changed
+sockclnt_create definition changed
+sockclnt_create_reply definition changed
+sr_set_encap_hop_limit only in image
+sr_set_encap_hop_limit_reply only in image
+stn_add_del_rule definition changed
+stn_rules_details definition changed
+svs_details definition changed
+svs_enable_disable definition changed
+svs_route_add_del definition changed
+svs_table_add_del definition changed
+sw_if_l2tpv3_tunnel_details definition changed
+sw_interface_add_del_address definition changed
+sw_interface_add_del_mac_address only in image
+sw_interface_add_del_mac_address_reply only in image
+sw_interface_bond_details definition changed
+sw_interface_clear_stats definition changed
+sw_interface_details definition changed
+sw_interface_dump definition changed
+sw_interface_event definition changed
+sw_interface_get_mac_address definition changed
+sw_interface_get_mac_address_reply definition changed
+sw_interface_get_table definition changed
+sw_interface_ip6_enable_disable definition changed
+sw_interface_ip6_set_link_local_address definition changed
+sw_interface_ip6nd_ra_config definition changed
+sw_interface_ip6nd_ra_prefix definition changed
+sw_interface_lacp_details definition changed
+sw_interface_rx_placement_details definition changed
+sw_interface_rx_placement_dump definition changed
+sw_interface_set_bond_weight only in image
+sw_interface_set_bond_weight_reply only in image
+sw_interface_set_flags definition changed
+sw_interface_set_geneve_bypass definition changed
+sw_interface_set_gtpu_bypass definition changed
+sw_interface_set_ip_directed_broadcast definition changed
+sw_interface_set_mac_address definition changed
+sw_interface_set_mpls_enable definition changed
+sw_interface_set_mtu definition changed
+sw_interface_set_rx_mode definition changed
+sw_interface_set_rx_placement definition changed
+sw_interface_set_table definition changed
+sw_interface_set_unnumbered definition changed
+sw_interface_set_vxlan_gbp_bypass definition changed
+sw_interface_slave_details definition changed
+sw_interface_slave_dump definition changed
+sw_interface_tag_add_del definition changed
+sw_interface_tap_v2_details definition changed
+sw_interface_tap_v2_dump definition changed
+sw_interface_vhost_user_details definition changed
+sw_interface_vhost_user_dump definition changed
+sw_interface_virtio_pci_details definition changed
+syslog_get_sender_reply definition changed
+syslog_set_sender definition changed
+tap_create_v2 definition changed
+tap_create_v2_reply definition changed
+tap_delete_v2 definition changed
+tcp_configure_src_addresses definition changed
+tls_openssl_set_engine only in image
+tls_openssl_set_engine_reply only in image
+trace_plugin_msg_ids definition changed
+udp_encap_add definition changed
+udp_encap_details definition changed
+udp_ping_add_del definition changed
+udp_ping_export definition changed
+virtio_pci_create definition changed
+virtio_pci_create_reply definition changed
+virtio_pci_delete definition changed
+vmxnet3_create definition changed
+vmxnet3_create_reply definition changed
+vmxnet3_delete definition changed
+vmxnet3_details definition changed
+vxlan_gbp_tunnel_add_del definition changed
+vxlan_gbp_tunnel_add_del_reply definition changed
+vxlan_gbp_tunnel_details definition changed
+vxlan_gbp_tunnel_dump definition changed
+vxlan_gpe_ioam_enable definition changed
+vxlan_gpe_ioam_export_enable_disable definition changed
+vxlan_gpe_ioam_transit_disable definition changed
+vxlan_gpe_ioam_transit_enable definition changed
+vxlan_gpe_ioam_vni_disable definition changed
+vxlan_gpe_ioam_vni_enable definition changed
+want_bfd_events definition changed
+want_dhcp6_pd_reply_events definition changed
+want_ip4_arp_events only in file
+want_ip4_arp_events_reply only in file
+want_ip6_nd_events only in file
+want_ip6_nd_events_reply only in file
+want_ip6_ra_events definition changed
+want_ip_neighbor_events only in image
+want_ip_neighbor_events_reply only in image
+want_l2_arp_term_events only in image
+want_l2_arp_term_events_reply only in image
+want_l2_macs_events definition changed
+================================================ ==================
+
+Found 493 api message signature differences
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+``src/vlibmemory/memclnt.api``
+
+* `8e388390d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8e388390d>`_ vlib: use explicit types in api
+* `daa4bff16 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=daa4bff16>`_ api: memclnt api use string type.
+* `7adaa226e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7adaa226e>`_ api: revert use string type for strings in memclnt.api
+* `2959d42fe <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2959d42fe>`_ api: use string type for strings in memclnt.api
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+
+``src/examples/sample-plugin/sample/sample.api``
+
+* `33a58171e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=33a58171e>`_ api: autogenerate api trace print/endian
+* `78d91cf9a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=78d91cf9a>`_ sample-plugin: refactor .api to use explicit types
+
+``src/vnet/interface.api``
+
+* `418ebb711 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=418ebb711>`_ papi: support default for type alias decaying to basetype
+* `9485d99bd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9485d99bd>`_ interface: Allow VLAN tag-rewrite on non-sub-interfaces too.
+* `c12eae73f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c12eae73f>`_ interface: shmemioerror while getting name_filter arg
+* `de312c2d5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=de312c2d5>`_ interface: dump the interface device type
+* `e0792fdff <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e0792fdff>`_ interface: callback to manage extra MAC addresses
+* `75761b933 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75761b933>`_ api: split vl_api_prefix into two
+* `e5ff5a36d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e5ff5a36d>`_ api: enforce vla is last and fixed string type
+* `053204ab0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=053204ab0>`_ api: Cleanup APIs interface.api
+* `0ad4a439d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0ad4a439d>`_ Fix vpp crash bug while deleting dhcp client
+* `9a29f795a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9a29f795a>`_ vpp_papi_provider.py: update defautmapping.
+* `b8591ac91 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b8591ac91>`_ API sw_interface_dump: Dump all if index is zero
+* `4a7240636 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4a7240636>`_ Make sw_interface_dump more compatible with 2.2.0
+* `6407ba56a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6407ba56a>`_ api: Add to interface crud - read by sw_if_index.
+
+``src/vnet/interface_types.api``
+
+* `053204ab0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=053204ab0>`_ api: Cleanup APIs interface.api
+
+``src/vnet/session/session.api``
+
+* `c4c4cf506 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c4c4cf506>`_ session: move add/del segment msg to mq
+* `79f89537c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=79f89537c>`_ session: Add certificate store
+* `e5ff5a36d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e5ff5a36d>`_ api: enforce vla is last and fixed string type
+* `458089bba <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=458089bba>`_ session: move ctrl messages from bapi to mq
+* `8ac1d6d05 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8ac1d6d05>`_ session: Use parent_handle instead of transport_opts
+* `ba65ca496 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ba65ca496>`_ Add transport_opts to connect_sock bapi
+
+``src/vnet/classify/classify.api``
+
+* `692bfc85f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=692bfc85f>`_ classify: API cleanup
+
+``src/vnet/l2tp/l2tp.api``
+
+* `3ae526271 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3ae526271>`_ l2: l2tp API cleanup
+
+``src/vnet/gre/gre.api``
+
+* `5f8f61733 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5f8f61733>`_ gre: Multi-point interfaces
+* `814f15948 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=814f15948>`_ gre: update gre.api with explicit types
+* `d0aed2eb3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d0aed2eb3>`_ GRE: set gre_tunnel_type init value to zero in API
+* `5a8844bdb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5a8844bdb>`_ GRE: API update
+
+``src/vnet/fib/fib_types.api``
+
+* `1dbcf30b7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1dbcf30b7>`_ fib: Support the POP of a Psuedo Wire Control Word
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+
+``src/vnet/lisp-cp/one.api``
+
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+
+``src/vnet/lisp-cp/lisp.api``
+
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+
+``src/vnet/feature/feature.api``
+
+* `bf6c5c158 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bf6c5c158>`_ feature: API cleanup
+
+``src/vnet/nhrp/nhrp.api``
+
+* `5f8f61733 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5f8f61733>`_ gre: Multi-point interfaces
+
+``src/vnet/qos/qos.api``
+
+* `4b76c58be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4b76c58be>`_ qos: api clenup
+* `83832e7ce <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=83832e7ce>`_ qos: Store function
+* `5281a9029 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5281a9029>`_ qos: QoS dump APIs
+
+``src/vnet/ipsec/ipsec.api``
+
+* `dbf68c9aa <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dbf68c9aa>`_ ipsec: Changes to make ipsec encoder/decoders reusable by the plugins
+* `12989b538 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=12989b538>`_ ipsec: remove dedicated IPSec tunnels
+* `c87b66c86 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c87b66c86>`_ ipsec: ipsec-tun protect
+* `f2922422d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f2922422d>`_ ipsec: remove the set_key API
+* `80f6fd53f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=80f6fd53f>`_ IPSEC: Pass the algorithm salt (used in GCM) over the API
+
+``src/vnet/ipsec/ipsec_types.api``
+
+* `dbf68c9aa <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dbf68c9aa>`_ ipsec: Changes to make ipsec encoder/decoders reusable by the plugins
+
+``src/vnet/lisp-gpe/lisp_gpe.api``
+
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+
+``src/vnet/pci/pci_types.api``
+
+* `2c504f89c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2c504f89c>`_ devices: virtio API cleanup
+
+``src/vnet/bonding/bond.api``
+
+* `3d1ef873d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3d1ef873d>`_ bonding: API cleanup
+* `a1876b84e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a1876b84e>`_ bonding: add weight support for active-backup mode
+* `751e3f382 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=751e3f382>`_ bonding: add support for numa-only in lacp mode
+
+``src/vnet/tcp/tcp.api``
+
+* `956819afa <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=956819afa>`_ tcp: api clenup
+
+``src/vnet/cop/cop.api``
+
+* `aa4438a31 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=aa4438a31>`_ cop: API cleanup
+
+``src/vnet/ip-neighbor/ip_neighbor.api``
+
+* `cbe25aab3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cbe25aab3>`_ ip: Protocol Independent IP Neighbors
+
+``src/vnet/ethernet/p2p_ethernet.api``
+
+* `8edca1361 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8edca1361>`_ p2p ethernet: update p2p_ethernet.api with explicit types.
+
+``src/vnet/ethernet/ethernet_types.api``
+
+* `33a58171e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=33a58171e>`_ api: autogenerate api trace print/endian
+
+``src/vnet/vxlan-gbp/vxlan_gbp.api``
+
+* `fb27096ee <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fb27096ee>`_ vxlan-gbp: api cleanup
+
+``src/vnet/arp/arp.api``
+
+* `cbe25aab3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cbe25aab3>`_ ip: Protocol Independent IP Neighbors
+
+``src/vnet/ipip/ipip.api``
+
+* `9534696b4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9534696b4>`_ ipip: Tunnel flags controlling copying data to/from payload/encap
+* `288e09362 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=288e09362>`_ ipip: refactor ipip.api with explicit types
+* `cbd0824d6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cbd0824d6>`_ IPIP tunnel: use address types on API
+
+``src/vnet/ipip/ipip_types.api``
+
+* `9534696b4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9534696b4>`_ ipip: Tunnel flags controlling copying data to/from payload/encap
+
+``src/vnet/bfd/bfd.api``
+
+* `4682feb1f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4682feb1f>`_ bfd: API cleanup
+
+``src/vnet/l2/l2.api``
+
+* `cbe25aab3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cbe25aab3>`_ ip: Protocol Independent IP Neighbors
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `bc764c8bc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bc764c8bc>`_ l2: BD ARP termination entry API update
+* `54bc5e40c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=54bc5e40c>`_ Update API description
+* `5e6f7348c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5e6f7348c>`_ l2: Add support for arp unicast forwarding
+
+``src/vnet/ip6-nd/ip6_nd.api``
+
+* `cbe25aab3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cbe25aab3>`_ ip: Protocol Independent IP Neighbors
+
+``src/vnet/ip6-nd/rd_cp.api``
+
+* `cbe25aab3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cbe25aab3>`_ ip: Protocol Independent IP Neighbors
+
+``src/vnet/udp/udp.api``
+
+* `10dc2eabd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=10dc2eabd>`_ udp: fix copyright typo
+
+``src/vnet/mpls/mpls.api``
+
+* `3eb8f207b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3eb8f207b>`_ mpls: api cleanup
+* `75761b933 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75761b933>`_ api: split vl_api_prefix into two
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+
+``src/vnet/mfib/mfib_types.api``
+
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+
+``src/vnet/ip/ip_types.api``
+
+* `75761b933 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75761b933>`_ api: split vl_api_prefix into two
+* `33a58171e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=33a58171e>`_ api: autogenerate api trace print/endian
+* `515eed425 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=515eed425>`_ api: add prefix matcher typedef
+* `038e1dfbd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=038e1dfbd>`_ dhcp ip: DSCP settings for transmitted DHCP packets
+* `53c501512 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53c501512>`_ api: add DSCP definitions to ip_types.api
+* `ab05508e1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ab05508e1>`_ api: refactor format_vl_api_prefix_t return keys
+* `b538dd868 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b538dd868>`_ Punt: specify packets by IP protocol Type
+* `50f0ac0f0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=50f0ac0f0>`_ Punt: socket register for exception dispatched/punted packets based on reason
+
+``src/vnet/ip/punt.api``
+
+* `f158944cc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f158944cc>`_ ip: trivial typos in docs
+* `f72ad93d6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f72ad93d6>`_ ip: punt API cleanup
+* `e5ff5a36d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e5ff5a36d>`_ api: enforce vla is last and fixed string type
+* `719beb709 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=719beb709>`_ ip ipsec: Remove IPSec SPI-0 punt reason
+* `b538dd868 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b538dd868>`_ Punt: specify packets by IP protocol Type
+* `50f0ac0f0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=50f0ac0f0>`_ Punt: socket register for exception dispatched/punted packets based on reason
+
+``src/vnet/ip/ip.api``
+
+* `58989a37d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=58989a37d>`_ ip: API cleanup
+* `cbe25aab3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cbe25aab3>`_ ip: Protocol Independent IP Neighbors
+* `668605fc8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=668605fc8>`_ ip: use explicit types in api
+* `9db6ada77 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9db6ada77>`_ fib: Table Replace
+* `de34c35fc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=de34c35fc>`_ ip: add shallow virtual reassembly functionality
+* `75761b933 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75761b933>`_ api: split vl_api_prefix into two
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+* `3a343d42d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3a343d42d>`_ reassembly: prevent long chain attack
+
+``src/vnet/pg/pg.api``
+
+* `22e9cfd76 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=22e9cfd76>`_ pg: add GSO support
+
+``src/vnet/bier/bier.api``
+
+* `f1f5a8a1a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f1f5a8a1a>`_ bier: API cleanup
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+* `e6eefb6e3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e6eefb6e3>`_ Trivial Typo's in bier comments/docs.
+
+``src/vnet/ipfix-export/ipfix_export.api``
+
+* `2f71a8889 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2f71a8889>`_ ip: ipfix-export API update
+* `21b83e96d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=21b83e96d>`_ api: implement ipfix_flush
+
+``src/vnet/gso/gso.api``
+
+* `29467b534 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=29467b534>`_ gso: Add gso feature arc
+
+``src/vnet/devices/af_packet/af_packet.api``
+
+* `97c998c28 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=97c998c28>`_ docs: devices-- add FEATURES.yaml
+* `3b2db9002 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3b2db9002>`_ devices: af_packet API cleanup
+
+``src/vnet/devices/virtio/vhost_user.api``
+
+* `5d4c99f27 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5d4c99f27>`_ devices: vhost API cleanup
+* `4208a4ce8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4208a4ce8>`_ devices interface tests: vhosst GSO support
+
+``src/vnet/devices/virtio/virtio_types.api``
+
+* `5d4c99f27 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5d4c99f27>`_ devices: vhost API cleanup
+
+``src/vnet/devices/virtio/virtio.api``
+
+* `6d4af8918 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6d4af8918>`_ virtio: split gso and checksum offload functionality
+* `2c504f89c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2c504f89c>`_ devices: virtio API cleanup
+* `97c998c28 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=97c998c28>`_ docs: devices-- add FEATURES.yaml
+* `bbd6b746e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bbd6b746e>`_ virtio: Add gso support for native virtio driver
+* `43b512cac <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=43b512cac>`_ virtio: remove configurable queue size support
+
+``src/vnet/devices/pipe/pipe.api``
+
+* `97c998c28 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=97c998c28>`_ docs: devices-- add FEATURES.yaml
+* `df40cb5b5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df40cb5b5>`_ devices: pipe API cleanup
+
+``src/vnet/devices/tap/tapv2.api``
+
+* `ba0061feb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ba0061feb>`_ tap: split gso and checksum offload functionality
+* `5de4fb707 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5de4fb707>`_ devices: tap API cleanup
+* `44d06916b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=44d06916b>`_ tap: Move client registration check to top
+* `97c998c28 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=97c998c28>`_ docs: devices-- add FEATURES.yaml
+* `97d54ed43 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=97d54ed43>`_ tap: add support to configure tap interface host MTU size
+
+``src/vnet/srv6/sr.api``
+
+* `eeb5fb3a5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=eeb5fb3a5>`_ sr: add "set sr encaps hop-limit" command
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+
+``src/vnet/geneve/geneve.api``
+
+* `2d3282e17 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2d3282e17>`_ geneve: API cleanup
+
+``src/plugins/marvell/pp2/pp2.api``
+
+* `4a65b910a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4a65b910a>`_ marvell: use explicit types in api
+* `859b59133 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=859b59133>`_ api: Add API support for marvell PP2 plugin
+
+``src/plugins/svs/svs.api``
+
+* `5e913f374 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5e913f374>`_ svs: use explicit types in api
+
+``src/plugins/acl/acl_types.api``
+
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `bb2e5221a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bb2e5221a>`_ api acl: breakout acl_types.api for reuse by others
+
+``src/plugins/acl/acl.api``
+
+* `b5076cbe1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b5076cbe1>`_ acl: add missing square brackets to vat_help option in acl api
+* `709dad304 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=709dad304>`_ acl: remove api boilerplate
+* `bb2e5221a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bb2e5221a>`_ api acl: breakout acl_types.api for reuse by others
+* `f995c7122 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f995c7122>`_ acl: implement counters
+
+``src/plugins/memif/memif.api``
+
+* `3ae9f5a90 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3ae9f5a90>`_ memif: remove api boilerplate
+* `546f955b3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=546f955b3>`_ memif: API cleanup
+
+``src/plugins/cdp/cdp.api``
+
+* `07e557a73 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=07e557a73>`_ cdp: use explicit types in api
+
+``src/plugins/dhcp/dhcp6_ia_na_client_cp.api``
+
+* `02bfd641b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=02bfd641b>`_ dhcp: Move to plugin
+
+``src/plugins/dhcp/dhcp6_pd_client_cp.api``
+
+* `d5262831a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d5262831a>`_ dhcp: dhcp6_pd_client_cp API cleanup
+* `02bfd641b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=02bfd641b>`_ dhcp: Move to plugin
+
+``src/plugins/dhcp/dhcp.api``
+
+* `6bcc6a455 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bcc6a455>`_ dhcp: fix crash on unicast renewal send
+* `02bfd641b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=02bfd641b>`_ dhcp: Move to plugin
+
+``src/plugins/avf/avf.api``
+
+* `a0bf06d74 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a0bf06d74>`_ avf: explicit types in api
+* `74af6f081 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=74af6f081>`_ avf: remote api boilerplate
+
+``src/plugins/dpdk/api/dpdk.api``
+
+* `6d75c20a6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6d75c20a6>`_ dpdk: use explicit types in api
+* `025166dc7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=025166dc7>`_ dpdk: remove api boilerplate
+
+``src/plugins/builtinurl/builtinurl.api``
+
+* `43765e2b4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=43765e2b4>`_ builtinurl: initial working attempt
+
+``src/plugins/mactime/mactime.api``
+
+* `7b22df06f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7b22df06f>`_ mactime: update api to use explicit types
+* `2c41a61d5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2c41a61d5>`_ mactime: add a "top" command to watch device stats
+* `7071952df <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7071952df>`_ mactime: remove api boilerplate
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `7681b1c46 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7681b1c46>`_ mactime: add per-mac allow-with-quota feature
+* `0c6ac791d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0c6ac791d>`_ mactime: upstream new features
+
+``src/plugins/ikev2/ikev2.api``
+
+* `6aaee8c7c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6aaee8c7c>`_ ikev2: use explicit api types
+* `fc7b77db7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fc7b77db7>`_ ikev2: remove api boilerplate
+
+``src/plugins/http_static/http_static.api``
+
+* `e5ff5a36d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e5ff5a36d>`_ api: enforce vla is last and fixed string type
+* `68b24e2c9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=68b24e2c9>`_ plugins: http_static. Migrate to use api string type.
+* `22bc2c46e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=22bc2c46e>`_ Static http server
+
+``src/plugins/gbp/gbp.api``
+
+* `38277e407 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=38277e407>`_ gbp: use explicit types in api
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `3918bdbcb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3918bdbcb>`_ gbp: update gbp-ext-itf API
+* `3c0d84c98 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3c0d84c98>`_ gbp: add anonymous l3-out subnets
+* `cfc7a107e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cfc7a107e>`_ gbp: add anonymous l3-out external interfaces
+* `160c923f9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=160c923f9>`_ gbp: VRF scoped contracts
+
+``src/plugins/l2e/l2e.api``
+
+* `b2e463a10 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b2e463a10>`_ l2e: use explicit api types
+
+``src/plugins/gtpu/gtpu.api``
+
+* `55636cb62 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=55636cb62>`_ gtpu: use explicit types in api
+* `49228efce <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=49228efce>`_ gtpu: remove api boilerplate
+
+``src/plugins/igmp/igmp.api``
+
+* `4a7fc4cf1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4a7fc4cf1>`_ igmp: use explicit types in api
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `4ff09ae34 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4ff09ae34>`_ API: Python and Unix domain socket improvement
+
+``src/plugins/ioam/lib-vxlan-gpe/ioam_vxlan_gpe.api``
+
+* `0fa66d618 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0fa66d618>`_ ioam: use explicit api types
+
+``src/plugins/ioam/udp-ping/udp_ping.api``
+
+* `0fa66d618 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0fa66d618>`_ ioam: use explicit api types
+
+``src/plugins/ioam/export/ioam_export.api``
+
+* `0fa66d618 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0fa66d618>`_ ioam: use explicit api types
+
+``src/plugins/ioam/ip6/ioam_cache.api``
+
+* `0fa66d618 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0fa66d618>`_ ioam: use explicit api types
+
+``src/plugins/ioam/lib-pot/pot.api``
+
+* `0fa66d618 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0fa66d618>`_ ioam: use explicit api types
+
+``src/plugins/ioam/export-vxlan-gpe/vxlan_gpe_ioam_export.api``
+
+* `0fa66d618 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0fa66d618>`_ ioam: use explicit api types
+
+``src/plugins/stn/stn.api``
+
+* `7929f9f5c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7929f9f5c>`_ stn: use explicit types in api
+
+``src/plugins/map/map.api``
+
+* `be31c2a25 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=be31c2a25>`_ map: use explicit types in api
+* `7b2e9fb1a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7b2e9fb1a>`_ map: use ip6-full-reassembly instead of own code
+* `640edcd90 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=640edcd90>`_ map: use SVR for MAP-T
+* `e5ff5a36d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e5ff5a36d>`_ api: enforce vla is last and fixed string type
+* `ff47fb645 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ff47fb645>`_ vppapigen map: raise ValueError when fieldname is python keyword
+* `4d376f67a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4d376f67a>`_ map: Use vl_api_string macros.
+
+``src/plugins/oddbuf/oddbuf.api``
+
+* `7ff64fb97 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7ff64fb97>`_ oddbuf: remove api boilerplate
+* `a287a30dd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a287a30dd>`_ misc: fix coverity warning in the oddbuf plugin
+* `c4abafd83 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c4abafd83>`_ ip: fix udp/tcp checksum corner cases
+
+``src/plugins/l3xc/l3xc.api``
+
+* `60f5108a9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=60f5108a9>`_ l3xc: use explicit types in api
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `59fa121f8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=59fa121f8>`_ L3 cross connect
+
+``src/plugins/pppoe/pppoe.api``
+
+* `04338e85a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=04338e85a>`_ pppoe: use explicit types in api
+* `25fe57821 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=25fe57821>`_ pppoe: remove api boilerplate
+
+``src/plugins/mdata/mdata.api``
+
+* `d7b306657 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d7b306657>`_ mdata: buffer metadata change tracker plugin
+
+``src/plugins/lb/lb.api``
+
+* `ae0724034 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ae0724034>`_ lb: remove api boilerplate
+* `33538a150 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=33538a150>`_ lb: add APIs for set interface nat4 and nat6
+* `75761b933 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75761b933>`_ api: split vl_api_prefix into two
+* `3efcd0d7c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3efcd0d7c>`_ lb: vip and as dump/detail api's
+* `a0cb32cb9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a0cb32cb9>`_ lb: update api.c to use scaffolding from latest skel
+
+``src/plugins/lb/lb_types.api``
+
+* `75761b933 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=75761b933>`_ api: split vl_api_prefix into two
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `3efcd0d7c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3efcd0d7c>`_ lb: vip and as dump/detail api's
+
+``src/plugins/nsim/nsim.api``
+
+* `e06e7c672 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e06e7c672>`_ nsim: use explicit api types
+* `2e7a43ca4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2e7a43ca4>`_ nsim: remove api boilerplate
+* `7c91007e1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7c91007e1>`_ Make the loss / delay sim available as an output feature
+
+``src/plugins/vmxnet3/vmxnet3.api``
+
+* `277f03f06 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=277f03f06>`_ vmxnet3: use explicit types in api
+* `10bbfce02 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=10bbfce02>`_ vmxnet3: remove api boilerplate
+* `2985e0af6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2985e0af6>`_ vmxnet3: per interface gso support
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+
+``src/plugins/tlsopenssl/tls_openssl.api``
+
+* `1e582206a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1e582206a>`_ tls: remove api boilerplate
+* `dd0cc9ec3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dd0cc9ec3>`_ tls: some rework based on TLS openssl C API
+* `be4d1aa2c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=be4d1aa2c>`_ tls: Add C API for TLS openssl to set engine
+
+``src/plugins/abf/abf.api``
+
+* `bdde58534 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdde58534>`_ abf: use explicit types in api
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `097fa66b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=097fa66b9>`_ fib: fib api updates
+
+``src/plugins/nat/nat.api``
+
+* `f126e746f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f126e746f>`_ nat: use SVR
+* `e5ff5a36d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e5ff5a36d>`_ api: enforce vla is last and fixed string type
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `e6e09a4ac <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e6e09a4ac>`_ nat: elog rewrite for multi-worker support
+* `c1f93067e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c1f93067e>`_ Add default value for API Nat flags
+* `dd1e3e780 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dd1e3e780>`_ NAT: VPP-1531 api cleanup & update
+* `89fec713f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=89fec713f>`_ Revert "NAT: VPP-1531 api cleanup & update"
+* `bed1421b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bed1421b9>`_ NAT: VPP-1531 api cleanup & update
+
+``src/plugins/rdma/rdma.api``
+
+* `d8c1ef925 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d8c1ef925>`_ rdma: api: prepare support for direct verb
+* `b644eb54f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b644eb54f>`_ rdma: add explicit types in api
+* `812afe712 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=812afe712>`_ rdma: add rdma API
+
+``src/plugins/sctp/sctp.api``
+
+* `3ffe6cadf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3ffe6cadf>`_ sctp: move to plugins, disabled by default
+
+``src/plugins/ct6/ct6.api``
+
+* `d4efce2e0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d4efce2e0>`_ ct6: use explicit type in api
+* `ee98904e0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ee98904e0>`_ ct6: remove api boilerplate
+
+``src/plugins/nsh/nsh.api``
+
+* `d3f0a4869 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d3f0a4869>`_ nsh: use explicit api types
+
+``src/plugins/flowprobe/flowprobe.api``
+
+* `3013e6988 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3013e6988>`_ flowprobe: use explicit types in api
+* `2a1ca787b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2a1ca787b>`_ api: split api generated files
+
+``src/plugins/lacp/lacp.api``
+
+* `ebef4a9e5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ebef4a9e5>`_ lacp: use explit api types
+
+``src/plugins/dns/dns.api``
+
+* `b922f16ba <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b922f16ba>`_ dns: remove api boilerplate
+* `34af0ccf5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=34af0ccf5>`_ dns: make the dns name resolver a plugin
+
+``src/vpp/api/vpe.api``
+
+* `e5ff5a36d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e5ff5a36d>`_ api: enforce vla is last and fixed string type
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `a47a5f20a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a47a5f20a>`_ api papi: add alias for timestamp(datetime)/timedelta
+* `888640a39 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=888640a39>`_ map gbp papi: match endianess of f64
+* `03f1af23b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=03f1af23b>`_ api: Implement log_dump/log_details
+* `c87b66c86 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c87b66c86>`_ ipsec: ipsec-tun protect
+* `9ac113815 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9ac113815>`_ API: Add support for limits to language.
+
+``src/vpp/api/vpe_types.api``
+
+* `e71748291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e71748291>`_ vppapigen: remove support for legacy typedefs
+* `a47a5f20a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a47a5f20a>`_ api papi: add alias for timestamp(datetime)/timedelta
+* `3cf9e67f5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3cf9e67f5>`_ api: add vl_api_version_t type
diff --git a/docs/aboutvpp/releasenotes/v20.05.1.rst b/docs/aboutvpp/releasenotes/v20.05.1.rst
new file mode 100644
index 0000000..7a8f650
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v20.05.1.rst
@@ -0,0 +1,9 @@
+Release notes for VPP 20.05.1
+=============================
+
+This is bug fix release.
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/2005>`__
diff --git a/docs/aboutvpp/releasenotes/v20.05.rst b/docs/aboutvpp/releasenotes/v20.05.rst
new file mode 100644
index 0000000..a510bf5
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v20.05.rst
@@ -0,0 +1,848 @@
+Release notes for VPP 20.05
+===========================
+
+More than 751 commits since the 20.01 release.
+
+Release Highlights
+------------------
+
+Feature Highlights
+~~~~~~~~~~~~~~~~~~
+
+As per commits involving FEATURE.yaml edits between the previous release
+and this release. They are mentioned in the below “features” section as
+well, together with the corresponding commits.
+
+- TAP Drivers
+
+ - Implement sw_interface_tap_v2_dump filtering by sw_if_index
+ - Add support for persistence
+
+- Native Virtio Drivers
+
+ - Support virtio 1.1 packed ring in vhost
+
+- gso
+
+ - Add support for IP-IP
+ - Add vxlan tunnel support
+
+- VRRP
+
+ - Add plugin providing VRRP support
+
+Ongoing Work On More Semantic-Typed API
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This release, like the 20.01, continues the journey on defining the
+semantic-based types instead of storage-based types within the API, so
+you may have noticed this in the API changes.
+
+Some of the changes are related to the infrastructure, and may be
+bugfixes, they do not change the CRC of the message but affect the
+representation on the wire. One particular commit we want you to pay
+attention to, is
+`b5c0d35f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b5c0d35f>`__,
+which fixes the bug with the enum representation on the wire - before
+it, even the enums declared as u8 or u16 were represented as u32 in the
+API messages.
+
+Another important commit we would like to call out explicitly as well is
+`7dd63e5c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7dd63e5c>`__,
+which pinned the address_family and ip_proto enum types to be u8 instead
+of the default u32.
+
+The above two commits will be primarily interesting for those who work
+with the low-level APIs on VPP - the API frameworks should make these
+under-the-hood changes transparent. However, we decided to call these
+out, given that for those affected these will be pretty important
+changes.
+
+Another commit, that does not have the immediate impact at the moment,
+but that is poised to improve the user interaction with the API is
+`5c318c70 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5c318c70>`__.
+This adds the tooling and ability to implement a structured process, by
+which the API messages can evolve, while minimizing the impact to the
+API users.
+
+Features
+--------
+
+- Binary API Compiler for Python
+
+ - Api crc checker (5c318c70d)
+
+- Binary API Libraries
+
+ - Add macro that zeros out api reply buffer (f24de1795)
+
+- Build System
+
+ - Add snap packaging (experimental) (6d97e62c0)
+ - Support arch-specific compiling for Neoverse N1 (690ce8672)
+
+- Crypto native Plugin
+
+ - Add ARMv8 AES-CBC implementation (776644efe)
+ - Add AArch64 AES-GCM native implementation (622b5ce61)
+ - Calculate ghash using vpclmulqdq instructions (627fb6a16)
+ - GCM implementation with vector AESNI instructions (47d8f5dcd)
+
+- Infrastructure Library
+
+ - Add x86 CPU definitions (38e0413b2)
+ - Numa vector placement support (a690fdbfe)
+ - Add cmake option to grow vectors by 1 (98bd75778)
+ - Add tw_timer_2t_2w_512sl variant (907678977)
+
+- Link Bonding
+
+ - Add GSO support (2e1fa54b7)
+
+- Plugins
+
+ - DPDK
+
+ - Output switch information (2347278d9)
+ - Use port_id as interface name suffix for representors
+ (a80f8f371)
+ - Add iova-mode to startup (4e96ddaec)
+ - Bump DPDK version to 20.02 (76be887d8)
+ - Enable DPDK iAVF PMD (162ea767c)
+ - DPDK 20.05 iavf flow director backporting to DPDK 20.02
+ (7f83738b4)
+
+ - GTPU
+
+ - Offload RX flow (00fdf53c7)
+ - RX offload for IPv6 payload supporting (ed63a0ff7)
+
+ - Host Stack Applications
+
+ - Proxy rcv wnd update acks after full fifos (dda2dbeda)
+
+ - IPv6 Segment Routing Mobile
+
+ - Support GTP4/6.DT and User Plane message mapping (9e722bd46)
+
+ - Internet Key Exchange (IKEv2) Protocol
+
+ - Configure a profile with an existing interface (44476c6b2)
+ - Responder honours the protected tunnel config (685001f0a)
+ - Add support for custom ipsec-over-udp port (e5d34919b)
+ - Dead peer detection (c415d0a8e)
+
+ - NAT
+
+ - In2out-output nodes work with acl reflect (d539e256b)
+ - Api & cli command for forcing session cleanup (edf777272)
+ - Dslite ce mode in separate config entry (958919f36)
+
+ - QUIC protocol
+
+ - Update quicly to v0.0.10-VPP (62b1cea6e)
+ - Quicly crypto offloading (92de6b65b)
+ - Check quicly version tag at compile time (ffdc72da4)
+
+ - RDMA (ibverb) driver
+
+ - Bunp rdma-core version to v28.0 (eb89b9093)
+ - Add Mellanox mlx5 Direct Verbs receive support (dd648aac0)
+ - Introduce direct verb for Cx4/5 tx (dc812d9a7)
+
+ - Unicast Reverse Path forwarding
+
+ - Unicast reverse Path Forwarding (plugin) (d724e4f43)
+
+ - VRRP
+
+ - Add plugin providing vrrp support (39e9428b9)
+
+- SVM Library
+
+ - Numa awareness for ssvm segments (6fe8998fe)
+ - Support multi-chunk fifo chunk alloc (8e755a16a)
+ - Chunk alloc stats (d35887297)
+ - New FIFO design/architecture (f22f4e562)
+ - Fifo test (64e96613d)
+
+- Test Infrastructure
+
+ - Add running_gcov_tests to framework.py (d498c9eb2)
+ - Implement ipaddress convenience methods (e64e5fff4)
+
+- VNET
+
+ - Crypto Infra
+
+ - Add chained buffer support in ipsecmb (AES-GCM) (2fc409131)
+ - Add support for testing quad loops in crypto algos (a9075dcf6)
+ - Introduce async crypto infra (f539578ba)
+
+ - Ethernet
+
+ - Configure system default ethernet MTU (5fa452554)
+
+ - FLOW
+
+ - Add vlan tagged types for IPv4/IPv6 5-tuple flows (f13830ce7)
+ - Add RSS support (24e2c50bf)
+ - Add l2tpv3oip flow (8b43aaaf1)
+
+ - GRE
+
+ - Tunnel encap/decap flags (e5b94dded)
+
+ - GSO
+
+ - Add vxlan tunnel support (0b04209ed)
+ - Add support for IP-IP (84f91fa9c)
+
+ - IP Neighbors
+
+ - Populate neighbor age via API (9c1928f81)
+ - Replace feature for the ip-neighbor data-base (c87fbb417)
+ - Add flush API (240dcb24a)
+
+ - IPIP
+
+ - Multi-point interface (14053c9db)
+
+ - IPSec
+
+ - Add support for chained buffers (efcad1a9d)
+ - IPSec protection for multi-point tunnel interfaces (282872127)
+ - Add input node bypass/discard functionality (0546483ce)
+ - User can choose the UDP source port (abc5660c6)
+ - Support 4o6 and 6o4 for SPD tunnel mode SAs (b1fd80f09)
+
+ - IPv4 LPM
+
+ - More detailed show reassembly commands (a877cf9f3)
+ - Replace Sematics for Interface IP addresses (59f71132e)
+
+ - MPLS
+
+ - Add user defined name tag to mpls tunnels (39ae0a07a)
+
+ - Native Virtio Drivers
+
+ - Support virtio 1.1 packed ring in vhost (bc0d9ff67)
+
+ - Packet Generator
+
+ - Set vnet buffer flags in pg streams (08eb2bb20)
+
+ - Segment Routing (IPv6 and MPLS)
+
+ - Change the CLI keyword from address to prefix. (b24e287b9)
+ - Support uSID function. (ec9cb9668)
+
+ - Session Layer
+
+ - Tracking segment memory usage (234fe894d)
+ - Basic fifo-tuning-logic (d8f48e216)
+ - Api to add new transport types (07063b8ea)
+ - Support connect on listeners (0a1e183e5)
+ - Adding debug events (7357043d2)
+ - Add option to preallocate fifo headers (9845c20d7)
+
+ - TAP Drivers
+
+ - Add support for persistance (b49bc1ae6)
+ - Add initial support for tun (206acf84d)
+ - Implement sw_interface_tap_v2_dump filtering by sw_if_index
+ (073d74d0b)
+
+ - TCP
+
+ - Add option to avoid endpoint cleanup (43818c1e0)
+ - Minimal set of worker stats (5e6305fb0)
+ - Allow custom mss on connects (ff19e3bf4)
+
+ - TLS and TLS engine plugins
+
+ - Picotls engine symmetric crypto enhancement by VPP crypto
+ framework (3b8518164)
+
+ - UDP
+
+ - Track connection port sharing (a039620c2)
+
+- VPP Comms Library
+
+ - Udp session migration notifications (68b7e5888)
+ - Propagate cleanup notifications to apps (9ace36d0f)
+
+- Vector Library
+
+ - Add plugin override support (8dc954a4e)
+ - Calculate per-worker loops/second metric (000a029e4)
+ - Leave SIGPROF signal with its default handler (6f533d780)
+ - Add nosyslog unix option (e31820af1)
+
+- Gomemif
+
+ - Introduce gomemif (07363a45f)
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Fixed issues
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/2005>`__
+
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+- *Only in image*: indicates the API is new for this release.
+- *Only in file*: indicates the API has been removed in this release.
+
+======================================== ==================
+Message Name Result
+======================================== ==================
+acl_add_replace definition changed
+acl_details definition changed
+acl_interface_add_del definition changed
+acl_interface_etype_whitelist_details definition changed
+acl_interface_etype_whitelist_dump definition changed
+acl_interface_list_details definition changed
+acl_interface_list_dump definition changed
+acl_interface_set_acl_list definition changed
+acl_interface_set_etype_whitelist definition changed
+add_node_next definition changed
+app_attach definition changed
+app_attach_reply definition changed
+app_cut_through_registration_add only in file
+app_cut_through_registration_add_reply only in file
+app_namespace_add_del definition changed
+app_worker_add_del definition changed
+app_worker_add_del_reply definition changed
+application_attach only in file
+application_attach_reply only in file
+bd_ip_mac_add_del definition changed
+bind_sock only in file
+bind_sock_reply only in file
+bind_uri only in file
+bind_uri_reply only in file
+bridge_domain_add_del definition changed
+bridge_domain_details definition changed
+bridge_domain_dump definition changed
+bridge_flags definition changed
+bvi_create_reply definition changed
+bvi_delete definition changed
+connect_sock only in file
+connect_sock_reply only in file
+connect_uri only in file
+connect_uri_reply only in file
+create_vhost_user_if definition changed
+disconnect_session only in file
+disconnect_session_reply only in file
+get_next_index definition changed
+get_node_index definition changed
+gpe_add_del_fwd_entry definition changed
+gpe_add_del_iface definition changed
+gpe_add_del_native_fwd_rpath definition changed
+gpe_enable_disable definition changed
+gpe_fwd_entries_get_reply definition changed
+gpe_fwd_entry_path_details definition changed
+gpe_native_fwd_rpaths_get definition changed
+gpe_native_fwd_rpaths_get_reply definition changed
+gpe_set_encap_mode definition changed
+gre_tunnel_add_del definition changed
+gre_tunnel_details definition changed
+gtpu_offload_rx only in image
+gtpu_offload_rx_reply only in image
+ikev2_profile_set_ipsec_udp_port only in image
+ikev2_profile_set_ipsec_udp_port_reply only in image
+ikev2_profile_set_liveness only in image
+ikev2_profile_set_liveness_reply only in image
+ikev2_profile_set_udp_encap only in image
+ikev2_profile_set_udp_encap_reply only in image
+ikev2_set_local_key definition changed
+ikev2_set_tunnel_interface only in image
+ikev2_set_tunnel_interface_reply only in image
+ip_neighbor_details definition changed
+ip_neighbor_flush only in image
+ip_neighbor_flush_reply only in image
+ip_neighbor_replace_begin only in image
+ip_neighbor_replace_begin_reply only in image
+ip_neighbor_replace_end only in image
+ip_neighbor_replace_end_reply only in image
+ip_route_lookup only in image
+ip_route_lookup_reply only in image
+ip_source_check_interface_add_del only in file
+ip_source_check_interface_add_del_reply only in file
+ipfix_classify_table_add_del definition changed
+ipfix_classify_table_details definition changed
+ipip_add_tunnel definition changed
+ipip_tunnel_details definition changed
+ipsec_backend_details definition changed
+ipsec_interface_add_del_spd definition changed
+ipsec_sa_details definition changed
+ipsec_sad_entry_add_del definition changed
+ipsec_select_backend definition changed
+ipsec_spd_add_del definition changed
+ipsec_spd_details definition changed
+ipsec_spd_entry_add_del definition changed
+ipsec_spd_interface_details definition changed
+ipsec_tunnel_if_add_del definition changed
+ipsec_tunnel_if_add_del_reply definition changed
+ipsec_tunnel_if_set_sa definition changed
+ipsec_tunnel_protect_del definition changed
+ipsec_tunnel_protect_details definition changed
+ipsec_tunnel_protect_update definition changed
+l2_fib_table_details definition changed
+l2_flags definition changed
+l2_interface_efp_filter definition changed
+l2_interface_pbb_tag_rewrite definition changed
+l2_interface_vlan_tag_rewrite definition changed
+l2_macs_event definition changed
+l2_patch_add_del definition changed
+l2_xconnect_details definition changed
+l2fib_add_del definition changed
+l2fib_flush_int definition changed
+lisp_add_del_adjacency definition changed
+lisp_add_del_local_eid definition changed
+lisp_add_del_locator definition changed
+lisp_add_del_locator_set definition changed
+lisp_add_del_map_request_itr_rlocs definition changed
+lisp_add_del_map_resolver definition changed
+lisp_add_del_map_server definition changed
+lisp_add_del_remote_mapping definition changed
+lisp_adjacencies_get_reply definition changed
+lisp_eid_table_add_del_map definition changed
+lisp_eid_table_details definition changed
+lisp_eid_table_dump definition changed
+lisp_eid_table_map_dump definition changed
+lisp_enable_disable definition changed
+lisp_get_map_request_itr_rlocs_reply definition changed
+lisp_locator_details definition changed
+lisp_locator_dump definition changed
+lisp_locator_set_details definition changed
+lisp_locator_set_dump definition changed
+lisp_map_register_enable_disable definition changed
+lisp_map_request_mode definition changed
+lisp_map_resolver_details definition changed
+lisp_map_server_details definition changed
+lisp_pitr_set_locator_set definition changed
+lisp_rloc_probe_enable_disable definition changed
+lisp_use_petr definition changed
+lldp_config definition changed
+macip_acl_add definition changed
+macip_acl_add_replace definition changed
+macip_acl_details definition changed
+macip_acl_interface_add_del definition changed
+macip_acl_interface_list_details definition changed
+macip_acl_interface_list_dump definition changed
+map_another_segment only in file
+map_another_segment_reply only in file
+modify_vhost_user_if definition changed
+mpls_tunnel_add_del definition changed
+mpls_tunnel_details definition changed
+nat44_del_user only in image
+nat44_del_user_reply only in image
+nat44_session_cleanup only in image
+nat44_session_cleanup_reply only in image
+nat44_set_session_limit only in image
+nat44_set_session_limit_reply only in image
+nat_show_config_reply definition changed
+netmap_create only in file
+netmap_create_reply only in file
+netmap_delete only in file
+netmap_delete_reply only in file
+nhrp_details only in file
+nhrp_dump only in file
+nhrp_entry_add_del only in file
+nhrp_entry_add_del_reply only in file
+one_add_del_adjacency definition changed
+one_add_del_l2_arp_entry definition changed
+one_add_del_local_eid definition changed
+one_add_del_locator definition changed
+one_add_del_locator_set definition changed
+one_add_del_map_request_itr_rlocs definition changed
+one_add_del_map_resolver definition changed
+one_add_del_map_server definition changed
+one_add_del_ndp_entry definition changed
+one_add_del_remote_mapping definition changed
+one_adjacencies_get_reply definition changed
+one_eid_table_add_del_map definition changed
+one_eid_table_details definition changed
+one_eid_table_dump definition changed
+one_eid_table_map_dump definition changed
+one_enable_disable definition changed
+one_enable_disable_petr_mode definition changed
+one_enable_disable_pitr_mode definition changed
+one_enable_disable_xtr_mode definition changed
+one_get_map_request_itr_rlocs_reply definition changed
+one_l2_arp_entries_get_reply definition changed
+one_locator_details definition changed
+one_locator_dump definition changed
+one_locator_set_details definition changed
+one_locator_set_dump definition changed
+one_map_register_enable_disable definition changed
+one_map_request_mode definition changed
+one_map_resolver_details definition changed
+one_map_server_details definition changed
+one_ndp_entries_get_reply definition changed
+one_nsh_set_locator_set definition changed
+one_pitr_set_locator_set definition changed
+one_rloc_probe_enable_disable definition changed
+one_show_petr_mode_reply definition changed
+one_show_pitr_mode_reply definition changed
+one_show_xtr_mode_reply definition changed
+one_stats_details definition changed
+one_stats_enable_disable definition changed
+one_use_petr definition changed
+pg_capture definition changed
+pg_create_interface definition changed
+pg_create_interface_reply definition changed
+pg_enable_disable definition changed
+policer_add_del definition changed
+policer_details definition changed
+policer_dump definition changed
+session_enable_disable definition changed
+session_rule_add_del definition changed
+session_rules_details definition changed
+show_lisp_map_register_state_reply definition changed
+show_lisp_map_request_mode_reply definition changed
+show_lisp_pitr_reply definition changed
+show_lisp_rloc_probe_state_reply definition changed
+show_lisp_status_reply definition changed
+show_lisp_use_petr_reply definition changed
+show_one_map_register_state_reply definition changed
+show_one_map_request_mode_reply definition changed
+show_one_nsh_mapping_reply definition changed
+show_one_pitr_reply definition changed
+show_one_rloc_probe_state_reply definition changed
+show_one_stats_enable_disable_reply definition changed
+show_one_status_reply definition changed
+show_one_use_petr_reply definition changed
+show_threads_reply definition changed
+sr_localsid_add_del definition changed
+sr_localsids_details definition changed
+sr_mpls_policy_add definition changed
+sr_mpls_policy_assign_endpoint_color definition changed
+sr_mpls_policy_mod definition changed
+sr_mpls_steering_add_del definition changed
+sr_policies_details definition changed
+sr_policy_add definition changed
+sr_policy_del definition changed
+sr_policy_mod definition changed
+sr_set_encap_source definition changed
+sr_steering_add_del definition changed
+sr_steering_pol_details definition changed
+sw_interface_address_replace_begin only in image
+sw_interface_address_replace_begin_reply only in image
+sw_interface_address_replace_end only in image
+sw_interface_address_replace_end_reply only in image
+sw_interface_set_l2_bridge definition changed
+sw_interface_set_l2_xconnect definition changed
+sw_interface_set_lldp definition changed
+sw_interface_set_vpath definition changed
+sw_interface_set_vxlan_bypass definition changed
+sw_interface_set_vxlan_gpe_bypass definition changed
+sw_interface_span_details definition changed
+sw_interface_span_dump definition changed
+sw_interface_span_enable_disable definition changed
+teib_details only in image
+teib_dump only in image
+teib_entry_add_del only in image
+teib_entry_add_del_reply only in image
+unbind_sock only in file
+unbind_sock_reply only in file
+unbind_uri only in file
+unbind_uri_reply only in file
+unmap_segment only in file
+unmap_segment_reply only in file
+urpf_update only in image
+urpf_update_reply only in image
+vrrp_vr_add_del only in image
+vrrp_vr_add_del_reply only in image
+vrrp_vr_details only in image
+vrrp_vr_dump only in image
+vrrp_vr_peer_details only in image
+vrrp_vr_peer_dump only in image
+vrrp_vr_set_peers only in image
+vrrp_vr_set_peers_reply only in image
+vrrp_vr_start_stop only in image
+vrrp_vr_start_stop_reply only in image
+vrrp_vr_track_if_add_del only in image
+vrrp_vr_track_if_add_del_reply only in image
+vrrp_vr_track_if_details only in image
+vrrp_vr_track_if_dump only in image
+vxlan_add_del_tunnel definition changed
+vxlan_add_del_tunnel_reply definition changed
+vxlan_gpe_add_del_tunnel definition changed
+vxlan_gpe_add_del_tunnel_reply definition changed
+vxlan_gpe_tunnel_details definition changed
+vxlan_gpe_tunnel_dump definition changed
+vxlan_offload_rx definition changed
+vxlan_tunnel_details definition changed
+vxlan_tunnel_dump definition changed
+======================================== ==================
+
+Found 279 api message signature differences
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``extras/deprecated/dpdk-hqos/api/dpdk.api``
+
+* `548d70de6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=548d70de6>`_ misc: deprecate dpdk hqos
+
+``extras/deprecated/netmap/netmap.api``
+
+* `7db6ab03d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7db6ab03d>`_ misc: deprecate netmap and ixge drivers
+
+``src/vpp/api/vpe.api``
+
+* `933fcf489 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=933fcf489>`_ api: API cleanup
+* `7db6ab03d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7db6ab03d>`_ misc: deprecate netmap and ixge drivers
+
+``src/vnet/tunnel/tunnel_types.api``
+
+* `14053c9db <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=14053c9db>`_ ipip: Multi-point interface
+* `59ff918ea <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=59ff918ea>`_ tunnel: Common types for IP tunnels
+
+``src/vnet/policer/policer_types.api``
+
+* `cd01fb423 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cd01fb423>`_ policer: API cleanup
+
+``src/vnet/policer/policer.api``
+
+* `cd01fb423 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cd01fb423>`_ policer: API cleanup
+
+``src/vnet/lisp-gpe/lisp_gpe.api``
+
+* `58db6e16c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=58db6e16c>`_ lisp: API cleanup
+
+``src/vnet/teib/teib.api``
+
+* `03ce46219 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=03ce46219>`_ teib: Rename NHRP to TEIB
+
+``src/vnet/ip-neighbor/ip_neighbor.api``
+
+* `240dcb24a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=240dcb24a>`_ ip-neighbor: Add flush API
+* `e64e5fff4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e64e5fff4>`_ tests: implement ipaddress convenience methods
+* `c87fbb417 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c87fbb417>`_ ip-neighbor: Replace feature for the ip-neighbor data-base
+* `8e7fdddd3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8e7fdddd3>`_ ip-neighbor: add description to the age parameter
+* `9c1928f81 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9c1928f81>`_ ip-neighbor: populate neighbor age via API
+
+``src/vnet/session/session.api``
+
+* `6fdd7a5f7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6fdd7a5f7>`_ session: improve .api comments slightly
+* `9845c20d7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9845c20d7>`_ session: add option to preallocate fifo headers
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+* `256779c85 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=256779c85>`_ udp: remove connected udp transport proto
+* `888d9f05e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=888d9f05e>`_ session: remove obsolete apis
+* `07063b8ea <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=07063b8ea>`_ session: api to add new transport types
+* `b4e5e50fe <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b4e5e50fe>`_ session: API cleanup
+* `2de9c0f92 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2de9c0f92>`_ svm: minimal initial fifo
+
+``src/vnet/interface_types.api``
+
+* `c4ae0fffb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c4ae0fffb>`_ interface: fix interface_types.api enums
+
+``src/vnet/vxlan/vxlan.api``
+
+* `7c0eb56f4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7c0eb56f4>`_ vxlan: vxlan/vxlan.api API cleanup
+
+``src/vnet/vxlan-gbp/vxlan_gbp.api``
+
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+
+``src/vnet/gre/gre.api``
+
+* `48ac1c2b2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=48ac1c2b2>`_ gre: improve .api descriptions
+* `8ab4e507c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8ab4e507c>`_ gre: add missing .api edits
+* `e5b94dded <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e5b94dded>`_ gre: Tunnel encap/decap flags
+* `59ff918ea <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=59ff918ea>`_ tunnel: Common types for IP tunnels
+
+``src/vnet/span/span.api``
+
+* `908965db7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=908965db7>`_ span: API cleanup
+
+``src/vnet/srv6/sr.api``
+
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+* `0938eba15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0938eba15>`_ sr: srv6 API cleanup
+* `79bfd2725 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=79bfd2725>`_ sr: SRv6 uN behavior
+
+``src/vnet/srv6/sr_types.api``
+
+* `0938eba15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0938eba15>`_ sr: srv6 API cleanup
+
+``src/vnet/pg/pg.api``
+
+* `db86329ab <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=db86329ab>`_ pg: API cleanup
+
+``src/vnet/l2/l2.api``
+
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+* `145e330f0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=145e330f0>`_ l2: API cleanup
+
+``src/vnet/lldp/lldp.api``
+
+* `1c684f9af <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1c684f9af>`_ lldp: API cleanup
+
+``src/vnet/vxlan-gpe/vxlan_gpe.api``
+
+* `1c2002a31 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1c2002a31>`_ vxlan: vxlan-gpe/vxlan-gpe.cpi API cleanup
+
+``src/vnet/lisp-cp/one.api``
+
+* `58db6e16c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=58db6e16c>`_ lisp: API cleanup
+
+``src/vnet/lisp-cp/lisp_types.api``
+
+* `58db6e16c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=58db6e16c>`_ lisp: API cleanup
+
+``src/vnet/lisp-cp/lisp.api``
+
+* `58db6e16c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=58db6e16c>`_ lisp: API cleanup
+
+``src/vnet/devices/tap/tapv2.api``
+
+* `d88fc0fce <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d88fc0fce>`_ tap: refactor existing flags
+* `073d74d0b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=073d74d0b>`_ tap: implement sw_interface_tap_v2_dump filtering by sw_if_index
+* `206acf84d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=206acf84d>`_ tap: add initial support for tun
+* `b49bc1ae6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b49bc1ae6>`_ tap: add support for persistance
+
+``src/vnet/devices/virtio/vhost_user.api``
+
+* `bc0d9ff67 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bc0d9ff67>`_ virtio: support virtio 1.1 packed ring in vhost
+
+``src/vnet/devices/virtio/virtio.api``
+
+* `53f06a014 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53f06a014>`_ vlib: move pci api types from vnet/pci to vlib/pci
+
+``src/vnet/ipsec/ipsec_types.api``
+
+* `abc5660c6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=abc5660c6>`_ ipsec: User can choose the UDP source port
+* `287d5e109 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=287d5e109>`_ ipsec: API cleanup
+* `5893747d7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5893747d7>`_ api: ipsec: add missing IS_INBOUND flag.
+* `2fcd265d3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2fcd265d3>`_ ipsec: Revert API cleanup
+* `666ece35c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=666ece35c>`_ ipsec: API cleanup
+
+``src/vnet/ipsec/ipsec.api``
+
+* `48d32b43c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=48d32b43c>`_ ipsec: provide stat index in sa details
+* `287d5e109 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=287d5e109>`_ ipsec: API cleanup
+* `2fcd265d3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2fcd265d3>`_ ipsec: Revert API cleanup
+* `666ece35c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=666ece35c>`_ ipsec: API cleanup
+* `282872127 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=282872127>`_ ipsec: IPSec protection for multi-point tunnel interfaces
+
+``src/vnet/ethernet/p2p_ethernet.api``
+
+* `bdfe5955f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bdfe5955f>`_ ethernet: add sanity checks to p2p_ethernet_add/del
+
+``src/vnet/bonding/bond.api``
+
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+
+``src/vnet/mpls/mpls.api``
+
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+* `39ae0a07a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=39ae0a07a>`_ mpls: add user defined name tag to mpls tunnels
+
+``src/vnet/syslog/syslog.api``
+
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+
+``src/vnet/interface.api``
+
+* `59f71132e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=59f71132e>`_ ip: Replace Sematics for Interface IP addresses
+
+``src/vnet/ipip/ipip.api``
+
+* `14053c9db <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=14053c9db>`_ ipip: Multi-point interface
+* `59ff918ea <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=59ff918ea>`_ tunnel: Common types for IP tunnels
+
+``src/vnet/srmpls/sr_mpls.api``
+
+* `0938eba15 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0938eba15>`_ sr: srv6 API cleanup
+* `00ec4019b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=00ec4019b>`_ sr: API cleanup
+
+``src/vnet/ip/ip.api``
+
+* `f5d38e05a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f5d38e05a>`_ api: ip: add IP_ROUTE_LOOKUP API
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+* `d724e4f43 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d724e4f43>`_ urpf: Unicast reverse Path Forwarding (plugin)
+
+``src/vnet/ip/ip_types.api``
+
+* `164c44f0b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=164c44f0b>`_ ip: Fix the AH/ESP protocol numbers on the API
+* `7dd63e5cc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7dd63e5cc>`_ ip: change ip API enums address_family and ip_proto size to u8
+* `3ec09e924 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3ec09e924>`_ ip: ip_address_t uses ip46_address_t
+
+``src/plugins/map/map.api``
+
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+
+``src/plugins/ikev2/ikev2.api``
+
+* `933c4ca5a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=933c4ca5a>`_ ikev2: fix string in api
+* `59fea5a6a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=59fea5a6a>`_ ikev2: make liveness params configurable
+* `8ceb44a89 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8ceb44a89>`_ ikev2: fix typo in .api description
+* `e5d34919b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e5d34919b>`_ ikev2: add support for custom ipsec-over-udp port
+* `b29d523af <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b29d523af>`_ ikev2: make UDP encap flag configurable
+* `44476c6b2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=44476c6b2>`_ ikev2: Configure a profile with an existing interface
+
+``src/plugins/urpf/urpf.api``
+
+* `d724e4f43 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d724e4f43>`_ urpf: Unicast reverse Path Forwarding (plugin)
+
+``src/plugins/lb/lb.api``
+
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+
+``src/plugins/gtpu/gtpu.api``
+
+* `00fdf53c7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=00fdf53c7>`_ gtpu: offload RX flow
+
+``src/plugins/acl/acl_types.api``
+
+* `2f8cd9145 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2f8cd9145>`_ acl: API cleanup
+* `492a5d0bd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=492a5d0bd>`_ acl: revert acl: api cleanup
+* `aad1ee149 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=aad1ee149>`_ acl: API cleanup
+
+``src/plugins/acl/acl.api``
+
+* `c0e9441e7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0e9441e7>`_ tests: move defaults from defaultmapping to .api files
+* `2f8cd9145 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2f8cd9145>`_ acl: API cleanup
+* `492a5d0bd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=492a5d0bd>`_ acl: revert acl: api cleanup
+* `aad1ee149 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=aad1ee149>`_ acl: API cleanup
+
+``src/plugins/nat/dslite/dslite.api``
+
+* `2c6639c69 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2c6639c69>`_ nat: move dslite to separate sub-plugin
+
+``src/plugins/nat/nat.api``
+
+* `6bb080f1e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bb080f1e>`_ nat: per vrf session limits
+* `61717cc38 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=61717cc38>`_ nat: use correct data types for memory sizes
+* `98301bd56 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=98301bd56>`_ nat: user deletion function & extra metrics
+* `edf777272 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=edf777272>`_ nat: api & cli command for forcing session cleanup
+* `2c6639c69 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2c6639c69>`_ nat: move dslite to separate sub-plugin
+
+``src/plugins/vrrp/vrrp.api``
+
+* `3fccd0278 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3fccd0278>`_ vrrp: do not define _details as autoreply
+* `39e9428b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=39e9428b9>`_ vrrp: add plugin providing vrrp support
+
+``src/vlib/pci/pci_types.api``
+
+* `53f06a014 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=53f06a014>`_ vlib: move pci api types from vnet/pci to vlib/pci
diff --git a/docs/aboutvpp/releasenotes/v20.09.rst b/docs/aboutvpp/releasenotes/v20.09.rst
new file mode 100644
index 0000000..5a13223
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v20.09.rst
@@ -0,0 +1,714 @@
+Release notes for VPP 20.09
+===========================
+
+More than 458 commits since the previous release, including 266 fixes.
+
+Release Highlights
+------------------
+
+The FD.io VPP 20.09 release added a number of notable new features. In
+plugins, the I/O layer added support for the Linux AF_XDP interface with
+the AF_XDP plugin. New plugins where added supporting both the Wireguard
+security protocol and CNAT destination based address translation, and
+the existing IKEv2 plugin added support for NAT-T. In the cryptography
+layer, support was added for synchronous software crypto engines,
+enabling users to allocate dedicated crypto worker threads. The flow
+layer added support for steering IPSEC ESP/AH flows to worker threads.
+GRO support was added to the packet coalescing library.
+
+This release introduces the new FD.io VPP API change policy to ensure
+backwards-compatibility. The policy will ensure seamless upgrades to new
+versions of FD.io VPP in future, provided no “in-progress” or deprecated
+APIs are in use. Enabling the FD.io community to enjoy the benefits of
+new releases, while minimizing the work involved in staying current.
+
+If you dive into the implementation, you will note that policy in
+action. A number of modified API messages have had their original
+versions maintained to ensure compatibility.
+
+Reflecting the new policy we added two new sections to the release notes
+describing: - Newly deprecated API messages: please note that if you are
+using a deprecated message, they will soon be removed in a subsequent
+release. Collaborate with the feature maintainer on the best approach to
+mitigate. - In-progress API messages: They are work-in-progress, and are
+*not* subject to the policy, and may change or even be removed at any
+time. Please collaborate with the feature maintainer on plans to
+productize the message before using in any product. In-progress APIs
+must eventually become stable or be removed.
+
+Features
+--------
+
+- VNET
+
+ - Crypto Infra
+
+ - Add chacha20-poly1305 algo (61f49aa38)
+ - Asynchronous crypto engines (2284817ea)
+ - Add asynchronous crypto APIs (0c936b147)
+ - Added support for optimized cryptodev API (ef80ad6bf)
+
+ - FLOW
+
+ - Added ability to steer IPSec ESP/AH flows to worker threads
+ (d4c3666b9)
+ - Added the vnet/flow API (d0236f725)
+
+ - GENEVE
+
+ - Support geneve interface acting as a bvi (7fc88cf3a)
+
+ - GSO
+
+ - Added software GRO support (f382b06fe)
+
+ - IPSec
+
+ - Dedicated IPSec interface type (dd4ccf262)
+ - Deprecate old interface API (e6df80de4)
+
+ - Interface Common
+
+ - Support configuring RSS steering queues (c4665093c)
+
+ - Native Virtio Drivers
+
+ - Add vhost sw_if_index filter for sw_interface_vhost_user_dump
+ (a0e8d9669)
+ - Add modern device support (379aac395)
+ - Add virtio 1.1 api flags (518251bc8)
+
+ - TAP Drivers
+
+ - Add gro support (9e2a78564)
+ - Add virtio 1.1 API flag (50bd16559)
+
+ - TCP
+
+ - Track reorder with selective acknowledgments (cc4d6d022)
+
+- Plugins
+
+ - AF_XDP driver
+
+ - New plugin for Linux AF_XDP input (4a76d6f6d)
+
+ - CNat
+
+ - New plugin for destination based NAT (29f3c7d2e)
+
+ - Wireguard
+
+ - New plugin, initial implementation of wireguard protocol
+ (edca1325c)
+
+ - Crypto - OpenSSL
+
+ - Add chacha20-poly1305 support to crypto-openssl (1b6ed022e)
+
+ - DPDK
+
+ - Device_id sorted order for cryptodev (5a849e3b3)
+ - Call the meson-based build instead of Makefiles (73903d7e8)
+
+ - Internet Key Exchange (IKEv2) Protocol
+
+ - Add support for NAT traversal (NAT-T) (4362baa33)
+ - Add profile dump API (6a9bd8188)
+ - Add support for AES-GCM cipher in IKE (a7b963df2)
+ - Add SA dump API (a340fe1ac)
+
+ - Network Delay Simulator
+
+ - Basic reorder support (e6c3e8f0e)
+
+- VPP Comms Library
+
+ - Nest vcl_mq_epfd to support epoll_wait without high CPU usage
+ (4266d4d5f)
+ - Support connected udp listens (1e96617d9)
+ - Support inter worker rpc (40c07ce7a)
+ - Support multi-threads with session migration (a3a489691)
+
+- Vector Library
+
+ - Add recursive macro expander to debug cli (961e3c842)
+
+- Binary API Libraries
+
+ - Add new stream message convention (f5db3711b)
+ - Make VPP api handlers endian independent (e796a1873)
+
+- Infrastructure Library
+
+ - Multiarch support for OCTEONTX2 SoC (e2f5236dc)
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Fixed issues
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/2009>`__
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+- *Only in image*: indicates the API is new for this release.
+- *Only in file*: indicates the API has been removed in this release.
+
+========================================== ==================
+Message Name Result
+========================================== ==================
+adl_allowlist_enable_disable only in image
+adl_allowlist_enable_disable_reply only in image
+adl_interface_enable_disable only in image
+adl_interface_enable_disable_reply only in image
+bond_add_member only in image
+bond_add_member_reply only in image
+bond_create2 only in image
+bond_create2_reply only in image
+bond_detach_member only in image
+bond_detach_member_reply only in image
+cnat_add_del_snat_prefix only in image
+cnat_add_del_snat_prefix_reply only in image
+cnat_session_details only in image
+cnat_session_dump only in image
+cnat_session_purge only in image
+cnat_session_purge_reply only in image
+cnat_set_snat_addresses only in image
+cnat_set_snat_addresses_reply only in image
+cnat_translation_del only in image
+cnat_translation_del_reply only in image
+cnat_translation_details only in image
+cnat_translation_dump only in image
+cnat_translation_update only in image
+cnat_translation_update_reply only in image
+crypto_set_async_dispatch only in image
+crypto_set_async_dispatch_reply only in image
+crypto_set_handler only in image
+crypto_set_handler_reply only in image
+crypto_sw_scheduler_set_worker only in image
+crypto_sw_scheduler_set_worker_reply only in image
+det44_add_del_map only in image
+det44_add_del_map_reply only in image
+det44_close_session_in only in image
+det44_close_session_in_reply only in image
+det44_close_session_out only in image
+det44_close_session_out_reply only in image
+det44_forward only in image
+det44_forward_reply only in image
+det44_get_timeouts only in image
+det44_get_timeouts_reply only in image
+det44_interface_add_del_feature only in image
+det44_interface_add_del_feature_reply only in image
+det44_interface_details only in image
+det44_interface_dump only in image
+det44_map_details only in image
+det44_map_dump only in image
+det44_plugin_enable_disable only in image
+det44_plugin_enable_disable_reply only in image
+det44_reverse only in image
+det44_reverse_reply only in image
+det44_session_details only in image
+det44_session_dump only in image
+det44_set_timeouts only in image
+det44_set_timeouts_reply only in image
+flow_add only in image
+flow_add_reply only in image
+flow_del only in image
+flow_del_reply only in image
+flow_disable only in image
+flow_disable_reply only in image
+flow_enable only in image
+flow_enable_reply only in image
+geneve_add_del_tunnel2 only in image
+geneve_add_del_tunnel2_reply only in image
+gtpu_add_del_tunnel definition changed
+gtpu_tunnel_details definition changed
+gtpu_tunnel_update_tteid only in image
+gtpu_tunnel_update_tteid_reply only in image
+ikev2_child_sa_details only in image
+ikev2_child_sa_dump only in image
+ikev2_nonce_get only in image
+ikev2_nonce_get_reply only in image
+ikev2_profile_details only in image
+ikev2_profile_dump only in image
+ikev2_profile_set_ts definition changed
+ikev2_sa_details only in image
+ikev2_sa_dump only in image
+ikev2_set_esp_transforms definition changed
+ikev2_set_ike_transforms definition changed
+ikev2_set_responder definition changed
+ikev2_traffic_selector_details only in image
+ikev2_traffic_selector_dump only in image
+ipsec_itf_create only in image
+ipsec_itf_create_reply only in image
+ipsec_itf_delete only in image
+ipsec_itf_delete_reply only in image
+ipsec_itf_details only in image
+ipsec_itf_dump only in image
+ipsec_set_async_mode only in image
+ipsec_set_async_mode_reply only in image
+map_domains_get only in image
+map_domains_get_reply only in image
+nat44_add_del_static_mapping_v2 only in image
+nat44_add_del_static_mapping_v2_reply only in image
+nat_show_config_2 only in image
+nat_show_config_2_reply only in image
+nsim_configure2 only in image
+nsim_configure2_reply only in image
+pg_interface_enable_disable_coalesce only in image
+pg_interface_enable_disable_coalesce_reply only in image
+sr_policies_with_sl_index_details only in image
+sr_policies_with_sl_index_dump only in image
+sw_bond_interface_details only in image
+sw_bond_interface_dump only in image
+sw_member_interface_details only in image
+sw_member_interface_dump only in image
+trace_details only in image
+trace_dump only in image
+trace_dump_reply only in image
+virtio_pci_create_v2 only in image
+virtio_pci_create_v2_reply only in image
+wireguard_interface_create only in image
+wireguard_interface_create_reply only in image
+wireguard_interface_delete only in image
+wireguard_interface_delete_reply only in image
+wireguard_interface_details only in image
+wireguard_interface_dump only in image
+wireguard_peer_add only in image
+wireguard_peer_add_reply only in image
+wireguard_peer_remove only in image
+wireguard_peer_remove_reply only in image
+wireguard_peers_details only in image
+wireguard_peers_dump only in image
+========================================== ==================
+
+Found 123 api message signature differences
+
+Newly deprecated API messages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+These messages are still there in the API, but can and probably will
+disappear in the next release.
+
+- bond_create
+- bond_detach_slave
+- bond_detach_slave_reply
+- bond_enslave
+- cop_interface_enable_disable
+- cop_interface_enable_disable_reply
+- cop_whitelist_enable_disable
+- cop_whitelist_enable_disable_reply
+- geneve_add_del_tunnel
+- ipsec_tunnel_if_add_del
+- ipsec_tunnel_if_set_sa
+- ipsec_tunnel_if_set_sa_reply
+- map_domain_dump
+- nat_det_add_del_map
+- nat_det_add_del_map_reply
+- nat_det_close_session_in
+- nat_det_close_session_in_reply
+- nat_det_close_session_out
+- nat_det_close_session_out_reply
+- nat_det_forward
+- nat_det_forward_reply
+- nat_det_map_details
+- nat_det_map_dump
+- nat_det_reverse
+- nat_det_reverse_reply
+- nat_det_session_details
+- nat_det_session_dump
+- nat_show_config
+- nsim_configure
+- nsim_configure_reply
+- sw_interface_bond_dump
+- sw_interface_slave_dump
+- virtio_pci_create
+- virtio_pci_create_reply
+
+In-progress API messages
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+These messages are provided for testing and experimentation only. They
+are *not* subject to any compatibility process, and therefore can
+arbitrarily change or disappear at *any* moment. Also they may have less
+than satisfactory testing, making them unsuitable for other use than the
+technology preview. If you are intending to use these messages in
+production projects, please collaborate with the feature maintainer on
+their productization.
+
+- abf_itf_attach_add_del
+- abf_itf_attach_add_del_reply
+- abf_itf_attach_details
+- abf_itf_attach_dump
+- abf_plugin_get_version
+- abf_plugin_get_version_reply
+- abf_policy_add_del
+- abf_policy_add_del_reply
+- abf_policy_details
+- abf_policy_dump
+- adl_allowlist_enable_disable
+- adl_allowlist_enable_disable_reply
+- adl_interface_enable_disable
+- adl_interface_enable_disable_reply
+- af_xdp_create
+- af_xdp_create_reply
+- af_xdp_delete
+- af_xdp_delete_reply
+- cnat_add_del_snat_prefix
+- cnat_add_del_snat_prefix_reply
+- cnat_session_details
+- cnat_session_dump
+- cnat_session_purge
+- cnat_session_purge_reply
+- cnat_set_snat_addresses
+- cnat_set_snat_addresses_reply
+- cnat_translation_del
+- cnat_translation_del_reply
+- cnat_translation_details
+- cnat_translation_dump
+- cnat_translation_update
+- cnat_translation_update_reply
+- crypto_sw_scheduler_set_worker
+- crypto_sw_scheduler_set_worker_reply
+- det44_get_timeouts_reply
+- det44_interface_add_del_feature
+- det44_interface_add_del_feature_reply
+- det44_interface_details
+- det44_interface_dump
+- det44_plugin_enable_disable
+- det44_plugin_enable_disable_reply
+- det44_set_timeouts
+- det44_set_timeouts_reply
+- flow_add
+- flow_add_reply
+- flow_del
+- flow_del_reply
+- flow_disable
+- flow_disable_reply
+- flow_enable
+- flow_enable_reply
+- gbp_bridge_domain_add
+- gbp_bridge_domain_add_reply
+- gbp_bridge_domain_del
+- gbp_bridge_domain_del_reply
+- gbp_bridge_domain_details
+- gbp_bridge_domain_dump
+- gbp_bridge_domain_dump_reply
+- gbp_contract_add_del
+- gbp_contract_add_del_reply
+- gbp_contract_details
+- gbp_contract_dump
+- gbp_endpoint_add
+- gbp_endpoint_add_reply
+- gbp_endpoint_del
+- gbp_endpoint_del_reply
+- gbp_endpoint_details
+- gbp_endpoint_dump
+- gbp_endpoint_group_add
+- gbp_endpoint_group_add_reply
+- gbp_endpoint_group_del
+- gbp_endpoint_group_del_reply
+- gbp_endpoint_group_details
+- gbp_endpoint_group_dump
+- gbp_ext_itf_add_del
+- gbp_ext_itf_add_del_reply
+- gbp_ext_itf_details
+- gbp_ext_itf_dump
+- gbp_recirc_add_del
+- gbp_recirc_add_del_reply
+- gbp_recirc_details
+- gbp_recirc_dump
+- gbp_route_domain_add
+- gbp_route_domain_add_reply
+- gbp_route_domain_del
+- gbp_route_domain_del_reply
+- gbp_route_domain_details
+- gbp_route_domain_dump
+- gbp_route_domain_dump_reply
+- gbp_subnet_add_del
+- gbp_subnet_add_del_reply
+- gbp_subnet_details
+- gbp_subnet_dump
+- gbp_vxlan_tunnel_add
+- gbp_vxlan_tunnel_add_reply
+- gbp_vxlan_tunnel_del
+- gbp_vxlan_tunnel_del_reply
+- gbp_vxlan_tunnel_details
+- gbp_vxlan_tunnel_dump
+- ikev2_child_sa_details
+- ikev2_child_sa_dump
+- ikev2_initiate_del_child_sa
+- ikev2_initiate_del_child_sa_reply
+- ikev2_initiate_del_ike_sa
+- ikev2_initiate_del_ike_sa_reply
+- ikev2_initiate_rekey_child_sa
+- ikev2_initiate_rekey_child_sa_reply
+- ikev2_initiate_sa_init
+- ikev2_initiate_sa_init_reply
+- ikev2_nonce_get
+- ikev2_nonce_get_reply
+- ikev2_profile_add_del
+- ikev2_profile_add_del_reply
+- ikev2_profile_details
+- ikev2_profile_dump
+- ikev2_profile_set_auth
+- ikev2_profile_set_auth_reply
+- ikev2_profile_set_id
+- ikev2_profile_set_id_reply
+- ikev2_profile_set_ipsec_udp_port
+- ikev2_profile_set_ipsec_udp_port_reply
+- ikev2_profile_set_liveness
+- ikev2_profile_set_liveness_reply
+- ikev2_profile_set_ts
+- ikev2_profile_set_ts_reply
+- ikev2_profile_set_udp_encap
+- ikev2_profile_set_udp_encap_reply
+- ikev2_sa_details
+- ikev2_sa_dump
+- ikev2_set_esp_transforms
+- ikev2_set_esp_transforms_reply
+- ikev2_set_ike_transforms
+- ikev2_set_ike_transforms_reply
+- ikev2_set_local_key
+- ikev2_set_local_key_reply
+- ikev2_set_responder
+- ikev2_set_responder_reply
+- ikev2_set_sa_lifetime
+- ikev2_set_sa_lifetime_reply
+- ikev2_set_tunnel_interface
+- ikev2_set_tunnel_interface_reply
+- ikev2_traffic_selector_details
+- ikev2_traffic_selector_dump
+- l2_emulation
+- l2_emulation_reply
+- mdata_enable_disable
+- mdata_enable_disable_reply
+- nat44_add_del_static_mapping_v2
+- nat44_add_del_static_mapping_v2_reply
+- oddbuf_enable_disable
+- oddbuf_enable_disable_reply
+- pg_interface_enable_disable_coalesce
+- pg_interface_enable_disable_coalesce_reply
+- sample_macswap_enable_disable
+- sample_macswap_enable_disable_reply
+- sr_policies_with_sl_index_details
+- sr_policies_with_sl_index_dump
+- sw_interface_set_vxlan_gbp_bypass
+- sw_interface_set_vxlan_gbp_bypass_reply
+- trace_details
+- trace_dump
+- trace_dump_reply
+- vxlan_gbp_tunnel_add_del
+- vxlan_gbp_tunnel_add_del_reply
+- vxlan_gbp_tunnel_details
+- vxlan_gbp_tunnel_dump
+- wireguard_interface_create
+- wireguard_interface_create_reply
+- wireguard_interface_delete
+- wireguard_interface_delete_reply
+- wireguard_interface_details
+- wireguard_interface_dump
+- wireguard_peer_add
+- wireguard_peer_add_reply
+- wireguard_peer_remove
+- wireguard_peer_remove_reply
+- wireguard_peers_details
+- wireguard_peers_dump
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``src/vpp/api/vpe.api``
+
+* `d0236f725 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d0236f725>`_ flow: add vnet/flow formal API
+
+``src/vnet/crypto/crypto.api``
+
+* `4035daffd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4035daffd>`_ crypto: Crypto set handler API to support set all as CLI
+* `0c936b147 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0c936b147>`_ crypto: Add async crypto APIs
+
+``src/vnet/cop/cop.api``
+
+* `00f21fb2f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=00f21fb2f>`_ api: clean up use of deprecated flag
+* `ac0326fc5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ac0326fc5>`_ adl: move allow/deny list function to plugin
+
+``src/vnet/lisp-gpe/lisp_gpe.api``
+
+* `4ab5190eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4ab5190eb>`_ lisp: API cleanup
+
+``src/vnet/vxlan-gbp/vxlan_gbp.api``
+
+* `f72b1aff7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f72b1aff7>`_ vxlan-gbp: Mark APIs as in-progress
+
+``src/vnet/flow/flow_types.api``
+
+* `34bfa50b6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=34bfa50b6>`_ flow: code refactor
+* `d0236f725 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d0236f725>`_ flow: add vnet/flow formal API
+
+``src/vnet/flow/flow.api``
+
+* `d0236f725 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d0236f725>`_ flow: add vnet/flow formal API
+
+``src/vnet/srv6/sr.api``
+
+* `30fa97dc6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=30fa97dc6>`_ sr: new messages created to return sl index for segment lists in a sr policy
+
+``src/vnet/pg/pg.api``
+
+* `f382b06fe <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f382b06fe>`_ gso: packet coalesce library
+* `0cf528233 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0cf528233>`_ gso: fix the udp checksum in test
+
+``src/vnet/geneve/geneve.api``
+
+* `00f21fb2f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=00f21fb2f>`_ api: clean up use of deprecated flag
+* `7fc88cf3a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7fc88cf3a>`_ geneve: support geneve interface acting as a bvi
+
+``src/vnet/lisp-cp/one.api``
+
+* `4ab5190eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4ab5190eb>`_ lisp: API cleanup
+
+``src/vnet/lisp-cp/lisp.api``
+
+* `4ab5190eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4ab5190eb>`_ lisp: API cleanup
+
+``src/vnet/devices/tap/tapv2.api``
+
+* `50bd16559 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=50bd16559>`_ tap: add virtio 1.1 API flag
+
+``src/vnet/devices/virtio/vhost_user.api``
+
+* `a0e8d9669 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a0e8d9669>`_ virtio: add vhost sw_if_index filter for sw_interface_vhost_user_dump
+
+``src/vnet/devices/virtio/virtio.api``
+
+* `00f21fb2f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=00f21fb2f>`_ api: clean up use of deprecated flag
+* `518251bc8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=518251bc8>`_ virtio: add virtio 1.1 api flags
+
+``src/vnet/ipsec/ipsec.api``
+
+* `00f21fb2f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=00f21fb2f>`_ api: clean up use of deprecated flag
+* `2e84d6655 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2e84d6655>`_ ipsec: add ipsec set async mode api
+* `e6df80de4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e6df80de4>`_ ipsec: Deprecate old interface API
+* `dd4ccf262 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dd4ccf262>`_ ipsec: Dedicated IPSec interface type
+
+``src/vnet/bonding/bond.api``
+
+* `ea7178631 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ea7178631>`_ bonding: add bond_create2 API to include gso option
+* `4c4223edf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4c4223edf>`_ bonding lacp: replace slave string with member
+
+``src/vnet/ip/ip_types.api``
+
+* `d0236f725 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d0236f725>`_ flow: add vnet/flow formal API
+
+``src/plugins/wireguard/wireguard.api``
+
+* `edca1325c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=edca1325c>`_ wireguard: initial implementation of wireguard protocol
+
+``src/plugins/map/map.api``
+
+* `00f21fb2f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=00f21fb2f>`_ api: clean up use of deprecated flag
+* `ac0326fc5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ac0326fc5>`_ adl: move allow/deny list function to plugin
+* `f5db3711b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f5db3711b>`_ api: add new stream message convention
+
+``src/plugins/lacp/lacp.api``
+
+* `4c4223edf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4c4223edf>`_ bonding lacp: replace slave string with member
+
+``src/plugins/l2e/l2e.api``
+
+* `f733e7ade <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f733e7ade>`_ l2e: mark API as in-progress
+
+``src/plugins/ikev2/ikev2.api``
+
+* `a340fe1ac <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a340fe1ac>`_ ikev2: add SA dump API
+* `459d17bb7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=459d17bb7>`_ ikev2: refactor and test profile dump API
+* `ac46e3b1d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ac46e3b1d>`_ ikev2: API downgrade due to lack of ikev2 tests
+* `6a9bd8188 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6a9bd8188>`_ ikev2: add profile dump API
+
+``src/plugins/ikev2/ikev2_types.api``
+
+* `a340fe1ac <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a340fe1ac>`_ ikev2: add SA dump API
+* `459d17bb7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=459d17bb7>`_ ikev2: refactor and test profile dump API
+* `6a9bd8188 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6a9bd8188>`_ ikev2: add profile dump API
+
+``src/plugins/tracedump/tracedump.api``
+
+* `65b65a469 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=65b65a469>`_ misc: add tracedump API plugin
+
+``src/plugins/gtpu/gtpu.api``
+
+* `9ebbb5c41 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9ebbb5c41>`_ gtpu: support separate rx-decap and encap-tx teid values
+
+``src/plugins/gbp/gbp.api``
+
+* `d2f8fb9c7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d2f8fb9c7>`_ gbp: mark APIs as in-progress
+
+``src/plugins/acl/acl.api``
+
+* `24ee40a5c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=24ee40a5c>`_ acl: correct acl vat help message
+
+``src/plugins/nat/dslite/dslite.api``
+
+* `603e75465 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=603e75465>`_ nat: move deterministic nat to det44 sub feature
+
+``src/plugins/nat/det44/det44.api``
+
+* `00f21fb2f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=00f21fb2f>`_ api: clean up use of deprecated flag
+* `603e75465 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=603e75465>`_ nat: move deterministic nat to det44 sub feature
+
+``src/plugins/nat/nat_types.api``
+
+* `96068d6b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=96068d6b9>`_ nat: nat66 to plugin
+
+``src/plugins/nat/nat.api``
+
+* `6484f4b9c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6484f4b9c>`_ nat: twice-nat static mapping pool address
+* `edc816355 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=edc816355>`_ nat: fix type in api message
+* `603e75465 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=603e75465>`_ nat: move deterministic nat to det44 sub feature
+* `96068d6b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=96068d6b9>`_ nat: nat66 to plugin
+
+``src/plugins/nat/nat66/nat66.api``
+
+* `96068d6b9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=96068d6b9>`_ nat: nat66 to plugin
+
+``src/plugins/cnat/cnat.api``
+
+* `29f3c7d2e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=29f3c7d2e>`_ cnat: Destination based NAT
+
+``src/plugins/abf/abf.api``
+
+* `df494dafa <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df494dafa>`_ abf: mark API as in-progress
+
+``src/plugins/adl/adl.api``
+
+* `ac0326fc5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ac0326fc5>`_ adl: move allow/deny list function to plugin
+
+``src/plugins/nsim/nsim.api``
+
+* `00f21fb2f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=00f21fb2f>`_ api: clean up use of deprecated flag
+* `e6c3e8f0e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e6c3e8f0e>`_ nsim: basic reorder support
+
+``src/plugins/crypto_sw_scheduler/crypto_sw_scheduler.api``
+
+* `0c936b147 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0c936b147>`_ crypto: Add async crypto APIs
+
+``src/plugins/dhcp/dhcp.api``
+
+* `bad679291 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bad679291>`_ api: register endian handlers for reply messages
+
+``src/plugins/af_xdp/af_xdp.api``
+
+* `4a76d6f6d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4a76d6f6d>`_ af_xdp: AF_XDP input plugin
diff --git a/docs/aboutvpp/releasenotes/v21.01.rst b/docs/aboutvpp/releasenotes/v21.01.rst
new file mode 100644
index 0000000..e0070b7
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v21.01.rst
@@ -0,0 +1,764 @@
+Release notes for VPP 21.01
+===========================
+
+Release Highlights
+------------------
+
+The FD.io VPP 20.09 release comprises more than 562 commits since the
+previous release, including 274 fixes. Notable changes in this release
+were the Virtio driver adding packet buffering on the transmit path to
+handle slow back-ends which often have jitter and delays in free’ing
+buffers, and also adding support for Virtio 1.1 packed rings. FD.io VPP
+IPSEC added support for multi-point on IPSec interfaces, brings the
+meshed benefits of IP to IPSec tunnels. The FD.io VPP Cloud NAT (CNAT),
+added support source NAT ICMP and DHCP. The FD.io VPP FIB added support
+for source address selection (SAS) and flow hashing on the inner
+packets. Finally the FD.io VPP Perfmon plugin has been substantially
+rewritten, to support measuring bundles of counters, and reporting
+statistics per graph node.
+
+Reflecting the API change policy introduced in the 20.09, please review
+the following sections below: - Newly deprecated API messages: please
+note that if you are using a deprecated message, they will soon be
+removed in a subsequent release. Collaborate with the feature maintainer
+on the best approach to mitigate. - In-progress API messages: They are
+work-in-progress, and are *not* subject to the policy, and may change or
+even be removed at any time. Please collaborate with the feature
+maintainer on plans to productize the message before using in any
+product. In-progress APIs must eventually become stable or be removed.
+
+Features
+--------
+
+- Binary API Libraries
+
+ - Vat2 and JSON autogeneration for API messages
+ (`df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`__)
+
+- Plugins
+
+ - AF_XDP driver
+
+ - Add option to claim all available RX queues
+ (`d4e109138 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d4e109138>`__)
+
+ - CNat
+
+ - Disable default scanner process
+ (`d63f73b83 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d63f73b83>`__)
+ - IP ICMP error support
+ (`ece39214b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ece39214b>`__)
+ - Add support for SNat ICMP
+ (`613b2c3c7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=613b2c3c7>`__)
+ - Introduce parametric source policy
+ (`ce25b60de <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ce25b60de>`__)
+ - Add DHCP support
+ (`af897c5e3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=af897c5e3>`__)
+
+ - Crypto - ipsecmb
+
+ - Bump to intel-ipsec-mb version 0.55
+ (`b5df85e24 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b5df85e24>`__)
+
+ - DPDK
+
+ - Call the meson-based build instead of Makefiles
+ (`4c4633cad <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4c4633cad>`__)
+ - Telemetry thread is off by default.
+ (`83f37fc3b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=83f37fc3b>`__)
+ - Bump to DPDK 20.11
+ (`f0419a0c8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f0419a0c8>`__)
+
+ - Internet Key Exchange (IKEv2) Protocol
+
+ - Support IPv6 traffic selectors & overlay
+ (`84962d19b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=84962d19b>`__)
+ - CLI for disabling dead peer detection
+ (`af4a414eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=af4a414eb>`__)
+ - Add option to disable NAT traversal
+ (`d7fc12f07 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d7fc12f07>`__)
+
+ - RDMA (ibverb) driver
+
+ - Add RSS support for IPv6 and TCP
+ (`91603958d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=91603958d>`__)
+
+ - VRRP
+
+ - Asynchronous events on VR state change
+ (`78f487e11 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=78f487e11>`__)
+
+ - Wireguard
+
+ - Return public key in API
+ (`de22111b5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=de22111b5>`__)
+
+ - Flowprobe
+
+ - Add show commands for params and list of interfaces for
+ recording
+ (`d1146f6dd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d1146f6dd>`__)
+
+- Python binding for the VPP API
+
+ - add support for enumflag part 1 of 2
+ (`3825d93af <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3825d93af>`__)
+
+- SVM Library
+
+ - Support for multi-segment enqueues
+ (`c95cfa218 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c95cfa218>`__)
+
+- Statistics Segment
+
+ - Counters data model
+ (`148c7b768 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=148c7b768>`__)
+
+- VNET
+
+ - FIB
+
+ - Source Address Selection
+ (`e2fe09742 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e2fe09742>`__)
+ - Adjacency flag for midchain to perfom flow hash (on inner
+ packet)
+ (`5c544c8c3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5c544c8c3>`__)
+
+ - Feature Arcs
+
+ - Add packet trace API
+ (`c0b195450 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0b195450>`__)
+
+ - IPSec
+
+ - Support for multipoint on IPSec interfaces
+ (`6ba4e41d3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6ba4e41d3>`__)
+ - Tunnel SA DSCP behaviour
+ (`041add7d1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=041add7d1>`__)
+
+ - Native Virtio Drivers
+
+ - Add packet buffering on transmit path
+ (`e347acbc3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e347acbc3>`__)
+ - Virtio: implement packed queues from virtio 1.1
+ (`b977d3f7c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b977d3f7c>`__)
+
+ - Segment Routing (IPv6 and MPLS)
+
+ - Show IPv6 address used as SRv6 Encaps source
+ (`448bc81d3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=448bc81d3>`__)
+ - Show the hop-limit value used for SRv6 encapsulation
+ (`80f0b88fc <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=80f0b88fc>`__)
+
+ - Session Layer
+
+ - Add Unix socket API for app attachment
+ (`61ae056bd <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=61ae056bd>`__)
+ - Per worker state for ct sessions
+ (`2d0e3de14 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2d0e3de14>`__)
+
+ - TAP Drivers
+
+ - Allow change of carrier state on host
+ (`bd50ed18d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bd50ed18d>`__)
+ - Add function to set speed
+ (`a6c34a19d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a6c34a19d>`__)
+
+- VPP Comms Library
+
+ - Add support for app socket API
+ (`935ce75cb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=935ce75cb>`__)
+ - Provide apps access to fifo chunks
+ (`d68faf855 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d68faf855>`__)
+
+- VPP Executable
+
+ - Use VPP heap for libc
+ (`ec4749a20 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ec4749a20>`__)
+
+- Vector Library - Buffer Management
+
+ - Add page-size config
+ (`61559029d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=61559029d>`__)
+
+Known issues
+------------
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Fixed issues
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/2101>`__
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+- *Only in image*: indicates the API is new for this release.
+- *Only in file*: indicates the API has been removed in this release.
+
+================================= ==================
+Message Name Result
+================================= ==================
+classify_pcap_get_tables only in image
+classify_pcap_get_tables_reply only in image
+classify_pcap_lookup_table only in image
+classify_pcap_lookup_table_reply only in image
+classify_pcap_set_table only in image
+classify_pcap_set_table_reply only in image
+classify_trace_get_tables only in image
+classify_trace_get_tables_reply only in image
+classify_trace_lookup_table only in image
+classify_trace_lookup_table_reply only in image
+classify_trace_set_table only in image
+classify_trace_set_table_reply only in image
+cnat_get_snat_addresses only in image
+cnat_get_snat_addresses_reply only in image
+cnat_session_details definition changed
+cnat_set_snat_addresses definition changed
+cnat_translation_details definition changed
+cnat_translation_update definition changed
+det44_plugin_enable_disable definition changed
+graph_node_details only in image
+graph_node_get only in image
+graph_node_get_reply only in image
+ikev2_profile_details definition changed
+ikev2_profile_disable_natt only in image
+ikev2_profile_disable_natt_reply only in image
+ikev2_profile_set_ts definition changed
+ikev2_sa_details definition changed
+ikev2_set_responder definition changed
+ikev2_traffic_selector_details definition changed
+ip_mroute_add_del definition changed
+ip_mroute_details definition changed
+ip_neighbor_event_v2 only in image
+ipsec_sa_v2_details only in image
+ipsec_sa_v2_dump only in image
+ipsec_sad_entry_add_del_v2 only in image
+ipsec_sad_entry_add_del_v2_reply only in image
+nat44_plugin_enable_disable only in image
+nat44_plugin_enable_disable_reply only in image
+nat44_show_running_config only in image
+nat44_show_running_config_reply only in image
+nat64_get_timeouts only in image
+nat64_get_timeouts_reply only in image
+nat64_plugin_enable_disable only in image
+nat64_plugin_enable_disable_reply only in image
+nat64_set_timeouts only in image
+nat64_set_timeouts_reply only in image
+pppoe_add_del_cp only in image
+pppoe_add_del_cp_reply only in image
+rdma_create_v2 only in image
+rdma_create_v2_reply only in image
+sw_vmxnet3_interface_details only in image
+sw_vmxnet3_interface_dump only in image
+trace_capture_packets only in image
+trace_capture_packets_reply only in image
+trace_clear_capture only in image
+trace_clear_capture_reply only in image
+trace_details definition changed
+trace_set_filters only in image
+trace_set_filters_reply only in image
+vrrp_vr_event only in image
+want_ip_neighbor_events_v2 only in image
+want_ip_neighbor_events_v2_reply only in image
+want_vrrp_vr_events only in image
+want_vrrp_vr_events_reply only in image
+wireguard_interface_create definition changed
+wireguard_interface_details definition changed
+================================= ==================
+
+Found 66 api message signature differences
+
+Newly deprecated API messages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+These messages are still there in the API, but can and probably will
+disappear in the next release.
+
+- geneve_add_del_tunnel
+- ip_neighbor_event
+- nat44_forwarding_enable_disable
+- nat44_forwarding_enable_disable_reply
+- nat44_forwarding_is_enabled
+- nat44_forwarding_is_enabled_reply
+- nat44_session_cleanup
+- nat44_session_cleanup_reply
+- nat_control_ping
+- nat_control_ping_reply
+- nat_get_timeouts
+- nat_get_timeouts_reply
+- nat_ipfix_enable_disable
+- nat_ipfix_enable_disable_reply
+- nat_set_log_level
+- nat_set_log_level_reply
+- nat_set_timeouts
+- nat_set_timeouts_reply
+- nat_show_config
+- nat_show_config_2
+- nat_show_config_2_reply
+- nat_show_config_reply
+- rdma_create
+- vmxnet3_dump
+- want_ip_neighbor_events
+- want_ip_neighbor_events_reply
+
+In-progress API messages
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+These messages are provided for testing and experimentation only. They
+are *not* subject to any compatibility process, and therefore can
+arbitrarily change or disappear at *any* moment. Also they may have less
+than satisfactory testing, making them unsuitable for other use than the
+technology preview. If you are intending to use these messages in
+production projects, please collaborate with the feature maintainer on
+their productization.
+
+- abf_itf_attach_add_del
+- abf_itf_attach_add_del_reply
+- abf_itf_attach_details
+- abf_itf_attach_dump
+- abf_plugin_get_version
+- abf_plugin_get_version_reply
+- abf_policy_add_del
+- abf_policy_add_del_reply
+- abf_policy_details
+- abf_policy_dump
+- adl_allowlist_enable_disable
+- adl_allowlist_enable_disable_reply
+- adl_interface_enable_disable
+- adl_interface_enable_disable_reply
+- af_xdp_create
+- af_xdp_create_reply
+- af_xdp_delete
+- af_xdp_delete_reply
+- cnat_add_del_snat_prefix
+- cnat_add_del_snat_prefix_reply
+- cnat_get_snat_addresses
+- cnat_get_snat_addresses_reply
+- cnat_session_details
+- cnat_session_dump
+- cnat_session_purge
+- cnat_session_purge_reply
+- cnat_set_snat_addresses
+- cnat_set_snat_addresses_reply
+- cnat_translation_del
+- cnat_translation_del_reply
+- cnat_translation_details
+- cnat_translation_dump
+- cnat_translation_update
+- cnat_translation_update_reply
+- crypto_sw_scheduler_set_worker
+- crypto_sw_scheduler_set_worker_reply
+- det44_get_timeouts_reply
+- det44_interface_add_del_feature
+- det44_interface_add_del_feature_reply
+- det44_interface_details
+- det44_interface_dump
+- det44_plugin_enable_disable
+- det44_plugin_enable_disable_reply
+- det44_set_timeouts
+- det44_set_timeouts_reply
+- flow_add
+- flow_add_reply
+- flow_del
+- flow_del_reply
+- flow_disable
+- flow_disable_reply
+- flow_enable
+- flow_enable_reply
+- gbp_bridge_domain_add
+- gbp_bridge_domain_add_reply
+- gbp_bridge_domain_del
+- gbp_bridge_domain_del_reply
+- gbp_bridge_domain_details
+- gbp_bridge_domain_dump
+- gbp_bridge_domain_dump_reply
+- gbp_contract_add_del
+- gbp_contract_add_del_reply
+- gbp_contract_details
+- gbp_contract_dump
+- gbp_endpoint_add
+- gbp_endpoint_add_reply
+- gbp_endpoint_del
+- gbp_endpoint_del_reply
+- gbp_endpoint_details
+- gbp_endpoint_dump
+- gbp_endpoint_group_add
+- gbp_endpoint_group_add_reply
+- gbp_endpoint_group_del
+- gbp_endpoint_group_del_reply
+- gbp_endpoint_group_details
+- gbp_endpoint_group_dump
+- gbp_ext_itf_add_del
+- gbp_ext_itf_add_del_reply
+- gbp_ext_itf_details
+- gbp_ext_itf_dump
+- gbp_recirc_add_del
+- gbp_recirc_add_del_reply
+- gbp_recirc_details
+- gbp_recirc_dump
+- gbp_route_domain_add
+- gbp_route_domain_add_reply
+- gbp_route_domain_del
+- gbp_route_domain_del_reply
+- gbp_route_domain_details
+- gbp_route_domain_dump
+- gbp_route_domain_dump_reply
+- gbp_subnet_add_del
+- gbp_subnet_add_del_reply
+- gbp_subnet_details
+- gbp_subnet_dump
+- gbp_vxlan_tunnel_add
+- gbp_vxlan_tunnel_add_reply
+- gbp_vxlan_tunnel_del
+- gbp_vxlan_tunnel_del_reply
+- gbp_vxlan_tunnel_details
+- gbp_vxlan_tunnel_dump
+- ikev2_child_sa_details
+- ikev2_child_sa_dump
+- ikev2_initiate_del_child_sa
+- ikev2_initiate_del_child_sa_reply
+- ikev2_initiate_del_ike_sa
+- ikev2_initiate_del_ike_sa_reply
+- ikev2_initiate_rekey_child_sa
+- ikev2_initiate_rekey_child_sa_reply
+- ikev2_initiate_sa_init
+- ikev2_initiate_sa_init_reply
+- ikev2_nonce_get
+- ikev2_nonce_get_reply
+- ikev2_profile_add_del
+- ikev2_profile_add_del_reply
+- ikev2_profile_details
+- ikev2_profile_disable_natt
+- ikev2_profile_disable_natt_reply
+- ikev2_profile_dump
+- ikev2_profile_set_auth
+- ikev2_profile_set_auth_reply
+- ikev2_profile_set_id
+- ikev2_profile_set_id_reply
+- ikev2_profile_set_ipsec_udp_port
+- ikev2_profile_set_ipsec_udp_port_reply
+- ikev2_profile_set_liveness
+- ikev2_profile_set_liveness_reply
+- ikev2_profile_set_ts
+- ikev2_profile_set_ts_reply
+- ikev2_profile_set_udp_encap
+- ikev2_profile_set_udp_encap_reply
+- ikev2_sa_details
+- ikev2_sa_dump
+- ikev2_set_esp_transforms
+- ikev2_set_esp_transforms_reply
+- ikev2_set_ike_transforms
+- ikev2_set_ike_transforms_reply
+- ikev2_set_local_key
+- ikev2_set_local_key_reply
+- ikev2_set_responder
+- ikev2_set_responder_reply
+- ikev2_set_sa_lifetime
+- ikev2_set_sa_lifetime_reply
+- ikev2_set_tunnel_interface
+- ikev2_set_tunnel_interface_reply
+- ikev2_traffic_selector_details
+- ikev2_traffic_selector_dump
+- l2_emulation
+- l2_emulation_reply
+- mdata_enable_disable
+- mdata_enable_disable_reply
+- nat44_add_del_static_mapping_v2
+- nat44_add_del_static_mapping_v2_reply
+- nat44_show_running_config
+- nat44_show_running_config_reply
+- nat64_plugin_enable_disable
+- nat64_plugin_enable_disable_reply
+- oddbuf_enable_disable
+- oddbuf_enable_disable_reply
+- pg_interface_enable_disable_coalesce
+- pg_interface_enable_disable_coalesce_reply
+- sample_macswap_enable_disable
+- sample_macswap_enable_disable_reply
+- sr_policies_with_sl_index_details
+- sr_policies_with_sl_index_dump
+- sw_interface_set_vxlan_gbp_bypass
+- sw_interface_set_vxlan_gbp_bypass_reply
+- test_enum
+- test_enum_reply
+- test_prefix
+- test_prefix_reply
+- trace_capture_packets
+- trace_capture_packets_reply
+- trace_clear_capture
+- trace_clear_capture_reply
+- trace_details
+- trace_dump
+- trace_dump_reply
+- trace_set_filters
+- trace_set_filters_reply
+- vxlan_gbp_tunnel_add_del
+- vxlan_gbp_tunnel_add_del_reply
+- vxlan_gbp_tunnel_details
+- vxlan_gbp_tunnel_dump
+- wireguard_interface_create
+- wireguard_interface_create_reply
+- wireguard_interface_delete
+- wireguard_interface_delete_reply
+- wireguard_interface_details
+- wireguard_interface_dump
+- wireguard_peer_add
+- wireguard_peer_add_reply
+- wireguard_peer_remove
+- wireguard_peer_remove_reply
+- wireguard_peers_details
+- wireguard_peers_dump
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``src/vpp/api/vpe_types.api``
+
+* `dc01471be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc01471be>`_ api: add missing version info
+
+``src/vat2/test/vat2_test.api``
+
+* `58a6e7725 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=58a6e7725>`_ api: crchcecker ignore version < 1.0.0 and outside of src directory
+* `510aaa891 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=510aaa891>`_ api: crchcecker ignore version < 1.0.0 and outside of src directory
+* `793be4632 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=793be4632>`_ api: fromjson/tojson enum flag support
+
+``src/vnet/mpls/mpls.api``
+
+* `df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`_ api: vat2 and json autogeneration for api messages
+
+``src/vnet/ipip/ipip.api``
+
+* `33c45f56a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=33c45f56a>`_ fib: supporting inner flow hash on tunnels
+
+``src/vnet/vxlan-gbp/vxlan_gbp.api``
+
+* `b468773aa <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b468773aa>`_ vxlan-gbp: Mark APIs as in-progress
+
+``src/vnet/ipsec/ipsec.api``
+
+* `041add7d1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=041add7d1>`_ ipsec: Tunnel SA DSCP behaviour
+* `f916414b3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f916414b3>`_ api: clean up use of deprecated flag
+
+``src/vnet/ipsec/ipsec_types.api``
+
+* `041add7d1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=041add7d1>`_ ipsec: Tunnel SA DSCP behaviour
+
+``src/vnet/tunnel/tunnel_types.api``
+
+* `dc01471be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc01471be>`_ api: add missing version info
+* `33c45f56a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=33c45f56a>`_ fib: supporting inner flow hash on tunnels
+
+``src/vnet/classify/classify.api``
+
+* `5c1e48c01 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5c1e48c01>`_ classify: add pcap/trace classfier mgmt API calls
+
+``src/vnet/ipfix-export/ipfix_export.api``
+
+* `f6cf57ceb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f6cf57ceb>`_ misc: fix api in ipfix_classify_table_add/details
+
+``src/vnet/mfib/mfib_types.api``
+
+* `dc01471be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc01471be>`_ api: add missing version info
+* `990f69450 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=990f69450>`_ ip: convert u32 entry_flags to vl_api_mfib_entry_flags_t on mroute API
+
+``src/vnet/gre/gre.api``
+
+* `33c45f56a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=33c45f56a>`_ fib: supporting inner flow hash on tunnels
+
+``src/vnet/ip/ip_types.api``
+
+* `6dc0c8d14 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6dc0c8d14>`_ ip: Sub Address Family types. Feature enable for each SAFI
+
+``src/vnet/ip/ip.api``
+
+* `df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`_ api: vat2 and json autogeneration for api messages
+* `990f69450 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=990f69450>`_ ip: convert u32 entry_flags to vl_api_mfib_entry_flags_t on mroute API
+
+``src/vnet/ethernet/ethernet_types.api``
+
+* `dc01471be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc01471be>`_ api: add missing version info
+
+``src/vnet/l2/l2.api``
+
+* `df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`_ api: vat2 and json autogeneration for api messages
+
+``src/vnet/cop/cop.api``
+
+* `6c8cdf78b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6c8cdf78b>`_ misc: cop - clean up stray doxygen block
+* `f916414b3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f916414b3>`_ api: clean up use of deprecated flag
+
+``src/vnet/crypto/crypto.api``
+
+* `8c91b2ae2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8c91b2ae2>`_ crypto: Crypto set handler API to support set all as CLI
+
+``src/vnet/devices/virtio/virtio.api``
+
+* `e347acbc3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e347acbc3>`_ virtio: add packet buffering on transmit path
+* `f916414b3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f916414b3>`_ api: clean up use of deprecated flag
+
+``src/vnet/interface_types.api``
+
+* `dc01471be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc01471be>`_ api: add missing version info
+
+``src/vnet/ip-neighbor/ip_neighbor.api``
+
+* `4ac36bcb1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4ac36bcb1>`_ ip-neighbor: Send API event when neighbor is removed
+
+``src/vnet/policer/policer_types.api``
+
+* `dc01471be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc01471be>`_ api: add missing version info
+
+``src/vnet/srv6/sr_types.api``
+
+* `dc01471be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc01471be>`_ api: add missing version info
+
+``src/plugins/map/map.api``
+
+* `148c7b768 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=148c7b768>`_ stats: counters data model
+* `f916414b3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f916414b3>`_ api: clean up use of deprecated flag
+
+``src/plugins/nat/nat64/nat64.api``
+
+* `1f36023d2 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1f36023d2>`_ nat: move nat64 to a subfeature
+
+``src/plugins/nat/det44/det44.api``
+
+* `d1762e614 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d1762e614>`_ nat: det44 plugin fix style and api cleanup
+* `f916414b3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f916414b3>`_ api: clean up use of deprecated flag
+
+``src/plugins/nat/nat44.api``
+
+* `df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`_ api: vat2 and json autogeneration for api messages
+* `25fd8ad03 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=25fd8ad03>`_ nat: cleanup & reorganization
+* `b227aa699 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b227aa699>`_ nat: api,cli and test update & cleanup
+
+``src/plugins/nat/nat_types.api``
+
+* `25fd8ad03 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=25fd8ad03>`_ nat: cleanup & reorganization
+
+``src/plugins/lisp/lisp-cp/one.api``
+
+* `2b202bc4b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2b202bc4b>`_ lisp: Move to plugin
+
+``src/plugins/lisp/lisp-cp/lisp.api``
+
+* `068ad25c1 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=068ad25c1>`_ lisp: .api dont set defaults in reply messages
+* `2b202bc4b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2b202bc4b>`_ lisp: Move to plugin
+
+``src/plugins/lisp/lisp-cp/lisp_types.api``
+
+* `2b202bc4b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2b202bc4b>`_ lisp: Move to plugin
+
+``src/plugins/lisp/lisp-gpe/lisp_gpe.api``
+
+* `2b202bc4b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2b202bc4b>`_ lisp: Move to plugin
+
+``src/plugins/nsim/nsim.api``
+
+* `f916414b3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f916414b3>`_ api: clean up use of deprecated flag
+
+``src/plugins/lb/lb_types.api``
+
+* `dc01471be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc01471be>`_ api: add missing version info
+
+``src/plugins/lb/lb.api``
+
+* `df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`_ api: vat2 and json autogeneration for api messages
+
+``src/plugins/pppoe/pppoe.api``
+
+* `340b10a38 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=340b10a38>`_ pppoe: make pppoe plugin work with dot1q subinterfaces
+
+``src/plugins/geneve/geneve.api``
+
+* `3a6adc52f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3a6adc52f>`_ geneve: Move to plugin
+
+``src/plugins/vmxnet3/vmxnet3.api``
+
+* `490e077fb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=490e077fb>`_ vmxnet3: add sw_if_index filter to vmxnet3 interface dump
+
+``src/plugins/wireguard/wireguard.api``
+
+* `de22111b5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=de22111b5>`_ wireguard: return public key in api
+
+``src/plugins/l2tp/l2tp.api``
+
+* `6810a77da <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6810a77da>`_ misc: Move l2tp to plugin
+
+``src/plugins/acl/acl.api``
+
+* `df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`_ api: vat2 and json autogeneration for api messages
+
+``src/plugins/acl/acl_types.api``
+
+* `dc01471be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc01471be>`_ api: add missing version info
+
+``src/plugins/rdma/rdma.api``
+
+* `798267aaa <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=798267aaa>`_ rdma: implement multiseg rx without striding rq
+
+``src/plugins/ikev2/ikev2.api``
+
+* `d7fc12f07 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d7fc12f07>`_ ikev2: add option to disable NAT traversal
+* `84962d19b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=84962d19b>`_ ikev2: support ipv6 traffic selectors & overlay
+
+``src/plugins/ikev2/ikev2_types.api``
+
+* `dc01471be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc01471be>`_ api: add missing version info
+* `d7fc12f07 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d7fc12f07>`_ ikev2: add option to disable NAT traversal
+* `84962d19b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=84962d19b>`_ ikev2: support ipv6 traffic selectors & overlay
+
+``src/plugins/cnat/cnat.api``
+
+* `2082835fe <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2082835fe>`_ cnat: allow max_u16 translation backends
+* `af897c5e3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=af897c5e3>`_ cnat: Add DHCP support
+
+``src/plugins/tracedump/tracedump.api``
+
+* `c0b195450 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0b195450>`_ feature: Add packet trace API
+
+``src/plugins/tracedump/graph.api``
+
+* `c0b195450 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c0b195450>`_ feature: Add packet trace API
+
+``src/plugins/vrrp/vrrp.api``
+
+* `78f487e11 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=78f487e11>`_ vrrp: asynchronous events on VR state change
+
+``src/plugins/flowprobe/flowprobe.api``
+
+* `df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`_ api: vat2 and json autogeneration for api messages
+
+``src/plugins/lldp/lldp.api``
+
+* `3f9fdd984 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3f9fdd984>`_ lldp: Move to plugin
+
+``src/plugins/memif/memif.api``
+
+* `6223766f9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6223766f9>`_ libmemif: clean up typos
+
+``src/plugins/dns/dns.api``
+
+* `df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`_ api: vat2 and json autogeneration for api messages
+
+``src/plugins/stn/stn.api``
+
+* `df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`_ api: vat2 and json autogeneration for api messages
+
+``src/plugins/af_xdp/af_xdp.api``
+
+* `d4e109138 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=d4e109138>`_ af_xdp: add option to claim all available rx queues
+
+``src/plugins/gbp/gbp.api``
+
+* `df87f8092 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=df87f8092>`_ api: vat2 and json autogeneration for api messages
+
diff --git a/docs/aboutvpp/releasenotes/v21.06.rst b/docs/aboutvpp/releasenotes/v21.06.rst
new file mode 100644
index 0000000..0c5820d
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v21.06.rst
@@ -0,0 +1,1477 @@
+Release notes for VPP 21.06
+===========================
+
+More than 787 commits since the previous release, including 364 fixes.
+
+Release Highlights
+------------------
+
+There are many excellent new features in this release, however a few of
+them deserve a special mention.
+
+Linux Control Plane Plugin (linux-cp)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One of the more significant new features included in this release is a
+linux control plane (linux-cp) plugin. It enables the near-seamless
+integration of VPP with the host control plane, by mirroring the VPP
+interfaces into a TUN or TAP device created in the linux kernel. All of
+the punted packets received on the VPP interface will be sent to the
+linux counterpart, and in the reverse direction, packets sent by linux
+kernel will be transmitted out the VPP interface. This plugin lays the
+foundation for the much easier integration of external software with
+VPP.
+
+Performance Monitor Plugin (perfmon)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Another interesting feature is the performance monitor (perfmon) plugin.
+It allows collection of detailed low-level CPU statistics on a per-node
+basis. It provides a useful advanced troubleshooting tool, should you
+encounter that a specific node’s performance is not on par with what it
+should be. Note, that the correct functioning of this plugin may require
+changing the /proc/sys/kernel/perf_event_paranoid setting to enable
+access to the performance counters.
+
+API CRC Substitution Table Removal
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+And finally a clarification, not a feature per se. Some messages in the
+API changes table for this release have a new marking: “message CRC32
+fix”. The history of this issue goes back to summer of 2020, when it was
+discovered that for a considerable amount of time (several months) the
+calculation of the CRC of API messages was incorrect. Specifically, all
+but the first user-defined types in the message failed to be included in
+the CRC calculation. In plain words, this means that one might end up
+with a situation where client and VPP layout of message in memory could
+be quite different.
+
+At the time of discovery, there were no API changes that were affected
+by that bug. However, simply fixing it meant the CRC of about half of
+the VPP API messages would be altered for no reason which would result
+in a significant amount of pain to the consumers of VPP. A message CRC
+is just an opague number for anyone using it, and the only property of
+it for the user is that the messages with the same value of CRC have the
+same layout on the wire with a sufficiently high certainty.
+
+Therefore a fix
+(`9f84e70c6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9f84e70c6>`__)
+was merged that also contained a “band-aid” to avoid this pain. In
+addition to fixing the CRC generation algorithm, the fix captured the
+“new” CRC values for those messages that had their CRCs arbitrarily
+changed by this fix and created a substituion table with [message name,
+new CRC, old CRC] triplets. For a given message, if the CRC matched the
+recorded new value, the code would substitute it with the old value,
+thus trading in a reduction in collision resistance (two values of CRC
+out of 2^32 space) for not forcing users to adapt to several hundred of
+messages which changed the CRCs.
+
+This band-aid also had the property that whenever a message did change
+the definition, it would automatically get a “correct” calculation of
+CRC32 and no longer use the slot in the table. The table naturally
+shrinks over time, thus allowing a painless transition, while also
+preserving the integrity check for the affected messages. If any fields
+changed, the CRC would no longer match the “new” value thus no
+substitution would be made. Since the choice of CRC32 is just an
+implementation detail which is supposed to be opaque to the user, the
+band-aid was deemed a reasonable approach to avoid a major burden on VPP
+consumers.
+
+However, in practice this solution was not accepted well. After a
+notification period, the API CRC Substitution table was removed by the
+patch
+(`da1b76aa8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=da1b76aa8>`__).
+Thus all of the API CRC changes that did not happen in 9f84e70c6,
+happened in da1b76aa8. Hopefully the notification of these changes has
+reduced the inconvenience. VPP users are asked to not rely on any other
+property of the message CRC other than changes to its value when the
+message layout changes.
+
+Features
+--------
+
+- Binary API Compiler for Python
+
+ - Support an ‘autoendian’ keyword for message definitions in .api
+ files
+ (`9302cfea9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9302cfea9>`__)
+
+- Build System
+
+ - Make rpath optional
+ (`2c91922eb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2c91922eb>`__)
+
+- Infrastructure Library
+
+ - Add option to use libexecinfo
+ (`67d7acd05 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=67d7acd05>`__)
+ - Add bihash with 32 byte key
+ (`f613a4402 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f613a4402>`__)
+ - Add missing %o
+ (`04a14133c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=04a14133c>`__)
+
+- Plugins
+
+ - ARPing CLI
+
+ - Add arping command
+ (`a77ae4708 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a77ae4708>`__)
+
+ - AVF Device driver
+
+ - Add avf flow framework
+ (`ffe9a5489 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ffe9a5489>`__)
+
+ - CNat
+
+ - Add maglev support
+ (`4d237874e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4d237874e>`__)
+ - Add input feature node
+ (`cc9a1a0d3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cc9a1a0d3>`__)
+ - Add calico/k8s src policy
+ (`516b0adf6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=516b0adf6>`__)
+
+ - Crypto - ipsecmb
+
+ - Add support for AES CTR
+ (`fe7ff320b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fe7ff320b>`__)
+
+ - DPDK
+
+ - Rebase cryptodev engine for DPDK 20.11
+ (`25f371ee0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=25f371ee0>`__)
+ - Allow configure individual VMBUS devices
+ (`982272974 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=982272974>`__)
+ - Implement interrupt mode
+ (`19ff0c369 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=19ff0c369>`__)
+
+ - IPv6 Segment Routing Flow-Based Dynamic Proxy
+
+ - SRv6 Per-Flow Dynamic Proxy
+ (`ed7c62a30 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ed7c62a30>`__)
+
+ - Internet Key Exchange (IKEv2) Protocol
+
+ - Use new counters data model & add more counters
+ (`fab5e7f39 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fab5e7f39>`__)
+ - Add per SA stats
+ (`68d275356 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=68d275356>`__)
+ - Support responder hostname
+ (`af2cc6425 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=af2cc6425>`__)
+
+ - NAT
+
+ - 1:1 policy NAT
+ (`18327be5d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=18327be5d>`__)
+ - Pnat copy and clear byte instructions
+ (`ab3151c52 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ab3151c52>`__)
+
+ - QUIC protocol
+
+ - Quicly v0.1.2 update
+ (`2e4523816 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=2e4523816>`__)
+ - Update quicly to v0.1.3
+ (`db36fda74 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=db36fda74>`__)
+
+ - RDMA (ibverb) driver
+
+ - Add support for RSS configuration
+ (`f5a45680e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f5a45680e>`__)
+
+ - SRTP
+
+ - Basic implementation based on libsrtp2
+ (`6621abf49 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6621abf49>`__)
+
+ - TCP MSS Clamping
+
+ - TCP MSS clamping plugin
+ (`bf55e9931 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bf55e9931>`__)
+
+ - Linux-cp
+
+ - Linux Interface Mirroring for Control Plane Integration
+ (`44db1caef <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=44db1caef>`__)
+
+ - Memif device driver
+
+ - Adapt to new rxq framework
+ (`755941865 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=755941865>`__)
+
+ - Performance counter
+
+ - New perfmon plugin
+ (`8b60fb0fe <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8b60fb0fe>`__)
+
+- Python binding for the VPP API
+
+ - Expose vpp_papi version to client
+ (`b552ff2e9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b552ff2e9>`__)
+
+- SVM Library
+
+ - Allow mq attachments at random offsets
+ (`b46241889 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b46241889>`__)
+ - Per app rx message queues
+ (`41d5f541d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=41d5f541d>`__)
+
+- Statistics Segment
+
+ - Adding symlinks for nodes and interfaces in the stat segment
+ (`db0238090 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=db0238090>`__)
+ - Memory heap counters
+ (`a606d9210 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a606d9210>`__)
+
+- VNET
+
+ - Crypto Infra
+
+ - Add support for aes-ctr+sha-1 chains
+ (`40ee2003b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=40ee2003b>`__)
+ - Support hashing operations
+ (`06111a837 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=06111a837>`__)
+ - Add chacha20-poly1305 support to ipsecmb
+ (`106e24bd9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=106e24bd9>`__)
+
+ - FIB
+
+ - Allow the creation of new source on the API
+ (`976b259be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=976b259be>`__)
+
+ - FLOW
+
+ - Add API implementation of IP4/IP6, IP4_VXLAN/IP6_VXLAN
+ (`c7e7819ad <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c7e7819ad>`__)
+
+ - IPIP
+
+ - Support MPLS over IP
+ (`e294de6f8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e294de6f8>`__)
+
+ - IPSec
+
+ - Support MPLS over IPSec[46] interface
+ (`4a58e49cf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4a58e49cf>`__)
+ - Add support for AES CTR
+ (`490b92738 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=490b92738>`__)
+ - CLI improvement for udp port encap
+ (`048189e7a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=048189e7a>`__)
+ - Use the new tunnel API types to add flow label and TTL copy
+ support
+ (`c7eaa711f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c7eaa711f>`__)
+ - Use the new tunnel API types to add flow label and TTL copy
+ support
+ (`9ec846c26 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9ec846c26>`__)
+ - Support async mode per-SA
+ (`f16e9a550 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f16e9a550>`__)
+
+ - IPv4 LPM
+
+ - Add API to retrieve IPv6 link-layer address
+ (`58a1915b5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=58a1915b5>`__)
+ - Router ID included in flow hash
+ (`3d5f08a82 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3d5f08a82>`__)
+ - Path MTU
+ (`8f5fef2c7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8f5fef2c7>`__)
+ - Extend punt CLI for exception packets
+ (`45723b8d3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=45723b8d3>`__)
+ - Extend show cmd of ip reassembly configuration
+ (`74a4a70ef <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=74a4a70ef>`__)
+
+ - Interface Common
+
+ - RX/TX direction type in API
+ (`6a999d67d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6a999d67d>`__)
+ - Add promisc on/off in api
+ (`fd0b399ff <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fd0b399ff>`__)
+
+ - L2
+
+ - Add per bridge domain learn limit
+ (`5f93e3b7f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5f93e3b7f>`__)
+ - Separating scan-delay and learn-limit into a separate API from
+ want_l2_macs_events
+ (`0f8d10035 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0f8d10035>`__)
+
+ - Session Layer
+
+ - Basic support for interrupt mode
+ (`7da8829d8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=7da8829d8>`__)
+ - Api to update connection attributes
+ (`04ae8273f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=04ae8273f>`__)
+
+ - TLS and TLS engine plugins
+
+ - Dtls initial implementation
+ (`4b47ee26c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4b47ee26c>`__)
+
+ - Vhost User Driver
+
+ - Add event index for interrupt notification to driver
+ (`27ba5008a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=27ba5008a>`__)
+
+ - Tunnel
+
+ - Support copying TTL and flow label from inner to outer
+ (`a91cb4590 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a91cb4590>`__)
+
+- VPP Comms Library
+
+ - Extended connect/listen configuration
+ (`4ac258497 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4ac258497>`__)
+
+- Libmemif
+
+ - Set next free buffer
+ (`47e68de22 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=47e68de22>`__)
+ - Set data offset for memif buffer
+ (`1421748e3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=1421748e3>`__)
+
+Known issues
+------------
+
+Coverity Issues
+~~~~~~~~~~~~~~~
+
+Starting with this release, we add the section about the section with
+the unresolved Coverity Issues into the Release Notes. In order to view
+the issues, visit https://scan.coverity.com/, add yourself to fd.io VPP
+project and click on the matching IDs.
+
+Plugin - PPPoE:
+^^^^^^^^^^^^^^^
+
+- BUG 218437 in function: pppoe_input_node_fn, file:
+ /src/plugins/pppoe/pppoe_decap.c
+- BUG 218401 in function: pppoe_input_node_fn, file:
+ /src/plugins/pppoe/pppoe_decap.c #### VNET - IP6 Neighbor Discovery:
+- BUG 218382 in function: set_ip6_nd_proxy_cmd, file:
+ /src/vnet/ip6-nd/ip6_nd_proxy.c #### Plugin - TCP MSS Clamping:
+- BUG 219550 in function: vl_api_mss_clamp_enable_disable_t_handler,
+ file: /src/plugins/mss_clamp/mss_clamp_api.c #### Plugin -
+ performance counter:
+- BUG 216295 in function: format_text_cell, file:
+ /src/plugins/perfmon/table.c
+- BUG 218459 in function: intel_uncore_init, file:
+ /src/plugins/perfmon/intel/uncore.c
+- BUG 216249 in function: perfmon_reset, file:
+ /src/plugins/perfmon/perfmon.c #### Plugin - DPDK:
+- BUG 220290 in function: dpdk_lib_init, file:
+ /src/plugins/dpdk/device/init.c
+- BUG 220289 in function: dpdk_lib_init, file:
+ /src/plugins/dpdk/device/init.c
+- BUG 220105 in function: cryptodev_get_common_capabilities, file:
+ /src/plugins/dpdk/cryptodev/cryptodev.c #### VNET IPv4 LPM:
+- BUG 216981 in function: icmp_to_icmp6, file:
+ /src/vnet/ip/ip4_to_ip6.h
+- BUG 214755 in function: ip_in_out_acl_inline, file:
+ /src/vnet/ip/ip_in_out_acl.c
+- BUG 220099 in function: vl_api_ip_route_lookup_v2_t_handler, file:
+ /src/vnet/ip/ip_api.c #### Plugin - Unit Tests:
+- BUG 218446 in function: test_crypto_perf, file:
+ /src/plugins/unittest/crypto_test.c #### Plugin - NSH:
+- BUG 218432 in function: nsh_add_del_entry, file:
+ /src/plugins/nsh/nsh_api.c #### Vector Library - PCI:
+- BUG 218391 in function: vlib_pci_device_open, file:
+ /src/vlib/linux/pci.c
+- BUG 218396 in function: linux_pci_init, file: /src/vlib/linux/pci.c
+ #### VNET Segment Routing (IPv6 and MPLS):
+- BUG 218375 in function: sr_policy_del, file:
+ /src/vnet/srv6/sr_policy_rewrite.c
+- BUG 218409 in function: sr_steering_policy, file:
+ /src/vnet/srv6/sr_steering.c
+- BUG 218427 in function: sr_policy_mod, file:
+ /src/vnet/srv6/sr_policy_rewrite.c
+- BUG 180995 in function: sr_mpls_policy_assign_endpoint_color, file:
+ /src/vnet/srmpls/sr_mpls_policy.c #### Vector Library:
+- BUG 218552 in function: add_sub_command, file: /src/vlib/cli.c ####
+ VNET FIB:
+- BUG 216057 in function: fib_sas6_get, file: /src/vnet/fib/fib_sas.c
+ #### VNET Ethernet:
+- BUG 214973 in function: ethernet_input_inline, file:
+ /src/vnet/ethernet/node.c
+- BUG 218549 in function: identify_subint, file:
+ /src/vnet/ethernet/node.c #### Infrastructure Library:
+- BUG 236112 in function: extract_bits, file: /src/vppinfra/clib.h ####
+ Binary API Compiler for C and C++:
+- BUG 236138 in function: test_loopbacks_2, file:
+ /src/vpp-api/vapi/vapi_cpp_test.cpp
+- BUG 236140 in function: test_loopbacks_1, file:
+ /src/vpp-api/vapi/vapi_cpp_test.cpp
+- BUG 236139 in function: Create_loopback_cb, file:
+ /src/vpp-api/vapi/vapi_cpp_test.cpp
+- BUG 236136 in function: test_api_strings, file:
+ /src/vpp-api/vapi/vapi_c_test.c
+- BUG 236137 in function: Delete_loopback_cb, file:
+ /src/vpp-api/vapi/vapi_cpp_test.cpp #### Plugin - IPv6 Segment
+ Routing Masquerading Proxy:
+- BUG 218441 in function: srv6_am_localsid_removal_fn, file:
+ /src/plugins/srv6-am/am.c #### VNET Policer:
+- BUG 218398 in function: show_policer_command_fn, file:
+ /src/vnet/policer/policer.c #### Plugin - DHCP:
+- BUG 218381 in function: dhcpv6_proxy_to_client_input, file:
+ /src/plugins/dhcp/dhcp6_proxy_node.c #### Plugin - IOAM:
+- BUG 216232 in function: ioam_cache_ts_table_destroy, file:
+ /src/plugins/ioam/ip6/ioam_cache.h #### VNET IPv6 LPM:
+- BUG 216981 in function: icmp_to_icmp6, file:
+ /src/vnet/ip/ip4_to_ip6.h
+- BUG 214755 in function: ip_in_out_acl_inline, file:
+ /src/vnet/ip/ip_in_out_acl.c
+- BUG 220099 in function: vl_api_ip_route_lookup_v2_t_handler, file:
+ /src/vnet/ip/ip_api.c
+
+Jira Issues
+~~~~~~~~~~~
+
+For the full list of issues please refer to fd.io
+`JIRA <https://jira.fd.io>`__.
+
+Fixed issues
+------------
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/2106>`__
+
+
+API changes
+-----------
+
+Description of results:
+
+- *Definition changed*: indicates that the API file was modified
+ between releases.
+- *Only in image*: indicates the API is new for this release.
+- *Only in file*: indicates the API has been removed in this release.
+- *Message CRC32 fix*: please refer to release highlights for
+ description.
+
+=============================================== ==================
+Message Name Result
+=============================================== ==================
+abf_policy_add_del message CRC32 fix
+abf_policy_details message CRC32 fix
+acl_add_replace message CRC32 fix
+acl_details message CRC32 fix
+af_xdp_create definition changed
+arping only in image
+arping_reply only in image
+bd_ip_mac_add_del message CRC32 fix
+bd_ip_mac_details message CRC32 fix
+bfd_udp_add message CRC32 fix
+bfd_udp_auth_activate message CRC32 fix
+bfd_udp_auth_deactivate message CRC32 fix
+bfd_udp_del message CRC32 fix
+bfd_udp_get_echo_source_reply message CRC32 fix
+bfd_udp_mod message CRC32 fix
+bfd_udp_session_details message CRC32 fix
+bfd_udp_session_event only in image
+bfd_udp_session_set_flags message CRC32 fix
+bier_disp_entry_add_del message CRC32 fix
+bier_disp_entry_details message CRC32 fix
+bier_route_add_del message CRC32 fix
+bier_route_details message CRC32 fix
+bond_create message CRC32 fix
+bond_enslave message CRC32 fix
+bridge_domain_details message CRC32 fix
+bridge_domain_set_default_learn_limit only in image
+bridge_domain_set_default_learn_limit_reply only in image
+bridge_domain_set_learn_limit only in image
+bridge_domain_set_learn_limit_reply only in image
+cnat_add_del_snat_prefix only in file
+cnat_add_del_snat_prefix_reply only in file
+cnat_session_details definition changed
+cnat_set_snat_policy only in image
+cnat_set_snat_policy_reply only in image
+cnat_snat_policy_add_del_exclude_pfx only in image
+cnat_snat_policy_add_del_exclude_pfx_reply only in image
+cnat_snat_policy_add_del_if only in image
+cnat_snat_policy_add_del_if_reply only in image
+cnat_translation_details definition changed
+cnat_translation_update definition changed
+cop_interface_enable_disable only in file
+cop_interface_enable_disable_reply only in file
+cop_whitelist_enable_disable only in file
+cop_whitelist_enable_disable_reply only in file
+create_subif message CRC32 fix
+create_vhost_user_if_v2 only in image
+create_vhost_user_if_v2_reply only in image
+dhcp6_pd_reply_event message CRC32 fix
+dhcp6_pd_send_client_message message CRC32 fix
+dhcp6_reply_event message CRC32 fix
+dhcp6_send_client_message message CRC32 fix
+dhcp_client_config message CRC32 fix
+dhcp_client_details message CRC32 fix
+dhcp_compl_event message CRC32 fix
+dhcp_proxy_config message CRC32 fix
+dhcp_proxy_details message CRC32 fix
+dslite_add_del_pool_addr_range message CRC32 fix
+dslite_get_aftr_addr_reply message CRC32 fix
+dslite_get_b4_addr_reply message CRC32 fix
+dslite_set_aftr_addr message CRC32 fix
+dslite_set_b4_addr message CRC32 fix
+fib_source_add only in image
+fib_source_add_reply only in image
+fib_source_details only in image
+fib_source_dump only in image
+flow_add definition changed
+gbp_bridge_domain_add message CRC32 fix
+gbp_bridge_domain_details message CRC32 fix
+gbp_contract_add_del message CRC32 fix
+gbp_contract_details message CRC32 fix
+gbp_endpoint_add message CRC32 fix
+gbp_endpoint_details message CRC32 fix
+gbp_endpoint_group_add message CRC32 fix
+gbp_endpoint_group_details message CRC32 fix
+gbp_ext_itf_add_del message CRC32 fix
+gbp_ext_itf_details message CRC32 fix
+gbp_route_domain_add message CRC32 fix
+gbp_route_domain_details message CRC32 fix
+gbp_subnet_add_del message CRC32 fix
+gbp_subnet_details message CRC32 fix
+geneve_add_del_tunnel message CRC32 fix
+geneve_tunnel_details message CRC32 fix
+gpe_add_del_fwd_entry message CRC32 fix
+gpe_add_del_native_fwd_rpath message CRC32 fix
+gpe_fwd_entries_get_reply message CRC32 fix
+gpe_fwd_entry_path_details message CRC32 fix
+gpe_native_fwd_rpaths_get_reply message CRC32 fix
+gre_tunnel_add_del message CRC32 fix
+gre_tunnel_details message CRC32 fix
+gtpu_add_del_tunnel message CRC32 fix
+gtpu_tunnel_details message CRC32 fix
+gtpu_tunnel_update_tteid message CRC32 fix
+igmp_details message CRC32 fix
+igmp_event message CRC32 fix
+igmp_group_prefix_details message CRC32 fix
+igmp_group_prefix_set message CRC32 fix
+igmp_listen message CRC32 fix
+ikev2_sa_details definition changed
+ikev2_set_responder_hostname only in image
+ikev2_set_responder_hostname_reply only in image
+ioam_export_ip6_enable_disable message CRC32 fix
+ip6_add_del_address_using_prefix message CRC32 fix
+ip6_ra_event message CRC32 fix
+ip6nd_proxy_add_del message CRC32 fix
+ip6nd_proxy_details message CRC32 fix
+ip_address_details message CRC32 fix
+ip_container_proxy_add_del message CRC32 fix
+ip_container_proxy_details message CRC32 fix
+ip_neighbor_add_del message CRC32 fix
+ip_neighbor_details message CRC32 fix
+ip_neighbor_dump message CRC32 fix
+ip_neighbor_event message CRC32 fix
+ip_path_mtu_details only in image
+ip_path_mtu_get only in image
+ip_path_mtu_get_reply only in image
+ip_path_mtu_replace_begin only in image
+ip_path_mtu_replace_begin_reply only in image
+ip_path_mtu_replace_end only in image
+ip_path_mtu_replace_end_reply only in image
+ip_path_mtu_update only in image
+ip_path_mtu_update_reply only in image
+ip_punt_redirect message CRC32 fix
+ip_punt_redirect_details message CRC32 fix
+ip_reassembly_enable_disable message CRC32 fix
+ip_route_add_del message CRC32 fix
+ip_route_add_del_v2 only in image
+ip_route_add_del_v2_reply only in image
+ip_route_details message CRC32 fix
+ip_route_lookup message CRC32 fix
+ip_route_lookup_reply message CRC32 fix
+ip_route_lookup_v2 only in image
+ip_route_lookup_v2_reply only in image
+ip_route_v2_details only in image
+ip_route_v2_dump only in image
+ip_source_and_port_range_check_add_del message CRC32 fix
+ip_unnumbered_details message CRC32 fix
+ipfix_exporter_details message CRC32 fix
+ipip_6rd_add_tunnel message CRC32 fix
+ipip_add_tunnel message CRC32 fix
+ipip_tunnel_details message CRC32 fix
+ipsec_sa_details message CRC32 fix
+ipsec_sa_v3_details only in image
+ipsec_sa_v3_dump only in image
+ipsec_sad_entry_add_del message CRC32 fix
+ipsec_sad_entry_add_del_v3 only in image
+ipsec_sad_entry_add_del_v3_reply only in image
+ipsec_spd_details message CRC32 fix
+ipsec_spd_entry_add_del message CRC32 fix
+ipsec_tunnel_if_add_del only in file
+ipsec_tunnel_if_add_del_reply only in file
+ipsec_tunnel_if_set_sa only in file
+ipsec_tunnel_if_set_sa_reply only in file
+ipsec_tunnel_protect_del message CRC32 fix
+ipsec_tunnel_protect_details message CRC32 fix
+ipsec_tunnel_protect_update message CRC32 fix
+l2_arp_term_event message CRC32 fix
+l2_fib_table_details message CRC32 fix
+l2_interface_pbb_tag_rewrite message CRC32 fix
+l2_macs_event message CRC32 fix
+l2_patch_add_del message CRC32 fix
+l2_xconnect_details message CRC32 fix
+l2fib_add_del message CRC32 fix
+l2fib_set_scan_delay only in image
+l2fib_set_scan_delay_reply only in image
+l2tpv3_create_tunnel message CRC32 fix
+l3xc_details message CRC32 fix
+l3xc_update message CRC32 fix
+lb_add_del_as message CRC32 fix
+lb_add_del_vip message CRC32 fix
+lb_as_details message CRC32 fix
+lb_conf message CRC32 fix
+lb_vip_details message CRC32 fix
+lb_vip_dump message CRC32 fix
+lisp_add_del_adjacency message CRC32 fix
+lisp_add_del_local_eid message CRC32 fix
+lisp_add_del_map_resolver message CRC32 fix
+lisp_add_del_map_server message CRC32 fix
+lisp_add_del_remote_mapping message CRC32 fix
+lisp_adjacencies_get_reply message CRC32 fix
+lisp_eid_table_details message CRC32 fix
+lisp_eid_table_dump message CRC32 fix
+lisp_locator_details message CRC32 fix
+lisp_map_resolver_details message CRC32 fix
+lisp_map_server_details message CRC32 fix
+lisp_use_petr message CRC32 fix
+log_details message CRC32 fix
+macip_acl_add message CRC32 fix
+macip_acl_add_replace message CRC32 fix
+macip_acl_details message CRC32 fix
+mactime_add_del_range message CRC32 fix
+mactime_details message CRC32 fix
+map_add_domain message CRC32 fix
+map_domain_details message CRC32 fix
+map_param_add_del_pre_resolve message CRC32 fix
+map_param_get_reply message CRC32 fix
+memif_details message CRC32 fix
+mfib_signal_details message CRC32 fix
+modify_vhost_user_if_v2 only in image
+modify_vhost_user_if_v2_reply only in image
+mpls_ip_bind_unbind message CRC32 fix
+mpls_route_add_del message CRC32 fix
+mpls_route_details message CRC32 fix
+mpls_tunnel_add_del message CRC32 fix
+mpls_tunnel_details message CRC32 fix
+mss_clamp_details only in image
+mss_clamp_enable_disable only in image
+mss_clamp_enable_disable_reply only in image
+mss_clamp_get only in image
+mss_clamp_get_reply only in image
+nat44_add_del_address_range message CRC32 fix
+nat44_add_del_identity_mapping message CRC32 fix
+nat44_add_del_interface_addr message CRC32 fix
+nat44_add_del_lb_static_mapping message CRC32 fix
+nat44_add_del_static_mapping message CRC32 fix
+nat44_address_details message CRC32 fix
+nat44_del_session message CRC32 fix
+nat44_ed_plugin_enable_disable only in image
+nat44_ed_plugin_enable_disable_reply only in image
+nat44_ed_set_fq_options only in image
+nat44_ed_set_fq_options_reply only in image
+nat44_ed_show_fq_options only in image
+nat44_ed_show_fq_options_reply only in image
+nat44_ei_add_del_address_range only in image
+nat44_ei_add_del_address_range_reply only in image
+nat44_ei_add_del_identity_mapping only in image
+nat44_ei_add_del_identity_mapping_reply only in image
+nat44_ei_add_del_interface_addr only in image
+nat44_ei_add_del_interface_addr_reply only in image
+nat44_ei_add_del_static_mapping only in image
+nat44_ei_add_del_static_mapping_reply only in image
+nat44_ei_address_details only in image
+nat44_ei_address_dump only in image
+nat44_ei_del_session only in image
+nat44_ei_del_session_reply only in image
+nat44_ei_del_user only in image
+nat44_ei_del_user_reply only in image
+nat44_ei_forwarding_enable_disable only in image
+nat44_ei_forwarding_enable_disable_reply only in image
+nat44_ei_get_addr_and_port_alloc_alg only in image
+nat44_ei_get_addr_and_port_alloc_alg_reply only in image
+nat44_ei_get_mss_clamping only in image
+nat44_ei_get_mss_clamping_reply only in image
+nat44_ei_ha_flush only in image
+nat44_ei_ha_flush_reply only in image
+nat44_ei_ha_get_failover only in image
+nat44_ei_ha_get_failover_reply only in image
+nat44_ei_ha_get_listener only in image
+nat44_ei_ha_get_listener_reply only in image
+nat44_ei_ha_resync only in image
+nat44_ei_ha_resync_completed_event only in image
+nat44_ei_ha_resync_reply only in image
+nat44_ei_ha_set_failover only in image
+nat44_ei_ha_set_failover_reply only in image
+nat44_ei_ha_set_listener only in image
+nat44_ei_ha_set_listener_reply only in image
+nat44_ei_identity_mapping_details only in image
+nat44_ei_identity_mapping_dump only in image
+nat44_ei_interface_add_del_feature only in image
+nat44_ei_interface_add_del_feature_reply only in image
+nat44_ei_interface_add_del_output_feature only in image
+nat44_ei_interface_add_del_output_feature_reply only in image
+nat44_ei_interface_addr_details only in image
+nat44_ei_interface_addr_dump only in image
+nat44_ei_interface_details only in image
+nat44_ei_interface_dump only in image
+nat44_ei_interface_output_feature_details only in image
+nat44_ei_interface_output_feature_dump only in image
+nat44_ei_ipfix_enable_disable only in image
+nat44_ei_ipfix_enable_disable_reply only in image
+nat44_ei_plugin_enable_disable only in image
+nat44_ei_plugin_enable_disable_reply only in image
+nat44_ei_set_addr_and_port_alloc_alg only in image
+nat44_ei_set_addr_and_port_alloc_alg_reply only in image
+nat44_ei_set_fq_options only in image
+nat44_ei_set_fq_options_reply only in image
+nat44_ei_set_log_level only in image
+nat44_ei_set_log_level_reply only in image
+nat44_ei_set_mss_clamping only in image
+nat44_ei_set_mss_clamping_reply only in image
+nat44_ei_set_timeouts only in image
+nat44_ei_set_timeouts_reply only in image
+nat44_ei_set_workers only in image
+nat44_ei_set_workers_reply only in image
+nat44_ei_show_fq_options only in image
+nat44_ei_show_fq_options_reply only in image
+nat44_ei_show_running_config only in image
+nat44_ei_show_running_config_reply only in image
+nat44_ei_static_mapping_details only in image
+nat44_ei_static_mapping_dump only in image
+nat44_ei_user_details only in image
+nat44_ei_user_dump only in image
+nat44_ei_user_session_details only in image
+nat44_ei_user_session_dump only in image
+nat44_ei_worker_details only in image
+nat44_ei_worker_dump only in image
+nat44_identity_mapping_details message CRC32 fix
+nat44_interface_addr_details message CRC32 fix
+nat44_lb_static_mapping_add_del_local message CRC32 fix
+nat44_lb_static_mapping_details message CRC32 fix
+nat44_static_mapping_details message CRC32 fix
+nat44_user_session_details message CRC32 fix
+nat64_add_del_pool_addr_range message CRC32 fix
+nat64_add_del_static_bib message CRC32 fix
+nat64_bib_details message CRC32 fix
+nat64_st_details message CRC32 fix
+nat66_add_del_static_mapping message CRC32 fix
+nat66_plugin_enable_disable only in image
+nat66_plugin_enable_disable_reply only in image
+nat66_static_mapping_details message CRC32 fix
+nat_det_add_del_map message CRC32 fix
+nat_det_close_session_in message CRC32 fix
+nat_det_close_session_out message CRC32 fix
+nat_det_map_details message CRC32 fix
+nsh_add_del_map message CRC32 fix
+nsh_map_details message CRC32 fix
+nsim_cross_connect_enable_disable message CRC32 fix
+one_add_del_adjacency message CRC32 fix
+one_add_del_l2_arp_entry message CRC32 fix
+one_add_del_local_eid message CRC32 fix
+one_add_del_map_resolver message CRC32 fix
+one_add_del_map_server message CRC32 fix
+one_add_del_ndp_entry message CRC32 fix
+one_add_del_remote_mapping message CRC32 fix
+one_adjacencies_get_reply message CRC32 fix
+one_eid_table_details message CRC32 fix
+one_eid_table_dump message CRC32 fix
+one_l2_arp_entries_get_reply message CRC32 fix
+one_locator_details message CRC32 fix
+one_map_resolver_details message CRC32 fix
+one_map_server_details message CRC32 fix
+one_ndp_entries_get_reply message CRC32 fix
+one_stats_details message CRC32 fix
+one_use_petr message CRC32 fix
+p2p_ethernet_add message CRC32 fix
+p2p_ethernet_del message CRC32 fix
+pipe_create_reply message CRC32 fix
+pipe_details message CRC32 fix
+pnat_binding_add only in image
+pnat_binding_add_reply only in image
+pnat_binding_attach only in image
+pnat_binding_attach_reply only in image
+pnat_binding_del only in image
+pnat_binding_del_reply only in image
+pnat_binding_detach only in image
+pnat_binding_detach_reply only in image
+pnat_bindings_details only in image
+pnat_bindings_get only in image
+pnat_bindings_get_reply only in image
+pnat_interfaces_details only in image
+pnat_interfaces_get only in image
+pnat_interfaces_get_reply only in image
+policer_add_del message CRC32 fix
+policer_bind only in image
+policer_bind_reply only in image
+policer_details message CRC32 fix
+policer_input only in image
+policer_input_reply only in image
+pppoe_add_del_session message CRC32 fix
+pppoe_session_details message CRC32 fix
+proxy_arp_add_del message CRC32 fix
+proxy_arp_details message CRC32 fix
+punt_socket_deregister message CRC32 fix
+punt_socket_details message CRC32 fix
+punt_socket_register message CRC32 fix
+qos_record_details message CRC32 fix
+qos_record_enable_disable message CRC32 fix
+qos_store_details message CRC32 fix
+qos_store_enable_disable message CRC32 fix
+rdma_create_v3 only in image
+rdma_create_v3_reply only in image
+session_rule_add_del message CRC32 fix
+session_rules_details message CRC32 fix
+set_ip_flow_hash_router_id only in image
+set_ip_flow_hash_router_id_reply only in image
+set_ip_flow_hash_v2 only in image
+set_ip_flow_hash_v2_reply only in image
+set_ipfix_exporter message CRC32 fix
+set_punt message CRC32 fix
+show_lisp_use_petr_reply message CRC32 fix
+show_one_use_petr_reply message CRC32 fix
+sr_localsid_add_del message CRC32 fix
+sr_localsids_details message CRC32 fix
+sr_mpls_policy_assign_endpoint_color message CRC32 fix
+sr_mpls_steering_add_del message CRC32 fix
+sr_policies_details message CRC32 fix
+sr_policy_add message CRC32 fix
+sr_policy_mod message CRC32 fix
+sr_steering_add_del message CRC32 fix
+sr_steering_pol_details message CRC32 fix
+stn_add_del_rule message CRC32 fix
+stn_rules_details message CRC32 fix
+svs_details message CRC32 fix
+svs_route_add_del message CRC32 fix
+sw_if_l2tpv3_tunnel_details message CRC32 fix
+sw_interface_add_del_address message CRC32 fix
+sw_interface_bond_details message CRC32 fix
+sw_interface_details message CRC32 fix
+sw_interface_event message CRC32 fix
+sw_interface_ip6_get_link_local_address only in image
+sw_interface_ip6_get_link_local_address_reply only in image
+sw_interface_ip6_set_link_local_address message CRC32 fix
+sw_interface_ip6nd_ra_prefix message CRC32 fix
+sw_interface_lacp_details message CRC32 fix
+sw_interface_rx_placement_details message CRC32 fix
+sw_interface_set_flags message CRC32 fix
+sw_interface_set_l2_bridge message CRC32 fix
+sw_interface_set_l2_xconnect message CRC32 fix
+sw_interface_set_lldp message CRC32 fix
+sw_interface_set_mac_address message CRC32 fix
+sw_interface_set_promisc only in image
+sw_interface_set_promisc_reply only in image
+sw_interface_set_rx_mode message CRC32 fix
+sw_interface_set_unnumbered message CRC32 fix
+sw_interface_span_details message CRC32 fix
+sw_interface_span_enable_disable message CRC32 fix
+sw_interface_tap_v2_details message CRC32 fix
+sw_interface_vhost_user_details message CRC32 fix
+sw_interface_virtio_pci_details message CRC32 fix
+syslog_get_sender_reply message CRC32 fix
+syslog_set_sender message CRC32 fix
+tap_create_v2 message CRC32 fix
+tcp_configure_src_addresses message CRC32 fix
+teib_details message CRC32 fix
+teib_entry_add_del message CRC32 fix
+udp_encap_add message CRC32 fix
+udp_encap_details message CRC32 fix
+udp_ping_add_del message CRC32 fix
+virtio_pci_create message CRC32 fix
+vmxnet3_details message CRC32 fix
+vrrp_vr_add_del message CRC32 fix
+vrrp_vr_details message CRC32 fix
+vrrp_vr_peer_details message CRC32 fix
+vrrp_vr_set_peers message CRC32 fix
+vrrp_vr_track_if_add_del message CRC32 fix
+vrrp_vr_track_if_details message CRC32 fix
+vxlan_add_del_tunnel message CRC32 fix
+vxlan_add_del_tunnel_v2 only in image
+vxlan_add_del_tunnel_v2_reply only in image
+vxlan_add_del_tunnel_v3 only in image
+vxlan_add_del_tunnel_v3_reply only in image
+vxlan_gbp_tunnel_add_del message CRC32 fix
+vxlan_gbp_tunnel_details message CRC32 fix
+vxlan_gpe_add_del_tunnel message CRC32 fix
+vxlan_gpe_ioam_export_enable_disable message CRC32 fix
+vxlan_gpe_ioam_transit_disable message CRC32 fix
+vxlan_gpe_ioam_transit_enable message CRC32 fix
+vxlan_gpe_ioam_vni_disable message CRC32 fix
+vxlan_gpe_ioam_vni_enable message CRC32 fix
+vxlan_gpe_tunnel_details message CRC32 fix
+vxlan_offload_rx message CRC32 fix
+vxlan_tunnel_details message CRC32 fix
+vxlan_tunnel_v2_details only in image
+vxlan_tunnel_v2_dump only in image
+want_ip_neighbor_events message CRC32 fix
+want_l2_macs_events2 only in image
+want_l2_macs_events2_reply only in image
+=============================================== ==================
+
+Found 456 api message signature differences
+
+Newly deprecated API messages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+These messages are still there in the API, but can and probably will
+disappear in the next release.
+
+- application_tls_cert_add
+- application_tls_cert_add_reply
+- application_tls_key_add
+- application_tls_key_add_reply
+- create_vhost_user_if
+- create_vhost_user_if_reply
+- ipsec_sa_details
+- ipsec_sa_dump
+- ipsec_sad_entry_add_del
+- ipsec_sad_entry_add_del_reply
+- modify_vhost_user_if
+- modify_vhost_user_if_reply
+- nat44_ei_add_del_identity_mapping
+- nat44_ei_add_del_identity_mapping_reply
+- nat44_ei_add_del_interface_addr
+- nat44_ei_add_del_interface_addr_reply
+- nat44_ei_get_addr_and_port_alloc_alg
+- nat44_ei_get_addr_and_port_alloc_alg_reply
+- nat44_ei_get_mss_clamping
+- nat44_ei_get_mss_clamping_reply
+- nat44_ei_ha_get_failover
+- nat44_ei_ha_get_failover_reply
+- nat44_ei_ha_get_listener
+- nat44_ei_ha_get_listener_reply
+- nat44_ei_identity_mapping_details
+- nat44_ei_identity_mapping_dump
+- nat44_ei_interface_add_del_output_feature
+- nat44_ei_interface_add_del_output_feature_reply
+- nat44_ei_interface_addr_details
+- nat44_ei_interface_addr_dump
+- nat44_ei_interface_output_feature_details
+- nat44_ei_interface_output_feature_dump
+- nat44_ei_set_log_level
+- nat44_ei_set_log_level_reply
+- nat44_forwarding_enable_disable
+- nat44_forwarding_enable_disable_reply
+- nat44_forwarding_is_enabled
+- nat44_forwarding_is_enabled_reply
+- nat44_plugin_enable_disable
+- nat44_plugin_enable_disable_reply
+- nat44_session_cleanup
+- nat44_session_cleanup_reply
+- nat_control_ping
+- nat_control_ping_reply
+- nat_get_timeouts
+- nat_get_timeouts_reply
+- nat_ipfix_enable_disable
+- nat_ipfix_enable_disable_reply
+- nat_set_log_level
+- nat_set_log_level_reply
+- nat_set_timeouts
+- nat_set_timeouts_reply
+- nat_show_config
+- nat_show_config_2
+- nat_show_config_2_reply
+- nat_show_config_reply
+- rdma_create_v2
+- set_ip_flow_hash
+- set_ip_flow_hash_reply
+- want_l2_macs_events
+- want_l2_macs_events_reply
+
+In-progress API messages
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+These messages are provided for testing and experimentation only. They
+are *not* subject to any compatibility process, and therefore can
+arbitrarily change or disappear at *any* moment. Also they may have less
+than satisfactory testing, making them unsuitable for other use than the
+technology preview. If you are intending to use these messages in
+production projects, please collaborate with the feature maintainer on
+their productization.
+
+- abf_itf_attach_add_del
+- abf_itf_attach_add_del_reply
+- abf_itf_attach_details
+- abf_itf_attach_dump
+- abf_plugin_get_version
+- abf_plugin_get_version_reply
+- abf_policy_add_del
+- abf_policy_add_del_reply
+- abf_policy_details
+- abf_policy_dump
+- adl_allowlist_enable_disable
+- adl_allowlist_enable_disable_reply
+- adl_interface_enable_disable
+- adl_interface_enable_disable_reply
+- af_xdp_create
+- af_xdp_create_reply
+- af_xdp_delete
+- af_xdp_delete_reply
+- cnat_get_snat_addresses
+- cnat_get_snat_addresses_reply
+- cnat_session_details
+- cnat_session_dump
+- cnat_session_purge
+- cnat_session_purge_reply
+- cnat_set_snat_addresses
+- cnat_set_snat_addresses_reply
+- cnat_set_snat_policy
+- cnat_set_snat_policy_reply
+- cnat_snat_policy_add_del_exclude_pfx
+- cnat_snat_policy_add_del_exclude_pfx_reply
+- cnat_snat_policy_add_del_if
+- cnat_snat_policy_add_del_if_reply
+- cnat_translation_del
+- cnat_translation_del_reply
+- cnat_translation_details
+- cnat_translation_dump
+- cnat_translation_update
+- cnat_translation_update_reply
+- crypto_sw_scheduler_set_worker
+- crypto_sw_scheduler_set_worker_reply
+- det44_get_timeouts_reply
+- det44_interface_add_del_feature
+- det44_interface_add_del_feature_reply
+- det44_interface_details
+- det44_interface_dump
+- det44_plugin_enable_disable
+- det44_plugin_enable_disable_reply
+- det44_set_timeouts
+- det44_set_timeouts_reply
+- flow_add
+- flow_add_reply
+- flow_del
+- flow_del_reply
+- flow_disable
+- flow_disable_reply
+- flow_enable
+- flow_enable_reply
+- gbp_bridge_domain_add
+- gbp_bridge_domain_add_reply
+- gbp_bridge_domain_del
+- gbp_bridge_domain_del_reply
+- gbp_bridge_domain_details
+- gbp_bridge_domain_dump
+- gbp_bridge_domain_dump_reply
+- gbp_contract_add_del
+- gbp_contract_add_del_reply
+- gbp_contract_details
+- gbp_contract_dump
+- gbp_endpoint_add
+- gbp_endpoint_add_reply
+- gbp_endpoint_del
+- gbp_endpoint_del_reply
+- gbp_endpoint_details
+- gbp_endpoint_dump
+- gbp_endpoint_group_add
+- gbp_endpoint_group_add_reply
+- gbp_endpoint_group_del
+- gbp_endpoint_group_del_reply
+- gbp_endpoint_group_details
+- gbp_endpoint_group_dump
+- gbp_ext_itf_add_del
+- gbp_ext_itf_add_del_reply
+- gbp_ext_itf_details
+- gbp_ext_itf_dump
+- gbp_recirc_add_del
+- gbp_recirc_add_del_reply
+- gbp_recirc_details
+- gbp_recirc_dump
+- gbp_route_domain_add
+- gbp_route_domain_add_reply
+- gbp_route_domain_del
+- gbp_route_domain_del_reply
+- gbp_route_domain_details
+- gbp_route_domain_dump
+- gbp_route_domain_dump_reply
+- gbp_subnet_add_del
+- gbp_subnet_add_del_reply
+- gbp_subnet_details
+- gbp_subnet_dump
+- gbp_vxlan_tunnel_add
+- gbp_vxlan_tunnel_add_reply
+- gbp_vxlan_tunnel_del
+- gbp_vxlan_tunnel_del_reply
+- gbp_vxlan_tunnel_details
+- gbp_vxlan_tunnel_dump
+- ikev2_child_sa_details
+- ikev2_child_sa_dump
+- ikev2_initiate_del_child_sa
+- ikev2_initiate_del_child_sa_reply
+- ikev2_initiate_del_ike_sa
+- ikev2_initiate_del_ike_sa_reply
+- ikev2_initiate_rekey_child_sa
+- ikev2_initiate_rekey_child_sa_reply
+- ikev2_initiate_sa_init
+- ikev2_initiate_sa_init_reply
+- ikev2_nonce_get
+- ikev2_nonce_get_reply
+- ikev2_profile_add_del
+- ikev2_profile_add_del_reply
+- ikev2_profile_details
+- ikev2_profile_disable_natt
+- ikev2_profile_disable_natt_reply
+- ikev2_profile_dump
+- ikev2_profile_set_auth
+- ikev2_profile_set_auth_reply
+- ikev2_profile_set_id
+- ikev2_profile_set_id_reply
+- ikev2_profile_set_ipsec_udp_port
+- ikev2_profile_set_ipsec_udp_port_reply
+- ikev2_profile_set_liveness
+- ikev2_profile_set_liveness_reply
+- ikev2_profile_set_ts
+- ikev2_profile_set_ts_reply
+- ikev2_profile_set_udp_encap
+- ikev2_profile_set_udp_encap_reply
+- ikev2_sa_details
+- ikev2_sa_dump
+- ikev2_set_esp_transforms
+- ikev2_set_esp_transforms_reply
+- ikev2_set_ike_transforms
+- ikev2_set_ike_transforms_reply
+- ikev2_set_local_key
+- ikev2_set_local_key_reply
+- ikev2_set_responder
+- ikev2_set_responder_hostname
+- ikev2_set_responder_hostname_reply
+- ikev2_set_responder_reply
+- ikev2_set_sa_lifetime
+- ikev2_set_sa_lifetime_reply
+- ikev2_set_tunnel_interface
+- ikev2_set_tunnel_interface_reply
+- ikev2_traffic_selector_details
+- ikev2_traffic_selector_dump
+- ip_route_add_del_v2
+- ip_route_add_del_v2_reply
+- ip_route_lookup_v2
+- ip_route_lookup_v2_reply
+- ip_route_v2_details
+- ip_route_v2_dump
+- l2_emulation
+- l2_emulation_reply
+- mdata_enable_disable
+- mdata_enable_disable_reply
+- nat44_add_del_static_mapping_v2
+- nat44_add_del_static_mapping_v2_reply
+- nat44_ed_plugin_enable_disable
+- nat44_ed_plugin_enable_disable_reply
+- nat44_ed_set_fq_options
+- nat44_ed_set_fq_options_reply
+- nat44_ed_show_fq_options
+- nat44_ed_show_fq_options_reply
+- nat44_ei_add_del_address_range
+- nat44_ei_add_del_address_range_reply
+- nat44_ei_add_del_static_mapping
+- nat44_ei_add_del_static_mapping_reply
+- nat44_ei_address_details
+- nat44_ei_address_dump
+- nat44_ei_del_session
+- nat44_ei_del_session_reply
+- nat44_ei_del_user
+- nat44_ei_del_user_reply
+- nat44_ei_forwarding_enable_disable
+- nat44_ei_forwarding_enable_disable_reply
+- nat44_ei_ha_flush
+- nat44_ei_ha_flush_reply
+- nat44_ei_ha_resync
+- nat44_ei_ha_resync_completed_event
+- nat44_ei_ha_resync_reply
+- nat44_ei_ha_set_failover
+- nat44_ei_ha_set_failover_reply
+- nat44_ei_ha_set_listener
+- nat44_ei_ha_set_listener_reply
+- nat44_ei_interface_add_del_feature
+- nat44_ei_interface_add_del_feature_reply
+- nat44_ei_interface_details
+- nat44_ei_interface_dump
+- nat44_ei_ipfix_enable_disable
+- nat44_ei_ipfix_enable_disable_reply
+- nat44_ei_plugin_enable_disable
+- nat44_ei_plugin_enable_disable_reply
+- nat44_ei_set_addr_and_port_alloc_alg
+- nat44_ei_set_addr_and_port_alloc_alg_reply
+- nat44_ei_set_fq_options
+- nat44_ei_set_fq_options_reply
+- nat44_ei_set_mss_clamping
+- nat44_ei_set_mss_clamping_reply
+- nat44_ei_set_timeouts
+- nat44_ei_set_timeouts_reply
+- nat44_ei_set_workers
+- nat44_ei_set_workers_reply
+- nat44_ei_show_fq_options
+- nat44_ei_show_fq_options_reply
+- nat44_ei_show_running_config
+- nat44_ei_show_running_config_reply
+- nat44_ei_static_mapping_details
+- nat44_ei_static_mapping_dump
+- nat44_ei_user_details
+- nat44_ei_user_dump
+- nat44_ei_user_session_details
+- nat44_ei_user_session_dump
+- nat44_ei_worker_details
+- nat44_ei_worker_dump
+- nat44_show_running_config
+- nat44_show_running_config_reply
+- nat64_plugin_enable_disable
+- nat64_plugin_enable_disable_reply
+- oddbuf_enable_disable
+- oddbuf_enable_disable_reply
+- pg_interface_enable_disable_coalesce
+- pg_interface_enable_disable_coalesce_reply
+- pnat_binding_add
+- pnat_binding_add_reply
+- pnat_binding_attach
+- pnat_binding_attach_reply
+- pnat_binding_del
+- pnat_binding_del_reply
+- pnat_binding_detach
+- pnat_binding_detach_reply
+- pnat_bindings_details
+- pnat_bindings_get
+- pnat_bindings_get_reply
+- pnat_interfaces_details
+- pnat_interfaces_get
+- pnat_interfaces_get_reply
+- sample_macswap_enable_disable
+- sample_macswap_enable_disable_reply
+- sr_policies_with_sl_index_details
+- sr_policies_with_sl_index_dump
+- sw_interface_set_vxlan_gbp_bypass
+- sw_interface_set_vxlan_gbp_bypass_reply
+- test_addresses
+- test_addresses2
+- test_addresses2_reply
+- test_addresses3
+- test_addresses3_reply
+- test_addresses_reply
+- test_empty
+- test_empty_reply
+- test_enum
+- test_enum_reply
+- test_interface
+- test_interface_reply
+- test_prefix
+- test_prefix_reply
+- test_string
+- test_string2
+- test_string2_reply
+- test_string_reply
+- test_vla
+- test_vla2
+- test_vla2_reply
+- test_vla3
+- test_vla3_reply
+- test_vla4
+- test_vla4_reply
+- test_vla5
+- test_vla5_reply
+- test_vla_reply
+- trace_capture_packets
+- trace_capture_packets_reply
+- trace_clear_capture
+- trace_clear_capture_reply
+- trace_details
+- trace_dump
+- trace_dump_reply
+- trace_set_filters
+- trace_set_filters_reply
+- vxlan_gbp_tunnel_add_del
+- vxlan_gbp_tunnel_add_del_reply
+- vxlan_gbp_tunnel_details
+- vxlan_gbp_tunnel_dump
+- wireguard_interface_create
+- wireguard_interface_create_reply
+- wireguard_interface_delete
+- wireguard_interface_delete_reply
+- wireguard_interface_details
+- wireguard_interface_dump
+- wireguard_peer_add
+- wireguard_peer_add_reply
+- wireguard_peer_remove
+- wireguard_peer_remove_reply
+- wireguard_peers_details
+- wireguard_peers_dump
+
+Patches that changed API definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. |clk| replace:: https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=
+
+``src/vnet/policer/policer_types.api``
+
+* `c5299ff30 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c5299ff30>`_ policer: remove SSE2 prefix
+
+``src/vnet/policer/policer.api``
+
+* `b04683017 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=b04683017>`_ policer: add api to configure input policing
+* `48e26367c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=48e26367c>`_ policer: add api to bind policer to worker
+
+``src/vnet/session/session.api``
+
+* `a5a9efd4d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a5a9efd4d>`_ vcl session: switch to generic cert key apis
+
+``src/vnet/ipfix-export/ipfix_export.api``
+
+* `baa18701b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=baa18701b>`_ misc: ipfix process node wait time adjustment
+
+``src/vnet/ipsec/ipsec_types.api``
+
+* `f16e9a550 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f16e9a550>`_ ipsec: Support async mode per-SA
+* `9ec846c26 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9ec846c26>`_ ipsec: Use the new tunnel API types to add flow label and TTL copy support
+* `751bb131e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=751bb131e>`_ Revert "ipsec: Use the new tunnel API types to add flow label and TTL copy"
+* `c7eaa711f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c7eaa711f>`_ ipsec: Use the new tunnel API types to add flow label and TTL copsupport
+
+``src/vnet/ipsec/ipsec.api``
+
+* `9ec846c26 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9ec846c26>`_ ipsec: Use the new tunnel API types to add flow label and TTL copsupport
+* `751bb131e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=751bb131e>`_ Revert "ipsec: Use the new tunnel API types to add flow label and TTL copy"
+* `c7eaa711f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c7eaa711f>`_ ipsec: Use the new tunnel API types to add flow label and TTL copy support
+* `a9e2774f5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a9e2774f5>`_ ipsec: Deprecated the old IPsec Tunnel interface
+* `95f59f380 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=95f59f380>`_ ipsec: Mark the interface create reply deprecated
+
+``src/vnet/devices/virtio/vhost_user.api``
+
+* `27ba5008a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=27ba5008a>`_ vhost: Add event index for interrupt notification to driver
+
+``src/vnet/ip/ip.api``
+
+* `976b259be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=976b259be>`_ fib: Allow the creation of new source on the API
+* `8f5fef2c7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=8f5fef2c7>`_ ip: Path MTU
+* `3d5f08a82 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3d5f08a82>`_ ip: Router ID included in flow hash
+* `f2984bbb0 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f2984bbb0>`_ ip: use IPv6 flowlabel in flow hash computation
+* `58a1915b5 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=58a1915b5>`_ ip: add API to retrieve IPv6 link-layer address
+
+``src/vnet/l2/l2.api``
+
+* `0f8d10035 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0f8d10035>`_ l2: Separating scan-delay and learn-limit into a separate API from want_l2_macs_events
+* `5f93e3b7f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=5f93e3b7f>`_ l2: add per bridge domain learn limit
+
+``src/vnet/flow/flow_types.api``
+
+* `91f102ed8 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=91f102ed8>`_ flow: The type of vni in VxLAN flow should be u32
+
+``src/vnet/flow/flow.api``
+
+* `c7e7819ad <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=c7e7819ad>`_ flow: Add API implementation of IP4/IP6, IP4_VXLAN/IP6_VXLAN
+
+``src/vnet/fib/fib_types.api``
+
+* `976b259be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=976b259be>`_ fib: Allow the creation of new source on the API
+
+``src/vnet/fib/fib.api``
+
+* `976b259be <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=976b259be>`_ fib: Allow the creation of new source on the API
+
+``src/vnet/vxlan/vxlan.api``
+
+* `3e38422ab <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3e38422ab>`_ vxlan: Fix L3 mode
+* `839dcc0fb <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=839dcc0fb>`_ vxlan: add udp-port configuration support
+
+``src/vnet/bfd/bfd.api``
+
+* `4376ab2a9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4376ab2a9>`_ tests: use socket transport instead of shared memory
+
+``src/vnet/interface.api``
+
+* `fd0b399ff <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fd0b399ff>`_ interface: Add promisc on/off in api
+
+``src/vnet/tunnel/tunnel_types.api``
+
+* `a91cb4590 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a91cb4590>`_ tunnel: support copying TTL and flow label from inner to outer
+
+``src/vnet/interface_types.api``
+
+* `6a999d67d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6a999d67d>`_ interface: RX/TX direction type in API
+
+``src/vat2/test/vat2_test.api``
+
+* `ab9f57355 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ab9f57355>`_ api: crchcecker ignore version < 1.0.0 and outside of src directory
+* `71134f26a <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=71134f26a>`_ vat2: jsonconvert return checking - coverity
+* `93c4b1bb3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=93c4b1bb3>`_ vppapigen: more _fromjson autogeneration coverity fixes
+* `316967cfa <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=316967cfa>`_ vppapigen: fix coverity issues in jsonconvert
+* `cf0102b3b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cf0102b3b>`_ vppapigen: coverity issues in autogenerated code pass 3.
+* `fb0afab7f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fb0afab7f>`_ vppapigen: fix fromjson coverity errors in generation
+
+``src/plugins/gbp/gbp.api``
+
+* `dc22c839f <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=dc22c839f>`_ tests: clean up gbp calls from vpp_papi_provider
+
+``src/plugins/map/map.api``
+
+* `9302cfea9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=9302cfea9>`_ vppapigen: Support an 'autoendian' keyword for message definitions i.api files |
+
+``src/plugins/arping/arping.api``
+
+* `a77ae4708 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a77ae4708>`_ arping: add arping command
+
+``src/plugins/linux-cp/lcp.api``
+
+* `6bb77dec7 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=6bb77dec7>`_ linux-cp: A V2 variant of pair create API that returns the hosinterface created |
+* `4376ab2a9 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4376ab2a9>`_ tests: use socket transport instead of shared memory
+* `44db1caef <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=44db1caef>`_ linux-cp: Linux Interface Mirroring for Control Plane Integration
+
+``src/plugins/ikev2/ikev2_types.api``
+
+* `68d275356 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=68d275356>`_ ikev2: add per SA stats
+
+``src/plugins/ikev2/ikev2.api``
+
+* `af2cc6425 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=af2cc6425>`_ ikev2: support responder hostname
+* `68d275356 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=68d275356>`_ ikev2: add per SA stats
+* `fab5e7f39 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=fab5e7f39>`_ ikev2: use new counters data model & add more counters
+
+``src/plugins/nat/det44/det44.api``
+
+* `0eaf4e678 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0eaf4e678>`_ nat: Final NAT44 EI/ED split patch
+
+``src/plugins/nat/nat44-ed/nat44_ed.api``
+
+* `e3f078fcf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e3f078fcf>`_ nat: fix byte order on ipfix message fields
+* `e7a80a98b <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e7a80a98b>`_ nat: NAT44ED fail if using old plugin option
+* `0eaf4e678 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0eaf4e678>`_ nat: Final NAT44 EI/ED split patch
+
+``src/plugins/nat/nat64/nat64.api``
+
+* `0eaf4e678 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0eaf4e678>`_ nat: Final NAT44 EI/ED split patch
+
+``src/plugins/nat/nat66/nat66.api``
+
+* `ed2ee5e57 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ed2ee5e57>`_ nat: NAT66 plugin enable&disable calls update
+* `0eaf4e678 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0eaf4e678>`_ nat: Final NAT44 EI/ED split patch
+
+``src/plugins/nat/pnat/pnat.api``
+
+* `ec34fb772 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ec34fb772>`_ pnat: coverity errors
+* `ab3151c52 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=ab3151c52>`_ nat: pnat copy and clear byte instructions
+* `18327be5d <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=18327be5d>`_ nat: 1:1 policy NAT
+
+``src/plugins/nat/lib/nat_types.api``
+
+* `0eaf4e678 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0eaf4e678>`_ nat: Final NAT44 EI/ED split patch
+
+``src/plugins/nat/nat44-ei/nat44_ei.api``
+
+* `01930f568 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=01930f568>`_ nat: report correct EI per-user session limit
+* `e3f078fcf <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=e3f078fcf>`_ nat: fix byte order on ipfix message fields
+* `0eaf4e678 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=0eaf4e678>`_ nat: Final NAT44 EI/ED split patch
+
+``src/plugins/af_xdp/af_xdp.api``
+
+* `a42c41be4 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=a42c41be4>`_ af_xdp: workaround kernel race between poll() and sendmsg()
+
+``src/plugins/mss_clamp/mss_clamp.api``
+
+* `bf55e9931 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=bf55e9931>`_ mss_clamp: TCP MSS clamping plugin
+
+``src/plugins/rdma/rdma.api``
+
+* `f5a45680e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=f5a45680e>`_ rdma: add support for RSS configuration
+
+``src/plugins/cnat/cnat.api``
+
+* `516b0adf6 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=516b0adf6>`_ cnat: Add calico/k8s src policy
+* `3fd77f7de <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=3fd77f7de>`_ cnat: Prepare extended snat policies
+* `cc9a1a0d3 <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=cc9a1a0d3>`_ cnat: add input feature node
+* `4d237874e <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=4d237874e>`_ cnat: Add maglev support
+* `27647a27c <https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commit;h=27647a27c>`_ cnat: fixes & prepare maglev
diff --git a/docs/aboutvpp/releasenotes/v21.10.rst b/docs/aboutvpp/releasenotes/v21.10.rst
new file mode 100644
index 0000000..ba3a01a
--- /dev/null
+++ b/docs/aboutvpp/releasenotes/v21.10.rst
@@ -0,0 +1,4 @@
+Release notes for VPP 21.10
+===========================
+
+TBD
diff --git a/docs/whatisvpp/scalar-vs-vector-packet-processing.rst b/docs/aboutvpp/scalar-vs-vector-packet-processing.rst
similarity index 97%
rename from docs/whatisvpp/scalar-vs-vector-packet-processing.rst
rename to docs/aboutvpp/scalar-vs-vector-packet-processing.rst
index ffa54a3..173efec 100644
--- a/docs/whatisvpp/scalar-vs-vector-packet-processing.rst
+++ b/docs/aboutvpp/scalar-vs-vector-packet-processing.rst
@@ -19,7 +19,7 @@
Interface, and processes it through a set of functions: fooA calls fooB calls
fooC and so on.
-.. code-block:: none
+.. code-block:: none
+---> fooA(packet1) +---> fooB(packet1) +---> fooC(packet1)
+---> fooA(packet2) +---> fooB(packet2) +---> fooC(packet2)
@@ -45,14 +45,14 @@
processes the vector through a set of functions: fooA calls fooB calls fooC and
so on.
-.. code-block:: none
+.. code-block:: none
+---> fooA([packet1, +---> fooB([packet1, +---> fooC([packet1, +--->
packet2, packet2, packet2,
... ... ...
packet256]) packet256]) packet256])
-This approach fixes:
+This approach fixes:
* The I-cache thrashing problem described above, by amortizing the cost of
I-cache loads across multiple packets.
diff --git a/docs/whatisvpp/supported.rst b/docs/aboutvpp/supported.rst
similarity index 72%
rename from docs/whatisvpp/supported.rst
rename to docs/aboutvpp/supported.rst
index a9390cc..077090b 100644
--- a/docs/whatisvpp/supported.rst
+++ b/docs/aboutvpp/supported.rst
@@ -2,13 +2,13 @@
.. toctree::
-Architectures and Operating Systems
-***********************************
+Supported archs and OS
+**********************
The following architectures and operating systems are supported in VPP:
Architectures
------------------------
+-------------
* The FD.io VPP platform supports:
@@ -26,4 +26,3 @@
* Debian
* Ubuntu
-For more about VPP performance press next.
diff --git a/docs/cli-reference/gettingstarted/index.rst b/docs/cli-reference/gettingstarted/index.rst
new file mode 100644
index 0000000..6528725
--- /dev/null
+++ b/docs/cli-reference/gettingstarted/index.rst
@@ -0,0 +1,126 @@
+.. _cli_getting_started:
+
+Getting Started with the debug CLI
+==================================
+
+The VPP network stack comes equipped with a set of commands that are useful
+for debugging.
+
+The easiest way to access the CLI (with proper permissions) is to use the
+vppctl command:
+
+.. code-block:: console
+
+ sudo vppctl <cli-command>
+
+
+The CLI parser matches static keyword strings, eventually invoking an action
+function. Unambiguous partial keyword matching always occurs. The action
+functions consume input until satisfied or until they fail. This model makes
+for easy coding, but does not guarantee useful "help" output. It's up to the
+CLI command writer to add useful help strings.
+
+You can find the source code of CLI commands by searching for instances of the
+``VLIB_CLI_COMMAND`` macro in the code source files.
+
+Please help maintain and improve this document to make and keep these commands
+clear and useful!
+
+.. _debug_telnet_cli:
+
+Debug and Telnet CLI
+--------------------
+
+The debug CLI is enabled with the unix interactive parameter or startup
+configuration option. This causes VPP to start without daemonizing and
+presents a command line interface on the terminal where it is run.
+
+The Telnet CLI is enabled with the ``cli-listen localhost:5002`` option which
+will cause VPP to listen for TCP connections on the localhost address port
+``5002``. A Telnet client can then connect to this port (for example, ``telnet
+localhost 5002``) and will receive a command line prompt.
+
+This configuration will enable both mechanisms:
+
+.. code-block:: console
+
+ unix {
+ interactive
+ cli-listen localhost:5002
+ }
+
+
+The debug CLI can operate in line mode, which may be useful when running
+inside an IDE like Emacs. This is enabled with the option
+``unix cli-line-mode``. Several other options exist that alter how this
+CLI works, see the @ref syscfg section for details.
+
+The CLI starts with a banner graphic (which can be disabled) and a prompt. The
+prompt will typically read ``vpp`` for a release version of VPP and ``DBGvpp#``
+for a development version with debugging enabled, for example:
+
+.. code-block:: console
+
+ _______ _ _ _____ ___
+ __/ __/ _ \ (_)__ | | / / _ \/ _ \
+ _/ _// // / / / _ \ | |/ / ___/ ___/
+ /_/ /____(_)_/\___/ |___/_/ /_/
+
+ vpp#
+
+
+
+versus:
+
+.. code-block:: console
+
+ _______ _ _ _____ ___
+ __/ __/ _ \ (_)__ | | / / _ \/ _ \
+ _/ _// // / / / _ \ | |/ / ___/ ___/
+ /_/ /____(_)_/\___/ |___/_/ /_/
+
+ DBGvpp#
+
+
+This prompt can be configured with the ``unix cli-prompt`` setting and the
+banner is disabled with ``unix cli-no-banner``.
+
+.. _cli_features:
+
+CLI features
+------------
+
+The CLI has several editing features that make it easy to use.
+
+- Cursor keys ``left/right`` will move the cursor within a command line;
+ typing will insert at the cursor; erase will erase at the cursor.
+
+- ``Ctrl-left/right`` will search for the start of the next word to
+ the left or right.
+- ``Home/end`` will jump the cursor to the start and end of the line.
+- Cursor keys up/down and ``^P/^N`` iterate through the command history
+ buffer. Lines from the history buffer may be edited. New commands
+ are added to the end of the buffer when executed; though
+ duplicates of the previous command are not added.
+- ``^U`` erases the line contents from the left of the cursor to the
+ start.
+- ``^K`` erases the contents from the cursor to the end.
+- ``^S/^R`` will search the command history forwards or in reverse for
+ a command; start typing for matches to auto complete.
+- ``^L`` will clear the screen (if supported by the terminal) and repaint
+ the prompt and any current line. The cursor position is also
+ retained.
+- The CLI can be closed with the quit command. Alternatively, ``^D`` on
+ an empty input line will also close the session. Closing the debug
+ session will also shutdown VPP.
+
+Output that exceeds the length of a terminal page will be buffered, up to a
+limit.
+
+- ``Space`` or ``page-down`` displays the next page.
+- ``Enter`` or ``down-arrow`` displays the next line.
+- ``Page-up`` goes back a page.
+- ``Up-arrow`` goes up a line.
+- ``Home/end`` jump to the start/end of the buffered output.
+- The key ``q`` quits the pager. ``Space`` and ``enter`` will also quit the
+ pager if the end of the buffer has been reached.
\ No newline at end of file
diff --git a/docs/reference/cmdreference/interface/basic.rst b/docs/cli-reference/interface/basic.rst
similarity index 97%
rename from docs/reference/cmdreference/interface/basic.rst
rename to docs/cli-reference/interface/basic.rst
index 9848005..2d4202d 100644
--- a/docs/reference/cmdreference/interface/basic.rst
+++ b/docs/cli-reference/interface/basic.rst
@@ -18,9 +18,6 @@
++++++++++++++++
Shows software interface information including counters and features.
-Summary/Usage
--------------
-
.. code-block:: shell
show interface [address|addr|features|feat] [<interface> [<interface> [..]]]
@@ -73,9 +70,6 @@
Clear the statistics for all interfaces (statistics associated with the
'*show interface*' command).
-Summary/Usage
--------------
-
.. code-block:: shell
clear interfaces
diff --git a/docs/reference/cmdreference/interface/create_interface.rst b/docs/cli-reference/interface/create_interface.rst
similarity index 92%
rename from docs/reference/cmdreference/interface/create_interface.rst
rename to docs/cli-reference/interface/create_interface.rst
index fa50c6a..abd1f40 100644
--- a/docs/reference/cmdreference/interface/create_interface.rst
+++ b/docs/cli-reference/interface/create_interface.rst
@@ -15,10 +15,11 @@
Create Host-Interface
++++++++++++++++++++++
-Summary/Usage
--------------
-create host-interface name <*ifname*> [*hw-addr <*mac-addr*>]
+.. code-block:: console
+
+ create host-interface name <*ifname*> [*hw-addr <*mac-addr*>]
+
Description
------------
@@ -59,10 +60,10 @@
Create Interface Memif
+++++++++++++++++++++++
-Summary/Usage
--------------
+.. code-block:: console
-create interface memif [id <*id*>] [socket-id <*socket-id*>] [ring-size <*size*>] [buffer-size <*size*>] [hw-addr <*mac-address*>] <master|slave> [rx-queues <*number*>] [tx-queues <*number*>] [mode ip] [secret <*string*>]
+ create interface memif [id <*id*>] [socket-id <*socket-id*>] [ring-size <*size*>] [buffer-size <*size*>] [hw-addr <*mac-address*>] <master|slave> [rx-queues <*number*>] [tx-queues <*number*>] [mode ip] [secret <*string*>]
+
Declaration and Implementation
-------------------------------
@@ -75,16 +76,14 @@
Create Loopback Interface
++++++++++++++++++++++++++
-Summary/Usage
---------------
-
-create loopback interface [mac <*mac-addr*>] [instance <*instance*>]
-
-Description
-------------
Create a loopback interface. Optionally, a MAC Address can be provided. If not provided, de:ad:00:00:00:<*loopId*> will be used.
+.. code-block:: console
+
+ create loopback interface [mac <*mac-addr*>] [instance <*instance*>]
+
+
Example Usage
--------------
The following two command syntaxes are equivalent:
@@ -118,7 +117,7 @@
provide, the VLAN ID is assumed to be the *subId*. The VLAN ID and
*subId* can be different, but this is not recommended.
-This command has several variations:
+This command has several variations:
- **create sub-interfaces** <*interface*> <*subId*> - Create a subinterface
to process packets with a given 802.1q VLAN ID (same value as the
@@ -221,4 +220,4 @@
.. code-block:: console
vpp# set interface GigabitEthernet2/0/0.7 up
-
+
diff --git a/docs/reference/cmdreference/interface/hardware.rst b/docs/cli-reference/interface/hardware.rst
similarity index 94%
rename from docs/reference/cmdreference/interface/hardware.rst
rename to docs/cli-reference/interface/hardware.rst
index 2c28d65..0d12438 100644
--- a/docs/reference/cmdreference/interface/hardware.rst
+++ b/docs/cli-reference/interface/hardware.rst
@@ -4,7 +4,7 @@
Hardware-Interfaces Commands
============================
-This section contains those interface commands that are related to hardware-interfaces:
+This section contains those interface commands that are related to hardware-interfaces:
* `Show Bridge-Domain`_
@@ -16,17 +16,15 @@
Show Bridge-Domain
+++++++++++++++++++
-Summary/Usage
--------------
-
-show bridge-domain [*bridge-domain-id* [detail|int|arp| *bd-tag* ]]
-
-Description
------------
-
Show a summary of all the bridge-domain instances or detailed view of a single bridge-domain.
Bridge-domains are created by adding an interface to a bridge using the **set interface l2 bridge** command.
+
+.. code-block:: console
+
+ show bridge-domain [*bridge-domain-id* [detail|int|arp| *bd-tag* ]]
+
+
Example Usage
-------------
.. code-block:: console
@@ -70,17 +68,16 @@
- **detail**: Also display all remaining attributes and extended
statistics.
-.. note::
+.. note::
To limit the output of the command to bonded interfaces and their
slave interfaces, use the '*bond*' optional parameter.
-Summary/Usage
--------------
.. code-block:: shell
show hardware-interfaces [brief|verbose|detail] [bond] [<interface> [<interface> [..]]] [<sw_idx> [<sw_idx> [..]]].
+
Examples
--------
Example of how to display default data for all interfaces:
@@ -139,13 +136,10 @@
(statistics associated with the **show hardware-interfaces** command).
-Summary/Usage
--------------
-
.. code-block:: shell
clear hardware-interfaces [<interface> [<interface> [..]]] [<sw_idx> [<sw_idx> [..]]].
-
+
Examples
--------
@@ -158,7 +152,7 @@
vpp# clear hardware-interfaces
Example of how to clear the extended statistics for an interface by name and software index
-(where 2 is the software index):
+(where 2 is the software index):
.. code-block:: console
diff --git a/docs/reference/cmdreference/interface/index.rst b/docs/cli-reference/interface/index.rst
similarity index 97%
rename from docs/reference/cmdreference/interface/index.rst
rename to docs/cli-reference/interface/index.rst
index 9e7fd17..7fca8f1 100644
--- a/docs/reference/cmdreference/interface/index.rst
+++ b/docs/cli-reference/interface/index.rst
@@ -10,9 +10,8 @@
.. toctree::
:maxdepth: 2
-
+
basic
hardware
create_interface
setinterface
-
\ No newline at end of file
diff --git a/docs/cli-reference/interface/setinterface.rst b/docs/cli-reference/interface/setinterface.rst
new file mode 100644
index 0000000..7eb14ad
--- /dev/null
+++ b/docs/cli-reference/interface/setinterface.rst
@@ -0,0 +1,182 @@
+======================
+Set Interface Commands
+======================
+
+This section covers those commands that are related to setting an
+interface:
+
+- `Set Interface IP Address <#set-interface-ip-address>`__
+- `Set Interface L2 Bridge <#set-interface-l2-bridge>`__
+- `Set Interface MTU <#set-interface-mtu>`__
+- `Set Interface Promiscuous <#set-interface-promiscuous>`__
+- `Set Interface State <#set-interface-state>`__
+
+.. note::
+
+ For a complete list of CLI Debug commands refer to the Debug CLI
+ section of the `Source Code
+ Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`__ .
+
+Set Interface IP Address
+========================
+
+.. code:: console
+
+ set interface ip address [del] <*interface*> <*ip-addr*>/<*mask*> | [all]
+
+Add an IP Address to an interface or remove and IP Address from an
+interface. The IP Address can be an IPv4 or an IPv6 address. Interfaces
+may have multiple IPv4 and IPv6 addresses. There is no concept of
+primary vs. secondary interface addresses; they're just addresses.
+
+To display the addresses associated with a given interface, use the
+command **show interface address** <*interface*>.
+
+.. note::
+
+ The debug CLI does not enforce classful mask-width / addressing
+ constraints.
+
+Example Usage
+-------------
+
+An example of how to add an IPv4 address to an interface:
+
+.. code:: console
+
+ vpp# set interface ip address GigabitEthernet2/0/0 172.16.2.12/24
+
+An example of how to add an IPv6 address to an interface:
+
+.. code:: console
+
+ vpp# set interface ip address GigabitEthernet2/0/0 ::a:1:1:0:7/126
+
+To delete a specific interface ip address:
+
+.. code:: console
+
+ vpp# set interface ip address GigabitEthernet2/0/0 172.16.2.12/24 del
+
+To delete all interfaces addresses (IPv4 and IPv6):
+
+.. code:: console
+
+ vpp# set interface ip address GigabitEthernet2/0/0 del all
+
+Declaration and Implementation
+------------------------------
+
+**Declaration:** set_interface_ip_address_command
+(src/vnet/ip/ip46_cli.c line 216)
+
+**Implementation:** add_del_ip_address
+
+Set Interface L2 Bridge
+=======================
+
+.. code:: console
+
+ set interface l2 bridge <*interface*> <*bridge-domain-id*> [bvi|uu-fwd]
+ [shg]
+
+Use this command put an interface into Layer 2 bridge domain. If a
+bridge-domain with the provided bridge-domain-id does not exist, it will
+be created. Interfaces in a bridge-domain forward packets to other
+interfaces in the same bridge-domain based on destination mac address.
+To remove an interface from a the Layer 2 bridge domain, put the
+interface in a different mode, for example Layer 3 mode.
+
+Optionally, an interface can be added to a Layer 2 bridge-domain as a
+Bridged Virtual Interface (bvi). Only one interface in a Layer 2
+bridge-domain can be a bvi.
+
+Optionally, a split-horizon group can also be specified. This defaults
+to 0 if not specified.
+
+.. _example-usage-1:
+
+Example Usage
+-------------
+
+Example of how to configure a Layer 2 bridge-domain with three
+interfaces (where 200 is the bridge-domain-id):
+
+.. code:: console
+
+ vpp# set interface l2 bridge GigabitEthernet0/8/0.200 200
+
+This interface is added a BVI interface:
+
+.. code:: console
+
+ vpp# set interface l2 bridge GigabitEthernet0/9/0.200 200 bvi
+
+This interface also has a split-horizon group of 1 specified:
+
+.. code:: console
+
+ vpp# set interface l2 bridge GigabitEthernet0/a/0.200 200 1
+
+Example of how to remove an interface from a Layer2 bridge-domain:
+
+.. code:: console
+
+ vpp# set interface l3 GigabitEthernet0/a/0.200
+
+.. _declaration-and-implementation-1:
+
+Declaration and Implementation
+------------------------------
+
+**Declaration:** int_l2_bridge_cli (src/vnet/l2/l2_input.c line 949)
+
+**Implementation:** int_l2_bridge
+
+Set Interface MTU
+=================
+
+.. code:: shell
+
+ set interface mtu [packet|ip4|ip6|mpls] <value> <interface>
+
+Set Interface Promiscuous
+=========================
+
+.. code:: shell
+
+ set interface promiscuous [on|off] <interface>.
+
+.. _setintstate:
+
+Set Interface State
+===================
+
+This command is used to change the admin state (up/down) of an
+interface.
+
+If an interface is down, the optional *punt* flag can also be set. The
+*punt* flag implies the interface is disabled for forwarding but punt
+all traffic to slow-path. Use the *enable* flag to clear *punt* flag
+(interface is still down).
+
+.. code:: shell
+
+ set interface state <interface> [up|down|punt|enable].
+
+.. _example-usage-2:
+
+Example Usage
+-------------
+
+Example of how to configure the admin state of an interface to **up**:
+
+.. code:: console
+
+ vpp# set interface state GigabitEthernet2/0/0 up
+
+Example of how to configure the admin state of an interface to **down**:
+
+.. code:: console
+
+ vpp# set interface state GigabitEthernet2/0/0 down
diff --git a/docs/conf.py b/docs/conf.py
index 09c4d9b..8a2aef5 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -16,6 +16,8 @@
# import sys
# sys.path.insert(0, os.path.abspath('.'))
+import subprocess
+
# -- Project information -----------------------------------------------------
project = u'The Vector Packet Processor'
@@ -23,9 +25,9 @@
author = u'FD.io VPP Community'
# The short X.Y version
-version = u'master'
+version = subprocess.run(["git", "describe"], stdout=subprocess.PIPE, text=True).stdout
# The full version, including alpha/beta/rc tags
-release = u'20.01'
+release = subprocess.run(["git", "describe", "--long"], stdout=subprocess.PIPE, text=True).stdout
# -- General configuration ---------------------------------------------------
@@ -44,6 +46,10 @@
'sphinxcontrib.spelling']
spelling_word_list_filename = 'spelling_wordlist.txt'
+
+# do not spell check those files
+spelling_exclude_patterns = ['aboutvpp/releasenotes/*']
+
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@@ -68,7 +74,7 @@
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path .
-exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ['Thumbs.db', '.DS_Store', '_scripts', 'venv', '_generated']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'default'
diff --git a/docs/configuration/config_getting_started.rst b/docs/configuration/config_getting_started.rst
new file mode 100644
index 0000000..9321ce6
--- /dev/null
+++ b/docs/configuration/config_getting_started.rst
@@ -0,0 +1,84 @@
+.. _config_getting_started:
+
+=======================================
+Getting started with the configuration
+=======================================
+
+After a successful installation, VPP installs a startup config file named
+*startup.conf* in the */etc/vpp/* directory. This file can be tailored to
+make VPP run as desired, but contains default values for typical installations.
+
+Below are more details about this file and some of the the parameters and values
+it contains.
+
+Command-line Arguments
+----------------------
+
+Before we describe details of the startup configuration file (startup.conf) it
+should be mentioned that VPP can be started without a startup configuration
+file.
+
+Parameters are grouped by a section name. When providing more than one
+parameter to a section, all parameters for that section must be wrapped in
+curly braces. For example, to start VPP with configuration data via the
+command line with the section name *'unix'*:
+
+.. code-block:: console
+
+ $ sudo /usr/bin/vpp unix { interactive cli-listen 127.0.0.1:5002 }
+
+The command line can be presented as a single string or as several; anything
+given on the command line is concatenated with spaces into a single string
+before parsing. VPP applications must be able to locate their own executable
+images. The simplest way to ensure this will work is to invoke a VPP
+application by giving its absolute path. For example:
+*'/usr/bin/vpp <options>'* At startup, VPP applications parse through their
+own ELF-sections [primarily] to make lists of init, configuration, and exit
+handlers.
+
+When developing with VPP, in gdb it's often sufficient to start an application
+like this:
+
+.. code-block:: console
+
+ (gdb) run unix interactive
+
+
+Configuration File (startup.conf)
+-----------------------------------------
+
+The more typical way to specify the startup configuration to VPP is with the
+startup configuration file (startup.conf).
+
+The path of the file is provided to the VPP application on the command line.
+This is typically at /etc/vpp/startup.conf. If VPP is installed as a package
+a default startup.conf file is provided at this location.
+
+The format of the configuration file is a simple text file with the same content
+as the command line.
+
+**A very simple startup.conf file:**
+
+.. code-block:: console
+
+ $ cat /etc/vpp/startup.conf
+ unix {
+ nodaemon
+ log /var/log/vpp/vpp.log
+ full-coredump
+ cli-listen localhost:5002
+ }
+
+ api-trace {
+ on
+ }
+
+ dpdk {
+ dev 0000:03:00.0
+ }
+
+VPP is instructed to load this file with the -c option. For example:
+
+.. code-block:: console
+
+ $ sudo /usr/bin/vpp -c /etc/vpp/startup.conf
\ No newline at end of file
diff --git a/docs/gettingstarted/users/configuring/startup.rst b/docs/configuration/reference.rst
similarity index 92%
rename from docs/gettingstarted/users/configuring/startup.rst
rename to docs/configuration/reference.rst
index e594b6a..84b2fd7 100644
--- a/docs/gettingstarted/users/configuring/startup.rst
+++ b/docs/configuration/reference.rst
@@ -1,90 +1,7 @@
-.. _startup:
+.. _configuration_reference:
-==========================================
-VPP Configuration - CLI and 'startup.conf'
-==========================================
-
-After a successful installation, VPP installs a startup config file named
-*startup.conf* in the */etc/vpp/* directory. This file can be tailored to
-make VPP run as desired, but contains default values for typical installations.
-
-Below are more details about this file and some of the the parameters and values
-it contains.
-
-Command-line Arguments
-----------------------
-
-Before we describe details of the startup configuration file (startup.conf) it
-should be mentioned that VPP can be started without a startup configuration
-file.
-
-Parameters are grouped by a section name. When providing more than one
-parameter to a section, all parameters for that section must be wrapped in
-curly braces. For example, to start VPP with configuration data via the
-command line with the section name *'unix'*:
-
-.. code-block:: console
-
- $ sudo /usr/bin/vpp unix { interactive cli-listen 127.0.0.1:5002 }
-
-The command line can be presented as a single string or as several; anything
-given on the command line is concatenated with spaces into a single string
-before parsing. VPP applications must be able to locate their own executable
-images. The simplest way to ensure this will work is to invoke a VPP
-application by giving its absolute path. For example:
-*'/usr/bin/vpp <options>'* At startup, VPP applications parse through their
-own ELF-sections [primarily] to make lists of init, configuration, and exit
-handlers.
-
-When developing with VPP, in gdb it's often sufficient to start an application
-like this:
-
-.. code-block:: console
-
- (gdb) run unix interactive
-
-
-Startup Configuration File (startup.conf)
------------------------------------------
-
-The more typical way to specify the startup configuration to VPP is with the
-startup configuration file (startup.conf).
-
-The path of the file is provided to the VPP application on the command line.
-This is typically at /etc/vpp/startup.conf. If VPP is installed as a package
-a default startup.conf file is provided at this location.
-
-The format of the configuration file is a simple text file with the same content
-as the command line.
-
-**A very simple startup.conf file:**
-
-.. code-block:: console
-
- $ cat /etc/vpp/startup.conf
- unix {
- nodaemon
- log /var/log/vpp/vpp.log
- full-coredump
- cli-listen localhost:5002
- }
-
- api-trace {
- on
- }
-
- dpdk {
- dev 0000:03:00.0
- }
-
-VPP is instructed to load this file with the -c option. For example:
-
-.. code-block:: console
-
- $ sudo /usr/bin/vpp -c /etc/vpp/startup.conf
-
-Configuration Parameters
-------------------------
+Configuration Reference
+=======================
Below is the list of some section names and their associated parameters.
This is not an exhaustive list, but should give you an idea of how VPP can be configured.
@@ -587,7 +504,7 @@
Increase number of buffers allocated, needed only in scenarios with
large number of interfaces and worker threads. Value is per numa node.
-Default is 16384 (8192 if running unpriviledged)
+Default is 16384 (8192 if running unprivileged)
.. code-block:: console
diff --git a/docs/gettingstarted/developers/gitreview.rst b/docs/contributing/gitreview.rst
similarity index 98%
rename from docs/gettingstarted/developers/gitreview.rst
rename to docs/contributing/gitreview.rst
index 3f865c1..b2608d1 100644
--- a/docs/gettingstarted/developers/gitreview.rst
+++ b/docs/contributing/gitreview.rst
@@ -32,9 +32,9 @@
$ ssh-keygen -t rsa
$ keychain
- $ cat ~/.ssh/id_rsa.pub
+ $ cat ~/.ssh/id_rsa.pub
-Copy **all** the contents of the public key (id_rsa.pub) output by the above **cat** command. Then go to your `SSH Public keys settings page <https://gerrit.fd.io/r/#/settings/ssh-keys>`_, click **Add Key ...**, paste your public key, and finally click **Add**.
+Copy **all** the contents of the public key (id_rsa.pub) output by the above **cat** command. Then go to your `SSH Public keys settings page <https://gerrit.fd.io/r/#/settings/ssh-keys>`_, click **Add Key ...**, paste your public key, and finally click **Add**.
.. _clone-ssh:
@@ -184,7 +184,7 @@
$ git review
In the upload-failure case, use caution: carefully **save your work**
-before you do anything else!
+before you do anything else!
Rebase your patch and try again. Please **do not** re-download ["git
review -d"] the patch from the gerrit server...:
diff --git a/docs/contributing/reportingissues/index.rst b/docs/contributing/reportingissues/index.rst
new file mode 100644
index 0000000..b16a2a9
--- /dev/null
+++ b/docs/contributing/reportingissues/index.rst
@@ -0,0 +1,23 @@
+.. _reportingissues:
+
+Reporting Bugs
+==============
+
+Although every situation is different, this section describes how to
+collect data which will help make efficient use of everyone's time
+when dealing with vpp bugs.
+
+Before you press the Jira button to create a bug report - or email
+vpp-dev@lists.fd.io - please ask yourself whether there's enough
+information for someone else to understand and to reproduce the issue
+given a reasonable amount of effort. **Unicast emails to maintainers,
+committers, and the project PTL are strongly discouraged.**
+
+A good strategy for clear-cut bugs: file a detailed Jira ticket, and
+then send a short description of the issue to vpp-dev@lists.fd.io,
+perhaps from the Jira ticket description. It's fine to send email to
+vpp-dev@lists.fd.io to ask a few questions **before** filing Jira tickets.
+
+.. toctree::
+
+ reportingissues
diff --git a/docs/troubleshooting/reportingissues/reportingissues.rst b/docs/contributing/reportingissues/reportingissues.rst
similarity index 91%
rename from docs/troubleshooting/reportingissues/reportingissues.rst
rename to docs/contributing/reportingissues/reportingissues.rst
index 3ccd494..970f340 100644
--- a/docs/troubleshooting/reportingissues/reportingissues.rst
+++ b/docs/contributing/reportingissues/reportingissues.rst
@@ -2,24 +2,6 @@
.. toctree::
-Reporting Bugs
-==============
-
-Although every situation is different, this section describes how to
-collect data which will help make efficient use of everyone's time
-when dealing with vpp bugs.
-
-Before you press the Jira button to create a bug report - or email
-vpp-dev@lists.fd.io - please ask yourself whether there's enough
-information for someone else to understand and to reproduce the issue
-given a reasonable amount of effort. **Unicast emails to maintainers,
-committers, and the project PTL are strongly discouraged.**
-
-A good strategy for clear-cut bugs: file a detailed Jira ticket, and
-then send a short description of the issue to vpp-dev@lists.fd.io,
-perhaps from the Jira ticket description. It's fine to send email to
-vpp-dev@lists.fd.io to ask a few questions **before** filing Jira tickets.
-
Data to include in bug reports
==============================
diff --git a/docs/contributing/writingdocs.rst b/docs/contributing/writingdocs.rst
new file mode 100644
index 0000000..ba37130
--- /dev/null
+++ b/docs/contributing/writingdocs.rst
@@ -0,0 +1,52 @@
+.. _buildingrst:
+
+Writing VPP Documentation
+=========================
+
+These instructions show how the VPP documentation sources are built.
+
+The VPP Documents are written using `reStructuredText <http://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html>`_ (rst),
+or markdown (md). These files are then built using the Sphinx build system `Sphinx <http://www.sphinx-doc.org/en/master/>`_.
+
+Building the docs
+-----------------
+
+Start with a clone of the vpp repository.
+
+.. code-block:: console
+
+ $ git clone https://gerrit.fd.io/r/vpp
+ $ cd vpp
+
+Build the html **index.html** file:
+
+.. code-block:: console
+
+ $ make docs
+
+Delete all the generated files with the following:
+
+.. code-block:: console
+
+ $ make docs-clean
+
+View the results
+----------------
+
+If there are no errors during the build process, you should now have an ``index.html`` file in your ``vpp/docs/_build/html`` directory, which you can then view in your browser.
+
+Whenever you make changes to your ``.rst`` files that you want to see, repeat this build process.
+
+Writing Docs and merging
+------------------------
+
+Documentation should be added as ``.rst`` file in the ``./src/`` tree next to the code it refers to. A symlink should be added at the relevant place in the ``./docs`` folder and a link in the appropriate place in the tree.
+
+To ensure documentation is correctly inserted, you can run
+
+.. code-block:: console
+
+ $ ./extras/scripts/check_documentation.sh
+
+VPP documents are reviewed and merged like and other source code. Refer to :ref:`gitreview`
+to get your changes reviewed and merged.
diff --git a/docs/gettingstarted/developers/building.rst b/docs/developer/build-run-debug/building.rst
similarity index 96%
rename from docs/gettingstarted/developers/building.rst
rename to docs/developer/build-run-debug/building.rst
index 4d7af16..1df838a 100644
--- a/docs/gettingstarted/developers/building.rst
+++ b/docs/developer/build-run-debug/building.rst
@@ -5,7 +5,7 @@
Building VPP
============
-To get started developing with VPP, you need to get the required VPP sources and then build the packages.
+To get started developing with VPP, you need to get the required VPP sources and then build the packages.
For more detailed information on the build system please refer to :ref:`buildsystem`.
.. _setupproxies:
@@ -13,7 +13,7 @@
Set up Proxies
--------------------------
-Depending on the environment you are operating in, proxies may need to be set.
+Depending on the environment you are operating in, proxies may need to be set.
Run these proxy commands to specify the *proxy-server-name* and corresponding *port-number*:
.. code-block:: console
@@ -40,14 +40,14 @@
.. code-block:: console
- $ dpkg -l | grep vpp
+ $ dpkg -l | grep vpp
$ dpkg -l | grep DPDK
There should be no output, or no packages shown after the above commands are run.
-Run the following **make** command to install the dependencies for FD.io VPP.
+Run the following **make** command to install the dependencies for FD.io VPP.
-If the download hangs at any point, then you may need to
+If the download hangs at any point, then you may need to
:ref:`set up proxies <setupproxies>` for the download to work.
.. code-block:: console
@@ -137,8 +137,8 @@
.. code-block:: console
- $ make pkg-deb
-
+ $ make pkg-deb
+
.. _rpmpackages:
Building RPM Packages
@@ -153,7 +153,7 @@
Once the packages are built they can be found in the build-root directory.
.. code-block:: console
-
+
$ ls *.deb
If the packages are built correctly, then this should be the corresponding output:
diff --git a/docs/gettingstarted/developers/cross_compile_macos.rst b/docs/developer/build-run-debug/cross_compile_macos.rst
similarity index 100%
rename from docs/gettingstarted/developers/cross_compile_macos.rst
rename to docs/developer/build-run-debug/cross_compile_macos.rst
diff --git a/docs/gettingstarted/developers/gdb_examples.rst b/docs/developer/build-run-debug/gdb_examples.rst
similarity index 97%
rename from docs/gettingstarted/developers/gdb_examples.rst
rename to docs/developer/build-run-debug/gdb_examples.rst
index 8939877..2a33f17 100644
--- a/docs/gettingstarted/developers/gdb_examples.rst
+++ b/docs/developer/build-run-debug/gdb_examples.rst
@@ -59,7 +59,7 @@
1 breakpoint keep y 0x00007ffff6b9c00b in ip4_icmp_input at /scratch/vpp-master/build-data/../src/vnet/ip/icmp4.c:142
breakpoint already hit 3 times
2 breakpoint keep y 0x00007ffff6b9c00b in ip4_icmp_input at /scratch/vpp-master/build-data/../src/vnet/ip/icmp4.c:142
- 3 breakpoint keep y 0x00007ffff640f646 in tw_timer_expire_timers_internal_1t_3w_1024sl_ov
+ 3 breakpoint keep y 0x00007ffff640f646 in tw_timer_expire_timers_internal_1t_3w_1024sl_ov
at /scratch/vpp-master/build-data/../src/vppinfra/tw_timer_template.c:775
Delete a breakpoint:
@@ -71,7 +71,7 @@
Num Type Disp Enb Address What
1 breakpoint keep y 0x00007ffff6b9c00b in ip4_icmp_input at /scratch/vpp-master/build-data/../src/vnet/ip/icmp4.c:142
breakpoint already hit 3 times
- 3 breakpoint keep y 0x00007ffff640f646 in tw_timer_expire_timers_internal_1t_3w_1024sl_ov
+ 3 breakpoint keep y 0x00007ffff640f646 in tw_timer_expire_timers_internal_1t_3w_1024sl_ov
at /scratch/vpp-master/build-data/../src/vppinfra/tw_timer_template.c:775
Step/Next/List
@@ -90,10 +90,10 @@
(gdb) list
202 vlib_put_next_frame (vm, node, next, n_left_to_next);
203 }
- 204
+ 204
205 return frame->n_vectors;
206 }
- 207
+ 207
208 /* *INDENT-OFF* */
209 VLIB_REGISTER_NODE (ip4_icmp_input_node,static) = {
210 .function = ip4_icmp_input,
@@ -122,7 +122,7 @@
as_u16 = {0xa, 0xa00}, as_u32 = 0xa00000a}}}}, {checksum_data_64 =
{0x40704954000045, 0x200000a2ddd0140}, checksum_data_64_32 = {0xa00000a}},
{checksum_data_32 = {0x54000045, 0x407049, 0x2ddd0140, 0x200000a, 0xa00000a}}}
-
+
Then the icmp header
.. code-block:: console
diff --git a/docs/developer/build-run-debug/index.rst b/docs/developer/build-run-debug/index.rst
new file mode 100644
index 0000000..f8bfeab
--- /dev/null
+++ b/docs/developer/build-run-debug/index.rst
@@ -0,0 +1,14 @@
+.. _build_run_debug:
+
+=======================
+Build, Run & Debug
+=======================
+
+.. toctree::
+ :maxdepth: 1
+
+ building
+ running_vpp
+ testing_vpp
+ gdb_examples
+ cross_compile_macos
diff --git a/docs/gettingstarted/developers/running_vpp.rst b/docs/developer/build-run-debug/running_vpp.rst
similarity index 100%
rename from docs/gettingstarted/developers/running_vpp.rst
rename to docs/developer/build-run-debug/running_vpp.rst
diff --git a/docs/developer/build-run-debug/testing_vpp.rst b/docs/developer/build-run-debug/testing_vpp.rst
new file mode 100644
index 0000000..ca9a09e
--- /dev/null
+++ b/docs/developer/build-run-debug/testing_vpp.rst
@@ -0,0 +1,140 @@
+Testing VPP
+===========
+
+As of this writing, the vpp source tree includes over 1,000 unit test
+vectors. Best practices prior to pushing patches for code review: make
+sure that all of the “make test” test vectors pass.
+
+We attempt to maintain the top-level “make test-help” command so that it
+accurately describes all of the “make test” options.
+
+Examples
+--------
+
+Basic test run, all test vectors, single-vpp instance, optimized image:
+
+::
+
+ $ make test
+
+10-way parallel basic test run:
+
+::
+
+ $ make TEST_JOBS=10 test
+
+Run a specific test suite (mpls, in this case):
+
+::
+
+ $ make TEST=test_mpls test
+
+Run a specific test suite, debug image, pause prior to running the test
+suite; attach to the vpp image in gdb:
+
+::
+
+ $ make TEST=xxx DEBUG=gdb test-debug
+
+Detailed Documentation
+----------------------
+
+Current “make test-help” output:
+
+::
+
+ $ make test-help
+ test - build and run (basic) functional tests
+ test-debug - build and run (basic) functional tests (debug build)
+ test-all - build and run functional and extended tests
+ test-all-debug - build and run functional and extended tests (debug build)
+ retest - run functional tests
+ retest-debug - run functional tests (debug build)
+ retest-all - run functional and extended tests
+ retest-all-debug - run functional and extended tests (debug build)
+ test-cov - generate code coverage report for test framework
+ test-gcov - build and run functional tests (gcov build)
+ test-wipe - wipe (temporary) files generated by unit tests
+ test-wipe-cov - wipe code coverage report for test framework
+ test-wipe-doc - wipe documentation for test framework
+ test-wipe-papi - rebuild vpp_papi sources
+ test-wipe-all - wipe (temporary) files generated by unit tests, docs, and coverage
+ test-shell - enter shell with test environment
+ test-shell-debug - enter shell with test environment (debug build)
+ test-checkstyle - check PEP8 compliance for test framework
+ test-refresh-deps - refresh the Python dependencies for the tests
+
+ Arguments controlling test runs:
+ V=[0|1|2] - set test verbosity level
+ 0=ERROR, 1=INFO, 2=DEBUG
+ TEST_JOBS=[<n>|auto] - use at most <n> parallel python processes for test execution, if auto, set to number of available cpus (default: 1)
+ MAX_VPP_CPUS=[<n>|auto]- use at most <n> cpus for running vpp main and worker threads, if auto, set to number of available cpus (default: auto)
+ CACHE_OUTPUT=[0|1] - cache VPP stdout/stderr and log as one block after test finishes (default: 1)
+ FAILFAST=[0|1] - fail fast if 1, complete all tests if 0
+ TIMEOUT=<timeout> - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)
+ RETRIES=<n> - retry failed tests <n> times
+ DEBUG=<type> - set VPP debugging kind
+ DEBUG=core - detect coredump and load it in gdb on crash
+ DEBUG=gdb - allow easy debugging by printing VPP PID
+ and waiting for user input before running
+ and tearing down a testcase
+ DEBUG=gdbserver - run gdb inside a gdb server, otherwise
+ same as above
+ DEBUG=attach - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb)
+
+ STEP=[yes|no] - ease debugging by stepping through a testcase
+ SANITY=[yes|no] - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)
+ EXTENDED_TESTS=[1|y] - used by '[re]test-all' & '[re]test-all-debug' to run extended tests
+ TEST=<filter> - filter the set of tests:
+ by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py
+ by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py
+ by wildcard - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'
+ e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name
+ TEST='bfd.*.*' is equivalent to above example of filter by file-suffix
+ TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class
+ TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase
+ TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes
+
+ VARIANT=<variant> - specify which march node variant to unit test
+ e.g. VARIANT=skx test the skx march variants
+ e.g. VARIANT=icl test the icl march variants
+
+ COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp
+ e.g. COREDUMP_SIZE=4g
+ COREDUMP_SIZE=unlimited
+ COREDUMP_COMPRESS=1 - compress core files if not debugging them
+ EXTERN_TESTS=<path> - path to out-of-tree test_<name>.py files containing test cases
+ EXTERN_PLUGINS=<path> - path to out-of-tree plugins to be loaded by vpp under test
+ EXTERN_COV_DIR=<path> - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report
+
+ PROFILE=1 - enable profiling of test framework via cProfile module
+ PROFILE_SORT_BY=opt - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)
+ PROFILE_OUTPUT=file - output profiling info to file - use absolute path (default: stdout)
+
+ TEST_DEBUG=1 - turn on debugging of the test framework itself (expert)
+
+ SKIP_AARCH64=1 - skip tests that are failing on the ARM platorm in FD.io CI
+
+ RND_SEED=seed - Seed RND with given seed
+
+ Starting VPP in GDB for use with DEBUG=attach:
+
+ test-start-vpp-in-gdb - start VPP in gdb (release)
+ test-start-vpp-debug-in-gdb - start VPP in gdb (debug)
+
+ Arguments controlling VPP in GDB runs:
+
+ VPP_IN_GDB_TMP_DIR - specify directory to run VPP IN (default: /tmp/unittest-attach-gdb)
+ VPP_IN_GDB_NO_RMDIR=0 - don't remove existing tmp dir but fail instead
+ VPP_IN_GDB_CMDLINE=1 - add 'interactive' to VPP arguments to run with command line
+
+ Creating test documentation
+ test-doc - generate documentation for test framework
+ test-wipe-doc - wipe documentation for test framework
+
+ Creating test code coverage report
+ test-cov - generate code coverage report for test framework
+ test-wipe-cov - wipe code coverage report for test framework
+
+ Verifying code-style
+ test-checkstyle - check PEP8 compliance
diff --git a/docs/developer/corearchitecture/bihash.rst b/docs/developer/corearchitecture/bihash.rst
new file mode 100644
index 0000000..9b62baa
--- /dev/null
+++ b/docs/developer/corearchitecture/bihash.rst
@@ -0,0 +1,313 @@
+Bounded-index Extensible Hashing (bihash)
+=========================================
+
+Vpp uses bounded-index extensible hashing to solve a variety of
+exact-match (key, value) lookup problems. Benefits of the current
+implementation:
+
+- Very high record count scaling, tested to 100,000,000 records.
+- Lookup performance degrades gracefully as the number of records
+ increases
+- No reader locking required
+- Template implementation, it’s easy to support arbitrary (key,value)
+ types
+
+Bounded-index extensible hashing has been widely used in databases for
+decades.
+
+Bihash uses a two-level data structure:
+
+::
+
+ +-----------------+
+ | bucket-0 |
+ | log2_size |
+ | backing store |
+ +-----------------+
+ | bucket-1 |
+ | log2_size | +--------------------------------+
+ | backing store | --------> | KVP_PER_PAGE * key-value-pairs |
+ +-----------------+ | page 0 |
+ ... +--------------------------------+
+ +-----------------+ | KVP_PER_PAGE * key-value-pairs |
+ | bucket-2**N-1 | | page 1 |
+ | log2_size | +--------------------------------+
+ | backing store | ---
+ +-----------------+ +--------------------------------+
+ | KVP_PER_PAGE * key-value-pairs |
+ | page 2**(log2(size)) - 1 |
+ +--------------------------------+
+
+Discussion of the algorithm
+---------------------------
+
+This structure has a couple of major advantages. In practice, each
+bucket entry fits into a 64-bit integer. Coincidentally, vpp’s target
+CPU architectures support 64-bit atomic operations. When modifying the
+contents of a specific bucket, we do the following:
+
+- Make a working copy of the bucket’s backing storage
+- Atomically swap a pointer to the working copy into the bucket array
+- Change the original backing store data
+- Atomically swap back to the original
+
+So, no reader locking is required to search a bihash table.
+
+At lookup time, the implementation computes a key hash code. We use the
+least-significant N bits of the hash to select the bucket.
+
+With the bucket in hand, we learn log2 (nBackingPages) for the selected
+bucket. At this point, we use the next log2_size bits from the hash code
+to select the specific backing page in which the (key,value) page will
+be found.
+
+Net result: we search **one** backing page, not 2**log2_size pages. This
+is a key property of the algorithm.
+
+When sufficient collisions occur to fill the backing pages for a given
+bucket, we double the bucket size, rehash, and deal the bucket contents
+into a double-sized set of backing pages. In the future, we may
+represent the size as a linear combination of two powers-of-two, to
+increase space efficiency.
+
+To solve the “jackpot case” where a set of records collide under hashing
+in a bad way, the implementation will fall back to linear search across
+2**log2_size backing pages on a per-bucket basis.
+
+To maintain *space* efficiency, we should configure the bucket array so
+that backing pages are effectively utilized. Lookup performance tends to
+change *very little* if the bucket array is too small or too large.
+
+Bihash depends on selecting an effective hash function. If one were to
+use a truly broken hash function such as “return 1ULL.” bihash would
+still work, but it would be equivalent to poorly-programmed linear
+search.
+
+We often use cpu intrinsic functions - think crc32 - to rapidly compute
+a hash code which has decent statistics.
+
+Bihash Cookbook
+---------------
+
+Using current (key,value) template instance types
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It’s quite easy to use one of the template instance types. As of this
+writing, …/src/vppinfra provides pre-built templates for 8, 16, 20, 24,
+40, and 48 byte keys, u8 \* vector keys, and 8 byte values.
+
+See …/src/vppinfra/{bihash\_\_8}.h
+
+To define the data types, #include a specific template instance, most
+often in a subsystem header file:
+
+.. code:: c
+
+ #include <vppinfra/bihash_8_8.h>
+
+If you’re building a standalone application, you’ll need to define the
+various functions by #including the method implementation file in a C
+source file.
+
+The core vpp engine currently uses most if not all of the known bihash
+types, so you probably won’t need to #include the method implementation
+file.
+
+.. code:: c
+
+ #include <vppinfra/bihash_template.c>
+
+Add an instance of the selected bihash data structure to e.g. a “main_t”
+structure:
+
+.. code:: c
+
+ typedef struct
+ {
+ ...
+ BVT (clib_bihash) hash_table;
+ or
+ clib_bihash_8_8_t hash_table;
+ ...
+ } my_main_t;
+
+The BV macro concatenate its argument with the value of the preprocessor
+symbol BIHASH_TYPE. The BVT macro concatenates its argument with the
+value of BIHASH_TYPE and the fixed-string “_t”. So in the above example,
+BVT (clib_bihash) generates “clib_bihash_8_8_t”.
+
+If you’re sure you won’t decide to change the template / type name
+later, it’s perfectly OK to code “clib_bihash_8_8_t” and so forth.
+
+In fact, if you #include multiple template instances in a single source
+file, you **must** use fully-enumerated type names. The macros stand no
+chance of working.
+
+Initializing a bihash table
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Call the init function as shown. As a rough guide, pick a number of
+buckets which is approximately
+number_of_expected_records/BIHASH_KVP_PER_PAGE from the relevant
+template instance header-file. See previous discussion.
+
+The amount of memory selected should easily contain all of the records,
+with a generous allowance for hash collisions. Bihash memory is
+allocated separately from the main heap, and won’t cost anything except
+kernel PTE’s until touched, so it’s OK to be reasonably generous.
+
+For example:
+
+.. code:: c
+
+ my_main_t *mm = &my_main;
+ clib_bihash_8_8_t *h;
+
+ h = &mm->hash_table;
+
+ clib_bihash_init_8_8 (h, "test", (u32) number_of_buckets,
+ (uword) memory_size);
+
+Add or delete a key/value pair
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Use BV(clib_bihash_add_del), or the explicit type variant:
+
+.. code:: c
+
+ clib_bihash_kv_8_8_t kv;
+ clib_bihash_8_8_t * h;
+ my_main_t *mm = &my_main;
+ clib_bihash_8_8_t *h;
+
+ h = &mm->hash_table;
+ kv.key = key_to_add_or_delete;
+ kv.value = value_to_add_or_delete;
+
+ clib_bihash_add_del_8_8 (h, &kv, is_add /* 1=add, 0=delete */);
+
+In the delete case, kv.value is irrelevant. To change the value
+associated with an existing (key,value) pair, simply re-add the [new]
+pair.
+
+Simple search
+~~~~~~~~~~~~~
+
+The simplest possible (key, value) search goes like so:
+
+.. code:: c
+
+ clib_bihash_kv_8_8_t search_kv, return_kv;
+ clib_bihash_8_8_t * h;
+ my_main_t *mm = &my_main;
+ clib_bihash_8_8_t *h;
+
+ h = &mm->hash_table;
+ search_kv.key = key_to_add_or_delete;
+
+ if (clib_bihash_search_8_8 (h, &search_kv, &return_kv) < 0)
+ key_not_found();
+ else
+ key_found();
+
+Note that it’s perfectly fine to collect the lookup result
+
+.. code:: c
+
+ if (clib_bihash_search_8_8 (h, &search_kv, &search_kv))
+ key_not_found();
+ etc.
+
+Bihash vector processing
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+When processing a vector of packets which need a certain lookup
+performed, it’s worth the trouble to compute the key hash, and prefetch
+the correct bucket ahead of time.
+
+Here’s a sketch of one way to write the required code:
+
+Dual-loop: \* 6 packets ahead, prefetch 2x vlib_buffer_t’s and 2x packet
+data required to form the record keys \* 4 packets ahead, form 2x record
+keys and call BV(clib_bihash_hash) or the explicit hash function to
+calculate the record hashes. Call 2x BV(clib_bihash_prefetch_bucket) to
+prefetch the buckets \* 2 packets ahead, call 2x
+BV(clib_bihash_prefetch_data) to prefetch 2x (key,value) data pages. \*
+In the processing section, call 2x
+BV(clib_bihash_search_inline_with_hash) to perform the search
+
+Programmer’s choice whether to stash the hash code somewhere in
+vnet_buffer(b) metadata, or to use local variables.
+
+Single-loop: \* Use simple search as shown above.
+
+Walking a bihash table
+~~~~~~~~~~~~~~~~~~~~~~
+
+A fairly common scenario to build “show” commands involves walking a
+bihash table. It’s simple enough:
+
+.. code:: c
+
+ my_main_t *mm = &my_main;
+ clib_bihash_8_8_t *h;
+ void callback_fn (clib_bihash_kv_8_8_t *, void *);
+
+ h = &mm->hash_table;
+
+ BV(clib_bihash_foreach_key_value_pair) (h, callback_fn, (void *) arg);
+
+To nobody’s great surprise: clib_bihash_foreach_key_value_pair iterates
+across the entire table, calling callback_fn with active entries.
+
+Bihash table iteration safety
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The iterator template “clib_bihash_foreach_key_value_pair” must be used
+with a certain amount of care. For one thing, the iterator template does
+*not* take the bihash hash table writer lock. If your use-case requires
+it, lock the table.
+
+For another, the iterator template is not safe under all conditions:
+
+- It’s **OK to delete** bihash table entries during a table-walk. The
+ iterator checks whether the current bucket has been freed after each
+ *callback_fn(…)* invocation.
+
+- It is **not OK to add** entries during a table-walk.
+
+The add-during-walk case involves a jackpot: while processing a
+key-value-pair in a particular bucket, add a certain number of entries.
+By luck, assume that one or more of the added entries causes the
+**current bucket** to split-and-rehash.
+
+Since we rehash KVP’s to different pages based on what amounts to a
+different hash function, either of these things can go wrong:
+
+- We may revisit previously-visited entries. Depending on how one coded
+ the use-case, we could end up in a recursive-add situation.
+
+- We may skip entries that have not been visited
+
+One could build an add-safe iterator, at a significant cost in
+performance: copy the entire bucket, and walk the copy.
+
+It’s hard to imagine a worthwhile add-during walk use-case in the first
+place; let alone one which couldn’t be implemented by walking the table
+without modifying it, then adding a set of records.
+
+Creating a new template instance
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Creating a new template is easy. Use one of the existing templates as a
+model, and make the obvious changes. The hash and key_compare methods
+are performance-critical in multiple senses.
+
+If the key compare method is slow, every lookup will be slow. If the
+hash function is slow, same story. If the hash function has poor
+statistical properties, space efficiency will suffer. In the limit, a
+bad enough hash function will cause large portions of the table to
+revert to linear search.
+
+Use of the best available vector unit is well worth the trouble in the
+hash and key_compare functions.
diff --git a/docs/developer/corearchitecture/buffer_metadata.rst b/docs/developer/corearchitecture/buffer_metadata.rst
new file mode 100644
index 0000000..545c31f
--- /dev/null
+++ b/docs/developer/corearchitecture/buffer_metadata.rst
@@ -0,0 +1,237 @@
+Buffer Metadata
+===============
+
+Each vlib_buffer_t (packet buffer) carries buffer metadata which
+describes the current packet-processing state. The underlying techniques
+have been used for decades, across multiple packet processing
+environments.
+
+We will examine vpp buffer metadata in some detail, but folks who need
+to manipulate and/or extend the scheme should expect to do a certain
+level of code inspection.
+
+Vlib (Vector library) primary buffer metadata
+---------------------------------------------
+
+The first 64 octets of each vlib_buffer_t carries the primary buffer
+metadata. See …/src/vlib/buffer.h for full details.
+
+Important fields:
+
+- i16 current_data: the signed offset in data[], pre_data[] that we are
+ currently processing. If negative current header points into the
+ pre-data (rewrite space) area.
+- u16 current_length: nBytes between current_data and the end of this
+ buffer.
+- u32 flags: Buffer flag bits. Heavily used, not many bits left
+
+ - src/vlib/buffer.h flag bits
+
+ - VLIB_BUFFER_IS_TRACED: buffer is traced
+ - VLIB_BUFFER_NEXT_PRESENT: buffer has multiple chunks
+ - VLIB_BUFFER_TOTAL_LENGTH_VALID:
+ total_length_not_including_first_buffer is valid (see below)
+
+ - src/vnet/buffer.h flag bits
+
+ - VNET_BUFFER_F_L4_CHECKSUM_COMPUTED: tcp/udp checksum has been
+ computed
+ - VNET_BUFFER_F_L4_CHECKSUM_CORRECT: tcp/udp checksum is correct
+ - VNET_BUFFER_F_VLAN_2_DEEP: two vlan tags present
+ - VNET_BUFFER_F_VLAN_1_DEEP: one vlan tag present
+ - VNET_BUFFER_F_SPAN_CLONE: packet has already been cloned (span
+ feature)
+ - VNET_BUFFER_F_LOOP_COUNTER_VALID: packet look-up loop count
+ valid
+ - VNET_BUFFER_F_LOCALLY_ORIGINATED: packet built by vpp
+ - VNET_BUFFER_F_IS_IP4: packet is ipv4, for checksum offload
+ - VNET_BUFFER_F_IS_IP6: packet is ipv6, for checksum offload
+ - VNET_BUFFER_F_OFFLOAD_IP_CKSUM: hardware ip checksum offload
+ requested
+ - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM: hardware tcp checksum offload
+ requested
+ - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM: hardware udp checksum offload
+ requested
+ - VNET_BUFFER_F_IS_NATED: natted packet, skip input checks
+ - VNET_BUFFER_F_L2_HDR_OFFSET_VALID: L2 header offset valid
+ - VNET_BUFFER_F_L3_HDR_OFFSET_VALID: L3 header offset valid
+ - VNET_BUFFER_F_L4_HDR_OFFSET_VALID: L4 header offset valid
+ - VNET_BUFFER_F_FLOW_REPORT: packet is an ipfix packet
+ - VNET_BUFFER_F_IS_DVR: packet to be reinjected into the l2
+ output path
+ - VNET_BUFFER_F_QOS_DATA_VALID: QoS data valid in
+ vnet_buffer_opaque2
+ - VNET_BUFFER_F_GSO: generic segmentation offload requested
+ - VNET_BUFFER_F_AVAIL1: available bit
+ - VNET_BUFFER_F_AVAIL2: available bit
+ - VNET_BUFFER_F_AVAIL3: available bit
+ - VNET_BUFFER_F_AVAIL4: available bit
+ - VNET_BUFFER_F_AVAIL5: available bit
+ - VNET_BUFFER_F_AVAIL6: available bit
+ - VNET_BUFFER_F_AVAIL7: available bit
+
+- u32 flow_id: generic flow identifier
+- u8 ref_count: buffer reference / clone count (e.g. for span
+ replication)
+- u8 buffer_pool_index: buffer pool index which owns this buffer
+- vlib_error_t (u16) error: error code for buffers enqueued to error
+ handler
+- u32 next_buffer: buffer index of next buffer in chain. Only valid if
+ VLIB_BUFFER_NEXT_PRESENT is set
+- union
+
+ - u32 current_config_index: current index on feature arc
+ - u32 punt_reason: reason code once packet punted. Mutually
+ exclusive with current_config_index
+
+- u32 opaque[10]: primary vnet-layer opaque data (see below)
+- END of first cache line / data initialized by the buffer allocator
+- u32 trace_index: buffer’s index in the packet trace subsystem
+- u32 total_length_not_including_first_buffer: see
+ VLIB_BUFFER_TOTAL_LENGTH_VALID above
+- u32 opaque2[14]: secondary vnet-layer opaque data (see below)
+- u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]: rewrite space, often used to
+ prepend tunnel encapsulations
+- u8 data[0]: buffer data received from the wire. Ordinarily, hardware
+ devices use b->data[0] as the DMA target but there are exceptions. Do
+ not write code which blindly assumes that packet data starts in
+ b->data[0]. Use vlib_buffer_get_current(…).
+
+Vnet (network stack) primary buffer metadata
+--------------------------------------------
+
+Vnet primary buffer metadata occupies space reserved in the vlib opaque
+field shown above, and has the type name vnet_buffer_opaque_t.
+Ordinarily accessed using the vnet_buffer(b) macro. See
+../src/vnet/buffer.h for full details.
+
+Important fields:
+
+- u32 sw_if_index[2]: RX and TX interface handles. At the ip lookup
+ stage, vnet_buffer(b)->sw_if_index[VLIB_TX] is interpreted as a FIB
+ index.
+- i16 l2_hdr_offset: offset from b->data[0] of the packet L2 header.
+ Valid only if b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID is set
+- i16 l3_hdr_offset: offset from b->data[0] of the packet L3 header.
+ Valid only if b->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID is set
+- i16 l4_hdr_offset: offset from b->data[0] of the packet L4 header.
+ Valid only if b->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID is set
+- u8 feature_arc_index: feature arc that the packet is currently
+ traversing
+- union
+
+ - ip
+
+ - u32 adj_index[2]: adjacency from dest IP lookup in [VLIB_TX],
+ adjacency from source ip lookup in [VLIB_RX], set to ~0 until
+ source lookup done
+ - union
+
+ - generic fields
+ - ICMP fields
+ - reassembly fields
+
+ - mpls fields
+ - l2 bridging fields, only valid in the L2 path
+ - l2tpv3 fields
+ - l2 classify fields
+ - vnet policer fields
+ - MAP fields
+ - MAP-T fields
+ - ip fragmentation fields
+ - COP (whitelist/blacklist filter) fields
+ - LISP fields
+ - TCP fields
+
+ - connection index
+ - sequence numbers
+ - header and data offsets
+ - data length
+ - flags
+
+ - SCTP fields
+ - NAT fields
+ - u32 unused[6]
+
+Vnet (network stack) secondary buffer metadata
+----------------------------------------------
+
+Vnet primary buffer metadata occupies space reserved in the vlib opaque2
+field shown above, and has the type name vnet_buffer_opaque2_t.
+Ordinarily accessed using the vnet_buffer2(b) macro. See
+../src/vnet/buffer.h for full details.
+
+Important fields:
+
+- qos fields
+
+ - u8 bits
+ - u8 source
+
+- u8 loop_counter: used to detect and report internal forwarding loops
+- group-based policy fields
+
+ - u8 flags
+ - u16 sclass: the packet’s source class
+
+- u16 gso_size: L4 payload size, persists all the way to
+ interface-output in case GSO is not enabled
+- u16 gso_l4_hdr_sz: size of the L4 protocol header
+- union
+
+ - packet trajectory tracer (largely deprecated)
+
+ - u16 \*trajectory_trace; only #if VLIB_BUFFER_TRACE_TRAJECTORY >
+ 0
+
+ - packet generator
+
+ - u64 pg_replay_timestamp: timestamp for replayed pcap trace
+ packets
+
+ - u32 unused[8]
+
+Buffer Metadata Extensions
+--------------------------
+
+Plugin developers may wish to extend either the primary or secondary
+vnet buffer opaque unions. Please perform a manual live variable
+analysis, otherwise nodes which use shared buffer metadata space may
+break things.
+
+It’s not OK to add plugin or proprietary metadata to the core vpp engine
+header files named above. Instead, proceed as follows. The example
+concerns the vnet primary buffer opaque union vlib_buffer_opaque_t. It’s
+a very simple variation to use the vnet secondary buffer opaque union
+vlib_buffer_opaque2_t.
+
+In a plugin header file:
+
+::
+
+ /* Add arbitrary buffer metadata */
+ #include <vnet/buffer.h>
+
+ typedef struct
+ {
+ u32 my_stuff[6];
+ } my_buffer_opaque_t;
+
+ STATIC_ASSERT (sizeof (my_buffer_opaque_t) <=
+ STRUCT_SIZE_OF (vnet_buffer_opaque_t, unused),
+ "Custom meta-data too large for vnet_buffer_opaque_t");
+
+ #define my_buffer_opaque(b) \
+ ((my_buffer_opaque_t *)((u8 *)((b)->opaque) + STRUCT_OFFSET_OF (vnet_buffer_opaque_t, unused)))
+
+To set data in the custom buffer opaque type given a vlib_buffer_t \*b:
+
+::
+
+ my_buffer_opaque (b)->my_stuff[2] = 123;
+
+To read data from the custom buffer opaque type:
+
+::
+
+ stuff0 = my_buffer_opaque (b)->my_stuff[2];
diff --git a/docs/gettingstarted/developers/buildsystem/buildrootmakefile.rst b/docs/developer/corearchitecture/buildsystem/buildrootmakefile.rst
similarity index 98%
rename from docs/gettingstarted/developers/buildsystem/buildrootmakefile.rst
rename to docs/developer/corearchitecture/buildsystem/buildrootmakefile.rst
index 84647d0..1eb4e6b 100644
--- a/docs/gettingstarted/developers/buildsystem/buildrootmakefile.rst
+++ b/docs/developer/corearchitecture/buildsystem/buildrootmakefile.rst
@@ -147,20 +147,20 @@
::
MACHINE=$(shell uname -m)
-
+
vpp_arch = native
ifeq ($(TARGET_PLATFORM),thunderx)
vpp_dpdk_target = arm64-thunderx-linuxapp-gcc
endif
vpp_native_tools = vppapigen
-
+
vpp_uses_dpdk = yes
-
+
# Uncomment to enable building unit tests
# vpp_enable_tests = yes
-
+
vpp_root_packages = vpp
-
+
# DPDK configuration parameters
# vpp_uses_dpdk_mlx4_pmd = yes
# vpp_uses_dpdk_mlx5_pmd = yes
@@ -168,17 +168,17 @@
# vpp_dpdk_inc_dir = /usr/include/dpdk
# vpp_dpdk_lib_dir = /usr/lib
# vpp_dpdk_shared_lib = yes
-
+
# Use '--without-libnuma' for non-numa aware architecture
# Use '--enable-dlmalloc' to use dlmalloc instead of mheap
vpp_configure_args_vpp = --enable-dlmalloc
sample-plugin_configure_args_vpp = --enable-dlmalloc
-
+
# load balancer plugin is not portable on 32 bit platform
ifeq ($(MACHINE),i686)
vpp_configure_args_vpp += --disable-lb-plugin
endif
-
+
vpp_debug_TAG_CFLAGS = -g -O0 -DCLIB_DEBUG \
-fstack-protector-all -fPIC -Werror
vpp_debug_TAG_CXXFLAGS = -g -O0 -DCLIB_DEBUG \
@@ -234,18 +234,18 @@
+------------------+----------------------+--------------------------------------------------------------------------------------+
| bootstrap-tools | none | Builds the set of native tools needed by the vpp build system to |
| | | build images. Example: vppapigen. In a full cross compilation case might include |
-| | | include "make", "git", "find", and "tar |
-+------------------+----------------------+--------------------------------------------------------------------------------------+
+| | | include "make", "git", "find", and "tar |
++------------------+----------------------+--------------------------------------------------------------------------------------+
| install-tools | PLATFORM | Builds the tool chain for the indicated <platform>. Not used in vpp builds |
-+------------------+----------------------+--------------------------------------------------------------------------------------+
++------------------+----------------------+--------------------------------------------------------------------------------------+
| distclean | none | Roto-rooters everything in sight: toolchains, images, and so forth. |
-+------------------+----------------------+--------------------------------------------------------------------------------------+
++------------------+----------------------+--------------------------------------------------------------------------------------+
| install-deb | PLATFORM and TAG | Build Debian packages comprising components listed in <platform>_root_packages, |
| | | using compile / link options defined by TAG. |
-+------------------+----------------------+--------------------------------------------------------------------------------------+
++------------------+----------------------+--------------------------------------------------------------------------------------+
| install-rpm | PLATFORM and TAG | Build RPMs comprising components listed in <platform>_root_packages, |
| | | using compile / link options defined by TAG. |
-+------------------+----------------------+--------------------------------------------------------------------------------------+
++------------------+----------------------+--------------------------------------------------------------------------------------+
Additional build-root/Makefile environment variable settings
------------------------------------------------------------
@@ -257,11 +257,11 @@
+======================+======================+=====================================================================================+
| BUILD_DEBUG=vx | Directs Makefile et al. to make a good-faith effort to show what's going on in excruciating detail. |
| | Use it as follows: "make ... BUILD_DEBUG=vx". Fairly effective in Makefile debug situations. |
-+----------------------+------------------------------------------------------------------------------------------------------------+
++----------------------+------------------------------------------------------------------------------------------------------------+
| V=1 | print detailed cc / ld command lines. Useful for discovering if -DFOO=11 is in the command line or not |
-+----------------------+------------------------------------------------------------------------------------------------------------+
++----------------------+------------------------------------------------------------------------------------------------------------+
| CC=mygcc | Override the configured C-compiler |
-+----------------------+------------------------------------------------------------------------------------------------------------+
++----------------------+------------------------------------------------------------------------------------------------------------+
.../build-root/config.site
--------------------------
@@ -272,7 +272,7 @@
::
- # glibc needs these setting for cross compiling
+ # glibc needs these setting for cross compiling
libc_cv_forced_unwind=yes
libc_cv_c_cleanup=yes
libc_cv_ssp=no
@@ -320,25 +320,25 @@
| | configure this component until you've successfully built the indicated targets. Almost always, |
| | xxx_configure_depend will list a set of "yyy-install" targets. Note the pattern: |
| | "variable names contain underscores, make target names contain hyphens" |
-+----------------------+------------------------------------------------------------------------------------------------------------+
++----------------------+------------------------------------------------------------------------------------------------------------+
| xxx_configure_args | (optional) Lists any additional arguments to pass to the xxx component "configure" script. |
| | The main Makefile %-configure rule adds the required settings for --libdir, --prefix, and |
| | --host (when cross-compiling) |
-+----------------------+------------------------------------------------------------------------------------------------------------+
++----------------------+------------------------------------------------------------------------------------------------------------+
| xxx_CPPFLAGS | Adds -I stanzas to CPPFLAGS for components upon which xxx depends. |
| | Almost invariably "xxx_CPPFLAGS = $(call installed_includes_fn, dep1 dep2 dep3)", where dep1, dep2, and |
| | dep3 are listed in xxx_configure_depend. It is bad practice to set "-g -O3" here. Those settings |
| | belong in a TAG. |
-+----------------------+------------------------------------------------------------------------------------------------------------+
++----------------------+------------------------------------------------------------------------------------------------------------+
| xxx_LDFLAGS | Adds -Wl,-rpath -Wl,depN stanzas to LDFLAGS for components upon which xxx depends. |
| | Almost invariably "xxx_LDFLAGS = $(call installed_lib_fn, dep1 dep2 dep3)", where dep1, dep2, and |
| | dep3 are listed in xxx_configure_depend. It is bad manners to set "-liberty-or-death" here. |
| | Those settings belong in Makefile.am. |
-+----------------------+------------------------------------------------------------------------------------------------------------+
++----------------------+------------------------------------------------------------------------------------------------------------+
When dealing with "irritating" components built with raw Makefiles
which only work when building in the source tree, we use a specific
-strategy in the xxx.mk file.
+strategy in the xxx.mk file.
The strategy is simple for those components: We copy the source tree
into .../vpp/build-root/build-xxx. This works, but completely defeats
@@ -350,4 +350,4 @@
and performs the BSD equivalent of "autoreconf -i -f" to configure the
build area. The rest of the file is similar: a bunch of hand-rolled
glue code which manages to make the dpdk act like a good vpp build
-citizen even though it is not.
+citizen even though it is not.
diff --git a/docs/gettingstarted/developers/buildsystem/cmakeandninja.rst b/docs/developer/corearchitecture/buildsystem/cmakeandninja.rst
similarity index 100%
rename from docs/gettingstarted/developers/buildsystem/cmakeandninja.rst
rename to docs/developer/corearchitecture/buildsystem/cmakeandninja.rst
diff --git a/docs/gettingstarted/developers/buildsystem/index.rst b/docs/developer/corearchitecture/buildsystem/index.rst
similarity index 100%
rename from docs/gettingstarted/developers/buildsystem/index.rst
rename to docs/developer/corearchitecture/buildsystem/index.rst
diff --git a/docs/gettingstarted/developers/buildsystem/mainmakefile.md b/docs/developer/corearchitecture/buildsystem/mainmakefile.rst
similarity index 97%
rename from docs/gettingstarted/developers/buildsystem/mainmakefile.md
rename to docs/developer/corearchitecture/buildsystem/mainmakefile.rst
index ddf0661..96b9749 100644
--- a/docs/gettingstarted/developers/buildsystem/mainmakefile.md
+++ b/docs/developer/corearchitecture/buildsystem/mainmakefile.rst
@@ -1,3 +1,2 @@
Introduction to the top-level Makefile
======================================
-
diff --git a/docs/developer/corearchitecture/featurearcs.rst b/docs/developer/corearchitecture/featurearcs.rst
new file mode 100644
index 0000000..89c50e3
--- /dev/null
+++ b/docs/developer/corearchitecture/featurearcs.rst
@@ -0,0 +1,225 @@
+Feature Arcs
+============
+
+A significant number of vpp features are configurable on a per-interface
+or per-system basis. Rather than ask feature coders to manually
+construct the required graph arcs, we built a general mechanism to
+manage these mechanics.
+
+Specifically, feature arcs comprise ordered sets of graph nodes. Each
+feature node in an arc is independently controlled. Feature arc nodes
+are generally unaware of each other. Handing a packet to “the next
+feature node” is quite inexpensive.
+
+The feature arc implementation solves the problem of creating graph arcs
+used for steering.
+
+At the beginning of a feature arc, a bit of setup work is needed, but
+only if at least one feature is enabled on the arc.
+
+On a per-arc basis, individual feature definitions create a set of
+ordering dependencies. Feature infrastructure performs a topological
+sort of the ordering dependencies, to determine the actual feature
+order. Missing dependencies **will** lead to runtime disorder. See
+https://gerrit.fd.io/r/#/c/12753 for an example.
+
+If no partial order exists, vpp will refuse to run. Circular dependency
+loops of the form “a then b, b then c, c then a” are impossible to
+satisfy.
+
+Adding a feature to an existing feature arc
+-------------------------------------------
+
+To nobody’s great surprise, we set up feature arcs using the typical
+“macro -> constructor function -> list of declarations” pattern:
+
+.. code:: c
+
+ VNET_FEATURE_INIT (mactime, static) =
+ {
+ .arc_name = "device-input",
+ .node_name = "mactime",
+ .runs_before = VNET_FEATURES ("ethernet-input"),
+ };
+
+This creates a “mactime” feature on the “device-input” arc.
+
+Once per frame, dig up the vnet_feature_config_main_t corresponding to
+the “device-input” feature arc:
+
+.. code:: c
+
+ vnet_main_t *vnm = vnet_get_main ();
+ vnet_interface_main_t *im = &vnm->interface_main;
+ u8 arc = im->output_feature_arc_index;
+ vnet_feature_config_main_t *fcm;
+
+ fcm = vnet_feature_get_config_main (arc);
+
+Note that in this case, we’ve stored the required arc index - assigned
+by the feature infrastructure - in the vnet_interface_main_t. Where to
+put the arc index is a programmer’s decision when creating a feature
+arc.
+
+Per packet, set next0 to steer packets to the next node they should
+visit:
+
+.. code:: c
+
+ vnet_get_config_data (&fcm->config_main,
+ &b0->current_config_index /* value-result */,
+ &next0, 0 /* # bytes of config data */);
+
+Configuration data is per-feature arc, and is often unused. Note that
+it’s normal to reset next0 to divert packets elsewhere; often, to drop
+them for cause:
+
+.. code:: c
+
+ next0 = MACTIME_NEXT_DROP;
+ b0->error = node->errors[DROP_CAUSE];
+
+Creating a feature arc
+----------------------
+
+Once again, we create feature arcs using constructor macros:
+
+.. code:: c
+
+ VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
+ {
+ .arc_name = "ip4-unicast",
+ .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
+ .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
+ };
+
+In this case, we configure two arc start nodes to handle the
+“hardware-verified ip checksum or not” cases. During initialization, the
+feature infrastructure stores the arc index as shown.
+
+In the head-of-arc node, do the following to send packets along the
+feature arc:
+
+.. code:: c
+
+ ip_lookup_main_t *lm = &im->lookup_main;
+ arc = lm->ucast_feature_arc_index;
+
+Once per packet, initialize packet metadata to walk the feature arc:
+
+.. code:: c
+
+ vnet_feature_arc_start (arc, sw_if_index0, &next, b0);
+
+Enabling / Disabling features
+-----------------------------
+
+Simply call vnet_feature_enable_disable to enable or disable a specific
+feature:
+
+.. code:: c
+
+ vnet_feature_enable_disable ("device-input", /* arc name */
+ "mactime", /* feature name */
+ sw_if_index, /* Interface sw_if_index */
+ enable_disable, /* 1 => enable */
+ 0 /* (void *) feature_configuration */,
+ 0 /* feature_configuration_nbytes */);
+
+The feature_configuration opaque is seldom used.
+
+If you wish to make a feature a *de facto* system-level concept, pass
+sw_if_index=0 at all times. Sw_if_index 0 is always valid, and
+corresponds to the “local” interface.
+
+Related “show” commands
+-----------------------
+
+To display the entire set of features, use “show features [verbose]”.
+The verbose form displays arc indices, and feature indicies within the
+arcs
+
+::
+
+ $ vppctl show features verbose
+ Available feature paths
+ <snip>
+ [14] ip4-unicast:
+ [ 0]: nat64-out2in-handoff
+ [ 1]: nat64-out2in
+ [ 2]: nat44-ed-hairpin-dst
+ [ 3]: nat44-hairpin-dst
+ [ 4]: ip4-dhcp-client-detect
+ [ 5]: nat44-out2in-fast
+ [ 6]: nat44-in2out-fast
+ [ 7]: nat44-handoff-classify
+ [ 8]: nat44-out2in-worker-handoff
+ [ 9]: nat44-in2out-worker-handoff
+ [10]: nat44-ed-classify
+ [11]: nat44-ed-out2in
+ [12]: nat44-ed-in2out
+ [13]: nat44-det-classify
+ [14]: nat44-det-out2in
+ [15]: nat44-det-in2out
+ [16]: nat44-classify
+ [17]: nat44-out2in
+ [18]: nat44-in2out
+ [19]: ip4-qos-record
+ [20]: ip4-vxlan-gpe-bypass
+ [21]: ip4-reassembly-feature
+ [22]: ip4-not-enabled
+ [23]: ip4-source-and-port-range-check-rx
+ [24]: ip4-flow-classify
+ [25]: ip4-inacl
+ [26]: ip4-source-check-via-rx
+ [27]: ip4-source-check-via-any
+ [28]: ip4-policer-classify
+ [29]: ipsec-input-ip4
+ [30]: vpath-input-ip4
+ [31]: ip4-vxlan-bypass
+ [32]: ip4-lookup
+ <snip>
+
+Here, we learn that the ip4-unicast feature arc has index 14, and that
+e.g. ip4-inacl is the 25th feature in the generated partial order.
+
+To display the features currently active on a specific interface, use
+“show interface features”:
+
+::
+
+ $ vppctl show interface GigabitEthernet3/0/0 features
+ Feature paths configured on GigabitEthernet3/0/0...
+ <snip>
+ ip4-unicast:
+ nat44-out2in
+ <snip>
+
+Table of Feature Arcs
+---------------------
+
+Simply search for name-strings to track down the arc definition,
+location of the arc index, etc.
+
+::
+
+ | Arc Name |
+ |------------------|
+ | device-input |
+ | ethernet-output |
+ | interface-output |
+ | ip4-drop |
+ | ip4-local |
+ | ip4-multicast |
+ | ip4-output |
+ | ip4-punt |
+ | ip4-unicast |
+ | ip6-drop |
+ | ip6-local |
+ | ip6-multicast |
+ | ip6-output |
+ | ip6-punt |
+ | ip6-unicast |
+ | mpls-input |
+ | mpls-output |
+ | nsh-output |
diff --git a/docs/developer/corearchitecture/index.rst b/docs/developer/corearchitecture/index.rst
new file mode 100644
index 0000000..ecd5a3c
--- /dev/null
+++ b/docs/developer/corearchitecture/index.rst
@@ -0,0 +1,21 @@
+.. _corearchitecture:
+
+=================
+Core Architecture
+=================
+
+.. toctree::
+ :maxdepth: 1
+
+ softwarearchitecture
+ infrastructure
+ vlib
+ vnet
+ featurearcs
+ buffer_metadata
+ multiarch/index
+ bihash
+ buildsystem/index
+ mem
+ multi_thread
+
diff --git a/docs/developer/corearchitecture/infrastructure.rst b/docs/developer/corearchitecture/infrastructure.rst
new file mode 100644
index 0000000..b4e1065
--- /dev/null
+++ b/docs/developer/corearchitecture/infrastructure.rst
@@ -0,0 +1,612 @@
+VPPINFRA (Infrastructure)
+=========================
+
+The files associated with the VPP Infrastructure layer are located in
+the ``./src/vppinfra`` folder.
+
+VPPinfra is a collection of basic c-library services, quite sufficient
+to build standalone programs to run directly on bare metal. It also
+provides high-performance dynamic arrays, hashes, bitmaps,
+high-precision real-time clock support, fine-grained event-logging, and
+data structure serialization.
+
+One fair comment / fair warning about vppinfra: you can't always tell a
+macro from an inline function from an ordinary function simply by name.
+Macros are used to avoid function calls in the typical case, and to
+cause (intentional) side-effects.
+
+Vppinfra has been around for almost 20 years and tends not to change
+frequently. The VPP Infrastructure layer contains the following
+functions:
+
+Vectors
+-------
+
+Vppinfra vectors are ubiquitous dynamically resized arrays with by user
+defined "headers". Many vpppinfra data structures (e.g. hash, heap,
+pool) are vectors with various different headers.
+
+The memory layout looks like this:
+
+::
+
+ User header (optional, uword aligned)
+ Alignment padding (if needed)
+ Vector length in elements
+ User's pointer -> Vector element 0
+ Vector element 1
+ ...
+ Vector element N-1
+
+As shown above, the vector APIs deal with pointers to the 0th element of
+a vector. Null pointers are valid vectors of length zero.
+
+To avoid thrashing the memory allocator, one often resets the length of
+a vector to zero while retaining the memory allocation. Set the vector
+length field to zero via the vec_reset_length(v) macro. [Use the macro!
+It’s smart about NULL pointers.]
+
+Typically, the user header is not present. User headers allow for other
+data structures to be built atop vppinfra vectors. Users may specify the
+alignment for first data element of a vector via the [vec]()*_aligned
+macros.
+
+Vector elements can be any C type e.g. (int, double, struct bar). This
+is also true for data types built atop vectors (e.g. heap, pool, etc.).
+Many macros have \_a variants supporting alignment of vector elements
+and \_h variants supporting non-zero-length vector headers. The \_ha
+variants support both. Additionally cacheline alignment within a vector
+element structure can be specified using the
+``[CLIB_CACHE_LINE_ALIGN_MARK]()`` macro.
+
+Inconsistent usage of header and/or alignment related macro variants
+will cause delayed, confusing failures.
+
+Standard programming error: memorize a pointer to the ith element of a
+vector, and then expand the vector. Vectors expand by 3/2, so such code
+may appear to work for a period of time. Correct code almost always
+memorizes vector **indices** which are invariant across reallocations.
+
+In typical application images, one supplies a set of global functions
+designed to be called from gdb. Here are a few examples:
+
+- vl(v) - prints vec_len(v)
+- pe(p) - prints pool_elts(p)
+- pifi(p, index) - prints pool_is_free_index(p, index)
+- debug_hex_bytes (p, nbytes) - hex memory dump nbytes starting at p
+
+Use the “show gdb” debug CLI command to print the current set.
+
+Bitmaps
+-------
+
+Vppinfra bitmaps are dynamic, built using the vppinfra vector APIs.
+Quite handy for a variety jobs.
+
+Pools
+-----
+
+Vppinfra pools combine vectors and bitmaps to rapidly allocate and free
+fixed-size data structures with independent lifetimes. Pools are perfect
+for allocating per-session structures.
+
+Hashes
+------
+
+Vppinfra provides several hash flavors. Data plane problems involving
+packet classification / session lookup often use
+./src/vppinfra/bihash_template.[ch] bounded-index extensible hashes.
+These templates are instantiated multiple times, to efficiently service
+different fixed-key sizes.
+
+Bihashes are thread-safe. Read-locking is not required. A simple
+spin-lock ensures that only one thread writes an entry at a time.
+
+The original vppinfra hash implementation in ./src/vppinfra/hash.[ch]
+are simple to use, and are often used in control-plane code which needs
+exact-string-matching.
+
+In either case, one almost always looks up a key in a hash table to
+obtain an index in a related vector or pool. The APIs are simple enough,
+but one must take care when using the unmanaged arbitrary-sized key
+variant. Hash_set_mem (hash_table, key_pointer, value) memorizes
+key_pointer. It is usually a bad mistake to pass the address of a vector
+element as the second argument to hash_set_mem. It is perfectly fine to
+memorize constant string addresses in the text segment.
+
+Timekeeping
+-----------
+
+Vppinfra includes high-precision, low-cost timing services. The datatype
+clib_time_t and associated functions reside in ./src/vppinfra/time.[ch].
+Call clib_time_init (clib_time_t \*cp) to initialize the clib_time_t
+object.
+
+Clib_time_init(…) can use a variety of different ways to establish the
+hardware clock frequency. At the end of the day, vppinfra timekeeping
+takes the attitude that the operating system’s clock is the closest
+thing to a gold standard it has handy.
+
+When properly configured, NTP maintains kernel clock synchronization
+with a highly accurate off-premises reference clock. Notwithstanding
+network propagation delays, a synchronized NTP client will keep the
+kernel clock accurate to within 50ms or so.
+
+Why should one care? Simply put, oscillators used to generate CPU ticks
+aren’t super accurate. They work pretty well, but a 0.1% error wouldn’t
+be out of the question. That’s a minute and a half’s worth of error in 1
+day. The error changes constantly, due to temperature variation, and a
+host of other physical factors.
+
+It’s far too expensive to use system calls for timing, so we’re left
+with the problem of continuously adjusting our view of the CPU tick
+register’s clocks_per_second parameter.
+
+The clock rate adjustment algorithm measures the number of cpu ticks and
+the “gold standard” reference time across an interval of approximately
+16 seconds. We calculate clocks_per_second for the interval: use rdtsc
+(on x86_64) and a system call to get the latest cpu tick count and the
+kernel’s latest nanosecond timestamp. We subtract the previous interval
+end values, and use exponential smoothing to merge the new clock rate
+sample into the clocks_per_second parameter.
+
+As of this writing, we maintain the clock rate by way of the following
+first-order differential equation:
+
+.. code:: c
+
+ clocks_per_second(t) = clocks_per_second(t-1) * K + sample_cps(t)*(1-K)
+ where K = e**(-1.0/3.75);
+
+This yields a per observation “half-life” of 1 minute. Empirically, the
+clock rate converges within 5 minutes, and appears to maintain
+near-perfect agreement with the kernel clock in the face of ongoing NTP
+time adjustments.
+
+See ./src/vppinfra/time.c:clib_time_verify_frequency(…) to look at the
+rate adjustment algorithm. The code rejects frequency samples
+corresponding to the sort of adjustment which might occur if someone
+changes the gold standard kernel clock by several seconds.
+
+Monotonic timebase support
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Particularly during system initialization, the “gold standard” system
+reference clock can change by a large amount, in an instant. It’s not a
+best practice to yank the reference clock - in either direction - by
+hours or days. In fact, some poorly-constructed use-cases do so.
+
+To deal with this reality, clib_time_now(…) returns the number of
+seconds since vpp started, *guaranteed to be monotonically increasing,
+no matter what happens to the system reference clock*.
+
+This is first-order important, to avoid breaking every active timer in
+the system. The vpp host stack alone may account for tens of millions of
+active timers. It’s utterly impractical to track down and fix timers, so
+we must deal with the issue at the timebase level.
+
+Here’s how it works. Prior to adjusting the clock rate, we collect the
+kernel reference clock and the cpu clock:
+
+.. code:: c
+
+ /* Ask the kernel and the CPU what time it is... */
+ now_reference = unix_time_now ();
+ now_clock = clib_cpu_time_now ();
+
+Compute changes for both clocks since the last rate adjustment, roughly
+15 seconds ago:
+
+.. code:: c
+
+ /* Compute change in the reference clock */
+ delta_reference = now_reference - c->last_verify_reference_time;
+
+ /* And change in the CPU clock */
+ delta_clock_in_seconds = (f64) (now_clock - c->last_verify_cpu_time) *
+ c->seconds_per_clock;
+
+Delta_reference is key. Almost 100% of the time, delta_reference and
+delta_clock_in_seconds are identical modulo one system-call time.
+However, NTP or a privileged user can yank the system reference time -
+in either direction - by an hour, a day, or a decade.
+
+As described above, clib_time_now(…) must return monotonically
+increasing answers to the question “how long has it been since vpp
+started, in seconds.” To do that, the clock rate adjustment algorithm
+begins by recomputing the initial reference time:
+
+.. code:: c
+
+ c->init_reference_time += (delta_reference - delta_clock_in_seconds);
+
+It’s easy to convince yourself that if the reference clock changes by
+15.000000 seconds and the cpu clock tick time changes by 15.000000
+seconds, the initial reference time won’t change.
+
+If, on the other hand, delta_reference is -86400.0 and delta clock is
+15.0 - reference time jumped backwards by exactly one day in a 15-second
+rate update interval - we add -86415.0 to the initial reference time.
+
+Given the corrected initial reference time, we recompute the total
+number of cpu ticks which have occurred since the corrected initial
+reference time, at the current clock tick rate:
+
+.. code:: c
+
+ c->total_cpu_time = (now_reference - c->init_reference_time)
+ * c->clocks_per_second;
+
+Timebase precision
+~~~~~~~~~~~~~~~~~~
+
+Cognoscenti may notice that vlib/clib_time_now(…) return a 64-bit
+floating-point value; the number of seconds since vpp started.
+
+Please see `this Wikipedia
+article <https://en.wikipedia.org/wiki/Double-precision_floating-point_format>`__
+for more information. C double-precision floating point numbers (called
+f64 in the vpp code base) have a 53-bit effective mantissa, and can
+accurately represent 15 decimal digits’ worth of precision.
+
+There are 315,360,000.000001 seconds in ten years plus one microsecond.
+That string has exactly 15 decimal digits. The vpp time base retains 1us
+precision for roughly 30 years.
+
+vlib/clib_time_now do *not* provide precision in excess of 1e-6 seconds.
+If necessary, please use clib_cpu_time_now(…) for direct access to the
+CPU clock-cycle counter. Note that the number of CPU clock cycles per
+second varies significantly across CPU architectures.
+
+Timer Wheels
+------------
+
+Vppinfra includes configurable timer wheel support. See the source code
+in …/src/vppinfra/tw_timer_template.[ch], as well as a considerable
+number of template instances defined in …/src/vppinfra/tw_timer\_.[ch].
+
+Instantiation of tw_timer_template.h generates named structures to
+implement specific timer wheel geometries. Choices include: number of
+timer wheels (currently, 1 or 2), number of slots per ring (a power of
+two), and the number of timers per “object handle”.
+
+Internally, user object/timer handles are 32-bit integers, so if one
+selects 16 timers/object (4 bits), the resulting timer wheel handle is
+limited to 2**28 objects.
+
+Here are the specific settings required to generate a single 2048 slot
+wheel which supports 2 timers per object:
+
+.. code:: c
+
+ #define TW_TIMER_WHEELS 1
+ #define TW_SLOTS_PER_RING 2048
+ #define TW_RING_SHIFT 11
+ #define TW_RING_MASK (TW_SLOTS_PER_RING -1)
+ #define TW_TIMERS_PER_OBJECT 2
+ #define LOG2_TW_TIMERS_PER_OBJECT 1
+ #define TW_SUFFIX _2t_1w_2048sl
+ #define TW_FAST_WHEEL_BITMAP 0
+ #define TW_TIMER_ALLOW_DUPLICATE_STOP 0
+
+See tw_timer_2t_1w_2048sl.h for a complete example.
+
+tw_timer_template.h is not intended to be #included directly. Client
+codes can include multiple timer geometry header files, although extreme
+caution would required to use the TW and TWT macros in such a case.
+
+API usage examples
+~~~~~~~~~~~~~~~~~~
+
+The unit test code in …/src/vppinfra/test_tw_timer.c provides a concrete
+API usage example. It uses a synthetic clock to rapidly exercise the
+underlying tw_timer_expire_timers(…) template.
+
+There are not many API routines to call.
+
+Initialize a two-timer, single 2048-slot wheel w/ a 1-second timer granularity
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: c
+
+ tw_timer_wheel_init_2t_1w_2048sl (&tm->single_wheel,
+ expired_timer_single_callback,
+ 1.0 / * timer interval * / );
+
+Start a timer
+^^^^^^^^^^^^^
+
+.. code:: c
+
+ handle = tw_timer_start_2t_1w_2048sl (&tm->single_wheel, elt_index,
+ [0 | 1] / * timer id * / ,
+ expiration_time_in_u32_ticks);
+
+Stop a timer
+^^^^^^^^^^^^
+
+.. code:: c
+
+ tw_timer_stop_2t_1w_2048sl (&tm->single_wheel, handle);
+
+An expired timer callback
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: c
+
+ static void
+ expired_timer_single_callback (u32 * expired_timers)
+ {
+ int i;
+ u32 pool_index, timer_id;
+ tw_timer_test_elt_t *e;
+ tw_timer_test_main_t *tm = &tw_timer_test_main;
+
+ for (i = 0; i < vec_len (expired_timers);
+ {
+ pool_index = expired_timers[i] & 0x7FFFFFFF;
+ timer_id = expired_timers[i] >> 31;
+
+ ASSERT (timer_id == 1);
+
+ e = pool_elt_at_index (tm->test_elts, pool_index);
+
+ if (e->expected_to_expire != tm->single_wheel.current_tick)
+ {
+ fformat (stdout, "[%d] expired at %d not %d\n",
+ e - tm->test_elts, tm->single_wheel.current_tick,
+ e->expected_to_expire);
+ }
+ pool_put (tm->test_elts, e);
+ }
+ }
+
+We use wheel timers extensively in the vpp host stack. Each TCP session
+needs 5 timers, so supporting 10 million flows requires up to 50 million
+concurrent timers.
+
+Timers rarely expire, so it’s of utmost important that stopping and
+restarting a timer costs as few clock cycles as possible.
+
+Stopping a timer costs a doubly-linked list dequeue. Starting a timer
+involves modular arithmetic to determine the correct timer wheel and
+slot, and a list head enqueue.
+
+Expired timer processing generally involves bulk link-list retirement
+with user callback presentation. Some additional complexity at wheel
+wrap time, to relocate timers from slower-turning timer wheels into
+faster-turning wheels.
+
+Format
+------
+
+Vppinfra format is roughly equivalent to printf.
+
+Format has a few properties worth mentioning. Format’s first argument is
+a (u8 \*) vector to which it appends the result of the current format
+operation. Chaining calls is very easy:
+
+.. code:: c
+
+ u8 * result;
+
+ result = format (0, "junk = %d, ", junk);
+ result = format (result, "more junk = %d\n", more_junk);
+
+As previously noted, NULL pointers are perfectly proper 0-length
+vectors. Format returns a (u8 \*) vector, **not** a C-string. If you
+wish to print a (u8 \*) vector, use the “%v” format string. If you need
+a (u8 \*) vector which is also a proper C-string, either of these
+schemes may be used:
+
+.. code:: c
+
+ vec_add1 (result, 0)
+ or
+ result = format (result, "<whatever>%c", 0);
+
+Remember to vec_free() the result if appropriate. Be careful not to pass
+format an uninitialized (u8 \*).
+
+Format implements a particularly handy user-format scheme via the “%U”
+format specification. For example:
+
+.. code:: c
+
+ u8 * format_junk (u8 * s, va_list *va)
+ {
+ junk = va_arg (va, u32);
+ s = format (s, "%s", junk);
+ return s;
+ }
+
+ result = format (0, "junk = %U, format_junk, "This is some junk");
+
+format_junk() can invoke other user-format functions if desired. The
+programmer shoulders responsibility for argument type-checking. It is
+typical for user format functions to blow up spectacularly if the
+va_arg(va, type) macros don’t match the caller’s idea of reality.
+
+Unformat
+--------
+
+Vppinfra unformat is vaguely related to scanf, but considerably more
+general.
+
+A typical use case involves initializing an unformat_input_t from either
+a C-string or a (u8 \*) vector, then parsing via unformat() as follows:
+
+.. code:: c
+
+ unformat_input_t input;
+ u8 *s = "<some-C-string>";
+
+ unformat_init_string (&input, (char *) s, strlen((char *) s));
+ /* or */
+ unformat_init_vector (&input, <u8-vector>);
+
+Then loop parsing individual elements:
+
+.. code:: c
+
+ while (unformat_check_input (&input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (&input, "value1 %d", &value1))
+ ;/* unformat sets value1 */
+ else if (unformat (&input, "value2 %d", &value2)
+ ;/* unformat sets value2 */
+ else
+ return clib_error_return (0, "unknown input '%U'",
+ format_unformat_error, input);
+ }
+
+As with format, unformat implements a user-unformat function capability
+via a “%U” user unformat function scheme. Generally, one can trivially
+transform “format (s,”foo %d”, foo) -> “unformat (input,”foo %d”,
+&foo)“.
+
+Unformat implements a couple of handy non-scanf-like format specifiers:
+
+.. code:: c
+
+ unformat (input, "enable %=", &enable, 1 /* defaults to 1 */);
+ unformat (input, "bitzero %|", &mask, (1<<0));
+ unformat (input, "bitone %|", &mask, (1<<1));
+ <etc>
+
+The phrase “enable %=” means “set the supplied variable to the default
+value” if unformat parses the “enable” keyword all by itself. If
+unformat parses “enable 123” set the supplied variable to 123.
+
+We could clean up a number of hand-rolled “verbose” + “verbose %d”
+argument parsing codes using “%=”.
+
+The phrase “bitzero %\|” means “set the specified bit in the supplied
+bitmask” if unformat parses “bitzero”. Although it looks like it could
+be fairly handy, it’s very lightly used in the code base.
+
+``%_`` toggles whether or not to skip input white space.
+
+For transition from skip to no-skip in middle of format string, skip
+input white space. For example, the following:
+
+.. code:: c
+
+ fmt = "%_%d.%d%_->%_%d.%d%_"
+ unformat (input, fmt, &one, &two, &three, &four);
+
+matches input “1.2 -> 3.4”. Without this, the space after -> does not
+get skipped.
+
+
+How to parse a single input line
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Debug CLI command functions MUST NOT accidentally consume input
+belonging to other debug CLI commands. Otherwise, it's impossible to
+script a set of debug CLI commands which "work fine" when issued one
+at a time.
+
+This bit of code is NOT correct:
+
+.. code:: c
+
+ /* Eats script input NOT beloging to it, and chokes! */
+ while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (input, ...))
+ ;
+ else if (unformat (input, ...))
+ ;
+ else
+ return clib_error_return (0, "parse error: '%U'",
+ format_unformat_error, input);
+ }
+ }
+
+When executed as part of a script, such a function will return “parse
+error: ‘’” every time, unless it happens to be the last command in the
+script.
+
+Instead, use “unformat_line_input” to consume the rest of a line’s worth
+of input - everything past the path specified in the VLIB_CLI_COMMAND
+declaration.
+
+For example, unformat_line_input with “my_command” set up as shown below
+and user input “my path is clear” will produce an unformat_input_t that
+contains “is clear”.
+
+.. code:: c
+
+ VLIB_CLI_COMMAND (...) = {
+ .path = "my path",
+ };
+
+Here’s a bit of code which shows the required mechanics, in full:
+
+.. code:: c
+
+ static clib_error_t *
+ my_command_fn (vlib_main_t * vm,
+ unformat_input_t * input,
+ vlib_cli_command_t * cmd)
+ {
+ unformat_input_t _line_input, *line_input = &_line_input;
+ u32 this, that;
+ clib_error_t *error = 0;
+
+ if (!unformat_user (input, unformat_line_input, line_input))
+ return 0;
+
+ /*
+ * Here, UNFORMAT_END_OF_INPUT is at the end of the line we consumed,
+ * not at the end of the script...
+ */
+ while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (line_input, "this %u", &this))
+ ;
+ else if (unformat (line_input, "that %u", &that))
+ ;
+ else
+ {
+ error = clib_error_return (0, "parse error: '%U'",
+ format_unformat_error, line_input);
+ goto done;
+ }
+ }
+
+ <do something based on "this" and "that", etc>
+
+ done:
+ unformat_free (line_input);
+ return error;
+ }
+ VLIB_CLI_COMMAND (my_command, static) = {
+ .path = "my path",
+ .function = my_command_fn",
+ };
+
+Vppinfra errors and warnings
+----------------------------
+
+Many functions within the vpp dataplane have return-values of type
+clib_error_t \*. Clib_error_t’s are arbitrary strings with a bit of
+metadata [fatal, warning] and are easy to announce. Returning a NULL
+clib_error_t \* indicates “A-OK, no error.”
+
+Clib_warning(format-args) is a handy way to add debugging output; clib
+warnings prepend function:line info to unambiguously locate the message
+source. Clib_unix_warning() adds perror()-style Linux system-call
+information. In production images, clib_warnings result in syslog
+entries.
+
+Serialization
+-------------
+
+Vppinfra serialization support allows the programmer to easily serialize
+and unserialize complex data structures.
+
+The underlying primitive serialize/unserialize functions use network
+byte-order, so there are no structural issues serializing on a
+little-endian host and unserializing on a big-endian host.
diff --git a/docs/developer/corearchitecture/mem.rst b/docs/developer/corearchitecture/mem.rst
new file mode 120000
index 0000000..0fc53ea
--- /dev/null
+++ b/docs/developer/corearchitecture/mem.rst
@@ -0,0 +1 @@
+../../../src/vpp/mem/mem.rst
\ No newline at end of file
diff --git a/docs/developer/corearchitecture/multi_thread.rst b/docs/developer/corearchitecture/multi_thread.rst
new file mode 100644
index 0000000..195a9b7
--- /dev/null
+++ b/docs/developer/corearchitecture/multi_thread.rst
@@ -0,0 +1,169 @@
+.. _vpp_multi_thread:
+
+Multi-threading in VPP
+======================
+
+Modes
+-----
+
+VPP can work in 2 different modes:
+
+- single-thread
+- multi-thread with worker threads
+
+Single-thread
+~~~~~~~~~~~~~
+
+In a single-thread mode there is one main thread which handles both
+packet processing and other management functions (Command-Line Interface
+(CLI), API, stats). This is the default setup. There is no special
+startup config needed.
+
+Multi-thread with Worker Threads
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In this mode, the main threads handles management functions(debug CLI,
+API, stats collection) and one or more worker threads handle packet
+processing from input to output of the packet.
+
+Each worker thread polls input queues on subset of interfaces.
+
+With RSS (Receive Side Scaling) enabled multiple threads can service one
+physical interface (RSS function on NIC distributes traffic between
+different queues which are serviced by different worker threads).
+
+Thread placement
+----------------
+
+Thread placement is defined in the startup config under the cpu { … }
+section.
+
+The VPP platform can place threads automatically or manually. Automatic
+placement works in the following way:
+
+- if “skip-cores X” is defined first X cores will not be used
+- if “main-core X” is defined, VPP main thread will be placed on core
+ X, otherwise 1st available one will be used
+- if “workers N” is defined vpp will allocate first N available cores
+ and it will run threads on them
+- if “corelist-workers A,B1-Bn,C1-Cn” is defined vpp will automatically
+ assign those CPU cores to worker threads
+
+User can see active placement of cores by using the VPP debug CLI
+command show threads:
+
+.. code-block:: console
+
+ vpd# show threads
+ ID Name Type LWP lcore Core Socket State
+ 0 vpe_main 59723 2 2 0 wait
+ 1 vpe_wk_0 workers 59755 4 4 0 running
+ 2 vpe_wk_1 workers 59756 5 5 0 running
+ 3 vpe_wk_2 workers 59757 6 0 1 running
+ 4 vpe_wk_3 workers 59758 7 1 1 running
+ 5 stats 59775
+ vpd#
+
+The sample output above shows the main thread running on core 2 (2nd
+core on the CPU socket 0), worker threads running on cores 4-7.
+
+Sample Configurations
+---------------------
+
+By default, at start-up VPP uses
+configuration values from: ``/etc/vpp/startup.conf``
+
+The following sections describe some of the additional changes that can be made to this file.
+This file is initially populated from the files located in the following directory ``/vpp/vpp/conf/``
+
+Manual Placement
+~~~~~~~~~~~~~~~~
+
+Manual placement places the main thread on core 1, workers on cores
+4,5,20,21.
+
+.. code-block:: console
+
+ cpu {
+ main-core 1
+ corelist-workers 4-5,20-21
+ }
+
+Auto placement
+--------------
+
+Auto placement is likely to place the main thread on core 1 and workers
+on cores 2,3,4.
+
+.. code-block:: console
+
+ cpu {
+ skip-cores 1
+ workers 3
+ }
+
+Buffer Memory Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The VPP platform is NUMA aware. It can allocate memory for buffers on
+different CPU sockets (NUMA nodes). The amount of memory allocated can
+be defined in the startup config for each CPU socket by using the
+socket-mem A[[,B],C] statement inside the dpdk { … } section.
+
+For example:
+
+.. code-block:: console
+
+ dpdk {
+ socket-mem 1024,1024
+ }
+
+The above configuration allocates 1GB of memory on NUMA#0 and 1GB on
+NUMA#1. Each worker thread uses buffers which are local to itself.
+
+Buffer memory is allocated from hugepages. VPP prefers 1G pages if they
+are available. If not 2MB pages will be used.
+
+VPP takes care of mounting/unmounting hugepages file-system
+automatically so there is no need to do that manually.
+
+’‘’NOTE’’’: If you are running latest VPP release, there is no need for
+specifying socket-mem manually. VPP will discover all NUMA nodes and it
+will allocate 512M on each by default. socket-mem is only needed if
+bigger number of mbufs is required (default is 16384 per socket and can
+be changed with num-mbufs startup config command).
+
+Interface Placement in Multi-thread Setup
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On startup, the VPP platform assigns interfaces (or interface, queue
+pairs if RSS is used) to different worker threads in round robin
+fashion.
+
+The following example shows debug CLI commands to show and change
+interface placement:
+
+.. code-block:: console
+
+ vpd# sh dpdk interface placement
+ Thread 1 (vpp_wk_0 at lcore 5):
+ TenGigabitEthernet2/0/0 queue 0
+ TenGigabitEthernet2/0/1 queue 0
+ Thread 2 (vpp_wk_1 at lcore 6):
+ TenGigabitEthernet2/0/0 queue 1
+ TenGigabitEthernet2/0/1 queue 1
+
+The following shows an example of moving TenGigabitEthernet2/0/1 queue 1
+processing to 1st worker thread:
+
+.. code-block:: console
+
+ vpd# set interface placement TenGigabitEthernet2/0/1 queue 1 thread 1
+
+ vpp# sh dpdk interface placement
+ Thread 1 (vpp_wk_0 at lcore 5):
+ TenGigabitEthernet2/0/0 queue 0
+ TenGigabitEthernet2/0/1 queue 0
+ TenGigabitEthernet2/0/1 queue 1
+ Thread 2 (vpp_wk_1 at lcore 6):
+ TenGigabitEthernet2/0/0 queue 1
diff --git a/docs/gettingstarted/developers/multiarch/arbfns.rst b/docs/developer/corearchitecture/multiarch/arbfns.rst
similarity index 100%
rename from docs/gettingstarted/developers/multiarch/arbfns.rst
rename to docs/developer/corearchitecture/multiarch/arbfns.rst
diff --git a/docs/gettingstarted/developers/multiarch/index.rst b/docs/developer/corearchitecture/multiarch/index.rst
similarity index 100%
rename from docs/gettingstarted/developers/multiarch/index.rst
rename to docs/developer/corearchitecture/multiarch/index.rst
diff --git a/docs/gettingstarted/developers/multiarch/nodefns.rst b/docs/developer/corearchitecture/multiarch/nodefns.rst
similarity index 97%
rename from docs/gettingstarted/developers/multiarch/nodefns.rst
rename to docs/developer/corearchitecture/multiarch/nodefns.rst
index a43d40e..9647e64 100644
--- a/docs/gettingstarted/developers/multiarch/nodefns.rst
+++ b/docs/developer/corearchitecture/multiarch/nodefns.rst
@@ -23,9 +23,9 @@
-------
Declare the node dispatch function as shown, using the VLIB\_NODE\_FN macro. The
-name of the node function **MUST** match the name of the graph node.
+name of the node function **MUST** match the name of the graph node.
-::
+::
VLIB_NODE_FN (ip4_sdp_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * frame)
@@ -36,7 +36,7 @@
else
return ip46_sdp_inline (vm, node, frame, 1 /* is_ip4 */ ,
0 /* is_trace */ );
- }
+ }
We need to generate *precisely one copy* of the
vlib_node_registration_t, error strings, and packet trace decode function.
@@ -103,13 +103,13 @@
::
always_inline uword
- ip46_sdp_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+ ip46_sdp_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * frame,
int is_ip4, int is_trace)
{ ... }
Otherwise, the compiler is highly likely NOT to build multiple
-versions of the guts of your dispatch function.
+versions of the guts of your dispatch function.
It's fairly easy to spot this mistake in "perf top." If you see, for
example, a bunch of functions with names of the form
@@ -130,7 +130,7 @@
add_vpp_plugin(myplugin
SOURCES
new_multiarch_node.c
- ...
+ ...
MULTIARCH_SOURCES
new_ multiarch_node.c
diff --git a/docs/developer/corearchitecture/softwarearchitecture.rst b/docs/developer/corearchitecture/softwarearchitecture.rst
new file mode 100644
index 0000000..7f8a0e0
--- /dev/null
+++ b/docs/developer/corearchitecture/softwarearchitecture.rst
@@ -0,0 +1,47 @@
+Software Architecture
+=====================
+
+The fd.io vpp implementation is a third-generation vector packet
+processing implementation specifically related to US Patent 7,961,636,
+as well as earlier work. Note that the Apache-2 license specifically
+grants non-exclusive patent licenses; we mention this patent as a point
+of historical interest.
+
+For performance, the vpp dataplane consists of a directed graph of
+forwarding nodes which process multiple packets per invocation. This
+schema enables a variety of micro-processor optimizations: pipelining
+and prefetching to cover dependent read latency, inherent I-cache phase
+behavior, vector instructions. Aside from hardware input and hardware
+output nodes, the entire forwarding graph is portable code.
+
+Depending on the scenario at hand, we often spin up multiple worker
+threads which process ingress-hashes packets from multiple queues using
+identical forwarding graph replicas.
+
+VPP Layers - Implementation Taxonomy
+------------------------------------
+
+.. figure:: /_images/VPP_Layering.png
+ :alt: image
+
+ image
+
+- VPP Infra - the VPP infrastructure layer, which contains the core
+ library source code. This layer performs memory functions, works with
+ vectors and rings, performs key lookups in hash tables, and works
+ with timers for dispatching graph nodes.
+- VLIB - the vector processing library. The vlib layer also handles
+ various application management functions: buffer, memory and graph
+ node management, maintaining and exporting counters, thread
+ management, packet tracing. Vlib implements the debug CLI (command
+ line interface).
+- VNET - works with VPP's networking interface (layers 2, 3, and 4)
+ performs session and traffic management, and works with devices and
+ the data control plane.
+- Plugins - Contains an increasingly rich set of data-plane plugins, as
+ noted in the above diagram.
+- VPP - the container application linked against all of the above.
+
+It’s important to understand each of these layers in a certain amount of
+detail. Much of the implementation is best dealt with at the API level
+and otherwise left alone.
diff --git a/docs/developer/corearchitecture/vlib.rst b/docs/developer/corearchitecture/vlib.rst
new file mode 100644
index 0000000..f542d33
--- /dev/null
+++ b/docs/developer/corearchitecture/vlib.rst
@@ -0,0 +1,888 @@
+VLIB (Vector Processing Library)
+================================
+
+The files associated with vlib are located in the ./src/{vlib, vlibapi,
+vlibmemory} folders. These libraries provide vector processing support
+including graph-node scheduling, reliable multicast support,
+ultra-lightweight cooperative multi-tasking threads, a CLI, plug in .DLL
+support, physical memory and Linux epoll support. Parts of this library
+embody US Patent 7,961,636.
+
+Init function discovery
+-----------------------
+
+vlib applications register for various [initialization] events by
+placing structures and \__attribute__((constructor)) functions into the
+image. At appropriate times, the vlib framework walks
+constructor-generated singly-linked structure lists, performs a
+topological sort based on specified constraints, and calls the indicated
+functions. Vlib applications create graph nodes, add CLI functions,
+start cooperative multi-tasking threads, etc. etc. using this mechanism.
+
+vlib applications invariably include a number of VLIB_INIT_FUNCTION
+(my_init_function) macros.
+
+Each init / configure / etc. function has the return type clib_error_t
+\*. Make sure that the function returns 0 if all is well, otherwise the
+framework will announce an error and exit.
+
+vlib applications must link against vppinfra, and often link against
+other libraries such as VNET. In the latter case, it may be necessary to
+explicitly reference symbol(s) otherwise large portions of the library
+may be AWOL at runtime.
+
+Init function construction and constraint specification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It’s easy to add an init function:
+
+.. code:: c
+
+ static clib_error_t *my_init_function (vlib_main_t *vm)
+ {
+ /* ... initialize things ... */
+
+ return 0; // or return clib_error_return (0, "BROKEN!");
+ }
+ VLIB_INIT_FUNCTION(my_init_function);
+
+As given, my_init_function will be executed “at some point,” but with no
+ordering guarantees.
+
+Specifying ordering constraints is easy:
+
+.. code:: c
+
+ VLIB_INIT_FUNCTION(my_init_function) =
+ {
+ .runs_before = VLIB_INITS("we_run_before_function_1",
+ "we_run_before_function_2"),
+ .runs_after = VLIB_INITS("we_run_after_function_1",
+ "we_run_after_function_2),
+ };
+
+It’s also easy to specify bulk ordering constraints of the form “a then
+b then c then d”:
+
+.. code:: c
+
+ VLIB_INIT_FUNCTION(my_init_function) =
+ {
+ .init_order = VLIB_INITS("a", "b", "c", "d"),
+ };
+
+It’s OK to specify all three sorts of ordering constraints for a single
+init function, although it’s hard to imagine why it would be necessary.
+
+Node Graph Initialization
+-------------------------
+
+vlib packet-processing applications invariably define a set of graph
+nodes to process packets.
+
+One constructs a vlib_node_registration_t, most often via the
+VLIB_REGISTER_NODE macro. At runtime, the framework processes the set of
+such registrations into a directed graph. It is easy enough to add nodes
+to the graph at runtime. The framework does not support removing nodes.
+
+vlib provides several types of vector-processing graph nodes, primarily
+to control framework dispatch behaviors. The type member of the
+vlib_node_registration_t functions as follows:
+
+- VLIB_NODE_TYPE_PRE_INPUT - run before all other node types
+- VLIB_NODE_TYPE_INPUT - run as often as possible, after pre_input
+ nodes
+- VLIB_NODE_TYPE_INTERNAL - only when explicitly made runnable by
+ adding pending frames for processing
+- VLIB_NODE_TYPE_PROCESS - only when explicitly made runnable.
+ “Process” nodes are actually cooperative multi-tasking threads. They
+ **must** explicitly suspend after a reasonably short period of time.
+
+For a precise understanding of the graph node dispatcher, please read
+./src/vlib/main.c:vlib_main_loop.
+
+Graph node dispatcher
+---------------------
+
+Vlib_main_loop() dispatches graph nodes. The basic vector processing
+algorithm is diabolically simple, but may not be obvious from even a
+long stare at the code. Here’s how it works: some input node, or set of
+input nodes, produce a vector of work to process. The graph node
+dispatcher pushes the work vector through the directed graph,
+subdividing it as needed, until the original work vector has been
+completely processed. At that point, the process recurs.
+
+This scheme yields a stable equilibrium in frame size, by construction.
+Here’s why: as the frame size increases, the per-frame-element
+processing time decreases. There are several related forces at work; the
+simplest to describe is the effect of vector processing on the CPU L1
+I-cache. The first frame element [packet] processed by a given node
+warms up the node dispatch function in the L1 I-cache. All subsequent
+frame elements profit. As we increase the number of frame elements, the
+cost per element goes down.
+
+Under light load, it is a crazy waste of CPU cycles to run the graph
+node dispatcher flat-out. So, the graph node dispatcher arranges to wait
+for work by sitting in a timed epoll wait if the prevailing frame size
+is low. The scheme has a certain amount of hysteresis to avoid
+constantly toggling back and forth between interrupt and polling mode.
+Although the graph dispatcher supports interrupt and polling modes, our
+current default device drivers do not.
+
+The graph node scheduler uses a hierarchical timer wheel to reschedule
+process nodes upon timer expiration.
+
+Graph dispatcher internals
+--------------------------
+
+This section may be safely skipped. It’s not necessary to understand
+graph dispatcher internals to create graph nodes.
+
+Vector Data Structure
+---------------------
+
+In vpp / vlib, we represent vectors as instances of the vlib_frame_t
+type:
+
+.. code:: c
+
+ typedef struct vlib_frame_t
+ {
+ /* Frame flags. */
+ u16 flags;
+
+ /* Number of scalar bytes in arguments. */
+ u8 scalar_size;
+
+ /* Number of bytes per vector argument. */
+ u8 vector_size;
+
+ /* Number of vector elements currently in frame. */
+ u16 n_vectors;
+
+ /* Scalar and vector arguments to next node. */
+ u8 arguments[0];
+ } vlib_frame_t;
+
+Note that one *could* construct all kinds of vectors - including vectors
+with some associated scalar data - using this structure. In the vpp
+application, vectors typically use a 4-byte vector element size, and
+zero bytes’ worth of associated per-frame scalar data.
+
+Frames are always allocated on CLIB_CACHE_LINE_BYTES boundaries. Frames
+have u32 indices which make use of the alignment property, so the
+maximum feasible main heap offset of a frame is CLIB_CACHE_LINE_BYTES \*
+0xFFFFFFFF: 64*4 = 256 Gbytes.
+
+Scheduling Vectors
+------------------
+
+As you can see, vectors are not directly associated with graph nodes. We
+represent that association in a couple of ways. The simplest is the
+vlib_pending_frame_t:
+
+.. code:: c
+
+ /* A frame pending dispatch by main loop. */
+ typedef struct
+ {
+ /* Node and runtime for this frame. */
+ u32 node_runtime_index;
+
+ /* Frame index (in the heap). */
+ u32 frame_index;
+
+ /* Start of next frames for this node. */
+ u32 next_frame_index;
+
+ /* Special value for next_frame_index when there is no next frame. */
+ #define VLIB_PENDING_FRAME_NO_NEXT_FRAME ((u32) ~0)
+ } vlib_pending_frame_t;
+
+Here is the code in …/src/vlib/main.c:vlib_main_or_worker_loop() which
+processes frames:
+
+.. code:: c
+
+ /*
+ * Input nodes may have added work to the pending vector.
+ * Process pending vector until there is nothing left.
+ * All pending vectors will be processed from input -> output.
+ */
+ for (i = 0; i < _vec_len (nm->pending_frames); i++)
+ cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
+ /* Reset pending vector for next iteration. */
+
+The pending frame node_runtime_index associates the frame with the node
+which will process it.
+
+Complications
+-------------
+
+Fasten your seatbelt. Here’s where the story - and the data structures -
+become quite complicated…
+
+At 100,000 feet: vpp uses a directed graph, not a directed *acyclic*
+graph. It’s really quite normal for a packet to visit ip[46]-lookup
+multiple times. The worst-case: a graph node which enqueues packets to
+itself.
+
+To deal with this issue, the graph dispatcher must force allocation of a
+new frame if the current graph node’s dispatch function happens to
+enqueue a packet back to itself.
+
+There are no guarantees that a pending frame will be processed
+immediately, which means that more packets may be added to the
+underlying vlib_frame_t after it has been attached to a
+vlib_pending_frame_t. Care must be taken to allocate new frames and
+pending frames if a (pending_frame, frame) pair fills.
+
+Next frames, next frame ownership
+---------------------------------
+
+The vlib_next_frame_t is the last key graph dispatcher data structure:
+
+.. code:: c
+
+ typedef struct
+ {
+ /* Frame index. */
+ u32 frame_index;
+
+ /* Node runtime for this next. */
+ u32 node_runtime_index;
+
+ /* Next frame flags. */
+ u32 flags;
+
+ /* Reflects node frame-used flag for this next. */
+ #define VLIB_FRAME_NO_FREE_AFTER_DISPATCH \
+ VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
+
+ /* This next frame owns enqueue to node
+ corresponding to node_runtime_index. */
+ #define VLIB_FRAME_OWNER (1 << 15)
+
+ /* Set when frame has been allocated for this next. */
+ #define VLIB_FRAME_IS_ALLOCATED VLIB_NODE_FLAG_IS_OUTPUT
+
+ /* Set when frame has been added to pending vector. */
+ #define VLIB_FRAME_PENDING VLIB_NODE_FLAG_IS_DROP
+
+ /* Set when frame is to be freed after dispatch. */
+ #define VLIB_FRAME_FREE_AFTER_DISPATCH VLIB_NODE_FLAG_IS_PUNT
+
+ /* Set when frame has traced packets. */
+ #define VLIB_FRAME_TRACE VLIB_NODE_FLAG_TRACE
+
+ /* Number of vectors enqueue to this next since last overflow. */
+ u32 vectors_since_last_overflow;
+ } vlib_next_frame_t;
+
+Graph node dispatch functions call vlib_get_next_frame (…) to set “(u32
+\*)to_next” to the right place in the vlib_frame_t corresponding to the
+ith arc (aka next0) from the current node to the indicated next node.
+
+After some scuffling around - two levels of macros - processing reaches
+vlib_get_next_frame_internal (…). Get-next-frame-internal digs up the
+vlib_next_frame_t corresponding to the desired graph arc.
+
+The next frame data structure amounts to a graph-arc-centric frame
+cache. Once a node finishes adding element to a frame, it will acquire a
+vlib_pending_frame_t and end up on the graph dispatcher’s run-queue. But
+there’s no guarantee that more vector elements won’t be added to the
+underlying frame from the same (source_node, next_index) arc or from a
+different (source_node, next_index) arc.
+
+Maintaining consistency of the arc-to-frame cache is necessary. The
+first step in maintaining consistency is to make sure that only one
+graph node at a time thinks it “owns” the target vlib_frame_t.
+
+Back to the graph node dispatch function. In the usual case, a certain
+number of packets will be added to the vlib_frame_t acquired by calling
+vlib_get_next_frame (…).
+
+Before a dispatch function returns, it’s required to call
+vlib_put_next_frame (…) for all of the graph arcs it actually used. This
+action adds a vlib_pending_frame_t to the graph dispatcher’s pending
+frame vector.
+
+Vlib_put_next_frame makes a note in the pending frame of the frame
+index, and also of the vlib_next_frame_t index.
+
+dispatch_pending_node actions
+-----------------------------
+
+The main graph dispatch loop calls dispatch pending node as shown above.
+
+Dispatch_pending_node recovers the pending frame, and the graph node
+runtime / dispatch function. Further, it recovers the next_frame
+currently associated with the vlib_frame_t, and detaches the
+vlib_frame_t from the next_frame.
+
+In …/src/vlib/main.c:dispatch_pending_node(…), note this stanza:
+
+.. code:: c
+
+ /* Force allocation of new frame while current frame is being
+ dispatched. */
+ restore_frame_index = ~0;
+ if (nf->frame_index == p->frame_index)
+ {
+ nf->frame_index = ~0;
+ nf->flags &= ~VLIB_FRAME_IS_ALLOCATED;
+ if (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH))
+ restore_frame_index = p->frame_index;
+ }
+
+dispatch_pending_node is worth a hard stare due to the several
+second-order optimizations it implements. Almost as an afterthought, it
+calls dispatch_node which actually calls the graph node dispatch
+function.
+
+Process / thread model
+----------------------
+
+vlib provides an ultra-lightweight cooperative multi-tasking thread
+model. The graph node scheduler invokes these processes in much the same
+way as traditional vector-processing run-to-completion graph nodes;
+plus-or-minus a setjmp/longjmp pair required to switch stacks. Simply
+set the vlib_node_registration_t type field to vlib_NODE_TYPE_PROCESS.
+Yes, process is a misnomer. These are cooperative multi-tasking threads.
+
+As of this writing, the default stack size is 2<<15 = 32kb. Initialize
+the node registration’s process_log2_n_stack_bytes member as needed. The
+graph node dispatcher makes some effort to detect stack overrun, e.g. by
+mapping a no-access page below each thread stack.
+
+Process node dispatch functions are expected to be “while(1) { }” loops
+which suspend when not otherwise occupied, and which must not run for
+unreasonably long periods of time.
+
+“Unreasonably long” is an application-dependent concept. Over the years,
+we have constructed frame-size sensitive control-plane nodes which will
+use a much higher fraction of the available CPU bandwidth when the frame
+size is low. The classic example: modifying forwarding tables. So long
+as the table-builder leaves the forwarding tables in a valid state, one
+can suspend the table builder to avoid dropping packets as a result of
+control-plane activity.
+
+Process nodes can suspend for fixed amounts of time, or until another
+entity signals an event, or both. See the next section for a description
+of the vlib process event mechanism.
+
+When running in vlib process context, one must pay strict attention to
+loop invariant issues. If one walks a data structure and calls a
+function which may suspend, one had best know by construction that it
+cannot change. Often, it’s best to simply make a snapshot copy of a data
+structure, walk the copy at leisure, then free the copy.
+
+Process events
+--------------
+
+The vlib process event mechanism API is extremely lightweight and easy
+to use. Here is a typical example:
+
+.. code:: c
+
+ vlib_main_t *vm = &vlib_global_main;
+ uword event_type, * event_data = 0;
+
+ while (1)
+ {
+ vlib_process_wait_for_event_or_clock (vm, 5.0 /* seconds */);
+
+ event_type = vlib_process_get_events (vm, &event_data);
+
+ switch (event_type) {
+ case EVENT1:
+ handle_event1s (event_data);
+ break;
+
+ case EVENT2:
+ handle_event2s (event_data);
+ break;
+
+ case ~0: /* 5-second idle/periodic */
+ handle_idle ();
+ break;
+
+ default: /* bug! */
+ ASSERT (0);
+ }
+
+ vec_reset_length(event_data);
+ }
+
+In this example, the VLIB process node waits for an event to occur, or
+for 5 seconds to elapse. The code demuxes on the event type, calling the
+appropriate handler function. Each call to vlib_process_get_events
+returns a vector of per-event-type data passed to successive
+vlib_process_signal_event calls; it is a serious error to process only
+event_data[0].
+
+Resetting the event_data vector-length to 0 [instead of calling
+vec_free] means that the event scheme doesn’t burn cycles continuously
+allocating and freeing the event data vector. This is a common vppinfra
+/ vlib coding pattern, well worth using when appropriate.
+
+Signaling an event is easy, for example:
+
+.. code:: c
+
+ vlib_process_signal_event (vm, process_node_index, EVENT1,
+ (uword)arbitrary_event1_data); /* and so forth */
+
+One can either know the process node index by construction - dig it out
+of the appropriate vlib_node_registration_t - or by finding the
+vlib_node_t with vlib_get_node_by_name(…).
+
+Buffers
+-------
+
+vlib buffering solves the usual set of packet-processing problems,
+albeit at high performance. Key in terms of performance: one ordinarily
+allocates / frees N buffers at a time rather than one at a time. Except
+when operating directly on a specific buffer, one deals with buffers by
+index, not by pointer.
+
+Packet-processing frames are u32[] arrays, not vlib_buffer_t[] arrays.
+
+Packets comprise one or more vlib buffers, chained together as required.
+Multiple particle sizes are supported; hardware input nodes simply ask
+for the required size(s). Coalescing support is available. For obvious
+reasons one is discouraged from writing one’s own wild and wacky buffer
+chain traversal code.
+
+vlib buffer headers are allocated immediately prior to the buffer data
+area. In typical packet processing this saves a dependent read wait:
+given a buffer’s address, one can prefetch the buffer header [metadata]
+at the same time as the first cache line of buffer data.
+
+Buffer header metadata (vlib_buffer_t) includes the usual rewrite
+expansion space, a current_data offset, RX and TX interface indices,
+packet trace information, and a opaque areas.
+
+The opaque data is intended to control packet processing in arbitrary
+subgraph-dependent ways. The programmer shoulders responsibility for
+data lifetime analysis, type-checking, etc.
+
+Buffers have reference-counts in support of e.g. multicast replication.
+
+Shared-memory message API
+-------------------------
+
+Local control-plane and application processes interact with the vpp
+dataplane via asynchronous message-passing in shared memory over
+unidirectional queues. The same application APIs are available via
+sockets.
+
+Capturing API traces and replaying them in a simulation environment
+requires a disciplined approach to the problem. This seems like a
+make-work task, but it is not. When something goes wrong in the
+control-plane after 300,000 or 3,000,000 operations, high-speed replay
+of the events leading up to the accident is a huge win.
+
+The shared-memory message API message allocator vl_api_msg_alloc uses a
+particularly cute trick. Since messages are processed in order, we try
+to allocate message buffering from a set of fixed-size, preallocated
+rings. Each ring item has a “busy” bit. Freeing one of the preallocated
+message buffers merely requires the message consumer to clear the busy
+bit. No locking required.
+
+Debug CLI
+---------
+
+Adding debug CLI commands to VLIB applications is very simple.
+
+Here is a complete example:
+
+.. code:: c
+
+ static clib_error_t *
+ show_ip_tuple_match (vlib_main_t * vm,
+ unformat_input_t * input,
+ vlib_cli_command_t * cmd)
+ {
+ vlib_cli_output (vm, "%U\n", format_ip_tuple_match_tables, &routing_main);
+ return 0;
+ }
+
+ static VLIB_CLI_COMMAND (show_ip_tuple_command) =
+ {
+ .path = "show ip tuple match",
+ .short_help = "Show ip 5-tuple match-and-broadcast tables",
+ .function = show_ip_tuple_match,
+ };
+
+This example implements the “show ip tuple match” debug cli command. In
+ordinary usage, the vlib cli is available via the “vppctl” application,
+which sends traffic to a named pipe. One can configure debug CLI telnet
+access on a configurable port.
+
+The cli implementation has an output redirection facility which makes it
+simple to deliver cli output via shared-memory API messaging,
+
+Particularly for debug or “show tech support” type commands, it would be
+wasteful to write vlib application code to pack binary data, write more
+code elsewhere to unpack the data and finally print the answer. If a
+certain cli command has the potential to hurt packet processing
+performance by running for too long, do the work incrementally in a
+process node. The client can wait.
+
+Macro expansion
+~~~~~~~~~~~~~~~
+
+The vpp debug CLI engine includes a recursive macro expander. This is
+quite useful for factoring out address and/or interface name specifics:
+
+::
+
+ define ip1 192.168.1.1/24
+ define ip2 192.168.2.1/24
+ define iface1 GigabitEthernet3/0/0
+ define iface2 loop1
+
+ set int ip address $iface1 $ip1
+ set int ip address $iface2 $(ip2)
+
+ undefine ip1
+ undefine ip2
+ undefine iface1
+ undefine iface2
+
+Each socket (or telnet) debug CLI session has its own macro tables. All
+debug CLI sessions which use CLI_INBAND binary API messages share a
+single table.
+
+The macro expander recognizes circular definitions:
+
+::
+
+ define foo \$(bar)
+ define bar \$(mumble)
+ define mumble \$(foo)
+
+At 8 levels of recursion, the macro expander throws up its hands and
+replies “CIRCULAR.”
+
+Macro-related debug CLI commands
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In addition to the “define” and “undefine” debug CLI commands, use “show
+macro [noevaluate]” to dump the macro table. The “echo” debug CLI
+command will evaluate and print its argument:
+
+::
+
+ vpp# define foo This\ Is\ Foo
+ vpp# echo $foo
+ This Is Foo
+
+Handing off buffers between threads
+-----------------------------------
+
+Vlib includes an easy-to-use mechanism for handing off buffers between
+worker threads. A typical use-case: software ingress flow hashing. At a
+high level, one creates a per-worker-thread queue which sends packets to
+a specific graph node in the indicated worker thread. With the queue in
+hand, enqueue packets to the worker thread of your choice.
+
+Initialize a handoff queue
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Simple enough, call vlib_frame_queue_main_init:
+
+.. code:: c
+
+ main_ptr->frame_queue_index
+ = vlib_frame_queue_main_init (dest_node.index, frame_queue_size);
+
+Frame_queue_size means what it says: the number of frames which may be
+queued. Since frames contain 1…256 packets, frame_queue_size should be a
+reasonably small number (32…64). If the frame queue producer(s) are
+faster than the frame queue consumer(s), congestion will occur. Suggest
+letting the enqueue operator deal with queue congestion, as shown in the
+enqueue example below.
+
+Under the floorboards, vlib_frame_queue_main_init creates an input queue
+for each worker thread.
+
+Please do NOT create frame queues until it’s clear that they will be
+used. Although the main dispatch loop is reasonably smart about how
+often it polls the (entire set of) frame queues, polling unused frame
+queues is a waste of clock cycles.
+
+Hand off packets
+~~~~~~~~~~~~~~~~
+
+The actual handoff mechanics are simple, and integrate nicely with a
+typical graph-node dispatch function:
+
+.. code:: c
+
+ always_inline uword
+ do_handoff_inline (vlib_main_t * vm,
+ vlib_node_runtime_t * node, vlib_frame_t * frame,
+ int is_ip4, int is_trace)
+ {
+ u32 n_left_from, *from;
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
+ u16 thread_indices [VLIB_FRAME_SIZE];
+ u16 nexts[VLIB_FRAME_SIZE], *next;
+ u32 n_enq;
+ htest_main_t *hmp = &htest_main;
+ int i;
+
+ from = vlib_frame_vector_args (frame);
+ n_left_from = frame->n_vectors;
+
+ vlib_get_buffers (vm, from, bufs, n_left_from);
+ next = nexts;
+ b = bufs;
+
+ /*
+ * Typical frame traversal loop, details vary with
+ * use case. Make sure to set thread_indices[i] with
+ * the desired destination thread index. You may
+ * or may not bother to set next[i].
+ */
+
+ for (i = 0; i < frame->n_vectors; i++)
+ {
+ <snip>
+ /* Pick a thread to handle this packet */
+ thread_indices[i] = f (packet_data_or_whatever);
+ <snip>
+
+ b += 1;
+ next += 1;
+ n_left_from -= 1;
+ }
+
+ /* Enqueue buffers to threads */
+ n_enq =
+ vlib_buffer_enqueue_to_thread (vm, node, hmp->frame_queue_index,
+ from, thread_indices, frame->n_vectors,
+ 1 /* drop on congestion */);
+ /* Typical counters,
+ if (n_enq < frame->n_vectors)
+ vlib_node_increment_counter (vm, node->node_index,
+ XXX_ERROR_CONGESTION_DROP,
+ frame->n_vectors - n_enq);
+ vlib_node_increment_counter (vm, node->node_index,
+ XXX_ERROR_HANDED_OFF, n_enq);
+ return frame->n_vectors;
+ }
+
+Notes about calling vlib_buffer_enqueue_to_thread(…):
+
+- If you pass “drop on congestion” non-zero, all packets in the inbound
+ frame will be consumed one way or the other. This is the recommended
+ setting.
+
+- In the drop-on-congestion case, please don’t try to “help” in the
+ enqueue node by freeing dropped packets, or by pushing them to
+ “error-drop.” Either of those actions would be a severe error.
+
+- It’s perfectly OK to enqueue packets to the current thread.
+
+Handoff Demo Plugin
+-------------------
+
+Check out the sample (plugin) example in …/src/examples/handoffdemo. If
+you want to build the handoff demo plugin:
+
+::
+
+ $ cd .../src/plugins
+ $ ln -s ../examples/handoffdemo
+
+This plugin provides a simple example of how to hand off packets between
+threads. We used it to debug packet-tracer handoff tracing support.
+
+Packet generator input script
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ packet-generator new {
+ name x
+ limit 5
+ size 128-128
+ interface local0
+ node handoffdemo-1
+ data {
+ incrementing 30
+ }
+ }
+
+Start vpp with 2 worker threads
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The demo plugin hands packets from worker 1 to worker 2.
+
+Enable tracing, and start the packet generator
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ trace add pg-input 100
+ packet-generator enable
+
+Sample Run
+~~~~~~~~~~
+
+::
+
+ DBGvpp# ex /tmp/pg_input_script
+ DBGvpp# pa en
+ DBGvpp# sh err
+ Count Node Reason
+ 5 handoffdemo-1 packets handed off processed
+ 5 handoffdemo-2 completed packets
+ DBGvpp# show run
+ Thread 1 vpp_wk_0 (lcore 0)
+ Time 133.9, average vectors/node 5.00, last 128 main loops 0.00 per node 0.00
+ vector rates in 3.7331e-2, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ handoffdemo-1 active 1 5 0 4.76e3 5.00
+ pg-input disabled 2 5 0 5.58e4 2.50
+ unix-epoll-input polling 22760 0 0 2.14e7 0.00
+ ---------------
+ Thread 2 vpp_wk_1 (lcore 2)
+ Time 133.9, average vectors/node 5.00, last 128 main loops 0.00 per node 0.00
+ vector rates in 0.0000e0, out 0.0000e0, drop 3.7331e-2, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ drop active 1 5 0 1.35e4 5.00
+ error-drop active 1 5 0 2.52e4 5.00
+ handoffdemo-2 active 1 5 0 2.56e4 5.00
+ unix-epoll-input polling 22406 0 0 2.18e7 0.00
+
+Enable the packet tracer and run it again…
+
+::
+
+ DBGvpp# trace add pg-input 100
+ DBGvpp# pa en
+ DBGvpp# sh trace
+ sh trace
+ ------------------- Start of thread 0 vpp_main -------------------
+ No packets in trace buffer
+ ------------------- Start of thread 1 vpp_wk_0 -------------------
+ Packet 1
+
+ 00:06:50:520688: pg-input
+ stream x, 128 bytes, 0 sw_if_index
+ current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000000
+ 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000
+ 00000020: 0000000000000000000000000000000000000000000000000000000000000000
+ 00000040: 0000000000000000000000000000000000000000000000000000000000000000
+ 00000060: 0000000000000000000000000000000000000000000000000000000000000000
+ 00:06:50:520762: handoffdemo-1
+ HANDOFFDEMO: current thread 1
+
+ Packet 2
+
+ 00:06:50:520688: pg-input
+ stream x, 128 bytes, 0 sw_if_index
+ current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000001
+ 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000
+ 00000020: 0000000000000000000000000000000000000000000000000000000000000000
+ 00000040: 0000000000000000000000000000000000000000000000000000000000000000
+ 00000060: 0000000000000000000000000000000000000000000000000000000000000000
+ 00:06:50:520762: handoffdemo-1
+ HANDOFFDEMO: current thread 1
+
+ Packet 3
+
+ 00:06:50:520688: pg-input
+ stream x, 128 bytes, 0 sw_if_index
+ current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000002
+ 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000
+ 00000020: 0000000000000000000000000000000000000000000000000000000000000000
+ 00000040: 0000000000000000000000000000000000000000000000000000000000000000
+ 00000060: 0000000000000000000000000000000000000000000000000000000000000000
+ 00:06:50:520762: handoffdemo-1
+ HANDOFFDEMO: current thread 1
+
+ Packet 4
+
+ 00:06:50:520688: pg-input
+ stream x, 128 bytes, 0 sw_if_index
+ current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000003
+ 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000
+ 00000020: 0000000000000000000000000000000000000000000000000000000000000000
+ 00000040: 0000000000000000000000000000000000000000000000000000000000000000
+ 00000060: 0000000000000000000000000000000000000000000000000000000000000000
+ 00:06:50:520762: handoffdemo-1
+ HANDOFFDEMO: current thread 1
+
+ Packet 5
+
+ 00:06:50:520688: pg-input
+ stream x, 128 bytes, 0 sw_if_index
+ current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000004
+ 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000
+ 00000020: 0000000000000000000000000000000000000000000000000000000000000000
+ 00000040: 0000000000000000000000000000000000000000000000000000000000000000
+ 00000060: 0000000000000000000000000000000000000000000000000000000000000000
+ 00:06:50:520762: handoffdemo-1
+ HANDOFFDEMO: current thread 1
+
+ ------------------- Start of thread 2 vpp_wk_1 -------------------
+ Packet 1
+
+ 00:06:50:520796: handoff_trace
+ HANDED-OFF: from thread 1 trace index 0
+ 00:06:50:520796: handoffdemo-2
+ HANDOFFDEMO: current thread 2
+ 00:06:50:520867: error-drop
+ rx:local0
+ 00:06:50:520914: drop
+ handoffdemo-2: completed packets
+
+ Packet 2
+
+ 00:06:50:520796: handoff_trace
+ HANDED-OFF: from thread 1 trace index 1
+ 00:06:50:520796: handoffdemo-2
+ HANDOFFDEMO: current thread 2
+ 00:06:50:520867: error-drop
+ rx:local0
+ 00:06:50:520914: drop
+ handoffdemo-2: completed packets
+
+ Packet 3
+
+ 00:06:50:520796: handoff_trace
+ HANDED-OFF: from thread 1 trace index 2
+ 00:06:50:520796: handoffdemo-2
+ HANDOFFDEMO: current thread 2
+ 00:06:50:520867: error-drop
+ rx:local0
+ 00:06:50:520914: drop
+ handoffdemo-2: completed packets
+
+ Packet 4
+
+ 00:06:50:520796: handoff_trace
+ HANDED-OFF: from thread 1 trace index 3
+ 00:06:50:520796: handoffdemo-2
+ HANDOFFDEMO: current thread 2
+ 00:06:50:520867: error-drop
+ rx:local0
+ 00:06:50:520914: drop
+ handoffdemo-2: completed packets
+
+ Packet 5
+
+ 00:06:50:520796: handoff_trace
+ HANDED-OFF: from thread 1 trace index 4
+ 00:06:50:520796: handoffdemo-2
+ HANDOFFDEMO: current thread 2
+ 00:06:50:520867: error-drop
+ rx:local0
+ 00:06:50:520914: drop
+ handoffdemo-2: completed packets
+ DBGvpp#
diff --git a/docs/developer/corearchitecture/vnet.rst b/docs/developer/corearchitecture/vnet.rst
new file mode 100644
index 0000000..812e2fb
--- /dev/null
+++ b/docs/developer/corearchitecture/vnet.rst
@@ -0,0 +1,807 @@
+VNET (VPP Network Stack)
+========================
+
+The files associated with the VPP network stack layer are located in the
+*./src/vnet* folder. The Network Stack Layer is basically an
+instantiation of the code in the other layers. This layer has a vnet
+library that provides vectorized layer-2 and 3 networking graph nodes, a
+packet generator, and a packet tracer.
+
+In terms of building a packet processing application, vnet provides a
+platform-independent subgraph to which one connects a couple of
+device-driver nodes.
+
+Typical RX connections include “ethernet-input” [full software
+classification, feeds ipv4-input, ipv6-input, arp-input etc.] and
+“ipv4-input-no-checksum” [if hardware can classify, perform ipv4 header
+checksum].
+
+Effective graph dispatch function coding
+----------------------------------------
+
+Over the 15 years, multiple coding styles have emerged: a
+single/dual/quad loop coding model (with variations) and a
+fully-pipelined coding model.
+
+Single/dual loops
+-----------------
+
+The single/dual/quad loop model variations conveniently solve problems
+where the number of items to process is not known in advance: typical
+hardware RX-ring processing. This coding style is also very effective
+when a given node will not need to cover a complex set of dependent
+reads.
+
+Here is an quad/single loop which can leverage up-to-avx512 SIMD vector
+units to convert buffer indices to buffer pointers:
+
+.. code:: c
+
+ static uword
+ simulated_ethernet_interface_tx (vlib_main_t * vm,
+ vlib_node_runtime_t *
+ node, vlib_frame_t * frame)
+ {
+ u32 n_left_from, *from;
+ u32 next_index = 0;
+ u32 n_bytes;
+ u32 thread_index = vm->thread_index;
+ vnet_main_t *vnm = vnet_get_main ();
+ vnet_interface_main_t *im = &vnm->interface_main;
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
+ u16 nexts[VLIB_FRAME_SIZE], *next;
+
+ n_left_from = frame->n_vectors;
+ from = vlib_frame_vector_args (frame);
+
+ /*
+ * Convert up to VLIB_FRAME_SIZE indices in "from" to
+ * buffer pointers in bufs[]
+ */
+ vlib_get_buffers (vm, from, bufs, n_left_from);
+ b = bufs;
+ next = nexts;
+
+ /*
+ * While we have at least 4 vector elements (pkts) to process..
+ */
+ while (n_left_from >= 4)
+ {
+ /* Prefetch next quad-loop iteration. */
+ if (PREDICT_TRUE (n_left_from >= 8))
+ {
+ vlib_prefetch_buffer_header (b[4], STORE);
+ vlib_prefetch_buffer_header (b[5], STORE);
+ vlib_prefetch_buffer_header (b[6], STORE);
+ vlib_prefetch_buffer_header (b[7], STORE);
+ }
+
+ /*
+ * $$$ Process 4x packets right here...
+ * set next[0..3] to send the packets where they need to go
+ */
+
+ do_something_to (b[0]);
+ do_something_to (b[1]);
+ do_something_to (b[2]);
+ do_something_to (b[3]);
+
+ /* Process the next 0..4 packets */
+ b += 4;
+ next += 4;
+ n_left_from -= 4;
+ }
+ /*
+ * Clean up 0...3 remaining packets at the end of the incoming frame
+ */
+ while (n_left_from > 0)
+ {
+ /*
+ * $$$ Process one packet right here...
+ * set next[0..3] to send the packets where they need to go
+ */
+ do_something_to (b[0]);
+
+ /* Process the next packet */
+ b += 1;
+ next += 1;
+ n_left_from -= 1;
+ }
+
+ /*
+ * Send the packets along their respective next-node graph arcs
+ * Considerable locality of reference is expected, most if not all
+ * packets in the inbound vector will traverse the same next-node
+ * arc
+ */
+ vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
+
+ return frame->n_vectors;
+ }
+
+Given a packet processing task to implement, it pays to scout around
+looking for similar tasks, and think about using the same coding
+pattern. It is not uncommon to recode a given graph node dispatch
+function several times during performance optimization.
+
+Creating Packets from Scratch
+-----------------------------
+
+At times, it’s necessary to create packets from scratch and send them.
+Tasks like sending keepalives or actively opening connections come to
+mind. Its not difficult, but accurate buffer metadata setup is required.
+
+Allocating Buffers
+~~~~~~~~~~~~~~~~~~
+
+Use vlib_buffer_alloc, which allocates a set of buffer indices. For
+low-performance applications, it’s OK to allocate one buffer at a time.
+Note that vlib_buffer_alloc(…) does NOT initialize buffer metadata. See
+below.
+
+In high-performance cases, allocate a vector of buffer indices, and hand
+them out from the end of the vector; decrement \_vec_len(..) as buffer
+indices are allocated. See tcp_alloc_tx_buffers(…) and
+tcp_get_free_buffer_index(…) for an example.
+
+Buffer Initialization Example
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following example shows the **main points**, but is not to be
+blindly cut-’n-pasted.
+
+.. code:: c
+
+ u32 bi0;
+ vlib_buffer_t *b0;
+ ip4_header_t *ip;
+ udp_header_t *udp;
+
+ /* Allocate a buffer */
+ if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
+ return -1;
+
+ b0 = vlib_get_buffer (vm, bi0);
+
+ /* At this point b0->current_data = 0, b0->current_length = 0 */
+
+ /*
+ * Copy data into the buffer. This example ASSUMES that data will fit
+ * in a single buffer, and is e.g. an ip4 packet.
+ */
+ if (have_packet_rewrite)
+ {
+ clib_memcpy (b0->data, data, vec_len (data));
+ b0->current_length = vec_len (data);
+ }
+ else
+ {
+ /* OR, build a udp-ip packet (for example) */
+ ip = vlib_buffer_get_current (b0);
+ udp = (udp_header_t *) (ip + 1);
+ data_dst = (u8 *) (udp + 1);
+
+ ip->ip_version_and_header_length = 0x45;
+ ip->ttl = 254;
+ ip->protocol = IP_PROTOCOL_UDP;
+ ip->length = clib_host_to_net_u16 (sizeof (*ip) + sizeof (*udp) +
+ vec_len(udp_data));
+ ip->src_address.as_u32 = src_address->as_u32;
+ ip->dst_address.as_u32 = dst_address->as_u32;
+ udp->src_port = clib_host_to_net_u16 (src_port);
+ udp->dst_port = clib_host_to_net_u16 (dst_port);
+ udp->length = clib_host_to_net_u16 (vec_len (udp_data));
+ clib_memcpy (data_dst, udp_data, vec_len(udp_data));
+
+ if (compute_udp_checksum)
+ {
+ /* RFC 7011 section 10.3.2. */
+ udp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip);
+ if (udp->checksum == 0)
+ udp->checksum = 0xffff;
+ }
+ b0->current_length = vec_len (sizeof (*ip) + sizeof (*udp) +
+ vec_len (udp_data));
+
+ }
+ b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
+
+ /* sw_if_index 0 is the "local" interface, which always exists */
+ vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
+
+ /* Use the default FIB index for tx lookup. Set non-zero to use another fib */
+ vnet_buffer (b0)->sw_if_index[VLIB_TX] = 0;
+
+If your use-case calls for large packet transmission, use
+vlib_buffer_chain_append_data_with_alloc(…) to create the requisite
+buffer chain.
+
+Enqueueing packets for lookup and transmission
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The simplest way to send a set of packets is to use
+vlib_get_frame_to_node(…) to allocate fresh frame(s) to ip4_lookup_node
+or ip6_lookup_node, add the constructed buffer indices, and dispatch the
+frame using vlib_put_frame_to_node(…).
+
+.. code:: c
+
+ vlib_frame_t *f;
+ f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
+ f->n_vectors = vec_len(buffer_indices_to_send);
+ to_next = vlib_frame_vector_args (f);
+
+ for (i = 0; i < vec_len (buffer_indices_to_send); i++)
+ to_next[i] = buffer_indices_to_send[i];
+
+ vlib_put_frame_to_node (vm, ip4_lookup_node_index, f);
+
+It is inefficient to allocate and schedule single packet frames. That’s
+typical in case you need to send one packet per second, but should
+**not** occur in a for-loop!
+
+Packet tracer
+-------------
+
+Vlib includes a frame element [packet] trace facility, with a simple
+debug CLI interface. The cli is straightforward: “trace add
+input-node-name count” to start capturing packet traces.
+
+To trace 100 packets on a typical x86_64 system running the dpdk plugin:
+“trace add dpdk-input 100”. When using the packet generator: “trace add
+pg-input 100”
+
+To display the packet trace: “show trace”
+
+Each graph node has the opportunity to capture its own trace data. It is
+almost always a good idea to do so. The trace capture APIs are simple.
+
+The packet capture APIs snapshoot binary data, to minimize processing at
+capture time. Each participating graph node initialization provides a
+vppinfra format-style user function to pretty-print data when required
+by the VLIB “show trace” command.
+
+Set the VLIB node registration “.format_trace” member to the name of the
+per-graph node format function.
+
+Here’s a simple example:
+
+.. code:: c
+
+ u8 * my_node_format_trace (u8 * s, va_list * args)
+ {
+ vlib_main_t * vm = va_arg (*args, vlib_main_t *);
+ vlib_node_t * node = va_arg (*args, vlib_node_t *);
+ my_node_trace_t * t = va_arg (*args, my_trace_t *);
+
+ s = format (s, "My trace data was: %d", t-><whatever>);
+
+ return s;
+ }
+
+The trace framework hands the per-node format function the data it
+captured as the packet whizzed by. The format function pretty-prints the
+data as desired.
+
+Graph Dispatcher Pcap Tracing
+-----------------------------
+
+The vpp graph dispatcher knows how to capture vectors of packets in pcap
+format as they’re dispatched. The pcap captures are as follows:
+
+::
+
+ VPP graph dispatch trace record description:
+
+ 0 1 2 3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Major Version | Minor Version | NStrings | ProtoHint |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Buffer index (big endian) |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ + VPP graph node name ... ... | NULL octet |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Buffer Metadata ... ... | NULL octet |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Buffer Opaque ... ... | NULL octet |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Buffer Opaque 2 ... ... | NULL octet |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | VPP ASCII packet trace (if NStrings > 4) | NULL octet |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | Packet data (up to 16K) |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+Graph dispatch records comprise a version stamp, an indication of how
+many NULL-terminated strings will follow the record header and preceed
+packet data, and a protocol hint.
+
+The buffer index is an opaque 32-bit cookie which allows consumers of
+these data to easily filter/track single packets as they traverse the
+forwarding graph.
+
+Multiple records per packet are normal, and to be expected. Packets will
+appear multiple times as they traverse the vpp forwarding graph. In this
+way, vpp graph dispatch traces are significantly different from regular
+network packet captures from an end-station. This property complicates
+stateful packet analysis.
+
+Restricting stateful analysis to records from a single vpp graph node
+such as “ethernet-input” seems likely to improve the situation.
+
+As of this writing: major version = 1, minor version = 0. Nstrings
+SHOULD be 4 or 5. Consumers SHOULD be wary values less than 4 or greater
+than 5. They MAY attempt to display the claimed number of strings, or
+they MAY treat the condition as an error.
+
+Here is the current set of protocol hints:
+
+.. code:: c
+
+ typedef enum
+ {
+ VLIB_NODE_PROTO_HINT_NONE = 0,
+ VLIB_NODE_PROTO_HINT_ETHERNET,
+ VLIB_NODE_PROTO_HINT_IP4,
+ VLIB_NODE_PROTO_HINT_IP6,
+ VLIB_NODE_PROTO_HINT_TCP,
+ VLIB_NODE_PROTO_HINT_UDP,
+ VLIB_NODE_N_PROTO_HINTS,
+ } vlib_node_proto_hint_t;
+
+Example: VLIB_NODE_PROTO_HINT_IP6 means that the first octet of packet
+data SHOULD be 0x60, and should begin an ipv6 packet header.
+
+Downstream consumers of these data SHOULD pay attention to the protocol
+hint. They MUST tolerate inaccurate hints, which MAY occur from time to
+time.
+
+Dispatch Pcap Trace Debug CLI
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To start a dispatch trace capture of up to 10,000 trace records:
+
+::
+
+ pcap dispatch trace on max 10000 file dispatch.pcap
+
+To start a dispatch trace which will also include standard vpp packet
+tracing for packets which originate in dpdk-input:
+
+::
+
+ pcap dispatch trace on max 10000 file dispatch.pcap buffer-trace dpdk-input 1000
+
+To save the pcap trace, e.g. in /tmp/dispatch.pcap:
+
+::
+
+ pcap dispatch trace off
+
+Wireshark dissection of dispatch pcap traces
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It almost goes without saying that we built a companion wireshark
+dissector to display these traces. As of this writing, we have
+upstreamed the wireshark dissector.
+
+Since it will be a while before wireshark/master/latest makes it into
+all of the popular Linux distros, please see the “How to build a vpp
+dispatch trace aware Wireshark” page for build info.
+
+Here is a sample packet dissection, with some fields omitted for
+clarity. The point is that the wireshark dissector accurately displays
+**all** of the vpp buffer metadata, and the name of the graph node in
+question.
+
+::
+
+ Frame 1: 2216 bytes on wire (17728 bits), 2216 bytes captured (17728 bits)
+ Encapsulation type: USER 13 (58)
+ [Protocols in frame: vpp:vpp-metadata:vpp-opaque:vpp-opaque2:eth:ethertype:ip:tcp:data]
+ VPP Dispatch Trace
+ BufferIndex: 0x00036663
+ NodeName: ethernet-input
+ VPP Buffer Metadata
+ Metadata: flags:
+ Metadata: current_data: 0, current_length: 102
+ Metadata: current_config_index: 0, flow_id: 0, next_buffer: 0
+ Metadata: error: 0, n_add_refs: 0, buffer_pool_index: 0
+ Metadata: trace_index: 0, recycle_count: 0, len_not_first_buf: 0
+ Metadata: free_list_index: 0
+ Metadata:
+ VPP Buffer Opaque
+ Opaque: raw: 00000007 ffffffff 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
+ Opaque: sw_if_index[VLIB_RX]: 7, sw_if_index[VLIB_TX]: -1
+ Opaque: L2 offset 0, L3 offset 0, L4 offset 0, feature arc index 0
+ Opaque: ip.adj_index[VLIB_RX]: 0, ip.adj_index[VLIB_TX]: 0
+ Opaque: ip.flow_hash: 0x0, ip.save_protocol: 0x0, ip.fib_index: 0
+ Opaque: ip.save_rewrite_length: 0, ip.rpf_id: 0
+ Opaque: ip.icmp.type: 0 ip.icmp.code: 0, ip.icmp.data: 0x0
+ Opaque: ip.reass.next_index: 0, ip.reass.estimated_mtu: 0
+ Opaque: ip.reass.fragment_first: 0 ip.reass.fragment_last: 0
+ Opaque: ip.reass.range_first: 0 ip.reass.range_last: 0
+ Opaque: ip.reass.next_range_bi: 0x0, ip.reass.ip6_frag_hdr_offset: 0
+ Opaque: mpls.ttl: 0, mpls.exp: 0, mpls.first: 0, mpls.save_rewrite_length: 0, mpls.bier.n_bytes: 0
+ Opaque: l2.feature_bitmap: 00000000, l2.bd_index: 0, l2.l2_len: 0, l2.shg: 0, l2.l2fib_sn: 0, l2.bd_age: 0
+ Opaque: l2.feature_bitmap_input: none configured, L2.feature_bitmap_output: none configured
+ Opaque: l2t.next_index: 0, l2t.session_index: 0
+ Opaque: l2_classify.table_index: 0, l2_classify.opaque_index: 0, l2_classify.hash: 0x0
+ Opaque: policer.index: 0
+ Opaque: ipsec.flags: 0x0, ipsec.sad_index: 0
+ Opaque: map.mtu: 0
+ Opaque: map_t.v6.saddr: 0x0, map_t.v6.daddr: 0x0, map_t.v6.frag_offset: 0, map_t.v6.l4_offset: 0
+ Opaque: map_t.v6.l4_protocol: 0, map_t.checksum_offset: 0, map_t.mtu: 0
+ Opaque: ip_frag.mtu: 0, ip_frag.next_index: 0, ip_frag.flags: 0x0
+ Opaque: cop.current_config_index: 0
+ Opaque: lisp.overlay_afi: 0
+ Opaque: tcp.connection_index: 0, tcp.seq_number: 0, tcp.seq_end: 0, tcp.ack_number: 0, tcp.hdr_offset: 0, tcp.data_offset: 0
+ Opaque: tcp.data_len: 0, tcp.flags: 0x0
+ Opaque: sctp.connection_index: 0, sctp.sid: 0, sctp.ssn: 0, sctp.tsn: 0, sctp.hdr_offset: 0
+ Opaque: sctp.data_offset: 0, sctp.data_len: 0, sctp.subconn_idx: 0, sctp.flags: 0x0
+ Opaque: snat.flags: 0x0
+ Opaque:
+ VPP Buffer Opaque2
+ Opaque2: raw: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
+ Opaque2: qos.bits: 0, qos.source: 0
+ Opaque2: loop_counter: 0
+ Opaque2: gbp.flags: 0, gbp.src_epg: 0
+ Opaque2: pg_replay_timestamp: 0
+ Opaque2:
+ Ethernet II, Src: 06:d6:01:41:3b:92 (06:d6:01:41:3b:92), Dst: IntelCor_3d:f6 Transmission Control Protocol, Src Port: 22432, Dst Port: 54084, Seq: 1, Ack: 1, Len: 36
+ Source Port: 22432
+ Destination Port: 54084
+ TCP payload (36 bytes)
+ Data (36 bytes)
+
+ 0000 cf aa 8b f5 53 14 d4 c7 29 75 3e 56 63 93 9d 11 ....S...)u>Vc...
+ 0010 e5 f2 92 27 86 56 4c 21 ce c5 23 46 d7 eb ec 0d ...'.VL!..#F....
+ 0020 a8 98 36 5a ..6Z
+ Data: cfaa8bf55314d4c729753e5663939d11e5f2922786564c21…
+ [Length: 36]
+
+It’s a matter of a couple of mouse-clicks in Wireshark to filter the
+trace to a specific buffer index. With that specific kind of filtration,
+one can watch a packet walk through the forwarding graph; noting any/all
+metadata changes, header checksum changes, and so forth.
+
+This should be of significant value when developing new vpp graph nodes.
+If new code mispositions b->current_data, it will be completely obvious
+from looking at the dispatch trace in wireshark.
+
+pcap rx, tx, and drop tracing
+-----------------------------
+
+vpp also supports rx, tx, and drop packet capture in pcap format,
+through the “pcap trace” debug CLI command.
+
+This command is used to start or stop a packet capture, or show the
+status of packet capture. Each of “pcap trace rx”, “pcap trace tx”, and
+“pcap trace drop” is implemented. Supply one or more of “rx”, “tx”, and
+“drop” to enable multiple simultaneous capture types.
+
+These commands have the following optional parameters:
+
+- rx - trace received packets.
+
+- tx - trace transmitted packets.
+
+- drop - trace dropped packets.
+
+- max *nnnn*\ - file size, number of packet captures. Once packets
+ have been received, the trace buffer buffer is flushed to the
+ indicated file. Defaults to 1000. Can only be updated if packet
+ capture is off.
+
+- max-bytes-per-pkt *nnnn*\ - maximum number of bytes to trace on a
+ per-packet basis. Must be >32 and less than 9000. Default value:
+
+ 512.
+
+- filter - Use the pcap rx / tx / drop trace filter, which must be
+ configured. Use classify filter pcap… to configure the filter. The
+ filter will only be executed if the per-interface or any-interface
+ tests fail.
+
+- intfc *interface* \| *any*\ - Used to specify a given interface, or
+ use ‘any’ to run packet capture on all interfaces. ‘any’ is the
+ default if not provided. Settings from a previous packet capture are
+ preserved, so ‘any’ can be used to reset the interface setting.
+
+- file *filename*\ - Used to specify the output filename. The file
+ will be placed in the ‘/tmp’ directory. If *filename* already exists,
+ file will be overwritten. If no filename is provided, ‘/tmp/rx.pcap
+ or tx.pcap’ will be used, depending on capture direction. Can only be
+ updated when pcap capture is off.
+
+- status - Displays the current status and configured attributes
+ associated with a packet capture. If packet capture is in progress,
+ ‘status’ also will return the number of packets currently in the
+ buffer. Any additional attributes entered on command line with a
+ ‘status’ request will be ignored.
+
+- filter - Capture packets which match the current packet trace filter
+ set. See next section. Configure the capture filter first.
+
+packet trace capture filtering
+------------------------------
+
+The “classify filter pcap \| \| trace” debug CLI command constructs an
+arbitrary set of packet classifier tables for use with “pcap rx \| tx \|
+drop trace,” and with the vpp packet tracer on a per-interface or
+system-wide basis.
+
+Packets which match a rule in the classifier table chain will be traced.
+The tables are automatically ordered so that matches in the most
+specific table are tried first.
+
+It’s reasonably likely that folks will configure a single table with one
+or two matches. As a result, we configure 8 hash buckets and 128K of
+match rule space by default. One can override the defaults by specifying
+“buckets ” and “memory-size ” as desired.
+
+To build up complex filter chains, repeatedly issue the classify filter
+debug CLI command. Each command must specify the desired mask and match
+values. If a classifier table with a suitable mask already exists, the
+CLI command adds a match rule to the existing table. If not, the CLI
+command add a new table and the indicated mask rule
+
+Configure a simple pcap classify filter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ classify filter pcap mask l3 ip4 src match l3 ip4 src 192.168.1.11
+ pcap trace rx max 100 filter
+
+Configure a simple per-interface capture filter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ classify filter GigabitEthernet3/0/0 mask l3 ip4 src match l3 ip4 src 192.168.1.11"
+ pcap trace rx max 100 intfc GigabitEthernet3/0/0
+
+Note that per-interface capture filters are *always* applied.
+
+Clear per-interface capture filters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ classify filter GigabitEthernet3/0/0 del
+
+Configure another fairly simple pcap classify filter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ classify filter pcap mask l3 ip4 src dst match l3 ip4 src 192.168.1.10 dst 192.168.2.10
+ pcap trace tx max 100 filter
+
+Configure a vpp packet tracer filter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ classify filter trace mask l3 ip4 src dst match l3 ip4 src 192.168.1.10 dst 192.168.2.10
+ trace add dpdk-input 100 filter
+
+Clear all current classifier filters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ classify filter [pcap | <interface> | trace] del
+
+To inspect the classifier tables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ show classify table [verbose]
+
+The verbose form displays all of the match rules, with hit-counters.
+
+Terse description of the “mask ” syntax:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ l2 src dst proto tag1 tag2 ignore-tag1 ignore-tag2 cos1 cos2 dot1q dot1ad
+ l3 ip4 <ip4-mask> ip6 <ip6-mask>
+ <ip4-mask> version hdr_length src[/width] dst[/width]
+ tos length fragment_id ttl protocol checksum
+ <ip6-mask> version traffic-class flow-label src dst proto
+ payload_length hop_limit protocol
+ l4 tcp <tcp-mask> udp <udp_mask> src_port dst_port
+ <tcp-mask> src dst # ports
+ <udp-mask> src_port dst_port
+
+To construct **matches**, add the values to match after the indicated
+keywords in the mask syntax. For example: “… mask l3 ip4 src” -> “…
+match l3 ip4 src 192.168.1.11”
+
+VPP Packet Generator
+--------------------
+
+We use the VPP packet generator to inject packets into the forwarding
+graph. The packet generator can replay pcap traces, and generate packets
+out of whole cloth at respectably high performance.
+
+The VPP pg enables quite a variety of use-cases, ranging from functional
+testing of new data-plane nodes to regression testing to performance
+tuning.
+
+PG setup scripts
+----------------
+
+PG setup scripts describe traffic in detail, and leverage vpp debug CLI
+mechanisms. It’s reasonably unusual to construct a pg setup script which
+doesn’t include a certain amount of interface and FIB configuration.
+
+For example:
+
+::
+
+ loop create
+ set int ip address loop0 192.168.1.1/24
+ set int state loop0 up
+
+ packet-generator new {
+ name pg0
+ limit 100
+ rate 1e6
+ size 300-300
+ interface loop0
+ node ethernet-input
+ data { IP4: 1.2.3 -> 4.5.6
+ UDP: 192.168.1.10 - 192.168.1.254 -> 192.168.2.10
+ UDP: 1234 -> 2345
+ incrementing 286
+ }
+ }
+
+A packet generator stream definition includes two major sections: -
+Stream Parameter Setup - Packet Data
+
+Stream Parameter Setup
+~~~~~~~~~~~~~~~~~~~~~~
+
+Given the example above, let’s look at how to set up stream parameters:
+
+- **name pg0** - Name of the stream, in this case “pg0”
+
+- **limit 1000** - Number of packets to send when the stream is
+ enabled. “limit 0” means send packets continuously.
+
+- **maxframe <nnn>** - Maximum frame size. Handy for injecting multiple
+ frames no larger than <nnn>. Useful for checking dual / quad loop
+ codes
+
+- **rate 1e6** - Packet injection rate, in this case 1 MPPS. When not
+ specified, the packet generator injects packets as fast as possible
+
+- **size 300-300** - Packet size range, in this case send 300-byte
+ packets
+
+- **interface loop0** - Packets appear as if they were received on the
+ specified interface. This datum is used in multiple ways: to select
+ graph arc feature configuration, to select IP FIBs. Configure
+ features e.g. on loop0 to exercise those features.
+
+- **tx-interface <name>** - Packets will be transmitted on the
+ indicated interface. Typically required only when injecting packets
+ into post-IP-rewrite graph nodes.
+
+- **pcap <filename>** - Replay packets from the indicated pcap capture
+ file. “make test” makes extensive use of this feature: generate
+ packets using scapy, save them in a .pcap file, then inject them into
+ the vpp graph via a vpp pg “pcap <filename>” stream definition
+
+- **worker <nn>** - Generate packets for the stream using the indicated
+ vpp worker thread. The vpp pg generates and injects O(10 MPPS /
+ core). Use multiple stream definitions and worker threads to generate
+ and inject enough traffic to easily fill a 40 gbit pipe with small
+ packets.
+
+Data definition
+~~~~~~~~~~~~~~~
+
+Packet generator data definitions make use of a layered implementation
+strategy. Networking layers are specified in order, and the notation can
+seem a bit counter-intuitive. In the example above, the data definition
+stanza constructs a set of L2-L4 headers layers, and uses an
+incrementing fill pattern to round out the requested 300-byte packets.
+
+- **IP4: 1.2.3 -> 4.5.6** - Construct an L2 (MAC) header with the ip4
+ ethertype (0x800), src MAC address of 00:01:00:02:00:03 and dst MAC
+ address of 00:04:00:05:00:06. Mac addresses may be specified in
+ either *xxxx.xxxx.xxxx* format or *xx:xx:xx:xx:xx:xx* format.
+
+- **UDP: 192.168.1.10 - 192.168.1.254 -> 192.168.2.10** - Construct an
+ incrementing set of L3 (IPv4) headers for successive packets with
+ source addresses ranging from .10 to .254. All packets in the stream
+ have a constant dest address of 192.168.2.10. Set the protocol field
+ to 17, UDP.
+
+- **UDP: 1234 -> 2345** - Set the UDP source and destination ports to
+ 1234 and 2345, respectively
+
+- **incrementing 256** - Insert up to 256 incrementing data bytes.
+
+Obvious variations involve “s/IP4/IP6/” in the above, along with
+changing from IPv4 to IPv6 address notation.
+
+The vpp pg can set any / all IPv4 header fields, including tos, packet
+length, mf / df / fragment id and offset, ttl, protocol, checksum, and
+src/dst addresses. Take a look at ../src/vnet/ip/ip[46]_pg.c for
+details.
+
+If all else fails, specify the entire packet data in hex:
+
+- **hex 0xabcd…** - copy hex data verbatim into the packet
+
+When replaying pcap files (“**pcap <filename>**”), do not specify a data
+stanza.
+
+Diagnosing “packet-generator new” parse failures
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you want to inject packets into a brand-new graph node, remember to
+tell the packet generator debug CLI how to parse the packet data stanza.
+
+If the node expects L2 Ethernet MAC headers, specify “.unformat_buffer =
+unformat_ethernet_header”:
+
+.. code:: c
+
+ VLIB_REGISTER_NODE (ethernet_input_node) =
+ {
+ <snip>
+ .unformat_buffer = unformat_ethernet_header,
+ <snip>
+ };
+
+Beyond that, it may be necessary to set breakpoints in
+…/src/vnet/pg/cli.c. Debug image suggested.
+
+When debugging new nodes, it may be far simpler to directly inject
+ethernet frames - and add a corresponding vlib_buffer_advance in the new
+node - than to modify the packet generator.
+
+Debug CLI
+---------
+
+The descriptions above describe the “packet-generator new” debug CLI in
+detail.
+
+Additional debug CLI commands include:
+
+::
+
+ vpp# packet-generator enable [<stream-name>]
+
+which enables the named stream, or all streams.
+
+::
+
+ vpp# packet-generator disable [<stream-name>]
+
+disables the named stream, or all streams.
+
+::
+
+ vpp# packet-generator delete <stream-name>
+
+Deletes the named stream.
+
+::
+
+ vpp# packet-generator configure <stream-name> [limit <nnn>]
+ [rate <f64-pps>] [size <nn>-<nn>]
+
+Changes stream parameters without having to recreate the entire stream
+definition. Note that re-issuing a “packet-generator new” command will
+correctly recreate the named stream.
diff --git a/docs/developer/corefeatures/bfd_doc.rst b/docs/developer/corefeatures/bfd_doc.rst
new file mode 120000
index 0000000..6e9fdd5
--- /dev/null
+++ b/docs/developer/corefeatures/bfd_doc.rst
@@ -0,0 +1 @@
+../../../src/vnet/bfd/bfd_doc.rst
\ No newline at end of file
diff --git a/docs/gettingstarted/developers/eventviewer.rst b/docs/developer/corefeatures/eventviewer.rst
similarity index 96%
rename from docs/gettingstarted/developers/eventviewer.rst
rename to docs/developer/corefeatures/eventviewer.rst
index 59aaef0..21d5fa9 100644
--- a/docs/gettingstarted/developers/eventviewer.rst
+++ b/docs/developer/corefeatures/eventviewer.rst
@@ -96,7 +96,7 @@
will almost certainly produce bad results in worker threads.
G2 graphical event viewer
-==========================
+-------------------------
The G2 graphical event viewer can display serialized vppinfra event logs
directly, or via the c2cpel tool. G2 is a fine-grained event-log viewer. It's
@@ -105,12 +105,12 @@
and also supports the CPEL file format, as described in this section.
Building G2
------------
+~~~~~~~~~~~
This link describes :ref:`how to build G2 <building-g2>`
Setting the Display Preferences
-------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The file $<*HOMEDIR*>/.g2 contains display preferences, which can be overridden.
Simply un-comment one of the stanzas shown below, or experiment as desired.
@@ -142,7 +142,7 @@
*/
Screen Taxonomy
-----------------------------
+~~~~~~~~~~~~~~~
Here is an annotated G2 viewer screenshot, corresponding to activity during BGP
prefix download. This data was captured on a Cisco IOS-XR system:
@@ -159,7 +159,7 @@
Using these tools -- and some patience -- you can understand a given event log.
Mouse Gestures
--------------------------
+~~~~~~~~~~~~~~~
G2 has three fairly sophisticated mouse gesture interfaces, which are worth describing
in detail. First, a left mouse click on a display event pops up a per-event detail box.
@@ -181,7 +181,7 @@
A click on any of the figures will show them at full resolution, right-click will open figures in new tabs,
Time Ruler
-------------------
+~~~~~~~~~~
To use a time ruler, press and hold the right mouse button; drag right or left
until the ruler measures the region of interest. If the time axis scale is coarse,
@@ -192,7 +192,7 @@
:scale: 75%
Event Selection
--------------------------
+~~~~~~~~~~~~~~~
Changing the Event Selector setup controls the set of points displayed in an
obvious way. Here, we suppress all events except "this thread is now running on the CPU":
@@ -210,7 +210,7 @@
above, the "THREAD/THREADY pid:491720 tid:12" detail box appears in this fashion.
Snapshot Ring
------------------------
+~~~~~~~~~~~~~
Three buttons in lower left-hand corner of the g2 main window control the snapshot
ring. Snapshots are simply saved views: maneuver the viewer into an "interesting"
@@ -223,7 +223,7 @@
to save/restore the snapshot ring from CPEL and vppinfra event log files.
Chasing Events
-------------------------
+~~~~~~~~~~~~~~
Event chasing sorts the trace axis by occurrence of the last selected event. For
example, if one selects an event which means "thread running on the CPU" the first
@@ -241,7 +241,7 @@
:scale: 75%
Burying Boring Tracks
------------------------------------
+~~~~~~~~~~~~~~~~~~~~~
The sequence <ctrl><left-mouse-click> moves the track under the mouse to the end
of the set of tracks, effectively burying it. The sequence <shift><left-mouse-click>
@@ -250,7 +250,7 @@
stack to provide precise thread exhumation.
Summary Mode
--------------------------
+~~~~~~~~~~~~
Summary mode declutters the screen by rendering events as short vertical line
segments instead of numbered boxes. Event detail display is unaffected. G2 starts
@@ -260,7 +260,7 @@
to enable boxed numeric event display.
Hotkeys
--------------
+~~~~~~~
G2 supports the following hotkey actions, supposedly (circa 1996) Quake-like
according to the feature's original author:
diff --git a/docs/gettingstarted/developers/fib20/attachedexport.rst b/docs/developer/corefeatures/fib/attachedexport.rst
similarity index 100%
rename from docs/gettingstarted/developers/fib20/attachedexport.rst
rename to docs/developer/corefeatures/fib/attachedexport.rst
diff --git a/docs/gettingstarted/developers/fib20/barnacles.rst b/docs/developer/corefeatures/fib/barnacles.rst
similarity index 96%
rename from docs/gettingstarted/developers/fib20/barnacles.rst
rename to docs/developer/corefeatures/fib/barnacles.rst
index b5b89a3..08e842a 100644
--- a/docs/gettingstarted/developers/fib20/barnacles.rst
+++ b/docs/developer/corefeatures/fib/barnacles.rst
@@ -26,7 +26,7 @@
of [an ordered set of] ABF policies to an interface. The attachment is
consulted on the ingress path of the IP DP (as an input
feature). If the ACL matches then the associated forwarding is
-followed, if not, the packet continues along the DP. Simples.
+followed, if not, the packet continues along the DP. Simple.
Layer 3 Cross Connect
^^^^^^^^^^^^^^^^^^^^^
@@ -34,7 +34,7 @@
An L3 cross-connect (L3XC) matches all packets
that ingress the interface and then forwards using the supplied FIB
path-list. Naturally it runs as an input feature in the IP
-path. Super simples.
+path. Super simple.
IP Punt
^^^^^^^
diff --git a/docs/gettingstarted/developers/fib20/controlplane.rst b/docs/developer/corefeatures/fib/controlplane.rst
similarity index 100%
rename from docs/gettingstarted/developers/fib20/controlplane.rst
rename to docs/developer/corefeatures/fib/controlplane.rst
diff --git a/docs/gettingstarted/developers/fib20/dataplane.rst b/docs/developer/corefeatures/fib/dataplane.rst
similarity index 95%
rename from docs/gettingstarted/developers/fib20/dataplane.rst
rename to docs/developer/corefeatures/fib/dataplane.rst
index 34886e1..94e11d1 100644
--- a/docs/gettingstarted/developers/fib20/dataplane.rst
+++ b/docs/developer/corefeatures/fib/dataplane.rst
@@ -8,7 +8,7 @@
the actions to perform on the packet. Each object type has an associated VLIB
graph node. For a packet to forward walk the graph is therefore to move from one
VLIB node to the next, with each performing the required actions. This is the
-heart of the VPP model.
+heart of the VPP model.
The data-plane graph is composed of generic data-path objects (DPOs). A parent
DPO is identified by the tuple:{type,index,next_node}. The *next_node* parameter
@@ -17,7 +17,7 @@
not need to be read [#f17]_ whilst processing the child. Specialisations [#f18]_ of the DPO
perform distinct actions. The most common DPOs and briefly what they represent are:
-- Load-balance: a choice in an ECMP set.
+- Load-balance: a choice in an ECMP set.
- Adjacency: apply a rewrite and forward through an interface
- MPLS-label: impose an MPLS label.
- Lookup: perform another lookup in a different table.
@@ -42,7 +42,7 @@
Each *fib_entry_t* contributes it own *load_balance_t*, for three reasons;
-- The result of a lookup in a IPv[46] table is a single 32 bit unsigned integer. This is an index into a memory pool. Consequently the object type must be the same for each result. Some routes will need a load-balance and some will not, but to insert another object in the graph to represent this choice is a waste of cycles, so the load-balance object is always the result. If the route does not have ECMP, then the load-balance has only one choice.
+- The result of a lookup in a IPv[46] table is a single 32 bit unsigned integer. This is an index into a memory pool. Consequently the object type must be the same for each result. Some routes will need a load-balance and some will not, but to insert another object in the graph to represent this choice is a waste of cycles, so the load-balance object is always the result. If the route does not have ECMP, then the load-balance has only one choice.
- In order to collect per-route counters, the lookup result must in some way uniquely identify the *fib_entry_t*. A shared load-balance (contributed by the path-list) would not allow this.
- In the case the *fib_entry_t* has MPLS out labels, and hence a *fib_path_ext_t*, then the load-balance must be per-prefix, since the MPLS labels that are its parents are themselves per-fib_entry_t.
@@ -74,13 +74,13 @@
into fewer DPOs and nodes. There are two ways we might consider doing
this:
-- write custom DPOs/nodes for combinded functions, e.g. pop MPLS label
+- write custom DPOs/nodes for combined functions, e.g. pop MPLS label
and lookup in v4 table. This has the disadvantage that the number of
such nodes would be, well, combinatorial, and resolving a path via
a combined DPO would be more difficult as it would involve a
forward walk of the graph to determine what the combination
is. However, VPP power users might consider this option for a
- limited set of their use cases where performance is truely king.
+ limited set of their use cases where performance is truly king.
- collapse multiple levels of load-balancing into one. For example,
if there were two levels of load-balancing each with two choices,
this could equally be represented by one level with 4 choices.
diff --git a/docs/gettingstarted/developers/fib20/debugging.rst b/docs/developer/corefeatures/fib/debugging.rst
similarity index 95%
rename from docs/gettingstarted/developers/fib20/debugging.rst
rename to docs/developer/corefeatures/fib/debugging.rst
index e0a85fd..750ad65 100644
--- a/docs/gettingstarted/developers/fib20/debugging.rst
+++ b/docs/developer/corefeatures/fib/debugging.rst
@@ -32,7 +32,7 @@
automatically generated by VPP).
- fib-index:0; in the VPP pool of FIB objects, this is index 0
- flow hash:[src dst sport dport proto ]: When calculating the flow
- hash to use for load-balanacing, these are the fields in the packet
+ hash to use for load-balancing, these are the fields in the packet
that are used. There is an API to change this per-table.
- epoch:0; Used during mark-n-sweep.
- flags:none; use the force, to find the per-table flags.
@@ -87,7 +87,7 @@
path-list 24. It's ip4 has a weight of 1 and a preference of 0. It's
of type 'attached-nexthop' and currently resolved - woohoo.
It is a path 'via 10.0.0.1 loop0'. It is contributing an incomplete adjacency.
-
+
next line:
.. code-block:: console
@@ -101,6 +101,6 @@
above.
Here we see load-balance object 11, which has 1 bucket/choice. It is
also linked to uRPF instance 11 (which it got from path-list 24).
-In bucket 0 there is the incomplete adjacnecy that was contributed by
+In bucket 0 there is the incomplete adjacency that was contributed by
path 26.
diff --git a/docs/gettingstarted/developers/fib20/fastconvergence.rst b/docs/developer/corefeatures/fib/fastconvergence.rst
similarity index 97%
rename from docs/gettingstarted/developers/fib20/fastconvergence.rst
rename to docs/developer/corefeatures/fib/fastconvergence.rst
index b07e08c..e1c5d0c 100644
--- a/docs/gettingstarted/developers/fib20/fastconvergence.rst
+++ b/docs/developer/corefeatures/fib/fastconvergence.rst
@@ -55,9 +55,9 @@
convergence.
Whenever an interface goes down, VPP issues a callback to all
-registerd clients. The adjacency code is such a client. The adjacency
+registered clients. The adjacency code is such a client. The adjacency
is a leaf node in the FIB control-plane graph (containing fib_path_t,
-fib_entry_t etc). A back-walk from the adjacnecy will trigger a
+fib_entry_t etc). A back-walk from the adjacency will trigger a
re-resolution of the paths.
FIB is a client of BFD in order to receive BFD notifications. BFD
@@ -81,7 +81,7 @@
networks are built in layers, it's how you scale them. We'll take
here a hypothetical service provider (SP) network, but the concepts
apply equally to data center leaf-spines. This is a rudimentary
-description, but it should serve our purpose.
+description, but it should serve our purpose.
An SP manages a BGP autonomous system (AS). The SP's goal is both to
attract traffic into its network to serve its customers, but also to
@@ -207,7 +207,7 @@
[0] [@12]: dpo-load-balance: [proto:ip4 index:17 buckets:2 uRPF:22 to:[0:0]]
[0] [@5]: ipv4 via 10.0.0.2 GigEthernet0/0/0: mtu:9000 next:3 001111111111dead000000000800
[1] [@5]: ipv4 via 10.0.1.2 GigEthernet0/0/1: mtu:9000 next:4 001111111111dead000000010800
-
+
the load-balance object used by this route is index 20, but note that
the next load-balance in the chain is index 17, i.e. it is exactly
the same instance that appears in the forwarding chain for the IGP
@@ -225,12 +225,12 @@
.. code-block:: console
- DBGvpp# sh ip fib 1.1.1.1/32
+ DBGvpp# sh ip fib 1.1.1.1/32
ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] epoch:0 flags:none locks:[adjacency:1, recursive-resolution:1, default-route:1, ]
1.1.1.1/32 fib:0 index:15 locks:4
API refs:1 src-flags:added,contributing,active,
path-list:[23] locks:2 flags:shared, uPRF-list:25 len:2 itfs:[1, 2, ]
- path:[27] pl-index:23 ip4 weight=1 pref=0 attached-nexthop:
+ path:[27] pl-index:23 ip4 weight=1 pref=0 attached-nexthop:
10.0.0.2 GigEthernet0/0/0
[@0]: arp-ipv4: via 10.0.0.2 GigEthernet0/0/0
path:[28] pl-index:23 ip4 weight=1 pref=0 attached-nexthop: oper-flags:resolved,
@@ -374,7 +374,7 @@
.. code-block:: console
- DBGvpp# sh ip fib 8.0.0.0/32
+ DBGvpp# sh ip fib 8.0.0.0/32
ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] epoch:0 flags:none locks:[adjacency:1, recursive-resolution:2, default-route:1, ]
8.0.0.0/16 fib:0 index:18 locks:2
API refs:1 src-flags:added,contributing,active,
@@ -426,7 +426,7 @@
.. code-block:: console
- DBGvpp# sh ip fib 8.8.0.0
+ DBGvpp# sh ip fib 8.8.0.0
ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] epoch:0 flags:none locks:[adjacency:1, recursive-resolution:4, default-route:1, ]
8.8.0.0/16 fib:0 index:77 locks:2
API refs:1 src-flags:added,contributing,active,
@@ -527,7 +527,7 @@
triggers a synchronous walk of the children of the /32 route, we want
a synchronous walk because we want to converge ASAP. This synchronous
walk will encounter path-lists in the /32 route's child dependent list.
-These path-lists (and thier LB maps) will be updated. If a path-list is
+These path-lists (and their LB maps) will be updated. If a path-list is
popular, then it will spawn a async walk of the path-list's child
dependent routes, if not it will walk those routes. So the walk
effectively proceeds breadth first across the path-lists, then returns
@@ -541,7 +541,7 @@
DBGvpp# ip route del 1.1.1.2/32 via 10.0.1.2 GigEthernet0/0/1
- DBGvpp# sh ip fib 8.8.0.0
+ DBGvpp# sh ip fib 8.8.0.0
ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] epoch:0 flags:none locks:[adjacency:1, recursive-resolution:4, default-route:1, ]
8.8.0.0/16 fib:0 index:77 locks:2
API refs:1 src-flags:added,contributing,active,
diff --git a/docs/gettingstarted/developers/fib20/graphs.rst b/docs/developer/corefeatures/fib/graphs.rst
similarity index 100%
rename from docs/gettingstarted/developers/fib20/graphs.rst
rename to docs/developer/corefeatures/fib/graphs.rst
diff --git a/docs/gettingstarted/developers/fib20/graphwalks.rst b/docs/developer/corefeatures/fib/graphwalks.rst
similarity index 100%
rename from docs/gettingstarted/developers/fib20/graphwalks.rst
rename to docs/developer/corefeatures/fib/graphwalks.rst
diff --git a/docs/gettingstarted/developers/fib20/hacking.rst b/docs/developer/corefeatures/fib/hacking.rst
similarity index 100%
rename from docs/gettingstarted/developers/fib20/hacking.rst
rename to docs/developer/corefeatures/fib/hacking.rst
diff --git a/docs/gettingstarted/developers/fib20/index.rst b/docs/developer/corefeatures/fib/index.rst
similarity index 65%
rename from docs/gettingstarted/developers/fib20/index.rst
rename to docs/developer/corefeatures/fib/index.rst
index 6a3115b..37c548b 100644
--- a/docs/gettingstarted/developers/fib20/index.rst
+++ b/docs/developer/corefeatures/fib/index.rst
@@ -1,8 +1,11 @@
.. _fib20:
-FIB 2.0 Hierarchical, Protocol, Independent
+The FIB
===========================================
+This describe the FIB (Forwarding information base) implementation :
+Hierarchical, Protocol, Independent
+
.. toctree::
prerequisites
diff --git a/docs/gettingstarted/developers/fib20/marknsweep.rst b/docs/developer/corefeatures/fib/marknsweep.rst
similarity index 100%
rename from docs/gettingstarted/developers/fib20/marknsweep.rst
rename to docs/developer/corefeatures/fib/marknsweep.rst
diff --git a/docs/gettingstarted/developers/fib20/missing.rst b/docs/developer/corefeatures/fib/missing.rst
similarity index 98%
rename from docs/gettingstarted/developers/fib20/missing.rst
rename to docs/developer/corefeatures/fib/missing.rst
index c5aa51f..0beccb1 100644
--- a/docs/gettingstarted/developers/fib20/missing.rst
+++ b/docs/developer/corefeatures/fib/missing.rst
@@ -91,7 +91,7 @@
- A recursive prefix with only one path and a path-list that is not
- popular, could stack directly on the LB of the via entry.
+ popular, could stack directly on the LB of the via entry.
- A recursive prefix with only multiple paths and a path-list that is not
popular, could construct a new load balance using the choices
present in each bucket of its via entries. The choices in the new LB
diff --git a/docs/gettingstarted/developers/fib20/mplsfib.rst b/docs/developer/corefeatures/fib/mplsfib.rst
similarity index 97%
rename from docs/gettingstarted/developers/fib20/mplsfib.rst
rename to docs/developer/corefeatures/fib/mplsfib.rst
index f66724e..50b1730 100644
--- a/docs/gettingstarted/developers/fib20/mplsfib.rst
+++ b/docs/developer/corefeatures/fib/mplsfib.rst
@@ -19,14 +19,14 @@
non-EOS packets cannot take this path, because to do so would expose
the neighbouring router to a label that it did not allocate.
-The desgin choice to make with an MPLS FIB table is therefore:
+The design choice to make with an MPLS FIB table is therefore:
- 20 bit key: label only. When the EOS and non-EOS actions differ the result is a 'EOS-choice' object.
- 21 bit key: label and EOS-bit. The result is then the specific action based on EOS-bit.
20 bit key
- Advantages:lower memory overhead, since there are few DB entries.
- Disadvantages: slower DP performance in the case the path-lists
- differ, as more objects are encounterd in the switch path
+ differ, as more objects are encountered in the switch path
21 bit key
- Advantages: faster DP performance
diff --git a/docs/gettingstarted/developers/fib20/multicast.rst b/docs/developer/corefeatures/fib/multicast.rst
similarity index 97%
rename from docs/gettingstarted/developers/fib20/multicast.rst
rename to docs/developer/corefeatures/fib/multicast.rst
index 7cffba8..37c5673 100644
--- a/docs/gettingstarted/developers/fib20/multicast.rst
+++ b/docs/developer/corefeatures/fib/multicast.rst
@@ -13,7 +13,7 @@
which it has been explicitly configured to accept.
The other factor that influences the design of the mFIB is that the
-match criteria (the prefix) is different. For multicast it is
+match criteria (the prefix) is different. For multicast it is
necessary to be able to match on source and destination/group
addresses (termed an (S,G)) and only on a destination prefix (a (\*,
G/m)). This prefix is much bigger than a unicast prefix, and since
diff --git a/docs/gettingstarted/developers/fib20/neighbors.rst b/docs/developer/corefeatures/fib/neighbors.rst
similarity index 96%
rename from docs/gettingstarted/developers/fib20/neighbors.rst
rename to docs/developer/corefeatures/fib/neighbors.rst
index f460955..13a3f07 100644
--- a/docs/gettingstarted/developers/fib20/neighbors.rst
+++ b/docs/developer/corefeatures/fib/neighbors.rst
@@ -10,7 +10,7 @@
Figure 1 shows the data model for IP neighbours. An IP neighbour contains the mapping
between a peer, identified by an IPv4 or IPv6 address, and its MAC address on a given
interface. An IP-table (VRF) is not part of the neighbour's
-data/identity. This is because the virtualisation of a router into
+data/identity. This is because the virtualization of a router into
different tables (VRFs) is performed at the interface level, i.e. an
IP-table is bound to a particular interface. A neighbour, which is
attached to an interface, is thus implicitly in that table, and
@@ -69,7 +69,7 @@
* A Glean Adjacency (key={interface}). This is a representation of the need to discover
a peer on the given interface. It is used when it is known that the
- packet is destined to an undiscoverd peer on that interface. The
+ packet is destined to an undiscovered peer on that interface. The
difference between the glean adjacency and an
incomplete neighbour adjacency is that in the forwarding path the
glean adjacency will construct an ARP/ND request for the peer as
diff --git a/docs/gettingstarted/developers/fib20/prefixes.rst b/docs/developer/corefeatures/fib/prefixes.rst
similarity index 100%
rename from docs/gettingstarted/developers/fib20/prefixes.rst
rename to docs/developer/corefeatures/fib/prefixes.rst
diff --git a/docs/gettingstarted/developers/fib20/prerequisites.rst b/docs/developer/corefeatures/fib/prerequisites.rst
similarity index 100%
rename from docs/gettingstarted/developers/fib20/prerequisites.rst
rename to docs/developer/corefeatures/fib/prerequisites.rst
diff --git a/docs/gettingstarted/developers/fib20/routes.rst b/docs/developer/corefeatures/fib/routes.rst
similarity index 98%
rename from docs/gettingstarted/developers/fib20/routes.rst
rename to docs/developer/corefeatures/fib/routes.rst
index 313a86c..a43cbd1 100644
--- a/docs/gettingstarted/developers/fib20/routes.rst
+++ b/docs/developer/corefeatures/fib/routes.rst
@@ -97,7 +97,7 @@
whose next-hop is the all zeros address and describes the only
peer on the link.
- * Recursive: The path is described only via the next-hop and table-id.
+ * Recursive: The path is described only via the next-hop and table-id.
* De-aggregate: The path is described only via the special all
zeros address and a table-id. This implies a subsequent lookup
@@ -127,7 +127,7 @@
are displayed at the top of the diagram, their parents below them. Forward walks are
thus from top to bottom, back walks bottom to top. The diagram shows the objects
that are shared, the path-list and adjacency. Sharing objects is critical to fast
-convergence (see section :ref:`fastconvergence`).
+convergence (see section :ref:`fastconvergence`).
FIB sources
"""""""""""
@@ -148,7 +148,7 @@
.. code-block:: console
- $ set interface ip address GigabitEthernet0/8/0 192.168.1.1/24
+ $ set interface ip address GigabitEthernet0/8/0 192.168.1.1/24
results in the addition of two FIB entries; 192.168.1.0/24 which is connected and
attached, and 192.168.1.1/32 which is connected and local (a.k.a.
@@ -157,7 +157,7 @@
the accidental or nefarious addition of identical prefixes does not prevent the
router from correctly forwarding. Packets matching a connected prefix will
generate an ARP request for the packets destination address, this process is known
-as a *glean*.
+as a *glean*.
An *attached* prefix also results in a glean, but the router does not have its own
address in that sub-net. The following configuration will result in an attached
@@ -251,8 +251,8 @@
The RR sourced FIB entry becomes the parent of the *fib_path_t* and will contribute its
forwarding information to that path, so that the child's FIB entry can construct its own
-forwarding information.
-
+forwarding information.
+
Figure 5 shows the object instances created to represent the recursive route and
its resolving route also shown.
@@ -319,7 +319,7 @@
FIB tracking sources a host-prefix entry in the FIB using the 'recusive
resolution (RR)' source, it exactly the same way that a recursive path
-does. If the entry did not previsouly exist, then the RR source will
+does. If the entry did not previously exist, then the RR source will
inherit (and track) forwarding from its covering prefix, therefore all
packets that match this entry are forwarded in the same way as if the
entry did not exist. The tunnel that is tracking this FIB entry will
@@ -330,7 +330,7 @@
FIB provides a wrapper to the sourcing of the host-prefix using a
delegate attached to the entry, and the entry is RR sourced only once.
-. The benefit of this aproach is that each time a new client tracks
+. The benefit of this approach is that each time a new client tracks
the entry it doesn't RR source it. When an entry is sourced all its
children are updated. Thus, new clients tracking an entry is
O(n^2). With the tracker as indirection, the entry is sourced only once.
diff --git a/docs/gettingstarted/developers/fib20/scale.rst b/docs/developer/corefeatures/fib/scale.rst
similarity index 87%
rename from docs/gettingstarted/developers/fib20/scale.rst
rename to docs/developer/corefeatures/fib/scale.rst
index 4100f9a..2ec8c6a 100644
--- a/docs/gettingstarted/developers/fib20/scale.rst
+++ b/docs/developer/corefeatures/fib/scale.rst
@@ -20,7 +20,7 @@
$ ip table add 1
-or the default table, comprises 2 *ip4_fib_t* objects.
+or the default table, comprises 2 *ip4_fib_t* objects.
The 'non-forwarding' *ip4_fib_t* contains all the entries in the table
and, the 'forwarding' contains the entries that are matched against in
the data-plane. The difference between the two sets are the entries
@@ -31,7 +31,7 @@
To see the amount of memory consumed by the IPv4 tables use:
.. code-block:: console
-
+
vpp# sh ip fib mem
ipv4-VRF:0 mtrie:335744 hash:4663
ipv4-VRF:1 mtrie:333056 hash:3499
@@ -71,7 +71,7 @@
.. code-block:: console
- vpp# sh ip6 fib mem
+ vpp# sh ip6 fib mem
IPv6 Non-Forwarding Hash Table:
Hash table ip6 FIB non-fwding table
7 active elements 7 active buckets
@@ -87,7 +87,7 @@
0 linear search buckets
arena: base 7f2fe48bf000, next 803c0
used 525248 b (0 Mbytes) of 33554432 b (32 Mbytes)
-
+
as we scale to 128k IPv6 entries:
.. code-block:: console
@@ -178,48 +178,48 @@
vpp# sh fib mem
FIB memory
Tables:
- SAFI Number Bytes
- IPv4 unicast 1 33619968
- IPv6 unicast 2 118502784
- MPLS 0 0
- IPv4 multicast 1 1175
- IPv6 multicast 1 525312
+ SAFI Number Bytes
+ IPv4 unicast 1 33619968
+ IPv6 unicast 2 118502784
+ MPLS 0 0
+ IPv4 multicast 1 1175
+ IPv6 multicast 1 525312
Nodes:
Name Size in-use /allocated totals
- Entry 72 1048589/ 1048589 75498408/75498408
- Entry Source 40 1048589/ 1048589 41943560/41943560
- Entry Path-Extensions 76 0 / 0 0/0
- multicast-Entry 192 6 / 6 1152/1152
- Path-list 40 18 / 18 720/720
- uRPF-list 16 14 / 14 224/224
- Path 72 22 / 22 1584/1584
- Node-list elements 20 1048602/ 1048602 20972040/20972040
- Node-list heads 8 24 / 24 192/192
+ Entry 72 1048589/ 1048589 75498408/75498408
+ Entry Source 40 1048589/ 1048589 41943560/41943560
+ Entry Path-Extensions 76 0 / 0 0/0
+ multicast-Entry 192 6 / 6 1152/1152
+ Path-list 40 18 / 18 720/720
+ uRPF-list 16 14 / 14 224/224
+ Path 72 22 / 22 1584/1584
+ Node-list elements 20 1048602/ 1048602 20972040/20972040
+ Node-list heads 8 24 / 24 192/192
and with 2M
.. code-block:: console
-
- vpp# sh fib mem
+
+ vpp# sh fib mem
FIB memory
Tables:
- SAFI Number Bytes
- IPv4 unicast 1 33619968
- IPv6 unicast 2 252743040
- MPLS 0 0
- IPv4 multicast 1 1175
- IPv6 multicast 1 525312
+ SAFI Number Bytes
+ IPv4 unicast 1 33619968
+ IPv6 unicast 2 252743040
+ MPLS 0 0
+ IPv4 multicast 1 1175
+ IPv6 multicast 1 525312
Nodes:
Name Size in-use /allocated totals
- Entry 72 2097165/ 2097165 150995880/150995880
- Entry Source 40 2097165/ 2097165 83886600/83886600
- Entry Path-Extensions 76 0 / 0 0/0
- multicast-Entry 192 6 / 6 1152/1152
- Path-list 40 18 / 19 720/760
- uRPF-list 16 18 / 18 288/288
- Path 72 22 / 23 1584/1656
- Node-list elements 20 2097178/ 2097178 41943560/41943560
- Node-list heads 8 24 / 24 192/192
+ Entry 72 2097165/ 2097165 150995880/150995880
+ Entry Source 40 2097165/ 2097165 83886600/83886600
+ Entry Path-Extensions 76 0 / 0 0/0
+ multicast-Entry 192 6 / 6 1152/1152
+ Path-list 40 18 / 19 720/760
+ uRPF-list 16 18 / 18 288/288
+ Path 72 22 / 23 1584/1656
+ Node-list elements 20 2097178/ 2097178 41943560/41943560
+ Node-list heads 8 24 / 24 192/192
However, the situation is not a simple as that. All of the 1M prefixes
added above were reachable via the same next-hop, so the path-list
diff --git a/docs/gettingstarted/developers/fib20/thedatamodel.rst b/docs/developer/corefeatures/fib/thedatamodel.rst
similarity index 100%
rename from docs/gettingstarted/developers/fib20/thedatamodel.rst
rename to docs/developer/corefeatures/fib/thedatamodel.rst
diff --git a/docs/gettingstarted/developers/fib20/tunnels.rst b/docs/developer/corefeatures/fib/tunnels.rst
similarity index 99%
rename from docs/gettingstarted/developers/fib20/tunnels.rst
rename to docs/developer/corefeatures/fib/tunnels.rst
index f53d622..d948a5e 100644
--- a/docs/gettingstarted/developers/fib20/tunnels.rst
+++ b/docs/developer/corefeatures/fib/tunnels.rst
@@ -9,7 +9,7 @@
beforehand, so the second lookup can be avoided if the packet can follow the
already constructed data-plane graph for the tunnel's destination. This process
of joining to DP graphs together is termed *stacking*.
-
+
.. figure:: /_images/fib20fig11.png
Figure 11: Tunnel control plane object diagram
diff --git a/docs/developer/corefeatures/index.rst b/docs/developer/corefeatures/index.rst
new file mode 100644
index 0000000..adc086e
--- /dev/null
+++ b/docs/developer/corefeatures/index.rst
@@ -0,0 +1,21 @@
+.. _corefeatures:
+
+=======================
+Core Features
+=======================
+
+.. toctree::
+ :maxdepth: 1
+
+ fib/index
+ sr/index
+ punt
+ ipsec
+ bfd_doc
+ ipfix_doc
+ span_doc
+ mtu
+ sylog_doc
+ eventviewer
+ stats
+ selinux_doc
diff --git a/docs/developer/corefeatures/ipfix_doc.rst b/docs/developer/corefeatures/ipfix_doc.rst
new file mode 120000
index 0000000..a4242e1
--- /dev/null
+++ b/docs/developer/corefeatures/ipfix_doc.rst
@@ -0,0 +1 @@
+../../../src/vnet/ipfix-export/ipfix_doc.rst
\ No newline at end of file
diff --git a/docs/gettingstarted/developers/ipsec.rst b/docs/developer/corefeatures/ipsec.rst
similarity index 100%
rename from docs/gettingstarted/developers/ipsec.rst
rename to docs/developer/corefeatures/ipsec.rst
diff --git a/docs/developer/corefeatures/mtu.rst b/docs/developer/corefeatures/mtu.rst
new file mode 120000
index 0000000..08930ff
--- /dev/null
+++ b/docs/developer/corefeatures/mtu.rst
@@ -0,0 +1 @@
+../../../src/vnet/mtu.rst
\ No newline at end of file
diff --git a/docs/gettingstarted/developers/punt.rst b/docs/developer/corefeatures/punt.rst
similarity index 100%
rename from docs/gettingstarted/developers/punt.rst
rename to docs/developer/corefeatures/punt.rst
diff --git a/docs/developer/corefeatures/selinux_doc.rst b/docs/developer/corefeatures/selinux_doc.rst
new file mode 120000
index 0000000..f0b3f5c
--- /dev/null
+++ b/docs/developer/corefeatures/selinux_doc.rst
@@ -0,0 +1 @@
+../../../extras/selinux/selinux_doc.rst
\ No newline at end of file
diff --git a/docs/developer/corefeatures/span_doc.rst b/docs/developer/corefeatures/span_doc.rst
new file mode 120000
index 0000000..a8a1cb3
--- /dev/null
+++ b/docs/developer/corefeatures/span_doc.rst
@@ -0,0 +1 @@
+../../../src/vnet/span/span_doc.rst
\ No newline at end of file
diff --git a/docs/developer/corefeatures/sr/index.rst b/docs/developer/corefeatures/sr/index.rst
new file mode 100644
index 0000000..8c87e2c
--- /dev/null
+++ b/docs/developer/corefeatures/sr/index.rst
@@ -0,0 +1,14 @@
+.. _corefeature_sr:
+
+===============
+Segment routing
+===============
+
+.. toctree::
+ :maxdepth: 1
+
+ sr_doc
+ sr_localsid
+ sr_mpls
+ sr_policy
+ sr_steering
\ No newline at end of file
diff --git a/docs/developer/corefeatures/sr/sr_doc.rst b/docs/developer/corefeatures/sr/sr_doc.rst
new file mode 120000
index 0000000..94bdfaa
--- /dev/null
+++ b/docs/developer/corefeatures/sr/sr_doc.rst
@@ -0,0 +1 @@
+../../../../src/vnet/srv6/sr_doc.rst
\ No newline at end of file
diff --git a/docs/developer/corefeatures/sr/sr_localsid.rst b/docs/developer/corefeatures/sr/sr_localsid.rst
new file mode 120000
index 0000000..0492fc7
--- /dev/null
+++ b/docs/developer/corefeatures/sr/sr_localsid.rst
@@ -0,0 +1 @@
+../../../../src/vnet/srv6/sr_localsid.rst
\ No newline at end of file
diff --git a/docs/developer/corefeatures/sr/sr_mpls.rst b/docs/developer/corefeatures/sr/sr_mpls.rst
new file mode 120000
index 0000000..d2fe402
--- /dev/null
+++ b/docs/developer/corefeatures/sr/sr_mpls.rst
@@ -0,0 +1 @@
+../../../../src/vnet/srmpls/sr_doc.rst
\ No newline at end of file
diff --git a/docs/developer/corefeatures/sr/sr_policy.rst b/docs/developer/corefeatures/sr/sr_policy.rst
new file mode 120000
index 0000000..bbd8734
--- /dev/null
+++ b/docs/developer/corefeatures/sr/sr_policy.rst
@@ -0,0 +1 @@
+../../../../src/vnet/srv6/sr_policy.rst
\ No newline at end of file
diff --git a/docs/developer/corefeatures/sr/sr_steering.rst b/docs/developer/corefeatures/sr/sr_steering.rst
new file mode 120000
index 0000000..7edf737
--- /dev/null
+++ b/docs/developer/corefeatures/sr/sr_steering.rst
@@ -0,0 +1 @@
+../../../../src/vnet/srv6/sr_steering.rst
\ No newline at end of file
diff --git a/docs/developer/corefeatures/stats.rst b/docs/developer/corefeatures/stats.rst
new file mode 120000
index 0000000..ef18296
--- /dev/null
+++ b/docs/developer/corefeatures/stats.rst
@@ -0,0 +1 @@
+../../../src/vpp/stats/stats.rst
\ No newline at end of file
diff --git a/docs/developer/corefeatures/sylog_doc.rst b/docs/developer/corefeatures/sylog_doc.rst
new file mode 120000
index 0000000..731267b
--- /dev/null
+++ b/docs/developer/corefeatures/sylog_doc.rst
@@ -0,0 +1 @@
+../../../src/vnet/syslog/sylog_doc.rst
\ No newline at end of file
diff --git a/docs/developer/devicedrivers/af_xdp.rst b/docs/developer/devicedrivers/af_xdp.rst
new file mode 120000
index 0000000..ae41d1f
--- /dev/null
+++ b/docs/developer/devicedrivers/af_xdp.rst
@@ -0,0 +1 @@
+../../../src/plugins/af_xdp/af_xdp_doc.rst
\ No newline at end of file
diff --git a/docs/developer/devicedrivers/avf.rst b/docs/developer/devicedrivers/avf.rst
new file mode 120000
index 0000000..06a84f5
--- /dev/null
+++ b/docs/developer/devicedrivers/avf.rst
@@ -0,0 +1 @@
+../../../src/plugins/avf/README.rst
\ No newline at end of file
diff --git a/docs/developer/devicedrivers/index.rst b/docs/developer/devicedrivers/index.rst
new file mode 100644
index 0000000..e1194e7
--- /dev/null
+++ b/docs/developer/devicedrivers/index.rst
@@ -0,0 +1,15 @@
+.. _devicedrivers:
+
+==============
+Device drivers
+==============
+
+
+
+.. toctree::
+ :maxdepth: 1
+
+ avf
+ rdma
+ vmxnet3
+ af_xdp
diff --git a/docs/developer/devicedrivers/rdma.rst b/docs/developer/devicedrivers/rdma.rst
new file mode 120000
index 0000000..d16203c
--- /dev/null
+++ b/docs/developer/devicedrivers/rdma.rst
@@ -0,0 +1 @@
+../../../src/plugins/rdma/rdma_doc.rst
\ No newline at end of file
diff --git a/docs/developer/devicedrivers/vmxnet3.rst b/docs/developer/devicedrivers/vmxnet3.rst
new file mode 120000
index 0000000..361b899
--- /dev/null
+++ b/docs/developer/devicedrivers/vmxnet3.rst
@@ -0,0 +1 @@
+../../../src/plugins/vmxnet3/README.rst
\ No newline at end of file
diff --git a/docs/developer/extras/index.rst b/docs/developer/extras/index.rst
new file mode 100644
index 0000000..200caf5
--- /dev/null
+++ b/docs/developer/extras/index.rst
@@ -0,0 +1,17 @@
+.. _vpp_extras:
+
+===============
+VPP extra tools
+===============
+
+.. toctree::
+ :maxdepth: 2
+
+ lcov
+ snap
+ strongswan
+ vpp_config
+ vpp_if_stats
+ vpp_stats_fs
+ vpptop
+ vcl_ldpreload
diff --git a/docs/developer/extras/lcov.rst b/docs/developer/extras/lcov.rst
new file mode 120000
index 0000000..9f8de24
--- /dev/null
+++ b/docs/developer/extras/lcov.rst
@@ -0,0 +1 @@
+../../../extras/lcov/README.rst
\ No newline at end of file
diff --git a/docs/developer/extras/snap.rst b/docs/developer/extras/snap.rst
new file mode 120000
index 0000000..773ec9b
--- /dev/null
+++ b/docs/developer/extras/snap.rst
@@ -0,0 +1 @@
+../../../extras/snap/README.rst
\ No newline at end of file
diff --git a/docs/developer/extras/strongswan.rst b/docs/developer/extras/strongswan.rst
new file mode 120000
index 0000000..1fef5a0
--- /dev/null
+++ b/docs/developer/extras/strongswan.rst
@@ -0,0 +1 @@
+../../../extras/strongswan/README.rst
\ No newline at end of file
diff --git a/docs/developer/extras/vcl_ldpreload.rst b/docs/developer/extras/vcl_ldpreload.rst
new file mode 120000
index 0000000..3740db2
--- /dev/null
+++ b/docs/developer/extras/vcl_ldpreload.rst
@@ -0,0 +1 @@
+../../../extras/vcl-ldpreload/README.rst
\ No newline at end of file
diff --git a/docs/developer/extras/vpp_config.rst b/docs/developer/extras/vpp_config.rst
new file mode 120000
index 0000000..0db2e6f
--- /dev/null
+++ b/docs/developer/extras/vpp_config.rst
@@ -0,0 +1 @@
+../../../extras/vpp_config/README.rst
\ No newline at end of file
diff --git a/docs/developer/extras/vpp_if_stats.rst b/docs/developer/extras/vpp_if_stats.rst
new file mode 120000
index 0000000..857cdee
--- /dev/null
+++ b/docs/developer/extras/vpp_if_stats.rst
@@ -0,0 +1 @@
+../../../extras/vpp_if_stats/README.rst
\ No newline at end of file
diff --git a/docs/developer/extras/vpp_stats_fs.rst b/docs/developer/extras/vpp_stats_fs.rst
new file mode 120000
index 0000000..5e67c6a
--- /dev/null
+++ b/docs/developer/extras/vpp_stats_fs.rst
@@ -0,0 +1 @@
+../../../extras/vpp_stats_fs/README.rst
\ No newline at end of file
diff --git a/docs/developer/extras/vpptop.rst b/docs/developer/extras/vpptop.rst
new file mode 120000
index 0000000..cf7f78e
--- /dev/null
+++ b/docs/developer/extras/vpptop.rst
@@ -0,0 +1 @@
+../../../extras/vpptop/README.rst
\ No newline at end of file
diff --git a/docs/gettingstarted/developers/add_plugin.rst b/docs/developer/plugindoc/add_plugin.rst
similarity index 100%
rename from docs/gettingstarted/developers/add_plugin.rst
rename to docs/developer/plugindoc/add_plugin.rst
diff --git a/docs/developer/plugindoc/handoffdemo.rst b/docs/developer/plugindoc/handoffdemo.rst
new file mode 120000
index 0000000..d9eaebf
--- /dev/null
+++ b/docs/developer/plugindoc/handoffdemo.rst
@@ -0,0 +1 @@
+../../../src/examples/handoffdemo/handoffdemo.rst
\ No newline at end of file
diff --git a/docs/developer/plugindoc/index.rst b/docs/developer/plugindoc/index.rst
new file mode 100644
index 0000000..2fcddb4
--- /dev/null
+++ b/docs/developer/plugindoc/index.rst
@@ -0,0 +1,13 @@
+.. _add_new_plugin:
+
+==============================
+Adding a new plugin or feature
+==============================
+
+
+.. toctree::
+ :maxdepth: 2
+
+ add_plugin
+ sample_plugin_doc
+ handoffdemo
diff --git a/docs/developer/plugindoc/sample_plugin_doc.rst b/docs/developer/plugindoc/sample_plugin_doc.rst
new file mode 120000
index 0000000..5f4c67a
--- /dev/null
+++ b/docs/developer/plugindoc/sample_plugin_doc.rst
@@ -0,0 +1 @@
+../../../src/examples/sample-plugin/sample_plugin_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/acl_hash_lookup.rst b/docs/developer/plugins/acl_hash_lookup.rst
new file mode 120000
index 0000000..934be28
--- /dev/null
+++ b/docs/developer/plugins/acl_hash_lookup.rst
@@ -0,0 +1 @@
+../../../src/plugins/acl/acl_hash_lookup_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/acl_lookup_context.rst b/docs/developer/plugins/acl_lookup_context.rst
new file mode 120000
index 0000000..e9e15ed
--- /dev/null
+++ b/docs/developer/plugins/acl_lookup_context.rst
@@ -0,0 +1 @@
+../../../src/plugins/acl/acl_lookup_context.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/acl_multicore.rst b/docs/developer/plugins/acl_multicore.rst
new file mode 120000
index 0000000..564b6a1
--- /dev/null
+++ b/docs/developer/plugins/acl_multicore.rst
@@ -0,0 +1 @@
+../../../src/plugins/acl/acl_multicore_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/bufmon_doc.rst b/docs/developer/plugins/bufmon_doc.rst
new file mode 120000
index 0000000..3578898
--- /dev/null
+++ b/docs/developer/plugins/bufmon_doc.rst
@@ -0,0 +1 @@
+../../../src/plugins/bufmon/bufmon_doc.rst
\ No newline at end of file
diff --git a/docs/gettingstarted/developers/cnat.rst b/docs/developer/plugins/cnat.rst
similarity index 100%
rename from docs/gettingstarted/developers/cnat.rst
rename to docs/developer/plugins/cnat.rst
diff --git a/docs/developer/plugins/dhcp6_pd.rst b/docs/developer/plugins/dhcp6_pd.rst
new file mode 120000
index 0000000..80bd946
--- /dev/null
+++ b/docs/developer/plugins/dhcp6_pd.rst
@@ -0,0 +1 @@
+../../../src/plugins/dhcp/dhcp6_pd_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/flowprobe.rst b/docs/developer/plugins/flowprobe.rst
new file mode 120000
index 0000000..7e2fbd2
--- /dev/null
+++ b/docs/developer/plugins/flowprobe.rst
@@ -0,0 +1 @@
+../../../src/plugins/flowprobe/flowprobe_plugin_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/index.rst b/docs/developer/plugins/index.rst
new file mode 100644
index 0000000..9aedc89
--- /dev/null
+++ b/docs/developer/plugins/index.rst
@@ -0,0 +1,41 @@
+.. _vpp_plugins:
+
+=======
+Plugins
+=======
+
+vlib implements a straightforward plug-in DLL mechanism. VLIB client
+applications specify a directory to search for plug-in .DLLs, and a name
+filter to apply (if desired). VLIB needs to load plug-ins very early.
+
+Once loaded, the plug-in DLL mechanism uses dlsym to find and verify a
+vlib\_plugin\_registration data structure in the newly-loaded plug-in.
+
+For more on plugins please refer to :ref:`add_plugin`.
+
+
+.. toctree::
+ :maxdepth: 2
+
+ quic
+ cnat
+ lcp
+ srv6/index
+ marvell
+ lldp
+ nat64
+ nat44_ei_ha
+ pnat
+ lb
+ lacp
+ flowprobe
+ map_lw4o6
+ mdata
+ dhcp6_pd
+ ioam
+ wireguard
+ srtp
+ acl_multicore
+ acl_hash_lookup
+ acl_lookup_context
+ bufmon_doc
diff --git a/docs/developer/plugins/ioam.rst b/docs/developer/plugins/ioam.rst
new file mode 120000
index 0000000..7a8faf6
--- /dev/null
+++ b/docs/developer/plugins/ioam.rst
@@ -0,0 +1 @@
+../../../src/plugins/ioam/ioam_plugin_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/lacp.rst b/docs/developer/plugins/lacp.rst
new file mode 120000
index 0000000..4d41024
--- /dev/null
+++ b/docs/developer/plugins/lacp.rst
@@ -0,0 +1 @@
+../../../src/plugins/lacp/lacp_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/lb.rst b/docs/developer/plugins/lb.rst
new file mode 120000
index 0000000..de3183a
--- /dev/null
+++ b/docs/developer/plugins/lb.rst
@@ -0,0 +1 @@
+../../../src/plugins/lb/lb_plugin_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/lcp.rst b/docs/developer/plugins/lcp.rst
new file mode 120000
index 0000000..2fbbe1f
--- /dev/null
+++ b/docs/developer/plugins/lcp.rst
@@ -0,0 +1 @@
+../../../src/plugins/linux-cp/lcp.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/lldp.rst b/docs/developer/plugins/lldp.rst
new file mode 120000
index 0000000..b913eca
--- /dev/null
+++ b/docs/developer/plugins/lldp.rst
@@ -0,0 +1 @@
+../../../src/plugins/lldp/lldp_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/map_lw4o6.rst b/docs/developer/plugins/map_lw4o6.rst
new file mode 120000
index 0000000..427f189
--- /dev/null
+++ b/docs/developer/plugins/map_lw4o6.rst
@@ -0,0 +1 @@
+../../../src/plugins/map/map_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/marvell.rst b/docs/developer/plugins/marvell.rst
new file mode 120000
index 0000000..28f0cd0
--- /dev/null
+++ b/docs/developer/plugins/marvell.rst
@@ -0,0 +1 @@
+../../../src/plugins/marvell/README.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/mdata.rst b/docs/developer/plugins/mdata.rst
new file mode 120000
index 0000000..87e5c92
--- /dev/null
+++ b/docs/developer/plugins/mdata.rst
@@ -0,0 +1 @@
+../../../src/plugins/mdata/mdata_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/nat44_ei_ha.rst b/docs/developer/plugins/nat44_ei_ha.rst
new file mode 120000
index 0000000..a8a00f402
--- /dev/null
+++ b/docs/developer/plugins/nat44_ei_ha.rst
@@ -0,0 +1 @@
+../../../src/plugins/nat/nat44-ei/nat44_ei_ha_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/nat64.rst b/docs/developer/plugins/nat64.rst
new file mode 120000
index 0000000..22c6bbe
--- /dev/null
+++ b/docs/developer/plugins/nat64.rst
@@ -0,0 +1 @@
+../../../src/plugins/nat/nat64/nat64_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/pnat.rst b/docs/developer/plugins/pnat.rst
new file mode 120000
index 0000000..4a8da6e
--- /dev/null
+++ b/docs/developer/plugins/pnat.rst
@@ -0,0 +1 @@
+../../../src/plugins/nat/pnat/pnat.rst
\ No newline at end of file
diff --git a/docs/gettingstarted/developers/quic_plugin.rst b/docs/developer/plugins/quic.rst
similarity index 100%
rename from docs/gettingstarted/developers/quic_plugin.rst
rename to docs/developer/plugins/quic.rst
diff --git a/docs/developer/plugins/srtp.rst b/docs/developer/plugins/srtp.rst
new file mode 120000
index 0000000..51013cc
--- /dev/null
+++ b/docs/developer/plugins/srtp.rst
@@ -0,0 +1 @@
+../../../src/plugins/srtp/srtp_plugin.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/srv6/ad_flow_plugin_doc.rst b/docs/developer/plugins/srv6/ad_flow_plugin_doc.rst
new file mode 120000
index 0000000..f944884
--- /dev/null
+++ b/docs/developer/plugins/srv6/ad_flow_plugin_doc.rst
@@ -0,0 +1 @@
+../../../../src/plugins/srv6-ad-flow/ad_flow_plugin_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/srv6/ad_plugin_doc.rst b/docs/developer/plugins/srv6/ad_plugin_doc.rst
new file mode 120000
index 0000000..46ec8a8
--- /dev/null
+++ b/docs/developer/plugins/srv6/ad_plugin_doc.rst
@@ -0,0 +1 @@
+../../../../src/plugins/srv6-ad/ad_plugin_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/srv6/am_plugin_doc.rst b/docs/developer/plugins/srv6/am_plugin_doc.rst
new file mode 120000
index 0000000..842426a
--- /dev/null
+++ b/docs/developer/plugins/srv6/am_plugin_doc.rst
@@ -0,0 +1 @@
+../../../../src/plugins/srv6-am/am_plugin_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/srv6/as_plugin_doc.rst b/docs/developer/plugins/srv6/as_plugin_doc.rst
new file mode 120000
index 0000000..90dd28e
--- /dev/null
+++ b/docs/developer/plugins/srv6/as_plugin_doc.rst
@@ -0,0 +1 @@
+../../../../src/plugins/srv6-as/as_plugin_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/srv6/index.rst b/docs/developer/plugins/srv6/index.rst
new file mode 100644
index 0000000..eb8f3e8
--- /dev/null
+++ b/docs/developer/plugins/srv6/index.rst
@@ -0,0 +1,16 @@
+.. _dplugins_srv6:
+
+============
+SRv6 Plugins
+============
+
+.. toctree::
+ :maxdepth: 2
+
+ ad_flow_plugin_doc
+ ad_plugin_doc
+ am_plugin_doc
+ as_plugin_doc
+ mobile_plugin_doc
+ runner_doc
+ srv6_sample_localsid_doc
diff --git a/docs/developer/plugins/srv6/mobile_plugin_doc.rst b/docs/developer/plugins/srv6/mobile_plugin_doc.rst
new file mode 120000
index 0000000..546f5f2
--- /dev/null
+++ b/docs/developer/plugins/srv6/mobile_plugin_doc.rst
@@ -0,0 +1 @@
+../../../../src/plugins/srv6-mobile/mobile_plugin_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/srv6/runner_doc.rst b/docs/developer/plugins/srv6/runner_doc.rst
new file mode 120000
index 0000000..1621703
--- /dev/null
+++ b/docs/developer/plugins/srv6/runner_doc.rst
@@ -0,0 +1 @@
+../../../../src/plugins/srv6-mobile/extra/runner_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/srv6/srv6_sample_localsid_doc.rst b/docs/developer/plugins/srv6/srv6_sample_localsid_doc.rst
new file mode 120000
index 0000000..db92456
--- /dev/null
+++ b/docs/developer/plugins/srv6/srv6_sample_localsid_doc.rst
@@ -0,0 +1 @@
+../../../../src/examples/srv6-sample-localsid/srv6_sample_localsid_doc.rst
\ No newline at end of file
diff --git a/docs/developer/plugins/wireguard.rst b/docs/developer/plugins/wireguard.rst
new file mode 120000
index 0000000..3cba6b4
--- /dev/null
+++ b/docs/developer/plugins/wireguard.rst
@@ -0,0 +1 @@
+../../../src/plugins/wireguard/README.rst
\ No newline at end of file
diff --git a/docs/developer/tests/overview.rst b/docs/developer/tests/overview.rst
new file mode 100644
index 0000000..cc9836e
--- /dev/null
+++ b/docs/developer/tests/overview.rst
@@ -0,0 +1,450 @@
+.. _unittest: https://docs.python.org/2/library/unittest.html
+.. _TestCase: https://docs.python.org/2/library/unittest.html#unittest.TestCase
+.. _AssertionError: https://docs.python.org/2/library/exceptions.html#exceptions.AssertionError
+.. _SkipTest: https://docs.python.org/2/library/unittest.html#unittest.SkipTest
+.. _virtualenv: http://docs.python-guide.org/en/latest/dev/virtualenvs/
+.. _scapy: http://www.secdev.org/projects/scapy/
+.. _logging: https://docs.python.org/2/library/logging.html
+.. _process: https://docs.python.org/2/library/multiprocessing.html#the-process-class
+.. _pipes: https://docs.python.org/2/library/multiprocessing.html#multiprocessing.Pipe
+.. _managed: https://docs.python.org/2/library/multiprocessing.html#managers
+
+.. |vtf| replace:: VPP Test Framework
+
+|vtf|
+=====
+
+.. contents::
+ :local:
+ :depth: 1
+
+Overview
+########
+
+The goal of the |vtf| is to ease writing, running and debugging
+unit tests for the VPP. For this, python was chosen as a high level language
+allowing rapid development with scapy_ providing the necessary tool for creating
+and dissecting packets.
+
+Anatomy of a test case
+######################
+
+Python's unittest_ is used as the base framework upon which the VPP test
+framework is built. A test suite in the |vtf| consists of multiple classes
+derived from `VppTestCase`, which is itself derived from TestCase_.
+The test class defines one or more test functions, which act as test cases.
+
+Function flow when running a test case is:
+
+1. `setUpClass <VppTestCase.setUpClass>`:
+ This function is called once for each test class, allowing a one-time test
+ setup to be executed. If this functions throws an exception,
+ none of the test functions are executed.
+2. `setUp <VppTestCase.setUp>`:
+ The setUp function runs before each of the test functions. If this function
+ throws an exception other than AssertionError_ or SkipTest_, then this is
+ considered an error, not a test failure.
+3. *test_<name>*:
+ This is the guts of the test case. It should execute the test scenario
+ and use the various assert functions from the unittest framework to check
+ necessary. Multiple test_<name> methods can exist in a test case.
+4. `tearDown <VppTestCase.tearDown>`:
+ The tearDown function is called after each test function with the purpose
+ of doing partial cleanup.
+5. `tearDownClass <VppTestCase.tearDownClass>`:
+ Method called once after running all of the test functions to perform
+ the final cleanup.
+
+Logging
+#######
+
+Each test case has a logger automatically created for it, stored in
+'logger' property, based on logging_. Use the logger's standard methods
+debug(), info(), error(), ... to emit log messages to the logger.
+
+All the log messages go always into a log file in temporary directory
+(see below).
+
+To control the messages printed to console, specify the V= parameter.
+
+.. code-block:: shell
+
+ make test # minimum verbosity
+ make test V=1 # moderate verbosity
+ make test V=2 # maximum verbosity
+
+Parallel test execution
+#######################
+
+|vtf| test suites can be run in parallel. Each test suite is executed
+in a separate process spawned by Python multiprocessing process_.
+
+The results from child test suites are sent to parent through pipes_, which are
+aggregated and summarized at the end of the run.
+
+Stdout, stderr and logs logged in child processes are redirected to individual
+parent managed_ queues. The data from these queues are then emitted to stdout
+of the parent process in the order the test suites have finished. In case there
+are no finished test suites (such as at the beginning of the run), the data
+from last started test suite are emitted in real time.
+
+To enable parallel test run, specify the number of parallel processes:
+
+.. code-block:: shell
+
+ make test TEST_JOBS=n # at most n processes will be spawned
+ make test TEST_JOBS=auto # chosen based on the number of cores
+ # and the size of shared memory
+
+Test temporary directory and VPP life cycle
+###########################################
+
+Test separation is achieved by separating the test files and vpp instances.
+Each test creates a temporary directory and it's name is used to create
+a shared memory prefix which is used to run a VPP instance.
+The temporary directory name contains the testcase class name for easy
+reference, so for testcase named 'TestVxlan' the directory could be named
+e.g. vpp-unittest-TestVxlan-UNUP3j.
+This way, there is no conflict between any other VPP instances running
+on the box and the test VPP. Any temporary files created by the test case
+are stored in this temporary test directory.
+
+The test temporary directory holds the following interesting files:
+
+* log.txt - this contains the logger output on max verbosity
+* pg*_in.pcap - last injected packet stream into VPP, named after the interface,
+ so for pg0, the file will be named pg0_in.pcap
+* pg*_out.pcap - last capture file created by VPP for interface, similarly,
+ named after the interface, so for e.g. pg1, the file will be named
+ pg1_out.pcap
+* history files - whenever the capture is restarted or a new stream is added,
+ the existing files are rotated and renamed, so all the pcap files
+ are always saved for later debugging if needed
+* core - if vpp dumps a core, it'll be stored in the temporary directory
+* vpp_stdout.txt - file containing output which vpp printed to stdout
+* vpp_stderr.txt - file containing output which vpp printed to stderr
+
+*NOTE*: existing temporary directories named vpp-unittest-* are automatically
+removed when invoking 'make test*' or 'make retest*' to keep the temporary
+directory clean.
+
+Virtual environment
+###################
+
+Virtualenv_ is a python module which provides a means to create an environment
+containing the dependencies required by the |vtf|, allowing a separation
+from any existing system-wide packages. |vtf|'s Makefile automatically
+creates a virtualenv_ inside build-root and installs the required packages
+in that environment. The environment is entered whenever executing a test
+via one of the make test targets.
+
+Naming conventions
+##################
+
+Most unit tests do some kind of packet manipulation - sending and receiving
+packets between VPP and virtual hosts connected to the VPP. Referring
+to the sides, addresses, etc. is always done as if looking from the VPP side,
+thus:
+
+* *local_* prefix is used for the VPP side.
+ So e.g. `local_ip4 <VppInterface.local_ip4>` address is the IPv4 address
+ assigned to the VPP interface.
+* *remote_* prefix is used for the virtual host side.
+ So e.g. `remote_mac <VppInterface.remote_mac>` address is the MAC address
+ assigned to the virtual host connected to the VPP.
+
+Automatically generated addresses
+#################################
+
+To send packets, one needs to typically provide some addresses, otherwise
+the packets will be dropped. The interface objects in |vtf| automatically
+provide addresses based on (typically) their indexes, which ensures
+there are no conflicts and eases debugging by making the addressing scheme
+consistent.
+
+The developer of a test case typically doesn't need to work with the actual
+numbers, rather using the properties of the objects. The addresses typically
+come in two flavors: '<address>' and '<address>n' - note the 'n' suffix.
+The former address is a Python string, while the latter is translated using
+socket.inet_pton to raw format in network byte order - this format is suitable
+for passing as an argument to VPP APIs.
+
+e.g. for the IPv4 address assigned to the VPP interface:
+
+* local_ip4 - Local IPv4 address on VPP interface (string)
+* local_ip4n - Local IPv4 address - raw, suitable as API parameter.
+
+These addresses need to be configured in VPP to be usable using e.g.
+`VppInterface.config_ip4` API. Please see the documentation to
+`VppInterface` for more details.
+
+By default, there is one remote address of each kind created for L3:
+remote_ip4 and remote_ip6. If the test needs more addresses, because it's
+simulating more remote hosts, they can be generated using
+`generate_remote_hosts` API and the entries for them inserted into the ARP
+table using `configure_ipv4_neighbors` API.
+
+Packet flow in the |vtf|
+########################
+
+Test framework -> VPP
+~~~~~~~~~~~~~~~~~~~~~
+
+|vtf| doesn't send any packets to VPP directly. Traffic is instead injected
+using packet-generator interfaces, represented by the `VppPGInterface` class.
+Packets are written into a temporary .pcap file, which is then read by the VPP
+and the packets are injected into the VPP world.
+
+To add a list of packets to an interface, call the `VppPGInterface.add_stream`
+method on that interface. Once everything is prepared, call `pg_start` method to
+start the packet generator on the VPP side.
+
+VPP -> test framework
+~~~~~~~~~~~~~~~~~~~~~
+
+Similarly, VPP doesn't send any packets to |vtf| directly. Instead, packet
+capture feature is used to capture and write traffic to a temporary .pcap file,
+which is then read and analyzed by the |vtf|.
+
+The following APIs are available to the test case for reading pcap files.
+
+* `VppPGInterface.get_capture`: this API is suitable for bulk & batch
+ style of test, where a list of packets is prepared & sent, then the
+ received packets are read and verified. The API needs the number of
+ packets which are expected to be captured (ignoring filtered
+ packets - see below) to know when the pcap file is completely
+ written by the VPP. If using packet infos for verifying packets,
+ then the counts of the packet infos can be automatically used by
+ `VppPGInterface.get_capture` to get the proper count (in this case
+ the default value None can be supplied as expected_count or omitted
+ altogether).
+* `VppPGInterface.wait_for_packet`: this API is suitable for
+ interactive style of test, e.g. when doing session management,
+ three-way handshakes, etc. This API waits for and returns a single
+ packet, keeping the capture file in place and remembering
+ context. Repeated invocations return following packets (or raise
+ Exception if timeout is reached) from the same capture file (=
+ packets arriving on the same interface).
+
+*NOTE*: it is not recommended to mix these APIs unless you understand
+how they work internally. None of these APIs rotate the pcap capture
+file, so calling e.g. `VppPGInterface.get_capture` after
+`VppPGInterface.wait_for_packet` will return already read packets. It
+is safe to switch from one API to another after calling
+`VppPGInterface.enable_capture` as that API rotates the capture file.
+
+Automatic filtering of packets:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Both APIs (`VppPGInterface.get_capture` and
+`VppPGInterface.wait_for_packet`) by default filter the packet
+capture, removing known uninteresting packets from it - these are IPv6
+Router Advertisements and IPv6 Router Alerts. These packets are
+unsolicited and from the point of |vtf| are random. If a test wants
+to receive these packets, it should specify either None or a custom
+filtering function as the value to the 'filter_out_fn' argument.
+
+Common API flow for sending/receiving packets:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We will describe a simple scenario, where packets are sent from pg0 to pg1
+interface, assuming that the interfaces were created using
+`create_pg_interfaces` API.
+
+1. Create a list of packets for pg0::
+
+ packet_count = 10
+ packets = create_packets(src=self.pg0, dst=self.pg1,
+ count=packet_count)
+
+2. Add that list of packets to the source interface::
+
+ self.pg0.add_stream(packets)
+
+3. Enable capture on the destination interface::
+
+ self.pg1.enable_capture()
+
+4. Start the packet generator::
+
+ self.pg_start()
+
+5. Wait for capture file to appear and read it::
+
+ capture = self.pg1.get_capture(expected_count=packet_count)
+
+6. Verify packets match sent packets::
+
+ self.verify_capture(send=packets, captured=capture)
+
+Test framework objects
+######################
+
+The following objects provide VPP abstraction and provide a means to do
+common tasks easily in the test cases.
+
+* `VppInterface`: abstract class representing generic VPP interface
+ and contains some common functionality, which is then used by derived classes
+* `VppPGInterface`: class representing VPP packet-generator interface.
+ The interface is created/destroyed when the object is created/destroyed.
+* `VppSubInterface`: VPP sub-interface abstract class, containing common
+ functionality for e.g. `VppDot1QSubint` and `VppDot1ADSubint` classes
+
+How VPP APIs/CLIs are called
+############################
+
+Vpp provides python bindings in a python module called vpp-papi, which the test
+framework installs in the virtual environment. A shim layer represented by
+the `VppPapiProvider` class is built on top of the vpp-papi, serving these
+purposes:
+
+1. Automatic return value checks:
+ After each API is called, the return value is checked against the expected
+ return value (by default 0, but can be overridden) and an exception
+ is raised if the check fails.
+2. Automatic call of hooks:
+
+ a. `before_cli <Hook.before_cli>` and `before_api <Hook.before_api>` hooks
+ are used for debug logging and stepping through the test
+ b. `after_cli <Hook.after_cli>` and `after_api <Hook.after_api>` hooks
+ are used for monitoring the vpp process for crashes
+3. Simplification of API calls:
+ Many of the VPP APIs take a lot of parameters and by providing sane defaults
+ for these, the API is much easier to use in the common case and the code is
+ more readable. E.g. ip_add_del_route API takes ~25 parameters, of which
+ in the common case, only 3 are needed.
+
+Utility methods
+###############
+
+Some interesting utility methods are:
+
+* `ppp`: 'Pretty Print Packet' - returns a string containing the same output
+ as Scapy's packet.show() would print
+* `ppc`: 'Pretty Print Capture' - returns a string containing printout of
+ a capture (with configurable limit on the number of packets printed from it)
+ using `ppp`
+
+*NOTE*: Do not use Scapy's packet.show() in the tests, because it prints
+the output to stdout. All output should go to the logger associated with
+the test case.
+
+Example: how to add a new test
+##############################
+
+In this example, we will describe how to add a new test case which tests
+basic IPv4 forwarding.
+
+1. Add a new file called test_ip4_fwd.py in the test directory, starting
+ with a few imports::
+
+ from framework import VppTestCase
+ from scapy.layers.l2 import Ether
+ from scapy.packet import Raw
+ from scapy.layers.inet import IP, UDP
+ from random import randint
+
+2. Create a class inherited from the VppTestCase::
+
+ class IP4FwdTestCase(VppTestCase):
+ """ IPv4 simple forwarding test case """
+
+3. Add a setUpClass function containing the setup needed for our test to run::
+
+ @classmethod
+ def setUpClass(self):
+ super(IP4FwdTestCase, self).setUpClass()
+ self.create_pg_interfaces(range(2)) # create pg0 and pg1
+ for i in self.pg_interfaces:
+ i.admin_up() # put the interface up
+ i.config_ip4() # configure IPv4 address on the interface
+ i.resolve_arp() # resolve ARP, so that we know VPP MAC
+
+4. Create a helper method to create the packets to send::
+
+ def create_stream(self, src_if, dst_if, count):
+ packets = []
+ for i in range(count):
+ # create packet info stored in the test case instance
+ info = self.create_packet_info(src_if, dst_if)
+ # convert the info into packet payload
+ payload = self.info_to_payload(info)
+ # create the packet itself
+ p = (Ether(dst=src_if.local_mac, src=src_if.remote_mac) /
+ IP(src=src_if.remote_ip4, dst=dst_if.remote_ip4) /
+ UDP(sport=randint(1000, 2000), dport=5678) /
+ Raw(payload))
+ # store a copy of the packet in the packet info
+ info.data = p.copy()
+ # append the packet to the list
+ packets.append(p)
+
+ # return the created packet list
+ return packets
+
+5. Create a helper method to verify the capture::
+
+ def verify_capture(self, src_if, dst_if, capture):
+ packet_info = None
+ for packet in capture:
+ try:
+ ip = packet[IP]
+ udp = packet[UDP]
+ # convert the payload to packet info object
+ payload_info = self.payload_to_info(packet[Raw])
+ # make sure the indexes match
+ self.assert_equal(payload_info.src, src_if.sw_if_index,
+ "source sw_if_index")
+ self.assert_equal(payload_info.dst, dst_if.sw_if_index,
+ "destination sw_if_index")
+ packet_info = self.get_next_packet_info_for_interface2(
+ src_if.sw_if_index,
+ dst_if.sw_if_index,
+ packet_info)
+ # make sure we didn't run out of saved packets
+ self.assertIsNotNone(packet_info)
+ self.assert_equal(payload_info.index, packet_info.index,
+ "packet info index")
+ saved_packet = packet_info.data # fetch the saved packet
+ # assert the values match
+ self.assert_equal(ip.src, saved_packet[IP].src,
+ "IP source address")
+ # ... more assertions here
+ self.assert_equal(udp.sport, saved_packet[UDP].sport,
+ "UDP source port")
+ except:
+ self.logger.error(ppp("Unexpected or invalid packet:",
+ packet))
+ raise
+ remaining_packet = self.get_next_packet_info_for_interface2(
+ src_if.sw_if_index,
+ dst_if.sw_if_index,
+ packet_info)
+ self.assertIsNone(remaining_packet,
+ "Interface %s: Packet expected from interface "
+ "%s didn't arrive" % (dst_if.name, src_if.name))
+
+6. Add the test code to test_basic function::
+
+ def test_basic(self):
+ count = 10
+ # create the packet stream
+ packets = self.create_stream(self.pg0, self.pg1, count)
+ # add the stream to the source interface
+ self.pg0.add_stream(packets)
+ # enable capture on both interfaces
+ self.pg0.enable_capture()
+ self.pg1.enable_capture()
+ # start the packet generator
+ self.pg_start()
+ # get capture - the proper count of packets was saved by
+ # create_packet_info() based on dst_if parameter
+ capture = self.pg1.get_capture()
+ # assert nothing captured on pg0 (always do this last, so that
+ # some time has already passed since pg_start())
+ self.pg0.assert_nothing_captured()
+ # verify capture
+ self.verify_capture(self.pg0, self.pg1, capture)
+
+7. Run the test by issuing 'make test' or, to run only this specific
+ test, issue 'make test TEST=test_ip4_fwd'.
diff --git a/docs/docsignore b/docs/docsignore
new file mode 100644
index 0000000..abd2eb5
--- /dev/null
+++ b/docs/docsignore
@@ -0,0 +1,8 @@
+# This is a list of ignored documentation files
+# in VPP. Not linking these files in ./docs won't
+# trigger an error in ./extra/scripts/check_documentation.sh
+extras/deprecated/dpdk-hqos/qos_doc.md
+extras/deprecated/dpdk-ipsec/dpdk_crypto_ipsec_doc.md
+src/vpp-api/lua/README.md
+src/vpp-api/lua/examples/cli/README.md
+src/vpp-api/lua/examples/lute/README.md
diff --git a/docs/dynamic_includes/.gitkeep b/docs/dynamic_includes/.gitkeep
deleted file mode 100644
index e69de29..0000000
--- a/docs/dynamic_includes/.gitkeep
+++ /dev/null
diff --git a/docs/gettingstarted/developers/VPPAPI.md b/docs/gettingstarted/developers/VPPAPI.md
deleted file mode 120000
index a5404c1..0000000
--- a/docs/gettingstarted/developers/VPPAPI.md
+++ /dev/null
@@ -1 +0,0 @@
-../../../src/tools/vppapigen/VPPAPI.md
\ No newline at end of file
diff --git a/docs/gettingstarted/developers/bihash.md b/docs/gettingstarted/developers/bihash.md
deleted file mode 100644
index b0f9886..0000000
--- a/docs/gettingstarted/developers/bihash.md
+++ /dev/null
@@ -1,308 +0,0 @@
-Bounded-index Extensible Hashing (bihash)
-=========================================
-
-Vpp uses bounded-index extensible hashing to solve a variety of
-exact-match (key, value) lookup problems. Benefits of the current
-implementation:
-
-* Very high record count scaling, tested to 100,000,000 records.
-* Lookup performance degrades gracefully as the number of records increases
-* No reader locking required
-* Template implementation, it's easy to support arbitrary (key,value) types
-
-Bounded-index extensible hashing has been widely used in databases for
-decades.
-
-Bihash uses a two-level data structure:
-
-```
- +-----------------+
- | bucket-0 |
- | log2_size |
- | backing store |
- +-----------------+
- | bucket-1 |
- | log2_size | +--------------------------------+
- | backing store | --------> | KVP_PER_PAGE * key-value-pairs |
- +-----------------+ | page 0 |
- ... +--------------------------------+
- +-----------------+ | KVP_PER_PAGE * key-value-pairs |
- | bucket-2**N-1 | | page 1 |
- | log2_size | +--------------------------------+
- | backing store | ---
- +-----------------+ +--------------------------------+
- | KVP_PER_PAGE * key-value-pairs |
- | page 2**(log2(size)) - 1 |
- +--------------------------------+
-```
-
-Discussion of the algorithm
----------------------------
-
-This structure has a couple of major advantages. In practice, each
-bucket entry fits into a 64-bit integer. Coincidentally, vpp's target
-CPU architectures support 64-bit atomic operations. When modifying the
-contents of a specific bucket, we do the following:
-
-* Make a working copy of the bucket's backing storage
-* Atomically swap a pointer to the working copy into the bucket array
-* Change the original backing store data
-* Atomically swap back to the original
-
-So, no reader locking is required to search a bihash table.
-
-At lookup time, the implementation computes a key hash code. We use
-the least-significant N bits of the hash to select the bucket.
-
-With the bucket in hand, we learn log2 (nBackingPages) for the
-selected bucket. At this point, we use the next log2_size bits from
-the hash code to select the specific backing page in which the
-(key,value) page will be found.
-
-Net result: we search **one** backing page, not 2**log2_size
-pages. This is a key property of the algorithm.
-
-When sufficient collisions occur to fill the backing pages for a given
-bucket, we double the bucket size, rehash, and deal the bucket
-contents into a double-sized set of backing pages. In the future, we
-may represent the size as a linear combination of two powers-of-two,
-to increase space efficiency.
-
-To solve the "jackpot case" where a set of records collide under
-hashing in a bad way, the implementation will fall back to linear
-search across 2**log2_size backing pages on a per-bucket basis.
-
-To maintain *space* efficiency, we should configure the bucket array
-so that backing pages are effectively utilized. Lookup performance
-tends to change *very little* if the bucket array is too small or too
-large.
-
-Bihash depends on selecting an effective hash function. If one were to
-use a truly broken hash function such as "return 1ULL." bihash would
-still work, but it would be equivalent to poorly-programmed linear
-search.
-
-We often use cpu intrinsic functions - think crc32 - to rapidly
-compute a hash code which has decent statistics.
-
-Bihash Cookbook
----------------
-
-### Using current (key,value) template instance types
-
-It's quite easy to use one of the template instance types. As of this
-writing, .../src/vppinfra provides pre-built templates for 8, 16, 20,
-24, 40, and 48 byte keys, u8 * vector keys, and 8 byte values.
-
-See .../src/vppinfra/{bihash_<key-size>_8}.h
-
-To define the data types, #include a specific template instance, most
-often in a subsystem header file:
-
-```c
- #include <vppinfra/bihash_8_8.h>
-```
-
-If you're building a standalone application, you'll need to define the
-various functions by #including the method implementation file in a C
-source file.
-
-The core vpp engine currently uses most if not all of the known bihash
-types, so you probably won't need to #include the method
-implementation file.
-
-
-```c
- #include <vppinfra/bihash_template.c>
-```
-
-Add an instance of the selected bihash data structure to e.g. a
-"main_t" structure:
-
-```c
- typedef struct
- {
- ...
- BVT (clib_bihash) hash_table;
- or
- clib_bihash_8_8_t hash_table;
- ...
- } my_main_t;
-```
-
-The BV macro concatenate its argument with the value of the
-preprocessor symbol BIHASH_TYPE. The BVT macro concatenates its
-argument with the value of BIHASH_TYPE and the fixed-string "_t". So
-in the above example, BVT (clib_bihash) generates "clib_bihash_8_8_t".
-
-If you're sure you won't decide to change the template / type name
-later, it's perfectly OK to code "clib_bihash_8_8_t" and so forth.
-
-In fact, if you #include multiple template instances in a single
-source file, you **must** use fully-enumerated type names. The macros
-stand no chance of working.
-
-### Initializing a bihash table
-
-Call the init function as shown. As a rough guide, pick a number of
-buckets which is approximately
-number_of_expected_records/BIHASH_KVP_PER_PAGE from the relevant
-template instance header-file. See previous discussion.
-
-The amount of memory selected should easily contain all of the
-records, with a generous allowance for hash collisions. Bihash memory
-is allocated separately from the main heap, and won't cost anything
-except kernel PTE's until touched, so it's OK to be reasonably
-generous.
-
-For example:
-
-```c
- my_main_t *mm = &my_main;
- clib_bihash_8_8_t *h;
-
- h = &mm->hash_table;
-
- clib_bihash_init_8_8 (h, "test", (u32) number_of_buckets,
- (uword) memory_size);
-```
-
-### Add or delete a key/value pair
-
-Use BV(clib_bihash_add_del), or the explicit type variant:
-
-```c
- clib_bihash_kv_8_8_t kv;
- clib_bihash_8_8_t * h;
- my_main_t *mm = &my_main;
- clib_bihash_8_8_t *h;
-
- h = &mm->hash_table;
- kv.key = key_to_add_or_delete;
- kv.value = value_to_add_or_delete;
-
- clib_bihash_add_del_8_8 (h, &kv, is_add /* 1=add, 0=delete */);
-```
-
-In the delete case, kv.value is irrelevant. To change the value associated
-with an existing (key,value) pair, simply re-add the [new] pair.
-
-### Simple search
-
-The simplest possible (key, value) search goes like so:
-
-```c
- clib_bihash_kv_8_8_t search_kv, return_kv;
- clib_bihash_8_8_t * h;
- my_main_t *mm = &my_main;
- clib_bihash_8_8_t *h;
-
- h = &mm->hash_table;
- search_kv.key = key_to_add_or_delete;
-
- if (clib_bihash_search_8_8 (h, &search_kv, &return_kv) < 0)
- key_not_found();
- else
- key_found();
-```
-
-Note that it's perfectly fine to collect the lookup result
-
-```c
- if (clib_bihash_search_8_8 (h, &search_kv, &search_kv))
- key_not_found();
- etc.
-```
-
-### Bihash vector processing
-
-When processing a vector of packets which need a certain lookup
-performed, it's worth the trouble to compute the key hash, and
-prefetch the correct bucket ahead of time.
-
-Here's a sketch of one way to write the required code:
-
-Dual-loop:
-* 6 packets ahead, prefetch 2x vlib_buffer_t's and 2x packet data
- required to form the record keys
-* 4 packets ahead, form 2x record keys and call BV(clib_bihash_hash)
- or the explicit hash function to calculate the record hashes.
- Call 2x BV(clib_bihash_prefetch_bucket) to prefetch the buckets
-* 2 packets ahead, call 2x BV(clib_bihash_prefetch_data) to prefetch
- 2x (key,value) data pages.
-* In the processing section, call 2x BV(clib_bihash_search_inline_with_hash)
- to perform the search
-
-Programmer's choice whether to stash the hash code somewhere in
-vnet_buffer(b) metadata, or to use local variables.
-
-Single-loop:
-* Use simple search as shown above.
-
-### Walking a bihash table
-
-A fairly common scenario to build "show" commands involves walking a
-bihash table. It's simple enough:
-
-```c
- my_main_t *mm = &my_main;
- clib_bihash_8_8_t *h;
- void callback_fn (clib_bihash_kv_8_8_t *, void *);
-
- h = &mm->hash_table;
-
- BV(clib_bihash_foreach_key_value_pair) (h, callback_fn, (void *) arg);
-```
-To nobody's great surprise: clib_bihash_foreach_key_value_pair
-iterates across the entire table, calling callback_fn with active
-entries.
-
-#### Bihash table iteration safety
-
-The iterator template "clib_bihash_foreach_key_value_pair" must be
-used with a certain amount of care. For one thing, the iterator
-template does _not_ take the bihash hash table writer lock. If your
-use-case requires it, lock the table.
-
-For another, the iterator template is not safe under all conditions:
-
-* It's __OK to delete__ bihash table entries during a table-walk. The
-iterator checks whether the current bucket has been freed after each
-_callback_fn(...)_ invocation.
-
-* It is __not OK to add__ entries during a table-walk.
-
-The add-during-walk case involves a jackpot: while processing a
-key-value-pair in a particular bucket, add a certain number of
-entries. By luck, assume that one or more of the added entries causes
-the __current bucket__ to split-and-rehash.
-
-Since we rehash KVP's to different pages based on what amounts to a
-different hash function, either of these things can go wrong:
-
-* We may revisit previously-visited entries. Depending on how one
-coded the use-case, we could end up in a recursive-add situation.
-
-* We may skip entries that have not been visited
-
-One could build an add-safe iterator, at a significant cost in
-performance: copy the entire bucket, and walk the copy.
-
-It's hard to imagine a worthwhile add-during walk use-case in the
-first place; let alone one which couldn't be implemented by walking
-the table without modifying it, then adding a set of records.
-
-### Creating a new template instance
-
-Creating a new template is easy. Use one of the existing templates as
-a model, and make the obvious changes. The hash and key_compare
-methods are performance-critical in multiple senses.
-
-If the key compare method is slow, every lookup will be slow. If the
-hash function is slow, same story. If the hash function has poor
-statistical properties, space efficiency will suffer. In the limit, a
-bad enough hash function will cause large portions of the table to
-revert to linear search.
-
-Use of the best available vector unit is well worth the trouble in the
-hash and key_compare functions.
diff --git a/docs/gettingstarted/developers/binary_api_support.rst b/docs/gettingstarted/developers/binary_api_support.rst
deleted file mode 100644
index f93bbc2..0000000
--- a/docs/gettingstarted/developers/binary_api_support.rst
+++ /dev/null
@@ -1,485 +0,0 @@
-.. _binary_api_support:
-
-.. toctree::
-
-Binary API Support
-==================
-
-VPP provides a binary API scheme to allow a wide variety of client
-codes to program data-plane tables. As of this writing, there are
-hundreds of binary APIs.
-
-Messages are defined in \*.api files. Today, there are about 80 api
-files, with more arriving as folks add programmable features. The API
-file compiler sources reside in src/tools/vppapigen.
-
-From `src/vnet/interface.api
-<https://docs.fd.io/vpp/18.11/de/d75/interface_8api.html>`_, here's a
-typical request/response message definition:
-
-.. code-block:: console
-
- autoreply define sw_interface_set_flags
- {
- u32 client_index;
- u32 context;
- u32 sw_if_index;
- /* 1 = up, 0 = down */
- u8 admin_up_down;
- };
-
-To a first approximation, the API compiler renders this definition
-into
-*vpp/build-root/install-vpp_debug-native/vpp/include/vnet/interface.api.h*
-as follows:
-
-.. code-block:: C
-
- /****** Message ID / handler enum ******/
-
- #ifdef vl_msg_id
- vl_msg_id(VL_API_SW_INTERFACE_SET_FLAGS, vl_api_sw_interface_set_flags_t_handler)
- vl_msg_id(VL_API_SW_INTERFACE_SET_FLAGS_REPLY, vl_api_sw_interface_set_flags_reply_t_handler)
- #endif
- /****** Message names ******/
-
- #ifdef vl_msg_name
- vl_msg_name(vl_api_sw_interface_set_flags_t, 1)
- vl_msg_name(vl_api_sw_interface_set_flags_reply_t, 1)
- #endif
- /****** Message name, crc list ******/
-
- #ifdef vl_msg_name_crc_list
- #define foreach_vl_msg_name_crc_interface \
- _(VL_API_SW_INTERFACE_SET_FLAGS, sw_interface_set_flags, f890584a) \
- _(VL_API_SW_INTERFACE_SET_FLAGS_REPLY, sw_interface_set_flags_reply, dfbf3afa) \
- #endif
- /****** Typedefs *****/
-
- #ifdef vl_typedefs
- #ifndef defined_sw_interface_set_flags
- #define defined_sw_interface_set_flags
- typedef VL_API_PACKED(struct _vl_api_sw_interface_set_flags {
- u16 _vl_msg_id;
- u32 client_index;
- u32 context;
- u32 sw_if_index;
- u8 admin_up_down;
- }) vl_api_sw_interface_set_flags_t;
- #endif
-
- #ifndef defined_sw_interface_set_flags_reply
- #define defined_sw_interface_set_flags_reply
- typedef VL_API_PACKED(struct _vl_api_sw_interface_set_flags_reply {
- u16 _vl_msg_id;
- u32 context;
- i32 retval;
- }) vl_api_sw_interface_set_flags_reply_t;
- #endif
- ...
- #endif /* vl_typedefs */
-
-To change the admin state of an interface, a binary api client sends a
-`vl_api_sw_interface_set_flags_t
-<https://docs.fd.io/vpp/18.11/dc/da3/structvl__api__sw__interface__set__flags__t.html>`_
-to VPP, which will respond with a
-vl_api_sw_interface_set_flags_reply_t message.
-
-Multiple layers of software, transport types, and shared libraries
-implement a variety of features:
-
-* API message allocation, tracing, pretty-printing, and replay.
-* Message transport via global shared memory, pairwise/private shared memory, and sockets.
-* Barrier synchronization of worker threads across thread-unsafe message handlers.
-
-Correctly-coded message handlers know nothing about the transport used
-to deliver messages to/from VPP. It's reasonably straightforward to use
-multiple API message transport types simultaneously.
-
-For historical reasons, binary api messages are (putatively) sent in
-network byte order. As of this writing, we're seriously considering
-whether that choice makes sense.
-
-Message Allocation
-__________________
-
-Since binary API messages are always processed in order, we allocate
-messages using a ring allocator whenever possible. This scheme is
-extremely fast when compared with a traditional memory allocator, and
-doesn't cause heap fragmentation. See `src/vlibmemory/memory_shared.c
-<https://docs.fd.io/vpp/18.11/dd/d0d/memory__shared_8c.html>`_
-`vl_msg_api_alloc_internal()
-<https://docs.fd.io/vpp/18.11/dd/d0d/memory__shared_8c.html#ac6b6797850e1a53bc68b206e6b8413fb>`_.
-
-Regardless of transport, binary api messages always follow a `msgbuf_t <https://docs.fd.io/vpp/18.11/d9/d65/structmsgbuf__.html>`_ header:
-
-.. code-block:: C
-
- /** Message header structure */
- typedef struct msgbuf_
- {
- svm_queue_t *q; /**< message allocated in this shmem ring */
- u32 data_len; /**< message length not including header */
- u32 gc_mark_timestamp; /**< message garbage collector mark TS */
- u8 data[0]; /**< actual message begins here */
- } msgbuf_t;
-
-This structure makes it easy to trace messages without having to
-decode them - simply save data_len bytes - and allows
-`vl_msg_api_free()
-<https://docs.fd.io/vpp/18.11/d6/d1b/api__common_8h.html#aff61e777fe5df789121d8e78134867e6>`_
-to rapidly dispose of message buffers:
-
-.. code-block:: C
-
- void
- vl_msg_api_free (void *a)
- {
- msgbuf_t *rv;
- void *oldheap;
- api_main_t *am = &api_main;
-
- rv = (msgbuf_t *) (((u8 *) a) - offsetof (msgbuf_t, data));
-
- /*
- * Here's the beauty of the scheme. Only one proc/thread has
- * control of a given message buffer. To free a buffer, we just clear the
- * queue field, and leave. No locks, no hits, no errors...
- */
- if (rv->q)
- {
- rv->q = 0;
- rv->gc_mark_timestamp = 0;
- <more code...>
- return;
- }
- <more code...>
- }
-
-Message Tracing and Replay
-__________________________
-
-It's extremely important that VPP can capture and replay sizeable
-binary API traces. System-level issues involving hundreds of thousands
-of API transactions can be re-run in a second or less. Partial replay
-allows one to binary-search for the point where the wheels fall
-off. One can add scaffolding to the data plane, to trigger when
-complex conditions obtain.
-
-With binary API trace, print, and replay, system-level bug reports of
-the form "after 300,000 API transactions, the VPP data-plane stopped
-forwarding traffic, FIX IT!" can be solved offline.
-
-More often than not, one discovers that a control-plane client
-misprograms the data plane after a long time or under complex
-circumstances. Without direct evidence, "it's a data-plane problem!"
-
-See `src/vlibmemory/memory_vlib::c
-<https://docs.fd.io/vpp/18.11/dd/d3e/vpp__get__metrics_8c.html#a7c3855ed3c45b48ff92a7e881bfede73>`_
-`vl_msg_api_process_file()
-<https://docs.fd.io/vpp/18.11/d0/d5b/vlib__api__cli_8c.html#a60194e3e91c0dc6a75906ea06f4ec113>`_,
-and `src/vlibapi/api_shared.c
-<https://docs.fd.io/vpp/18.11/d6/dd1/api__shared_8c.html>`_. See also
-the debug CLI command "api trace"
-
-API trace replay caveats
-________________________
-
-The vpp instance which replays a binary API trace must have the same
-message-ID numbering space as the vpp instance which captured the
-trace. The replay instance **must** load the same set of plugins as
-the capture instance. Otherwise, API messages will be processed by the
-**wrong** API message handlers!
-
-Always start vpp with command-line arguments which include an
-"api-trace on" stanza, so vpp will start tracing binary API messages
-from the beginning:
-
-.. code-block:: console
-
- api-trace {
- on
- }
-
-Given a binary api trace in /tmp/api_trace, do the following to work
-out the set of plugins:
-
-.. code-block:: console
-
- DBGvpp# api trace dump /tmp/api_trace
- vl_api_trace_plugin_msg_ids: arp_cfdf7292 first 49 last 56
- vl_api_trace_plugin_msg_ids: ip6_nd_ac628462 first 57 last 69
- vl_api_trace_plugin_msg_ids: rd_cp_8a996e86 first 70 last 71
- <etc>
-
-Here, we see the "abf," "acl," "cdp," and "flowprobe" plugins. Use the
-list of plugins to construct a matching "plugins" command-line argument
-stanza:
-
-.. code-block:: console
-
- plugins {
- ## Disable all plugins, selectively enable specific plugins
- plugin default { disable }
- plugin abf_plugin.so { enable }
- plugin acl_plugin.so { enable }
- plugin cdp_plugin.so { enable }
- plugin flowprobe_plugin.so { enable }
- }
-
-To begin with, use the same vpp image that captured a trace to replay
-it. It's perfectly fair to rebuild the vpp replay instance, to add
-scaffolding to facilitate setting gdb breakpoints on complex
-conditions or similar.
-
-API trace interface issues
-__________________________
-
-Along the same lines, it may be necessary to manufacture [simulated]
-physical interfaces so that an API trace will replay correctly. "show
-interface" on the trace origin system can help. An API trace
-dump as shown above may make it obvious how many loopback
-interfaces to create. If you see vhost interfaces being created and
-then configured, the first such configuration message in the trace
-will tell you how many physical interfaces were involved.
-
-.. code-block:: console
-
- SCRIPT: create_vhost_user_if socket /tmp/foosock server
- SCRIPT: sw_interface_set_flags sw_if_index 3 admin-up
-
-In this case, it's fair to guess that one needs to create two loopback
-interfaces to "help" the trace replay correctly.
-
-These issues can be mitigated to a certain extent by replaying the
-trace on the system which created it, but in a field debug case that's
-not a realistic.
-
-Client connection details
-_________________________
-
-Establishing a binary API connection to VPP from a C-language client is easy:
-
-.. code-block:: C
-
- int
- connect_to_vpe (char *client_name, int client_message_queue_length)
- {
- vat_main_t *vam = &vat_main;
- api_main_t *am = &api_main;
- if (vl_client_connect_to_vlib ("/vpe-api", client_name,
- client_message_queue_length) < 0)
- return -1;
- /* Memorize vpp's binary API message input queue address */
- vam->vl_input_queue = am->shmem_hdr->vl_input_queue;
- /* And our client index */
- vam->my_client_index = am->my_client_index;
- return 0;
- }
-
-32 is a typical value for client_message_queue_length. VPP *cannot*
-block when it needs to send an API message to a binary API client. The
-VPP-side binary API message handlers are very fast. So, when sending
-asynchronous messages, make sure to scrape the binary API rx ring with
-some enthusiasm!
-
-**Binary API message RX pthread**
-
-Calling `vl_client_connect_to_vlib
-<https://docs.fd.io/vpp/18.11/da/d25/memory__client_8h.html#a6654b42c91be33bfb6a4b4bfd2327920>`_
-spins up a binary API message RX pthread:
-
-.. code-block:: C
-
- static void *
- rx_thread_fn (void *arg)
- {
- svm_queue_t *q;
- memory_client_main_t *mm = &memory_client_main;
- api_main_t *am = &api_main;
- int i;
-
- q = am->vl_input_queue;
-
- /* So we can make the rx thread terminate cleanly */
- if (setjmp (mm->rx_thread_jmpbuf) == 0)
- {
- mm->rx_thread_jmpbuf_valid = 1;
- /*
- * Find an unused slot in the per-cpu-mheaps array,
- * and grab it for this thread. We need to be able to
- * push/pop the thread heap without affecting other thread(s).
- */
- if (__os_thread_index == 0)
- {
- for (i = 0; i < ARRAY_LEN (clib_per_cpu_mheaps); i++)
- {
- if (clib_per_cpu_mheaps[i] == 0)
- {
- /* Copy the main thread mheap pointer */
- clib_per_cpu_mheaps[i] = clib_per_cpu_mheaps[0];
- __os_thread_index = i;
- break;
- }
- }
- ASSERT (__os_thread_index > 0);
- }
- while (1)
- vl_msg_api_queue_handler (q);
- }
- pthread_exit (0);
- }
-
-To handle the binary API message queue yourself, use
-`vl_client_connect_to_vlib_no_rx_pthread
-<https://docs.fd.io/vpp/18.11/da/d25/memory__client_8h.html#a11b9577297106c57c0783b96ab190c36>`_.
-
-**Queue non-empty signalling**
-
-vl_msg_api_queue_handler(...) uses mutex/condvar signalling to wake
-up, process VPP -> client traffic, then sleep. VPP supplies a condvar
-broadcast when the VPP -> client API message queue transitions from
-empty to nonempty.
-
-VPP checks its own binary API input queue at a very high rate. VPP
-invokes message handlers in "process" context [aka cooperative
-multitasking thread context] at a variable rate, depending on
-data-plane packet processing requirements.
-
-Client disconnection details
-____________________________
-
-To disconnect from VPP, call `vl_client_disconnect_from_vlib
-<https://docs.fd.io/vpp/18.11/da/d25/memory__client_8h.html#a82c9ba6e7ead8362ae2175eefcf2fd12>`_. Please
-arrange to call this function if the client application terminates
-abnormally. VPP makes every effort to hold a decent funeral for dead
-clients, but VPP can't guarantee to free leaked memory in the shared
-binary API segment.
-
-Sending binary API messages to VPP
-__________________________________
-
-The point of the exercise is to send binary API messages to VPP, and
-to receive replies from VPP. Many VPP binary APIs comprise a client
-request message, and a simple status reply. For example, to set the
-admin status of an interface:
-
-.. code-block:: C
-
- vl_api_sw_interface_set_flags_t *mp;
- mp = vl_msg_api_alloc (sizeof (*mp));
- memset (mp, 0, sizeof (*mp));
- mp->_vl_msg_id = clib_host_to_net_u16 (VL_API_SW_INTERFACE_SET_FLAGS);
- mp->client_index = api_main.my_client_index;
- mp->sw_if_index = clib_host_to_net_u32 (<interface-sw-if-index>);
- vl_msg_api_send (api_main.shmem_hdr->vl_input_queue, (u8 *)mp);
-
-Key points:
-
-* Use `vl_msg_api_alloc <https://docs.fd.io/vpp/18.11/dc/d5a/memory__shared_8h.html#a109ff1e95ebb2c968d43c100c4a1c55a>`_ to allocate message buffers
-* Allocated message buffers are not initialized, and must be presumed to contain trash.
-* Don't forget to set the _vl_msg_id field!
-* As of this writing, binary API message IDs and data are sent in network byte order
-* The client-library global data structure `api_main <https://docs.fd.io/vpp/18.11/d6/dd1/api__shared_8c.html#af58e3e46b569573e9622b826b2f47a22>`_ keeps track of sufficient pointers and handles used to communicate with VPP
-
-Receiving binary API messages from VPP
-______________________________________
-
-Unless you've made other arrangements (see
-`vl_client_connect_to_vlib_no_rx_pthread
-<https://docs.fd.io/vpp/18.11/da/d25/memory__client_8h.html#a11b9577297106c57c0783b96ab190c36>`_),
-*messages are received on a separate rx pthread*. Synchronization with
-the client application main thread is the responsibility of the
-application!
-
-Set up message handlers about as follows:
-
-.. code-block:: C
-
- #define vl_typedefs /* define message structures */
- #include <vpp/api/vpe_all_api_h.h>
- #undef vl_typedefs
- /* declare message handlers for each api */
- #define vl_endianfun /* define message structures */
- #include <vpp/api/vpe_all_api_h.h>
- #undef vl_endianfun
- /* instantiate all the print functions we know about */
- #define vl_print(handle, ...)
- #define vl_printfun
- #include <vpp/api/vpe_all_api_h.h>
- #undef vl_printfun
- /* Define a list of all message that the client handles */
- #define foreach_vpe_api_reply_msg \
- _(SW_INTERFACE_SET_FLAGS_REPLY, sw_interface_set_flags_reply)
- static clib_error_t *
- my_api_hookup (vlib_main_t * vm)
- {
- api_main_t *am = &api_main;
- #define _(N,n) \
- vl_msg_api_set_handlers(VL_API_##N, #n, \
- vl_api_##n##_t_handler, \
- vl_noop_handler, \
- vl_api_##n##_t_endian, \
- vl_api_##n##_t_print, \
- sizeof(vl_api_##n##_t), 1);
- foreach_vpe_api_msg;
- #undef _
- return 0;
- }
-
-The key API used to establish message handlers is
-`vl_msg_api_set_handlers
-<https://docs.fd.io/vpp/18.11/d6/dd1/api__shared_8c.html#aa8a8e1f3876ec1a02f283c1862ecdb7a>`_
-, which sets values in multiple parallel vectors in the `api_main_t
-<https://docs.fd.io/vpp/18.11/dd/db2/structapi__main__t.html>`_
-structure. As of this writing: not all vector element values can be
-set through the API. You'll see sporadic API message registrations
-followed by minor adjustments of this form:
-
-.. code-block:: C
-
- /*
- * Thread-safe API messages
- */
- am->is_mp_safe[VL_API_IP_ADD_DEL_ROUTE] = 1;
- am->is_mp_safe[VL_API_GET_NODE_GRAPH] = 1;
-
-API message numbering in plugins
---------------------------------
-
-Binary API message numbering in plugins relies on vpp to issue a block
-of message-ID's for the plugin to use:
-
-.. code-block:: C
-
- static clib_error_t *
- my_init (vlib_main_t * vm)
- {
- my_main_t *mm = &my_main;
-
- name = format (0, "myplugin_%08x%c", api_version, 0);
-
- /* Ask for a correctly-sized block of API message decode slots */
- mm->msg_id_base = vl_msg_api_get_msg_ids
- ((char *) name, VL_MSG_FIRST_AVAILABLE);
-
- }
-
-Control-plane codes use the vl_client_get_first_plugin_msg_id (...) api
-to recover the message ID block base:
-
-.. code-block:: C
-
- /* Ask the vpp engine for the first assigned message-id */
- name = format (0, "myplugin_%08x%c", api_version, 0);
- sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name);
-
-It's a fairly common error to forget to add msg_id_base when
-registering message handlers, or when sending messages. Using macros
-from .../src/vlibapi/api_helper_macros.h can automate the process, but
-remember to #define REPLY_MSG_ID_BASE before #including the file:
-
-.. code-block:: C
-
- #define REPLY_MSG_ID_BASE mm->msg_id_base
- #include <vlibapi/api_helper_macros.h>
diff --git a/docs/gettingstarted/developers/buildwireshark.md b/docs/gettingstarted/developers/buildwireshark.md
deleted file mode 100644
index 5d87b7a..0000000
--- a/docs/gettingstarted/developers/buildwireshark.md
+++ /dev/null
@@ -1,72 +0,0 @@
-How to build a vpp dispatch trace aware Wireshark
-=================================================
-
-The vpp pcap dispatch trace dissector has been merged into the wireshark
-main branch, so the process is simple. Download wireshark, compile it,
-and install it.
-
-Download wireshark source code
-------------------------------
-
-The wireshark git repo is large, so it takes a while to clone.
-
-```
- git clone https://code.wireshark.org/review/wireshark
-```
-
-Install prerequisite packages
-------------------------------------
-
-Here is a list of prerequisite packages which must be present in order
-to compile wireshark, beyond what's typically installed on an Ubuntu
-18.04 system:
-
-```
- libgcrypt11-dev flex bison qtbase5-dev qttools5-dev-tools qttools5-dev
- qtmultimedia5-dev libqt5svg5-dev libpcap-dev qt5-default
-```
-
-Compile Wireshark
------------------
-
-Mercifully, Wireshark uses cmake, so it's relatively easy to build, at
-least on Ubuntu 18.04.
-
-
-```
- $ cd wireshark
- $ mkdir build
- $ cd build
- $ cmake -G Ninja ../
- $ ninja -j 8
- $ sudo ninja install
-```
-
-Make a pcap dispatch trace
---------------------------
-
-Configure vpp to pass traffic in some fashion or other, and then:
-
-
-```
- vpp# pcap dispatch trace on max 10000 file vppcapture buffer-trace dpdk-input 1000
-```
-
-
-or similar. Run traffic for long enough to capture some data. Save the
-dispatch trace capture like so:
-
-```
- vpp# pcap dispatch trace off
-```
-
-Display in Wireshark
---------------------
-
-Display /tmp/vppcapture in the vpp-enabled version of wireshark. With
-any luck, normal version of wireshark will refuse to process vpp
-dispatch trace pcap files because they won't understand the encap type.
-
-Set wireshark to filter on vpp.bufferindex to watch a single packet
-traverse the forwarding graph. Otherwise, you'll see a vector of packets
-in e.g. ip4-lookup, then a vector of packets in ip4-rewrite, etc.
diff --git a/docs/gettingstarted/developers/featurearcs.md b/docs/gettingstarted/developers/featurearcs.md
deleted file mode 100644
index f1e3ec4..0000000
--- a/docs/gettingstarted/developers/featurearcs.md
+++ /dev/null
@@ -1,224 +0,0 @@
-Feature Arcs
-============
-
-A significant number of vpp features are configurable on a per-interface
-or per-system basis. Rather than ask feature coders to manually
-construct the required graph arcs, we built a general mechanism to
-manage these mechanics.
-
-Specifically, feature arcs comprise ordered sets of graph nodes. Each
-feature node in an arc is independently controlled. Feature arc nodes
-are generally unaware of each other. Handing a packet to "the next
-feature node" is quite inexpensive.
-
-The feature arc implementation solves the problem of creating graph arcs
-used for steering.
-
-At the beginning of a feature arc, a bit of setup work is needed, but
-only if at least one feature is enabled on the arc.
-
-On a per-arc basis, individual feature definitions create a set of
-ordering dependencies. Feature infrastructure performs a topological
-sort of the ordering dependencies, to determine the actual feature
-order. Missing dependencies **will** lead to runtime disorder. See
-<https://gerrit.fd.io/r/#/c/12753> for an example.
-
-If no partial order exists, vpp will refuse to run. Circular dependency
-loops of the form "a then b, b then c, c then a" are impossible to
-satisfy.
-
-Adding a feature to an existing feature arc
--------------------------------------------
-
-To nobody's great surprise, we set up feature arcs using the typical
-"macro -> constructor function -> list of declarations" pattern:
-
-```c
- VNET_FEATURE_INIT (mactime, static) =
- {
- .arc_name = "device-input",
- .node_name = "mactime",
- .runs_before = VNET_FEATURES ("ethernet-input"),
- };
-```
-
-This creates a "mactime" feature on the "device-input" arc.
-
-Once per frame, dig up the vnet\_feature\_config\_main\_t corresponding
-to the "device-input" feature arc:
-
-```c
- vnet_main_t *vnm = vnet_get_main ();
- vnet_interface_main_t *im = &vnm->interface_main;
- u8 arc = im->output_feature_arc_index;
- vnet_feature_config_main_t *fcm;
-
- fcm = vnet_feature_get_config_main (arc);
-```
-
-Note that in this case, we've stored the required arc index - assigned
-by the feature infrastructure - in the vnet\_interface\_main\_t. Where
-to put the arc index is a programmer's decision when creating a feature
-arc.
-
-Per packet, set next0 to steer packets to the next node they should
-visit:
-
-```c
- vnet_get_config_data (&fcm->config_main,
- &b0->current_config_index /* value-result */,
- &next0, 0 /* # bytes of config data */);
-```
-
-Configuration data is per-feature arc, and is often unused. Note that
-it's normal to reset next0 to divert packets elsewhere; often, to drop
-them for cause:
-
-```c
- next0 = MACTIME_NEXT_DROP;
- b0->error = node->errors[DROP_CAUSE];
-```
-
-Creating a feature arc
-----------------------
-
-Once again, we create feature arcs using constructor macros:
-
-```c
- VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
- {
- .arc_name = "ip4-unicast",
- .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
- .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
- };
-```
-
-In this case, we configure two arc start nodes to handle the
-"hardware-verified ip checksum or not" cases. During initialization,
-the feature infrastructure stores the arc index as shown.
-
-In the head-of-arc node, do the following to send packets along the
-feature arc:
-
-```c
- ip_lookup_main_t *lm = &im->lookup_main;
- arc = lm->ucast_feature_arc_index;
-```
-
-Once per packet, initialize packet metadata to walk the feature arc:
-
-```c
-vnet_feature_arc_start (arc, sw_if_index0, &next, b0);
-```
-
-Enabling / Disabling features
------------------------------
-
-Simply call vnet_feature_enable_disable to enable or disable a specific
-feature:
-
-```c
- vnet_feature_enable_disable ("device-input", /* arc name */
- "mactime", /* feature name */
- sw_if_index, /* Interface sw_if_index */
- enable_disable, /* 1 => enable */
- 0 /* (void *) feature_configuration */,
- 0 /* feature_configuration_nbytes */);
-```
-
-The feature_configuration opaque is seldom used.
-
-If you wish to make a feature a _de facto_ system-level concept, pass
-sw_if_index=0 at all times. Sw_if_index 0 is always valid, and
-corresponds to the "local" interface.
-
-Related "show" commands
------------------------
-
-To display the entire set of features, use "show features [verbose]". The
-verbose form displays arc indices, and feature indicies within the arcs
-
-```
-$ vppctl show features verbose
-Available feature paths
-<snip>
-[14] ip4-unicast:
- [ 0]: nat64-out2in-handoff
- [ 1]: nat64-out2in
- [ 2]: nat44-ed-hairpin-dst
- [ 3]: nat44-hairpin-dst
- [ 4]: ip4-dhcp-client-detect
- [ 5]: nat44-out2in-fast
- [ 6]: nat44-in2out-fast
- [ 7]: nat44-handoff-classify
- [ 8]: nat44-out2in-worker-handoff
- [ 9]: nat44-in2out-worker-handoff
- [10]: nat44-ed-classify
- [11]: nat44-ed-out2in
- [12]: nat44-ed-in2out
- [13]: nat44-det-classify
- [14]: nat44-det-out2in
- [15]: nat44-det-in2out
- [16]: nat44-classify
- [17]: nat44-out2in
- [18]: nat44-in2out
- [19]: ip4-qos-record
- [20]: ip4-vxlan-gpe-bypass
- [21]: ip4-reassembly-feature
- [22]: ip4-not-enabled
- [23]: ip4-source-and-port-range-check-rx
- [24]: ip4-flow-classify
- [25]: ip4-inacl
- [26]: ip4-source-check-via-rx
- [27]: ip4-source-check-via-any
- [28]: ip4-policer-classify
- [29]: ipsec-input-ip4
- [30]: vpath-input-ip4
- [31]: ip4-vxlan-bypass
- [32]: ip4-lookup
-<snip>
-```
-
-Here, we learn that the ip4-unicast feature arc has index 14, and that
-e.g. ip4-inacl is the 25th feature in the generated partial order.
-
-To display the features currently active on a specific interface,
-use "show interface <name> features":
-
-```
-$ vppctl show interface GigabitEthernet3/0/0 features
-Feature paths configured on GigabitEthernet3/0/0...
-<snip>
-ip4-unicast:
- nat44-out2in
-<snip>
-```
-
-Table of Feature Arcs
----------------------
-
-Simply search for name-strings to track down the arc definition, location of
-the arc index, etc.
-
-```
- | Arc Name |
- |------------------|
- | device-input |
- | ethernet-output |
- | interface-output |
- | ip4-drop |
- | ip4-local |
- | ip4-multicast |
- | ip4-output |
- | ip4-punt |
- | ip4-unicast |
- | ip6-drop |
- | ip6-local |
- | ip6-multicast |
- | ip6-output |
- | ip6-punt |
- | ip6-unicast |
- | mpls-input |
- | mpls-output |
- | nsh-output |
-```
diff --git a/docs/gettingstarted/developers/index.rst b/docs/gettingstarted/developers/index.rst
deleted file mode 100644
index b47abcd..0000000
--- a/docs/gettingstarted/developers/index.rst
+++ /dev/null
@@ -1,48 +0,0 @@
-.. _gstarteddevel:
-
-###############
-For Developers
-###############
-
-The Developers section covers the following areas:
-
-* Describes how to build different types of VPP images
-* Explains how to run VPP with and without GDB, with some GDB examples
-* Describes the steps required to get a patch reviewed and merged
-* Describes the VPP software architecture and identifies the associated four VPP layers
-* Describes the different components that are associated with each VPP layer
-* Explains how to Create, Add, Enable/Disable different ARC features
-* Discusses different aspects of Bounded-index Extensible Hashing (bihash), and how it is used in database lookups
-* Describes the different types of API support and how to integrate a plugin
-
-.. toctree::
- :maxdepth: 2
-
- building
- running_vpp
- testing_vpp
- gdb_examples
- add_plugin
- add_plugin_goapi
- gitreview
- softwarearchitecture
- infrastructure
- vlib
- plugins
- vnet
- featurearcs
- metadata
- multiarch/index.rst
- bihash
- vpp_api_module
- binary_api_support
- buildsystem/index.rst
- eventviewer
- fib20/index.rst
- buildwireshark
- punt
- quic_plugin
- cross_compile_macos.rst
- cnat
- ipsec
- VPPAPI.md
diff --git a/docs/gettingstarted/developers/infrastructure.md b/docs/gettingstarted/developers/infrastructure.md
deleted file mode 100644
index a61068c..0000000
--- a/docs/gettingstarted/developers/infrastructure.md
+++ /dev/null
@@ -1,614 +0,0 @@
-VPPINFRA (Infrastructure)
-=========================
-
-The files associated with the VPP Infrastructure layer are located in
-the ./src/vppinfra folder.
-
-VPPinfra is a collection of basic c-library services, quite
-sufficient to build standalone programs to run directly on bare metal.
-It also provides high-performance dynamic arrays, hashes, bitmaps,
-high-precision real-time clock support, fine-grained event-logging, and
-data structure serialization.
-
-One fair comment / fair warning about vppinfra: you can\'t always tell a
-macro from an inline function from an ordinary function simply by name.
-Macros are used to avoid function calls in the typical case, and to
-cause (intentional) side-effects.
-
-Vppinfra has been around for almost 20 years and tends not to change
-frequently. The VPP Infrastructure layer contains the following
-functions:
-
-Vectors
--------
-
-Vppinfra vectors are ubiquitous dynamically resized arrays with by user
-defined \"headers\". Many vpppinfra data structures (e.g. hash, heap,
-pool) are vectors with various different headers.
-
-The memory layout looks like this:
-
-```
- User header (optional, uword aligned)
- Alignment padding (if needed)
- Vector length in elements
- User's pointer -> Vector element 0
- Vector element 1
- ...
- Vector element N-1
-```
-
-As shown above, the vector APIs deal with pointers to the 0th element of
-a vector. Null pointers are valid vectors of length zero.
-
-To avoid thrashing the memory allocator, one often resets the length of
-a vector to zero while retaining the memory allocation. Set the vector
-length field to zero via the vec\_reset\_length(v) macro. \[Use the
-macro! It's smart about NULL pointers.\]
-
-Typically, the user header is not present. User headers allow for other
-data structures to be built atop vppinfra vectors. Users may specify the
-alignment for first data element of a vector via the \[vec\]()\*\_aligned
-macros.
-
-Vector elements can be any C type e.g. (int, double, struct bar). This
-is also true for data types built atop vectors (e.g. heap, pool, etc.).
-Many macros have \_a variants supporting alignment of vector elements
-and \_h variants supporting non-zero-length vector headers. The \_ha
-variants support both. Additionally cacheline alignment within a
-vector element structure can be specified using the
-\[CLIB_CACHE_LINE_ALIGN_MARK\]() macro.
-
-Inconsistent usage of header and/or alignment related macro variants
-will cause delayed, confusing failures.
-
-Standard programming error: memorize a pointer to the ith element of a
-vector, and then expand the vector. Vectors expand by 3/2, so such code
-may appear to work for a period of time. Correct code almost always
-memorizes vector **indices** which are invariant across reallocations.
-
-In typical application images, one supplies a set of global functions
-designed to be called from gdb. Here are a few examples:
-
-- vl(v) - prints vec\_len(v)
-- pe(p) - prints pool\_elts(p)
-- pifi(p, index) - prints pool\_is\_free\_index(p, index)
-- debug\_hex\_bytes (p, nbytes) - hex memory dump nbytes starting at p
-
-Use the "show gdb" debug CLI command to print the current set.
-
-Bitmaps
--------
-
-Vppinfra bitmaps are dynamic, built using the vppinfra vector APIs.
-Quite handy for a variety jobs.
-
-Pools
------
-
-Vppinfra pools combine vectors and bitmaps to rapidly allocate and free
-fixed-size data structures with independent lifetimes. Pools are perfect
-for allocating per-session structures.
-
-Hashes
-------
-
-Vppinfra provides several hash flavors. Data plane problems involving
-packet classification / session lookup often use
-./src/vppinfra/bihash\_template.\[ch\] bounded-index extensible
-hashes. These templates are instantiated multiple times, to efficiently
-service different fixed-key sizes.
-
-Bihashes are thread-safe. Read-locking is not required. A simple
-spin-lock ensures that only one thread writes an entry at a time.
-
-The original vppinfra hash implementation in
-./src/vppinfra/hash.\[ch\] are simple to use, and are often used in
-control-plane code which needs exact-string-matching.
-
-In either case, one almost always looks up a key in a hash table to
-obtain an index in a related vector or pool. The APIs are simple enough,
-but one must take care when using the unmanaged arbitrary-sized key
-variant. Hash\_set\_mem (hash\_table, key\_pointer, value) memorizes
-key\_pointer. It is usually a bad mistake to pass the address of a
-vector element as the second argument to hash\_set\_mem. It is perfectly
-fine to memorize constant string addresses in the text segment.
-
-Timekeeping
------------
-
-Vppinfra includes high-precision, low-cost timing services. The
-datatype clib_time_t and associated functions reside in
-./src/vppinfra/time.\[ch\]. Call clib_time_init (clib_time_t \*cp) to
-initialize the clib_time_t object.
-
-Clib_time_init(...) can use a variety of different ways to establish
-the hardware clock frequency. At the end of the day, vppinfra
-timekeeping takes the attitude that the operating system's clock is
-the closest thing to a gold standard it has handy.
-
-When properly configured, NTP maintains kernel clock synchronization
-with a highly accurate off-premises reference clock. Notwithstanding
-network propagation delays, a synchronized NTP client will keep the
-kernel clock accurate to within 50ms or so.
-
-Why should one care? Simply put, oscillators used to generate CPU
-ticks aren't super accurate. They work pretty well, but a 0.1% error
-wouldn't be out of the question. That's a minute and a half's worth of
-error in 1 day. The error changes constantly, due to temperature
-variation, and a host of other physical factors.
-
-It's far too expensive to use system calls for timing, so we're left
-with the problem of continously adjusting our view of the CPU tick
-register's clocks_per_second parameter.
-
-The clock rate adjustment algorithm measures the number of cpu ticks
-and the "gold standard" reference time across an interval of
-approximately 16 seconds. We calculate clocks_per_second for the
-interval: use rdtsc (on x86_64) and a system call to get the latest
-cpu tick count and the kernel's latest nanosecond timestamp. We
-subtract the previous interval end values, and use exponential
-smoothing to merge the new clock rate sample into the clocks_per_second
-parameter.
-
-As of this writing, we maintain the clock rate by way of the following
-first-order differential equation:
-
-
-```
- clocks_per_second(t) = clocks_per_second(t-1) * K + sample_cps(t)*(1-K)
- where K = e**(-1.0/3.75);
-```
-
-This yields a per observation "half-life" of 1 minute. Empirically,
-the clock rate converges within 5 minutes, and appears to maintain
-near-perfect agreement with the kernel clock in the face of ongoing
-NTP time adjustments.
-
-See ./src/vppinfra/time.c:clib_time_verify_frequency(...) to look at
-the rate adjustment algorithm. The code rejects frequency samples
-corresponding to the sort of adjustment which might occur if someone
-changes the gold standard kernel clock by several seconds.
-
-### Monotonic timebase support
-
-Particularly during system initialization, the "gold standard" system
-reference clock can change by a large amount, in an instant. It's not
-a best practice to yank the reference clock - in either direction - by
-hours or days. In fact, some poorly-constructed use-cases do so.
-
-To deal with this reality, clib_time_now(...) returns the number of
-seconds since vpp started, *guaranteed to be monotonically
-increasing, no matter what happens to the system reference clock*.
-
-This is first-order important, to avoid breaking every active timer in
-the system. The vpp host stack alone may account for tens of millions
-of active timers. It's utterly impractical to track down and fix
-timers, so we must deal with the issue at the timebase level.
-
-Here's how it works. Prior to adjusting the clock rate, we collect the
-kernel reference clock and the cpu clock:
-
-```
- /* Ask the kernel and the CPU what time it is... */
- now_reference = unix_time_now ();
- now_clock = clib_cpu_time_now ();
-```
-
-Compute changes for both clocks since the last rate adjustment,
-roughly 15 seconds ago:
-
-```
- /* Compute change in the reference clock */
- delta_reference = now_reference - c->last_verify_reference_time;
-
- /* And change in the CPU clock */
- delta_clock_in_seconds = (f64) (now_clock - c->last_verify_cpu_time) *
- c->seconds_per_clock;
-```
-
-Delta_reference is key. Almost 100% of the time, delta_reference and
-delta_clock_in_seconds are identical modulo one system-call
-time. However, NTP or a privileged user can yank the system reference
-time - in either direction - by an hour, a day, or a decade.
-
-As described above, clib_time_now(...) must return monotonically
-increasing answers to the question "how long has it been since vpp
-started, in seconds." To do that, the clock rate adjustment algorithm
-begins by recomputing the initial reference time:
-
-```
- c->init_reference_time += (delta_reference - delta_clock_in_seconds);
-```
-
-It's easy to convince yourself that if the reference clock changes by
-15.000000 seconds and the cpu clock tick time changes by 15.000000
-seconds, the initial reference time won't change.
-
-If, on the other hand, delta_reference is -86400.0 and delta clock is
-15.0 - reference time jumped backwards by exactly one day in a
-15-second rate update interval - we add -86415.0 to the initial
-reference time.
-
-Given the corrected initial reference time, we recompute the total
-number of cpu ticks which have occurred since the corrected initial
-reference time, at the current clock tick rate:
-
-```
- c->total_cpu_time = (now_reference - c->init_reference_time)
- * c->clocks_per_second;
-```
-
-### Timebase precision
-
-Cognoscenti may notice that vlib/clib\_time\_now(...) return a 64-bit
-floating-point value; the number of seconds since vpp started.
-
-Please see [this Wikipedia
-article](https://en.wikipedia.org/wiki/Double-precision_floating-point_format)
-for more information. C double-precision floating point numbers
-(called f64 in the vpp code base) have a 53-bit effective mantissa,
-and can accurately represent 15 decimal digits' worth of precision.
-
-There are 315,360,000.000001 seconds in ten years plus one
-microsecond. That string has exactly 15 decimal digits. The vpp time
-base retains 1us precision for roughly 30 years.
-
-vlib/clib\_time\_now do *not* provide precision in excess of 1e-6
-seconds. If necessary, please use clib_cpu_time_now(...) for direct
-access to the CPU clock-cycle counter. Note that the number of CPU
-clock cycles per second varies significantly across CPU architectures.
-
-Timer Wheels
-------------
-
-Vppinfra includes configurable timer wheel support. See the source
-code in .../src/vppinfra/tw_timer_template.[ch], as well as a
-considerable number of template instances defined in
-.../src/vppinfra/tw_timer_<wheel-geometry-spec>.[ch].
-
-Instantiation of tw_timer_template.h generates named structures to
-implement specific timer wheel geometries. Choices include: number of
-timer wheels (currently, 1 or 2), number of slots per ring (a power of
-two), and the number of timers per "object handle".
-
-Internally, user object/timer handles are 32-bit integers, so if one
-selects 16 timers/object (4 bits), the resulting timer wheel handle is
-limited to 2**28 objects.
-
-Here are the specific settings required to generate a single 2048 slot
-wheel which supports 2 timers per object:
-
-```
- #define TW_TIMER_WHEELS 1
- #define TW_SLOTS_PER_RING 2048
- #define TW_RING_SHIFT 11
- #define TW_RING_MASK (TW_SLOTS_PER_RING -1)
- #define TW_TIMERS_PER_OBJECT 2
- #define LOG2_TW_TIMERS_PER_OBJECT 1
- #define TW_SUFFIX _2t_1w_2048sl
- #define TW_FAST_WHEEL_BITMAP 0
- #define TW_TIMER_ALLOW_DUPLICATE_STOP 0
-```
-
-See tw_timer_2t_1w_2048sl.h for a complete
-example.
-
-tw_timer_template.h is not intended to be #included directly. Client
-codes can include multiple timer geometry header files, although
-extreme caution would required to use the TW and TWT macros in such a
-case.
-
-### API usage examples
-
-The unit test code in .../src/vppinfra/test_tw_timer.c provides a
-concrete API usage example. It uses a synthetic clock to rapidly
-exercise the underlying tw_timer_expire_timers(...) template.
-
-There are not many API routines to call.
-
-#### Initialize a two-timer, single 2048-slot wheel w/ a 1-second timer granularity
-
-```
- tw_timer_wheel_init_2t_1w_2048sl (&tm->single_wheel,
- expired_timer_single_callback,
- 1.0 / * timer interval * / );
-```
-
-#### Start a timer
-
-```
- handle = tw_timer_start_2t_1w_2048sl (&tm->single_wheel, elt_index,
- [0 | 1] / * timer id * / ,
- expiration_time_in_u32_ticks);
-```
-
-#### Stop a timer
-
-```
- tw_timer_stop_2t_1w_2048sl (&tm->single_wheel, handle);
-```
-
-#### An expired timer callback
-
-```
- static void
- expired_timer_single_callback (u32 * expired_timers)
- {
- int i;
- u32 pool_index, timer_id;
- tw_timer_test_elt_t *e;
- tw_timer_test_main_t *tm = &tw_timer_test_main;
-
- for (i = 0; i < vec_len (expired_timers);
- {
- pool_index = expired_timers[i] & 0x7FFFFFFF;
- timer_id = expired_timers[i] >> 31;
-
- ASSERT (timer_id == 1);
-
- e = pool_elt_at_index (tm->test_elts, pool_index);
-
- if (e->expected_to_expire != tm->single_wheel.current_tick)
- {
- fformat (stdout, "[%d] expired at %d not %d\n",
- e - tm->test_elts, tm->single_wheel.current_tick,
- e->expected_to_expire);
- }
- pool_put (tm->test_elts, e);
- }
- }
-```
-
-We use wheel timers extensively in the vpp host stack. Each TCP
-session needs 5 timers, so supporting 10 million flows requires up to
-50 million concurrent timers.
-
-Timers rarely expire, so it's of utmost important that stopping and
-restarting a timer costs as few clock cycles as possible.
-
-Stopping a timer costs a doubly-linked list dequeue. Starting a timer
-involves modular arithmetic to determine the correct timer wheel and
-slot, and a list head enqueue.
-
-Expired timer processing generally involves bulk link-list retirement
-with user callback presentation. Some additional complexity at wheel
-wrap time, to relocate timers from slower-turning timer wheels into
-faster-turning wheels.
-
-Format
-------
-
-Vppinfra format is roughly equivalent to printf.
-
-Format has a few properties worth mentioning. Format's first argument is
-a (u8 \*) vector to which it appends the result of the current format
-operation. Chaining calls is very easy:
-
-```c
- u8 * result;
-
- result = format (0, "junk = %d, ", junk);
- result = format (result, "more junk = %d\n", more_junk);
-```
-
-As previously noted, NULL pointers are perfectly proper 0-length
-vectors. Format returns a (u8 \*) vector, **not** a C-string. If you
-wish to print a (u8 \*) vector, use the "%v" format string. If you need
-a (u8 \*) vector which is also a proper C-string, either of these
-schemes may be used:
-
-```c
- vec_add1 (result, 0)
- or
- result = format (result, "<whatever>%c", 0);
-```
-
-Remember to vec\_free() the result if appropriate. Be careful not to
-pass format an uninitialized (u8 \*).
-
-Format implements a particularly handy user-format scheme via the "%U"
-format specification. For example:
-
-```c
- u8 * format_junk (u8 * s, va_list *va)
- {
- junk = va_arg (va, u32);
- s = format (s, "%s", junk);
- return s;
- }
-
- result = format (0, "junk = %U, format_junk, "This is some junk");
-```
-
-format\_junk() can invoke other user-format functions if desired. The
-programmer shoulders responsibility for argument type-checking. It is
-typical for user format functions to blow up spectacularly if the
-va\_arg(va, type) macros don't match the caller's idea of reality.
-
-Unformat
---------
-
-Vppinfra unformat is vaguely related to scanf, but considerably more
-general.
-
-A typical use case involves initializing an unformat\_input\_t from
-either a C-string or a (u8 \*) vector, then parsing via unformat() as
-follows:
-
-```c
- unformat_input_t input;
- u8 *s = "<some-C-string>";
-
- unformat_init_string (&input, (char *) s, strlen((char *) s));
- /* or */
- unformat_init_vector (&input, <u8-vector>);
-```
-
-Then loop parsing individual elements:
-
-```c
- while (unformat_check_input (&input) != UNFORMAT_END_OF_INPUT)
- {
- if (unformat (&input, "value1 %d", &value1))
- ;/* unformat sets value1 */
- else if (unformat (&input, "value2 %d", &value2)
- ;/* unformat sets value2 */
- else
- return clib_error_return (0, "unknown input '%U'",
- format_unformat_error, input);
- }
-```
-
-As with format, unformat implements a user-unformat function capability
-via a "%U" user unformat function scheme. Generally, one can trivially
-transform "format (s, "foo %d", foo) -> "unformat (input, "foo %d", &foo)".
-
-Unformat implements a couple of handy non-scanf-like format specifiers:
-
-```c
- unformat (input, "enable %=", &enable, 1 /* defaults to 1 */);
- unformat (input, "bitzero %|", &mask, (1<<0));
- unformat (input, "bitone %|", &mask, (1<<1));
- <etc>
-```
-
-The phrase "enable %=" means "set the supplied variable to the default
-value" if unformat parses the "enable" keyword all by itself. If
-unformat parses "enable 123" set the supplied variable to 123.
-
-We could clean up a number of hand-rolled "verbose" + "verbose %d"
-argument parsing codes using "%=".
-
-The phrase "bitzero %|" means "set the specified bit in the supplied
-bitmask" if unformat parses "bitzero". Although it looks like it could
-be fairly handy, it's very lightly used in the code base.
-
-`%_` toggles whether or not to skip input white space.
-
-For transition from skip to no-skip in middle of format string, skip input white space. For example, the following:
-
-```c
-fmt = "%_%d.%d%_->%_%d.%d%_"
-unformat (input, fmt, &one, &two, &three, &four);
-```
-matches input "1.2 -> 3.4".
-Without this, the space after -> does not get skipped.
-
-
-```
-
-### How to parse a single input line
-
-Debug CLI command functions MUST NOT accidentally consume input
-belonging to other debug CLI commands. Otherwise, it's impossible to
-script a set of debug CLI commands which "work fine" when issued one
-at a time.
-
-This bit of code is NOT correct:
-
-```c
- /* Eats script input NOT beloging to it, and chokes! */
- while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
- {
- if (unformat (input, ...))
- ;
- else if (unformat (input, ...))
- ;
- else
- return clib_error_return (0, "parse error: '%U'",
- format_unformat_error, input);
- }
- }
-```
-
-When executed as part of a script, such a function will return "parse
-error: '<next-command-text>'" every time, unless it happens to be the
-last command in the script.
-
-Instead, use "unformat_line_input" to consume the rest of a line's
-worth of input - everything past the path specified in the
-VLIB_CLI_COMMAND declaration.
-
-For example, unformat_line_input with "my_command" set up as shown
-below and user input "my path is clear" will produce an
-unformat_input_t that contains "is clear".
-
-```c
- VLIB_CLI_COMMAND (...) = {
- .path = "my path",
- };
-```
-
-Here's a bit of code which shows the required mechanics, in full:
-
-```c
- static clib_error_t *
- my_command_fn (vlib_main_t * vm,
- unformat_input_t * input,
- vlib_cli_command_t * cmd)
- {
- unformat_input_t _line_input, *line_input = &_line_input;
- u32 this, that;
- clib_error_t *error = 0;
-
- if (!unformat_user (input, unformat_line_input, line_input))
- return 0;
-
- /*
- * Here, UNFORMAT_END_OF_INPUT is at the end of the line we consumed,
- * not at the end of the script...
- */
- while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
- {
- if (unformat (line_input, "this %u", &this))
- ;
- else if (unformat (line_input, "that %u", &that))
- ;
- else
- {
- error = clib_error_return (0, "parse error: '%U'",
- format_unformat_error, line_input);
- goto done;
- }
- }
-
- <do something based on "this" and "that", etc>
-
- done:
- unformat_free (line_input);
- return error;
- }
- /* *INDENT-OFF* */
- VLIB_CLI_COMMAND (my_command, static) = {
- .path = "my path",
- .function = my_command_fn",
- };
- /* *INDENT-ON* */
-
-```
-
-
-Vppinfra errors and warnings
-----------------------------
-
-Many functions within the vpp dataplane have return-values of type
-clib\_error\_t \*. Clib\_error\_t's are arbitrary strings with a bit of
-metadata \[fatal, warning\] and are easy to announce. Returning a NULL
-clib\_error\_t \* indicates "A-OK, no error."
-
-Clib\_warning(format-args) is a handy way to add debugging
-output; clib warnings prepend function:line info to unambiguously locate
-the message source. Clib\_unix\_warning() adds perror()-style Linux
-system-call information. In production images, clib\_warnings result in
-syslog entries.
-
-Serialization
--------------
-
-Vppinfra serialization support allows the programmer to easily serialize
-and unserialize complex data structures.
-
-The underlying primitive serialize/unserialize functions use network
-byte-order, so there are no structural issues serializing on a
-little-endian host and unserializing on a big-endian host.
diff --git a/docs/gettingstarted/developers/metadata.md b/docs/gettingstarted/developers/metadata.md
deleted file mode 100644
index 34400ef..0000000
--- a/docs/gettingstarted/developers/metadata.md
+++ /dev/null
@@ -1,190 +0,0 @@
-Buffer Metadata
-===============
-
-Each vlib_buffer_t (packet buffer) carries buffer metadata which
-describes the current packet-processing state. The underlying
-techniques have been used for decades, across multiple packet
-processing environments.
-
-We will examine vpp buffer metadata in some detail, but folks who need
-to manipulate and/or extend the scheme should expect to do a certain
-level of code inspection.
-
-Vlib (Vector library) primary buffer metadata
-----------------------------------------------
-
-The first 64 octets of each vlib_buffer_t carries the primary buffer
-metadata. See .../src/vlib/buffer.h for full details.
-
-Important fields:
-
-* i16 current_data: the signed offset in data[], pre_data[] that we
-are currently processing. If negative current header points into
-the pre-data (rewrite space) area.
-* u16 current_length: nBytes between current_data and the end of this buffer.
-* u32 flags: Buffer flag bits. Heavily used, not many bits left
- * src/vlib/buffer.h flag bits
- * VLIB_BUFFER_IS_TRACED: buffer is traced
- * VLIB_BUFFER_NEXT_PRESENT: buffer has multiple chunks
- * VLIB_BUFFER_TOTAL_LENGTH_VALID: total_length_not_including_first_buffer is valid (see below)
- * src/vnet/buffer.h flag bits
- * VNET_BUFFER_F_L4_CHECKSUM_COMPUTED: tcp/udp checksum has been computed
- * VNET_BUFFER_F_L4_CHECKSUM_CORRECT: tcp/udp checksum is correct
- * VNET_BUFFER_F_VLAN_2_DEEP: two vlan tags present
- * VNET_BUFFER_F_VLAN_1_DEEP: one vlan tag present
- * VNET_BUFFER_F_SPAN_CLONE: packet has already been cloned (span feature)
- * VNET_BUFFER_F_LOOP_COUNTER_VALID: packet look-up loop count valid
- * VNET_BUFFER_F_LOCALLY_ORIGINATED: packet built by vpp
- * VNET_BUFFER_F_IS_IP4: packet is ipv4, for checksum offload
- * VNET_BUFFER_F_IS_IP6: packet is ipv6, for checksum offload
- * VNET_BUFFER_F_OFFLOAD_IP_CKSUM: hardware ip checksum offload requested
- * VNET_BUFFER_F_OFFLOAD_TCP_CKSUM: hardware tcp checksum offload requested
- * VNET_BUFFER_F_OFFLOAD_UDP_CKSUM: hardware udp checksum offload requested
- * VNET_BUFFER_F_IS_NATED: natted packet, skip input checks
- * VNET_BUFFER_F_L2_HDR_OFFSET_VALID: L2 header offset valid
- * VNET_BUFFER_F_L3_HDR_OFFSET_VALID: L3 header offset valid
- * VNET_BUFFER_F_L4_HDR_OFFSET_VALID: L4 header offset valid
- * VNET_BUFFER_F_FLOW_REPORT: packet is an ipfix packet
- * VNET_BUFFER_F_IS_DVR: packet to be reinjected into the l2 output path
- * VNET_BUFFER_F_QOS_DATA_VALID: QoS data valid in vnet_buffer_opaque2
- * VNET_BUFFER_F_GSO: generic segmentation offload requested
- * VNET_BUFFER_F_AVAIL1: available bit
- * VNET_BUFFER_F_AVAIL2: available bit
- * VNET_BUFFER_F_AVAIL3: available bit
- * VNET_BUFFER_F_AVAIL4: available bit
- * VNET_BUFFER_F_AVAIL5: available bit
- * VNET_BUFFER_F_AVAIL6: available bit
- * VNET_BUFFER_F_AVAIL7: available bit
-* u32 flow_id: generic flow identifier
-* u8 ref_count: buffer reference / clone count (e.g. for span replication)
-* u8 buffer_pool_index: buffer pool index which owns this buffer
-* vlib_error_t (u16) error: error code for buffers enqueued to error handler
-* u32 next_buffer: buffer index of next buffer in chain. Only valid if VLIB_BUFFER_NEXT_PRESENT is set
-* union
- * u32 current_config_index: current index on feature arc
- * u32 punt_reason: reason code once packet punted. Mutually exclusive with current_config_index
-* u32 opaque[10]: primary vnet-layer opaque data (see below)
-* END of first cache line / data initialized by the buffer allocator
-* u32 trace_index: buffer's index in the packet trace subsystem
-* u32 total_length_not_including_first_buffer: see VLIB_BUFFER_TOTAL_LENGTH_VALID above
-* u32 opaque2[14]: secondary vnet-layer opaque data (see below)
-* u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]: rewrite space, often used to prepend tunnel encapsulations
-* u8 data[0]: buffer data received from the wire. Ordinarily, hardware devices use b->data[0] as the DMA target but there are exceptions. Do not write code which blindly assumes that packet data starts in b->data[0]. Use vlib_buffer_get_current(...).
-
-Vnet (network stack) primary buffer metadata
---------------------------------------------
-
-Vnet primary buffer metadata occupies space reserved in the vlib
-opaque field shown above, and has the type name
-vnet_buffer_opaque_t. Ordinarily accessed using the vnet_buffer(b)
-macro. See ../src/vnet/buffer.h for full details.
-
-Important fields:
-
-* u32 sw_if_index[2]: RX and TX interface handles. At the ip lookup
- stage, vnet_buffer(b)->sw_if_index[VLIB_TX] is interpreted as a FIB
- index.
-* i16 l2_hdr_offset: offset from b->data[0] of the packet L2 header.
- Valid only if b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID is set
-* i16 l3_hdr_offset: offset from b->data[0] of the packet L3 header.
- Valid only if b->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID is set
-* i16 l4_hdr_offset: offset from b->data[0] of the packet L4 header.
- Valid only if b->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID is set
-* u8 feature_arc_index: feature arc that the packet is currently traversing
-* union
- * ip
- * u32 adj_index[2]: adjacency from dest IP lookup in [VLIB_TX], adjacency
- from source ip lookup in [VLIB_RX], set to ~0 until source lookup done
- * union
- * generic fields
- * ICMP fields
- * reassembly fields
- * mpls fields
- * l2 bridging fields, only valid in the L2 path
- * l2tpv3 fields
- * l2 classify fields
- * vnet policer fields
- * MAP fields
- * MAP-T fields
- * ip fragmentation fields
- * COP (whitelist/blacklist filter) fields
- * LISP fields
- * TCP fields
- * connection index
- * sequence numbers
- * header and data offsets
- * data length
- * flags
- * SCTP fields
- * NAT fields
- * u32 unused[6]
-
-Vnet (network stack) secondary buffer metadata
------------------------------------------------
-
-Vnet primary buffer metadata occupies space reserved in the vlib
-opaque2 field shown above, and has the type name
-vnet_buffer_opaque2_t. Ordinarily accessed using the vnet_buffer2(b)
-macro. See ../src/vnet/buffer.h for full details.
-
-Important fields:
-
-* qos fields
- * u8 bits
- * u8 source
-* u8 loop_counter: used to detect and report internal forwarding loops
-* group-based policy fields
- * u8 flags
- * u16 sclass: the packet's source class
-* u16 gso_size: L4 payload size, persists all the way to
- interface-output in case GSO is not enabled
-* u16 gso_l4_hdr_sz: size of the L4 protocol header
-* union
- * packet trajectory tracer (largely deprecated)
- * u16 *trajectory_trace; only #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
- * packet generator
- * u64 pg_replay_timestamp: timestamp for replayed pcap trace packets
- * u32 unused[8]
-
-Buffer Metadata Extensions
-==========================
-
-Plugin developers may wish to extend either the primary or secondary
-vnet buffer opaque unions. Please perform a
-manual live variable analysis, otherwise nodes which use shared buffer metadata space may break things.
-
-It's not OK to add plugin or proprietary metadata to the core vpp
-engine header files named above. Instead, proceed as follows. The
-example concerns the vnet primary buffer opaque union
-vlib_buffer_opaque_t. It's a very simple variation to use the vnet
-secondary buffer opaque union vlib_buffer_opaque2_t.
-
-In a plugin header file:
-
-```
- /* Add arbitrary buffer metadata */
- #include <vnet/buffer.h>
-
- typedef struct
- {
- u32 my_stuff[6];
- } my_buffer_opaque_t;
-
- STATIC_ASSERT (sizeof (my_buffer_opaque_t) <=
- STRUCT_SIZE_OF (vnet_buffer_opaque_t, unused),
- "Custom meta-data too large for vnet_buffer_opaque_t");
-
- #define my_buffer_opaque(b) \
- ((my_buffer_opaque_t *)((u8 *)((b)->opaque) + STRUCT_OFFSET_OF (vnet_buffer_opaque_t, unused)))
-```
-To set data in the custom buffer opaque type given a vlib_buffer_t *b:
-
-```
- my_buffer_opaque (b)->my_stuff[2] = 123;
-```
-
-To read data from the custom buffer opaque type:
-
-```
- stuff0 = my_buffer_opaque (b)->my_stuff[2];
-```
diff --git a/docs/gettingstarted/developers/plugins.rst b/docs/gettingstarted/developers/plugins.rst
deleted file mode 100644
index 09db1d3..0000000
--- a/docs/gettingstarted/developers/plugins.rst
+++ /dev/null
@@ -1,13 +0,0 @@
-.. _dplugins:
-
-Plugins
-=======
-
-vlib implements a straightforward plug-in DLL mechanism. VLIB client
-applications specify a directory to search for plug-in .DLLs, and a name
-filter to apply (if desired). VLIB needs to load plug-ins very early.
-
-Once loaded, the plug-in DLL mechanism uses dlsym to find and verify a
-vlib\_plugin\_registration data structure in the newly-loaded plug-in.
-
-For more on plugins please refer to :ref:`add_plugin`.
diff --git a/docs/gettingstarted/developers/softwarearchitecture.md b/docs/gettingstarted/developers/softwarearchitecture.md
deleted file mode 100644
index a663134..0000000
--- a/docs/gettingstarted/developers/softwarearchitecture.md
+++ /dev/null
@@ -1,44 +0,0 @@
-Software Architecture
-=====================
-
-The fd.io vpp implementation is a third-generation vector packet
-processing implementation specifically related to US Patent 7,961,636,
-as well as earlier work. Note that the Apache-2 license specifically
-grants non-exclusive patent licenses; we mention this patent as a point
-of historical interest.
-
-For performance, the vpp dataplane consists of a directed graph of
-forwarding nodes which process multiple packets per invocation. This
-schema enables a variety of micro-processor optimizations: pipelining
-and prefetching to cover dependent read latency, inherent I-cache phase
-behavior, vector instructions. Aside from hardware input and hardware
-output nodes, the entire forwarding graph is portable code.
-
-Depending on the scenario at hand, we often spin up multiple worker
-threads which process ingress-hashes packets from multiple queues using
-identical forwarding graph replicas.
-
-VPP Layers - Implementation Taxonomy
-------------------------------------
-
-
-
-- VPP Infra - the VPP infrastructure layer, which contains the core
- library source code. This layer performs memory functions, works
- with vectors and rings, performs key lookups in hash tables, and
- works with timers for dispatching graph nodes.
-- VLIB - the vector processing library. The vlib layer also handles
- various application management functions: buffer, memory and graph
- node management, maintaining and exporting counters, thread
- management, packet tracing. Vlib implements the debug CLI (command
- line interface).
-- VNET - works with VPP\'s networking interface (layers 2, 3, and 4)
- performs session and traffic management, and works with devices and
- the data control plane.
-- Plugins - Contains an increasingly rich set of data-plane plugins,
- as noted in the above diagram.
-- VPP - the container application linked against all of the above.
-
-It's important to understand each of these layers in a certain amount of
-detail. Much of the implementation is best dealt with at the API level
-and otherwise left alone.
diff --git a/docs/gettingstarted/developers/testing_vpp.md b/docs/gettingstarted/developers/testing_vpp.md
deleted file mode 100644
index 4e4ca1a..0000000
--- a/docs/gettingstarted/developers/testing_vpp.md
+++ /dev/null
@@ -1,141 +0,0 @@
-Testing VPP
-===========
-
-As of this writing, the vpp source tree includes over 1,000 unit test
-vectors. Best practices prior to pushing patches for code review: make
-sure that all of the "make test" test vectors pass.
-
-We attempt to maintain the top-level "make test-help" command so that it
-accurately describes all of the "make test" options.
-
-Examples
---------
-
-Basic test run, all test vectors, single-vpp instance, optimized image:
-
-```
- $ make test
-```
-
-10-way parallel basic test run:
-
-```
- $ make TEST_JOBS=10 test
-```
-
-Run a specific test suite (mpls, in this case):
-
-```
- $ make TEST=test_mpls test
-```
-
-Run a specifc test suite, debug image, pause prior to running the test
-suite; attach to the vpp image in gdb:
-
-```
- $ make TEST=xxx DEBUG=gdb test-debug
-```
-
-Detailed Documentation
-----------------------
-
-Current "make test-help" output:
-
-
-```
- $ make test-help
- test - build and run (basic) functional tests
- test-debug - build and run (basic) functional tests (debug build)
- test-all - build and run functional and extended tests
- test-all-debug - build and run functional and extended tests (debug build)
- retest - run functional tests
- retest-debug - run functional tests (debug build)
- retest-all - run functional and extended tests
- retest-all-debug - run functional and extended tests (debug build)
- test-cov - generate code coverage report for test framework
- test-gcov - build and run functional tests (gcov build)
- test-wipe - wipe (temporary) files generated by unit tests
- test-wipe-cov - wipe code coverage report for test framework
- test-wipe-doc - wipe documentation for test framework
- test-wipe-papi - rebuild vpp_papi sources
- test-wipe-all - wipe (temporary) files generated by unit tests, docs, and coverage
- test-shell - enter shell with test environment
- test-shell-debug - enter shell with test environment (debug build)
- test-checkstyle - check PEP8 compliance for test framework
- test-refresh-deps - refresh the Python dependencies for the tests
-
- Arguments controlling test runs:
- V=[0|1|2] - set test verbosity level
- 0=ERROR, 1=INFO, 2=DEBUG
- TEST_JOBS=[<n>|auto] - use at most <n> parallel python processes for test execution, if auto, set to number of available cpus (default: 1)
- MAX_VPP_CPUS=[<n>|auto]- use at most <n> cpus for running vpp main and worker threads, if auto, set to number of available cpus (default: auto)
- CACHE_OUTPUT=[0|1] - cache VPP stdout/stderr and log as one block after test finishes (default: 1)
- FAILFAST=[0|1] - fail fast if 1, complete all tests if 0
- TIMEOUT=<timeout> - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)
- RETRIES=<n> - retry failed tests <n> times
- DEBUG=<type> - set VPP debugging kind
- DEBUG=core - detect coredump and load it in gdb on crash
- DEBUG=gdb - allow easy debugging by printing VPP PID
- and waiting for user input before running
- and tearing down a testcase
- DEBUG=gdbserver - run gdb inside a gdb server, otherwise
- same as above
- DEBUG=attach - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb)
-
- STEP=[yes|no] - ease debugging by stepping through a testcase
- SANITY=[yes|no] - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)
- EXTENDED_TESTS=[1|y] - used by '[re]test-all' & '[re]test-all-debug' to run extended tests
- TEST=<filter> - filter the set of tests:
- by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py
- by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py
- by wildcard - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'
- e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name
- TEST='bfd.*.*' is equivalent to above example of filter by file-suffix
- TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class
- TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase
- TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes
-
- VARIANT=<variant> - specify which march node variant to unit test
- e.g. VARIANT=skx test the skx march variants
- e.g. VARIANT=icl test the icl march variants
-
- COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp
- e.g. COREDUMP_SIZE=4g
- COREDUMP_SIZE=unlimited
- COREDUMP_COMPRESS=1 - compress core files if not debugging them
- EXTERN_TESTS=<path> - path to out-of-tree test_<name>.py files containing test cases
- EXTERN_PLUGINS=<path> - path to out-of-tree plugins to be loaded by vpp under test
- EXTERN_COV_DIR=<path> - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report
-
- PROFILE=1 - enable profiling of test framework via cProfile module
- PROFILE_SORT_BY=opt - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)
- PROFILE_OUTPUT=file - output profiling info to file - use absolute path (default: stdout)
-
- TEST_DEBUG=1 - turn on debugging of the test framework itself (expert)
-
- SKIP_AARCH64=1 - skip tests that are failing on the ARM platorm in FD.io CI
-
- RND_SEED=seed - Seed RND with given seed
-
- Starting VPP in GDB for use with DEBUG=attach:
-
- test-start-vpp-in-gdb - start VPP in gdb (release)
- test-start-vpp-debug-in-gdb - start VPP in gdb (debug)
-
- Arguments controlling VPP in GDB runs:
-
- VPP_IN_GDB_TMP_DIR - specify directory to run VPP IN (default: /tmp/unittest-attach-gdb)
- VPP_IN_GDB_NO_RMDIR=0 - don't remove existing tmp dir but fail instead
- VPP_IN_GDB_CMDLINE=1 - add 'interactive' to VPP arguments to run with command line
-
- Creating test documentation
- test-doc - generate documentation for test framework
- test-wipe-doc - wipe documentation for test framework
-
- Creating test code coverage report
- test-cov - generate code coverage report for test framework
- test-wipe-cov - wipe code coverage report for test framework
-
- Verifying code-style
- test-checkstyle - check PEP8 compliance
-```
diff --git a/docs/gettingstarted/developers/vlib.md b/docs/gettingstarted/developers/vlib.md
deleted file mode 100644
index 3a35978..0000000
--- a/docs/gettingstarted/developers/vlib.md
+++ /dev/null
@@ -1,891 +0,0 @@
-
-VLIB (Vector Processing Library)
-================================
-
-The files associated with vlib are located in the ./src/{vlib,
-vlibapi, vlibmemory} folders. These libraries provide vector
-processing support including graph-node scheduling, reliable multicast
-support, ultra-lightweight cooperative multi-tasking threads, a CLI,
-plug in .DLL support, physical memory and Linux epoll support. Parts of
-this library embody US Patent 7,961,636.
-
-Init function discovery
------------------------
-
-vlib applications register for various \[initialization\] events by
-placing structures and \_\_attribute\_\_((constructor)) functions into
-the image. At appropriate times, the vlib framework walks
-constructor-generated singly-linked structure lists, performs a
-topological sort based on specified constraints, and calls the
-indicated functions. Vlib applications create graph nodes, add CLI
-functions, start cooperative multi-tasking threads, etc. etc. using
-this mechanism.
-
-vlib applications invariably include a number of VLIB\_INIT\_FUNCTION
-(my\_init\_function) macros.
-
-Each init / configure / etc. function has the return type clib\_error\_t
-\*. Make sure that the function returns 0 if all is well, otherwise the
-framework will announce an error and exit.
-
-vlib applications must link against vppinfra, and often link against
-other libraries such as VNET. In the latter case, it may be necessary to
-explicitly reference symbol(s) otherwise large portions of the library
-may be AWOL at runtime.
-
-### Init function construction and constraint specification
-
-It's easy to add an init function:
-
-```
- static clib_error_t *my_init_function (vlib_main_t *vm)
- {
- /* ... initialize things ... */
-
- return 0; // or return clib_error_return (0, "BROKEN!");
- }
- VLIB_INIT_FUNCTION(my_init_function);
-```
-
-As given, my_init_function will be executed "at some point," but with
-no ordering guarantees.
-
-Specifying ordering constraints is easy:
-
-```
- VLIB_INIT_FUNCTION(my_init_function) =
- {
- .runs_before = VLIB_INITS("we_run_before_function_1",
- "we_run_before_function_2"),
- .runs_after = VLIB_INITS("we_run_after_function_1",
- "we_run_after_function_2),
- };
-```
-
-It's also easy to specify bulk ordering constraints of the form "a
-then b then c then d":
-
-```
- VLIB_INIT_FUNCTION(my_init_function) =
- {
- .init_order = VLIB_INITS("a", "b", "c", "d"),
- };
-```
-
-It's OK to specify all three sorts of ordering constraints for a
-single init function, although it's hard to imagine why it would be
-necessary.
-
-
-Node Graph Initialization
--------------------------
-
-vlib packet-processing applications invariably define a set of graph
-nodes to process packets.
-
-One constructs a vlib\_node\_registration\_t, most often via the
-VLIB\_REGISTER\_NODE macro. At runtime, the framework processes the set
-of such registrations into a directed graph. It is easy enough to add
-nodes to the graph at runtime. The framework does not support removing
-nodes.
-
-vlib provides several types of vector-processing graph nodes, primarily
-to control framework dispatch behaviors. The type member of the
-vlib\_node\_registration\_t functions as follows:
-
-- VLIB\_NODE\_TYPE\_PRE\_INPUT - run before all other node types
-- VLIB\_NODE\_TYPE\_INPUT - run as often as possible, after pre\_input
- nodes
-- VLIB\_NODE\_TYPE\_INTERNAL - only when explicitly made runnable by
- adding pending frames for processing
-- VLIB\_NODE\_TYPE\_PROCESS - only when explicitly made runnable.
- "Process" nodes are actually cooperative multi-tasking threads. They
- **must** explicitly suspend after a reasonably short period of time.
-
-For a precise understanding of the graph node dispatcher, please read
-./src/vlib/main.c:vlib\_main\_loop.
-
-Graph node dispatcher
----------------------
-
-Vlib\_main\_loop() dispatches graph nodes. The basic vector processing
-algorithm is diabolically simple, but may not be obvious from even a
-long stare at the code. Here's how it works: some input node, or set of
-input nodes, produce a vector of work to process. The graph node
-dispatcher pushes the work vector through the directed graph,
-subdividing it as needed, until the original work vector has been
-completely processed. At that point, the process recurs.
-
-This scheme yields a stable equilibrium in frame size, by construction.
-Here's why: as the frame size increases, the per-frame-element
-processing time decreases. There are several related forces at work; the
-simplest to describe is the effect of vector processing on the CPU L1
-I-cache. The first frame element \[packet\] processed by a given node
-warms up the node dispatch function in the L1 I-cache. All subsequent
-frame elements profit. As we increase the number of frame elements, the
-cost per element goes down.
-
-Under light load, it is a crazy waste of CPU cycles to run the graph
-node dispatcher flat-out. So, the graph node dispatcher arranges to wait
-for work by sitting in a timed epoll wait if the prevailing frame size
-is low. The scheme has a certain amount of hysteresis to avoid
-constantly toggling back and forth between interrupt and polling mode.
-Although the graph dispatcher supports interrupt and polling modes, our
-current default device drivers do not.
-
-The graph node scheduler uses a hierarchical timer wheel to reschedule
-process nodes upon timer expiration.
-
-Graph dispatcher internals
---------------------------
-
-This section may be safely skipped. It's not necessary to understand
-graph dispatcher internals to create graph nodes.
-
-Vector Data Structure
----------------------
-
-In vpp / vlib, we represent vectors as instances of the vlib_frame_t type:
-
-```c
- typedef struct vlib_frame_t
- {
- /* Frame flags. */
- u16 flags;
-
- /* Number of scalar bytes in arguments. */
- u8 scalar_size;
-
- /* Number of bytes per vector argument. */
- u8 vector_size;
-
- /* Number of vector elements currently in frame. */
- u16 n_vectors;
-
- /* Scalar and vector arguments to next node. */
- u8 arguments[0];
- } vlib_frame_t;
-```
-
-Note that one _could_ construct all kinds of vectors - including
-vectors with some associated scalar data - using this structure. In
-the vpp application, vectors typically use a 4-byte vector element
-size, and zero bytes' worth of associated per-frame scalar data.
-
-Frames are always allocated on CLIB_CACHE_LINE_BYTES boundaries.
-Frames have u32 indices which make use of the alignment property, so
-the maximum feasible main heap offset of a frame is
-CLIB_CACHE_LINE_BYTES * 0xFFFFFFFF: 64*4 = 256 Gbytes.
-
-Scheduling Vectors
-------------------
-
-As you can see, vectors are not directly associated with graph
-nodes. We represent that association in a couple of ways. The
-simplest is the vlib\_pending\_frame\_t:
-
-```c
- /* A frame pending dispatch by main loop. */
- typedef struct
- {
- /* Node and runtime for this frame. */
- u32 node_runtime_index;
-
- /* Frame index (in the heap). */
- u32 frame_index;
-
- /* Start of next frames for this node. */
- u32 next_frame_index;
-
- /* Special value for next_frame_index when there is no next frame. */
- #define VLIB_PENDING_FRAME_NO_NEXT_FRAME ((u32) ~0)
- } vlib_pending_frame_t;
-```
-
-Here is the code in .../src/vlib/main.c:vlib_main_or_worker_loop()
-which processes frames:
-
-```c
- /*
- * Input nodes may have added work to the pending vector.
- * Process pending vector until there is nothing left.
- * All pending vectors will be processed from input -> output.
- */
- for (i = 0; i < _vec_len (nm->pending_frames); i++)
- cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
- /* Reset pending vector for next iteration. */
-```
-
-The pending frame node_runtime_index associates the frame with the
-node which will process it.
-
-Complications
--------------
-
-Fasten your seatbelt. Here's where the story - and the data structures
-\- become quite complicated...
-
-At 100,000 feet: vpp uses a directed graph, not a directed _acyclic_
-graph. It's really quite normal for a packet to visit ip\[46\]-lookup
-multiple times. The worst-case: a graph node which enqueues packets to
-itself.
-
-To deal with this issue, the graph dispatcher must force allocation of
-a new frame if the current graph node's dispatch function happens to
-enqueue a packet back to itself.
-
-There are no guarantees that a pending frame will be processed
-immediately, which means that more packets may be added to the
-underlying vlib_frame_t after it has been attached to a
-vlib_pending_frame_t. Care must be taken to allocate new
-frames and pending frames if a (pending\_frame, frame) pair fills.
-
-Next frames, next frame ownership
----------------------------------
-
-The vlib\_next\_frame\_t is the last key graph dispatcher data structure:
-
-```c
- typedef struct
- {
- /* Frame index. */
- u32 frame_index;
-
- /* Node runtime for this next. */
- u32 node_runtime_index;
-
- /* Next frame flags. */
- u32 flags;
-
- /* Reflects node frame-used flag for this next. */
- #define VLIB_FRAME_NO_FREE_AFTER_DISPATCH \
- VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
-
- /* This next frame owns enqueue to node
- corresponding to node_runtime_index. */
- #define VLIB_FRAME_OWNER (1 << 15)
-
- /* Set when frame has been allocated for this next. */
- #define VLIB_FRAME_IS_ALLOCATED VLIB_NODE_FLAG_IS_OUTPUT
-
- /* Set when frame has been added to pending vector. */
- #define VLIB_FRAME_PENDING VLIB_NODE_FLAG_IS_DROP
-
- /* Set when frame is to be freed after dispatch. */
- #define VLIB_FRAME_FREE_AFTER_DISPATCH VLIB_NODE_FLAG_IS_PUNT
-
- /* Set when frame has traced packets. */
- #define VLIB_FRAME_TRACE VLIB_NODE_FLAG_TRACE
-
- /* Number of vectors enqueue to this next since last overflow. */
- u32 vectors_since_last_overflow;
- } vlib_next_frame_t;
-```
-
-Graph node dispatch functions call vlib\_get\_next\_frame (...) to
-set "(u32 \*)to_next" to the right place in the vlib_frame_t
-corresponding to the ith arc (aka next0) from the current node to the
-indicated next node.
-
-After some scuffling around - two levels of macros - processing
-reaches vlib\_get\_next\_frame_internal (...). Get-next-frame-internal
-digs up the vlib\_next\_frame\_t corresponding to the desired graph
-arc.
-
-The next frame data structure amounts to a graph-arc-centric frame
-cache. Once a node finishes adding element to a frame, it will acquire
-a vlib_pending_frame_t and end up on the graph dispatcher's
-run-queue. But there's no guarantee that more vector elements won't be
-added to the underlying frame from the same (source\_node,
-next\_index) arc or from a different (source\_node, next\_index) arc.
-
-Maintaining consistency of the arc-to-frame cache is necessary. The
-first step in maintaining consistency is to make sure that only one
-graph node at a time thinks it "owns" the target vlib\_frame\_t.
-
-Back to the graph node dispatch function. In the usual case, a certain
-number of packets will be added to the vlib\_frame\_t acquired by
-calling vlib\_get\_next\_frame (...).
-
-Before a dispatch function returns, it's required to call
-vlib\_put\_next\_frame (...) for all of the graph arcs it actually
-used. This action adds a vlib\_pending\_frame\_t to the graph
-dispatcher's pending frame vector.
-
-Vlib\_put\_next\_frame makes a note in the pending frame of the frame
-index, and also of the vlib\_next\_frame\_t index.
-
-dispatch\_pending\_node actions
--------------------------------
-
-The main graph dispatch loop calls dispatch pending node as shown
-above.
-
-Dispatch\_pending\_node recovers the pending frame, and the graph node
-runtime / dispatch function. Further, it recovers the next\_frame
-currently associated with the vlib\_frame\_t, and detaches the
-vlib\_frame\_t from the next\_frame.
-
-In .../src/vlib/main.c:dispatch\_pending\_node(...), note this stanza:
-
-```c
- /* Force allocation of new frame while current frame is being
- dispatched. */
- restore_frame_index = ~0;
- if (nf->frame_index == p->frame_index)
- {
- nf->frame_index = ~0;
- nf->flags &= ~VLIB_FRAME_IS_ALLOCATED;
- if (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH))
- restore_frame_index = p->frame_index;
- }
-```
-
-dispatch\_pending\_node is worth a hard stare due to the several
-second-order optimizations it implements. Almost as an afterthought,
-it calls dispatch_node which actually calls the graph node dispatch
-function.
-
-Process / thread model
-----------------------
-
-vlib provides an ultra-lightweight cooperative multi-tasking thread
-model. The graph node scheduler invokes these processes in much the same
-way as traditional vector-processing run-to-completion graph nodes;
-plus-or-minus a setjmp/longjmp pair required to switch stacks. Simply
-set the vlib\_node\_registration\_t type field to
-vlib\_NODE\_TYPE\_PROCESS. Yes, process is a misnomer. These are
-cooperative multi-tasking threads.
-
-As of this writing, the default stack size is 2<<15 = 32kb.
-Initialize the node registration's process\_log2\_n\_stack\_bytes member
-as needed. The graph node dispatcher makes some effort to detect stack
-overrun, e.g. by mapping a no-access page below each thread stack.
-
-Process node dispatch functions are expected to be "while(1) { }" loops
-which suspend when not otherwise occupied, and which must not run for
-unreasonably long periods of time.
-
-"Unreasonably long" is an application-dependent concept. Over the years,
-we have constructed frame-size sensitive control-plane nodes which will
-use a much higher fraction of the available CPU bandwidth when the frame
-size is low. The classic example: modifying forwarding tables. So long
-as the table-builder leaves the forwarding tables in a valid state, one
-can suspend the table builder to avoid dropping packets as a result of
-control-plane activity.
-
-Process nodes can suspend for fixed amounts of time, or until another
-entity signals an event, or both. See the next section for a description
-of the vlib process event mechanism.
-
-When running in vlib process context, one must pay strict attention to
-loop invariant issues. If one walks a data structure and calls a
-function which may suspend, one had best know by construction that it
-cannot change. Often, it's best to simply make a snapshot copy of a data
-structure, walk the copy at leisure, then free the copy.
-
-Process events
---------------
-
-The vlib process event mechanism API is extremely lightweight and easy
-to use. Here is a typical example:
-
-```c
- vlib_main_t *vm = &vlib_global_main;
- uword event_type, * event_data = 0;
-
- while (1)
- {
- vlib_process_wait_for_event_or_clock (vm, 5.0 /* seconds */);
-
- event_type = vlib_process_get_events (vm, &event_data);
-
- switch (event_type) {
- case EVENT1:
- handle_event1s (event_data);
- break;
-
- case EVENT2:
- handle_event2s (event_data);
- break;
-
- case ~0: /* 5-second idle/periodic */
- handle_idle ();
- break;
-
- default: /* bug! */
- ASSERT (0);
- }
-
- vec_reset_length(event_data);
- }
-```
-
-In this example, the VLIB process node waits for an event to occur, or
-for 5 seconds to elapse. The code demuxes on the event type, calling
-the appropriate handler function. Each call to
-vlib\_process\_get\_events returns a vector of per-event-type data
-passed to successive vlib\_process\_signal\_event calls; it is a
-serious error to process only event\_data\[0\].
-
-Resetting the event\_data vector-length to 0 \[instead of calling
-vec\_free\] means that the event scheme doesn't burn cycles continuously
-allocating and freeing the event data vector. This is a common vppinfra
-/ vlib coding pattern, well worth using when appropriate.
-
-Signaling an event is easy, for example:
-
-```c
- vlib_process_signal_event (vm, process_node_index, EVENT1,
- (uword)arbitrary_event1_data); /* and so forth */
-```
-
-One can either know the process node index by construction - dig it out
-of the appropriate vlib\_node\_registration\_t - or by finding the
-vlib\_node\_t with vlib\_get\_node\_by\_name(...).
-
-Buffers
--------
-
-vlib buffering solves the usual set of packet-processing problems,
-albeit at high performance. Key in terms of performance: one ordinarily
-allocates / frees N buffers at a time rather than one at a time. Except
-when operating directly on a specific buffer, one deals with buffers by
-index, not by pointer.
-
-Packet-processing frames are u32\[\] arrays, not
-vlib\_buffer\_t\[\] arrays.
-
-Packets comprise one or more vlib buffers, chained together as required.
-Multiple particle sizes are supported; hardware input nodes simply ask
-for the required size(s). Coalescing support is available. For obvious
-reasons one is discouraged from writing one's own wild and wacky buffer
-chain traversal code.
-
-vlib buffer headers are allocated immediately prior to the buffer data
-area. In typical packet processing this saves a dependent read wait:
-given a buffer's address, one can prefetch the buffer header
-\[metadata\] at the same time as the first cache line of buffer data.
-
-Buffer header metadata (vlib\_buffer\_t) includes the usual rewrite
-expansion space, a current\_data offset, RX and TX interface indices,
-packet trace information, and a opaque areas.
-
-The opaque data is intended to control packet processing in arbitrary
-subgraph-dependent ways. The programmer shoulders responsibility for
-data lifetime analysis, type-checking, etc.
-
-Buffers have reference-counts in support of e.g. multicast replication.
-
-Shared-memory message API
--------------------------
-
-Local control-plane and application processes interact with the vpp
-dataplane via asynchronous message-passing in shared memory over
-unidirectional queues. The same application APIs are available via
-sockets.
-
-Capturing API traces and replaying them in a simulation environment
-requires a disciplined approach to the problem. This seems like a
-make-work task, but it is not. When something goes wrong in the
-control-plane after 300,000 or 3,000,000 operations, high-speed replay
-of the events leading up to the accident is a huge win.
-
-The shared-memory message API message allocator vl\_api\_msg\_alloc uses
-a particularly cute trick. Since messages are processed in order, we try
-to allocate message buffering from a set of fixed-size, preallocated
-rings. Each ring item has a "busy" bit. Freeing one of the preallocated
-message buffers merely requires the message consumer to clear the busy
-bit. No locking required.
-
-Debug CLI
----------
-
-Adding debug CLI commands to VLIB applications is very simple.
-
-Here is a complete example:
-
-```c
- static clib_error_t *
- show_ip_tuple_match (vlib_main_t * vm,
- unformat_input_t * input,
- vlib_cli_command_t * cmd)
- {
- vlib_cli_output (vm, "%U\n", format_ip_tuple_match_tables, &routing_main);
- return 0;
- }
-
- /* *INDENT-OFF* */
- static VLIB_CLI_COMMAND (show_ip_tuple_command) =
- {
- .path = "show ip tuple match",
- .short_help = "Show ip 5-tuple match-and-broadcast tables",
- .function = show_ip_tuple_match,
- };
- /* *INDENT-ON* */
-```
-
-This example implements the "show ip tuple match" debug cli
-command. In ordinary usage, the vlib cli is available via the "vppctl"
-application, which sends traffic to a named pipe. One can configure
-debug CLI telnet access on a configurable port.
-
-The cli implementation has an output redirection facility which makes it
-simple to deliver cli output via shared-memory API messaging,
-
-Particularly for debug or "show tech support" type commands, it would be
-wasteful to write vlib application code to pack binary data, write more
-code elsewhere to unpack the data and finally print the answer. If a
-certain cli command has the potential to hurt packet processing
-performance by running for too long, do the work incrementally in a
-process node. The client can wait.
-
-### Macro expansion
-
-The vpp debug CLI engine includes a recursive macro expander. This
-is quite useful for factoring out address and/or interface name
-specifics:
-
-```
- define ip1 192.168.1.1/24
- define ip2 192.168.2.1/24
- define iface1 GigabitEthernet3/0/0
- define iface2 loop1
-
- set int ip address $iface1 $ip1
- set int ip address $iface2 $(ip2)
-
- undefine ip1
- undefine ip2
- undefine iface1
- undefine iface2
-```
-
-Each socket (or telnet) debug CLI session has its own macro
-tables. All debug CLI sessions which use CLI_INBAND binary API
-messages share a single table.
-
-The macro expander recognizes circular defintions:
-
-```
- define foo \$(bar)
- define bar \$(mumble)
- define mumble \$(foo)
-```
-
-At 8 levels of recursion, the macro expander throws up its hands and
-replies "CIRCULAR."
-
-### Macro-related debug CLI commands
-
-In addition to the "define" and "undefine" debug CLI commands, use
-"show macro [noevaluate]" to dump the macro table. The "echo" debug
-CLI command will evaluate and print its argument:
-
-```
- vpp# define foo This\ Is\ Foo
- vpp# echo $foo
- This Is Foo
-```
-
-Handing off buffers between threads
------------------------------------
-
-Vlib includes an easy-to-use mechanism for handing off buffers between
-worker threads. A typical use-case: software ingress flow hashing. At
-a high level, one creates a per-worker-thread queue which sends packets
-to a specific graph node in the indicated worker thread. With the
-queue in hand, enqueue packets to the worker thread of your choice.
-
-### Initialize a handoff queue
-
-Simple enough, call vlib_frame_queue_main_init:
-
-```c
- main_ptr->frame_queue_index
- = vlib_frame_queue_main_init (dest_node.index, frame_queue_size);
-```
-
-Frame_queue_size means what it says: the number of frames which may be
-queued. Since frames contain 1...256 packets, frame_queue_size should
-be a reasonably small number (32...64). If the frame queue producer(s)
-are faster than the frame queue consumer(s), congestion will
-occur. Suggest letting the enqueue operator deal with queue
-congestion, as shown in the enqueue example below.
-
-Under the floorboards, vlib_frame_queue_main_init creates an input queue
-for each worker thread.
-
-Please do NOT create frame queues until it's clear that they will be
-used. Although the main dispatch loop is reasonably smart about how
-often it polls the (entire set of) frame queues, polling unused frame
-queues is a waste of clock cycles.
-
-### Hand off packets
-
-The actual handoff mechanics are simple, and integrate nicely with
-a typical graph-node dispatch function:
-
-```c
- always_inline uword
- do_handoff_inline (vlib_main_t * vm,
- vlib_node_runtime_t * node, vlib_frame_t * frame,
- int is_ip4, int is_trace)
- {
- u32 n_left_from, *from;
- vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
- u16 thread_indices [VLIB_FRAME_SIZE];
- u16 nexts[VLIB_FRAME_SIZE], *next;
- u32 n_enq;
- htest_main_t *hmp = &htest_main;
- int i;
-
- from = vlib_frame_vector_args (frame);
- n_left_from = frame->n_vectors;
-
- vlib_get_buffers (vm, from, bufs, n_left_from);
- next = nexts;
- b = bufs;
-
- /*
- * Typical frame traversal loop, details vary with
- * use case. Make sure to set thread_indices[i] with
- * the desired destination thread index. You may
- * or may not bother to set next[i].
- */
-
- for (i = 0; i < frame->n_vectors; i++)
- {
- <snip>
- /* Pick a thread to handle this packet */
- thread_indices[i] = f (packet_data_or_whatever);
- <snip>
-
- b += 1;
- next += 1;
- n_left_from -= 1;
- }
-
- /* Enqueue buffers to threads */
- n_enq =
- vlib_buffer_enqueue_to_thread (vm, node, hmp->frame_queue_index,
- from, thread_indices, frame->n_vectors,
- 1 /* drop on congestion */);
- /* Typical counters,
- if (n_enq < frame->n_vectors)
- vlib_node_increment_counter (vm, node->node_index,
- XXX_ERROR_CONGESTION_DROP,
- frame->n_vectors - n_enq);
- vlib_node_increment_counter (vm, node->node_index,
- XXX_ERROR_HANDED_OFF, n_enq);
- return frame->n_vectors;
-}
-```
-
-Notes about calling vlib_buffer_enqueue_to_thread(...):
-
-* If you pass "drop on congestion" non-zero, all packets in the
-inbound frame will be consumed one way or the other. This is the
-recommended setting.
-
-* In the drop-on-congestion case, please don't try to "help" in the
-enqueue node by freeing dropped packets, or by pushing them to
-"error-drop." Either of those actions would be a severe error.
-
-* It's perfectly OK to enqueue packets to the current thread.
-
-Handoff Demo Plugin
--------------------
-
-Check out the sample (plugin) example in
-.../src/examples/handoffdemo. If you want to build the handoff demo plugin:
-
-```
-$ cd .../src/plugins
-$ ln -s ../examples/handoffdemo
-```
-
-This plugin provides a simple example of how to hand off packets
-between threads. We used it to debug packet-tracer handoff tracing
-support.
-
-# Packet generator input script
-
-```
- packet-generator new {
- name x
- limit 5
- size 128-128
- interface local0
- node handoffdemo-1
- data {
- incrementing 30
- }
- }
-```
-# Start vpp with 2 worker threads
-
-The demo plugin hands packets from worker 1 to worker 2.
-
-# Enable tracing, and start the packet generator
-
-```
- trace add pg-input 100
- packet-generator enable
-```
-
-# Sample Run
-
-```
- DBGvpp# ex /tmp/pg_input_script
- DBGvpp# pa en
- DBGvpp# sh err
- Count Node Reason
- 5 handoffdemo-1 packets handed off processed
- 5 handoffdemo-2 completed packets
- DBGvpp# show run
- Thread 1 vpp_wk_0 (lcore 0)
- Time 133.9, average vectors/node 5.00, last 128 main loops 0.00 per node 0.00
- vector rates in 3.7331e-2, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- handoffdemo-1 active 1 5 0 4.76e3 5.00
- pg-input disabled 2 5 0 5.58e4 2.50
- unix-epoll-input polling 22760 0 0 2.14e7 0.00
- ---------------
- Thread 2 vpp_wk_1 (lcore 2)
- Time 133.9, average vectors/node 5.00, last 128 main loops 0.00 per node 0.00
- vector rates in 0.0000e0, out 0.0000e0, drop 3.7331e-2, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- drop active 1 5 0 1.35e4 5.00
- error-drop active 1 5 0 2.52e4 5.00
- handoffdemo-2 active 1 5 0 2.56e4 5.00
- unix-epoll-input polling 22406 0 0 2.18e7 0.00
-```
-
-Enable the packet tracer and run it again...
-
-```
- DBGvpp# trace add pg-input 100
- DBGvpp# pa en
- DBGvpp# sh trace
- sh trace
- ------------------- Start of thread 0 vpp_main -------------------
- No packets in trace buffer
- ------------------- Start of thread 1 vpp_wk_0 -------------------
- Packet 1
-
- 00:06:50:520688: pg-input
- stream x, 128 bytes, 0 sw_if_index
- current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000000
- 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000
- 00000020: 0000000000000000000000000000000000000000000000000000000000000000
- 00000040: 0000000000000000000000000000000000000000000000000000000000000000
- 00000060: 0000000000000000000000000000000000000000000000000000000000000000
- 00:06:50:520762: handoffdemo-1
- HANDOFFDEMO: current thread 1
-
- Packet 2
-
- 00:06:50:520688: pg-input
- stream x, 128 bytes, 0 sw_if_index
- current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000001
- 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000
- 00000020: 0000000000000000000000000000000000000000000000000000000000000000
- 00000040: 0000000000000000000000000000000000000000000000000000000000000000
- 00000060: 0000000000000000000000000000000000000000000000000000000000000000
- 00:06:50:520762: handoffdemo-1
- HANDOFFDEMO: current thread 1
-
- Packet 3
-
- 00:06:50:520688: pg-input
- stream x, 128 bytes, 0 sw_if_index
- current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000002
- 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000
- 00000020: 0000000000000000000000000000000000000000000000000000000000000000
- 00000040: 0000000000000000000000000000000000000000000000000000000000000000
- 00000060: 0000000000000000000000000000000000000000000000000000000000000000
- 00:06:50:520762: handoffdemo-1
- HANDOFFDEMO: current thread 1
-
- Packet 4
-
- 00:06:50:520688: pg-input
- stream x, 128 bytes, 0 sw_if_index
- current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000003
- 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000
- 00000020: 0000000000000000000000000000000000000000000000000000000000000000
- 00000040: 0000000000000000000000000000000000000000000000000000000000000000
- 00000060: 0000000000000000000000000000000000000000000000000000000000000000
- 00:06:50:520762: handoffdemo-1
- HANDOFFDEMO: current thread 1
-
- Packet 5
-
- 00:06:50:520688: pg-input
- stream x, 128 bytes, 0 sw_if_index
- current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000004
- 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000
- 00000020: 0000000000000000000000000000000000000000000000000000000000000000
- 00000040: 0000000000000000000000000000000000000000000000000000000000000000
- 00000060: 0000000000000000000000000000000000000000000000000000000000000000
- 00:06:50:520762: handoffdemo-1
- HANDOFFDEMO: current thread 1
-
- ------------------- Start of thread 2 vpp_wk_1 -------------------
- Packet 1
-
- 00:06:50:520796: handoff_trace
- HANDED-OFF: from thread 1 trace index 0
- 00:06:50:520796: handoffdemo-2
- HANDOFFDEMO: current thread 2
- 00:06:50:520867: error-drop
- rx:local0
- 00:06:50:520914: drop
- handoffdemo-2: completed packets
-
- Packet 2
-
- 00:06:50:520796: handoff_trace
- HANDED-OFF: from thread 1 trace index 1
- 00:06:50:520796: handoffdemo-2
- HANDOFFDEMO: current thread 2
- 00:06:50:520867: error-drop
- rx:local0
- 00:06:50:520914: drop
- handoffdemo-2: completed packets
-
- Packet 3
-
- 00:06:50:520796: handoff_trace
- HANDED-OFF: from thread 1 trace index 2
- 00:06:50:520796: handoffdemo-2
- HANDOFFDEMO: current thread 2
- 00:06:50:520867: error-drop
- rx:local0
- 00:06:50:520914: drop
- handoffdemo-2: completed packets
-
- Packet 4
-
- 00:06:50:520796: handoff_trace
- HANDED-OFF: from thread 1 trace index 3
- 00:06:50:520796: handoffdemo-2
- HANDOFFDEMO: current thread 2
- 00:06:50:520867: error-drop
- rx:local0
- 00:06:50:520914: drop
- handoffdemo-2: completed packets
-
- Packet 5
-
- 00:06:50:520796: handoff_trace
- HANDED-OFF: from thread 1 trace index 4
- 00:06:50:520796: handoffdemo-2
- HANDOFFDEMO: current thread 2
- 00:06:50:520867: error-drop
- rx:local0
- 00:06:50:520914: drop
- handoffdemo-2: completed packets
- DBGvpp#
-```
diff --git a/docs/gettingstarted/developers/vnet.md b/docs/gettingstarted/developers/vnet.md
deleted file mode 100644
index 09f81e4..0000000
--- a/docs/gettingstarted/developers/vnet.md
+++ /dev/null
@@ -1,795 +0,0 @@
-
-VNET (VPP Network Stack)
-========================
-
-The files associated with the VPP network stack layer are located in the
-*./src/vnet* folder. The Network Stack Layer is basically an
-instantiation of the code in the other layers. This layer has a vnet
-library that provides vectorized layer-2 and 3 networking graph nodes, a
-packet generator, and a packet tracer.
-
-In terms of building a packet processing application, vnet provides a
-platform-independent subgraph to which one connects a couple of
-device-driver nodes.
-
-Typical RX connections include "ethernet-input" \[full software
-classification, feeds ipv4-input, ipv6-input, arp-input etc.\] and
-"ipv4-input-no-checksum" \[if hardware can classify, perform ipv4 header
-checksum\].
-
-Effective graph dispatch function coding
-----------------------------------------
-
-Over the 15 years, multiple coding styles have emerged: a
-single/dual/quad loop coding model (with variations) and a
-fully-pipelined coding model.
-
-Single/dual loops
------------------
-
-The single/dual/quad loop model variations conveniently solve problems
-where the number of items to process is not known in advance: typical
-hardware RX-ring processing. This coding style is also very effective
-when a given node will not need to cover a complex set of dependent
-reads.
-
-Here is an quad/single loop which can leverage up-to-avx512 SIMD vector
-units to convert buffer indices to buffer pointers:
-
-```c
- static uword
- simulated_ethernet_interface_tx (vlib_main_t * vm,
- vlib_node_runtime_t *
- node, vlib_frame_t * frame)
- {
- u32 n_left_from, *from;
- u32 next_index = 0;
- u32 n_bytes;
- u32 thread_index = vm->thread_index;
- vnet_main_t *vnm = vnet_get_main ();
- vnet_interface_main_t *im = &vnm->interface_main;
- vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
- u16 nexts[VLIB_FRAME_SIZE], *next;
-
- n_left_from = frame->n_vectors;
- from = vlib_frame_vector_args (frame);
-
- /*
- * Convert up to VLIB_FRAME_SIZE indices in "from" to
- * buffer pointers in bufs[]
- */
- vlib_get_buffers (vm, from, bufs, n_left_from);
- b = bufs;
- next = nexts;
-
- /*
- * While we have at least 4 vector elements (pkts) to process..
- */
- while (n_left_from >= 4)
- {
- /* Prefetch next quad-loop iteration. */
- if (PREDICT_TRUE (n_left_from >= 8))
- {
- vlib_prefetch_buffer_header (b[4], STORE);
- vlib_prefetch_buffer_header (b[5], STORE);
- vlib_prefetch_buffer_header (b[6], STORE);
- vlib_prefetch_buffer_header (b[7], STORE);
- }
-
- /*
- * $$$ Process 4x packets right here...
- * set next[0..3] to send the packets where they need to go
- */
-
- do_something_to (b[0]);
- do_something_to (b[1]);
- do_something_to (b[2]);
- do_something_to (b[3]);
-
- /* Process the next 0..4 packets */
- b += 4;
- next += 4;
- n_left_from -= 4;
- }
- /*
- * Clean up 0...3 remaining packets at the end of the incoming frame
- */
- while (n_left_from > 0)
- {
- /*
- * $$$ Process one packet right here...
- * set next[0..3] to send the packets where they need to go
- */
- do_something_to (b[0]);
-
- /* Process the next packet */
- b += 1;
- next += 1;
- n_left_from -= 1;
- }
-
- /*
- * Send the packets along their respective next-node graph arcs
- * Considerable locality of reference is expected, most if not all
- * packets in the inbound vector will traverse the same next-node
- * arc
- */
- vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
-
- return frame->n_vectors;
- }
-```
-
-Given a packet processing task to implement, it pays to scout around
-looking for similar tasks, and think about using the same coding
-pattern. It is not uncommon to recode a given graph node dispatch function
-several times during performance optimization.
-
-Creating Packets from Scratch
------------------------------
-
-At times, it's necessary to create packets from scratch and send
-them. Tasks like sending keepalives or actively opening connections
-come to mind. Its not difficult, but accurate buffer metadata setup is
-required.
-
-### Allocating Buffers
-
-Use vlib_buffer_alloc, which allocates a set of buffer indices. For
-low-performance applications, it's OK to allocate one buffer at a
-time. Note that vlib_buffer_alloc(...) does NOT initialize buffer
-metadata. See below.
-
-In high-performance cases, allocate a vector of buffer indices,
-and hand them out from the end of the vector; decrement _vec_len(..)
-as buffer indices are allocated. See tcp_alloc_tx_buffers(...) and
-tcp_get_free_buffer_index(...) for an example.
-
-### Buffer Initialization Example
-
-The following example shows the **main points**, but is not to be
-blindly cut-'n-pasted.
-
-```c
- u32 bi0;
- vlib_buffer_t *b0;
- ip4_header_t *ip;
- udp_header_t *udp;
-
- /* Allocate a buffer */
- if (vlib_buffer_alloc (vm, &bi0, 1) != 1)
- return -1;
-
- b0 = vlib_get_buffer (vm, bi0);
-
- /* At this point b0->current_data = 0, b0->current_length = 0 */
-
- /*
- * Copy data into the buffer. This example ASSUMES that data will fit
- * in a single buffer, and is e.g. an ip4 packet.
- */
- if (have_packet_rewrite)
- {
- clib_memcpy (b0->data, data, vec_len (data));
- b0->current_length = vec_len (data);
- }
- else
- {
- /* OR, build a udp-ip packet (for example) */
- ip = vlib_buffer_get_current (b0);
- udp = (udp_header_t *) (ip + 1);
- data_dst = (u8 *) (udp + 1);
-
- ip->ip_version_and_header_length = 0x45;
- ip->ttl = 254;
- ip->protocol = IP_PROTOCOL_UDP;
- ip->length = clib_host_to_net_u16 (sizeof (*ip) + sizeof (*udp) +
- vec_len(udp_data));
- ip->src_address.as_u32 = src_address->as_u32;
- ip->dst_address.as_u32 = dst_address->as_u32;
- udp->src_port = clib_host_to_net_u16 (src_port);
- udp->dst_port = clib_host_to_net_u16 (dst_port);
- udp->length = clib_host_to_net_u16 (vec_len (udp_data));
- clib_memcpy (data_dst, udp_data, vec_len(udp_data));
-
- if (compute_udp_checksum)
- {
- /* RFC 7011 section 10.3.2. */
- udp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip);
- if (udp->checksum == 0)
- udp->checksum = 0xffff;
- }
- b0->current_length = vec_len (sizeof (*ip) + sizeof (*udp) +
- vec_len (udp_data));
-
- }
- b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
-
- /* sw_if_index 0 is the "local" interface, which always exists */
- vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
-
- /* Use the default FIB index for tx lookup. Set non-zero to use another fib */
- vnet_buffer (b0)->sw_if_index[VLIB_TX] = 0;
-
-```
-
-If your use-case calls for large packet transmission, use
-vlib_buffer_chain_append_data_with_alloc(...) to create the requisite
-buffer chain.
-
-### Enqueueing packets for lookup and transmission
-
-The simplest way to send a set of packets is to use
-vlib_get_frame_to_node(...) to allocate fresh frame(s) to
-ip4_lookup_node or ip6_lookup_node, add the constructed buffer
-indices, and dispatch the frame using vlib_put_frame_to_node(...).
-
-```c
- vlib_frame_t *f;
- f = vlib_get_frame_to_node (vm, ip4_lookup_node.index);
- f->n_vectors = vec_len(buffer_indices_to_send);
- to_next = vlib_frame_vector_args (f);
-
- for (i = 0; i < vec_len (buffer_indices_to_send); i++)
- to_next[i] = buffer_indices_to_send[i];
-
- vlib_put_frame_to_node (vm, ip4_lookup_node_index, f);
-```
-
-It is inefficient to allocate and schedule single packet frames.
-That's typical in case you need to send one packet per second, but
-should **not** occur in a for-loop!
-
-Packet tracer
--------------
-
-Vlib includes a frame element \[packet\] trace facility, with a simple
-debug CLI interface. The cli is straightforward: "trace add
-input-node-name count" to start capturing packet traces.
-
-To trace 100 packets on a typical x86\_64 system running the dpdk
-plugin: "trace add dpdk-input 100". When using the packet generator:
-"trace add pg-input 100"
-
-To display the packet trace: "show trace"
-
-Each graph node has the opportunity to capture its own trace data. It is
-almost always a good idea to do so. The trace capture APIs are simple.
-
-The packet capture APIs snapshoot binary data, to minimize processing at
-capture time. Each participating graph node initialization provides a
-vppinfra format-style user function to pretty-print data when required
-by the VLIB "show trace" command.
-
-Set the VLIB node registration ".format\_trace" member to the name of
-the per-graph node format function.
-
-Here's a simple example:
-
-```c
- u8 * my_node_format_trace (u8 * s, va_list * args)
- {
- vlib_main_t * vm = va_arg (*args, vlib_main_t *);
- vlib_node_t * node = va_arg (*args, vlib_node_t *);
- my_node_trace_t * t = va_arg (*args, my_trace_t *);
-
- s = format (s, "My trace data was: %d", t-><whatever>);
-
- return s;
- }
-```
-
-The trace framework hands the per-node format function the data it
-captured as the packet whizzed by. The format function pretty-prints the
-data as desired.
-
-Graph Dispatcher Pcap Tracing
------------------------------
-
-The vpp graph dispatcher knows how to capture vectors of packets in pcap
-format as they're dispatched. The pcap captures are as follows:
-
-```
- VPP graph dispatch trace record description:
-
- 0 1 2 3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Major Version | Minor Version | NStrings | ProtoHint |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Buffer index (big endian) |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- + VPP graph node name ... ... | NULL octet |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Buffer Metadata ... ... | NULL octet |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Buffer Opaque ... ... | NULL octet |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Buffer Opaque 2 ... ... | NULL octet |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | VPP ASCII packet trace (if NStrings > 4) | NULL octet |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Packet data (up to 16K) |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-```
-
-Graph dispatch records comprise a version stamp, an indication of how
-many NULL-terminated strings will follow the record header and preceed
-packet data, and a protocol hint.
-
-The buffer index is an opaque 32-bit cookie which allows consumers of
-these data to easily filter/track single packets as they traverse the
-forwarding graph.
-
-Multiple records per packet are normal, and to be expected. Packets
-will appear multiple times as they traverse the vpp forwarding
-graph. In this way, vpp graph dispatch traces are significantly
-different from regular network packet captures from an end-station.
-This property complicates stateful packet analysis.
-
-Restricting stateful analysis to records from a single vpp graph node
-such as "ethernet-input" seems likely to improve the situation.
-
-As of this writing: major version = 1, minor version = 0. Nstrings
-SHOULD be 4 or 5. Consumers SHOULD be wary values less than 4 or
-greater than 5. They MAY attempt to display the claimed number of
-strings, or they MAY treat the condition as an error.
-
-Here is the current set of protocol hints:
-
-```c
- typedef enum
- {
- VLIB_NODE_PROTO_HINT_NONE = 0,
- VLIB_NODE_PROTO_HINT_ETHERNET,
- VLIB_NODE_PROTO_HINT_IP4,
- VLIB_NODE_PROTO_HINT_IP6,
- VLIB_NODE_PROTO_HINT_TCP,
- VLIB_NODE_PROTO_HINT_UDP,
- VLIB_NODE_N_PROTO_HINTS,
- } vlib_node_proto_hint_t;
-```
-
-Example: VLIB_NODE_PROTO_HINT_IP6 means that the first octet of packet
-data SHOULD be 0x60, and should begin an ipv6 packet header.
-
-Downstream consumers of these data SHOULD pay attention to the
-protocol hint. They MUST tolerate inaccurate hints, which MAY occur
-from time to time.
-
-### Dispatch Pcap Trace Debug CLI
-
-To start a dispatch trace capture of up to 10,000 trace records:
-
-```
- pcap dispatch trace on max 10000 file dispatch.pcap
-```
-
-To start a dispatch trace which will also include standard vpp packet
-tracing for packets which originate in dpdk-input:
-
-```
- pcap dispatch trace on max 10000 file dispatch.pcap buffer-trace dpdk-input 1000
-```
-To save the pcap trace, e.g. in /tmp/dispatch.pcap:
-
-```
- pcap dispatch trace off
-```
-
-### Wireshark dissection of dispatch pcap traces
-
-It almost goes without saying that we built a companion wireshark
-dissector to display these traces. As of this writing, we have
-upstreamed the wireshark dissector.
-
-Since it will be a while before wireshark/master/latest makes it into
-all of the popular Linux distros, please see the "How to build a vpp
-dispatch trace aware Wireshark" page for build info.
-
-Here is a sample packet dissection, with some fields omitted for
-clarity. The point is that the wireshark dissector accurately
-displays **all** of the vpp buffer metadata, and the name of the graph
-node in question.
-
-```
- Frame 1: 2216 bytes on wire (17728 bits), 2216 bytes captured (17728 bits)
- Encapsulation type: USER 13 (58)
- [Protocols in frame: vpp:vpp-metadata:vpp-opaque:vpp-opaque2:eth:ethertype:ip:tcp:data]
- VPP Dispatch Trace
- BufferIndex: 0x00036663
- NodeName: ethernet-input
- VPP Buffer Metadata
- Metadata: flags:
- Metadata: current_data: 0, current_length: 102
- Metadata: current_config_index: 0, flow_id: 0, next_buffer: 0
- Metadata: error: 0, n_add_refs: 0, buffer_pool_index: 0
- Metadata: trace_index: 0, recycle_count: 0, len_not_first_buf: 0
- Metadata: free_list_index: 0
- Metadata:
- VPP Buffer Opaque
- Opaque: raw: 00000007 ffffffff 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
- Opaque: sw_if_index[VLIB_RX]: 7, sw_if_index[VLIB_TX]: -1
- Opaque: L2 offset 0, L3 offset 0, L4 offset 0, feature arc index 0
- Opaque: ip.adj_index[VLIB_RX]: 0, ip.adj_index[VLIB_TX]: 0
- Opaque: ip.flow_hash: 0x0, ip.save_protocol: 0x0, ip.fib_index: 0
- Opaque: ip.save_rewrite_length: 0, ip.rpf_id: 0
- Opaque: ip.icmp.type: 0 ip.icmp.code: 0, ip.icmp.data: 0x0
- Opaque: ip.reass.next_index: 0, ip.reass.estimated_mtu: 0
- Opaque: ip.reass.fragment_first: 0 ip.reass.fragment_last: 0
- Opaque: ip.reass.range_first: 0 ip.reass.range_last: 0
- Opaque: ip.reass.next_range_bi: 0x0, ip.reass.ip6_frag_hdr_offset: 0
- Opaque: mpls.ttl: 0, mpls.exp: 0, mpls.first: 0, mpls.save_rewrite_length: 0, mpls.bier.n_bytes: 0
- Opaque: l2.feature_bitmap: 00000000, l2.bd_index: 0, l2.l2_len: 0, l2.shg: 0, l2.l2fib_sn: 0, l2.bd_age: 0
- Opaque: l2.feature_bitmap_input: none configured, L2.feature_bitmap_output: none configured
- Opaque: l2t.next_index: 0, l2t.session_index: 0
- Opaque: l2_classify.table_index: 0, l2_classify.opaque_index: 0, l2_classify.hash: 0x0
- Opaque: policer.index: 0
- Opaque: ipsec.flags: 0x0, ipsec.sad_index: 0
- Opaque: map.mtu: 0
- Opaque: map_t.v6.saddr: 0x0, map_t.v6.daddr: 0x0, map_t.v6.frag_offset: 0, map_t.v6.l4_offset: 0
- Opaque: map_t.v6.l4_protocol: 0, map_t.checksum_offset: 0, map_t.mtu: 0
- Opaque: ip_frag.mtu: 0, ip_frag.next_index: 0, ip_frag.flags: 0x0
- Opaque: cop.current_config_index: 0
- Opaque: lisp.overlay_afi: 0
- Opaque: tcp.connection_index: 0, tcp.seq_number: 0, tcp.seq_end: 0, tcp.ack_number: 0, tcp.hdr_offset: 0, tcp.data_offset: 0
- Opaque: tcp.data_len: 0, tcp.flags: 0x0
- Opaque: sctp.connection_index: 0, sctp.sid: 0, sctp.ssn: 0, sctp.tsn: 0, sctp.hdr_offset: 0
- Opaque: sctp.data_offset: 0, sctp.data_len: 0, sctp.subconn_idx: 0, sctp.flags: 0x0
- Opaque: snat.flags: 0x0
- Opaque:
- VPP Buffer Opaque2
- Opaque2: raw: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
- Opaque2: qos.bits: 0, qos.source: 0
- Opaque2: loop_counter: 0
- Opaque2: gbp.flags: 0, gbp.src_epg: 0
- Opaque2: pg_replay_timestamp: 0
- Opaque2:
- Ethernet II, Src: 06:d6:01:41:3b:92 (06:d6:01:41:3b:92), Dst: IntelCor_3d:f6 Transmission Control Protocol, Src Port: 22432, Dst Port: 54084, Seq: 1, Ack: 1, Len: 36
- Source Port: 22432
- Destination Port: 54084
- TCP payload (36 bytes)
- Data (36 bytes)
-
- 0000 cf aa 8b f5 53 14 d4 c7 29 75 3e 56 63 93 9d 11 ....S...)u>Vc...
- 0010 e5 f2 92 27 86 56 4c 21 ce c5 23 46 d7 eb ec 0d ...'.VL!..#F....
- 0020 a8 98 36 5a ..6Z
- Data: cfaa8bf55314d4c729753e5663939d11e5f2922786564c21…
- [Length: 36]
-```
-
-It's a matter of a couple of mouse-clicks in Wireshark to filter the
-trace to a specific buffer index. With that specific kind of filtration,
-one can watch a packet walk through the forwarding graph; noting any/all
-metadata changes, header checksum changes, and so forth.
-
-This should be of significant value when developing new vpp graph
-nodes. If new code mispositions b->current_data, it will be completely
-obvious from looking at the dispatch trace in wireshark.
-
-## pcap rx, tx, and drop tracing
-
-vpp also supports rx, tx, and drop packet capture in pcap format,
-through the "pcap trace" debug CLI command.
-
-This command is used to start or stop a packet capture, or show the
-status of packet capture. Each of "pcap trace rx", "pcap trace tx",
-and "pcap trace drop" is implemented. Supply one or more of "rx",
-"tx", and "drop" to enable multiple simultaneous capture types.
-
-These commands have the following optional parameters:
-
-- <b>rx</b> - trace received packets.
-
-- <b>tx</b> - trace transmitted packets.
-
-- <b>drop</b> - trace dropped packets.
-
-- <b>max _nnnn_</b> - file size, number of packet captures. Once
- <nnnn> packets have been received, the trace buffer buffer is flushed
- to the indicated file. Defaults to 1000. Can only be updated if packet
- capture is off.
-
-- <b>max-bytes-per-pkt _nnnn_</b> - maximum number of bytes to trace
- on a per-packet basis. Must be >32 and less than 9000. Default value:
- 512.
-
-- <b>filter</b> - Use the pcap rx / tx / drop trace filter, which must
- be configured. Use <b>classify filter pcap...</b> to configure the
- filter. The filter will only be executed if the per-interface or
- any-interface tests fail.
-
-- <b>intfc _interface_ | _any_</b> - Used to specify a given interface,
- or use '<em>any</em>' to run packet capture on all interfaces.
- '<em>any</em>' is the default if not provided. Settings from a previous
- packet capture are preserved, so '<em>any</em>' can be used to reset
- the interface setting.
-
-- <b>file _filename_</b> - Used to specify the output filename. The
- file will be placed in the '<em>/tmp</em>' directory. If _filename_
- already exists, file will be overwritten. If no filename is
- provided, '<em>/tmp/rx.pcap or tx.pcap</em>' will be used, depending
- on capture direction. Can only be updated when pcap capture is off.
-
-- <b>status</b> - Displays the current status and configured
- attributes associated with a packet capture. If packet capture is in
- progress, '<em>status</em>' also will return the number of packets
- currently in the buffer. Any additional attributes entered on
- command line with a '<em>status</em>' request will be ignored.
-
-- <b>filter</b> - Capture packets which match the current packet
- trace filter set. See next section. Configure the capture filter
- first.
-
-## packet trace capture filtering
-
-The "classify filter pcap | <interface-name> | trace" debug CLI command
-constructs an arbitrary set of packet classifier tables for use with
-"pcap rx | tx | drop trace," and with the vpp packet tracer on a
-per-interface or system-wide basis.
-
-Packets which match a rule in the classifier table chain will be
-traced. The tables are automatically ordered so that matches in the
-most specific table are tried first.
-
-It's reasonably likely that folks will configure a single table with
-one or two matches. As a result, we configure 8 hash buckets and 128K
-of match rule space by default. One can override the defaults by
-specifying "buckets <nnn>" and "memory-size <xxx>" as desired.
-
-To build up complex filter chains, repeatedly issue the classify
-filter debug CLI command. Each command must specify the desired mask
-and match values. If a classifier table with a suitable mask already
-exists, the CLI command adds a match rule to the existing table. If
-not, the CLI command add a new table and the indicated mask rule
-
-### Configure a simple pcap classify filter
-
-```
- classify filter pcap mask l3 ip4 src match l3 ip4 src 192.168.1.11
- pcap trace rx max 100 filter
-```
-
-### Configure a simple per-interface capture filter
-
-```
- classify filter GigabitEthernet3/0/0 mask l3 ip4 src match l3 ip4 src 192.168.1.11"
- pcap trace rx max 100 intfc GigabitEthernet3/0/0
-```
-
-Note that per-interface capture filters are _always_ applied.
-
-### Clear per-interface capture filters
-
-```
- classify filter GigabitEthernet3/0/0 del
-```
-
-### Configure another fairly simple pcap classify filter
-
-```
- classify filter pcap mask l3 ip4 src dst match l3 ip4 src 192.168.1.10 dst 192.168.2.10
- pcap trace tx max 100 filter
-```
-
-### Configure a vpp packet tracer filter
-
-```
- classify filter trace mask l3 ip4 src dst match l3 ip4 src 192.168.1.10 dst 192.168.2.10
- trace add dpdk-input 100 filter
-```
-
-### Clear all current classifier filters
-
-```
- classify filter [pcap | <interface> | trace] del
-```
-
-### To inspect the classifier tables
-
-```
- show classify table [verbose]
-```
-
-The verbose form displays all of the match rules, with hit-counters.
-
-### Terse description of the "mask <xxx>" syntax:
-
-```
- l2 src dst proto tag1 tag2 ignore-tag1 ignore-tag2 cos1 cos2 dot1q dot1ad
- l3 ip4 <ip4-mask> ip6 <ip6-mask>
- <ip4-mask> version hdr_length src[/width] dst[/width]
- tos length fragment_id ttl protocol checksum
- <ip6-mask> version traffic-class flow-label src dst proto
- payload_length hop_limit protocol
- l4 tcp <tcp-mask> udp <udp_mask> src_port dst_port
- <tcp-mask> src dst # ports
- <udp-mask> src_port dst_port
-```
-
-To construct **matches**, add the values to match after the indicated
-keywords in the mask syntax. For example: "... mask l3 ip4 src" ->
-"... match l3 ip4 src 192.168.1.11"
-
-## VPP Packet Generator
-
-We use the VPP packet generator to inject packets into the forwarding
-graph. The packet generator can replay pcap traces, and generate packets
-out of whole cloth at respectably high performance.
-
-The VPP pg enables quite a variety of use-cases, ranging from functional
-testing of new data-plane nodes to regression testing to performance
-tuning.
-
-## PG setup scripts
-
-PG setup scripts describe traffic in detail, and leverage vpp debug
-CLI mechanisms. It's reasonably unusual to construct a pg setup script
-which doesn't include a certain amount of interface and FIB configuration.
-
-For example:
-
-```
- loop create
- set int ip address loop0 192.168.1.1/24
- set int state loop0 up
-
- packet-generator new {
- name pg0
- limit 100
- rate 1e6
- size 300-300
- interface loop0
- node ethernet-input
- data { IP4: 1.2.3 -> 4.5.6
- UDP: 192.168.1.10 - 192.168.1.254 -> 192.168.2.10
- UDP: 1234 -> 2345
- incrementing 286
- }
- }
-```
-
-A packet generator stream definition includes two major sections:
-- Stream Parameter Setup
-- Packet Data
-
-### Stream Parameter Setup
-
-Given the example above, let's look at how to set up stream
-parameters:
-
-- **name pg0** - Name of the stream, in this case "pg0"
-
-- **limit 1000** - Number of packets to send when the stream is
-enabled. "limit 0" means send packets continuously.
-
-- **maxframe \<nnn\>** - Maximum frame size. Handy for injecting
-multiple frames no larger than \<nnn\>. Useful for checking dual /
-quad loop codes
-
-- **rate 1e6** - Packet injection rate, in this case 1 MPPS. When not
-specified, the packet generator injects packets as fast as possible
-
-- **size 300-300** - Packet size range, in this case send 300-byte packets
-
-- **interface loop0** - Packets appear as if they were received on the
-specified interface. This datum is used in multiple ways: to select
-graph arc feature configuration, to select IP FIBs. Configure
-features e.g. on loop0 to exercise those features.
-
-- **tx-interface \<name\>** - Packets will be transmitted on the
-indicated interface. Typically required only when injecting packets
-into post-IP-rewrite graph nodes.
-
-- **pcap \<filename\>** - Replay packets from the indicated pcap
-capture file. "make test" makes extensive use of this feature:
-generate packets using scapy, save them in a .pcap file, then inject
-them into the vpp graph via a vpp pg "pcap \<filename\>" stream
-definition
-
-- **worker \<nn\>** - Generate packets for the stream using the
-indicated vpp worker thread. The vpp pg generates and injects O(10
-MPPS / core). Use multiple stream definitions and worker threads to
-generate and inject enough traffic to easily fill a 40 gbit pipe with
-small packets.
-
-### Data definition
-
-Packet generator data definitions make use of a layered implementation
-strategy. Networking layers are specified in order, and the notation can
-seem a bit counter-intuitive. In the example above, the data
-definition stanza constructs a set of L2-L4 headers layers, and
-uses an incrementing fill pattern to round out the requested 300-byte
-packets.
-
-- **IP4: 1.2.3 -> 4.5.6** - Construct an L2 (MAC) header with the ip4
-ethertype (0x800), src MAC address of 00:01:00:02:00:03 and dst MAC
-address of 00:04:00:05:00:06. Mac addresses may be specified in either
-_xxxx.xxxx.xxxx_ format or _xx:xx:xx:xx:xx:xx_ format.
-
-- **UDP: 192.168.1.10 - 192.168.1.254 -> 192.168.2.10** - Construct an
-incrementing set of L3 (IPv4) headers for successive packets with
-source addresses ranging from .10 to .254. All packets in the stream
-have a constant dest address of 192.168.2.10. Set the protocol field
-to 17, UDP.
-
-- **UDP: 1234 -> 2345** - Set the UDP source and destination ports to
-1234 and 2345, respectively
-
-- **incrementing 256** - Insert up to 256 incrementing data bytes.
-
-Obvious variations involve "s/IP4/IP6/" in the above, along with
-changing from IPv4 to IPv6 address notation.
-
-The vpp pg can set any / all IPv4 header fields, including tos, packet
-length, mf / df / fragment id and offset, ttl, protocol, checksum, and
-src/dst addresses. Take a look at ../src/vnet/ip/ip[46]_pg.c for
-details.
-
-If all else fails, specify the entire packet data in hex:
-
-- **hex 0xabcd...** - copy hex data verbatim into the packet
-
-When replaying pcap files ("**pcap \<filename\>**"), do not specify a
-data stanza.
-
-### Diagnosing "packet-generator new" parse failures
-
-If you want to inject packets into a brand-new graph node, remember
-to tell the packet generator debug CLI how to parse the packet
-data stanza.
-
-If the node expects L2 Ethernet MAC headers, specify ".unformat_buffer
-= unformat_ethernet_header":
-
-```
- /* *INDENT-OFF* */
- VLIB_REGISTER_NODE (ethernet_input_node) =
- {
- <snip>
- .unformat_buffer = unformat_ethernet_header,
- <snip>
- };
-```
-
-Beyond that, it may be necessary to set breakpoints in
-.../src/vnet/pg/cli.c. Debug image suggested.
-
-When debugging new nodes, it may be far simpler to directly inject
-ethernet frames - and add a corresponding vlib_buffer_advance in the
-new node - than to modify the packet generator.
-
-## Debug CLI
-
-The descriptions above describe the "packet-generator new" debug CLI in
-detail.
-
-Additional debug CLI commands include:
-
-```
- vpp# packet-generator enable [<stream-name>]
-```
-
-which enables the named stream, or all streams.
-
-```
- vpp# packet-generator disable [<stream-name>]
-```
-
-disables the named stream, or all streams.
-
-
-```
- vpp# packet-generator delete <stream-name>
-```
-
-Deletes the named stream.
-
-```
- vpp# packet-generator configure <stream-name> [limit <nnn>]
- [rate <f64-pps>] [size <nn>-<nn>]
-```
-
-Changes stream parameters without having to recreate the entire stream
-definition. Note that re-issuing a "packet-generator new" command will
-correctly recreate the named stream.
diff --git a/docs/gettingstarted/developers/vpp_api_module.rst b/docs/gettingstarted/developers/vpp_api_module.rst
deleted file mode 100644
index 3cdea52..0000000
--- a/docs/gettingstarted/developers/vpp_api_module.rst
+++ /dev/null
@@ -1,122 +0,0 @@
-.. _vpp_api_module:
-
-.. toctree::
-
-VPP API module
-==============
-
-Overview
-________
-
-VPP API module allows communicating with VPP over shared memory interface. The API consists of 3 parts:
-
-* common code - low-level API
-* generated code - high-level API
-* code generator - to generate your own high-level API e.g. for custom plugins
-
-Common code
-___________
-
-**C**
-
-C common code represents the basic, low-level API, providing functions to connect/disconnect, perform message discovery and send/receive messages. The C variant is in vapi.h.
-
-**C++**
-
-C++ is provided by vapi.hpp and contains high-level API templates, which are specialized by generated code.
-
-Generated code
-______________
-
-Each API file present in the source tree is automatically translated to JSON file, which the code generator parses and generates either C (vapi_c_gen.py) or C++ (vapi_cpp_gen.py) code.
-
-This can then be included in the client application and provides convenient way to interact with VPP. This includes:
-
-* automatic byte-swapping
-* automatic request-response matching based on context
-* automatic casts to appropriate types (type-safety) when calling callbacks
-* automatic sending of control-pings for dump messages
-
-The API supports two modes of operation:
-
-* blocking
-* non-blocking
-
-In blocking mode, whenever an operation is initiated, the code waits until it can finish. This means that when sending a message, the call blocks until the message can be written to shared memory. Similarly, receiving a message blocks until a message becomes available. On higher level, this also means that when doing a request (e.g. show_version), the call blocks until a response comes back (e.g. show_version_reply).
-
-In non-blocking mode, these are decoupled, the API returns VAPI_EAGAIN whenever an operation cannot be performed and after sending a request, it's up to the client to wait for and process a response.
-
-Code generator
-______________
-
-Python code generator comes in two flavors - C and C++ and generates high-level API headers. All the code is stored in the headers.
-
-C Usage
-_______
-
-**Low-level API**
-
-Refer to inline API documentation in doxygen format in vapi.h header for description of functions. It's recommended to use the safer, high-level API provided by specialized headers (e.g. vpe.api.vapi.h or vpe.api.vapi.hpp).
-
-**C high-level API**
-
-*Callbacks*
-
-The C high-level API is strictly callback-based for maximum efficiency. Whenever an operation is initiated a callback with a callback context is part of that operation. The callback is then invoked when the response (or multiple responses) arrive which are tied to the request. Also, callbacks are invoked whenever an event arrives, if such callback is registered. All the pointers to responses/events point to shared memory and are immediately freed after callback finishes so the client needs to extract/copy any data in which it is interested in.
-
-**Blocking mode**
-
-In simple blocking mode, the whole operation (being a simple request or a dump) is finished and it's callback is called (potentially multiple times for dumps) during function call.
-
-Example pseudo-code for a simple request in this mode:
-
-vapi_show_version(message, callback, callback_context)
-
-#. generate unique internal context and assign it to message.header.context
-#. byteswap the message to network byte order
-#. send message to vpp (message is now consumed and vpp will free it)
-#. create internal "outstanding request context" which stores the callback, callback context and the internal context value
-#. call dispatch, which in this mode receives and processes responses until the internal "outstanding requests" queue is empty. In blocking mode, this queue always contains at most one item.
-
-.. note::
-
- It's possible for different - unrelated callbacks to be called before the response callbacks is called in cases where e.g. events are stored in shared memory queue.
-
-**Non-blocking mode**
-In non-blocking mode, all the requests are only byte-swapped and the context information along with callbacks is stored locally (so in the above example, only steps 1-4 are executed and step 5 is skipped). Calling dispatch is up to the client application. This allows to alternate between sending/receiving messages or have a dedicated thread which calls dispatch.
-
-C++ high level API
-__________________
-
-**Callbacks**
-
-In C++ API, the response is automatically tied to the corresponding Request, Dump or Event_registration object. Optionally a callback might be specified, which then gets called when the response is received.
-
-.. note::
-
- Responses take up shared memory space and should be freed either manually (in case of result sets) or automatically (by destroying the object owning them) when no longer needed. Once a Request or Dump object was executed, it cannot be re-sent, since the request itself (stores in shared memory) is consumed by vpp and inaccessible (set to nullptr) anymore.
-
-C++ Usage
-_________
-
-**Requests & dumps**
-
-*Create an object of Connection type and call connect() to connect to vpp.*
-
-#. Create an object of Request or Dump type using it's typedef (e.g. Show_version)
-#. Use get_request() to obtain and manipulate the underlying request if required.
-#. Issue execute() to send the request.
-#. Use either wait_for_response() or dispatch() to wait for the response.
-#. Use get_response_state() to get the state and get_response() to read the response.
-
-**Events**
-
-*Create a Connection and execute the appropriate Request to subscribe to events (e.g. Want_stats)*
-
-#. Create an Event_registration with a template argument being the type of event you are interested in.
-#. Call dispatch() or wait_for_response() to wait for the event. A callback will be called when an event occurs (if passed to Event_registration() constructor). Alternatively, read the result set.
-
-.. note::
-
- Events stored in the result set take up space in shared memory and should be freed regularly (e.g. in the callback, once the event is processed).
-
diff --git a/docs/gettingstarted/index.rst b/docs/gettingstarted/index.rst
deleted file mode 100644
index 08fc1cd..0000000
--- a/docs/gettingstarted/index.rst
+++ /dev/null
@@ -1,44 +0,0 @@
-.. _gettingstarted:
-
-######################
-Getting Started
-######################
-
-The Getting Started Guide is comprised of several different sections; a User section
-that describes a basic installation and configuration of VPP (either manually or
-using a config utility), another install for Developers, which contains additional
-code that provides tools that are used in a development environment.
-
-This section covers the following:
-
-* Describes how to manually install VPP Binaries on different OS platforms (Ubuntu, Centos) and then how to configure and use VPP.
-* Describes the different types of VPP packages, which are used in both basic and developer installs.
-* A VPP tutorial which is a great way to learn VPP basics.
-
-The Users section covers configuration operations; this
-
-* How to configure and Run VPP manually.
-* How to use the Configuration Utility to install, and then configure VPP.
-
-The Developers section covers the following areas:
-
-* Building VPP
-* Describes the components of the four VPP layers
-* How to Create, Add, Enable/Disable features
-* Discusses different aspects of Bounded-index Extensible Hashing (bihash)
-
-The Writing VPP Documentation section covers the following topics:
-
-* How to build VPP documents
-* How to push your changes to the VPP Docs Repository
-* Identifies the different styles associated to reStructuredText
-* Identifies the different styles associated to Markdown
-
-.. toctree::
- :maxdepth: 2
-
- installing/index.rst
- progressivevpp/index.rst
- users/index.rst
- developers/index.rst
- writingdocs/index.rst
diff --git a/docs/gettingstarted/installing/centos.rst b/docs/gettingstarted/installing/centos.rst
deleted file mode 100644
index 88691a2..0000000
--- a/docs/gettingstarted/installing/centos.rst
+++ /dev/null
@@ -1,176 +0,0 @@
-.. _centos:
-
-.. toctree::
-
-Setup the FD.io Repository - Centos 7
-=====================================
-
-Update the OS
--------------
-
-Before starting the repository setup, it is a good idea to first update and upgrade
-the OS; run the following commands to update the OS and get some packages.
-
-.. code-block:: console
-
- $ sudo yum update
- $ sudo yum install pygpgme yum-utils
-
-
-Package Cloud Repository
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-Build artifacts are also posted to a packagecloud.io Repository. This includes
-official point releases. To use any of these build artifacts, create a file
-*'/etc/yum.repos.d/fdio-release.repo'* with the content that points to the
-version needed. Below are some common examples of the content needed:
-
-
-VPP Latest Release
-""""""""""""""""""
-
-To allow *'yum'* access to the official VPP releases, create the file
-*'/etc/yum.repos.d/fdio-release.repo'* with the following content.
-
-.. code-block:: console
-
- $ cat /etc/yum.repos.d/fdio-release.repo
- [fdio_release]
- name=fdio_release
- baseurl=https://packagecloud.io/fdio/release/el/7/$basearch
- repo_gpgcheck=1
- gpgcheck=0
- enabled=1
- gpgkey=https://packagecloud.io/fdio/release/gpgkey
- sslverify=1
- sslcacert=/etc/pki/tls/certs/ca-bundle.crt
- metadata_expire=300
-
- [fdio_release-source]
- name=fdio_release-source
- baseurl=https://packagecloud.io/fdio/release/el/7/SRPMS
- repo_gpgcheck=1
- gpgcheck=0
- enabled=1
- gpgkey=https://packagecloud.io/fdio/release/gpgkey
- sslverify=1
- sslcacert=/etc/pki/tls/certs/ca-bundle.crt
- metadata_expire=300
-
-Update your local yum cache.
-
-.. code-block:: console
-
- $ sudo yum clean all
- $ sudo yum -q makecache -y --disablerepo='*' --enablerepo='fdio_release'
-
-The *'yum install vpp'* command will install the most recent release. To
-install older releases, run the following command to get the list of releases
-provided.
-
-.. code-block:: console
-
- $ sudo yum --showduplicates list vpp* | expand
-
-VPP Master Branch
-"""""""""""""""""""
-
-To allow *yum* access to the nightly builds from the VPP master branch, create
-the file *'/etc/yum.repos.d/fdio-release.repo'* with the following content.
-
-.. code-block:: console
-
- $ cat /etc/yum.repos.d/fdio-release.repo
- [fdio_master]
- name=fdio_master
- baseurl=https://packagecloud.io/fdio/master/el/7/$basearch
- repo_gpgcheck=1
- gpgcheck=0
- enabled=1
- gpgkey=https://packagecloud.io/fdio/master/gpgkey
- sslverify=1
- sslcacert=/etc/pki/tls/certs/ca-bundle.crt
- metadata_expire=300
-
- [fdio_master-source]
- name=fdio_master-source
- baseurl=https://packagecloud.io/fdio/master/el/7/SRPMS
- repo_gpgcheck=1
- gpgcheck=0
- enabled=1
- gpgkey=https://packagecloud.io/fdio/master/gpgkey
- sslverify=1
- sslcacert=/etc/pki/tls/certs/ca-bundle.crt
- metadata_expire=300
-
-Update your local yum cache.
-
-.. code-block:: console
-
- $ sudo yum clean all
- $ sudo yum -q makecache -y --disablerepo='*' --enablerepo='fdio_master'
-
-The *'yum install vpp'* command will install the most recent build on the
-branch. Run the following command to get the list of images produce by the
-branch.
-
-.. code-block:: console
-
- $ sudo yum clean all
- $ sudo yum --showduplicates list vpp* | expand
-
-Install VPP RPMs
-================
-
-To install the VPP packet engine, run the following command:
-
-.. code-block:: console
-
- $ sudo yum install vpp
-
-The *vpp* RPM depends on the *vpp-lib* and *vpp-selinux-policy*
-RPMs, so they will be installed as well.
-
-.. note::
-
- The *vpp-selinux-policy* will not enable SELinux on the system. It
- will install a Custom VPP SELinux policy that will be used if SELinux is
- enabled at any time.
-
-There are additional packages that are optional. These packages can be
-combined with the command above and installed all at once, or installed as
-needed:
-
-.. code-block:: console
-
- $ sudo yum install vpp-plugins vpp-devel vpp-api-python vpp-api-lua vpp-api-java vpp-debuginfo vpp-devel libvpp0
-
-Starting VPP
-============
-
-Once VPP is installed on the system, to run VPP as a systemd service on CentOS,
-run the following command:
-
-.. code-block:: console
-
- $ sudo systemctl start vpp
-
-Then to enable VPP to start on system reboot, run the following command:
-
-.. code-block:: console
-
- $ sudo systemctl enable vpp
-
-Outside of running VPP as a systemd service, VPP can be started manually or
-made to run within GDB for debugging. See :ref:`running` for more details and
-ways to tailor VPP to a specific system.
-
-
-Uninstall the VPP RPMs
-======================
-
-To uninstall a VPP RPM, run the following command:
-
-.. code-block:: console
-
- $ sudo yum autoremove vpp*
diff --git a/docs/gettingstarted/installing/index.rst b/docs/gettingstarted/installing/index.rst
index b482ea6..ba67375 100644
--- a/docs/gettingstarted/installing/index.rst
+++ b/docs/gettingstarted/installing/index.rst
@@ -27,14 +27,6 @@
ubuntu
-Installing on Centos
--------------------------------
-
-The following are instructions on how to install VPP on Centos.
-
-.. toctree::
-
- centos
Package Descriptions
----------------------------------
diff --git a/docs/gettingstarted/installing/packages.rst b/docs/gettingstarted/installing/packages.rst
index 6c4fae1..7eea532 100644
--- a/docs/gettingstarted/installing/packages.rst
+++ b/docs/gettingstarted/installing/packages.rst
@@ -35,7 +35,7 @@
Vector Packet Processing plugin modules.
-.. include:: ../../dynamic_includes/plugin_list.inc
+.. include:: ../../_generated/includes/plugin_list.inc
vpp-dbg
-------
@@ -48,7 +48,7 @@
Vector Packet Processing development support. This package contains
development support files for the VPP libraries.
-vpp-api-python
+vpp-api-python
--------------
Python binding for the VPP Binary API.
diff --git a/docs/gettingstarted/installing/ubuntu.rst b/docs/gettingstarted/installing/ubuntu.rst
index 2521b6d..f1bdb3e 100644
--- a/docs/gettingstarted/installing/ubuntu.rst
+++ b/docs/gettingstarted/installing/ubuntu.rst
@@ -67,7 +67,7 @@
sudo apt-get update
sudo apt-get install vpp vpp-plugin-core vpp-plugin-dpdk
-
+
Install the Optional Packages
==============================
diff --git a/docs/gettingstarted/progressivevpp/index.rst b/docs/gettingstarted/progressivevpp/index.rst
index ab257f2..7d1a2e2 100644
--- a/docs/gettingstarted/progressivevpp/index.rst
+++ b/docs/gettingstarted/progressivevpp/index.rst
@@ -14,8 +14,6 @@
This is **not** intended to be a 'How to Run in a Production Environment' set of instructions.
-For more information on using VPP with Virtual Box/Vagrant, please refer to :ref:`vppvagrant`
-
.. toctree::
settingupenvironment.rst
diff --git a/docs/gettingstarted/progressivevpp/interface.rst b/docs/gettingstarted/progressivevpp/interface.rst
index 112eaa9..7e35ca8 100644
--- a/docs/gettingstarted/progressivevpp/interface.rst
+++ b/docs/gettingstarted/progressivevpp/interface.rst
@@ -41,7 +41,7 @@
The initial state here is presumed to be the final state from the previous sections
of the tutorial.
-
+
Create veth interfaces on host
-------------------------------
@@ -104,7 +104,7 @@
__/ __/ _ \ (_)__ | | / / _ \/ _ \
_/ _// // / / / _ \ | |/ / ___/ ___/
/_/ /____(_)_/\___/ |___/_/ /_/
-
+
vpp#
Create a host interface attached to **vpp1out**.
diff --git a/docs/gettingstarted/progressivevpp/routing.rst b/docs/gettingstarted/progressivevpp/routing.rst
index c9586c1..38841b7 100644
--- a/docs/gettingstarted/progressivevpp/routing.rst
+++ b/docs/gettingstarted/progressivevpp/routing.rst
@@ -50,10 +50,10 @@
$ sudo ip route add 10.10.2.0/24 via 10.10.1.2
$ ip route
- default via 10.0.2.2 dev enp0s3
- 10.0.2.0/24 dev enp0s3 proto kernel scope link src 10.0.2.15
- 10.10.1.0/24 dev vpp1host proto kernel scope link src 10.10.1.1
- 10.10.2.0/24 via 10.10.1.2 dev vpp1host
+ default via 10.0.2.2 dev enp0s3
+ 10.0.2.0/24 dev enp0s3 proto kernel scope link src 10.0.2.15
+ 10.10.1.0/24 dev vpp1host proto kernel scope link src 10.10.1.1
+ 10.10.2.0/24 via 10.10.1.2 dev vpp1host
Setup return route on vpp2
---------------------------
diff --git a/docs/gettingstarted/progressivevpp/runningvpp.rst b/docs/gettingstarted/progressivevpp/runningvpp.rst
index 19d2f4e..c24d104 100644
--- a/docs/gettingstarted/progressivevpp/runningvpp.rst
+++ b/docs/gettingstarted/progressivevpp/runningvpp.rst
@@ -50,7 +50,7 @@
__/ __/ _ \ (_)__ | | / / _ \/ _ \
_/ _// // / / / _ \ | |/ / ___/ ___/
/_/ /____(_)_/\___/ |___/_/ /_/
-
+
vpp# show version
vpp v18.07-release built by root on c469eba2a593 at Mon Jul 30 23:27:03 UTC 2018
vpp#
diff --git a/docs/gettingstarted/progressivevpp/settingupenvironment.rst b/docs/gettingstarted/progressivevpp/settingupenvironment.rst
index c012f69..8c67c79 100644
--- a/docs/gettingstarted/progressivevpp/settingupenvironment.rst
+++ b/docs/gettingstarted/progressivevpp/settingupenvironment.rst
@@ -11,8 +11,7 @@
Install Virtual Box and Vagrant
-------------------------------
-You will need to install Virtual Box and Vagrant. If you have not installed Virtual Box or Vagrant please
-refer to :ref:`installingVboxVagrant` to install Virtual Box and Vagrant.
+You will need to install Virtual Box and Vagrant.
Create a Vagrant Directory
---------------------------
@@ -30,17 +29,17 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
-
+
Vagrant.configure(2) do |config|
-
+
config.vm.box = "bento/ubuntu-18.04"
config.vm.box_check_update = false
-
+
vmcpu=(ENV['VPP_VAGRANT_VMCPU'] || 2)
vmram=(ENV['VPP_VAGRANT_VMRAM'] || 4096)
-
+
config.ssh.forward_agent = true
-
+
config.vm.provider "virtualbox" do |vb|
vb.customize ["modifyvm", :id, "--ioapic", "on"]
vb.memory = "#{vmram}"
@@ -129,7 +128,7 @@
We will create some startup files for the use of this tutorial. Typically you will
modify the startup.conf file found in /etc/vpp/startup.conf. For more information
-on this file refer to :ref:`startup`.
+on this file refer to :ref:`configuration_reference`.
When running multiple VPP instances, each instance needs to have
specified a 'name' or 'prefix'. In the example below, the 'name' or 'prefix'
diff --git a/docs/gettingstarted/progressivevpp/switching.rst b/docs/gettingstarted/progressivevpp/switching.rst
index a28e860..7ee5598 100644
--- a/docs/gettingstarted/progressivevpp/switching.rst
+++ b/docs/gettingstarted/progressivevpp/switching.rst
@@ -80,8 +80,8 @@
.. code-block:: console
vpp# show bridge-domain
- ID Index Learning U-Forwrd UU-Flood Flooding ARP-Term BVI-Intf
- 0 0 off off off off off local0
+ ID Index Learning U-Forwrd UU-Flood Flooding ARP-Term BVI-Intf
+ 0 0 off off off off off local0
In the example above, there is bridge domain ID '0' already. Even though
sometimes we might get feedback as below:
@@ -175,15 +175,15 @@
.. code-block:: console
vpp# show l2fib verbose
- Mac Address BD Idx Interface Index static filter bvi Mac Age (min)
- de:ad:00:00:00:00 1 host-vpp1vpp2 2 0 0 0 disabled
- c2:f6:88:31:7b:8e 1 host-vpp1out 1 0 0 0 disabled
+ Mac Address BD Idx Interface Index static filter bvi Mac Age (min)
+ de:ad:00:00:00:00 1 host-vpp1vpp2 2 0 0 0 disabled
+ c2:f6:88:31:7b:8e 1 host-vpp1out 1 0 0 0 disabled
2 l2fib entries
.. code-block:: console
vpp# show l2fib verbose
- Mac Address BD Idx Interface Index static filter bvi Mac Age (min)
- de:ad:00:00:00:00 1 loop0 2 1 0 1 disabled
- c2:f6:88:31:7b:8e 1 host-vpp2vpp1 1 0 0 0 disabled
+ Mac Address BD Idx Interface Index static filter bvi Mac Age (min)
+ de:ad:00:00:00:00 1 loop0 2 1 0 1 disabled
+ c2:f6:88:31:7b:8e 1 host-vpp2vpp1 1 0 0 0 disabled
2 l2fib entries
diff --git a/docs/gettingstarted/progressivevpp/traces.rst b/docs/gettingstarted/progressivevpp/traces.rst
index a340858..d1dfad5 100644
--- a/docs/gettingstarted/progressivevpp/traces.rst
+++ b/docs/gettingstarted/progressivevpp/traces.rst
@@ -2,7 +2,7 @@
.. toctree::
-Using the trace command
+Using the trace command
========================
Skills to be Learned
@@ -21,26 +21,26 @@
Show trace buffer [max COUNT].
-.. code-block:: console
+.. code-block:: console
vpp# show trace
Clear trace buffer and free memory.
-.. code-block:: console
+.. code-block:: console
vpp# clear trace
filter trace output - include NODE COUNT | exclude NODE COUNT | none.
-.. code-block:: console
+.. code-block:: console
vpp# trace filter <include NODE COUNT | exclude NODE COUNT | none>
Add Trace
----------
-.. code-block:: console
+.. code-block:: console
vpp# trace add af-packet-input 10
@@ -69,7 +69,7 @@
Ping from Host to VPP
----------------------
-.. code-block:: console
+.. code-block:: console
vpp# q
$ ping -c 1 10.10.1.2
@@ -80,10 +80,10 @@
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.283/0.283/0.283/0.000 ms
-Examine Trace of ping from host to VPP
+Examine Trace of ping from host to VPP
----------------------------------------
-.. code-block:: console
+.. code-block:: console
$ sudo vppctl -s /run/vpp/cli-vpp1.sock
vpp# show trace
@@ -144,14 +144,14 @@
Clear trace buffer
-------------------
-.. code-block:: console
+.. code-block:: console
vpp# clear trace
-Ping from VPP to Host
+Ping from VPP to Host
-----------------------
-.. code-block:: console
+.. code-block:: console
vpp# ping 10.10.1.1
64 bytes from 10.10.1.1: icmp_seq=1 ttl=64 time=.0789 ms
@@ -167,7 +167,7 @@
The output will demonstrate FD.io VPP's trace of ping for all packets.
-.. code-block:: console
+.. code-block:: console
vpp# show trace
------------------- Start of thread 0 vpp_main -------------------
@@ -465,16 +465,16 @@
Examine ARP table (neighbors)
-----------------------------
-.. code-block:: console
+.. code-block:: console
vpp# show ip neighbors
Time IP4 Flags Ethernet Interface
- 1101.5636 10.10.1.1 D e2:0f:1e:59:ec:f7 host-vpp1out
+ 1101.5636 10.10.1.1 D e2:0f:1e:59:ec:f7 host-vpp1out
Examine routing tables
-----------------------
-.. code-block:: console
+.. code-block:: console
vpp# show ip fib
ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] locks:[src:plugin-hi:2, src:adjacency:1, src:default-route:1, ]
diff --git a/docs/gettingstarted/progressivevpp/twovppinstances.rst b/docs/gettingstarted/progressivevpp/twovppinstances.rst
index c6a7420..eadd4e6 100644
--- a/docs/gettingstarted/progressivevpp/twovppinstances.rst
+++ b/docs/gettingstarted/progressivevpp/twovppinstances.rst
@@ -64,7 +64,7 @@
__/ __/ _ \ (_)__ | | / / _ \/ _ \
_/ _// // / / / _ \ | |/ / ___/ ___/
/_/ /____(_)_/\___/ |___/_/ /_/
-
+
vpp# show version
vpp v18.07-release built by root on c469eba2a593 at Mon Jul 30 23:27:03 UTC 2018
vpp# quit
@@ -121,7 +121,7 @@
.. code-block:: console
- $ ping 10.10.2.2
+ $ ping 10.10.2.2
Ping 10.10.2.1 from vpp2
diff --git a/docs/gettingstarted/running/index.rst b/docs/gettingstarted/running/index.rst
new file mode 100644
index 0000000..626be6c
--- /dev/null
+++ b/docs/gettingstarted/running/index.rst
@@ -0,0 +1,121 @@
+.. _running:
+
+Running VPP
+===========
+
+.. toctree::
+
+Usergroup
+---------
+
+When VPP is installed, a new usergroup *'vpp'* is created. To avoid running the
+VPP CLI (vppctl) as root, add any existing users to the new group that need to
+interact with VPP:
+
+.. code-block:: console
+
+ $ sudo usermod -a -G vpp user1
+
+Update your current session for the group change to take effect:
+
+.. code-block:: console
+
+ $ newgrp vpp
+
+
+Systemd File vpp.service
+------------------------
+
+When the VPP is installed, a systemd service files is also installed. This
+file, vpp.service (Ubuntu: /lib/systemd/system/vpp.service and CentOS:
+/usr/lib/systemd/system/vpp.service), controls how VPP is run as a service. For
+example, whether or not to restart on failure and if so, with how much delay.
+Also, which UIO driver should be loaded and location of the *'startup.conf'*
+file.
+
+.. code-block:: console
+
+ $ cat /usr/lib/systemd/system/vpp.service
+ [Unit]
+ Description=Vector Packet Processing Process
+ After=syslog.target network.target auditd.service
+
+ [Service]
+ ExecStartPre=-/bin/rm -f /dev/shm/db /dev/shm/global_vm /dev/shm/vpe-api
+ ExecStartPre=-/sbin/modprobe uio_pci_generic
+ ExecStart=/usr/bin/vpp -c /etc/vpp/startup.conf
+ Type=simple
+ Restart=on-failure
+ RestartSec=5s
+
+ [Install]
+ WantedBy=multi-user.target
+
+.. note::
+
+ Some older versions of the *'uio_pci_generic'* driver don't bind all
+ the supported NICs properly, so the *'igb_uio'* driver built from DPDK
+ needs to be installed. This file controls which driver is loaded at boot.
+ *'startup.conf'* file controls which driver is used.
+
+Huge Pages
+----------
+
+VPP requires *hugepages* to run during VPP operation, to manage large pages of memory.
+During VPP installation, VPP will overwrite the existing hugepage settings.
+By default, VPP sets the number of hugepages on a system to 1024 2M hugepages.
+This is the number of hugepages on the system, not just used by VPP.
+
+When VPP is installed, the following configuration file is copied to the system. The
+hugepage settings are applied in the VPP installation and on system reboots. To set
+the hugepage settings, perform the following commands:
+
+.. code-block:: console
+
+ $ cat /etc/sysctl.d/80-vpp.conf
+ # Number of 2MB hugepages desired
+ vm.nr_hugepages=1024
+
+ # Must be greater than or equal to (2 * vm.nr_hugepages).
+ vm.max_map_count=3096
+
+ # All groups allowed to access hugepages
+ vm.hugetlb_shm_group=0
+
+ # Shared Memory Max must be greater or equal to the total size of hugepages.
+ # For 2MB pages, TotalHugepageSize = vm.nr_hugepages * 2 * 1024 * 1024
+ # If the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax)
+ # is greater than the calculated TotalHugepageSize then set this parameter
+ # to current shmmax value.
+ kernel.shmmax=2147483648
+
+Depending on how the system is being used, this configuration file can be updated to adjust
+the number of hugepages reserved on a system. Below are some examples of
+possible settings.
+
+For a small VM with minimal workload:
+
+.. code-block:: console
+
+ vm.nr_hugepages=512
+ vm.max_map_count=2048
+ kernel.shmmax=1073741824
+
+For a large system running multiple VMs, each needing its own set of hugepages:
+
+.. code-block:: console
+
+ vm.nr_hugepages=32768
+ vm.max_map_count=66560
+ kernel.shmmax=68719476736
+
+
+.. note::
+
+ If VPP is being run in a Virtual Machine (VM), the VM must have hugepage
+ backing. When VPP is installed, it will attempt to overwrite existing the
+ hugepage setting. If the VM does not have hugepage backing, the install will fail,
+ but the failure may go unnoticed. When the VM is rebooted, on system startup,
+ *'vm.nr_hugepages'* will be reapplied, and will fail, and the VM will abort kernel
+ boot, locking up the VM. To avoid this scenario, ensure the VM has enough
+ hugepage backing.
diff --git a/docs/troubleshooting/cpuusage.rst b/docs/gettingstarted/troubleshooting/cpuusage.rst
similarity index 97%
rename from docs/troubleshooting/cpuusage.rst
rename to docs/gettingstarted/troubleshooting/cpuusage.rst
index b9b8942..9b4514e 100644
--- a/docs/troubleshooting/cpuusage.rst
+++ b/docs/gettingstarted/troubleshooting/cpuusage.rst
@@ -58,7 +58,7 @@
alloc. from vector-expand: 3430
allocs: 52335 2027.54 clocks/call
frees: 30291 594.36 clocks/call
-
+
VPP CPU Load
============
@@ -83,7 +83,7 @@
Thread 0 vpp_main (lcore 8)
Time 6152.9, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
+ Name State Calls Vectors Suspends Clocks Vectors/Call
acl-plugin-fa-cleaner-process event wait 0 0 1 3.66e4 0.00
admin-up-down-process event wait 0 0 1 2.54e3 0.00
....
@@ -91,7 +91,7 @@
Thread 1 vpp_wk_0 (lcore 9)
Time 6152.9, average vectors/node 1.00, last 128 main loops 0.00 per node 0.00
vector rates in 1.3073e2, out 1.3073e2, drop 6.5009e-4, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
+ Name State Calls Vectors Suspends Clocks Vectors/Call
TenGigabitEthernet86/0/0-outpu active 804395 804395 0 6.17e2 1.00
TenGigabitEthernet86/0/0-tx active 804395 804395 0 7.29e2 1.00
arp-input active 2 2 0 3.82e4 1.00
diff --git a/docs/troubleshooting/index.rst b/docs/gettingstarted/troubleshooting/index.rst
similarity index 88%
rename from docs/troubleshooting/index.rst
rename to docs/gettingstarted/troubleshooting/index.rst
index 5dee98a..d70c190 100644
--- a/docs/troubleshooting/index.rst
+++ b/docs/gettingstarted/troubleshooting/index.rst
@@ -9,7 +9,6 @@
.. toctree::
- reportingissues/index.rst
cpuusage
sanitizer
mem
diff --git a/docs/troubleshooting/mem.rst b/docs/gettingstarted/troubleshooting/mem.rst
similarity index 95%
rename from docs/troubleshooting/mem.rst
rename to docs/gettingstarted/troubleshooting/mem.rst
index 207b277..630b0af 100644
--- a/docs/troubleshooting/mem.rst
+++ b/docs/gettingstarted/troubleshooting/mem.rst
@@ -66,7 +66,7 @@
==================
Internal VPP memory allocations rely on VPP main-heap, however when using
-external libraries, esp. in plugins (eg. OpenSSL library used by the IKEv2
+external libraries, esp. in plugins (e.g. OpenSSL library used by the IKEv2
plugin), those external libraries usually manages memory using the standard
libc malloc()/free()/... calls. This, in turn, makes use of the default
libc heap.
@@ -78,7 +78,7 @@
replaces standard libc memory management calls with version using VPP
main-heap.
-To use it, you need to use the `LD_PRELOAD` mechanism, eg.
+To use it, you need to use the `LD_PRELOAD` mechanism, e.g.
.. code-block:: console
diff --git a/docs/troubleshooting/sanitizer.rst b/docs/gettingstarted/troubleshooting/sanitizer.rst
similarity index 100%
rename from docs/troubleshooting/sanitizer.rst
rename to docs/gettingstarted/troubleshooting/sanitizer.rst
diff --git a/docs/gettingstarted/users/configuring/hugepages.rst b/docs/gettingstarted/users/configuring/hugepages.rst
deleted file mode 100644
index 7a1e0bd..0000000
--- a/docs/gettingstarted/users/configuring/hugepages.rst
+++ /dev/null
@@ -1,63 +0,0 @@
-.. _hugepages:
-
-Huge Pages
-----------
-
-VPP requires *hugepages* to run during VPP operation, to manage large pages of memory.
-During VPP installation, VPP will overwrite the existing hugepage settings.
-By default, VPP sets the number of hugepages on a system to 1024 2M hugepages.
-This is the number of hugepages on the system, not just used by VPP.
-
-When VPP is installed, the following configuration file is copied to the system. The
-hugepage settings are applied in the VPP installation and on system reboots. To set
-the hugepage settings, perform the following commands:
-
-.. code-block:: console
-
- $ cat /etc/sysctl.d/80-vpp.conf
- # Number of 2MB hugepages desired
- vm.nr_hugepages=1024
-
- # Must be greater than or equal to (2 * vm.nr_hugepages).
- vm.max_map_count=3096
-
- # All groups allowed to access hugepages
- vm.hugetlb_shm_group=0
-
- # Shared Memory Max must be greater or equal to the total size of hugepages.
- # For 2MB pages, TotalHugepageSize = vm.nr_hugepages * 2 * 1024 * 1024
- # If the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax)
- # is greater than the calculated TotalHugepageSize then set this parameter
- # to current shmmax value.
- kernel.shmmax=2147483648
-
-Depending on how the system is being used, this configuration file can be updated to adjust
-the number of hugepages reserved on a system. Below are some examples of
-possible settings.
-
-For a small VM with minimal workload:
-
-.. code-block:: console
-
- vm.nr_hugepages=512
- vm.max_map_count=2048
- kernel.shmmax=1073741824
-
-For a large system running multiple VMs, each needing its own set of hugepages:
-
-.. code-block:: console
-
- vm.nr_hugepages=32768
- vm.max_map_count=66560
- kernel.shmmax=68719476736
-
-
-.. note::
-
- If VPP is being run in a Virtual Machine (VM), the VM must have hugepage
- backing. When VPP is installed, it will attempt to overwrite existing the
- hugepage setting. If the VM does not have hugepage backing, the install will fail,
- but the failure may go unnoticed. When the VM is rebooted, on system startup,
- *'vm.nr_hugepages'* will be reapplied, and will fail, and the VM will abort kernel
- boot, locking up the VM. To avoid this scenario, ensure the VM has enough
- hugepage backing.
diff --git a/docs/gettingstarted/users/configuring/index.rst b/docs/gettingstarted/users/configuring/index.rst
deleted file mode 100644
index 0de77ad..0000000
--- a/docs/gettingstarted/users/configuring/index.rst
+++ /dev/null
@@ -1,13 +0,0 @@
-.. _configuring:
-
-Configuring VPP
-==================
-
-There is some basic configuration that needs to be performed before running
-FD.io VPP. This section describes the configuration process:
-
-.. toctree::
- :maxdepth: 2
-
- hugepages
- startup
diff --git a/docs/gettingstarted/users/index.rst b/docs/gettingstarted/users/index.rst
deleted file mode 100644
index e04889d..0000000
--- a/docs/gettingstarted/users/index.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-.. _users:
-
-###########
-For Users
-###########
-
-The Users section describe a basic VPP installation and configuration operation.
-The installation and configuration of VPP can be done either manually, or by
-using a configuration utility.
-
-This section covers the following areas:
-
-The Users section covers basic VPP installation and configuration operations. This
-section covers the following areas:
-
-* Describes the different types of VPP packages
-* Describes how to manually install VPP Binaries on different OS platforms (Ubuntu, Centos, openSUSE)
-* Explains how to manually configure, then run VPP
-* Explains how to install, then configure VPP using the Configuration Utility
-
-.. toctree::
- :maxdepth: 2
-
- configuring/index.rst
- running/index.rst
diff --git a/docs/gettingstarted/users/running/index.rst b/docs/gettingstarted/users/running/index.rst
deleted file mode 100644
index b36773e..0000000
--- a/docs/gettingstarted/users/running/index.rst
+++ /dev/null
@@ -1,59 +0,0 @@
-.. _running:
-
-Running VPP
-===========
-
-.. toctree::
-
-'vpp' Usergroup
----------------
-
-When VPP is installed, a new usergroup *'vpp'* is created. To avoid running the
-VPP CLI (vppctl) as root, add any existing users to the new group that need to
-interact with VPP:
-
-.. code-block:: console
-
- $ sudo usermod -a -G vpp user1
-
-Update your current session for the group change to take effect:
-
-.. code-block:: console
-
- $ newgrp vpp
-
-
-VPP Systemd File - 'vpp.service'
---------------------------------
-
-When the VPP is installed, a systemd service files is also installed. This
-file, vpp.service (Ubuntu: /lib/systemd/system/vpp.service and CentOS:
-/usr/lib/systemd/system/vpp.service), controls how VPP is run as a service. For
-example, whether or not to restart on failure and if so, with how much delay.
-Also, which UIO driver should be loaded and location of the *'startup.conf'*
-file.
-
-.. code-block:: console
-
- $ cat /usr/lib/systemd/system/vpp.service
- [Unit]
- Description=Vector Packet Processing Process
- After=syslog.target network.target auditd.service
-
- [Service]
- ExecStartPre=-/bin/rm -f /dev/shm/db /dev/shm/global_vm /dev/shm/vpe-api
- ExecStartPre=-/sbin/modprobe uio_pci_generic
- ExecStart=/usr/bin/vpp -c /etc/vpp/startup.conf
- Type=simple
- Restart=on-failure
- RestartSec=5s
-
- [Install]
- WantedBy=multi-user.target
-
-.. note::
-
- Some older versions of the *'uio_pci_generic'* driver don't bind all
- the supported NICs properly, so the *'igb_uio'* driver built from DPDK
- needs to be installed. This file controls which driver is loaded at boot.
- *'startup.conf'* file controls which driver is used.
diff --git a/docs/gettingstarted/writingdocs/buildingrst.rst b/docs/gettingstarted/writingdocs/buildingrst.rst
deleted file mode 100644
index 474257d..0000000
--- a/docs/gettingstarted/writingdocs/buildingrst.rst
+++ /dev/null
@@ -1,94 +0,0 @@
-.. _buildingrst:
-
-**************************
-Creating VPP Documents
-**************************
-
-These instructions show how the VPP documentation sources are built.
-
-The VPP Documents are written using `reStructuredText <http://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html>`_ (rst),
-or markdown (md). These files are then built using the Sphinx build system `Sphinx <http://www.sphinx-doc.org/en/master/>`_.
-
-Get the VPP sources
-=====================
-
-Start with a clone of the vpp repository.
-
-.. code-block:: console
-
- $ git clone https://gerrit.fd.io/r/vpp
- $ cd vpp
-
-
-Install the Necessary Packages
-===============================
-
-Before building the docs there are some packages that are needed. To install
-these packages on ubuntu execute the following.
-
-.. code-block:: console
-
- $ sudo apt-get install python3-all python3-setuptools python3-pip
-
-
-Create a Virtual Environment using virtualenv
-===============================================
-
-For more information on how to use the Python virtual environment check out
-`Installing packages using pip and virtualenv`_.
-
-.. _`Installing packages using pip and virtualenv`: https://packaging.python.org/guides/installing-using-pip-and-virtualenv/
-
-In the vpp root directory on your system, run:
-
-.. code-block:: console
-
- $ make docs-venv
-
-Which installs all the required applications into it's own, isolated, virtual environment, so as to not
-interfere with other builds that may use different versions of software.
-
-Build the html files
-======================
-
-Build the html **index.html** file:
-
-.. code-block:: console
-
- $ make docs
-
-Clean the environment
-======================
-
-Delete all the generated files with the following:
-
-.. code-block:: console
-
- $ make docs-clean
-
-View the results
-=================
-
-| If there are no errors during the build process, you should now have an **index.html** file in your
-| **vpp/docs/_build/html** directory, which you can then view in your browser.
-
-.. figure:: /_images/htmlBuild.png
- :alt: Figure: My directory containing the index.html file
- :scale: 35%
- :align: center
-
-Whenever you make changes to your **.rst** files that you want to see, repeat this build process.
-
-.. note::
-
- To exit from the virtual environment execute:
-
-.. code-block:: console
-
- $ deactivate
-
-Getting your documents reviewed and merged
-==========================================
-
-VPP documents are reviewed and merged like and other source code. Refer to :ref:`gitreview`
-to get your changes reviewed and merged.
diff --git a/docs/gettingstarted/writingdocs/index.rst b/docs/gettingstarted/writingdocs/index.rst
deleted file mode 100644
index 45a746d..0000000
--- a/docs/gettingstarted/writingdocs/index.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. _writingdocs:
-
-#########################
-Writing Documents
-#########################
-
-This section covers the following topics:
-
-* Building VPP Documents
-* Merging FD.io VPP documents and performing a Git review
-* reStructured Text Style Guide that describes different format styles
-* Markdown Style Guide that describes different format styles
-
-
-.. toctree::
- :maxdepth: 3
-
- buildingrst
- styleguide/index.rst
- styleguidemd/index.rst
diff --git a/docs/gettingstarted/writingdocs/styleguide/index.rst b/docs/gettingstarted/writingdocs/styleguide/index.rst
deleted file mode 100644
index 240b3a2..0000000
--- a/docs/gettingstarted/writingdocs/styleguide/index.rst
+++ /dev/null
@@ -1,31 +0,0 @@
-.. _styleguide:
-
-#############################
-reStructured Text Style Guide
-#############################
-
-Most of the these documents are written ins reStructured Text (rst). This chapter describes some of
-the Sphinx Markup Constructs used in these documents. The Sphinx style guide can be found at:
-`Sphinx Style Guide <http://documentation-style-guide-sphinx.readthedocs.io/en/latest/style-guide.html>`_
-For a more detailed list of Sphinx Markup Constructs please refer to:
-`Sphinx Markup Constructs <http://www.sphinx-doc.org/en/stable/markup/index.html>`_
-
-This document is also an example of a directory structure for a document that spans multiple pages.
-Notice we have the file **index.rst** and the then documents that are referenced in index.rst. The
-referenced documents are shown at the bottom of this page.
-
-A label is shown at the top of this page. Then the first construct describes a the document title
-**FD.io Style Guide**. Text usually follows under each title or heading.
-
-A **Table of Contents** structure is shown below. Using **toctree** in this way will show the headings
-in a nicely in the generated documents.
-
-.. toctree::
- :maxdepth: 2
-
- styleguide.rst
- styleguide02.rst
- styleguide02table.rst
- styleguide03.rst
- styleguide04.rst
- styleguide05.rst
diff --git a/docs/gettingstarted/writingdocs/styleguide/iperf-vm.xml b/docs/gettingstarted/writingdocs/styleguide/iperf-vm.xml
deleted file mode 100644
index be354c5..0000000
--- a/docs/gettingstarted/writingdocs/styleguide/iperf-vm.xml
+++ /dev/null
@@ -1,106 +0,0 @@
-<domain type='kvm' id='54'>
- <name>iperf-server</name>
- <memory unit='KiB'>1048576</memory>
- <currentMemory unit='KiB'>1048576</currentMemory>
- <memoryBacking>
- <hugepages>
- <page size='2048' unit='KiB'/>
- </hugepages>
- </memoryBacking>
- <vcpu placement='static'>1</vcpu>
- <resource>
- <partition>/machine</partition>
- </resource>
- <os>
- <type arch='x86_64' machine='pc-i440fx-xenial'>hvm</type>
- <boot dev='hd'/>
- </os>
- <features>
- <acpi/>
- <apic/>
- </features>
- <cpu mode='host-model'>
- <model fallback='allow'></model>
- <numa>
- <cell id='0' cpus='0' memory='262144' unit='KiB' memAccess='shared'/>
- </numa>
- </cpu>
- <clock offset='utc'>
- <timer name='rtc' tickpolicy='catchup'/>
- <timer name='pit' tickpolicy='delay'/>
- <timer name='hpet' present='no'/>
- </clock>
- <on_poweroff>destroy</on_poweroff>
- <on_reboot>restart</on_reboot>
- <on_crash>restart</on_crash>
- <pm>
- <suspend-to-mem enabled='no'/>
- <suspend-to-disk enabled='no'/>
- </pm>
- <devices>
- <emulator>/usr/bin/kvm</emulator>
- <disk type='file' device='disk'>
- <driver name='qemu' type='qcow2'/>
- <source file='/tmp/xenial-mod.img'/>
- <backingStore/>
- <target dev='vda' bus='virtio'/>
- <alias name='virtio-disk0'/>
- <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
- </disk>
- <disk type='file' device='cdrom'>
- <driver name='qemu' type='raw'/>
- <source file='/scratch/jdenisco/sae/configs/cloud-config.iso'/>
- <backingStore/>
- <target dev='hda' bus='ide'/>
- <readonly/>
- <alias name='ide0-0-0'/>
- <address type='drive' controller='0' bus='0' target='0' unit='0'/>
- </disk>
- <controller type='usb' index='0' model='ich9-ehci1'>
- <alias name='usb'/>
- <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x7'/>
- </controller>
- <controller type='pci' index='0' model='pci-root'>
- <alias name='pci.0'/>
- </controller>
- <controller type='ide' index='0'>
- <alias name='ide'/>
- <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
- </controller>
- <controller type='virtio-serial' index='0'>
- <alias name='virtio-serial0'/>
- <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
- </controller>
- <interface type='vhostuser'>
- <mac address='52:54:00:4c:47:f2'/>
- <source type='unix' path='/tmp//vm00.sock' mode='server'/>
- <model type='virtio'/>
- <alias name='net1'/>
- <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
- </interface>
- <serial type='pty'>
- <source path='/dev/pts/2'/>
- <target port='0'/>
- <alias name='serial0'/>
- </serial>
- <console type='pty' tty='/dev/pts/2'>
- <source path='/dev/pts/2'/>
- <target type='serial' port='0'/>
- <alias name='serial0'/>
- </console>
- <input type='mouse' bus='ps2'/>
- <input type='keyboard' bus='ps2'/>
- <graphics type='vnc' port='5900' autoport='yes' listen='127.0.0.1'>
- <listen type='address' address='127.0.0.1'/>
- </graphics>
- <memballoon model='virtio'>
- <alias name='balloon0'/>
- <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
- </memballoon>
- </devices>
- <seclabel type='dynamic' model='apparmor' relabel='yes'>
- <label>libvirt-2c4c9317-c7a5-4b37-b789-386ccda7348a</label>
- <imagelabel>libvirt-2c4c9317-c7a5-4b37-b789-386ccda7348a</imagelabel>
- </seclabel>
-</domain>
-
diff --git a/docs/gettingstarted/writingdocs/styleguide/rawhtml.rst b/docs/gettingstarted/writingdocs/styleguide/rawhtml.rst
deleted file mode 100644
index e94eb45..0000000
--- a/docs/gettingstarted/writingdocs/styleguide/rawhtml.rst
+++ /dev/null
@@ -1,10 +0,0 @@
-.. _rawhtml01:
-
-Raw HTML Example
-================
-
-This example shows how to include include a CSIT performance graph.
-
-.. raw:: html
-
- <iframe src="https://docs.fd.io/csit/rls1801/report/_static/vpp/64B-1t1c-l2-sel2-ndrdisc.html" width="850" height="1000" frameborder="0">
diff --git a/docs/gettingstarted/writingdocs/styleguide/styleguide.rst b/docs/gettingstarted/writingdocs/styleguide/styleguide.rst
deleted file mode 100644
index 2312ea9..0000000
--- a/docs/gettingstarted/writingdocs/styleguide/styleguide.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. _styleguide01:
-
-*********
-Heading 1
-*********
-
-This is the top heading level. More levels are shown below.
-
-Heading 2
-=========
-
-Heading 3
----------
-
-Heading 4
-^^^^^^^^^
-
-Heading 5
-"""""""""
-
diff --git a/docs/gettingstarted/writingdocs/styleguide/styleguide02.rst b/docs/gettingstarted/writingdocs/styleguide/styleguide02.rst
deleted file mode 100644
index af92c93..0000000
--- a/docs/gettingstarted/writingdocs/styleguide/styleguide02.rst
+++ /dev/null
@@ -1,83 +0,0 @@
-.. _styleguide02:
-
-*************************
-Bullets, Bold and Italics
-*************************
-
-Bold text can be show with **Bold Text**, Italics with *Italic text*. Bullets like so:
-
-* Bullet 1
-* Bullet 2
-
-#. Numbered Bullet 1
-#. Numbered Bullet 2
-
-*****
-Notes
-*****
-
-A note can be used to describe something not in the normal flow of the paragraph. This
-is an example of a note.
-
-.. note::
-
- Using **git commit** after adding your files saves a "Snapshot" of them, so it's very hard
- to lose your work if you *commit often*.
-
-***********
-Code Blocks
-***********
-
-This paragraph describes how to do **Console Commands**. When showing VPP commands it is recommended
-that the command be executed from the linux console as shown. The Highlighting in the final documents
-shows up nicely this way.
-
-.. code-block:: console
-
- $ sudo bash
- # vppctl show interface
- Name Idx State Counter Count
- TenGigabitEthernet86/0/0 1 up rx packets 6569213
- rx bytes 9928352943
- tx packets 50384
- tx bytes 3329279
- TenGigabitEthernet86/0/1 2 down
- VirtualEthernet0/0/0 3 up rx packets 50384
- rx bytes 3329279
- tx packets 6569213
- tx bytes 9928352943
- drops 1498
- local0 0 down
- #
-
-
-The **code-block** construct is also used for code samples. The following shows how to include a block of
-"C" code.
-
-.. code-block:: c
-
- #include <vlib/unix/unix.h>
- abf_policy_t *
- abf_policy_get (u32 index)
- {
- return (pool_elt_at_index (abf_policy_pool, index));
- }
-
-Diffs are generated in the final docs nicely with "::" at the end of the description like so::
-
- diff --git a/src/vpp/vnet/main.c b/src/vpp/vnet/main.c
- index 6e136e19..69189c93 100644
- --- a/src/vpp/vnet/main.c
- +++ b/src/vpp/vnet/main.c
- @@ -18,6 +18,8 @@
- #include <vlib/unix/unix.h>
- #include <vnet/plugin/plugin.h>
- #include <vnet/ethernet/ethernet.h>
- +#include <vnet/ip/ip4_packet.h>
- +#include <vnet/ip/format.h>
- #include <vpp/app/version.h>
- #include <vpp/api/vpe_msg_enum.h>
- #include <limits.h>
- @@ -400,6 +402,63 @@ VLIB_CLI_COMMAND (test_crash_command, static) = {
-
- #endif
diff --git a/docs/gettingstarted/writingdocs/styleguide/styleguide02table.rst b/docs/gettingstarted/writingdocs/styleguide/styleguide02table.rst
deleted file mode 100644
index 742888b..0000000
--- a/docs/gettingstarted/writingdocs/styleguide/styleguide02table.rst
+++ /dev/null
@@ -1,76 +0,0 @@
-.. _styleguide02table:
-
-******
-Tables
-******
-
-There are two types of tables with different syntax, `Grid Tables <http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#grid-tables>`_, and `Simple Tables <http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#simple-tables>`_.
-
-Grid Tables
-___________
-
-`Grid Tables <http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#grid-tables>`_ are described with a visual grid made up of the characters "-", "=", "|", and "+". The hyphen ("-") is used for horizontal lines (row separators). The equals sign ("=") may be used to separate optional header rows from the table body. The vertical bar ("|") is used for vertical lines (column separators). The plus sign ("+") is used for intersections of horizontal and vertical lines.
-
-Here is example code for a grid table in a *.rst* file:
-
-.. code-block:: console
-
- +------------------------+------------+----------+----------+
- | Header row, column 1 | Header 2 | Header 3 | Header 4 |
- | (header rows optional) | | | |
- +========================+============+==========+==========+
- | body row 1, column 1 | column 2 | column 3 | column 4 |
- +------------------------+------------+----------+----------+
- | body row 2 | Cells may span columns. |
- +------------------------+------------+---------------------+
- | body row 3 | Cells may | - Table cells |
- +------------------------+ span rows. | - contain |
- | body row 4 | | - body elements. |
- +------------------------+------------+---------------------+
-
-This example code generates a grid table that looks like this:
-
-+------------------------+------------+----------+----------+
-| Header row, column 1 | Header 2 | Header 3 | Header 4 |
-| (header rows optional) | | | |
-+========================+============+==========+==========+
-| body row 1, column 1 | column 2 | column 3 | column 4 |
-+------------------------+------------+----------+----------+
-| body row 2 | Cells may span columns. |
-+------------------------+------------+---------------------+
-| body row 3 | Cells may | - Table cells |
-+------------------------+ span rows. | - contain |
-| body row 4 | | - body elements. |
-+------------------------+------------+---------------------+
-
-
-Simple Tables
-_____________
-
-`Simple tables <http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#simple-tables>`_ are described with horizontal borders made up of "=" and "-" characters. The equals sign ("=") is used for top and bottom table borders, and to separate optional header rows from the table body. The hyphen ("-") is used to indicate column spans in a single row by underlining the joined columns, and may optionally be used to explicitly and/or visually separate rows.
-
-Simple tables are "simpler" to create than grid tables, but are more limited.
-
-Here is example code for a simple table in a *.rst* file.
-
-.. code-block:: console
-
- ===== ===== =======
- A B A and B
- ===== ===== =======
- False False False
- True False False
- False True False
- True True True
- ===== ===== =======
-
-This example code generates a simple table that looks like this:
-
-===== ===== =======
- A B A and B
-===== ===== =======
-False False False
-True False False
-False True False
-True True True
-===== ===== =======
\ No newline at end of file
diff --git a/docs/gettingstarted/writingdocs/styleguide/styleguide03.rst b/docs/gettingstarted/writingdocs/styleguide/styleguide03.rst
deleted file mode 100644
index df35643..0000000
--- a/docs/gettingstarted/writingdocs/styleguide/styleguide03.rst
+++ /dev/null
@@ -1,32 +0,0 @@
-.. _styleguide03:
-
-******************
-Labels, References
-******************
-
-A link or reference to other paragraphs within these documents can be done with
-following construct.
-
-In this example the reference points the label **showintcommand**. The label **styleguide03**
-is shown at the top of this page. A label used in this way must be above a heading or title.
-
-:ref:`showintcommand` command.
-
-**************
-External Links
-**************
-
-An external link is done with the following construct:
-
-`Sphinx Markup Constructs <http://www.sphinx-doc.org/en/stable/markup/index.html>`_
-
-******
-Images
-******
-
-Images should be placed in the directory docs/_images. They can then be referenced with
-following construct. This is the image created to show a pull request.
-
-.. figure:: /_images/examplePullReq.png
- :scale: 35%
- :align: center
diff --git a/docs/gettingstarted/writingdocs/styleguide/styleguide04.rst b/docs/gettingstarted/writingdocs/styleguide/styleguide04.rst
deleted file mode 100644
index 7dcea37..0000000
--- a/docs/gettingstarted/writingdocs/styleguide/styleguide04.rst
+++ /dev/null
@@ -1,12 +0,0 @@
-.. _styleguide04:
-
-****************
-Including a file
-****************
-
-A complete file should be included with the following construct. It is recommended it be included with
-it's own .rst file describing the file included. This is an example of an xml file is included.
-
-.. toctree::
-
- xmlexample.rst
diff --git a/docs/gettingstarted/writingdocs/styleguide/styleguide05.rst b/docs/gettingstarted/writingdocs/styleguide/styleguide05.rst
deleted file mode 100644
index 7ea9773..0000000
--- a/docs/gettingstarted/writingdocs/styleguide/styleguide05.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-.. _styleguide05:
-
-********
-Raw HTML
-********
-
-An html frame can be included with the following construct. It is recommended that references to raw html be included with it's own .rst file.
-
-.. toctree::
-
- rawhtml.rst
diff --git a/docs/gettingstarted/writingdocs/styleguide/xmlexample.rst b/docs/gettingstarted/writingdocs/styleguide/xmlexample.rst
deleted file mode 100644
index 575a639..0000000
--- a/docs/gettingstarted/writingdocs/styleguide/xmlexample.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-.. _xmlexample:
-
-An XML File
-===========
-
-An example of an XML file.
-
-.. literalinclude:: iperf-vm.xml
- :language: XML
- :emphasize-lines: 42-49, 74-80
-
diff --git a/docs/gettingstarted/writingdocs/styleguidemd/index.rst b/docs/gettingstarted/writingdocs/styleguidemd/index.rst
deleted file mode 100644
index f9dfbeb..0000000
--- a/docs/gettingstarted/writingdocs/styleguidemd/index.rst
+++ /dev/null
@@ -1,16 +0,0 @@
-.. _styleguidemd:
-
-####################
-Markdown Style Guide
-####################
-
-Most of these documents are written using :ref:`styleguide` (rst), but pages can also be
-written in Markdown. This chapter describes some constructs used to write these documents.
-For a more detailed description of Markdown refer to `Markdown Wikipedia <https://en.wikipedia.org/wiki/Markdown>`_
-
-.. toctree::
- :maxdepth: 2
-
- styleguide.md
- styleguide02.md
- styleguide03.md
diff --git a/docs/gettingstarted/writingdocs/styleguidemd/styleguide.md b/docs/gettingstarted/writingdocs/styleguidemd/styleguide.md
deleted file mode 100644
index 31ea822..0000000
--- a/docs/gettingstarted/writingdocs/styleguidemd/styleguide.md
+++ /dev/null
@@ -1,13 +0,0 @@
-Heading 1
-=========
-
-This is the top heading level. More levels are shown below.
-
-Heading 2
----------
-
-### Heading 3
-
-#### Heading 4
-
-##### Heading 5
diff --git a/docs/gettingstarted/writingdocs/styleguidemd/styleguide02.md b/docs/gettingstarted/writingdocs/styleguidemd/styleguide02.md
deleted file mode 100644
index 5582d72..0000000
--- a/docs/gettingstarted/writingdocs/styleguidemd/styleguide02.md
+++ /dev/null
@@ -1,66 +0,0 @@
-Bullets, Bold and Italics
-=========================
-
-Bold text can be show with **Bold Text**, Italics with *Italic text*.
-Bullets like so:
-
-- Bullet 1
-- Bullet 2
-
-Code Blocks
-===========
-
-This paragraph describes how to do **Console Commands**. When showing
-VPP commands it is recommended that the command be executed from the
-linux console as shown. The Highlighting in the final documents shows up
-nicely this way.
-
-``` console
-$ sudo bash
-# vppctl show interface
- Name Idx State Counter Count
-TenGigabitEthernet86/0/0 1 up rx packets 6569213
- rx bytes 9928352943
- tx packets 50384
- tx bytes 3329279
-TenGigabitEthernet86/0/1 2 down
-VirtualEthernet0/0/0 3 up rx packets 50384
- rx bytes 3329279
- tx packets 6569213
- tx bytes 9928352943
- drops 1498
-local0 0 down
-#
-```
-
-The **code-block** construct is also used for code samples. The
-following shows how to include a block of \"C\" code.
-
-``` c
-#include <vlib/unix/unix.h>
-abf_policy_t *
-abf_policy_get (u32 index)
-{
- return (pool_elt_at_index (abf_policy_pool, index));
-}
-```
-
-Diffs are generated in the final docs nicely with \":\" at the end of
-the description like so:
-
- diff --git a/src/vpp/vnet/main.c b/src/vpp/vnet/main.c
- index 6e136e19..69189c93 100644
- --- a/src/vpp/vnet/main.c
- +++ b/src/vpp/vnet/main.c
- @@ -18,6 +18,8 @@
- #include <vlib/unix/unix.h>
- #include <vnet/plugin/plugin.h>
- #include <vnet/ethernet/ethernet.h>
- +#include <vnet/ip/ip4_packet.h>
- +#include <vnet/ip/format.h>
- #include <vpp/app/version.h>
- #include <vpp/api/vpe_msg_enum.h>
- #include <limits.h>
- @@ -400,6 +402,63 @@ VLIB_CLI_COMMAND (test_crash_command, static) = {
-
- #endif
diff --git a/docs/gettingstarted/writingdocs/styleguidemd/styleguide03.md b/docs/gettingstarted/writingdocs/styleguidemd/styleguide03.md
deleted file mode 100644
index 4c66dc4..0000000
--- a/docs/gettingstarted/writingdocs/styleguidemd/styleguide03.md
+++ /dev/null
@@ -1,32 +0,0 @@
-Labels, References
-==================
-
-A link or reference to other paragraphs within these documents can be
-done with following construct.
-
-External Links
-==============
-
-An external link is done with the following construct:
-
-[Sphinx Markup
-Constructs](http://www.sphinx-doc.org/en/stable/markup/index.html)
-
-Images
-======
-
-Images should be placed in the directory docs/\_images. They can then be
-referenced with following construct. This is the image created to show a
-pull request.
-
-
-
-Tables
-======
-
-An example of a table in markdown is shown here:
-
-| This Column | This Column | This Column |
-| ------------|:-------------:|------------:|
-| is left | is | is right |
-| aligned | centered | aligned |
diff --git a/docs/includes_renderer.py b/docs/includes_renderer.py
deleted file mode 100644
index d36752d..0000000
--- a/docs/includes_renderer.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2020. Vinci Consulting Corp. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import glob
-import inspect
-import os.path
-import re
-
-
-class ContentRenderer:
- name = ""
- curr_path = os.path.abspath(inspect.getsourcefile(lambda: 0))
- vpp_root = curr_path.rsplit("/", 2)[0]
- output_dir = f"{vpp_root}/docs/dynamic_includes/"
-
- def render(self):
- raise NotImplementedError
-
-
-class PluginRenderer(ContentRenderer):
- name = "plugin_list.inc"
-
- plugin_dir = f"{ContentRenderer.vpp_root}/src/plugins"
-
- pattern = r'VLIB_PLUGIN_REGISTER\s?\(\)\s*=\s*{.*\.description\s?=\s?"([^"]*)".*};' # noqa: 501
- regex = re.compile(pattern, re.MULTILINE | re.DOTALL)
-
- def render(self):
- with open(f"{self.__class__.output_dir}{self.__class__.name}",
- "w") as output:
- with os.scandir(self.__class__.plugin_dir) as pdir:
- for entry in sorted(pdir, key=lambda entry: entry.name):
- if not entry.name.startswith('.') and entry.is_dir():
- description = "<no-description-found>"
- # we use glob because a plugin can (ioam for now)
- # define the plugin definition in
- # a further subdirectory.
- for f in glob.iglob(f'{self.__class__.plugin_dir}/'
- f'{entry.name}/**',
- recursive=True):
- if f.endswith('.c'):
- with open(f, "r", encoding="utf-8") \
- as src:
- for match in self.__class__.regex.finditer(
- src.read()):
- description = "%s" % (match.group(1))
-
- output.write(f"* {entry.name} - {description}\n")
-
-
-# if this list grows substantially, we can move the classes to
-# a folder and import them.
-renderers = [PluginRenderer,
- ]
-
-
-def main():
- print("rendering dynamic includes...")
- for renderer in renderers:
- renderer().render()
- print("done.")
-
-
-if __name__ == "__main__":
- main()
diff --git a/docs/index.rst b/docs/index.rst
index 238931e..fcb63ee 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -26,18 +26,105 @@
include complete symbol tables, and extensive consistency checking.
Some VPP Use-cases include vSwitches, vRouters, Gateways, Firewalls
-and Load Balancers, to name a few.
+and Load-Balancers, to name a few.
For more details click on the links below or press next.
.. toctree::
+ :caption: About VPP
+ :maxdepth: 1
+
+ aboutvpp/scalar-vs-vector-packet-processing
+ aboutvpp/extensible
+ aboutvpp/networkstack
+ aboutvpp/hoststack
+ aboutvpp/developer
+ aboutvpp/supported
+ aboutvpp/performance
+ aboutvpp/releasenotes/index
+ aboutvpp/featurelist
+
+.. toctree::
+ :caption: Use Cases
+ :maxdepth: 1
+
+ usecases/containers/index
+ usecases/simpleperf/index
+ usecases/vppcloud/index
+ usecases/vhost/index
+ usecases/vmxnet3
+ usecases/home_gateway
+ usecases/acls
+ usecases/networksim
+ usecases/webapp
+ usecases/container_test
+ usecases/trafficgen
+ usecases/ikev2/index
+ usecases/contiv/index
+
+
+.. toctree::
+ :caption: Getting started
:maxdepth: 2
- whatisvpp/index.rst
- gettingstarted/index
- links/index
- usecases/index
- troubleshooting/index
- reference/index
- relatedprojects/index
- about.rst
+ gettingstarted/installing/index
+ gettingstarted/running/index
+ gettingstarted/progressivevpp/index
+ gettingstarted/troubleshooting/index
+
+.. toctree::
+ :caption: Developer Documentation
+ :maxdepth: 2
+
+ developer/build-run-debug/index
+ developer/corearchitecture/index
+ developer/corefeatures/index
+ developer/plugindoc/index
+ developer/plugins/index
+ developer/devicedrivers/index
+ developer/tests/overview
+ developer/extras/index
+
+.. toctree::
+ :caption: Interfacing with VPP
+ :maxdepth: 2
+
+ interfacing/binapi/index
+ interfacing/go/index
+ interfacing/rust/index
+ interfacing/libmemif/index
+
+
+
+.. toctree::
+ :caption: Contributing
+ :maxdepth: 2
+
+ contributing/gitreview
+ contributing/writingdocs
+ contributing/reportingissues/index
+
+
+.. toctree::
+ :caption: Debug CLI
+ :maxdepth: 2
+
+ cli-reference/gettingstarted/index
+ cli-reference/interface/index
+ cli-reference/index
+
+
+.. toctree::
+ :caption: Configuration file
+ :maxdepth: 2
+
+ configuration/config_getting_started
+ configuration/reference
+
+
+About this documentation
+
+::
+
+ VPP Version : __VPP_VERSION__
+ Built on : __BUILT_ON__
diff --git a/docs/interfacing/binapi/index.rst b/docs/interfacing/binapi/index.rst
new file mode 100644
index 0000000..500c16d
--- /dev/null
+++ b/docs/interfacing/binapi/index.rst
@@ -0,0 +1,12 @@
+.. _vpp_binapi:
+
+==============
+The binary API
+==============
+
+.. toctree::
+ :maxdepth: 2
+
+ vpp_api_module
+ vpp_api_language
+ writing_api_handlers
diff --git a/docs/interfacing/binapi/vpp_api_language.rst b/docs/interfacing/binapi/vpp_api_language.rst
new file mode 120000
index 0000000..16a0788
--- /dev/null
+++ b/docs/interfacing/binapi/vpp_api_language.rst
@@ -0,0 +1 @@
+../../../src/tools/vppapigen/VPPAPI.rst
\ No newline at end of file
diff --git a/docs/interfacing/binapi/vpp_api_module.rst b/docs/interfacing/binapi/vpp_api_module.rst
new file mode 120000
index 0000000..d412ef6
--- /dev/null
+++ b/docs/interfacing/binapi/vpp_api_module.rst
@@ -0,0 +1 @@
+../../../src/vpp-api/vapi/vapi_doc.rst
\ No newline at end of file
diff --git a/docs/interfacing/binapi/writing_api_handlers.rst b/docs/interfacing/binapi/writing_api_handlers.rst
new file mode 120000
index 0000000..6c58111
--- /dev/null
+++ b/docs/interfacing/binapi/writing_api_handlers.rst
@@ -0,0 +1 @@
+../../../src/vlibapi/api_doc.rst
\ No newline at end of file
diff --git a/docs/gettingstarted/developers/add_plugin_goapi.rst b/docs/interfacing/go/add_plugin_goapi.rst
similarity index 100%
rename from docs/gettingstarted/developers/add_plugin_goapi.rst
rename to docs/interfacing/go/add_plugin_goapi.rst
diff --git a/docs/interfacing/go/index.rst b/docs/interfacing/go/index.rst
new file mode 100644
index 0000000..68c01e0
--- /dev/null
+++ b/docs/interfacing/go/index.rst
@@ -0,0 +1,10 @@
+.. _govpp:
+
+==============
+Go api (govpp)
+==============
+
+.. toctree::
+ :maxdepth: 2
+
+ add_plugin_goapi
diff --git a/docs/interfacing/libmemif/buildinstructions_doc.rst b/docs/interfacing/libmemif/buildinstructions_doc.rst
new file mode 120000
index 0000000..9a5cf40
--- /dev/null
+++ b/docs/interfacing/libmemif/buildinstructions_doc.rst
@@ -0,0 +1 @@
+../../../extras/libmemif/docs/buildinstructions_doc.rst
\ No newline at end of file
diff --git a/docs/interfacing/libmemif/example_setup_doc.rst b/docs/interfacing/libmemif/example_setup_doc.rst
new file mode 120000
index 0000000..79e7dcf
--- /dev/null
+++ b/docs/interfacing/libmemif/example_setup_doc.rst
@@ -0,0 +1 @@
+../../../extras/libmemif/examples/example_setup_doc.rst
\ No newline at end of file
diff --git a/docs/interfacing/libmemif/examples_doc.rst b/docs/interfacing/libmemif/examples_doc.rst
new file mode 120000
index 0000000..0f93bab
--- /dev/null
+++ b/docs/interfacing/libmemif/examples_doc.rst
@@ -0,0 +1 @@
+../../../extras/libmemif/examples/examples_doc.rst
\ No newline at end of file
diff --git a/docs/interfacing/libmemif/gettingstarted_doc.rst b/docs/interfacing/libmemif/gettingstarted_doc.rst
new file mode 120000
index 0000000..f9f20f6
--- /dev/null
+++ b/docs/interfacing/libmemif/gettingstarted_doc.rst
@@ -0,0 +1 @@
+../../../extras/libmemif/docs/gettingstarted_doc.rst
\ No newline at end of file
diff --git a/docs/interfacing/libmemif/index.rst b/docs/interfacing/libmemif/index.rst
new file mode 100644
index 0000000..4d4b24f
--- /dev/null
+++ b/docs/interfacing/libmemif/index.rst
@@ -0,0 +1,24 @@
+.. _libmemif_index:
+
+Memif library (libmemif)
+========================
+
+Shared memory packet interface (memif) provides high performance packet
+transmit and receive between user application and Vector Packet
+Processing (VPP) or multiple user applications. Using libmemif, user
+application can create shared memory interface in master or slave mode
+and connect to VPP or another application using libmemif. Once the
+connection is established, user application can receive or transmit
+packets using libmemif API.
+
+.. figure:: /_images/libmemif_architecture.png
+ :alt: Architecture
+
+.. toctree::
+ :maxdepth: 2
+
+ libmemif_doc
+ buildinstructions_doc
+ example_setup_doc
+ examples_doc
+ gettingstarted_doc
diff --git a/docs/interfacing/libmemif/libmemif_doc.rst b/docs/interfacing/libmemif/libmemif_doc.rst
new file mode 120000
index 0000000..394e83c
--- /dev/null
+++ b/docs/interfacing/libmemif/libmemif_doc.rst
@@ -0,0 +1 @@
+../../../extras/libmemif/libmemif_doc.rst
\ No newline at end of file
diff --git a/docs/interfacing/rust/index.rst b/docs/interfacing/rust/index.rst
new file mode 100644
index 0000000..31b6931
--- /dev/null
+++ b/docs/interfacing/rust/index.rst
@@ -0,0 +1,8 @@
+.. _rustvpp:
+
+===============
+Rust api client
+===============
+
+The VPP Rust API client is in alpha stage
+You can read more in `this blog post <https://dev.to/felixfaisal/rust-vpp-api-bindings-lfx-mentorship-project-2g8p>`_
diff --git a/docs/links/index.rst b/docs/links/index.rst
deleted file mode 100644
index 16c26de..0000000
--- a/docs/links/index.rst
+++ /dev/null
@@ -1,33 +0,0 @@
-.. _links:
-
-##################################
-VPP Wiki, Doxygen and Other Links
-##################################
-
-There are several places to find helpful Information regarding FD.io VPP.
-
-.. toctree::
-
-****************
-FD.io Main Site
-****************
-
-VPP is part of the FD.io Linux Foundation project. The link describing the FD.io Linux
-foundation project is here:
-
-`<https://fd.io>`_
-
-*********
-VPP Wiki
-*********
-
-Much of the VPP developer documentation can be found on the VPP wiki.
-
-`<https://wiki.fd.io/view/VPP>`_
-
-********************************
-Source code Documents (doxygen)
-********************************
-
-Some of VPP is documented in the sources with Doxygen. See
-`<https://docs.fd.io/vpp>`_ for links to Doxygen documentation by release.
diff --git a/docs/reference/cmdreference/index.rst b/docs/reference/cmdreference/index.rst
deleted file mode 100644
index c791992..0000000
--- a/docs/reference/cmdreference/index.rst
+++ /dev/null
@@ -1,57 +0,0 @@
-.. _cmdreference:
-
-
-Useful Debug CLI
-==================
-
-This is a reference guide for the vpp debug commands that are referenced within these documents. This is **NOT** a complete list. For a complete list refer to the Debug CLI section of the
-`Source Code Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`_.
-
-The debug CLI can be executed from a su (superuser) shell using the vppctl command.
-
-.. code-block:: console
-
- # sudo bash
- # vppctl show interface
-
- Name Idx State Counter Count
- TenGigabitEthernet86/0/0 1 up rx packets 6569213
- rx bytes 9928352943
- tx packets 50384
- tx bytes 3329279
- TenGigabitEthernet86/0/1 2 down
- VirtualEthernet0/0/0 3 up rx packets 50384
- rx bytes 3329279
- tx packets 6569213
- tx bytes 9928352943
- drops 1498
- local0 0 down
-
-Commands can also be executed from the vppctl shell.
-
-.. code-block:: console
-
- # vppctl
-
- vpp# show interface
- Name Idx State Counter Count
- TenGigabitEthernet86/0/0 1 up rx packets 6569213
- rx bytes 9928352943
- tx packets 50384
- tx bytes 3329279
- TenGigabitEthernet86/0/1 2 down
- VirtualEthernet0/0/0 3 up rx packets 50384
- rx bytes 3329279
- tx packets 6569213
- tx bytes 9928352943
- drops 1498
- local0 0 down
-
-.. toctree::
- :maxdepth: 3
-
- interface/index.rst
- ip/index.rst
- show/index.rst
- trace/index.rst
- vhost/index.rst
diff --git a/docs/reference/cmdreference/interface/setinterface.rst b/docs/reference/cmdreference/interface/setinterface.rst
deleted file mode 100644
index 6906358..0000000
--- a/docs/reference/cmdreference/interface/setinterface.rst
+++ /dev/null
@@ -1,193 +0,0 @@
-.. _interface:
-
-.. toctree::
-
-Set Interface Commands
-=======================
-This section covers those commands that are related to setting an interface:
-
-* `Set Interface`_
-* `Set Interface IP Address`_
-* `Set Interface L2 Bridge`_
-* `Set Interface MTU`_
-* `Set Interface Promiscuous`_
-* `Set Interface State`_
-
-.. note:: For a complete list of CLI Debug commands refer to the Debug CLI section of the `Source Code Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`_ .
-
-Set Interface
-++++++++++++++++
-
-Summary/Usage
--------------
-
-Interface commands.
-
-Declaration and Implementation
--------------------------------
-
-**Declaration:** vnet_cli_set_interface_command (src/vnet/interface_cli.c line 484)
-
-Set Interface IP Address
-+++++++++++++++++++++++++
-
-Summary/Usage
--------------
-
-set interface ip address [del] <*interface*> <*ip-addr*>/<*mask*> | [all]
-
-Description
--------------
-
-Add an IP Address to an interface or remove and IP Address from an interface.
-The IP Address can be an IPv4 or an IPv6 address. Interfaces may have multiple IPv4 and IPv6
-addresses. There is no concept of primary vs. secondary interface addresses; they're just addresses.
-
-To display the addresses associated with a given interface, use the command **show interface address** <*interface*>.
-
-.. note::
- The debug CLI does not enforce classful mask-width / addressing constraints.
-
-Example Usage
---------------
-
-An example of how to add an IPv4 address to an interface:
-
-.. code-block:: console
-
- vpp# set interface ip address GigabitEthernet2/0/0 172.16.2.12/24
-
-An example of how to add an IPv6 address to an interface:
-
-.. code-block:: console
-
- vpp# set interface ip address GigabitEthernet2/0/0 ::a:1:1:0:7/126
-
-To delete a specific interface ip address:
-
-.. code-block:: console
-
- vpp# set interface ip address GigabitEthernet2/0/0 172.16.2.12/24 del
-
-To delete all interfaces addresses (IPv4 and IPv6):
-
-.. code-block:: console
-
- vpp# set interface ip address GigabitEthernet2/0/0 del all
-
-Declaration and Implementation
--------------------------------
-
-**Declaration:** set_interface_ip_address_command (src/vnet/ip/ip46_cli.c line 216)
-
-**Implementation:** add_del_ip_address
-
-
-Set Interface L2 Bridge
-+++++++++++++++++++++++++
-
-Summary/Usage
--------------
-
-set interface l2 bridge <*interface*> <*bridge-domain-id*> [bvi|uu-fwd] [shg]
-
-Description
--------------
-
-Use this command put an interface into Layer 2 bridge domain. If a bridge-domain with the provided
-bridge-domain-id does not exist, it will be created. Interfaces in a bridge-domain forward packets
-to other interfaces in the same bridge-domain based on destination mac address. To remove an interface
-from a the Layer 2 bridge domain, put the interface in a different mode, for example Layer 3 mode.
-
-Optionally, an interface can be added to a Layer 2 bridge-domain as a Bridged Virtual Interface (bvi).
-Only one interface in a Layer 2 bridge-domain can be a bvi.
-
-Optionally, a split-horizon group can also be specified. This defaults to 0 if not specified.
-
-Example Usage
---------------
-Example of how to configure a Layer 2 bridge-domain with three interfaces (where 200 is the bridge-domain-id):
-
-.. code-block:: console
-
- vpp# set interface l2 bridge GigabitEthernet0/8/0.200 200
-
-This interface is added a BVI interface:
-
-.. code-block:: console
-
- vpp# set interface l2 bridge GigabitEthernet0/9/0.200 200 bvi
-
-This interface also has a split-horizon group of 1 specified:
-
-.. code-block:: console
-
- vpp# set interface l2 bridge GigabitEthernet0/a/0.200 200 1
-
-Example of how to remove an interface from a Layer2 bridge-domain:
-
-.. code-block:: console
-
- vpp# set interface l3 GigabitEthernet0/a/0.200
-
-Declaration and Implementation
--------------------------------
-
-**Declaration:** int_l2_bridge_cli (src/vnet/l2/l2_input.c line 949)
-
-**Implementation:** int_l2_bridge
-
-Set Interface MTU
-++++++++++++++++++
-
-Summary/Usage
--------------
-
-.. code-block:: shell
-
- set interface mtu [packet|ip4|ip6|mpls] <value> <interface>
-
-Set Interface Promiscuous
-++++++++++++++++++++++++++
-
-Summary/Usage
--------------
-
-.. code-block:: shell
-
- set interface promiscuous [on|off] <interface>.
-
-.. _setintstate:
-
-Set Interface State
-++++++++++++++++++++
-This command is used to change the admin state (up/down) of an
-interface.
-
-If an interface is down, the optional *punt* flag can also be set. The
-*punt* flag implies the interface is disabled for forwarding but punt
-all traffic to slow-path. Use the *enable* flag to clear *punt* flag
-(interface is still down).
-
-Summary/Usage
--------------
-
-.. code-block:: shell
-
- set interface state <interface> [up|down|punt|enable].
-
-Example Usage
-----------------
-
-Example of how to configure the admin state of an interface to **up**:
-
-.. code-block:: console
-
- vpp# set interface state GigabitEthernet2/0/0 up
-
-Example of how to configure the admin state of an interface to **down**:
-
-.. code-block:: console
-
- vpp# set interface state GigabitEthernet2/0/0 down
-
diff --git a/docs/reference/cmdreference/ip/index.rst b/docs/reference/cmdreference/ip/index.rst
deleted file mode 100644
index 1a06321..0000000
--- a/docs/reference/cmdreference/ip/index.rst
+++ /dev/null
@@ -1,12 +0,0 @@
-.. ip:
-
-.. note:: For a complete list of CLI Debug commands refer to the Debug CLI section of the `Source Code Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`_ .
-
-IP Commands
-============
-This section identifies the following types of IP commands:
-
-.. toctree::
- :maxdepth: 2
-
- ip
\ No newline at end of file
diff --git a/docs/reference/cmdreference/ip/ip.rst b/docs/reference/cmdreference/ip/ip.rst
deleted file mode 100644
index a709215..0000000
--- a/docs/reference/cmdreference/ip/ip.rst
+++ /dev/null
@@ -1,151 +0,0 @@
-.. _ip1:
-
-.. toctree::
-
-.. note:: For a complete list of CLI Debug commands refer to the Debug CLI section of the `Source Code Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`_ .
-
-IP Route
-==========
-
-Summary/Usage
---------------
-
-`ip route [add|del] [count <*n*>] <*dst-ip-addr*>/<*width*> [table <*table-id*>] via [next-hop-address[next-hop-interface] [next-hop-table <*value*>] [weight <*value*>] [preference <*value*>] [udp-encap-id <*value*>] [ip4-lookup-in-table <*value*>] [ip6-lookup-in-table <*value*>] [mpls-lookup-in-table <*value*>] [resolve-via-host] [resolve-via-connected] [rx-ip4 <*interface*>] [out-labels <*value value value*>]`
-
-Description
-------------
-
-This command is used to add or delete IPv4 or IPv6 routes. All IP Addresses
-('<*dst-ip-addr*>/<*width*>', '<*next-hop-ip-addr*>' and '<*adj-hop-ip-addr*>') can be IPv4 or IPv6,
-but all must be of the same form in a single command. To display the current set of routes,
-use the commands 'show ip fib' and 'show ip6 fib'.
-
-Example Usage
---------------
-
-Example of how to add a straight forward static route:
-
-.. code-block:: console
-
- vpp# ip route add 6.0.1.2/32 via 6.0.0.1 GigabitEthernet2/0/0
-
-Example of how to delete a straight forward static route:
-
-.. code-block:: console
-
- vpp# ip route del 6.0.1.2/32 via 6.0.0.1 GigabitEthernet2/0/0
-
-Mainly for route add/del performance testing, one can add or delete multiple routes by adding
-'count N' to the previous item:
-
-.. code-block:: console
-
- vpp# ip route add count 10 7.0.0.0/24 via 6.0.0.1 GigabitEthernet2/0/0
-
-Add multiple routes for the same destination to create equal-cost multipath:
-
-.. code-block:: console
-
- vpp# ip route add 7.0.0.1/32 via 6.0.0.1 GigabitEthernet2/0/0
-
- vpp# ip route add 7.0.0.1/32 via 6.0.0.2 GigabitEthernet2/0/0
-
-For unequal-cost multipath, specify the desired weights. This combination of weights
-results in 3/4 of the traffic following the second path, 1/4 following the first path:
-
-.. code-block:: console
-
- vpp# ip route add 7.0.0.1/32 via 6.0.0.1 GigabitEthernet2/0/0 weight 1
-
- vpp# ip route add 7.0.0.1/32 via 6.0.0.2 GigabitEthernet2/0/0 weight 3
-
-To add a route to a particular FIB table (VRF), use:
-
-.. code-block:: console
-
- vpp# ip route add 172.16.24.0/24 table 7 via GigabitEthernet2/0/0
-
-Declaration and Implementation
----------------------------------
-
-**Declaration:** ip_route_command (src/vnet/ip/lookup.c line 641)
-
-**Implementation:** vnet_ip_route_cmd
-
-Ping
-=====
-
-Summary/Usage
---------------
-
-ping {<*ip-addr*> | ipv4 <*ip4-addr*> | ipv6 <*ip6-addr*>} [ipv4 <*ip4-addr*> | ipv6 <*ip6-addr*>] [source <*interface*>] [size <*pktsize*>] [interval <*sec*>] [repeat <*cnt*>] [table-id <*id*>] [verbose]
-
-Description
-------------
-
-This command sends an ICMP ECHO_REQUEST to network hosts. The address can be an IPv4 or IPv6
-address (or both at the same time).
-
-Example Usage
---------------
-
-Example of how ping an IPv4 address:
-
-.. code-block:: console
-
- vpp# ping 172.16.1.2 source GigabitEthernet2/0/0 repeat 2
-
- 64 bytes from 172.16.1.2: icmp_seq=1 ttl=64 time=.1090 ms
- 64 bytes from 172.16.1.2: icmp_seq=2 ttl=64 time=.0914 ms
-
- Statistics: 2 sent, 2 received, 0% packet loss
-
- Example of how ping both an IPv4 address and IPv6 address at the same time:
-
- vpp# ping 172.16.1.2 ipv6 fe80::24a5:f6ff:fe9c:3a36 source GigabitEthernet2/0/0 repeat 2 verbose
-
- Adjacency index: 10, sw_if_index: 1
- Adj: ip6-discover-neighbor
- Adj Interface: 0
- Forced set interface: 1
- Adjacency index: 0, sw_if_index: 4294967295
- Adj: ip4-miss
- Adj Interface: 0
- Forced set interface: 1
- Source address: 172.16.1.1
- 64 bytes from 172.16.1.2: icmp_seq=1 ttl=64 time=.1899 ms
- Adjacency index: 10, sw_if_index: 1
- Adj: ip6-discover-neighbor
- Adj Interface: 0
- Forced set interface: 1
- Adjacency index: 0, sw_if_index: 4294967295
- Adj: ip4-miss
- Adj Interface: 0
- Forced set interface: 1
- Source address: 172.16.1.1
- 64 bytes from 172.16.1.2: icmp_seq=2 ttl=64 time=.0910 ms
-
- Statistics: 4 sent, 2 received, 50% packet loss
-
-Declaration and Implementation
--------------------------------
-
-Declaration: ping_command (src/vnet/ip/ping.c line 899)
-
-Implementation: ping_ip_address
-
-Set Interface IP Address
-=========================
-
-`Set Interface IP Address <../interface/setinterface.html#set-interface-ip-address>`_
-
-Show IP Arp
-=============
-
-`Show IP-Arp <../show/show.html#show-ip-arp>`_
-
-
-Show IP Fib
-============
-
-`Show IP-Fib <../show/show.html#show-ip-fib>`_
diff --git a/docs/reference/cmdreference/show/index.rst b/docs/reference/cmdreference/show/index.rst
deleted file mode 100644
index 6f1e012..0000000
--- a/docs/reference/cmdreference/show/index.rst
+++ /dev/null
@@ -1,12 +0,0 @@
-.. _interface:
-
-.. note:: For a complete list of CLI Debug commands refer to the Debug CLI section of the `Source Code Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`_ .
-
-Show Commands
-==================
-This section identifies the following types of show commands:
-
-.. toctree::
- :maxdepth: 2
-
- show
\ No newline at end of file
diff --git a/docs/reference/cmdreference/show/show.rst b/docs/reference/cmdreference/show/show.rst
deleted file mode 100644
index 61c5360..0000000
--- a/docs/reference/cmdreference/show/show.rst
+++ /dev/null
@@ -1,286 +0,0 @@
-.. _interface:
-
-.. toctree::
-
-.. note:: For a complete list of CLI Debug commands refer to the Debug CLI section of the `Source Code Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`_ .
-
-Show Bridge-Domain
-===================
-
-`Show Bridge-Domain <../interface/hardware.html#show-bridge-domain>`_
-
-Show Interface
-================
-
-`Show Interface <../interface/basic.html#show-interface>`_
-
-Show IP Arp
-============
-
-Summary/Usage
----------------
-
-show ip arp
-
-Description
-------------
-
-Display all the IPv4 ARP entries.
-
-Example Usage
---------------
-Example of how to display the IPv4 ARP table:
-
-.. code-block:: console
-
- vpp# **show ip arp**
- Time FIB IP4 Flags Ethernet Interface
- 346.3028 0 6.1.1.3 de:ad:be:ef:ba:be GigabitEthernet2/0/0
- 3077.4271 0 6.1.1.4 S de:ad:be:ef:ff:ff GigabitEthernet2/0/0
- 2998.6409 1 6.2.2.3 de:ad:be:ef:00:01 GigabitEthernet2/0/0
- Proxy arps enabled for:
- Fib_index 0 6.0.0.1 - 6.0.0.11
-
-
-Declaration and Implementation
--------------------------------
-
-**Declaration:** show_ip4_arp_command (src/vnet/ethernet/arp.c line 1465)
-
-**Implementation:** show_ip4_arp
-
-Show IP Fib
-=============
-
-Summary/Usage
----------------
-
-show ip fib [summary] [table <*table-id*>] [index <*fib-id*>] [<*ip4-addr*>[/<*mask*>]] [mtrie] [detail]
-
-Description
-------------
-
-This command displays the IPv4 FIB Tables (VRF Tables) and the route entries for each table.
-
-.. note::
- This command will run for a long time when the FIB tables are comprised of
- millions of entries. For those scenarios, consider displaying a single table or summary mode.
-
-Example Usage
---------------
-Example of how to display all the IPv4 FIB tables:
-
-.. code-block:: console
-
- vpp# **show ip fib**
-
- ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
- 0.0.0.0/0
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 0.0.0.0/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 6.0.1.2/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
- [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
- 7.0.0.1/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
- [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
- [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
- [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
- [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
- 224.0.0.0/8
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 240.0.0.0/8
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 255.255.255.255/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- ipv4-VRF:7, fib_index 1, flow hash: src dst sport dport proto
- 0.0.0.0/0
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 0.0.0.0/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 172.16.1.0/24
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
- [0] [@4]: ipv4-glean: af_packet0
- 172.16.1.1/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
- [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
- 172.16.1.2/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
- [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
- 172.16.2.0/24
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
- [0] [@4]: ipv4-glean: af_packet1
- 172.16.2.1/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
- [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
- 224.0.0.0/8
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 240.0.0.0/8
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 255.255.255.255/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
- [0] [@0]: dpo-drop ip6
-
-Example of how to display a single IPv4 FIB table:
-
-.. code-block:: console
-
- vpp# **show ip fib table 7**
-
- ipv4-VRF:7, fib_index 1, flow hash: src dst sport dport proto
- 0.0.0.0/0
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 0.0.0.0/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 172.16.1.0/24
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
- [0] [@4]: ipv4-glean: af_packet0
- 172.16.1.1/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
- [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
- 172.16.1.2/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
- [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -*> 26:a5:f6:9c:3a:36
- 172.16.2.0/24
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
- [0] [@4]: ipv4-glean: af_packet1
- 172.16.2.1/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
- [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
- 224.0.0.0/8
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 240.0.0.0/8
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
- [0] [@0]: dpo-drop ip6
- 255.255.255.255/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
- [0] [@0]: dpo-drop ip6
-
-Example of how to display a summary of all IPv4 FIB tables:
-
-.. code-block:: console
-
- vpp# **show ip fib summary**
-
- ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
- Prefix length Count
- 0 1
- 8 2
- 32 4
- ipv4-VRF:7, fib_index 1, flow hash: src dst sport dport proto
- Prefix length Count
- 0 1
- 8 2
- 24 2
- 32 4
-
-Declaration and Implementation
--------------------------------
-
-**Declaration:** ip4_show_fib_command (src/vnet/fib/ip4_fib.c line 873)
-
-**Implementation:** ip4_show_fib
-
-Show L2fib
-============
-
-Summary/Usage
-------------------
-
-show l2fib [all] | [bd_id <*nn*> | bd_index <*nn*>] [learn | add] | [raw]
-
-Description
-------------
-
-This command displays the MAC Address entries of the L2 FIB table.
-Output can be filtered to just get the number of MAC Addresses or display each
-MAC Address for all bridge domains or just a single bridge domain.
-
-Example Usage
---------------
-Example of how to display the number of MAC Address entries in the L2 FIB table:
-
-
-.. code-block:: console
-
- vpp# **show l2fib**
-
- 3 l2fib entries
-
- Example of how to display all the MAC Address entries in the L2 FIB table:
-
- vpp# **show l2fib all**
-
- Mac Address BD Idx Interface Index static filter bvi refresh timestamp
- 52:54:00:53:18:33 1 GigabitEthernet0/8/0.200 3 0 0 0 0 0
- 52:54:00:53:18:55 1 GigabitEthernet0/8/0.200 3 1 0 0 0 0
- 52:54:00:53:18:77 1 N/A -1 1 1 0 0 0
- 3 l2fib entries
-
-Declaration and Implementation
--------------------------------
-
-**Declaration:** show_l2fib_cli (src/vnet/l2/l2_fib.c line 311)
-
-**Implementation:** show_l2fib
-
-Show Trace
-===========
-
-Summary/Usage
---------------
-
-show trace buffer [max COUNT]
-
-Declaration and Implementation
-------------------------------
-
-**Declaration:** show_trace_cli (src/vlib/trace.c line 347)
-
-**Implementation:** cli_show_trace_buffer
-
-Show Vhost-User
-================
-
-`Show Vhost-User <../vhost/vhostuser.html#show-vhost-user>`_
-
diff --git a/docs/reference/cmdreference/trace/index.rst b/docs/reference/cmdreference/trace/index.rst
deleted file mode 100644
index 1dbbe2e..0000000
--- a/docs/reference/cmdreference/trace/index.rst
+++ /dev/null
@@ -1,12 +0,0 @@
-.. _interface:
-
-.. note:: For a complete list of CLI Debug commands refer to the Debug CLI section of the `Source Code Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`_ .
-
-Trace Commands
-===============
-This section identifies the following types of trace commands:
-
-.. toctree::
- :maxdepth: 2
-
- trace
\ No newline at end of file
diff --git a/docs/reference/cmdreference/trace/trace.rst b/docs/reference/cmdreference/trace/trace.rst
deleted file mode 100644
index 927342f..0000000
--- a/docs/reference/cmdreference/trace/trace.rst
+++ /dev/null
@@ -1,59 +0,0 @@
-.. _interface:
-
-.. toctree::
-
-.. note:: For a complete list of CLI Debug commands refer to the Debug CLI section of the `Source Code Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`_ .
-
-
-API Trace
-===========
-
-Summary/Usage
---------------
-
-api trace [tx][on|off][first <n>][last <n>][status][free]
- [post-mortem-on][dump|dump-file|dump-json|save|tojson|save-json|replay <file>][nitems <n>][initializers <file>]
-
-Description
-------------
-
-Display, replay, or save a binary API trace.
-
-Declaration and Implementation
--------------------------------
-
-**Declaration:** api_trace_command (src/vlibmemory/vlib_api_cli.c line 783)
-
-**Implementation:** api_trace_command_fn
-
-Clear Trace
-=============
-
-Summary/Usage
---------------
-Clear trace buffer and free memory.
-Declaration and implementation
-
-**Declaration:** clear_trace_cli (src/vlib/trace.c line 519)
-
-**Implementation:** cli_clear_trace_buffer
-
-Show Trace
-===========
-
-`Show Trace <../show/show.html#show-trace>`_
-
-Trace Add
-===========
-
-Summary/Usage
---------------
-
-Trace given number of packets.
-
-Declaration and Implementation
--------------------------------
-
-**Declaration:** add_trace_cli (src/vlib/trace.c line 405)
-
-**Implementation:** cli_add_trace_buffer
diff --git a/docs/reference/cmdreference/vhost/index.rst b/docs/reference/cmdreference/vhost/index.rst
deleted file mode 100644
index d1cce23..0000000
--- a/docs/reference/cmdreference/vhost/index.rst
+++ /dev/null
@@ -1,10 +0,0 @@
-.. _vhostcommands:
-
-.. note:: For a complete list of CLI Debug commands refer to the Debug CLI section of the `Source Code Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`_ .
-
-Vhost User Commands
-===================
-
-.. toctree::
-
- vhostuser
diff --git a/docs/reference/cmdreference/vhost/vhostuser.rst b/docs/reference/cmdreference/vhost/vhostuser.rst
deleted file mode 100644
index 0c2498b..0000000
--- a/docs/reference/cmdreference/vhost/vhostuser.rst
+++ /dev/null
@@ -1,271 +0,0 @@
-.. vhost:
-
-.. toctree::
-
-.. note:: For a complete list of CLI Debug commands refer to the Debug CLI section of the `Source Code Documents <https://docs.fd.io/vpp/18.07/clicmd.html>`_ .
-
-.. _createvhostuser:
-
-Create Vhost-User
-=================
-
-Create a vHost User interface. Once created, a new virtual interface
-will exist with the name '*VirtualEthernet0/0/x*', where '*x*' is the
-next free index.
-
-There are several parameters associated with a vHost interface:
-
-- **socket <*socket-filename*>** - Name of the linux socket used by
- hypervisor and VPP to manage the vHost interface. If in '*server*'
- mode, VPP will create the socket if it does not already exist. If in
- '*client*' mode, hypervisor will create the socket if it does not
- already exist. The VPP code is indifferent to the file location.
- However, if SELinux is enabled, then the socket needs to be created
- in '*/var/run/vpp/*'.
-- **server** - Optional flag to indicate that VPP should be the server
- for the linux socket. If not provided, VPP will be the client. In
- '*server*' mode, the VM can be reset without tearing down the vHost
- Interface. In '*client*' mode, VPP can be reset without bringing down
- the VM and tearing down the vHost Interface.
-- **feature-mask <hex>** - Optional virtio/vhost feature set negotiated
- at startup. **This is intended for debugging only.** It is
- recommended that this parameter not be used except by experienced
- users. By default, all supported features will be advertised.
- Otherwise, provide the set of features desired.
-
- - 0x000008000 (15) - VIRTIO_NET_F_MRG_RXBUF
- - 0x000020000 (17) - VIRTIO_NET_F_CTRL_VQ
- - 0x000200000 (21) - VIRTIO_NET_F_GUEST_ANNOUNCE
- - 0x000400000 (22) - VIRTIO_NET_F_MQ
- - 0x004000000 (26) - VHOST_F_LOG_ALL
- - 0x008000000 (27) - VIRTIO_F_ANY_LAYOUT
- - 0x010000000 (28) - VIRTIO_F_INDIRECT_DESC
- - 0x040000000 (30) - VHOST_USER_F_PROTOCOL_FEATURES
- - 0x100000000 (32) - VIRTIO_F_VERSION_1
-
-- **hwaddr <mac-addr>** - Optional ethernet address, can be in either
- X:X:X:X:X:X unix or X.X.X cisco format.
-- **renumber <dev_instance>** - Optional parameter which allows the
- instance in the name to be specified. If instance already exists,
- name will be used anyway and multiple instances will have the same
- name. Use with caution.
-
-
-Summary/Usage
--------------
-
-.. code-block:: shell
-
- create vhost-user socket <socket-filename> [server] [feature-mask <hex>] [hwaddr <mac-addr>] [renumber <dev_instance>]
-
-
-Examples
---------
-
-Example of how to create a vhost interface with VPP as the client
-and all features enabled:
-
-.. code-block:: console
-
- vpp# create vhost-user socket /var/run/vpp/vhost1.sock
- VirtualEthernet0/0/0
-
-Example of how to create a vhost interface with VPP as the server
-and with just multiple queues enabled:
-
-.. code-block:: console
-
- vpp# create vhost-user socket /var/run/vpp/vhost2.sock server feature-mask 0x40400000
- VirtualEthernet0/0/1
-
-Once the vHost interface is created, enable the interface using:
-
-.. code-block:: console
-
- vpp# set interface state VirtualEthernet0/0/0 up
-
-.. _showvhost:
-
-Show Vhost-User
-===============
-
-Display the attributes of a single vHost User interface (provide
-interface name), multiple vHost User interfaces (provide a list of
-interface names separated by spaces) or all Vhost User interfaces (omit
-an interface name to display all vHost interfaces).
-
-Summary/Usage
--------------
-
-.. code-block:: shell
-
- show vhost-user [<interface> [<interface> [..]]] [descriptors].
-
-Examples
---------
-Example of how to display a vhost interface:
-
-.. code-block:: console
-
- vpp# show vhost-user VirtualEthernet0/0/0
- Virtio vhost-user interfaces
- Global:
- coalesce frames 32 time 1e-3
- Interface: VirtualEthernet0/0/0 (ifindex 1)
- virtio_net_hdr_sz 12
- features mask (0xffffffffffffffff):
- features (0x50408000):
- VIRTIO_NET_F_MRG_RXBUF (15)
- VIRTIO_NET_F_MQ (22)
- VIRTIO_F_INDIRECT_DESC (28)
- VHOST_USER_F_PROTOCOL_FEATURES (30)
- protocol features (0x3)
- VHOST_USER_PROTOCOL_F_MQ (0)
- VHOST_USER_PROTOCOL_F_LOG_SHMFD (1)
-
- socket filename /var/run/vpp/vhost1.sock type client errno "Success"
-
- rx placement:
- thread 1 on vring 1
- thread 1 on vring 5
- thread 2 on vring 3
- thread 2 on vring 7
- tx placement: spin-lock
- thread 0 on vring 0
- thread 1 on vring 2
- thread 2 on vring 0
-
- Memory regions (total 2)
- region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr
- ====== ===== ================== ================== ================== ================== ==================
- 0 60 0x0000000000000000 0x00000000000a0000 0x00002aaaaac00000 0x0000000000000000 0x00002aab2b400000
- 1 61 0x00000000000c0000 0x000000003ff40000 0x00002aaaaacc0000 0x00000000000c0000 0x00002aababcc0000
-
- Virtqueue 0 (TX)
- qsz 256 last_avail_idx 0 last_used_idx 0
- avail.flags 1 avail.idx 128 used.flags 1 used.idx 0
- kickfd 62 callfd 64 errfd -1
-
- Virtqueue 1 (RX)
- qsz 256 last_avail_idx 0 last_used_idx 0
- avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
- kickfd 65 callfd 66 errfd -1
-
- Virtqueue 2 (TX)
- qsz 256 last_avail_idx 0 last_used_idx 0
- avail.flags 1 avail.idx 128 used.flags 1 used.idx 0
- kickfd 63 callfd 70 errfd -1
-
- Virtqueue 3 (RX)
- qsz 256 last_avail_idx 0 last_used_idx 0
- avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
- kickfd 72 callfd 74 errfd -1
-
- Virtqueue 4 (TX disabled)
- qsz 256 last_avail_idx 0 last_used_idx 0
- avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
- kickfd 76 callfd 78 errfd -1
-
- Virtqueue 5 (RX disabled)
- qsz 256 last_avail_idx 0 last_used_idx 0
- avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
- kickfd 80 callfd 82 errfd -1
-
- Virtqueue 6 (TX disabled)
- qsz 256 last_avail_idx 0 last_used_idx 0
- avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
- kickfd 84 callfd 86 errfd -1
-
- Virtqueue 7 (RX disabled)
- qsz 256 last_avail_idx 0 last_used_idx 0
- avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
- kickfd 88 callfd 90 errfd -1
-
-The optional '*descriptors*' parameter will display the same output as the
-previous example but will include the descriptor table for each queue. The output is truncated below:
-
-.. code-block:: console
-
- vpp# show vhost-user VirtualEthernet0/0/0 descriptors
-
- Virtio vhost-user interfaces
- Global:
- coalesce frames 32 time 1e-3
- Interface: VirtualEthernet0/0/0 (ifindex 1)
- virtio_net_hdr_sz 12
- features mask (0xffffffffffffffff):
- features (0x50408000):
- VIRTIO_NET_F_MRG_RXBUF (15)
- VIRTIO_NET_F_MQ (22)
- :
- Virtqueue 0 (TX)
- qsz 256 last_avail_idx 0 last_used_idx 0
- avail.flags 1 avail.idx 128 used.flags 1 used.idx 0
- kickfd 62 callfd 64 errfd -1
-
- descriptor table:
- id addr len flags next user_addr
- ===== ================== ===== ====== ===== ==================
- 0 0x0000000010b6e974 2060 0x0002 1 0x00002aabbc76e974
- 1 0x0000000010b6e034 2060 0x0002 2 0x00002aabbc76e034
- 2 0x0000000010b6d6f4 2060 0x0002 3 0x00002aabbc76d6f4
- 3 0x0000000010b6cdb4 2060 0x0002 4 0x00002aabbc76cdb4
- 4 0x0000000010b6c474 2060 0x0002 5 0x00002aabbc76c474
- 5 0x0000000010b6bb34 2060 0x0002 6 0x00002aabbc76bb34
- 6 0x0000000010b6b1f4 2060 0x0002 7 0x00002aabbc76b1f4
- 7 0x0000000010b6a8b4 2060 0x0002 8 0x00002aabbc76a8b4
- 8 0x0000000010b69f74 2060 0x0002 9 0x00002aabbc769f74
- 9 0x0000000010b69634 2060 0x0002 10 0x00002aabbc769634
- 10 0x0000000010b68cf4 2060 0x0002 11 0x00002aabbc768cf4
- :
- 249 0x0000000000000000 0 0x0000 250 0x00002aab2b400000
- 250 0x0000000000000000 0 0x0000 251 0x00002aab2b400000
- 251 0x0000000000000000 0 0x0000 252 0x00002aab2b400000
- 252 0x0000000000000000 0 0x0000 253 0x00002aab2b400000
- 253 0x0000000000000000 0 0x0000 254 0x00002aab2b400000
- 254 0x0000000000000000 0 0x0000 255 0x00002aab2b400000
- 255 0x0000000000000000 0 0x0000 32768 0x00002aab2b400000
-
- Virtqueue 1 (RX)
- qsz 256 last_avail_idx 0 last_used_idx 0
-
-
-Debug Vhost-User
-================
-Turn on/off debug for vhost.
-
-
-Summary/Usage
--------------
-
-.. code-block:: shell
-
- debug vhost-user <on | off>
-
-Delete Vhost-User
-========================
-Delete a vHost User interface using the interface name or the software
-interface index. Use the '*show interface*' command to determine the
-software interface index. On deletion, the linux socket will not be
-deleted.
-
-Summary/Usage
--------------
-
-.. code-block:: shell
-
- delete vhost-user {<interface> | sw_if_index <sw_idx>}
-
-Examples
---------
-Example of how to delete a vhost interface by name:
-
-.. code-block:: console
-
- vpp# delete vhost-user VirtualEthernet0/0/1
-
-Example of how to delete a vhost interface by software interface index:
-
-.. code-block:: console
-
- vpp# delete vhost-user sw_if_index 1
diff --git a/docs/reference/github/index.rst b/docs/reference/github/index.rst
deleted file mode 100644
index 1d7231a..0000000
--- a/docs/reference/github/index.rst
+++ /dev/null
@@ -1,237 +0,0 @@
-.. _pushingapatch:
-
-=================
-Github Repository
-=================
-
-**The github repository is only being used as a source for readthedocs.**
-**There should be no reason for the typical developer to use this repository.**
-**It should only be used by a document developer.**
-
-Overview
-________
-
-This section will cover how to fork your own branch of the `fdioDocs/vpp-docs <https://github.com/fdioDocs/vpp-docs>`_ repository, clone that repo locally to your computer, make changes to it, and how to issue a pull request when you want your changes to be reflected on the main repo.
-
-.. toctree::
-
-Forking your own branch
-_______________________
-
-In your browser, navigate to the repo you want to branch off of. In this case, the `fdioDocs/vpp-docs <https://github.com/fdioDocs/vpp-docs>`_ repo. At the top right of the page you should see this:
-
-.. figure:: /_images/ForkButtons.png
- :alt: Figure: Repository options on Github
- :scale: 50%
- :align: right
-
-|
-|
-|
-
-Click on "Fork", and then a pop-up should appear where you should then click your Github username. Once this is done, it should automatically take you to the Github page where your new branch is located, just like in the image below.
-
-.. figure:: /_images/usernameFork.png
- :alt: Figure: Your own branch of the main repo on Github
- :scale: 35%
- :align: center
-
-
-Now your **own branch** can be **cloned** to your computer using the URL (https://github.com/YOURUSERNAME/vpp-docs) of the Github page where your branch is located.
-
-
-Creating a local repository
-___________________________
-
-Now that you have your own branch of the main repository on Github, you can store it locally on your computer. In your shell, navigate to the directory where you want to store your branch/repo. Then execute:
-
-.. code-block:: console
-
- $ git clone https://github.com/YOURUSERNAME/vpp-docs
-
-This will create a directory on your computer named **vpp-docs**, the name of the repo.
-
-Now that your branch is on your computer, you can modify and build files however you wish.
-
-If you are not on the master branch, move to it.
-
-.. code-block:: console
-
- $ git checkout master
-
-
-Keeping your files in sync with the main repo
-_____________________________________________
-
-The following talks about remote branches, but keep in mind that there are currently *two* branches, your local "master" branch (on your computer), and your remote "origin or origin/master" branch (the one you created using "Fork" on the Github website).
-
-You can view your *remote* repositories with:
-
-.. code-block:: console
-
- $ git remote -v
-
-At this point, you may only see the remote branch that you cloned from.
-
-.. code-block:: console
-
- Macintosh:docs Andrew$ git remote -v
- origin https://github.com/a-olechtchouk/vpp-docs (fetch)
- origin https://github.com/a-olechtchouk/vpp-docs (push)
-
-Now you want to create a new remote repository of the main vpp-docs repo (naming it upstream).
-
-.. code-block:: console
-
- $ git remote add upstream https://github.com/fdioDocs/vpp-docs
-
-
-You can verify that you have added a remote repo using the previous **git remote -v** command.
-
-.. code-block:: console
-
- $ git remote -v
- origin https://github.com/a-olechtchouk/vpp-docs (fetch)
- origin https://github.com/a-olechtchouk/vpp-docs (push)
- upstream https://github.com/fdioDocs/vpp-docs (fetch)
- upstream https://github.com/fdioDocs/vpp-docs (push)
-
-
-If there have been any changes to files in the main repo (hopefully not the same files you were working on!), you want to make sure your local branch is in sync with them.
-
-To do so, fetch any changes that the main repo has made, and then merge them into your local master branch using:
-
-.. code-block:: console
-
- $ git fetch upstream
- $ git merge upstream/master
-
-
-.. note:: **This is optional, so don't do these commands if you just want one local branch!!!**
-
- You may want to have multiple branches, where each branch has its own different features, allowing you to have multiple pull requests out at a time. To create a new local branch:
-
-.. code-block:: shell
-
- $ git checkout -b cleanup-01
- $ git branch
- * cleanup-01
- master
- overview
-
- Now you can redo the previous steps for "Keeping your files in sync with the main repo" for your newly created local branch, and then depending on which branch you want to send out a pull reqest for, proceed below.
-
-
-Pushing to your branch
-______________________
-
-Now that your files are in sync, you want to add modified files, commit, and push them from *your local branch* to your *personal remote branch* (not the main fdioDocs repo).
-
-To check the status of your files, run:
-
-.. code-block:: console
-
- $ git status
-
-
-In the output example below, I deleted gettingsources.rst, made changes to index.rst and pushingapatch.rst, and have created a new file called buildingrst.rst.
-
-.. code-block:: console
-
- Macintosh:docs Andrew$ git status
- On branch master
- Your branch is up-to-date with 'origin/master'.
- Changes to be committed:
- (use "git reset HEAD <file>..." to unstage)
-
- deleted: tasks/writingdocs/gettingsources.rst
-
- Changes not staged for commit:
- (use "git add <file>..." to update what will be committed)
- (use "git checkout -- <file>..." to discard changes in working directory)
-
- modified: tasks/writingdocs/index.rst
- modified: tasks/writingdocs/pushingapatch.rst
-
- Untracked files:
- (use "git add <file>..." to include in what will be committed)
-
- tasks/writingdocs/buildingrst.rst
-
-
-
-To add files (use **git add -A** to add all modified files):
-
-.. code-block:: console
-
- $ git add FILENAME1 FILENAME2
-
-Commit and push using:
-
-.. code-block:: console
-
- $ git commit -m 'A descriptive commit message for two files.'
-
-Push your changes for the branch where your changes were made
-
-.. code-block:: console
-
- $ git push origin <branch name>
-
-Here, your personal remote branch is "origin" and your local branch is "master".
-
-.. note::
-
- Using **git commit** after adding your files saves a "Snapshot" of them, so it's very hard to lose your work if you *commit often*.
-
-
-
-Initiating a pull request (Code review)
-_______________________________________
-
-Once you've pushed your changes to your remote branch, go to your remote branch on Github (https://github.com/YOURUSERNAME/vpp-docs), and click on "New pull request".
-
-.. figure:: /_images/issuePullReq.png
- :alt: Figure: Your own branch of the main repo on Github
- :scale: 35%
- :align: center
-
-This will bring you to a "Comparing changes" page. Click "Create new pull request".
-
-.. figure:: /_images/createNewPullReq.png
- :scale: 35%
- :align: left
-
-|
-|
-|
-
-Which will open up text fields to add information to your pull request.
-
-.. figure:: /_images/examplePullReq.png
- :scale: 35%
- :align: center
-
-
- Then finally click "Create pull request" to complete the pull request.
-
-Your documents will be reviewed. To this same branch make the changes requested from the review and then push your new changes. There is no need to create another pull request.
-
-.. code-block:: console
-
- $ git commit -m 'A descriptive commit message for the new changes'
- $ git push origin <branch name>
-
-
-Additional Git commands
-_______________________
-
-You may find some of these Git commands useful:
-
-Use **git diff** to quickly show the file changes and repo differences of your commits.
-
-Use **git rm FILENAME** to stop tracking a file and to remove it from your remote branch and local directory. Use flag **-r** to remove folders/directories. E.g (**git rm -r oldfolder**)
-
-
-.. _fdioDocs: https://github.com/fdioDocs/vpp-docs
-
diff --git a/docs/reference/index.rst b/docs/reference/index.rst
deleted file mode 100644
index 76156a2..0000000
--- a/docs/reference/index.rst
+++ /dev/null
@@ -1,13 +0,0 @@
-.. _reference:
-
-######################
-Reference
-######################
-
-.. toctree::
- :maxdepth: 2
-
- cmdreference/index.rst
- vppvagrant/index.rst
- readthedocs/index.rst
- github/index.rst
diff --git a/docs/reference/readthedocs/index.rst b/docs/reference/readthedocs/index.rst
deleted file mode 100644
index 5ddade8..0000000
--- a/docs/reference/readthedocs/index.rst
+++ /dev/null
@@ -1,129 +0,0 @@
-.. _readthedocs:
-
-Read The Docs
-=================
-
-`Read the Docs <https://readthedocs.org/>`_ is a website that "simplifies software documentation by automating building, versioning, and hosting of your docs for you". Essentially, it accesses your Github repo to generate the **index.html** file, and then displays it on its own *Read the Docs* webpage so others can view your documentation.
-
-Create an account on *Read the Docs* if you haven't already.
-
-Go to your `dashboard <https://readthedocs.org/dashboard/>`_ , and click on "Import a Project".
-
-.. figure:: /_images/importReadDocs.png
- :scale: 35%
- :align: left
-
- This will bring you to a page where you can choose to import a repo from your Github account (only if you've linked your Github account to your Read the Docs account), or to import a repo manually. In this example, we'll do it manually. Click "Import Manually".
-
-|
-|
-|
-|
-|
-|
-|
-
-
-
-This will bring you to a page that asks for your repo details. Set "Name" to your forked repo name, or whatever you want. Set "Repository URL" to the URL of your forked repo (https://github.com/YOURUSERNAME/vpp-docs). "Repository type" should already be selected to "Git". Then click "Next".
-
-
-.. figure:: /_images/importRTDManually.png
- :scale: 35%
- :align: left
-
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-
-
-This will bring you to a project page of your repo on Read the Docs. You can confirm it's the correct repo by checking on the right side of the page the Repository URL.
-
-Then click on "Build Version".
-
-.. figure:: /_images/buildVerRTD.png
- :scale: 35%
- :align: left
-
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-
-Which takes you to another page showing your recent builds.
-
-Then click on "Build Version:". This should "Trigger" a build. After about a minute or so you can refresh the page and see that your build "Passed".
-
-
-.. figure:: /_images/passedBuild.png
- :scale: 35%
- :align: left
-
-
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-
-
-Now on your builds page from the previous image, you can click "View Docs" at the top-right, which will take you a *readthedocs.io* page of your generated build!
-
-.. figure:: /_images/rtdWebpage.png
- :scale: 30%
- :align: left
diff --git a/docs/reference/vppvagrant/VagrantVMSetup.rst b/docs/reference/vppvagrant/VagrantVMSetup.rst
deleted file mode 100644
index c13a87e..0000000
--- a/docs/reference/vppvagrant/VagrantVMSetup.rst
+++ /dev/null
@@ -1,56 +0,0 @@
-.. _VagrantVMSetup:
-
-.. toctree::
-
-Accessing your VM
-^^^^^^^^^^^^^^^^^
-ssh into the newly created box:
-
-.. code-block:: shell
-
- $ vagrant ssh <id>
-
-Sample output looks like:
-
-.. code-block:: console
-
- $ vagrant ssh c1c
- Welcome to Ubuntu 16.04 LTS (GNU/Linux 4.4.0-21-generic x86_64)
-
- * Documentation: https://help.ubuntu.com/
- Last login: Mon Jun 25 08:05:38 2018 from 10.0.2.2
- vagrant@localhost:~$
-
-
-.. note::
-
- Type **exit** in the command-line if you want to exit the VM.
-
-Become the root with:
-
-.. code-block:: shell
-
- $ sudo bash
-
-Now *install* VPP in the VM. Keep in mind that VPP is already built (but not yet installed) at this point based on the commands from the provisioned script *build.sh*.
-
-When you ssh into your Vagrant box you will be placed in the directory */home/vagrant*. Change directories to */vpp/build-root*, and run these commands to install VPP based on your OS and architecture:
-
-For Ubuntu systems:
-
-.. code-block:: shell
-
- # dpkg -i *.deb
-
-For CentOS systems:
-
-.. code-block:: shell
-
- # rpm -Uvh *.rpm
-
-
-Since VPP is now installed, you can start running VPP with:
-
-.. code-block:: shell
-
- # service vpp start
diff --git a/docs/reference/vppvagrant/Vagrantfile b/docs/reference/vppvagrant/Vagrantfile
deleted file mode 100644
index 642969f..0000000
--- a/docs/reference/vppvagrant/Vagrantfile
+++ /dev/null
@@ -1,112 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-Vagrant.configure(2) do |config|
-
- # Pick the right distro and bootstrap, default is ubuntu1604
- distro = ( ENV['VPP_VAGRANT_DISTRO'] || "ubuntu1604")
- if distro == 'centos7'
- config.vm.box = "centos/7"
- config.vm.box_version = "1708.01"
- config.ssh.insert_key = false
- else
- config.vm.box = "puppetlabs/ubuntu-16.04-64-nocm"
- end
- config.vm.box_check_update = false
-
- config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"update.sh")
- config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"build.sh"), :args => "/vpp vagrant"
-
- post_build = ( ENV['VPP_VAGRANT_POST_BUILD'] )
- if post_build == "test"
- config.vm.provision "shell", inline: "echo Testing VPP; cd /vpp; make test"
- elsif post_build == "install"
- config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"install.sh"), :args => "/vpp"
- config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"clearinterfaces.sh")
- config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"run.sh")
- end
-
- # Add .gnupg dir in so folks can sign patches
- # Note, as gnupg puts socket files in that dir, we have
- # to be cautious and make sure we are dealing with a plain file
- homedir = File.expand_path("~/")
- Dir["#{homedir}/.gnupg/**/*"].each do |fname|
- if File.file?(fname)
- destname = fname.sub(Regexp.escape("#{homedir}/"),'')
- config.vm.provision "file", source: fname, destination: destname
- end
- end
-
- # Copy in the .gitconfig if it exists
- if File.file?(File.expand_path("~/.gitconfig"))
- config.vm.provision "file", source: "~/.gitconfig", destination: ".gitconfig"
- end
-
- # vagrant-cachier caches apt/yum etc to speed subsequent
- # vagrant up
- # to enable, run
- # vagrant plugin install vagrant-cachier
- #
- if Vagrant.has_plugin?("vagrant-cachier")
- config.cache.scope = :box
- end
-
- # Define some physical ports for your VMs to be used by DPDK
- nics = (ENV['VPP_VAGRANT_NICS'] || "2").to_i(10)
- for i in 1..nics
- config.vm.network "private_network", type: "dhcp"
- end
-
- # use http proxy if available
- if ENV['http_proxy'] && Vagrant.has_plugin?("vagrant-proxyconf")
- config.proxy.http = ENV['http_proxy']
- config.proxy.https = ENV['https_proxy']
- config.proxy.no_proxy = "localhost,127.0.0.1"
- end
-
- vmcpu=(ENV['VPP_VAGRANT_VMCPU'] || 2)
- vmram=(ENV['VPP_VAGRANT_VMRAM'] || 4096)
-
- config.ssh.forward_agent = true
- config.ssh.forward_x11 = true
-
- config.vm.provider "virtualbox" do |vb|
- vb.customize ["modifyvm", :id, "--ioapic", "on"]
- vb.memory = "#{vmram}"
- vb.cpus = "#{vmcpu}"
-
- # rsync the vpp directory if provision hasn't happened yet
- unless File.exist? (".vagrant/machines/default/virtualbox/action_provision")
- config.vm.synced_folder "../../", "/vpp", type: "rsync",
- rsync__auto: false,
- rsync__exclude: [
- "build-root/build*/",
- "build-root/install*/",
- "build-root/images*/",
- "build-root/*.deb",
- "build-root/*.rpm",
- "build-root/*.changes",
- "build-root/python",
- "build-root/deb/debian/*.dkms",
- "build-root/deb/debian/*.install",
- "build-root/deb/debian/changes",
- "build-root/tools"]
- end
-
- #support for the SSE4.x instruction is required in some versions of VB.
- vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"]
- vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"]
- end
- config.vm.provider "vmware_fusion" do |fusion,override|
- fusion.vmx["memsize"] = "#{vmram}"
- fusion.vmx["numvcpus"] = "#{vmcpu}"
- end
- config.vm.provider "libvirt" do |lv|
- lv.memory = "#{vmram}"
- lv.cpus = "#{vmcpu}"
- end
- config.vm.provider "vmware_workstation" do |vws,override|
- vws.vmx["memsize"] = "#{vmram}"
- vws.vmx["numvcpus"] = "#{vmcpu}"
- end
-end
diff --git a/docs/reference/vppvagrant/boxSetup.rst b/docs/reference/vppvagrant/boxSetup.rst
deleted file mode 100644
index ed20c9d..0000000
--- a/docs/reference/vppvagrant/boxSetup.rst
+++ /dev/null
@@ -1,122 +0,0 @@
-.. _boxSetup:
-
-.. toctree::
-
-
-Vagrantfiles
-============
-
-A `Vagrantfile <https://www.vagrantup.com/docs/vagrantfile/>`_ contains the box and provision configuration settings for your VM. The syntax of Vagrantfiles is Ruby (Ruby experience is not necessary).
-
-The command **vagrant up** creates a *Vagrant Box* based on your Vagrantfile. A Vagrant box is one of the motivations for using Vagrant - its a "development-ready box" that can be copied to other machines to recreate the same environment.
-
-It's common for people to think that a Vagrant box *is* the VM. But rather, the VM is *inside* a Vagrant box, with the box containing additional configuration options you can set, such as VM options, scripts to run on boot, etc.
-
-This `Vagrant website for boxes <https://app.vagrantup.com/boxes/search>`_ shows you how to configure a basic Vagrantfile for your specific OS and VM software.
-
-
-Box configuration
-_________________
-
-
-Looking at the :ref:`vppVagrantfile`, we can see that the default OS is Ubuntu 16.04 (since the variable *distro* equals *ubuntu1604* if there is no VPP_VAGRANT_DISTRO variable set - thus the **else** case is executed.)
-
-.. code-block:: ruby
-
- # -*- mode: ruby -*-
- # vi: set ft=ruby :
-
- Vagrant.configure(2) do |config|
-
- # Pick the right distro and bootstrap, default is ubuntu1604
- distro = ( ENV['VPP_VAGRANT_DISTRO'] || "ubuntu1604")
- if distro == 'centos7'
- config.vm.box = "centos/7"
- config.vm.box_version = "1708.01"
- config.ssh.insert_key = false
- else
- config.vm.box = "puppetlabs/ubuntu-16.04-64-nocm"
-
-As mentioned in the previous page, you can specify which OS and VM provider you want for your Vagrant box from the `Vagrant boxes page <https://app.vagrantup.com/boxes/search>`_, and setting your ENV variable appropriately in *env.sh*.
-
-Next in the Vagrantfile, you see some *config.vm.provision* commands. As paraphrased from `Basic usage of Provisioners <https://www.vagrantup.com/docs/provisioning/basic_usage.html>`_, by default these are only run *once* - during the first boot of the box.
-
-.. code-block:: ruby
-
- config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"update.sh")
- config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"build.sh"), :args => "/vpp vagrant"
-
-The two lines above set the VM to run two scripts during its first bootup: an update script *update.sh* that does basic updating and installation of some useful tools, as well as *build.sh* that builds (but does **not** install) VPP in the VM. You can view these scripts on your own for more detail on the commands used.
-
-
-Looking further in the :ref:`vppVagrantfile`, you can see more Ruby variables being set to ENV's or to a default value:
-
-.. code-block:: ruby
-
- # Define some physical ports for your VMs to be used by DPDK
- nics = (ENV['VPP_VAGRANT_NICS'] || "2").to_i(10)
- for i in 1..nics
- config.vm.network "private_network", type: "dhcp"
- end
-
- # use http proxy if available
- if ENV['http_proxy'] && Vagrant.has_plugin?("vagrant-proxyconf")
- config.proxy.http = ENV['http_proxy']
- config.proxy.https = ENV['https_proxy']
- config.proxy.no_proxy = "localhost,127.0.0.1"
- end
-
- vmcpu=(ENV['VPP_VAGRANT_VMCPU'] || 2)
- vmram=(ENV['VPP_VAGRANT_VMRAM'] || 4096)
-
-
-You can see how the box or VM is configured, such as the amount of NICs (defaults to 3 NICs: 1 x NAT - host access and 2 x VPP DPDK enabled), CPUs (defaults to 2), and RAM (defaults to 4096 MB).
-
-
-Box bootup
-__________
-
-
-Once you're satisfied with your *Vagrantfile*, boot the box with:
-
-.. code-block:: shell
-
- $ vagrant up
-
-Doing this above command will take quite some time, since you are installing a VM and building VPP. Take a break and get some scooby snacks while you wait.
-
-To confirm it is up, show the status and information of Vagrant boxes with:
-
-.. code-block:: console
-
- $ vagrant global-status
- id name provider state directory
- -----------------------------------------------------------------------------------------
- d90a17b default virtualbox poweroff /home/centos/andrew-vpp/vppsb/vpp-userdemo
- 77b085e default virtualbox poweroff /home/centos/andrew-vpp/vppsb2/vpp-userdemo
- c1c8952 default virtualbox poweroff /home/centos/andrew-vpp/testingVPPSB/extras/vagrant
- c199140 default virtualbox running /home/centos/andrew-vpp/vppsb3/vpp-userdemo
-
- You will have only one machine running, but I have multiple as shown above.
-
-.. note::
-
- To poweroff your VM, type:
-
- .. code-block:: shell
-
- $ vagrant halt <id>
-
- To resume your VM, type:
-
- .. code-block:: shell
-
- $ vagrant resume <id>
-
- To destroy your VM, type:
-
- .. code-block:: shell
-
- $ vagrant destroy <id>
-
- Note that "destroying" a VM does not erase the box, but rather destroys all resources allocated for that VM. For other Vagrant commands, such as destroying a box, refer to the `Vagrant CLI Page <https://www.vagrantup.com/docs/cli/>`_.
diff --git a/docs/reference/vppvagrant/index.rst b/docs/reference/vppvagrant/index.rst
deleted file mode 100644
index 04da9e3..0000000
--- a/docs/reference/vppvagrant/index.rst
+++ /dev/null
@@ -1,15 +0,0 @@
-.. _vppvagrant:
-
-VM's with Vagrant
-=================
-
-This reference guide will cover using Vagrant to boot a VM (virtual machine).
-
-.. toctree::
-
- vagrantOverview
- installingVboxVagrant
- settingENV
- boxSetup
- VagrantVMSetup
- vppVagrantfile
diff --git a/docs/reference/vppvagrant/installingVboxVagrant.rst b/docs/reference/vppvagrant/installingVboxVagrant.rst
deleted file mode 100644
index 3263128..0000000
--- a/docs/reference/vppvagrant/installingVboxVagrant.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-.. _installingVboxVagrant:
-
-Installing Vbox and Vagrant
-===========================
-
-Installing VirtualBox
-_____________________
-
-First download VirtualBox, which is virtualization software for creating VM's.
-
-If you're on CentOS, follow the `steps here <https://wiki.centos.org/HowTos/Virtualization/VirtualBox>`_.
-
-
-If you're on Ubuntu, perform:
-
-.. code-block:: shell
-
- $ sudo apt-get install virtualbox
-
-If you want to download a newer version of Virtual Box or one specific to your OS and architecture, go to the `Virtual Box download page <https://www.virtualbox.org/wiki/Downloads>`_.
-
-Installing Vagrant
-__________________
-
-Here we are on a 64-bit version of CentOS, downloading and installing Vagrant 2.1.2:
-
-.. code-block:: shell
-
- $ yum -y install https://releases.hashicorp.com/vagrant/2.1.2/vagrant_2.1.2_x86_64.rpm
-
-This is a similar command, but on a 64-bit version of Debian:
-
-.. code-block:: shell
-
- $ sudo apt-get install https://releases.hashicorp.com/vagrant/2.1.2/vagrant_2.1.2_x86_64.deb
-
-
-If you want to download a newer version of Vagrant or one specific to your OS and architecture, go to the `Vagrant download page <https://www.vagrantup.com/downloads.html>`_.
diff --git a/docs/reference/vppvagrant/settingENV.rst b/docs/reference/vppvagrant/settingENV.rst
deleted file mode 100644
index 8bd7847..0000000
--- a/docs/reference/vppvagrant/settingENV.rst
+++ /dev/null
@@ -1,29 +0,0 @@
-.. _settingENV:
-
-.. toctree::
-
-
-Setting your ENV Variables
-==========================
-
-
-The :ref:`vppVagrantfile` used in the VPP repo sets the configuration options based on your ENV (environment) variables, or to default the configuration at specified values if your ENV variables are not initialized (if you did not run the *env.sh* script found below).
-
-This is the *env.sh* script found in *vpp/extras/vagrant*. When run, the script sets ENV variables using the **export** command.
-
-.. code-block:: bash
-
- export VPP_VAGRANT_DISTRO="ubuntu1604"
- export VPP_VAGRANT_NICS=2
- export VPP_VAGRANT_VMCPU=4
- export VPP_VAGRANT_VMRAM=4096
-
-In the :ref:`vppVagrantfile`, you can see these same ENV variables used (discussed on the next page).
-
-Adding your own ENV variables is easy. For example, if you wanted to setup proxies for your VM, you would add to this file the **export** commands found in the :ref:`building VPP commands section <building>`. Note that this only works if the ENV variable is defined in the :ref:`vppVagrantfile`.
-
-Once you're finished with *env.sh* script, and you are in the directory containing *env.sh*, run the script to set the ENV variables with:
-
-.. code-block:: shell
-
- $ source ./env.sh
diff --git a/docs/reference/vppvagrant/vagrantOverview.rst b/docs/reference/vppvagrant/vagrantOverview.rst
deleted file mode 100644
index 882b8a6..0000000
--- a/docs/reference/vppvagrant/vagrantOverview.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-.. _vagrantOverview:
-
-.. toctree::
-
-Overview
-^^^^^^^^
-
-This guide shows how the VPP :ref:`vppVagrantfile` interacts with shell scripts to configure a `Vagrant box <https://www.vagrantup.com/docs/boxes.html>`_ that boots a VM with VPP.
-
-.. _prereqlabel:
-
-Prerequisites
-_____________
-
-You have the `FD.io VPP repo <https://github.com/FDio/vpp>`_ cloned locally on your machine.
-
-
-This guide will refer to the following files from that repo: *Vagrantfile, build.sh, env.sh, and update.sh*.
-
-
-
-
-
-
-
diff --git a/docs/reference/vppvagrant/vppVagrantfile.rst b/docs/reference/vppvagrant/vppVagrantfile.rst
deleted file mode 100644
index d3c7055..0000000
--- a/docs/reference/vppvagrant/vppVagrantfile.rst
+++ /dev/null
@@ -1,14 +0,0 @@
-.. _vppVagrantfile:
-
-Vagrant File
-============
-
-This is the Vagrantfile provided in the `Git VPP repo <https://github.com/FDio/vpp/blob/master/extras/vagrant/Vagrantfile>`_.
-
-.. literalinclude:: Vagrantfile
- :language: Ruby
- :emphasize-lines: 4-17, 20-21, 57-71
-
-
-
-.. toctree::
diff --git a/docs/relatedprojects/index.rst b/docs/relatedprojects/index.rst
deleted file mode 100644
index 12d366a..0000000
--- a/docs/relatedprojects/index.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. _relatedprojects:
-
-######################
-Related FD.IO Projects
-######################
-
-.. toctree::
-
-***********
-CICN & HICN
-***********
-
-`<https://docs.fd.io/cicn>`_ and `<https://docs.fd.io/hicn>`_
-
-
-************************************************
-Continuous System Integration and Testing (CSIT)
-************************************************
-
-`<https://docs.fd.io/csit>`_
diff --git a/docs/scripts/sphinx-make.sh b/docs/scripts/sphinx-make.sh
deleted file mode 100755
index 532225e..0000000
--- a/docs/scripts/sphinx-make.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash -ex
-
-# Not refactored to root Makefile because CI calls this from
-# makefile in /docs (as if 'make -C docs').
-if [ -z "$PYTHON" ]
-then
-PYTHON_INTERP=python3
-else
-PYTHON_INTERP=$(PYTHON)
-fi
-
-# Get the OS
-OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-OS_VERSION=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-PIP_VERSION=$(grep 'PIP_VERSION=' $WS_ROOT/test/Makefile | cut -d'=' -f2)
-PIP_TOOLS_VERSION=$(grep 'PIP_TOOLS_VERSION=' $WS_ROOT/test/Makefile | cut -d'=' -f2)
-
-if [ "$1" == "venv" ]
-then
-
- # Install the virtual environment
- $PYTHON_INTERP -m venv $VENV_DIR
- source $VENV_DIR/bin/activate;
- $PYTHON_INTERP -m pip install pip==$PIP_VERSION
- $PYTHON_INTERP -m pip install pip-tools==$PIP_TOOLS_VERSION
- $PYTHON_INTERP -m pip install -r $WS_ROOT/test/requirements-3.txt
-else
- [ -n "$(declare -f deactivate)" ] && deactivate
- source $VENV_DIR/bin/activate;
- VERSION=`source $WS_ROOT/src/scripts/version`
- TM=`TZ=GMT date`
- sed -ie "s/**VPP Version:\*\* .*/**VPP Version:** $VERSION/" $DOCS_DIR/about.rst
- sed -ie "s/**Built on:\*\* .*/**Built on:** $TM/" $DOCS_DIR/about.rst
- rm $DOCS_DIR/about.rste
- make -C $DOCS_DIR $1
-fi
-
-deactivate
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index f5cbf4e..53f34c5 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -1,37 +1,615 @@
-aarch
-Aarch
AArch
-abf
-accelera
ACK
-acl
ACL
ACLs
-addr
-admin
-af
+API
+ASan
+Aarch
Agg
Ajourn
+Ananyev
+govpp
+unserialization
+archs
+symlink
+libmemf
+libme
+mif
+pwait
+tunnelling
+ctx
+efd
+optimised
+strerror
+syscall
+ligato
+congruential
+phc
+Anycast
+Arp
+icmpr
+pps
+bc
+midchain
+Midchain
+fmt
+intfcs
+libmemifs
+gethostbyname
+multithreading
+sourceNATing
+Mapp
+Headend
+pgm
+retransmits
+cheng
+iccrg
+filsfils
+Datagram
+syscalls
+Altivec
+memcpy
+mmap
+Warshall
+XXhash
+headend
+Fortville
+openbsd
+zx
+openbsd
+Diffie
+github
+criterias
+daddr
+gettingstarted
+Arping
+Arping
+Autogenerated
+BGP
+adaptively
+Defensic
+aa
+BIHASH
+BV
+Backtrace
+allocatable
+consumability
+Usermode
+isn
+shouldn
+combinatorial
+ufwd
+Barach
+Bfd
+receival
+ap
+Bihash
+emphasise
+Bihashes
+neighbouring
+Brockners
+LimitCORE
+eXtensible
+aabb
+aftr
+Recircs
+Bufmon
+iPv
+Builtinurl
+deallocation
+nonaddress
+CMakeLists
+CNat
+CPPFLAGS
+Castelino
+Centos
+efp
+preallocation
+Cisco
+Clib
+Cmake
+Cnat
+ufwd
+runit
+rulebase
+seatbelt
+iWARP
+strongswan
+unapplies
+subgraphs
+Codenomicon
+eeff
+unapply
+analyse
+Colour
+subdirectories
+Comms
+srcIP
+Config
+unreachables
+ager
+ConnectX
+str
+undirected
+Contiv
+reselects
+Coras
+unmanaged
+Coredumps
+Coverity
+Crypto
+Cryptodev
+DS
+DSR
+Dataplane
+Dataplanes
+Debuginfo
+Decapsulating
+Dest
+Diffs
+Distro
+Donley
+Doxygen
+Eifel
+Encaps
+Enqueueing
+Enums
+Erns
+Extranets
+FiB
+Flowprobe
+GByte
+GTP
+GbE
+Gbe
+Gbits
+Gbps
+Gbytes
+Geneve
+Gerrit
+GigE
+Gigabit
+Github
+Goncalves
+Gzip
+HQoS
+Heapsize
+Hotkeys
+Huawei
+Hypervisors
+IBverbs
+IKEv2
+IP
+IPsec
+IPv
+IPv4
+Ikev
+Init
+Instantiation
+Integrations
+Interoperation
+Iperf
+Jira
+KMG
+KOC
+KVP
+programmability
+decapsulate
+pseudocode
+libsrtp
+upf
+Konstantynowicz
+Kube
+Kubeadm
+Kubelet
+Kubernetes
+L3XC
+LB
+LDFLAGS
+LLDP
+Linuxcon
+Lldp
+balancer
+balancers
+Lockless
+Loopback
+Lua
+encap
+Luajit
+MPLS
+MPLS
+Maciek
+Makefile
+Makefiles
+Manohar
+Maryam
+Mbytes
+Mdata
+Mellanox
+Memif
+Metadata
+Mfib
+Monroy
+Mpps
+Mss
+Multicast
+Multicore
+Multihoming
+Multiqueue
+Multistream
+Multitenancy
+Mutlithreading
+NIC
+llist
+NSH
+NUMA
+Namespace
+mrvl
+mempool
+localsids
+msgbuf
+logrotate
+auditd
+ndp
+neighbour
+modularity
+masse
+snapcraft
+tgz
+lcov
+cd
+gcovr
+ggcov
+Bn
+Cn
+unmounting
+sealert
+ausearch
+semodule
+Snapcraft
+multipass
+awk
+ldpreload
+iPerf
+xterms
+ps
+centos
+statfs
+proc
+unmounts
+unmount
+fs
+mountpoint
+statSocket
+Unmounting
+fusermount
+Golang
+setUp
+tearDown
+Stdout
+stdout
+testcase
+unittest
+TestVxlan
+UNUP
+pton
+infos
+papi
+setUpClass
+nelts
+portranges
+wasn
+wi
+xcrw
+xcrw
+xtr
+includedir
+iface
+realised
+interoperation
+Infiniband
+Inband
+woohoo
+inex
+inet
+instantiation
+libibverb
+lc
+jitter
+incrementing
+libvppapiclient
+linearization
+recurse
+recirc
+teib
+th
+srcPortDefinition
+srcUP
+ssm
+spinlock
+te
+recode
+srh
+stderr
+sha
+sys
+conn
+sv
+iWARP
+et
+handoffdemo
+syscfg
+hbh
+timebase
+gtp
+fbfc
+tmpl
+preprocessor
+fixup
+psp
+fmask
+extranet
+spd
+DT
+DX
+undefine
+aren
+subgraph
+expander
+encaps
+backwalk
+ccdd
+cdbdec
+centric
+cdba
+cfd
+cksum
+reallocations
+preload
+reachability
+rdtsc
+putatively
+ppc
+pitr
+pmtu
+rekeying
+requeueing
+requeued
+recusive
+requeue
+resolvers
+recurse
+recurses
+ress
+rloc
+rr
+rxandtx
+runnable
+scrollbars
+scrollbar
+tw
+typedef
+wouldn
+zeroize
+untagged
+upstreamed
+vectorized
+vfr
+vtf
+pbb
+petr
+peerings
+pathname
+pipelined
+parsable
+parallelize
+parallelizes
+pipelining
+netdev
+noevaluate
+onlink
+outacl
+preallocated
+nexthop
+nosyslog
+nthrottle
+nsess
+Neighbour
+nbr
+nd
+musdk
+muppet
+makefile
+mult
+Netmap
+Netplan
+mdata
+mroute
+Nsh
+Nstrings
+NxN
+loopId
+macos
+O'Driscoll
+OAM
+Oddbuf
+Openssl
+subnets
+Optimisations
+PCI
+PSP
+Pcap
+Perfmon
+Performan
+Pfister
+Plugin
+Plugins
+Pnat
+PoC
+Policers
+Pollable
+Preallocate
+Prebuilt
+Promisc
+Pseudocode
+Qemu
+QoS
+rss
+strongSwan
+Quic
+verifier
+Quickstart
+Load
+RLOC
+Reass
+uri
+alg
+resync
+pnat
+membulk
+uRPF
+vr
+proto
+pmc
+macip
+unthrottle
+liveness
+sa
+snat
+vss
+uncomment
+Rebase
+tmpfile
+url
+lt
+eid
+itr
+rlocs
+vip
+vips
+flowtable
+lcp
+rawhtml
+myhugosite
+www
+Redhat
+javascript
+Refactor
+Refactored
+subdirectory
+Responder
+shortcodes
+shortcode
+Roadmap
+Roto
+SCHED_RR
+hugo
+SRTP
+SRv
+SRv6
+Scapy
+Shamir
+Shrikant
+Slideshow
+Specialisations
+Srmpls
+Srv
+Stateful
+Strongswan
+Subnet
+Subnets
+Sw
+Syslog
+Systemd
+TLS
+TPv
+TRex
+Tahhan
+Tahr
+Templated
+Tenant
+Terabit
+Terraform
+Timebase
+Tollet
+Tracedump
+Tsou
+Tunnelling
+UDP
+Unformat
+Uninstall
+Uninstalling
+Unittest
+Urpf
+Usergroup
+Userspace
+VM
+VMs
+VMware
+VPP
+VPP
+VRF
+VRF
+Vagrantfile
+Vagrantfiles
+Valgrind
+Vbox
+Vhost
+Virtio
+Virtualization
+Vlib
+Vmxnet
+Vnet
+Vpp
+Vppinfra
+Vrrp
+Vswitch
+Vxlan
+Warnicke
+Webinar
+Wiki
+Wikipedia
+Wireguard
+Wireshark
+Wl
+Workstream
+XXX
+Xenial
+Xeon
+Xerus
+Xtensa
+Yaml
+Zhao
+aarch
+abf
+accelera
+acl
+acyclic
+addr
+adl
+admin
+af
al
alloc
+allocator
+allowlist
amine
-Ananyev
-Anycast
+ansi
api
-API
apiTrace
app
apps
arg
args
arp
-Arp
-µs
+arping
async
+atomics
auth
autoconf
+autoconfig
+autoconfiguration
autodetect
-Autogenerated
automake
autoreconf
autotools
@@ -42,96 +620,128 @@
backend
backends
backtrace
-Backtrace
backtracing
-Barach
baseva
bd
behaviour
+bfd
+bift
+bigly
bihash
-Bihash
-BIHASH
-Bihashes
+bihash
binutils
bitmask
bitzero
+blackholing
+blake
bms
+boolean
bootup
-Brockners
+bsid
bss
bufferindex
+bufmon
bugfixing
buildingrst
builtin
+builtinurl
busybox
-BV
bvi
byteswap
+cPython
cacheline
-Castelino
ccache
ccmake
cdp
ce
-Centos
cfssl
cisco
-Cisco
cj
classful
cli
clib
-Clib
cmake
-Cmake
-CMakeLists
cmd
-Colour
-Comms
condvar
conf
config
-Config
connectable
contiv
-Contiv
copwhitelist
copwhtlistbase
-Coras
coredump
+macswap
+datagrams
+plen
coredumps
-Coredumps
+refcount
corefile
corefiles
+usecases
corekeeper
corelist
+fff
+middlebox
+backpressure
+ruleset
+lookups
+callees
+dstIP
+dstPortDefinition
+promisc
+netfilter
+multiprotocol
+pluggable
+eBPF
+libbpf
+hostname
+xffb
+lpc
+ec
+adjacencies
+neighbours
+pluggable
+lagrange
+reass
+lw
+desc
+dscp
+apis
+verboseness
+controlplane
+utilisation
+dsr
coremask
couldn
-Coverity
+mfib
+params
cp
+ei
+det
cpel
+decaps
+datapath
+daemontools
+daemonizing
+decapsulating
+decapsulated
cpp
-CPPFLAGS
cpu
-cPython
crc
+syntaxes
+urpf
crcs
crt
crypto
-Crypto
cryptodev
-Cryptodev
ctrl
ctwo
dataplane
-Dataplane
-Dataplanes
datastructures
dbg
de
debian
debuggability
-Debuginfo
decap
decapsulation
declutters
@@ -142,57 +752,59 @@
depN
deselection
dest
-Dest
dev
devbind
df
dhcp
+dhcp
dhcpd
-Diffs
+dhcpserver
dir
dirN
distclean
distro
-Distro
distros
dlsym
dns
dnsmasq
doesn
-Donley
dont
doxygen
-Doxygen
dpdk
+dpkg
dpo
+dport
dryrun
-DS
dslite
dst
dumpxml
-eal
eBGP
-Eifel
+eal
elog
elogs
elts
+emacs
+emerg
enablement
encap
endian
enic
enp
+enqueue
+enqueued
enqueueing
-Enqueueing
+enqueues
+enums
eos
epoll
-Erns
etcd
ethernet
ethertype
ethip
eventfd
evt
-Extranets
+executables
+extensibility
failover
fallback
favour
@@ -200,13 +812,12 @@
favours
fd
fdioDocs
-FiB
fifo
fifos
filesystem
+flavours
flowendns
flowprobe
-Flowprobe
flowstartns
fn
foo
@@ -214,28 +825,63 @@
fooB
fooC
foreach
-Gbe
-GbE
gbit
-Gbits
+responder
+cryptographic
+tabula
+srv
+nsh
+xc
gbp
-Gbps
-GByte
-Gbytes
gcc
gcov
+lxd
+testbed
+https
+configs
+zfs
+workspaces
+mss
+oddbuf
+srtp
+svs
+tlspicotls
+picotls
+respondhost
+initiatehost
+eth
+pem
+dhcpv
+subnet
+debootstrap
+sharepoint
+deps
gdb
ge
gerrit
-Gerrit
gettingsources
gid
-Gigabit
-GigE
-Github
+marvell
+lldp
+geneve
+openssl
+ipsecmb
+cnat
+xdp
+runtime
+ikev
+ioam
+http
+hs
+runtime
+DoS
+chachapoly
glibc
gmake
-Goncalves
+libc
+tracedump
+urpf
+wireguard
gpe
gre
grep
@@ -245,37 +891,33 @@
gtpu
gud
gui
-Gzip
hairpinning
handoff
handoffs
+hashtable
hdr
heapsize
-Heapsize
-hostnames
hostNetwork
+hostnames
hoststack
hotkey
-Hotkeys
hpp
hqos
-HQoS
hsa
html
htop
-Huawei
hugepage
hugepages
hugetlb
hw
hyperlink
hypervisor
-Hypervisors
+iACL
+iAcl
+iBGP
+iOAM
ia
iacl
-iAcl
-iACL
-iBGP
ibverb
icmp
ifndef
@@ -288,7 +930,6 @@
indicies
ing
init
-Init
inline
inlines
intel
@@ -296,29 +937,23 @@
interworkings
intfc
io
-iOAM
iommu
ip
-IP
+ipN
+ipX
ipaddress
iperf
-Iperf
ipfix
ipip
-ipN
ipsec
-IPsec
iptables
ipv
-IPv
-ipX
isc
isra
itf
ith
ixgbe
ixge
-Jira
jsmith
jsmithuser
json
@@ -329,24 +964,17 @@
keychain
kflows
kni
-KOC
-Konstantynowicz
ksr
kube
-Kube
kubeadm
-Kubeadm
kubectl
kubelet
-Kubelet
kubeproxy
kubernetes
-Kubernetes
kv
labelled
lacp
ld
-LDFLAGS
ldp
learnt
len
@@ -357,125 +985,92 @@
linting
linux
linuxapp
-Linuxcon
localhost
localsid
-Lockless
longjmp
loopback
-Loopback
lshw
lstack
lua
-Lua
-Luajit
lxc
-Maciek
+mFIB
+mIn
mactime
mainp
malloc
-Manohar
-Maryam
maximise
mb
mbedtls
mbit
mbufs
-Mbytes
mc
md
mem
memfd
memif
-Memif
metadata
-Metadata
mf
-mFIB
mheap
-mIn
mispositions
misprograms
mk
mldv
modelled
modernisation
-Monroy
mortem
mpls
-MPLS
mpps
-Mpps
msg
mtrie
mtu
multiarch
multicast
-Multicast
-Multihoming
multipath
multiqueue
-Multiqueue
-Multistream
-Multitenancy
mutex
-Mutlithreading
myapp
mygcc
myplugin
+nBackingPages
+nBytes
namespace
-Namespace
namespaces
nat
nats
natted
-nBackingPages
nbytes
-nBytes
ndrpdr
-Neighbour
netcfg
-Netmap
+netlink
+netmap
netplan
-Netplan
nginx
-NIC
nitems
nn
nnn
nodaemon
ns
nsim
-Nstrings
nullptr
num
numa
-NUMA
oacl
oam
-O'Driscoll
offline
ofr
online
ooo
-Openssl
openSUSE
-Optimisations
os
packagecloud
passthrough
passwordless
pcap
-Pcap
pci
-PCI
pe
perf
perfmon
-Perfmon
-Performan
perror
-Pfister
pfSense
phoneycomponent
physmem
@@ -484,15 +1079,10 @@
pifi
pktgen
plugin
-Plugin
plugins
-Plugins
pmalloc
-PoC
podSubnet
policer
-Policers
-Pollable
poweroff
pppoe
pre
@@ -513,51 +1103,42 @@
py
pypi
qemu
-Qemu
qos
-QoS
qs
qsz
quic
-Quic
quicly
rasa
rbtree
rc
rdma
+reStructured
+reStructuredText
realcomponent
realisation
realise
realsource
rebase
-Rebase
recognised
-Redhat
refactor
-Refactor
refactored
-Refactored
regex
reinjected
repo
repos
restack
-reStructured
-reStructuredText
rfc
rint
-RLOC
-Roadmap
-Roto
rpath
rsa
rst
rwlocks
rx
+sN
sbin
+scalable
scanf
scapy
-Scapy
sclass
scooby
screenshot
@@ -570,30 +1151,23 @@
sfr
shg
shm
-Shrikant
signalling
sixrd
sizeable
sl
-Slideshow
-sN
snapshoot
socketsvr
socksvr
-Specialisations
spinlocks
sr
src
-SRv
sshd
ssl
stateful
-Stateful
statseg
stdin
stl
stn
-Strongswan
struct
su
subfolder
@@ -603,26 +1177,17 @@
sudo
svm
sw
-Sw
sysctl
sysfs
syslog
-Syslog
systemd
-Systemd
sz
-Tahhan
-Tahr
tapcli
tcp
tcpcontrolbits
teardown
ted
-Templated
-Tenant
tep
-Terabit
-Terraform
threadLeave
threadplacement
tid
@@ -631,34 +1196,25 @@
tlsmbedtls
tlsopenssl
tmp
-Tollet
toolchain
toolchains
tos
tp
tpv
-TPv
-TRex
-Tsou
ttl
-Tunnelling
tuntap
twovppinstances
tx
txt
ubuntu
udp
-UDP
uid
uio
un
unformat
-Unformat
unicast
uninstall
-Uninstall
uninstalled
-Uninstalling
unix
unrouteable
unserialize
@@ -669,17 +1225,18 @@
upgradable
usec
usergroup
-Usergroup
username
userspace
-Userspace
usr
uu
+vHost
+vRouter
+vRouters
+vSwitch
+vSwitches
+vTEP
va
-Vagrantfile
-Vagrantfiles
vapi
-Vbox
vcl
ve
vec
@@ -690,85 +1247,74 @@
vfio
vft
vhost
-vHost
-Vhost
virsh
virtio
-Virtio
+virtualchnl
virtualenv
virtualization
-Virtualization
virtualized
vl
vlan
vlib
-Vlib
vlibapi
vlibmemory
vls
vm
-VM
-VMs
vmware
-VMware
vmx
vmxnet
-Vmxnet
vnet
-Vnet
+ap
vom
vpe
vpn
vpp
-Vpp
-VPP
vppapigen
vppapitrace
vppcapture
+bb
vppctl
vppinfra
-Vppinfra
vpppinfra
vpps
+crit
+eb
+ee
vpptrace
-VRF
+vrf
vrouter
-vRouter
+vrrp
+decrementing
vswitch
-vSwitch
-Vswitch
vswitches
-vTEP
vxlan
-Vxlan
walkthrough
-Warnicke
-Webinar
+dequeue
+dpd
+disp
webpage
+substring
+decrypted
+decrypt
+deregister
+superset
whitelist
whitelisting
whitelists
wiki
-Wiki
-Wikipedia
+ietf
wireshark
-Wireshark
-Wl
-Workstream
+xFFFFFFFF
+xTR
xchain
xconnect
-Xenial
-Xeon
-Xerus
-xFFFFFFFF
-Xtensa
-xTR
xxx
-XXX
xyz
yaml
-Yaml
yyy
-Zhao
zoomin
zoomout
+µs
+e.g.
+i.e.
+operationalize
\ No newline at end of file
diff --git a/docs/troubleshooting/reportingissues/index.rst b/docs/troubleshooting/reportingissues/index.rst
deleted file mode 100644
index 4d954ac..0000000
--- a/docs/troubleshooting/reportingissues/index.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-.. _reportingissues:
-
-How to Report an Issue
-======================
-
-.. toctree::
-
- reportingissues
diff --git a/docs/usecases/2_vpp.md b/docs/usecases/2_vpp.md
deleted file mode 100644
index d5f9281..0000000
--- a/docs/usecases/2_vpp.md
+++ /dev/null
@@ -1,128 +0,0 @@
-How to connect VPP instances using IKEv2
-========================================
-
-This section describes how to initiate IKEv2 session between two VPP instances
-using Linux veth interfaces and namespaces.
-
-
-Create veth interfaces and namespaces and configure it:
-
-```
-sudo ip link add ifresp type veth peer name ifinit
-sudo ip link set dev ifresp up
-sudo ip link set dev ifinit up
-
-sudo ip netns add clientns
-sudo ip netns add serverns
-sudo ip link add veth_client type veth peer name client
-sudo ip link add veth_server type veth peer name server
-sudo ip link set dev veth_client up netns clientns
-sudo ip link set dev veth_server up netns serverns
-
-sudo ip netns exec clientns \
- bash -c "
- ip link set dev lo up
- ip addr add 192.168.5.2/24 dev veth_client
- ip addr add fec5::2/16 dev veth_client
- ip route add 192.168.3.0/24 via 192.168.5.1
- ip route add fec3::0/16 via fec5::1
- "
-
-sudo ip netns exec serverns \
- bash -c "
- ip link set dev lo up
- ip addr add 192.168.3.2/24 dev veth_server
- ip addr add fec3::2/16 dev veth_server
- ip route add 192.168.5.0/24 via 192.168.3.1
- ip route add fec5::0/16 via fec3::1
- "
-```
-
-Run responder VPP:
-
-```
-sudo /usr/bin/vpp unix { \
- cli-listen /tmp/vpp_resp.sock \
- gid $(id -g) } \
- api-segment { prefix vpp } \
- plugins { plugin dpdk_plugin.so { disable } }
-```
-
-Configure the responder
-
-
-```
-create host-interface name ifresp
-set interface ip addr host-ifresp 192.168.10.2/24
-set interface state host-ifresp up
-
-create host-interface name server
-set interface ip addr host-server 192.168.3.1/24
-set interface state host-server up
-
-ikev2 profile add pr1
-ikev2 profile set pr1 auth shared-key-mic string Vpp123
-ikev2 profile set pr1 id local ipv4 192.168.10.2
-ikev2 profile set pr1 id remote ipv4 192.168.10.1
-
-ikev2 profile set pr1 traffic-selector local ip-range 192.168.3.0 - 192.168.3.255 port-range 0 - 65535 protocol 0
-ikev2 profile set pr1 traffic-selector remote ip-range 192.168.5.0 - 192.168.5.255 port-range 0 - 65535 protocol 0
-
-create ipip tunnel src 192.168.10.2 dst 192.168.10.1
-ikev2 profile set pr1 tunnel ipip0
-ip route add 192.168.5.0/24 via 192.168.10.1 ipip0
-set interface unnumbered ipip0 use host-ifresp
-```
-
-Run initiator VPP:
-
-```
-sudo /usr/bin/vpp unix { \
- cli-listen /tmp/vpp_init.sock \
- gid $(id -g) } \
- api-segment { prefix vpp } \
- plugins { plugin dpdk_plugin.so { disable } }
-```
-
-Configure initiator:
-```
-create host-interface name ifinit
-set interface ip addr host-ifinit 192.168.10.1/24
-set interface state host-ifinit up
-
-create host-interface name client
-set interface ip addr host-client 192.168.5.1/24
-set interface state host-client up
-
-ikev2 profile add pr1
-ikev2 profile set pr1 auth shared-key-mic string Vpp123
-ikev2 profile set pr1 id local ipv4 192.168.10.1
-ikev2 profile set pr1 id remote ipv4 192.168.10.2
-
-ikev2 profile set pr1 traffic-selector remote ip-range 192.168.3.0 - 192.168.3.255 port-range 0 - 65535 protocol 0
-ikev2 profile set pr1 traffic-selector local ip-range 192.168.5.0 - 192.168.5.255 port-range 0 - 65535 protocol 0
-
-ikev2 profile set pr1 responder host-ifinit 192.168.10.2
-ikev2 profile set pr1 ike-crypto-alg aes-gcm-16 256 ike-dh modp-2048
-ikev2 profile set pr1 esp-crypto-alg aes-gcm-16 256
-
-create ipip tunnel src 192.168.10.1 dst 192.168.10.2
-ikev2 profile set pr1 tunnel ipip0
-ip route add 192.168.3.0/24 via 192.168.10.2 ipip0
-set interface unnumbered ipip0 use host-ifinit
-```
-
-Initiate the IKEv2 connection:
-
-```
-vpp# ikev2 initiate sa-init pr1
-```
-
-Responder's and initiator's private networks are now connected with IPSEC tunnel:
-
-```
-$ sudo ip netns exec clientns ping 192.168.3.1
-PING 192.168.3.1 (192.168.3.1) 56(84) bytes of data.
-64 bytes from 192.168.3.1: icmp_seq=1 ttl=63 time=1.64 ms
-64 bytes from 192.168.3.1: icmp_seq=2 ttl=63 time=7.24 ms
-```
diff --git a/docs/usecases/acls.rst b/docs/usecases/acls.rst
index 0350af2..2dcb1f3 100644
--- a/docs/usecases/acls.rst
+++ b/docs/usecases/acls.rst
@@ -1,7 +1,7 @@
.. _aclwithvpp:
-Access Control Lists (ACLs) with FD.io VPP
-==========================================
+Access Control Lists with VPP
+=============================
This section is overview of the options available to implement ACLs in
FD.io VPP. As there are a number of way's to address ACL-like functionality,
@@ -106,36 +106,36 @@
.. code-block:: console
- DUT1:
- Thread 0 vpp_main (lcore 1)
- Time 3.8, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
- vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- acl-plugin-fa-cleaner-process any wait 0 0 14 1.29e3 0.00
- acl-plugin-fa-worker-cleaner-pinterrupt wa 7 0 0 9.18e2 0.00
- api-rx-from-ring active 0 0 52 8.96e4 0.00
- dpdk-process any wait 0 0 1 1.35e4 0.00
- fib-walk any wait 0 0 2 2.69e3 0.00
- ip6-icmp-neighbor-discovery-ev any wait 0 0 4 1.32e3 0.00
- lisp-retry-service any wait 0 0 2 2.90e3 0.00
- unix-epoll-input polling 7037 0 0 1.25e6 0.00
- vpe-oam-process any wait 0 0 2 2.28e3 0.00
+ DUT1:
+ Thread 0 vpp_main (lcore 1)
+ Time 3.8, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
+ vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ acl-plugin-fa-cleaner-process any wait 0 0 14 1.29e3 0.00
+ acl-plugin-fa-worker-cleaner-pinterrupt wa 7 0 0 9.18e2 0.00
+ api-rx-from-ring active 0 0 52 8.96e4 0.00
+ dpdk-process any wait 0 0 1 1.35e4 0.00
+ fib-walk any wait 0 0 2 2.69e3 0.00
+ ip6-icmp-neighbor-discovery-ev any wait 0 0 4 1.32e3 0.00
+ lisp-retry-service any wait 0 0 2 2.90e3 0.00
+ unix-epoll-input polling 7037 0 0 1.25e6 0.00
+ vpe-oam-process any wait 0 0 2 2.28e3 0.00
- Thread 1 vpp_wk_0 (lcore 2)
- Time 3.8, average vectors/node 249.02, last 128 main loops 32.00 per node 273.07
- vector rates in 6.1118e6, out 6.1118e6, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- TenGigabitEtherneta/0/0-output active 47106 11721472 0 9.47e0 248.83
- TenGigabitEtherneta/0/0-tx active 47106 11721472 0 4.22e1 248.83
- TenGigabitEtherneta/0/1-output active 47106 11721472 0 1.02e1 248.83
- TenGigabitEtherneta/0/1-tx active 47106 11721472 0 4.18e1 248.83
- acl-plugin-fa-worker-cleaner-pinterrupt wa 7 0 0 1.39e3 0.00
- acl-plugin-in-ip4-fa active 94107 23442944 0 1.75e2 249.11
- dpdk-input polling 47106 23442944 0 4.64e1 497.66
- ethernet-input active 94212 23442944 0 1.55e1 248.83
- ip4-input-no-checksum active 94107 23442944 0 3.23e1 249.11
- ip4-lookup active 94107 23442944 0 2.91e1 249.11
- ip4-rewrite active 94107 23442944 0 2.48e1 249.11
+ Thread 1 vpp_wk_0 (lcore 2)
+ Time 3.8, average vectors/node 249.02, last 128 main loops 32.00 per node 273.07
+ vector rates in 6.1118e6, out 6.1118e6, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ TenGigabitEtherneta/0/0-output active 47106 11721472 0 9.47e0 248.83
+ TenGigabitEtherneta/0/0-tx active 47106 11721472 0 4.22e1 248.83
+ TenGigabitEtherneta/0/1-output active 47106 11721472 0 1.02e1 248.83
+ TenGigabitEtherneta/0/1-tx active 47106 11721472 0 4.18e1 248.83
+ acl-plugin-fa-worker-cleaner-pinterrupt wa 7 0 0 1.39e3 0.00
+ acl-plugin-in-ip4-fa active 94107 23442944 0 1.75e2 249.11
+ dpdk-input polling 47106 23442944 0 4.64e1 497.66
+ ethernet-input active 94212 23442944 0 1.55e1 248.83
+ ip4-input-no-checksum active 94107 23442944 0 3.23e1 249.11
+ ip4-lookup active 94107 23442944 0 2.91e1 249.11
+ ip4-rewrite active 94107 23442944 0 2.48e1 249.11
unix-epoll-input polling 46 0 0 1.54e3 0.00
Input/Stateful
@@ -146,41 +146,41 @@
.. code-block:: console
- DUT1:
- Thread 0 vpp_main (lcore 1)
- Time 3.9, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
- vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- acl-plugin-fa-cleaner-process any wait 0 0 16 1.40e3 0.00
- acl-plugin-fa-worker-cleaner-pinterrupt wa 8 0 0 8.97e2 0.00
- api-rx-from-ring active 0 0 52 7.12e4 0.00
- dpdk-process any wait 0 0 1 1.69e4 0.00
- fib-walk any wait 0 0 2 2.55e3 0.00
- ip4-reassembly-expire-walk any wait 0 0 1 1.27e4 0.00
- ip6-icmp-neighbor-discovery-ev any wait 0 0 4 1.09e3 0.00
- ip6-reassembly-expire-walk any wait 0 0 1 2.57e3 0.00
- lisp-retry-service any wait 0 0 2 1.18e4 0.00
- statseg-collector-process time wait 0 0 1 6.38e3 0.00
- unix-epoll-input polling 6320 0 0 1.41e6 0.00
- vpe-oam-process any wait 0 0 2 7.53e3 0.00
+ DUT1:
+ Thread 0 vpp_main (lcore 1)
+ Time 3.9, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
+ vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ acl-plugin-fa-cleaner-process any wait 0 0 16 1.40e3 0.00
+ acl-plugin-fa-worker-cleaner-pinterrupt wa 8 0 0 8.97e2 0.00
+ api-rx-from-ring active 0 0 52 7.12e4 0.00
+ dpdk-process any wait 0 0 1 1.69e4 0.00
+ fib-walk any wait 0 0 2 2.55e3 0.00
+ ip4-reassembly-expire-walk any wait 0 0 1 1.27e4 0.00
+ ip6-icmp-neighbor-discovery-ev any wait 0 0 4 1.09e3 0.00
+ ip6-reassembly-expire-walk any wait 0 0 1 2.57e3 0.00
+ lisp-retry-service any wait 0 0 2 1.18e4 0.00
+ statseg-collector-process time wait 0 0 1 6.38e3 0.00
+ unix-epoll-input polling 6320 0 0 1.41e6 0.00
+ vpe-oam-process any wait 0 0 2 7.53e3 0.00
- Thread 1 vpp_wk_0 (lcore 2)
- Time 3.9, average vectors/node 252.74, last 128 main loops 32.00 per node 273.07
- vector rates in 7.5833e6, out 7.5833e6, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- TenGigabitEtherneta/0/0-output active 58325 14738944 0 9.41e0 252.70
- TenGigabitEtherneta/0/0-tx active 58325 14738944 0 4.32e1 252.70
- TenGigabitEtherneta/0/1-output active 58323 14738944 0 1.02e1 252.71
- TenGigabitEtherneta/0/1-tx active 58323 14738944 0 4.31e1 252.71
- acl-plugin-fa-worker-cleaner-pinterrupt wa 8 0 0 1.62e3 0.00
- acl-plugin-in-ip4-fa active 116628 29477888 0 1.01e2 252.75
- dpdk-input polling 58325 29477888 0 4.63e1 505.41
- ethernet-input active 116648 29477888 0 1.53e1 252.71
- ip4-input-no-checksum active 116628 29477888 0 3.21e1 252.75
- ip4-lookup active 116628 29477888 0 2.90e1 252.75
- ip4-rewrite active 116628 29477888 0 2.48e1 252.75
- unix-epoll-input polling 57 0 0 2.39e3 0.00
-
+ Thread 1 vpp_wk_0 (lcore 2)
+ Time 3.9, average vectors/node 252.74, last 128 main loops 32.00 per node 273.07
+ vector rates in 7.5833e6, out 7.5833e6, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ TenGigabitEtherneta/0/0-output active 58325 14738944 0 9.41e0 252.70
+ TenGigabitEtherneta/0/0-tx active 58325 14738944 0 4.32e1 252.70
+ TenGigabitEtherneta/0/1-output active 58323 14738944 0 1.02e1 252.71
+ TenGigabitEtherneta/0/1-tx active 58323 14738944 0 4.31e1 252.71
+ acl-plugin-fa-worker-cleaner-pinterrupt wa 8 0 0 1.62e3 0.00
+ acl-plugin-in-ip4-fa active 116628 29477888 0 1.01e2 252.75
+ dpdk-input polling 58325 29477888 0 4.63e1 505.41
+ ethernet-input active 116648 29477888 0 1.53e1 252.71
+ ip4-input-no-checksum active 116628 29477888 0 3.21e1 252.75
+ ip4-lookup active 116628 29477888 0 2.90e1 252.75
+ ip4-rewrite active 116628 29477888 0 2.48e1 252.75
+ unix-epoll-input polling 57 0 0 2.39e3 0.00
+
Output/Stateless
~~~~~~~~~~~~~~~~
@@ -189,39 +189,39 @@
.. code-block:: console
- DUT1:
- Thread 0 vpp_main (lcore 1)
- Time 3.8, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
- vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- acl-plugin-fa-cleaner-process any wait 0 0 14 1.43e3 0.00
- acl-plugin-fa-worker-cleaner-pinterrupt wa 7 0 0 9.23e2 0.00
- api-rx-from-ring active 0 0 52 8.01e4 0.00
- dpdk-process any wait 0 0 1 1.59e6 0.00
- fib-walk any wait 0 0 2 6.81e3 0.00
- ip6-icmp-neighbor-discovery-ev any wait 0 0 4 2.81e3 0.00
- lisp-retry-service any wait 0 0 2 3.64e3 0.00
- unix-epoll-input polling 4842 0 0 1.81e6 0.00
- vpe-oam-process any wait 0 0 1 2.24e4 0.00
-
- Thread 1 vpp_wk_0 (lcore 2)
- Time 3.8, average vectors/node 249.29, last 128 main loops 36.00 per node 271.06
- vector rates in 5.9196e6, out 5.9196e6, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- TenGigabitEtherneta/0/0-output active 45595 11363584 0 9.22e0 249.23
- TenGigabitEtherneta/0/0-tx active 45595 11363584 0 4.25e1 249.23
- TenGigabitEtherneta/0/1-output active 45594 11363584 0 9.75e0 249.23
- TenGigabitEtherneta/0/1-tx active 45594 11363584 0 4.21e1 249.23
- acl-plugin-fa-worker-cleaner-pinterrupt wa 7 0 0 1.28e3 0.00
- acl-plugin-out-ip4-fa active 91155 22727168 0 1.78e2 249.32
- dpdk-input polling 45595 22727168 0 4.64e1 498.46
- ethernet-input active 91189 22727168 0 1.56e1 249.23
- interface-output active 91155 22727168 0 1.13e1 249.32
- ip4-input-no-checksum active 91155 22727168 0 1.95e1 249.32
- ip4-lookup active 91155 22727168 0 2.88e1 249.32
- ip4-rewrite active 91155 22727168 0 3.53e1 249.32
- unix-epoll-input polling 44 0 0 1.53e3 0.00
-
+ DUT1:
+ Thread 0 vpp_main (lcore 1)
+ Time 3.8, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
+ vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ acl-plugin-fa-cleaner-process any wait 0 0 14 1.43e3 0.00
+ acl-plugin-fa-worker-cleaner-pinterrupt wa 7 0 0 9.23e2 0.00
+ api-rx-from-ring active 0 0 52 8.01e4 0.00
+ dpdk-process any wait 0 0 1 1.59e6 0.00
+ fib-walk any wait 0 0 2 6.81e3 0.00
+ ip6-icmp-neighbor-discovery-ev any wait 0 0 4 2.81e3 0.00
+ lisp-retry-service any wait 0 0 2 3.64e3 0.00
+ unix-epoll-input polling 4842 0 0 1.81e6 0.00
+ vpe-oam-process any wait 0 0 1 2.24e4 0.00
+
+ Thread 1 vpp_wk_0 (lcore 2)
+ Time 3.8, average vectors/node 249.29, last 128 main loops 36.00 per node 271.06
+ vector rates in 5.9196e6, out 5.9196e6, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ TenGigabitEtherneta/0/0-output active 45595 11363584 0 9.22e0 249.23
+ TenGigabitEtherneta/0/0-tx active 45595 11363584 0 4.25e1 249.23
+ TenGigabitEtherneta/0/1-output active 45594 11363584 0 9.75e0 249.23
+ TenGigabitEtherneta/0/1-tx active 45594 11363584 0 4.21e1 249.23
+ acl-plugin-fa-worker-cleaner-pinterrupt wa 7 0 0 1.28e3 0.00
+ acl-plugin-out-ip4-fa active 91155 22727168 0 1.78e2 249.32
+ dpdk-input polling 45595 22727168 0 4.64e1 498.46
+ ethernet-input active 91189 22727168 0 1.56e1 249.23
+ interface-output active 91155 22727168 0 1.13e1 249.32
+ ip4-input-no-checksum active 91155 22727168 0 1.95e1 249.32
+ ip4-lookup active 91155 22727168 0 2.88e1 249.32
+ ip4-rewrite active 91155 22727168 0 3.53e1 249.32
+ unix-epoll-input polling 44 0 0 1.53e3 0.00
+
Output/Stateful
~~~~~~~~~~~~~~~
@@ -230,42 +230,42 @@
.. code-block:: console
- DUT1:
- Thread 0 vpp_main (lcore 1)
- Time 3.8, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
- vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- acl-plugin-fa-cleaner-process any wait 0 0 16 1.47e3 0.00
- acl-plugin-fa-worker-cleaner-pinterrupt wa 8 0 0 8.51e2 0.00
- api-rx-from-ring active 0 0 50 7.24e4 0.00
- dpdk-process any wait 0 0 2 1.93e4 0.00
- fib-walk any wait 0 0 2 2.02e3 0.00
- ip4-reassembly-expire-walk any wait 0 0 1 3.96e3 0.00
- ip6-icmp-neighbor-discovery-ev any wait 0 0 4 9.84e2 0.00
- ip6-reassembly-expire-walk any wait 0 0 1 3.76e3 0.00
- lisp-retry-service any wait 0 0 2 1.49e4 0.00
- statseg-collector-process time wait 0 0 1 4.98e3 0.00
- unix-epoll-input polling 5653 0 0 1.55e6 0.00
- vpe-oam-process any wait 0 0 2 1.90e3 0.00
-
- Thread 1 vpp_wk_0 (lcore 2)
- Time 3.8, average vectors/node 250.85, last 128 main loops 36.00 per node 271.06
- vector rates in 7.2686e6, out 7.2686e6, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- TenGigabitEtherneta/0/0-output active 55639 13930752 0 9.33e0 250.38
- TenGigabitEtherneta/0/0-tx active 55639 13930752 0 4.27e1 250.38
- TenGigabitEtherneta/0/1-output active 55636 13930758 0 9.81e0 250.39
- TenGigabitEtherneta/0/1-tx active 55636 13930758 0 4.33e1 250.39
- acl-plugin-fa-worker-cleaner-pinterrupt wa 8 0 0 1.62e3 0.00
- acl-plugin-out-ip4-fa active 110988 27861510 0 1.04e2 251.03
- dpdk-input polling 55639 27861510 0 4.62e1 500.76
- ethernet-input active 111275 27861510 0 1.55e1 250.38
- interface-output active 110988 27861510 0 1.21e1 251.03
- ip4-input-no-checksum active 110988 27861510 0 1.95e1 251.03
- ip4-lookup active 110988 27861510 0 2.89e1 251.03
- ip4-rewrite active 110988 27861510 0 3.55e1 251.03
- unix-epoll-input polling 54 0 0 2.43e3 0.00
-
+ DUT1:
+ Thread 0 vpp_main (lcore 1)
+ Time 3.8, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
+ vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ acl-plugin-fa-cleaner-process any wait 0 0 16 1.47e3 0.00
+ acl-plugin-fa-worker-cleaner-pinterrupt wa 8 0 0 8.51e2 0.00
+ api-rx-from-ring active 0 0 50 7.24e4 0.00
+ dpdk-process any wait 0 0 2 1.93e4 0.00
+ fib-walk any wait 0 0 2 2.02e3 0.00
+ ip4-reassembly-expire-walk any wait 0 0 1 3.96e3 0.00
+ ip6-icmp-neighbor-discovery-ev any wait 0 0 4 9.84e2 0.00
+ ip6-reassembly-expire-walk any wait 0 0 1 3.76e3 0.00
+ lisp-retry-service any wait 0 0 2 1.49e4 0.00
+ statseg-collector-process time wait 0 0 1 4.98e3 0.00
+ unix-epoll-input polling 5653 0 0 1.55e6 0.00
+ vpe-oam-process any wait 0 0 2 1.90e3 0.00
+
+ Thread 1 vpp_wk_0 (lcore 2)
+ Time 3.8, average vectors/node 250.85, last 128 main loops 36.00 per node 271.06
+ vector rates in 7.2686e6, out 7.2686e6, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ TenGigabitEtherneta/0/0-output active 55639 13930752 0 9.33e0 250.38
+ TenGigabitEtherneta/0/0-tx active 55639 13930752 0 4.27e1 250.38
+ TenGigabitEtherneta/0/1-output active 55636 13930758 0 9.81e0 250.39
+ TenGigabitEtherneta/0/1-tx active 55636 13930758 0 4.33e1 250.39
+ acl-plugin-fa-worker-cleaner-pinterrupt wa 8 0 0 1.62e3 0.00
+ acl-plugin-out-ip4-fa active 110988 27861510 0 1.04e2 251.03
+ dpdk-input polling 55639 27861510 0 4.62e1 500.76
+ ethernet-input active 111275 27861510 0 1.55e1 250.38
+ interface-output active 110988 27861510 0 1.21e1 251.03
+ ip4-input-no-checksum active 110988 27861510 0 1.95e1 251.03
+ ip4-lookup active 110988 27861510 0 2.89e1 251.03
+ ip4-rewrite active 110988 27861510 0 3.55e1 251.03
+ unix-epoll-input polling 54 0 0 2.43e3 0.00
+
Performance
-----------
@@ -297,21 +297,21 @@
.. code-block:: console
- $ sudo vppctl ip_add_del_route 20.20.20.0/24 via 1.1.1.2 sw_if_index 1 resolve-attempts 10 count 1
- $ sudo vppctl acl_add_replace ipv4 permit src 30.30.30.1/32 dst 40.40.40.1/32 sport 1000 dport 1000, ipv4 permit+reflect src 10.10.10.0/24, ipv4 permit+reflect src 20.20.20.0/24
- $ sudo vppctl acl_interface_set_acl_list sw_if_index 2 input 0
- $ sudo vppctl acl_interface_set_acl_list sw_if_index 1 input 0
-
+ $ sudo vppctl ip_add_del_route 20.20.20.0/24 via 1.1.1.2 sw_if_index 1 resolve-attempts 10 count 1
+ $ sudo vppctl acl_add_replace ipv4 permit src 30.30.30.1/32 dst 40.40.40.1/32 sport 1000 dport 1000, ipv4 permit+reflect src 10.10.10.0/24, ipv4 permit+reflect src 20.20.20.0/24
+ $ sudo vppctl acl_interface_set_acl_list sw_if_index 2 input 0
+ $ sudo vppctl acl_interface_set_acl_list sw_if_index 1 input 0
+
Stateless
~~~~~~~~~
.. code-block:: console
- $ sudo vppctl ip_add_del_route 20.20.20.0/24 via 1.1.1.2 sw_if_index 1 resolve-attempts 10 count 1
- $ sudo vppctl acl_add_replace ipv4 permit src 30.30.30.1/32 dst 40.40.40.1/32 sport 1000 dport 1000, ipv4 permit src 10.10.10.0/24, ipv4 permit src 20.20.20.0/24
- $ sudo vppctl acl_interface_set_acl_list sw_if_index 2 input 0
+ $ sudo vppctl ip_add_del_route 20.20.20.0/24 via 1.1.1.2 sw_if_index 1 resolve-attempts 10 count 1
+ $ sudo vppctl acl_add_replace ipv4 permit src 30.30.30.1/32 dst 40.40.40.1/32 sport 1000 dport 1000, ipv4 permit src 10.10.10.0/24, ipv4 permit src 20.20.20.0/24
+ $ sudo vppctl acl_interface_set_acl_list sw_if_index 2 input 0
$ sudo vppctl acl_interface_set_acl_list sw_if_index 1 input 0
-
+
Links
~~~~~
@@ -348,40 +348,40 @@
.. code-block:: console
- DUT1:
- Thread 0 vpp_main (lcore 1)
- Time 3.9, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
- vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- api-rx-from-ring active 0 0 53 4.20e4 0.00
- dpdk-process any wait 0 0 1 1.75e4 0.00
- fib-walk any wait 0 0 2 1.59e3 0.00
- ip4-reassembly-expire-walk any wait 0 0 1 2.20e3 0.00
- ip6-icmp-neighbor-discovery-ev any wait 0 0 4 1.14e3 0.00
- ip6-reassembly-expire-walk any wait 0 0 1 1.50e3 0.00
- lisp-retry-service any wait 0 0 2 2.19e3 0.00
- statseg-collector-process time wait 0 0 1 2.48e3 0.00
- unix-epoll-input polling 2800 0 0 3.15e6 0.00
- vpe-oam-process any wait 0 0 2 7.00e2 0.00
+ DUT1:
+ Thread 0 vpp_main (lcore 1)
+ Time 3.9, average vectors/node 0.00, last 128 main loops 0.00 per node 0.00
+ vector rates in 0.0000e0, out 0.0000e0, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ api-rx-from-ring active 0 0 53 4.20e4 0.00
+ dpdk-process any wait 0 0 1 1.75e4 0.00
+ fib-walk any wait 0 0 2 1.59e3 0.00
+ ip4-reassembly-expire-walk any wait 0 0 1 2.20e3 0.00
+ ip6-icmp-neighbor-discovery-ev any wait 0 0 4 1.14e3 0.00
+ ip6-reassembly-expire-walk any wait 0 0 1 1.50e3 0.00
+ lisp-retry-service any wait 0 0 2 2.19e3 0.00
+ statseg-collector-process time wait 0 0 1 2.48e3 0.00
+ unix-epoll-input polling 2800 0 0 3.15e6 0.00
+ vpe-oam-process any wait 0 0 2 7.00e2 0.00
- Thread 1 vpp_wk_0 (lcore 2)
- Time 3.9, average vectors/node 220.84, last 128 main loops 20.87 per node 190.86
- vector rates in 1.0724e7, out 1.0724e7, drop 0.0000e0, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
- TenGigabitEtherneta/0/0-output active 94960 20698112 0 1.03e1 217.97
- TenGigabitEtherneta/0/0-tx active 94960 20698112 0 3.97e1 217.97
- TenGigabitEtherneta/0/1-output active 92238 20698112 0 9.92e0 224.39
- TenGigabitEtherneta/0/1-tx active 92238 20698112 0 4.26e1 224.39
- cop-input active 94960 20698112 0 1.98e1 217.97
- dpdk-input polling 95154 41396224 0 4.58e1 435.04
- ethernet-input active 92238 20698112 0 1.59e1 224.39
- ip4-cop-whitelist active 94960 20698112 0 3.24e1 217.97
- ip4-input active 94960 20698112 0 3.13e1 217.97
- ip4-input-no-checksum active 92238 20698112 0 2.23e1 224.39
- ip4-lookup active 187198 41396224 0 3.08e1 221.14
- ip4-rewrite active 187198 41396224 0 2.47e1 221.14
- unix-epoll-input polling 93 0 0 1.35e3 0.00
-
+ Thread 1 vpp_wk_0 (lcore 2)
+ Time 3.9, average vectors/node 220.84, last 128 main loops 20.87 per node 190.86
+ vector rates in 1.0724e7, out 1.0724e7, drop 0.0000e0, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ TenGigabitEtherneta/0/0-output active 94960 20698112 0 1.03e1 217.97
+ TenGigabitEtherneta/0/0-tx active 94960 20698112 0 3.97e1 217.97
+ TenGigabitEtherneta/0/1-output active 92238 20698112 0 9.92e0 224.39
+ TenGigabitEtherneta/0/1-tx active 92238 20698112 0 4.26e1 224.39
+ cop-input active 94960 20698112 0 1.98e1 217.97
+ dpdk-input polling 95154 41396224 0 4.58e1 435.04
+ ethernet-input active 92238 20698112 0 1.59e1 224.39
+ ip4-cop-whitelist active 94960 20698112 0 3.24e1 217.97
+ ip4-input active 94960 20698112 0 3.13e1 217.97
+ ip4-input-no-checksum active 92238 20698112 0 2.23e1 224.39
+ ip4-lookup active 187198 41396224 0 3.08e1 221.14
+ ip4-rewrite active 187198 41396224 0 2.47e1 221.14
+ unix-epoll-input polling 93 0 0 1.35e3 0.00
+
Performance
~~~~~~~~~~~
@@ -403,12 +403,12 @@
.. code-block:: console
- $ sudo vppctl ip_add_del_route 10.10.10.0/24 via 1.1.1.1 sw_if_index 2 resolve-attempts 10 count 1
- $ sudo vppctl ip_table_add_del table 1
- $ sudo vppctl ip_add_del_route 20.20.20.0/24 vrf 1 resolve-attempts 10 count 1 local
- $ sudo vppctl cop_whitelist_enable_disable sw_if_index 1 ip4 fib-id 1
- $ sudo vppctl cop_interface_enable_disable sw_if_index 1
-
+ $ sudo vppctl ip_add_del_route 10.10.10.0/24 via 1.1.1.1 sw_if_index 2 resolve-attempts 10 count 1
+ $ sudo vppctl ip_table_add_del table 1
+ $ sudo vppctl ip_add_del_route 20.20.20.0/24 vrf 1 resolve-attempts 10 count 1 local
+ $ sudo vppctl cop_whitelist_enable_disable sw_if_index 1 ip4 fib-id 1
+ $ sudo vppctl cop_interface_enable_disable sw_if_index 1
+
Links
~~~~~
@@ -478,7 +478,7 @@
$ sudo vppctl classify table mask l3 ip6 dst buckets 64
$ sudo vppctl classify session hit-next 0 table-index 0 match l3 ip6 dst 2001:db8:1::2 opaque-index 42
$ sudo vppctl set interface l2 input classify intfc host-s0_s1 ip6-table 0
-
+
Links
~~~~~
diff --git a/docs/usecases/container_test.md b/docs/usecases/container_test.md
deleted file mode 100644
index ad0bc2e..0000000
--- a/docs/usecases/container_test.md
+++ /dev/null
@@ -1,640 +0,0 @@
-Container-based network simulation
-==================================
-
-The "make test" framework provides a good way to test individual
-features. However, when testing several features at once - or
-validating nontrivial configurations - it may prove difficult or
-impossible to use the unit-test framework.
-
-This note explains how to set up lxc/lxd, and a 5-container testbed to
-test a split-tunnel nat + ikev2 + ipsec + ipv6 prefix-delegation
-scenario.
-
-OS / Distro test results
-------------------------
-
-This setup has been tested on an Ubuntu 18.04 LTS system. If you're
-feeling adventurous, the same scenario also worked on a recent Ubuntu
-20.04 "preview" daily build.
-
-Other distros may work fine, or not at all.
-
-Proxy Server
-------------
-
-If you need to use a proxy server e.g. from a lab system, you'll
-probably need to set HTTP_PROXY, HTTPS_PROXY, http_proxy and
-https_proxy in /etc/environment. Directly setting variables in the
-environment doesn't work. The lxd snap _daemon_ needs the proxy settings,
-not the user interface.
-
-Something like so:
-
-```
- HTTP_PROXY=http://my.proxy.server:8080
- HTTPS_PROXY=http://my.proxy.server:4333
- http_proxy=http://my.proxy.server:8080
- https_proxy=http://my.proxy.server:4333
-```
-
-Install and configure lxd
--------------------------
-
-Install the lxd snap. The lxd snap is up to date, as opposed to the
-results of "sudo apt-get install lxd".
-
-```
- # snap install lxd
- # lxd init
-```
-
-"lxd init" asks several questions. With the exception of the storage
-pool, take the defaults. To match the configs shown below, create a
-storage pool named "vpp." Storage pools of type "zfs" and "files" have
-been tested successfully.
-
-zfs is more space-efficient. "lxc copy" is infinitely faster with
-zfs. The path for the zfs storage pool is under /var. Do not replace
-it with a symbolic link, unless you want to rebuild all of your
-containers from scratch. Ask me how I know that.
-
-Create three network segments
------------------------------
-
-Aka, linux bridges.
-
-```
- # lxc network create respond
- # lxc network create internet
- # lxc network create initiate
-```
-
-We'll explain the test topology in a bit. Stay tuned.
-
-Set up the default container profile
-------------------------------------
-
-Execute "lxc profile edit default", and install the following
-configuration. Note that the "shared" directory should mount your vpp
-workspaces. With that trick, you can edit code from any of the
-containers, run vpp without installing it, etc.
-
-```
- config: {}
- description: Default LXD profile
- devices:
- eth0:
- name: eth0
- network: lxdbr0
- type: nic
- eth1:
- name: eth1
- nictype: bridged
- parent: internet
- type: nic
- eth2:
- name: eth2
- nictype: bridged
- parent: respond
- type: nic
- eth3:
- name: eth3
- nictype: bridged
- parent: initiate
- type: nic
- root:
- path: /
- pool: vpp
- type: disk
- shared:
- path: /scratch
- source: /scratch
- type: disk
- name: default
-```
-
-Set up the network configurations
----------------------------------
-
-Edit the fake "internet" backbone:
-
-```
- # lxc network edit internet
-```
-
-Install the ip addresses shown below, to avoid having to rebuild the vpp
-and host configuration:
-
-```
- config:
- ipv4.address: 10.26.68.1/24
- ipv4.dhcp.ranges: 10.26.68.10-10.26.68.50
- ipv4.nat: "true"
- ipv6.address: none
- ipv6.nat: "false"
- description: ""
- name: internet
- type: bridge
- used_by:
- managed: true
- status: Created
- locations:
- - none
-```
-
-Repeat the process with the "respond" and "initiate" networks, using these
-configurations:
-
-### respond network configuration
-
-```
- config:
- ipv4.address: 10.166.14.1/24
- ipv4.dhcp.ranges: 10.166.14.10-10.166.14.50
- ipv4.nat: "true"
- ipv6.address: none
- ipv6.nat: "false"
- description: ""
- name: respond
- type: bridge
- used_by:
- managed: true
- status: Created
- locations:
- - none
-```
-### initiate network configuration
-
-```
- config:
- ipv4.address: 10.219.188.1/24
- ipv4.dhcp.ranges: 10.219.188.10-10.219.188.50
- ipv4.nat: "true"
- ipv6.address: none
- ipv6.nat: "false"
- description: ""
- name: initiate
- type: bridge
- used_by:
- managed: true
- status: Created
- locations:
- - none
-```
-
-Create a "master" container image
----------------------------------
-
-The master container image should be set up so that you can
-build vpp, ssh into the container, edit source code, run gdb, etc.
-
-Make sure that e.g. public key auth ssh works.
-
-```
- # lxd launch ubuntu:18.04 respond
- <spew>
- # lxc exec respond bash
- respond# cd /scratch/my-vpp-workspace
- respond# apt-get install make ssh
- respond# make install-dep
- respond# exit
- # lxc stop respond
-```
-
-Mark the container image privileged. If you forget this step, you'll
-trip over a netlink error (-11) aka EAGAIN when you try to roll in the
-vpp configurations.
-
-```
- # lxc config set respond security.privileged "true"
-```
-
-Duplicate the "master" container image
---------------------------------------
-
-To avoid having to configure N containers, be sure that the master
-container image is fully set up before you help it have children:
-
-```
- # lxc copy respond respondhost
- # lxc copy respond initiate
- # lxc copy respond initiatehost
- # lxc copy respond dhcpserver # optional, to test ipv6 prefix delegation
-```
-
-Install handy script
---------------------
-
-See below for a handly script which executes lxc commands across the
-current set of running containers. I call it "lxc-foreach," feel free
-to call the script Ishmael if you like.
-
-Examples:
-
-```
- $ lxc-foreach start
- <issues "lxc start" for each container in the list>
-```
-
-After a few seconds, use this one to open an ssh connection to each
-container. The ssh command parses the output of "lxc info," which
-displays container ip addresses.
-
-```
- $ lxc-foreach ssh
-```
-
-Here's the script:
-
-```
- #!/bin/bash
-
- set -u
- export containers="respond respondhost initiate initiatehost dhcpserver"
-
- if [ x$1 = "x" ] ; then
- echo missing command
- exit 1
- fi
-
- if [ $1 = "ssh" ] ; then
- for c in $containers
- do
- inet=`lxc info $c | grep eth0 | grep -v inet6 | head -1 | cut -f 3`
- if [ x$inet = "x" ] ; then
- echo $c not started
- else
- gnome-terminal --command "/usr/bin/ssh $inet"
- fi
- done
- exit 0
- fi
-
- for c in $containers
- do
- echo lxc $1 $c
- lxc $1 $c
- done
-
- exit 0
-```
-
-Test topology
--------------
-
-Finally, we're ready to describe a test topology. First, a picture:
-
-```
- ===+======== management lan/bridge lxdbr0 (dhcp) ===========+===
- | | |
- | | |
- | | |
- v | v
- eth0 | eth0
- +------+ eth1 eth1 +------+
- | respond | 10.26.88.100 <= internet bridge => 10.26.88.101 | initiate |
- +------+ +------+
- eth2 / bvi0 10.166.14.2 | 10.219.188.2 eth3 / bvi0
- | | |
- | ("respond" bridge) | ("initiate" bridge) |
- | | |
- v | v
- eth2 10.166.14.3 | eth3 10.219.188.3
- +----------+ | +----------+
- | respondhost | | | respondhost |
- +----------+ | +----------+
- eth0 (management lan) <========+========> eth0 (management lan)
-```
-
-### Test topology discussion
-
-This topology is suitable for testing almost any tunnel encap/decap
-scenario. The two containers "respondhost" and "initiatehost" are end-stations
-connected to two vpp instances running on "respond" and "initiate".
-
-We leverage the Linux end-station network stacks to generate traffic
-of all sorts.
-
-The so-called "internet" bridge models the public internet. The "respond" and
-"initiate" bridges connect vpp instances to local hosts
-
-End station configs
--------------------
-
-The end-station Linux configurations set up the eth2 and eth3 ip
-addresses shown above, and add tunnel routes to the opposite
-end-station networks.
-
-### respondhost configuration
-
-```
- ifconfig eth2 10.166.14.3/24 up
- route add -net 10.219.188.0/24 gw 10.166.14.2
-```
-
-### initiatehost configuration
-
-```
- sudo ifconfig eth3 10.219.188.3/24 up
- sudo route add -net 10.166.14.0/24 gw 10.219.188.2
-```
-
-VPP configs
------------
-
-Split nat44 / ikev2 + ipsec tunneling, with ipv6 prefix delegation in
-the "respond" config.
-
-### respond configuration
-
-```
- set term pag off
-
- comment { "internet" }
- create host-interface name eth1
- set int ip address host-eth1 10.26.68.100/24
- set int ip6 table host-eth1 0
- set int state host-eth1 up
-
- comment { default route via initiate }
- ip route add 0.0.0.0/0 via 10.26.68.101
-
- comment { "respond-private-net" }
- create host-interface name eth2
- bvi create instance 0
- set int l2 bridge bvi0 1 bvi
- set int ip address bvi0 10.166.14.2/24
- set int state bvi0 up
- set int l2 bridge host-eth2 1
- set int state host-eth2 up
-
-
- nat44 add interface address host-eth1
- set interface nat44 in host-eth2 out host-eth1
- nat44 add identity mapping external host-eth1 udp 500
- nat44 add identity mapping external host-eth1 udp 4500
- comment { nat44 untranslated subnet 10.219.188.0/24 }
-
- comment { responder profile }
- ikev2 profile add initiate
- ikev2 profile set initiate udp-encap
- ikev2 profile set initiate auth rsa-sig cert-file /scratch/setups/respondcert.pem
- set ikev2 local key /scratch/setups/initiatekey.pem
- ikev2 profile set initiate id local fqdn initiator.my.net
- ikev2 profile set initiate id remote fqdn responder.my.net
- ikev2 profile set initiate traffic-selector remote ip-range 10.219.188.0 - 10.219.188.255 port-range 0 - 65535 protocol 0
- ikev2 profile set initiate traffic-selector local ip-range 10.166.14.0 - 10.166.14.255 port-range 0 - 65535 protocol 0
- create ipip tunnel src 10.26.68.100 dst 10.26.68.101
- ikev2 profile set initiate tunnel ipip0
-
- comment { ipv6 prefix delegation }
- ip6 nd address autoconfig host-eth1 default-route
- dhcp6 client host-eth1
- dhcp6 pd client host-eth1 prefix group hgw
- set ip6 address bvi0 prefix group hgw ::2/56
- ip6 nd address autoconfig bvi0 default-route
- ip6 nd bvi0 ra-interval 5 3 ra-lifetime 180
-
- set int mtu packet 1390 ipip0
- set int unnum ipip0 use host-eth1
- ip route add 10.219.188.0/24 via ipip0
-```
-
-### initiate configuration
-
-```
- set term pag off
-
- comment { "internet" }
- create host-interface name eth1
- comment { set dhcp client intfc host-eth1 hostname initiate }
- set int ip address host-eth1 10.26.68.101/24
- set int state host-eth1 up
-
- comment { default route via "internet gateway" }
- comment { ip route add 0.0.0.0/0 via 10.26.68.1 }
-
- comment { "initiate-private-net" }
- create host-interface name eth3
- bvi create instance 0
- set int l2 bridge bvi0 1 bvi
- set int ip address bvi0 10.219.188.2/24
- set int state bvi0 up
- set int l2 bridge host-eth3 1
- set int state host-eth3 up
-
- nat44 add interface address host-eth1
- set interface nat44 in bvi0 out host-eth1
- nat44 add identity mapping external host-eth1 udp 500
- nat44 add identity mapping external host-eth1 udp 4500
- comment { nat44 untranslated subnet 10.166.14.0/24 }
-
- comment { initiator profile }
- ikev2 profile add respond
- ikev2 profile set respond udp-encap
- ikev2 profile set respond auth rsa-sig cert-file /scratch/setups/initiatecert.pem
- set ikev2 local key /scratch/setups/respondkey.pem
- ikev2 profile set respond id local fqdn responder.my.net
- ikev2 profile set respond id remote fqdn initiator.my.net
-
- ikev2 profile set respond traffic-selector remote ip-range 10.166.14.0 - 10.166.14.255 port-range 0 - 65535 protocol 0
- ikev2 profile set respond traffic-selector local ip-range 10.219.188.0 - 10.219.188.255 port-range 0 - 65535 protocol 0
-
- ikev2 profile set respond responder host-eth1 10.26.68.100
- ikev2 profile set respond ike-crypto-alg aes-cbc 256 ike-integ-alg sha1-96 ike-dh modp-2048
- ikev2 profile set respond esp-crypto-alg aes-cbc 256 esp-integ-alg sha1-96 esp-dh ecp-256
- ikev2 profile set respond sa-lifetime 3600 10 5 0
-
- create ipip tunnel src 10.26.68.101 dst 10.26.68.100
- ikev2 profile set respond tunnel ipip0
- ikev2 initiate sa-init respond
-
- set int mtu packet 1390 ipip0
- set int unnum ipip0 use host-eth1
- ip route add 10.166.14.0/24 via ipip0
-```
-
-IKEv2 certificate setup
------------------------
-
-In both of the vpp configurations, you'll see "/scratch/setups/xxx.pem"
-mentioned. These certificates are used in the ikev2 key exchange.
-
-Here's how to generate the certificates:
-
-```
- openssl req -x509 -nodes -newkey rsa:4096 -keyout respondkey.pem -out respondcert.pem -days 3560
- openssl x509 -text -noout -in respondcert.pem
- openssl req -x509 -nodes -newkey rsa:4096 -keyout initiatekey.pem -out initiatecert.pem -days 3560
- openssl x509 -text -noout -in initiatecert.pem
-```
-
-Make sure that the "respond" and "initiate" configurations point to the certificates.
-
-DHCPv6 server setup
--------------------
-
-If you need an ipv6 dhcp server to test ipv6 prefix delegation,
-create the "dhcpserver" container as shown above.
-
-Install the "isc-dhcp-server" Debian package:
-
-```
- sudo apt-get install isc-dhcp-server
-```
-
-### /etc/dhcp/dhcpd6.conf
-
-Edit the dhcpv6 configuration and add an ipv6 subnet with prefix
-delegation. For example:
-
-```
- subnet6 2001:db01:0:1::/64 {
- range6 2001:db01:0:1::1 2001:db01:0:1::9;
- prefix6 2001:db01:0:100:: 2001:db01:0:200::/56;
- }
-```
-
-Add an ipv6 address on eth1, which is connected to the "internet"
-bridge, and start the dhcp server. I use the following trivial bash
-script, which runs the dhcp6 server in the foreground and produces
-dhcp traffic spew:
-
-```
- #!/bin/bash
- ifconfig eth1 inet6 add 2001:db01:0:1::10/64 || true
- dhcpd -6 -d -cf /etc/dhcp/dhcpd6.conf
-```
-
-The "|| true" bit keeps going if eth1 already has the indicated ipv6
-address.
-
-Container / Host Interoperation
--------------------------------
-
-Host / container interoperation is highly desirable. If the host and a
-set of containers don't run the same distro _and distro version_, it's
-reasonably likely that the glibc versions won't match. That, in turn,
-makes vpp binaries built in one environment fail in the other.
-
-Trying to install multiple versions of glibc - especially at the host
-level - often ends very badly and is _not recommended_. It's not just
-glibc, either. The dynamic loader ld-linux-xxx-so.2 is glibc version
-specific.
-
-Fortunately, it's reasonable easy to build lxd container images based on
-specific Ubuntu or Debian versions.
-
-### Create a custom root filesystem image
-
-First, install the "debootstrap" tool:
-
-```
- sudo apt-get install debootstrap
-```
-
-Make a temp directory, and use debootstrap to populate it. In this
-example, we create an Ubuntu 20.04 (focal fossa) base image:
-
-```
- # mkdir /tmp/myroot
- # debootstrap focal /tmp/myroot http://archive.ubuntu.com/ubuntu
-```
-
-To tinker with the base image (if desired):
-
-```
- # chroot /tmp/myroot
- <add packages, etc.>
- # exit
-```
-
-Make a compressed tarball of the base image:
-
-```
- # tar zcf /tmp/rootfs.tar.gz -C /tmp/myroot .
-```
-
-Create a "metadata.yaml" file which describes the base image:
-
-```
- architecture: "x86_64"
- # To get current date in Unix time, use `date +%s` command
- creation_date: 1458040200
- properties:
- architecture: "x86_64"
- description: "My custom Focal Fossa image"
- os: "Ubuntu"
- release: "focal"
-```
-
-Make a compressed tarball of metadata.yaml:
-
-```
- # tar zcf metadata.tar.gz metadata.yaml
-```
-
-Import the image into lxc / lxd:
-
-```
- $ lxc image import metadata.tar.gz rootfd.tar.gz --alias focal-base
-```
-
-### Create a container which uses the customized base image:
-
-```
- $ lxc launch focal-base focaltest
- $ lxc exec focaltest bash
-```
-
-The next several steps should be executed in the container, in the
-bash shell spun up by "lxc exec..."
-
-### Configure container networking
-
-In the container, create /etc/netplan/50-cloud-init.yaml:
-
-```
- network:
- version: 2
- ethernets:
- eth0:
- dhcp4: true
-```
-
-Use "cat > /etc/netplan/50-cloud-init.yaml", and cut-'n-paste if your
-favorite text editor is AWOL.
-
-Apply the configuration:
-
-```
- # netplan apply
-```
-
-At this point, eth0 should have an ip address, and you should see
-a default route with "route -n".
-
-### Configure apt
-
-Again, in the container, set up /etc/apt/sources.list via cut-'n-paste
-from a recently update "focal fossa" host. Something like so:
-
-```
- deb http://us.archive.ubuntu.com/ubuntu/ focal main restricted
- deb http://us.archive.ubuntu.com/ubuntu/ focal-updates main restricted
- deb http://us.archive.ubuntu.com/ubuntu/ focal universe
- deb http://us.archive.ubuntu.com/ubuntu/ focal-updates universe
- deb http://us.archive.ubuntu.com/ubuntu/ focal multiverse
- deb http://us.archive.ubuntu.com/ubuntu/ focal-updates multiverse
- deb http://us.archive.ubuntu.com/ubuntu/ focal-backports main restricted universe multiverse
- deb http://security.ubuntu.com/ubuntu focal-security main restricted
- deb http://security.ubuntu.com/ubuntu focal-security universe
- deb http://security.ubuntu.com/ubuntu focal-security multiverse
-```
-
-"apt-get update" and "apt-install" should produce reasonable results.
-Suggest "apt-get install make git".
-
-At this point, you can use the "/scratch" sharepoint (or similar) to
-execute "make install-dep install-ext-deps" to set up the container
-with the vpp toolchain; proceed as desired.
diff --git a/docs/usecases/container_test.rst b/docs/usecases/container_test.rst
new file mode 100644
index 0000000..8ad6285
--- /dev/null
+++ b/docs/usecases/container_test.rst
@@ -0,0 +1,655 @@
+Simulating networks with VPP
+============================
+
+The “make test” framework provides a good way to test individual
+features. However, when testing several features at once - or validating
+nontrivial configurations - it may prove difficult or impossible to use
+the unit-test framework.
+
+This note explains how to set up lxc/lxd, and a 5-container testbed to
+test a split-tunnel nat + ikev2 + ipsec + ipv6 prefix-delegation
+scenario.
+
+OS / Distro test results
+------------------------
+
+This setup has been tested on an Ubuntu 18.04 LTS system. If you’re
+feeling adventurous, the same scenario also worked on a recent Ubuntu
+20.04 “preview” daily build.
+
+Other distros may work fine, or not at all.
+
+Proxy Server
+------------
+
+If you need to use a proxy server e.g. from a lab system, you’ll
+probably need to set HTTP_PROXY, HTTPS_PROXY, http_proxy and https_proxy
+in /etc/environment. Directly setting variables in the environment
+doesn’t work. The lxd snap *daemon* needs the proxy settings, not the
+user interface.
+
+Something like so:
+
+::
+
+ HTTP_PROXY=http://my.proxy.server:8080
+ HTTPS_PROXY=http://my.proxy.server:4333
+ http_proxy=http://my.proxy.server:8080
+ https_proxy=http://my.proxy.server:4333
+
+Install and configure lxd
+-------------------------
+
+Install the lxd snap. The lxd snap is up to date, as opposed to the
+results of “sudo apt-get install lxd”.
+
+::
+
+ # snap install lxd
+ # lxd init
+
+“lxd init” asks several questions. With the exception of the storage
+pool, take the defaults. To match the configs shown below, create a
+storage pool named “vpp.” Storage pools of type “zfs” and “files” have
+been tested successfully.
+
+zfs is more space-efficient. “lxc copy” is infinitely faster with zfs.
+The path for the zfs storage pool is under /var. Do not replace it with
+a symbolic link, unless you want to rebuild all of your containers from
+scratch. Ask me how I know that.
+
+Create three network segments
+-----------------------------
+
+Aka, linux bridges.
+
+::
+
+ # lxc network create respond
+ # lxc network create internet
+ # lxc network create initiate
+
+We’ll explain the test topology in a bit. Stay tuned.
+
+Set up the default container profile
+------------------------------------
+
+Execute “lxc profile edit default”, and install the following
+configuration. Note that the “shared” directory should mount your vpp
+workspaces. With that trick, you can edit code from any of the
+containers, run vpp without installing it, etc.
+
+::
+
+ config: {}
+ description: Default LXD profile
+ devices:
+ eth0:
+ name: eth0
+ network: lxdbr0
+ type: nic
+ eth1:
+ name: eth1
+ nictype: bridged
+ parent: internet
+ type: nic
+ eth2:
+ name: eth2
+ nictype: bridged
+ parent: respond
+ type: nic
+ eth3:
+ name: eth3
+ nictype: bridged
+ parent: initiate
+ type: nic
+ root:
+ path: /
+ pool: vpp
+ type: disk
+ shared:
+ path: /scratch
+ source: /scratch
+ type: disk
+ name: default
+
+Set up the network configurations
+---------------------------------
+
+Edit the fake “internet” backbone:
+
+::
+
+ # lxc network edit internet
+
+Install the ip addresses shown below, to avoid having to rebuild the vpp
+and host configuration:
+
+::
+
+ config:
+ ipv4.address: 10.26.68.1/24
+ ipv4.dhcp.ranges: 10.26.68.10-10.26.68.50
+ ipv4.nat: "true"
+ ipv6.address: none
+ ipv6.nat: "false"
+ description: ""
+ name: internet
+ type: bridge
+ used_by:
+ managed: true
+ status: Created
+ locations:
+ - none
+
+Repeat the process with the “respond” and “initiate” networks, using
+these configurations:
+
+respond network configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ config:
+ ipv4.address: 10.166.14.1/24
+ ipv4.dhcp.ranges: 10.166.14.10-10.166.14.50
+ ipv4.nat: "true"
+ ipv6.address: none
+ ipv6.nat: "false"
+ description: ""
+ name: respond
+ type: bridge
+ used_by:
+ managed: true
+ status: Created
+ locations:
+ - none
+
+initiate network configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ config:
+ ipv4.address: 10.219.188.1/24
+ ipv4.dhcp.ranges: 10.219.188.10-10.219.188.50
+ ipv4.nat: "true"
+ ipv6.address: none
+ ipv6.nat: "false"
+ description: ""
+ name: initiate
+ type: bridge
+ used_by:
+ managed: true
+ status: Created
+ locations:
+ - none
+
+Create a “master” container image
+---------------------------------
+
+The master container image should be set up so that you can build vpp,
+ssh into the container, edit source code, run gdb, etc.
+
+Make sure that e.g. public key auth ssh works.
+
+::
+
+ # lxd launch ubuntu:18.04 respond
+ <spew>
+ # lxc exec respond bash
+ respond# cd /scratch/my-vpp-workspace
+ respond# apt-get install make ssh
+ respond# make install-dep
+ respond# exit
+ # lxc stop respond
+
+Mark the container image privileged. If you forget this step, you’ll
+trip over a netlink error (-11) aka EAGAIN when you try to roll in the
+vpp configurations.
+
+::
+
+ # lxc config set respond security.privileged "true"
+
+Duplicate the “master” container image
+--------------------------------------
+
+To avoid having to configure N containers, be sure that the master
+container image is fully set up before you help it have children:
+
+::
+
+ # lxc copy respond respondhost
+ # lxc copy respond initiate
+ # lxc copy respond initiatehost
+ # lxc copy respond dhcpserver # optional, to test ipv6 prefix delegation
+
+Install handy script
+--------------------
+
+See below for a handy script which executes lxc commands across the
+current set of running containers. I call it “lxc-foreach,” feel free to
+call the script Ishmael if you like.
+
+Examples:
+
+::
+
+ $ lxc-foreach start
+ <issues "lxc start" for each container in the list>
+
+After a few seconds, use this one to open an ssh connection to each
+container. The ssh command parses the output of “lxc info,” which
+displays container ip addresses.
+
+::
+
+ $ lxc-foreach ssh
+
+Here’s the script:
+
+::
+
+ #!/bin/bash
+
+ set -u
+ export containers="respond respondhost initiate initiatehost dhcpserver"
+
+ if [ x$1 = "x" ] ; then
+ echo missing command
+ exit 1
+ fi
+
+ if [ $1 = "ssh" ] ; then
+ for c in $containers
+ do
+ inet=`lxc info $c | grep eth0 | grep -v inet6 | head -1 | cut -f 3`
+ if [ x$inet = "x" ] ; then
+ echo $c not started
+ else
+ gnome-terminal --command "/usr/bin/ssh $inet"
+ fi
+ done
+ exit 0
+ fi
+
+ for c in $containers
+ do
+ echo lxc $1 $c
+ lxc $1 $c
+ done
+
+ exit 0
+
+Test topology
+-------------
+
+Finally, we’re ready to describe a test topology. First, a picture:
+
+::
+
+ ===+======== management lan/bridge lxdbr0 (dhcp) ===========+===
+ | | |
+ | | |
+ | | |
+ v | v
+ eth0 | eth0
+ +------+ eth1 eth1 +------+
+ | respond | 10.26.88.100 <= internet bridge => 10.26.88.101 | initiate |
+ +------+ +------+
+ eth2 / bvi0 10.166.14.2 | 10.219.188.2 eth3 / bvi0
+ | | |
+ | ("respond" bridge) | ("initiate" bridge) |
+ | | |
+ v | v
+ eth2 10.166.14.3 | eth3 10.219.188.3
+ +----------+ | +----------+
+ | respondhost | | | respondhost |
+ +----------+ | +----------+
+ eth0 (management lan) <========+========> eth0 (management lan)
+
+Test topology discussion
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+This topology is suitable for testing almost any tunnel encap/decap
+scenario. The two containers “respondhost” and “initiatehost” are
+end-stations connected to two vpp instances running on “respond” and
+“initiate”.
+
+We leverage the Linux end-station network stacks to generate traffic of
+all sorts.
+
+The so-called “internet” bridge models the public internet. The
+“respond” and “initiate” bridges connect vpp instances to local hosts
+
+End station configs
+-------------------
+
+The end-station Linux configurations set up the eth2 and eth3 ip
+addresses shown above, and add tunnel routes to the opposite end-station
+networks.
+
+respondhost configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ ifconfig eth2 10.166.14.3/24 up
+ route add -net 10.219.188.0/24 gw 10.166.14.2
+
+initiatehost configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ sudo ifconfig eth3 10.219.188.3/24 up
+ sudo route add -net 10.166.14.0/24 gw 10.219.188.2
+
+VPP configs
+-----------
+
+Split nat44 / ikev2 + ipsec tunneling, with ipv6 prefix delegation in
+the “respond” config.
+
+respond configuration
+~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ set term pag off
+
+ comment { "internet" }
+ create host-interface name eth1
+ set int ip address host-eth1 10.26.68.100/24
+ set int ip6 table host-eth1 0
+ set int state host-eth1 up
+
+ comment { default route via initiate }
+ ip route add 0.0.0.0/0 via 10.26.68.101
+
+ comment { "respond-private-net" }
+ create host-interface name eth2
+ bvi create instance 0
+ set int l2 bridge bvi0 1 bvi
+ set int ip address bvi0 10.166.14.2/24
+ set int state bvi0 up
+ set int l2 bridge host-eth2 1
+ set int state host-eth2 up
+
+
+ nat44 add interface address host-eth1
+ set interface nat44 in host-eth2 out host-eth1
+ nat44 add identity mapping external host-eth1 udp 500
+ nat44 add identity mapping external host-eth1 udp 4500
+ comment { nat44 untranslated subnet 10.219.188.0/24 }
+
+ comment { responder profile }
+ ikev2 profile add initiate
+ ikev2 profile set initiate udp-encap
+ ikev2 profile set initiate auth rsa-sig cert-file /scratch/setups/respondcert.pem
+ set ikev2 local key /scratch/setups/initiatekey.pem
+ ikev2 profile set initiate id local fqdn initiator.my.net
+ ikev2 profile set initiate id remote fqdn responder.my.net
+ ikev2 profile set initiate traffic-selector remote ip-range 10.219.188.0 - 10.219.188.255 port-range 0 - 65535 protocol 0
+ ikev2 profile set initiate traffic-selector local ip-range 10.166.14.0 - 10.166.14.255 port-range 0 - 65535 protocol 0
+ create ipip tunnel src 10.26.68.100 dst 10.26.68.101
+ ikev2 profile set initiate tunnel ipip0
+
+ comment { ipv6 prefix delegation }
+ ip6 nd address autoconfig host-eth1 default-route
+ dhcp6 client host-eth1
+ dhcp6 pd client host-eth1 prefix group hgw
+ set ip6 address bvi0 prefix group hgw ::2/56
+ ip6 nd address autoconfig bvi0 default-route
+ ip6 nd bvi0 ra-interval 5 3 ra-lifetime 180
+
+ set int mtu packet 1390 ipip0
+ set int unnum ipip0 use host-eth1
+ ip route add 10.219.188.0/24 via ipip0
+
+initiate configuration
+~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ set term pag off
+
+ comment { "internet" }
+ create host-interface name eth1
+ comment { set dhcp client intfc host-eth1 hostname initiate }
+ set int ip address host-eth1 10.26.68.101/24
+ set int state host-eth1 up
+
+ comment { default route via "internet gateway" }
+ comment { ip route add 0.0.0.0/0 via 10.26.68.1 }
+
+ comment { "initiate-private-net" }
+ create host-interface name eth3
+ bvi create instance 0
+ set int l2 bridge bvi0 1 bvi
+ set int ip address bvi0 10.219.188.2/24
+ set int state bvi0 up
+ set int l2 bridge host-eth3 1
+ set int state host-eth3 up
+
+ nat44 add interface address host-eth1
+ set interface nat44 in bvi0 out host-eth1
+ nat44 add identity mapping external host-eth1 udp 500
+ nat44 add identity mapping external host-eth1 udp 4500
+ comment { nat44 untranslated subnet 10.166.14.0/24 }
+
+ comment { initiator profile }
+ ikev2 profile add respond
+ ikev2 profile set respond udp-encap
+ ikev2 profile set respond auth rsa-sig cert-file /scratch/setups/initiatecert.pem
+ set ikev2 local key /scratch/setups/respondkey.pem
+ ikev2 profile set respond id local fqdn responder.my.net
+ ikev2 profile set respond id remote fqdn initiator.my.net
+
+ ikev2 profile set respond traffic-selector remote ip-range 10.166.14.0 - 10.166.14.255 port-range 0 - 65535 protocol 0
+ ikev2 profile set respond traffic-selector local ip-range 10.219.188.0 - 10.219.188.255 port-range 0 - 65535 protocol 0
+
+ ikev2 profile set respond responder host-eth1 10.26.68.100
+ ikev2 profile set respond ike-crypto-alg aes-cbc 256 ike-integ-alg sha1-96 ike-dh modp-2048
+ ikev2 profile set respond esp-crypto-alg aes-cbc 256 esp-integ-alg sha1-96 esp-dh ecp-256
+ ikev2 profile set respond sa-lifetime 3600 10 5 0
+
+ create ipip tunnel src 10.26.68.101 dst 10.26.68.100
+ ikev2 profile set respond tunnel ipip0
+ ikev2 initiate sa-init respond
+
+ set int mtu packet 1390 ipip0
+ set int unnum ipip0 use host-eth1
+ ip route add 10.166.14.0/24 via ipip0
+
+IKEv2 certificate setup
+-----------------------
+
+In both of the vpp configurations, you’ll see “/scratch/setups/xxx.pem”
+mentioned. These certificates are used in the ikev2 key exchange.
+
+Here’s how to generate the certificates:
+
+::
+
+ openssl req -x509 -nodes -newkey rsa:4096 -keyout respondkey.pem -out respondcert.pem -days 3560
+ openssl x509 -text -noout -in respondcert.pem
+ openssl req -x509 -nodes -newkey rsa:4096 -keyout initiatekey.pem -out initiatecert.pem -days 3560
+ openssl x509 -text -noout -in initiatecert.pem
+
+Make sure that the “respond” and “initiate” configurations point to the
+certificates.
+
+DHCPv6 server setup
+-------------------
+
+If you need an ipv6 dhcp server to test ipv6 prefix delegation, create
+the “dhcpserver” container as shown above.
+
+Install the “isc-dhcp-server” Debian package:
+
+::
+
+ sudo apt-get install isc-dhcp-server
+
+/etc/dhcp/dhcpd6.conf
+~~~~~~~~~~~~~~~~~~~~~
+
+Edit the dhcpv6 configuration and add an ipv6 subnet with prefix
+delegation. For example:
+
+::
+
+ subnet6 2001:db01:0:1::/64 {
+ range6 2001:db01:0:1::1 2001:db01:0:1::9;
+ prefix6 2001:db01:0:100:: 2001:db01:0:200::/56;
+ }
+
+Add an ipv6 address on eth1, which is connected to the “internet”
+bridge, and start the dhcp server. I use the following trivial bash
+script, which runs the dhcp6 server in the foreground and produces dhcp
+traffic spew:
+
+::
+
+ #!/bin/bash
+ ifconfig eth1 inet6 add 2001:db01:0:1::10/64 || true
+ dhcpd -6 -d -cf /etc/dhcp/dhcpd6.conf
+
+The “\|\| true” bit keeps going if eth1 already has the indicated ipv6
+address.
+
+Container / Host Interoperation
+-------------------------------
+
+Host / container interoperation is highly desirable. If the host and a
+set of containers don’t run the same distro *and distro version*, it’s
+reasonably likely that the glibc versions won’t match. That, in turn,
+makes vpp binaries built in one environment fail in the other.
+
+Trying to install multiple versions of glibc - especially at the host
+level - often ends very badly and is *not recommended*. It’s not just
+glibc, either. The dynamic loader ld-linux-xxx-so.2 is glibc version
+specific.
+
+Fortunately, it’s reasonable easy to build lxd container images based on
+specific Ubuntu or Debian versions.
+
+Create a custom root filesystem image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+First, install the “debootstrap” tool:
+
+::
+
+ sudo apt-get install debootstrap
+
+Make a temp directory, and use debootstrap to populate it. In this
+example, we create an Ubuntu 20.04 (focal fossa) base image:
+
+::
+
+ # mkdir /tmp/myroot
+ # debootstrap focal /tmp/myroot http://archive.ubuntu.com/ubuntu
+
+To tinker with the base image (if desired):
+
+::
+
+ # chroot /tmp/myroot
+ <add packages, etc.>
+ # exit
+
+Make a compressed tarball of the base image:
+
+::
+
+ # tar zcf /tmp/rootfs.tar.gz -C /tmp/myroot .
+
+Create a “metadata.yaml” file which describes the base image:
+
+::
+
+ architecture: "x86_64"
+ # To get current date in Unix time, use `date +%s` command
+ creation_date: 1458040200
+ properties:
+ architecture: "x86_64"
+ description: "My custom Focal Fossa image"
+ os: "Ubuntu"
+ release: "focal"
+
+Make a compressed tarball of metadata.yaml:
+
+::
+
+ # tar zcf metadata.tar.gz metadata.yaml
+
+Import the image into lxc / lxd:
+
+::
+
+ $ lxc image import metadata.tar.gz rootfd.tar.gz --alias focal-base
+
+Create a container which uses the customized base image:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ $ lxc launch focal-base focaltest
+ $ lxc exec focaltest bash
+
+The next several steps should be executed in the container, in the bash
+shell spun up by “lxc exec…”
+
+Configure container networking
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the container, create /etc/netplan/50-cloud-init.yaml:
+
+::
+
+ network:
+ version: 2
+ ethernets:
+ eth0:
+ dhcp4: true
+
+Use “cat > /etc/netplan/50-cloud-init.yaml”, and cut-’n-paste if your
+favorite text editor is AWOL.
+
+Apply the configuration:
+
+::
+
+ # netplan apply
+
+At this point, eth0 should have an ip address, and you should see a
+default route with “route -n”.
+
+Configure apt
+~~~~~~~~~~~~~
+
+Again, in the container, set up /etc/apt/sources.list via cut-’n-paste
+from a recently update “focal fossa” host. Something like so:
+
+::
+
+ deb http://us.archive.ubuntu.com/ubuntu/ focal main restricted
+ deb http://us.archive.ubuntu.com/ubuntu/ focal-updates main restricted
+ deb http://us.archive.ubuntu.com/ubuntu/ focal universe
+ deb http://us.archive.ubuntu.com/ubuntu/ focal-updates universe
+ deb http://us.archive.ubuntu.com/ubuntu/ focal multiverse
+ deb http://us.archive.ubuntu.com/ubuntu/ focal-updates multiverse
+ deb http://us.archive.ubuntu.com/ubuntu/ focal-backports main restricted universe multiverse
+ deb http://security.ubuntu.com/ubuntu focal-security main restricted
+ deb http://security.ubuntu.com/ubuntu focal-security universe
+ deb http://security.ubuntu.com/ubuntu focal-security multiverse
+
+“apt-get update” and “apt-install” should produce reasonable results.
+Suggest “apt-get install make git”.
+
+At this point, you can use the “/scratch” sharepoint (or similar) to
+execute “make install-dep install-ext-deps” to set up the container with
+the vpp toolchain; proceed as desired.
diff --git a/docs/usecases/Routing.rst b/docs/usecases/containers/Routing.rst
similarity index 93%
rename from docs/usecases/Routing.rst
rename to docs/usecases/containers/Routing.rst
index 31929d3..b9d3bc9 100644
--- a/docs/usecases/Routing.rst
+++ b/docs/usecases/containers/Routing.rst
@@ -10,7 +10,7 @@
Enter container *cone*, and check the current network configuration:
.. code-block:: console
-
+
root@cone:/# ip -o a
1: lo inet 127.0.0.1/8 scope host lo\ valid_lft forever preferred_lft forever
1: lo inet6 ::1/128 scope host \ valid_lft forever preferred_lft forever
@@ -43,7 +43,7 @@
Make sure your loopback interface is up, and assign an IP and gateway to veth_link1.
.. code-block:: console
-
+
root@cone:/# ip link set dev lo up
root@cone:/# ip addr add 172.16.1.2/24 dev veth_link1
root@cone:/# ip link set dev veth_link1 up
@@ -55,7 +55,7 @@
Run some commands to verify the changes:
.. code-block:: console
-
+
root@cone:/# ip -o a
1: lo inet 127.0.0.1/8 scope host lo\ valid_lft forever preferred_lft forever
1: lo inet6 ::1/128 scope host \ valid_lft forever preferred_lft forever
@@ -78,7 +78,7 @@
After that's done for *both* containers, exit from the container if you're in one:
.. code-block:: console
-
+
root@ctwo:/# exit
exit
root@localhost:~#
@@ -86,7 +86,7 @@
In the machine running the containers, run **ip link** to see the host *veth* network interfaces, and their link with their respective *container veth's*.
.. code-block:: console
-
+
root@localhost:~# ip link
1: lo: <LOOPBACK> mtu 65536 qdisc noqueue state DOWN mode DEFAULT group default qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
@@ -113,29 +113,29 @@
With VPP in the host machine, show current VPP interfaces:
.. code-block:: console
-
+
root@localhost:~# vppctl show inter
- Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
- local0 0 down 0/0/0/0
+ Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
+ local0 0 down 0/0/0/0
Which should only output local0.
Based on the names of the network interfaces discussed previously, which are specific to my systems, we can create VPP host-interfaces:
.. code-block:: console
-
+
root@localhost:~# vppctl create host-interface name vethQL7K0C
root@localhost:~# vppctl create host-interface name veth8NA72P
Verify they have been set up properly:
.. code-block:: console
-
+
root@localhost:~# vppctl show inter
- Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
- host-vethQL7K0C 1 down 9000/0/0/0
- host-veth8NA72P 2 down 9000/0/0/0
- local0 0 down 0/0/0/0
+ Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
+ host-vethQL7K0C 1 down 9000/0/0/0
+ host-veth8NA72P 2 down 9000/0/0/0
+ local0 0 down 0/0/0/0
Which should output *three network interfaces*, local0, and the other two host network interfaces linked to the container veth's.
@@ -143,7 +143,7 @@
Set their state to up:
.. code-block:: console
-
+
root@localhost:~# vppctl set interface state host-vethQL7K0C up
root@localhost:~# vppctl set interface state host-veth8NA72P up
@@ -152,16 +152,16 @@
.. code-block:: console
root@localhost:~# vppctl show inter
- Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
- host-vethQL7K0C 1 up 9000/0/0/0
- host-veth8NA72P 2 up 9000/0/0/0
- local0 0 down 0/0/0/0
+ Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
+ host-vethQL7K0C 1 up 9000/0/0/0
+ host-veth8NA72P 2 up 9000/0/0/0
+ local0 0 down 0/0/0/0
Add IP addresses for the other end of each veth link:
.. code-block:: console
-
+
root@localhost:~# vppctl set interface ip address host-vethQL7K0C 172.16.1.1/24
root@localhost:~# vppctl set interface ip address host-veth8NA72P 172.16.2.1/24
@@ -180,7 +180,7 @@
Or looking at the FIB by doing:
.. code-block:: console
-
+
root@localhost:~# vppctl show ip fib
ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] locks:[src:plugin-hi:2, src:default-route:1, ]
0.0.0.0/0
@@ -239,7 +239,7 @@
At long last you probably want to see some pings:
.. code-block:: console
-
+
root@localhost:~# lxc-attach -n cone -- ping -c3 172.16.2.2
PING 172.16.2.2 (172.16.2.2) 56(84) bytes of data.
64 bytes from 172.16.2.2: icmp_seq=1 ttl=63 time=0.102 ms
@@ -263,4 +263,4 @@
Which should send/receive three packets for each command.
-This is the end of this guide. Great work!
+This is the end of this guide. Great work!
diff --git a/docs/usecases/containerCreation.rst b/docs/usecases/containers/containerCreation.rst
similarity index 93%
rename from docs/usecases/containerCreation.rst
rename to docs/usecases/containers/containerCreation.rst
index 9b2cc12..bb11688 100644
--- a/docs/usecases/containerCreation.rst
+++ b/docs/usecases/containers/containerCreation.rst
@@ -29,7 +29,7 @@
Look at the contents of *default.conf*, which should initially look like this:
.. code-block:: console
-
+
# cat /etc/lxc/default.conf
lxc.network.type = veth
lxc.network.link = lxcbr0
@@ -40,7 +40,7 @@
Now you will *append to this file* so that each container you create will have an interface for a Linux bridge and an unconsumed second interface.
-You can do this by piping *echo* output into *tee*, where each line is separated with a newline character *\\n* as shown below. Alternatively, you can manually add to this file with a text editor such as **vi**, but make sure you have root privileges.
+You can do this by piping *echo* output into *tee*, where each line is separated with a newline character *\\n* as shown below. Alternatively, you can manually add to this file with a text editor such as **vi**, but make sure you have root privileges.
.. code-block:: console
@@ -72,7 +72,7 @@
If successful, you'll get an output similar to this:
.. code-block:: console
-
+
You just created an Ubuntu xenial amd64 (20180625_07:42) container.
To enable SSH, run: apt install openssh-server
@@ -98,17 +98,17 @@
Start the first container:
.. code-block:: console
-
+
# lxc-start --name cone
And verify its running:
.. code-block:: console
-
+
# lxc-ls --fancy
- NAME STATE AUTOSTART GROUPS IPV4 IPV6
- cone RUNNING 0 - - -
- ctwo STOPPED 0 - - -
+ NAME STATE AUTOSTART GROUPS IPV4 IPV6
+ cone RUNNING 0 - - -
+ ctwo STOPPED 0 - - -
.. note::
diff --git a/docs/usecases/containerSetup.rst b/docs/usecases/containers/containerSetup.rst
similarity index 93%
rename from docs/usecases/containerSetup.rst
rename to docs/usecases/containers/containerSetup.rst
index e0fd81e..8c458f7 100644
--- a/docs/usecases/containerSetup.rst
+++ b/docs/usecases/containers/containerSetup.rst
@@ -3,21 +3,21 @@
.. toctree::
Container packages
-__________________
+==================
Now we can go into container *cone* and install prerequisites such as VPP, and perform some additional commands:
To enter our container via the shell, type:
.. code-block:: console
-
+
# lxc-attach -n cone
root@cone:/#
-Run the linux DHCP setup and install VPP:
+Run the linux DHCP setup and install VPP:
.. code-block:: console
-
+
root@cone:/# resolvconf -d eth0
root@cone:/# dhclient
root@cone:/# apt-get install -y wget
@@ -29,7 +29,7 @@
After this is done, start VPP in this container:
.. code-block:: console
-
+
root@cone:/# service vpp start
Exit this container with the **exit** command (you *may* need to run **exit** twice):
diff --git a/docs/usecases/containers.rst b/docs/usecases/containers/index.rst
similarity index 100%
rename from docs/usecases/containers.rst
rename to docs/usecases/containers/index.rst
diff --git a/docs/usecases/contiv/BUG_REPORTS.md b/docs/usecases/contiv/BUG_REPORTS.md
deleted file mode 100644
index 5b9c3cf..0000000
--- a/docs/usecases/contiv/BUG_REPORTS.md
+++ /dev/null
@@ -1,333 +0,0 @@
-# Debugging and Reporting Bugs in Contiv-VPP
-
-## Bug Report Structure
-
-- [Deployment description](#describe-deployment):
-Briefly describes the deployment, where an issue was spotted,
-number of k8s nodes, is DHCP/STN/TAP used.
-
-- [Logs](#collecting-the-logs):
-Attach corresponding logs, at least from the vswitch pods.
-
-- [VPP config](#inspect-vpp-config):
-Attach output of the show commands.
-
-- [Basic Collection Example](#basic-example)
-
-### Describe Deployment
-Since contiv-vpp can be used with different configurations, it is helpful
-to attach the config that was applied. Either attach `values.yaml` passed to the helm chart,
-or attach the [corresponding part](https://github.com/contiv/vpp/blob/42b3bfbe8735508667b1e7f1928109a65dfd5261/k8s/contiv-vpp.yaml#L24-L38) from the deployment yaml file.
-
-```
- contiv.yaml: |-
- TCPstackDisabled: true
- UseTAPInterfaces: true
- TAPInterfaceVersion: 2
- NatExternalTraffic: true
- MTUSize: 1500
- IPAMConfig:
- PodSubnetCIDR: 10.1.0.0/16
- PodNetworkPrefixLen: 24
- PodIfIPCIDR: 10.2.1.0/24
- VPPHostSubnetCIDR: 172.30.0.0/16
- VPPHostNetworkPrefixLen: 24
- NodeInterconnectCIDR: 192.168.16.0/24
- VxlanCIDR: 192.168.30.0/24
- NodeInterconnectDHCP: False
-```
-
-Information that might be helpful:
- - Whether node IPs are statically assigned, or if DHCP is used
- - STN is enabled
- - Version of TAP interfaces used
- - Output of `kubectl get pods -o wide --all-namespaces`
-
-
-### Collecting the Logs
-
-The most essential thing that needs to be done when debugging and **reporting an issue**
-in Contiv-VPP is **collecting the logs from the contiv-vpp vswitch containers**.
-
-#### a) Collecting Vswitch Logs Using kubectl
-In order to collect the logs from individual vswitches in the cluster, connect to the master node
-and then find the POD names of the individual vswitch containers:
-
-```
-$ kubectl get pods --all-namespaces | grep vswitch
-kube-system contiv-vswitch-lqxfp 2/2 Running 0 1h
-kube-system contiv-vswitch-q6kwt 2/2 Running 0 1h
-```
-
-Then run the following command, with *pod name* replaced by the actual POD name:
-```
-$ kubectl logs <pod name> -n kube-system -c contiv-vswitch
-```
-
-Redirect the output to a file to save the logs, for example:
-
-```
-kubectl logs contiv-vswitch-lqxfp -n kube-system -c contiv-vswitch > logs-master.txt
-```
-
-#### b) Collecting Vswitch Logs Using Docker
-If option a) does not work, then you can still collect the same logs using the plain docker
-command. For that, you need to connect to each individual node in the k8s cluster, and find the container ID of the vswitch container:
-
-```
-$ docker ps | grep contivvpp/vswitch
-b682b5837e52 contivvpp/vswitch "/usr/bin/supervisor…" 2 hours ago Up 2 hours k8s_contiv-vswitch_contiv-vswitch-q6kwt_kube-system_d09b6210-2903-11e8-b6c9-08002723b076_0
-```
-
-Now use the ID from the first column to dump the logs into the `logs-master.txt` file:
-```
-$ docker logs b682b5837e52 > logs-master.txt
-```
-
-#### Reviewing the Vswitch Logs
-
-In order to debug an issue, it is good to start by grepping the logs for the `level=error` string, for example:
-```
-$ cat logs-master.txt | grep level=error
-```
-
-Also, VPP or contiv-agent may crash with some bugs. To check if some process crashed, grep for the string `exit`, for example:
-```
-$ cat logs-master.txt | grep exit
-2018-03-20 06:03:45,948 INFO exited: vpp (terminated by SIGABRT (core dumped); not expected)
-2018-03-20 06:03:48,948 WARN received SIGTERM indicating exit request
-```
-
-#### Collecting the STN Daemon Logs
-In STN (Steal The NIC) deployment scenarios, often need to collect and review the logs
-from the STN daemon. This needs to be done on each node:
-```
-$ docker logs contiv-stn > logs-stn-master.txt
-```
-
-#### Collecting Logs in Case of Crash Loop
-If the vswitch is crashing in a loop (which can be determined by increasing the number in the `RESTARTS`
-column of the `kubectl get pods --all-namespaces` output), the `kubectl logs` or `docker logs` would
-give us the logs of the latest incarnation of the vswitch. That might not be the original root cause
-of the very first crash, so in order to debug that, we need to disable k8s health check probes to not
-restart the vswitch after the very first crash. This can be done by commenting-out the `readinessProbe`
-and `livenessProbe` in the contiv-vpp deployment YAML:
-
-```diff
-diff --git a/k8s/contiv-vpp.yaml b/k8s/contiv-vpp.yaml
-index 3676047..ffa4473 100644
---- a/k8s/contiv-vpp.yaml
-+++ b/k8s/contiv-vpp.yaml
-@@ -224,18 +224,18 @@ spec:
- ports:
- # readiness + liveness probe
- - containerPort: 9999
-- readinessProbe:
-- httpGet:
-- path: /readiness
-- port: 9999
-- periodSeconds: 1
-- initialDelaySeconds: 15
-- livenessProbe:
-- httpGet:
-- path: /liveness
-- port: 9999
-- periodSeconds: 1
-- initialDelaySeconds: 60
-+ # readinessProbe:
-+ # httpGet:
-+ # path: /readiness
-+ # port: 9999
-+ # periodSeconds: 1
-+ # initialDelaySeconds: 15
-+ # livenessProbe:
-+ # httpGet:
-+ # path: /liveness
-+ # port: 9999
-+ # periodSeconds: 1
-+ # initialDelaySeconds: 60
- env:
- - name: MICROSERVICE_LABEL
- valueFrom:
-```
-
-If VPP is the crashing process, please follow the \[CORE_FILES\](CORE_FILES.html) guide and provide the coredump file.
-
-
-### Inspect VPP Config
-Inspect the following areas:
-- Configured interfaces (issues related basic node/pod connectivity issues):
-```
-vpp# sh int addr
-GigabitEthernet0/9/0 (up):
- 192.168.16.1/24
-local0 (dn):
-loop0 (up):
- l2 bridge bd_id 1 bvi shg 0
- 192.168.30.1/24
-tapcli-0 (up):
- 172.30.1.1/24
-```
-
-- IP forwarding table:
-```
-vpp# sh ip fib
-ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] locks:[src:(nil):2, src:adjacency:3, src:default-route:1, ]
-0.0.0.0/0
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:1 buckets:1 uRPF:0 to:[7:552]]
- [0] [@0]: dpo-drop ip4
-0.0.0.0/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:2 buckets:1 uRPF:1 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-
-...
-...
-
-255.255.255.255/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:5 buckets:1 uRPF:4 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-```
-- ARP Table:
-```
-vpp# sh ip arp
- Time IP4 Flags Ethernet Interface
- 728.6616 192.168.16.2 D 08:00:27:9c:0e:9f GigabitEthernet0/8/0
- 542.7045 192.168.30.2 S 1a:2b:3c:4d:5e:02 loop0
- 1.4241 172.30.1.2 D 86:41:d5:92:fd:24 tapcli-0
- 15.2485 10.1.1.2 SN 00:00:00:00:00:02 tapcli-1
- 739.2339 10.1.1.3 SN 00:00:00:00:00:02 tapcli-2
- 739.4119 10.1.1.4 SN 00:00:00:00:00:02 tapcli-3
-```
-- NAT configuration (issues related to services):
-```
-DBGvpp# sh nat44 addresses
-NAT44 pool addresses:
-192.168.16.10
- tenant VRF independent
- 0 busy udp ports
- 0 busy tcp ports
- 0 busy icmp ports
-NAT44 twice-nat pool addresses:
-```
-```
-vpp# sh nat44 static mappings
-NAT44 static mappings:
- tcp local 192.168.42.1:6443 external 10.96.0.1:443 vrf 0 out2in-only
- tcp local 192.168.42.1:12379 external 192.168.42.2:32379 vrf 0 out2in-only
- tcp local 192.168.42.1:12379 external 192.168.16.2:32379 vrf 0 out2in-only
- tcp local 192.168.42.1:12379 external 192.168.42.1:32379 vrf 0 out2in-only
- tcp local 192.168.42.1:12379 external 192.168.16.1:32379 vrf 0 out2in-only
- tcp local 192.168.42.1:12379 external 10.109.143.39:12379 vrf 0 out2in-only
- udp local 10.1.2.2:53 external 10.96.0.10:53 vrf 0 out2in-only
- tcp local 10.1.2.2:53 external 10.96.0.10:53 vrf 0 out2in-only
-```
-```
-vpp# sh nat44 interfaces
-NAT44 interfaces:
- loop0 in out
- GigabitEthernet0/9/0 out
- tapcli-0 in out
-```
-```
-vpp# sh nat44 sessions
-NAT44 sessions:
- 192.168.20.2: 0 dynamic translations, 3 static translations
- 10.1.1.3: 0 dynamic translations, 0 static translations
- 10.1.1.4: 0 dynamic translations, 0 static translations
- 10.1.1.2: 0 dynamic translations, 6 static translations
- 10.1.2.18: 0 dynamic translations, 2 static translations
-```
-- ACL config (issues related to policies):
-```
-vpp# sh acl-plugin acl
-```
-- "Steal the NIC (STN)" config (issues related to host connectivity when STN is active):
-```
-vpp# sh stn rules
-- rule_index: 0
- address: 10.1.10.47
- iface: tapcli-0 (2)
- next_node: tapcli-0-output (410)
-```
-- Errors:
-```
-vpp# sh errors
-```
-- Vxlan tunnels:
-```
-vpp# sh vxlan tunnels
-```
-- Vxlan tunnels:
-```
-vpp# sh vxlan tunnels
-```
-- Hardware interface information:
-```
-vpp# sh hardware-interfaces
-```
-
-### Basic Example
-
-[contiv-vpp-bug-report.sh][1] is an example of a script that may be a useful starting point to gathering the above information using kubectl.
-
-Limitations:
-- The script does not include STN daemon logs nor does it handle the special
- case of a crash loop
-
-Prerequisites:
-- The user specified in the script must have passwordless access to all nodes
- in the cluster; on each node in the cluster the user must have passwordless
- access to sudo.
-
-#### Setting up Prerequisites
-To enable logging into a node without a password, copy your public key to the following
-node:
-```
-ssh-copy-id <user-id>@<node-name-or-ip-address>
-```
-
-To enable running sudo without a password for a given user, enter:
-```
-$ sudo visudo
-```
-
-Append the following entry to run ALL command without a password for a given
-user:
-```
-<userid> ALL=(ALL) NOPASSWD:ALL
-```
-
-You can also add user `<user-id>` to group `sudo` and edit the `sudo`
-entry as follows:
-
-```
-# Allow members of group sudo to execute any command
-%sudo ALL=(ALL:ALL) NOPASSWD:ALL
-```
-
-Add user `<user-id>` to group `<group-id>` as follows:
-```
-sudo adduser <user-id> <group-id>
-```
-or as follows:
-```
-usermod -a -G <group-id> <user-id>
-```
-#### Working with the Contiv-VPP Vagrant Test Bed
-The script can be used to collect data from the [Contiv-VPP test bed created with Vagrant][2].
-To collect debug information from this Contiv-VPP test bed, do the
-following steps:
-* In the directory where you created your vagrant test bed, do:
-```
- vagrant ssh-config > vagrant-ssh.conf
-```
-* To collect the debug information do:
-```
- ./contiv-vpp-bug-report.sh -u vagrant -m k8s-master -f <path-to-your-vagrant-ssh-config-file>/vagrant-ssh.conf
-```
-
-[1]: https://github.com/contiv/vpp/tree/master/scripts/contiv-vpp-bug-report.sh
-[2]: https://github.com/contiv/vpp/blob/master/vagrant/README.md
diff --git a/docs/usecases/contiv/BUG_REPORTS.rst b/docs/usecases/contiv/BUG_REPORTS.rst
new file mode 100644
index 0000000..8e55d5b
--- /dev/null
+++ b/docs/usecases/contiv/BUG_REPORTS.rst
@@ -0,0 +1,401 @@
+Debugging and Reporting Bugs in Contiv-VPP
+==========================================
+
+Bug Report Structure
+--------------------
+
+- `Deployment description <#describe-deployment>`__: Briefly describes
+ the deployment, where an issue was spotted, number of k8s nodes, is
+ DHCP/STN/TAP used.
+
+- `Logs <#collecting-the-logs>`__: Attach corresponding logs, at least
+ from the vswitch pods.
+
+- `VPP config <#inspect-vpp-config>`__: Attach output of the show
+ commands.
+
+- `Basic Collection Example <#basic-example>`__
+
+Describe Deployment
+~~~~~~~~~~~~~~~~~~~
+
+Since contiv-vpp can be used with different configurations, it is
+helpful to attach the config that was applied. Either attach
+``values.yaml`` passed to the helm chart, or attach the `corresponding
+part <https://github.com/contiv/vpp/blob/42b3bfbe8735508667b1e7f1928109a65dfd5261/k8s/contiv-vpp.yaml#L24-L38>`__
+from the deployment yaml file.
+
+.. code:: yaml
+
+ contiv.yaml: |-
+ TCPstackDisabled: true
+ UseTAPInterfaces: true
+ TAPInterfaceVersion: 2
+ NatExternalTraffic: true
+ MTUSize: 1500
+ IPAMConfig:
+ PodSubnetCIDR: 10.1.0.0/16
+ PodNetworkPrefixLen: 24
+ PodIfIPCIDR: 10.2.1.0/24
+ VPPHostSubnetCIDR: 172.30.0.0/16
+ VPPHostNetworkPrefixLen: 24
+ NodeInterconnectCIDR: 192.168.16.0/24
+ VxlanCIDR: 192.168.30.0/24
+ NodeInterconnectDHCP: False
+
+Information that might be helpful: - Whether node IPs are statically
+assigned, or if DHCP is used - STN is enabled - Version of TAP
+interfaces used - Output of
+``kubectl get pods -o wide --all-namespaces``
+
+Collecting the Logs
+~~~~~~~~~~~~~~~~~~~
+
+The most essential thing that needs to be done when debugging and
+**reporting an issue** in Contiv-VPP is **collecting the logs from the
+contiv-vpp vswitch containers**.
+
+a) Collecting Vswitch Logs Using kubectl
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In order to collect the logs from individual vswitches in the cluster,
+connect to the master node and then find the POD names of the individual
+vswitch containers:
+
+::
+
+ $ kubectl get pods --all-namespaces | grep vswitch
+ kube-system contiv-vswitch-lqxfp 2/2 Running 0 1h
+ kube-system contiv-vswitch-q6kwt 2/2 Running 0 1h
+
+Then run the following command, with *pod name* replaced by the actual
+POD name:
+
+::
+
+ $ kubectl logs <pod name> -n kube-system -c contiv-vswitch
+
+Redirect the output to a file to save the logs, for example:
+
+::
+
+ kubectl logs contiv-vswitch-lqxfp -n kube-system -c contiv-vswitch > logs-master.txt
+
+b) Collecting Vswitch Logs Using Docker
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If option a) does not work, then you can still collect the same logs
+using the plain docker command. For that, you need to connect to each
+individual node in the k8s cluster, and find the container ID of the
+vswitch container:
+
+::
+
+ $ docker ps | grep contivvpp/vswitch
+ b682b5837e52 contivvpp/vswitch "/usr/bin/supervisor…" 2 hours ago Up 2 hours k8s_contiv-vswitch_contiv-vswitch-q6kwt_kube-system_d09b6210-2903-11e8-b6c9-08002723b076_0
+
+Now use the ID from the first column to dump the logs into the
+``logs-master.txt`` file:
+
+::
+
+ $ docker logs b682b5837e52 > logs-master.txt
+
+Reviewing the Vswitch Logs
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In order to debug an issue, it is good to start by grepping the logs for
+the ``level=error`` string, for example:
+
+::
+
+ $ cat logs-master.txt | grep level=error
+
+Also, VPP or contiv-agent may crash with some bugs. To check if some
+process crashed, grep for the string ``exit``, for example:
+
+::
+
+ $ cat logs-master.txt | grep exit
+ 2018-03-20 06:03:45,948 INFO exited: vpp (terminated by SIGABRT (core dumped); not expected)
+ 2018-03-20 06:03:48,948 WARN received SIGTERM indicating exit request
+
+Collecting the STN Daemon Logs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In STN (Steal The NIC) deployment scenarios, often need to collect and
+review the logs from the STN daemon. This needs to be done on each node:
+
+::
+
+ $ docker logs contiv-stn > logs-stn-master.txt
+
+Collecting Logs in Case of Crash Loop
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If the vswitch is crashing in a loop (which can be determined by
+increasing the number in the ``RESTARTS`` column of the
+``kubectl get pods --all-namespaces`` output), the ``kubectl logs`` or
+``docker logs`` would give us the logs of the latest incarnation of the
+vswitch. That might not be the original root cause of the very first
+crash, so in order to debug that, we need to disable k8s health check
+probes to not restart the vswitch after the very first crash. This can
+be done by commenting-out the ``readinessProbe`` and ``livenessProbe``
+in the contiv-vpp deployment YAML:
+
+.. code:: diff
+
+ diff --git a/k8s/contiv-vpp.yaml b/k8s/contiv-vpp.yaml
+ index 3676047..ffa4473 100644
+ --- a/k8s/contiv-vpp.yaml
+ +++ b/k8s/contiv-vpp.yaml
+ @@ -224,18 +224,18 @@ spec:
+ ports:
+ # readiness + liveness probe
+ - containerPort: 9999
+ - readinessProbe:
+ - httpGet:
+ - path: /readiness
+ - port: 9999
+ - periodSeconds: 1
+ - initialDelaySeconds: 15
+ - livenessProbe:
+ - httpGet:
+ - path: /liveness
+ - port: 9999
+ - periodSeconds: 1
+ - initialDelaySeconds: 60
+ + # readinessProbe:
+ + # httpGet:
+ + # path: /readiness
+ + # port: 9999
+ + # periodSeconds: 1
+ + # initialDelaySeconds: 15
+ + # livenessProbe:
+ + # httpGet:
+ + # path: /liveness
+ + # port: 9999
+ + # periodSeconds: 1
+ + # initialDelaySeconds: 60
+ env:
+ - name: MICROSERVICE_LABEL
+ valueFrom:
+
+If VPP is the crashing process, please follow the
+[CORE_FILES](CORE_FILES.html) guide and provide the coredump file.
+
+Inspect VPP Config
+~~~~~~~~~~~~~~~~~~
+
+Inspect the following areas: - Configured interfaces (issues related
+basic node/pod connectivity issues):
+
+::
+
+ vpp# sh int addr
+ GigabitEthernet0/9/0 (up):
+ 192.168.16.1/24
+ local0 (dn):
+ loop0 (up):
+ l2 bridge bd_id 1 bvi shg 0
+ 192.168.30.1/24
+ tapcli-0 (up):
+ 172.30.1.1/24
+
+- IP forwarding table:
+
+::
+
+ vpp# sh ip fib
+ ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] locks:[src:(nil):2, src:adjacency:3, src:default-route:1, ]
+ 0.0.0.0/0
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:1 buckets:1 uRPF:0 to:[7:552]]
+ [0] [@0]: dpo-drop ip4
+ 0.0.0.0/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:2 buckets:1 uRPF:1 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+
+ ...
+ ...
+
+ 255.255.255.255/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:5 buckets:1 uRPF:4 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+
+- ARP Table:
+
+::
+
+ vpp# sh ip arp
+ Time IP4 Flags Ethernet Interface
+ 728.6616 192.168.16.2 D 08:00:27:9c:0e:9f GigabitEthernet0/8/0
+ 542.7045 192.168.30.2 S 1a:2b:3c:4d:5e:02 loop0
+ 1.4241 172.30.1.2 D 86:41:d5:92:fd:24 tapcli-0
+ 15.2485 10.1.1.2 SN 00:00:00:00:00:02 tapcli-1
+ 739.2339 10.1.1.3 SN 00:00:00:00:00:02 tapcli-2
+ 739.4119 10.1.1.4 SN 00:00:00:00:00:02 tapcli-3
+
+- NAT configuration (issues related to services):
+
+::
+
+ DBGvpp# sh nat44 addresses
+ NAT44 pool addresses:
+ 192.168.16.10
+ tenant VRF independent
+ 0 busy udp ports
+ 0 busy tcp ports
+ 0 busy icmp ports
+ NAT44 twice-nat pool addresses:
+
+::
+
+ vpp# sh nat44 static mappings
+ NAT44 static mappings:
+ tcp local 192.168.42.1:6443 external 10.96.0.1:443 vrf 0 out2in-only
+ tcp local 192.168.42.1:12379 external 192.168.42.2:32379 vrf 0 out2in-only
+ tcp local 192.168.42.1:12379 external 192.168.16.2:32379 vrf 0 out2in-only
+ tcp local 192.168.42.1:12379 external 192.168.42.1:32379 vrf 0 out2in-only
+ tcp local 192.168.42.1:12379 external 192.168.16.1:32379 vrf 0 out2in-only
+ tcp local 192.168.42.1:12379 external 10.109.143.39:12379 vrf 0 out2in-only
+ udp local 10.1.2.2:53 external 10.96.0.10:53 vrf 0 out2in-only
+ tcp local 10.1.2.2:53 external 10.96.0.10:53 vrf 0 out2in-only
+
+::
+
+ vpp# sh nat44 interfaces
+ NAT44 interfaces:
+ loop0 in out
+ GigabitEthernet0/9/0 out
+ tapcli-0 in out
+
+::
+
+ vpp# sh nat44 sessions
+ NAT44 sessions:
+ 192.168.20.2: 0 dynamic translations, 3 static translations
+ 10.1.1.3: 0 dynamic translations, 0 static translations
+ 10.1.1.4: 0 dynamic translations, 0 static translations
+ 10.1.1.2: 0 dynamic translations, 6 static translations
+ 10.1.2.18: 0 dynamic translations, 2 static translations
+
+- ACL config (issues related to policies):
+
+::
+
+ vpp# sh acl-plugin acl
+
+- “Steal the NIC (STN)” config (issues related to host connectivity
+ when STN is active):
+
+::
+
+ vpp# sh stn rules
+ - rule_index: 0
+ address: 10.1.10.47
+ iface: tapcli-0 (2)
+ next_node: tapcli-0-output (410)
+
+- Errors:
+
+::
+
+ vpp# sh errors
+
+- Vxlan tunnels:
+
+::
+
+ vpp# sh vxlan tunnels
+
+- Vxlan tunnels:
+
+::
+
+ vpp# sh vxlan tunnels
+
+- Hardware interface information:
+
+::
+
+ vpp# sh hardware-interfaces
+
+Basic Example
+~~~~~~~~~~~~~
+
+`contiv-vpp-bug-report.sh <https://github.com/contiv/vpp/tree/master/scripts/contiv-vpp-bug-report.sh>`__
+is an example of a script that may be a useful starting point to
+gathering the above information using kubectl.
+
+Limitations: - The script does not include STN daemon logs nor does it
+handle the special case of a crash loop
+
+Prerequisites: - The user specified in the script must have passwordless
+access to all nodes in the cluster; on each node in the cluster the user
+must have passwordless access to sudo.
+
+Setting up Prerequisites
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+To enable logging into a node without a password, copy your public key
+to the following node:
+
+::
+
+ ssh-copy-id <user-id>@<node-name-or-ip-address>
+
+To enable running sudo without a password for a given user, enter:
+
+::
+
+ $ sudo visudo
+
+Append the following entry to run ALL command without a password for a
+given user:
+
+::
+
+ <userid> ALL=(ALL) NOPASSWD:ALL
+
+You can also add user ``<user-id>`` to group ``sudo`` and edit the
+``sudo`` entry as follows:
+
+::
+
+ # Allow members of group sudo to execute any command
+ %sudo ALL=(ALL:ALL) NOPASSWD:ALL
+
+Add user ``<user-id>`` to group ``<group-id>`` as follows:
+
+::
+
+ sudo adduser <user-id> <group-id>
+
+or as follows:
+
+::
+
+ usermod -a -G <group-id> <user-id>
+
+Working with the Contiv-VPP Vagrant Test Bed
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The script can be used to collect data from the `Contiv-VPP test bed
+created with
+Vagrant <https://github.com/contiv/vpp/blob/master/vagrant/README.md>`__.
+To collect debug information from this Contiv-VPP test bed, do the
+following steps: \* In the directory where you created your vagrant test
+bed, do:
+
+::
+
+ vagrant ssh-config > vagrant-ssh.conf
+
+- To collect the debug information do:
+
+::
+
+ ./contiv-vpp-bug-report.sh -u vagrant -m k8s-master -f <path-to-your-vagrant-ssh-config-file>/vagrant-ssh.conf
diff --git a/docs/usecases/contiv/CORE_FILES.md b/docs/usecases/contiv/CORE_FILES.md
deleted file mode 100644
index 5d269cd..0000000
--- a/docs/usecases/contiv/CORE_FILES.md
+++ /dev/null
@@ -1,73 +0,0 @@
-# Capturing VPP core dumps
-In order to debug a crash of VPP, it is required to provide a coredump file, which allows backtracing of the VPP issue. The following items are the requirements for capturing a coredump:
-
-#### 1. Disable k8s Probes to Prevent k8s from Restarting the POD with a Crashed VPP
-As described in [BUG_REPORTS.md](BUG_REPORTS.html#collecting-the-logs-in-case-of-crash-loop).
-
-#### 2. Modify VPP Startup config file
-In `/etc/vpp/contiv-vswitch.conf`, add the following lines into the `unix` section:
-
-```
-unix {
- ...
- coredump-size unlimited
- full-coredump
-}
-```
-
-#### 3. Turn on Coredumps in the Vswitch Container
-After re-deploying Contiv-VPP networking, enter bash shell in the vswitch
-container (use actual name of the vswitch POD - `contiv-vswitch-7whk7` in this case):
-```
-kubectl exec -it contiv-vswitch-7whk7 -n kube-system -c contiv-vswitch bash
-```
-
-Enable coredumps:
-```
-mkdir -p /tmp/dumps
-sysctl -w debug.exception-trace=1
-sysctl -w kernel.core_pattern="/tmp/dumps/%e-%t"
-ulimit -c unlimited
-echo 2 > /proc/sys/fs/suid_dumpable
-```
-
-#### 4. Let VPP Crash
-Now repeat the steps that lead to the VPP crash. You can also force VPP to crash at the point where it is
-running (e.g., if it is stuck) by using the SIGQUIT signal:
-```
-kill -3 `pidof vpp`
-```
-
-#### 5. Locate and Inspect the Core File
-The core file should appear in `/tmp/dumps` in the container:
-```
-cd /tmp/dumps
-ls
-vpp_main-1524124440
-```
-
-You can try to backtrace, after installing gdb:
-```
-apt-get update && apt-get install gdb
-gdb vpp vpp_main-1524124440
-(gdb) bt
-```
-
-#### 6. Copy the Core File Out of the Container
-Finally, copy the core file out of the container. First, while still inside the container,
-pack the core file into an archive:
-
-```
-cd /tmp/dumps
-tar cvzf vppdump.tar.gz vpp_main-1524124440
-```
-
-Now, on the host, determine the docker ID of the container, and then copy the file out of the host:
-```
-docker ps | grep vswitch_contiv
-d7aceb2e4876 c43a70ac3d01 "/usr/bin/supervisor…" 25 minutes ago Up 25 minutes k8s_contiv-vswitch_contiv-vswitch-zqzn6_kube-system_9923952f-43a6-11e8-be84-080027de08ea_0
-
-docker cp d7aceb2e4876:/tmp/dumps/vppdump.tar.gz .
-```
-
-Now you are ready to file a bug in [jira.fd.io](https://jira.fd.io/) and attach the core file.
\ No newline at end of file
diff --git a/docs/usecases/contiv/CORE_FILES.rst b/docs/usecases/contiv/CORE_FILES.rst
new file mode 100644
index 0000000..1888848
--- /dev/null
+++ b/docs/usecases/contiv/CORE_FILES.rst
@@ -0,0 +1,101 @@
+Capturing VPP core dumps
+========================
+
+In order to debug a crash of VPP, it is required to provide a coredump
+file, which allows backtracing of the VPP issue. The following items are
+the requirements for capturing a coredump:
+
+1. Disable k8s Probes to Prevent k8s from Restarting the POD with a Crashed VPP
+-------------------------------------------------------------------------------
+
+As described in
+`BUG_REPORTS.md <BUG_REPORTS.html#collecting-the-logs-in-case-of-crash-loop>`__.
+
+2. Modify VPP Startup config file
+---------------------------------
+
+In ``/etc/vpp/contiv-vswitch.conf``, add the following lines into the
+``unix`` section:
+
+::
+
+ unix {
+ ...
+ coredump-size unlimited
+ full-coredump
+ }
+
+3. Turn on Coredumps in the Vswitch Container
+---------------------------------------------
+
+After re-deploying Contiv-VPP networking, enter bash shell in the
+vswitch container (use actual name of the vswitch POD -
+``contiv-vswitch-7whk7`` in this case):
+
+::
+
+ kubectl exec -it contiv-vswitch-7whk7 -n kube-system -c contiv-vswitch bash
+
+Enable coredumps:
+
+::
+
+ mkdir -p /tmp/dumps
+ sysctl -w debug.exception-trace=1
+ sysctl -w kernel.core_pattern="/tmp/dumps/%e-%t"
+ ulimit -c unlimited
+ echo 2 > /proc/sys/fs/suid_dumpable
+
+4. Let VPP Crash
+----------------
+
+Now repeat the steps that lead to the VPP crash. You can also force VPP
+to crash at the point where it is running (e.g., if it is stuck) by
+using the SIGQUIT signal:
+
+::
+
+ kill -3 `pidof vpp`
+
+5. Locate and Inspect the Core File
+-----------------------------------
+
+The core file should appear in ``/tmp/dumps`` in the container:
+
+::
+
+ cd /tmp/dumps
+ ls
+ vpp_main-1524124440
+
+You can try to backtrace, after installing gdb:
+
+::
+
+ apt-get update && apt-get install gdb
+ gdb vpp vpp_main-1524124440
+ (gdb) bt
+
+6. Copy the Core File Out of the Container
+------------------------------------------
+
+Finally, copy the core file out of the container. First, while still
+inside the container, pack the core file into an archive:
+
+::
+
+ cd /tmp/dumps
+ tar cvzf vppdump.tar.gz vpp_main-1524124440
+
+Now, on the host, determine the docker ID of the container, and then
+copy the file out of the host:
+
+::
+
+ docker ps | grep vswitch_contiv
+ d7aceb2e4876 c43a70ac3d01 "/usr/bin/supervisor…" 25 minutes ago Up 25 minutes k8s_contiv-vswitch_contiv-vswitch-zqzn6_kube-system_9923952f-43a6-11e8-be84-080027de08ea_0
+
+ docker cp d7aceb2e4876:/tmp/dumps/vppdump.tar.gz .
+
+Now you are ready to file a bug in `jira.fd.io <https://jira.fd.io/>`__
+and attach the core file.
diff --git a/docs/usecases/contiv/CUSTOM_MGMT_NETWORK.md b/docs/usecases/contiv/CUSTOM_MGMT_NETWORK.md
deleted file mode 100644
index bf2937f..0000000
--- a/docs/usecases/contiv/CUSTOM_MGMT_NETWORK.md
+++ /dev/null
@@ -1,26 +0,0 @@
-### Setting Up a Custom Management Network on Multi-Homed Nodes
-
-If the interface you use for Kubernetes management traffic (for example, the
-IP address used for `kubeadm join`) is not the one that contains the default
-route out of the host, then you need to specify the management node IP address in
-the Kubelet config file. Add the following line to:
-(`/etc/systemd/system/kubelet.service.d/10-kubeadm.conf`):
-```
-Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false --node-ip=<node-management-ip-address>"
-```
-#### Example
-Consider a 2 node deployment where each node is connected to 2 networks -
-`10.0.2.0/24` and `192.168.56.0/24`, and the default route on each node points
-to the interface connected to the `10.0.2.0/24` subnet. We want to use subnet
-`192.168.56.0/24` for Kubernetes management traffic. Assume the addresses of
-nodes connected to `192.168.56.0/24` are `192.168.56.105` and `192.168.56.106`.
-
-On the `192.168.56.105` node you add the following line to `10-kubeadm.conf`:
-```
-Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false --node-ip=192.168.56.105"
-```
-On the `192.168.56.106` node you add the following line to `10-kubeadm.conf`:
-```
-Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false --node-ip=192.168.56.106"
-```
-
diff --git a/docs/usecases/contiv/CUSTOM_MGMT_NETWORK.rst b/docs/usecases/contiv/CUSTOM_MGMT_NETWORK.rst
new file mode 100644
index 0000000..b8cf2e6
--- /dev/null
+++ b/docs/usecases/contiv/CUSTOM_MGMT_NETWORK.rst
@@ -0,0 +1,36 @@
+Setting Up a Custom Management Network on Multi-Homed Nodes
+===========================================================
+
+If the interface you use for Kubernetes management traffic (for example,
+the IP address used for ``kubeadm join``) is not the one that contains
+the default route out of the host, then you need to specify the
+management node IP address in the Kubelet config file. Add the following
+line to: (``/etc/systemd/system/kubelet.service.d/10-kubeadm.conf``):
+
+::
+
+ Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false --node-ip=<node-management-ip-address>"
+
+Example
+-------
+
+Consider a 2 node deployment where each node is connected to 2 networks
+- ``10.0.2.0/24`` and ``192.168.56.0/24``, and the default route on each
+node points to the interface connected to the ``10.0.2.0/24`` subnet. We
+want to use subnet ``192.168.56.0/24`` for Kubernetes management
+traffic. Assume the addresses of nodes connected to ``192.168.56.0/24``
+are ``192.168.56.105`` and ``192.168.56.106``.
+
+On the ``192.168.56.105`` node you add the following line to
+``10-kubeadm.conf``:
+
+::
+
+ Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false --node-ip=192.168.56.105"
+
+On the ``192.168.56.106`` node you add the following line to
+``10-kubeadm.conf``:
+
+::
+
+ Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false --node-ip=192.168.56.106"
diff --git a/docs/usecases/contiv/K8s_Overview.md b/docs/usecases/contiv/K8s_Overview.md
deleted file mode 100644
index 69f144b..0000000
--- a/docs/usecases/contiv/K8s_Overview.md
+++ /dev/null
@@ -1,109 +0,0 @@
-# Contiv/VPP Kubernetes Network Plugin
-
-
-## Overview
-
-Kubernetes is a container orchestration system that efficiently manages Docker containers. The Docker containers and container platforms provide many advantages over traditional virtualization. Container isolation is done on the kernel level, which eliminates the need for a guest virtual operating system, and therefore makes containers much more efficient, faster, and lightweight. The containers in Contiv/VPP are referred to as PODs.
-
-Contiv/VPP is a Kubernetes network plugin that uses [FD.io VPP](https://fd.io/)
-to provide network connectivity between PODs in a k8s cluster (k8s is an abbreviated reference for kubernetes).
-It deploys itself as a set of system PODs in the `kube-system` namespace,
-some of them (`contiv-ksr`, `contiv-etcd`) on the master node, and some
-of them (`contiv-cni`, `contiv-vswitch`, `contiv-stn`) on each node in the cluster.
-
-Contiv/VPP is fully integrated with k8s via its components,
-and it automatically reprograms itself upon each change in the cluster
-via k8s API.
-
-The main component of the [VPP](https://fd.io/technology/#vpp) solution, which
-runs within the `contiv-vswitch` POD on each node in the cluster. The VPP solution also provides
-POD-to-POD connectivity across the nodes in the cluster, as well as host-to-POD
-and outside-to-POD connectivity. This solution also leverages
-VPP's fast data processing that runs completely in userspace, and uses
-[DPDK](https://dpdk.org/) for fast access to the network IO layer.
-
-Kubernetes services and policies are also a part of the VPP configuration,
-which means they are fully supported on VPP, without the need of forwarding
-packets into the Linux network stack (Kube Proxy), which makes them very
-effective and scalable.
-
-
-## Architecture
-
-Contiv/VPP consists of several components, each of them packed and shipped as
-a Docker container. Two of them deploy on Kubernetes master node only:
-
- - [Contiv KSR](#contiv-ksr)
- - [Contiv ETCD](#contiv-etcd)
-
-The rest of them deploy on all nodes within the k8s cluster (including the master node):
-
-- [Contiv vSwitch](#contiv-vswitch)
-- [Contiv CNI](#contiv-cni)
-- [Contiv STN](#contiv-stn-daemon)
-
-
-The following section briefly describes the individual Contiv components, which are displayed
-as orange boxes on the picture below:
-
-
-
-
-### Contiv KSR
-Contiv KSR (Kubernetes State Reflector) is an agent that subscribes to k8s control plane, watches k8s resources and
-propagates all relevant cluster-related information into the Contiv ETCD data store.
-Other Contiv components do not access the k8s API directly, they subscribe to
-Contiv ETCD instead. For more information on KSR, read the
-[KSR Readme](https://github.com/contiv/vpp/blob/master/cmd/contiv-ksr/README.md).
-
-
-### Contiv ETCD
-Contiv/VPP uses its own instance of the ETCD database for storage of k8s cluster-related data
-reflected by KSR, which are then accessed by Contiv vSwitch Agents running on
-individual nodes. Apart from the data reflected by KSR, ETCD also stores persisted VPP
-configuration of individual vswitches (mainly used to restore the operation after restarts),
-as well as some more internal metadata.
-
-
-### Contiv vSwitch
-vSwitch is the main networking component that provides the connectivity to PODs.
-It deploys on each node in the cluster, and consists of two main components packed
-into a single Docker container: VPP and Contiv VPP Agent.
-
-**VPP** is the data plane software that provides the connectivity between PODs, host Linux
-network stack, and data-plane NIC interface controlled by VPP:
- - PODs are connected to VPP using TAP interfaces wired between VPP, and each POD network namespace.
- - host network stack is connected to VPP using another TAP interface connected
- to the main (default) network namespace.
- - data-plane NIC is controlled directly by VPP using DPDK. Note, this means that
- this interface is not visible to the host Linux network stack, and the node either needs another
- management interface for k8s control plane communication, or
- \[STN (Steal The NIC)\](SINGLE_NIC_SETUP.html) deployment must be applied.
-
-**Contiv VPP Agent** is the control plane part of the vSwitch container. It is responsible
-for configuring the VPP according to the information gained from ETCD, and requests
-from Contiv STN. It is based on the [Ligato VPP Agent](https://github.com/ligato/vpp-agent) code with extensions that are related to k8s.
-
-For communication with VPP, it uses VPP binary API messages sent via shared memory using
-[GoVPP](https://wiki.fd.io/view/GoVPP).
-For connection with Contiv STN, the agent acts as a GRPC server serving CNI requests
-forwarded from the Contiv CNI.
-
-### Contiv CNI
-Contiv CNI (Container Network Interface) is a simple binary that implements the
-[Container Network Interface](https://github.com/containernetworking/cni)
-API and is being executed by Kubelet upon POD creation and deletion. The CNI binary
-just packs the request into a GRPC request and forwards it to the Contiv VPP Agent
-running on the same node, which then processes it (wires/unwires the container)
-and replies with a response, which is then forwarded back to Kubelet.
-
-
-### Contiv STN Daemon
-This section discusses how the Contiv \[STN (Steal The NIC)\](SINGLE_NIC_SETUP.html) daemon operation works. As already mentioned, the default setup of Contiv/VPP requires two network interfaces
-per node: one controlled by VPP for data facing PODs, and one controlled by the host
-network stack for k8s control plane communication. In case that your k8s nodes
-do not provide two network interfaces, Contiv/VPP can work in the single NIC setup,
-when the interface will be "stolen" from the host network stack just before starting
-the VPP and configured with the same IP address on VPP, as well as
-on the host-VPP interconnect TAP interface, as it had in the host before it.
-For more information on STN setup, read the \[Single NIC Setup README\](./SINGLE_NIC_SETUP.html)
diff --git a/docs/usecases/contiv/K8s_Overview.rst b/docs/usecases/contiv/K8s_Overview.rst
new file mode 100644
index 0000000..62e9e10
--- /dev/null
+++ b/docs/usecases/contiv/K8s_Overview.rst
@@ -0,0 +1,144 @@
+Contiv/VPP Kubernetes Network Plugin
+====================================
+
+Overview
+--------
+
+Kubernetes is a container orchestration system that efficiently manages
+Docker containers. The Docker containers and container platforms provide
+many advantages over traditional virtualization. Container isolation is
+done on the kernel level, which eliminates the need for a guest virtual
+operating system, and therefore makes containers much more efficient,
+faster, and lightweight. The containers in Contiv/VPP are referred to as
+PODs.
+
+Contiv/VPP is a Kubernetes network plugin that uses `FD.io
+VPP <https://fd.io/>`__ to provide network connectivity between PODs in
+a k8s cluster (k8s is an abbreviated reference for kubernetes). It
+deploys itself as a set of system PODs in the ``kube-system`` namespace,
+some of them (``contiv-ksr``, ``contiv-etcd``) on the master node, and
+some of them (``contiv-cni``, ``contiv-vswitch``, ``contiv-stn``) on
+each node in the cluster.
+
+Contiv/VPP is fully integrated with k8s via its components, and it
+automatically reprograms itself upon each change in the cluster via k8s
+API.
+
+The main component of the `VPP <https://fd.io/technology/#vpp>`__
+solution, which runs within the ``contiv-vswitch`` POD on each node in
+the cluster. The VPP solution also provides POD-to-POD connectivity
+across the nodes in the cluster, as well as host-to-POD and
+outside-to-POD connectivity. This solution also leverages VPP’s fast
+data processing that runs completely in userspace, and uses
+`DPDK <https://dpdk.org/>`__ for fast access to the network IO layer.
+
+Kubernetes services and policies are also a part of the VPP
+configuration, which means they are fully supported on VPP, without the
+need of forwarding packets into the Linux network stack (Kube Proxy),
+which makes them very effective and scalable.
+
+Architecture
+------------
+
+Contiv/VPP consists of several components, each of them packed and
+shipped as a Docker container. Two of them deploy on Kubernetes master
+node only:
+
+- `Contiv KSR <#contiv-ksr>`__
+- `Contiv ETCD <#contiv-etcd>`__
+
+The rest of them deploy on all nodes within the k8s cluster (including
+the master node):
+
+- `Contiv vSwitch <#contiv-vswitch>`__
+- `Contiv CNI <#contiv-cni>`__
+- `Contiv STN <#contiv-stn-daemon>`__
+
+The following section briefly describes the individual Contiv
+components, which are displayed as orange boxes on the picture below:
+
+.. figure:: ../../_images/contiv-arch.png
+ :alt: Contiv/VPP Architecture
+
+ Contiv/VPP Architecture
+
+Contiv KSR
+~~~~~~~~~~
+
+Contiv KSR (Kubernetes State Reflector) is an agent that subscribes to
+k8s control plane, watches k8s resources and propagates all relevant
+cluster-related information into the Contiv ETCD data store. Other
+Contiv components do not access the k8s API directly, they subscribe to
+Contiv ETCD instead. For more information on KSR, read the `KSR
+Readme <https://github.com/contiv/vpp/blob/master/cmd/contiv-ksr/README.md>`__.
+
+Contiv ETCD
+~~~~~~~~~~~
+
+Contiv/VPP uses its own instance of the ETCD database for storage of k8s
+cluster-related data reflected by KSR, which are then accessed by Contiv
+vSwitch Agents running on individual nodes. Apart from the data
+reflected by KSR, ETCD also stores persisted VPP configuration of
+individual vswitches (mainly used to restore the operation after
+restarts), as well as some more internal metadata.
+
+Contiv vSwitch
+~~~~~~~~~~~~~~
+
+vSwitch is the main networking component that provides the connectivity
+to PODs. It deploys on each node in the cluster, and consists of two
+main components packed into a single Docker container: VPP and Contiv
+VPP Agent.
+
+**VPP** is the data plane software that provides the connectivity
+between PODs, host Linux network stack, and data-plane NIC interface
+controlled by VPP:
+
+- PODs are connected to VPP using TAP interfaces wired between VPP, and
+ each POD network namespace.
+- host network stack is connected to VPP using another TAP interface
+ connected to the main (default) network namespace.
+- data-plane NIC is controlled directly by VPP using DPDK. Note, this
+ means that this interface is not visible to the host Linux network
+ stack, and the node either needs another management interface for k8s
+ control plane communication, or [STN (Steal The
+ NIC)](SINGLE_NIC_SETUP.html) deployment must be applied.
+
+**Contiv VPP Agent** is the control plane part of the vSwitch container.
+It is responsible for configuring the VPP according to the information
+gained from ETCD, and requests from Contiv STN. It is based on the
+`Ligato VPP Agent <https://github.com/ligato/vpp-agent>`__ code with
+extensions that are related to k8s.
+
+For communication with VPP, it uses VPP binary API messages sent via
+shared memory using `GoVPP <https://wiki.fd.io/view/GoVPP>`__. For
+connection with Contiv STN, the agent acts as a GRPC server serving CNI
+requests forwarded from the Contiv CNI.
+
+Contiv CNI
+~~~~~~~~~~
+
+Contiv CNI (Container Network Interface) is a simple binary that
+implements the `Container Network
+Interface <https://github.com/containernetworking/cni>`__ API and is
+being executed by Kubelet upon POD creation and deletion. The CNI binary
+just packs the request into a GRPC request and forwards it to the Contiv
+VPP Agent running on the same node, which then processes it
+(wires/unwires the container) and replies with a response, which is then
+forwarded back to Kubelet.
+
+Contiv STN Daemon
+~~~~~~~~~~~~~~~~~
+
+This section discusses how the Contiv [STN (Steal The
+NIC)](SINGLE_NIC_SETUP.html) daemon operation works. As already
+mentioned, the default setup of Contiv/VPP requires two network
+interfaces per node: one controlled by VPP for data facing PODs, and one
+controlled by the host network stack for k8s control plane
+communication. In case that your k8s nodes do not provide two network
+interfaces, Contiv/VPP can work in the single NIC setup, when the
+interface will be “stolen” from the host network stack just before
+starting the VPP and configured with the same IP address on VPP, as well
+as on the host-VPP interconnect TAP interface, as it had in the host
+before it. For more information on STN setup, read the [Single NIC Setup
+README](./SINGLE_NIC_SETUP.html)
diff --git a/docs/usecases/contiv/MANUAL_INSTALL.md b/docs/usecases/contiv/MANUAL_INSTALL.md
deleted file mode 100644
index 35506db..0000000
--- a/docs/usecases/contiv/MANUAL_INSTALL.md
+++ /dev/null
@@ -1,482 +0,0 @@
-# Manual Installation
-This document describes how to clone the Contiv repository and then use [kubeadm][1] to manually install Kubernetes
-with Contiv-VPP networking on one or more bare metal or VM hosts.
-
-## Clone the Contiv Repository
-To clone the Contiv repository enter the following command:
-```
-git clone https://github.com/contiv/vpp/<repository-name>
-```
-**Note:** Replace *<repository-name>* with the name you want assigned to your cloned contiv repository.
-
-The cloned repository has important folders that contain content that are referenced in this Contiv documentation; those folders are noted below:
-```
-vpp-contiv2$ ls
-build build-root doxygen gmod LICENSE Makefile RELEASE.md src
-build-data docs extras INFO.yaml MAINTAINERS README.md sphinx_venv test
-```
-## Preparing Your Hosts
-
-### Host-specific Configurations
-- **VmWare VMs**: the vmxnet3 driver is required on each interface that will
- be used by VPP. Please see [here][13] for instructions how to install the
- vmxnet3 driver on VmWare Fusion.
-
-### Setting up Network Adapter(s)
-#### Setting up DPDK
-DPDK setup must be completed **on each node** as follows:
-
-- Load the PCI UIO driver:
- ```
- $ sudo modprobe uio_pci_generic
- ```
-
-- Verify that the PCI UIO driver has loaded successfully:
- ```
- $ lsmod | grep uio
- uio_pci_generic 16384 0
- uio 20480 1 uio_pci_generic
- ```
-
- Please note that this driver needs to be loaded upon each server bootup,
- so you may want to add `uio_pci_generic` into the `/etc/modules` file,
- or a file in the `/etc/modules-load.d/` directory. For example, the
- `/etc/modules` file could look as follows:
- ```
- # /etc/modules: kernel modules to load at boot time.
- #
- # This file contains the names of kernel modules that should be loaded
- # at boot time, one per line. Lines beginning with "#" are ignored.
- uio_pci_generic
- ```
-#### Determining Network Adapter PCI Addresses
-You need the PCI address of the network interface that VPP will use for the multi-node pod interconnect. On Debian-based
-distributions, you can use `lshw`(*):
-
-```
-$ sudo lshw -class network -businfo
-Bus info Device Class Description
-====================================================
-pci@0000:00:03.0 ens3 network Virtio network device
-pci@0000:00:04.0 ens4 network Virtio network device
-```
-**Note:** On CentOS/RedHat/Fedora distributions, `lshw` may not be available by default, install it by issuing the following command:
- ```
- yum -y install lshw
- ```
-
-#### Configuring vswitch to Use Network Adapters
-Finally, you need to set up the vswitch to use the network adapters:
-
-- [Setup on a node with a single NIC][14]
-- [Setup a node with multiple NICs][15]
-
-### Using a Node Setup Script
-You can perform the above steps using the [node setup script][17].
-
-## Installing Kubernetes with Contiv-VPP CNI plugin
-After the nodes you will be using in your K8s cluster are prepared, you can
-install the cluster using [kubeadm][1].
-
-### (1/4) Installing Kubeadm on Your Hosts
-For first-time installation, see [Installing kubeadm][6]. To update an
-existing installation, you should do a `apt-get update && apt-get upgrade`
-or `yum update` to get the latest version of kubeadm.
-
-On each host with multiple NICs where the NIC that will be used for Kubernetes
-management traffic is not the one pointed to by the default route out of the
-host, a [custom management network][12] for Kubernetes must be configured.
-
-#### Using Kubernetes 1.10 and Above
-In K8s 1.10, support for huge pages in a pod has been introduced. For now, this
-feature must be either disabled or memory limit must be defined for vswitch container.
-
-To disable huge pages, perform the following
-steps as root:
-* Using your favorite editor, disable huge pages in the kubelet configuration
- file (`/etc/systemd/system/kubelet.service.d/10-kubeadm.conf` or `/etc/default/kubelet` for version 1.11+):
-```
- Environment="KUBELET_EXTRA_ARGS=--feature-gates HugePages=false"
-```
-* Restart the kubelet daemon:
-```
- systemctl daemon-reload
- systemctl restart kubelet
-```
-
-To define memory limit, append the following snippet to vswitch container in deployment yaml file:
-```
- resources:
- limits:
- hugepages-2Mi: 1024Mi
- memory: 1024Mi
-
-```
-or set `contiv.vswitch.defineMemoryLimits` to `true` in [helm values](https://github.com/contiv/vpp/blob/master/k8s/contiv-vpp/README.md).
-
-### (2/4) Initializing Your Master
-Before initializing the master, you may want to [remove][8] any
-previously installed K8s components. Then, proceed with master initialization
-as described in the [kubeadm manual][3]. Execute the following command as
-root:
-```
-kubeadm init --token-ttl 0 --pod-network-cidr=10.1.0.0/16
-```
-**Note:** `kubeadm init` will autodetect the network interface to advertise
-the master on as the interface with the default gateway. If you want to use a
-different interface (i.e. a custom management network setup), specify the
-`--apiserver-advertise-address=<ip-address>` argument to kubeadm init. For
-example:
-```
-kubeadm init --token-ttl 0 --pod-network-cidr=10.1.0.0/16 --apiserver-advertise-address=192.168.56.106
-```
-**Note:** The CIDR specified with the flag `--pod-network-cidr` is used by
-kube-proxy, and it **must include** the `PodSubnetCIDR` from the `IPAMConfig`
-section in the Contiv-vpp config map in Contiv-vpp's deployment file
-[contiv-vpp.yaml](https://github.com/contiv/vpp/blob/master/k8s/contiv-vpp/values.yaml). Pods in the host network namespace
-are a special case; they share their respective interfaces and IP addresses with
-the host. For proxying to work properly it is therefore required for services
-with backends running on the host to also **include the node management IP**
-within the `--pod-network-cidr` subnet. For example, with the default
-`PodSubnetCIDR=10.1.0.0/16` and `PodIfIPCIDR=10.2.1.0/24`, the subnet
-`10.3.0.0/16` could be allocated for the management network and
-`--pod-network-cidr` could be defined as `10.0.0.0/8`, so as to include IP
-addresses of all pods in all network namespaces:
-```
-kubeadm init --token-ttl 0 --pod-network-cidr=10.0.0.0/8 --apiserver-advertise-address=10.3.1.1
-```
-
-If Kubernetes was initialized successfully, it prints out this message:
-```
-Your Kubernetes master has initialized successfully!
-```
-
-After successful initialization, don't forget to set up your .kube directory
-as a regular user (as instructed by `kubeadm`):
-```bash
-mkdir -p $HOME/.kube
-sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
-sudo chown $(id -u):$(id -g) $HOME/.kube/config
-```
-
-### (3/4) Installing the Contiv-VPP Pod Network
-If you have already used the Contiv-VPP plugin before, you may need to pull
-the most recent Docker images on each node:
-```
-bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/pull-images.sh)
-```
-
-Install the Contiv-VPP network for your cluster as follows:
-
-- If you do not use the STN feature, install Contiv-vpp as follows:
- ```
- kubectl apply -f https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml
- ```
-
-- If you use the STN feature, download the `contiv-vpp.yaml` file:
- ```
- wget https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml
- ```
- Then edit the STN configuration as described [here][16]. Finally, create
- the Contiv-vpp deployment from the edited file:
- ```
- kubectl apply -f ./contiv-vpp.yaml
- ```
-
-Beware contiv-etcd data is persisted in `/var/etcd` by default. It has to be cleaned up manually after `kubeadm reset`.
-Otherwise outdated data will be loaded by a subsequent deployment.
-
-You can also generate random subfolder, alternatively:
-
-```
-curl --silent https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml | sed "s/\/var\/etcd\/contiv-data/\/var\/etcd\/contiv-data\/$RANDOM/g" | kubectl apply -f -
-```
-
-#### Deployment Verification
-After some time, all contiv containers should enter the running state:
-```
-root@cvpp:/home/jan# kubectl get pods -n kube-system -o wide | grep contiv
-NAME READY STATUS RESTARTS AGE IP NODE
-...
-contiv-etcd-gwc84 1/1 Running 0 14h 192.168.56.106 cvpp
-contiv-ksr-5c2vk 1/1 Running 2 14h 192.168.56.106 cvpp
-contiv-vswitch-l59nv 2/2 Running 0 14h 192.168.56.106 cvpp
-```
-In particular, make sure that the Contiv-VPP pod IP addresses are the same as
-the IP address specified in the `--apiserver-advertise-address=<ip-address>`
-argument to kubeadm init.
-
-Verify that the VPP successfully grabbed the network interface specified in
-the VPP startup config (`GigabitEthernet0/4/0` in our case):
-```
-$ sudo vppctl
-vpp# sh inter
- Name Idx State Counter Count
-GigabitEthernet0/4/0 1 up rx packets 1294
- rx bytes 153850
- tx packets 512
- tx bytes 21896
- drops 962
- ip4 1032
-host-40df9b44c3d42f4 3 up rx packets 126601
- rx bytes 44628849
- tx packets 132155
- tx bytes 27205450
- drops 24
- ip4 126585
- ip6 16
-host-vppv2 2 up rx packets 132162
- rx bytes 27205824
- tx packets 126658
- tx bytes 44634963
- drops 15
- ip4 132147
- ip6 14
-local0 0 down
-```
-
-You should also see the interface to kube-dns (`host-40df9b44c3d42f4`) and to the
-node's IP stack (`host-vppv2`).
-
-#### Master Isolation (Optional)
-By default, your cluster will not schedule pods on the master for security
-reasons. If you want to be able to schedule pods on the master, (e.g., for a
-single-machine Kubernetes cluster for development), then run:
-
-```
-kubectl taint nodes --all node-role.kubernetes.io/master-
-```
-More details about installing the pod network can be found in the
-[kubeadm manual][4].
-
-### (4/4) Joining Your Nodes
-To add a new node to your cluster, run as root the command that was output
-by kubeadm init. For example:
-```
-kubeadm join --token <token> <master-ip>:<master-port> --discovery-token-ca-cert-hash sha256:<hash>
-```
-More details can be found int the [kubeadm manual][5].
-
-#### Deployment Verification
-After some time, all contiv containers should enter the running state:
-```
-root@cvpp:/home/jan# kubectl get pods -n kube-system -o wide | grep contiv
-NAME READY STATUS RESTARTS AGE IP NODE
-contiv-etcd-gwc84 1/1 Running 0 14h 192.168.56.106 cvpp
-contiv-ksr-5c2vk 1/1 Running 2 14h 192.168.56.106 cvpp
-contiv-vswitch-h6759 2/2 Running 0 14h 192.168.56.105 cvpp-slave2
-contiv-vswitch-l59nv 2/2 Running 0 14h 192.168.56.106 cvpp
-etcd-cvpp 1/1 Running 0 14h 192.168.56.106 cvpp
-kube-apiserver-cvpp 1/1 Running 0 14h 192.168.56.106 cvpp
-kube-controller-manager-cvpp 1/1 Running 0 14h 192.168.56.106 cvpp
-kube-dns-545bc4bfd4-fr6j9 3/3 Running 0 14h 10.1.134.2 cvpp
-kube-proxy-q8sv2 1/1 Running 0 14h 192.168.56.106 cvpp
-kube-proxy-s8kv9 1/1 Running 0 14h 192.168.56.105 cvpp-slave2
-kube-scheduler-cvpp 1/1 Running 0 14h 192.168.56.106 cvpp
-```
-In particular, verify that a vswitch pod and a kube-proxy pod is running on
-each joined node, as shown above.
-
-On each joined node, verify that the VPP successfully grabbed the network
-interface specified in the VPP startup config (`GigabitEthernet0/4/0` in
-our case):
-```
-$ sudo vppctl
-vpp# sh inter
- Name Idx State Counter Count
-GigabitEthernet0/4/0 1 up
-...
-```
-From the vpp CLI on a joined node you can also ping kube-dns to verify
-node-to-node connectivity. For example:
-```
-vpp# ping 10.1.134.2
-64 bytes from 10.1.134.2: icmp_seq=1 ttl=64 time=.1557 ms
-64 bytes from 10.1.134.2: icmp_seq=2 ttl=64 time=.1339 ms
-64 bytes from 10.1.134.2: icmp_seq=3 ttl=64 time=.1295 ms
-64 bytes from 10.1.134.2: icmp_seq=4 ttl=64 time=.1714 ms
-64 bytes from 10.1.134.2: icmp_seq=5 ttl=64 time=.1317 ms
-
-Statistics: 5 sent, 5 received, 0% packet loss
-```
-### Deploying Example Applications
-#### Simple Deployment
-You can go ahead and create a simple deployment:
-```
-$ kubectl run nginx --image=nginx --replicas=2
-```
-
-Use `kubectl describe pod` to get the IP address of a pod, e.g.:
-```
-$ kubectl describe pod nginx | grep IP
-```
-You should see two ip addresses, for example:
-```
-IP: 10.1.1.3
-IP: 10.1.1.4
-```
-
-You can check the pods' connectivity in one of the following ways:
-* Connect to the VPP debug CLI and ping any pod:
-```
- sudo vppctl
- vpp# ping 10.1.1.3
-```
-* Start busybox and ping any pod:
-```
- kubectl run busybox --rm -ti --image=busybox /bin/sh
- If you don't see a command prompt, try pressing enter.
- / #
- / # ping 10.1.1.3
-
-```
-* You should be able to ping any pod from the host:
-```
- ping 10.1.1.3
-```
-
-#### Deploying Pods on Different Nodes
-to enable pod deployment on the master, untaint the master first:
-```
-kubectl taint nodes --all node-role.kubernetes.io/master-
-```
-
-In order to verify inter-node pod connectivity, we need to tell Kubernetes
-to deploy one pod on the master node and one POD on the worker. For this,
-we can use node selectors.
-
-In your deployment YAMLs, add the `nodeSelector` sections that refer to
-preferred node hostnames, e.g.:
-```
- nodeSelector:
- kubernetes.io/hostname: vm5
-```
-
-Example of whole JSONs:
-```
-apiVersion: v1
-kind: Pod
-metadata:
- name: nginx1
-spec:
- nodeSelector:
- kubernetes.io/hostname: vm5
- containers:
- - name: nginx
-
- : nginx
-```
-
-```
-apiVersion: v1
-kind: Pod
-metadata:
- name: nginx2
-spec:
- nodeSelector:
- kubernetes.io/hostname: vm6
- containers:
- - name: nginx
- image: nginx
-```
-
-After deploying the JSONs, verify they were deployed on different hosts:
-```
-$ kubectl get pods -o wide
-NAME READY STATUS RESTARTS AGE IP NODE
-nginx1 1/1 Running 0 13m 10.1.36.2 vm5
-nginx2 1/1 Running 0 13m 10.1.219.3 vm6
-```
-
-Now you can verify the connectivity to both nginx PODs from a busybox POD:
-```
-kubectl run busybox --rm -it --image=busybox /bin/sh
-
-/ # wget 10.1.36.2
-Connecting to 10.1.36.2 (10.1.36.2:80)
-index.html 100% |*******************************************************************************************************************************************************************| 612 0:00:00 ETA
-
-/ # rm index.html
-
-/ # wget 10.1.219.3
-Connecting to 10.1.219.3 (10.1.219.3:80)
-index.html 100% |*******************************************************************************************************************************************************************| 612 0:00:00 ETA
-```
-
-### Uninstalling Contiv-VPP
-To uninstall the network plugin itself, use `kubectl`:
-```
-kubectl delete -f https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml
-```
-
-### Tearing down Kubernetes
-* First, drain the node and make sure that the node is empty before
-shutting it down:
-```
- kubectl drain <node name> --delete-local-data --force --ignore-daemonsets
- kubectl delete node <node name>
-```
-* Next, on the node being removed, reset all kubeadm installed state:
-```
- rm -rf $HOME/.kube
- sudo su
- kubeadm reset
-```
-
-* If you added environment variable definitions into
- `/etc/systemd/system/kubelet.service.d/10-kubeadm.conf`, this would have been a process from the [Custom Management Network file][10], then remove the definitions now.
-
-### Troubleshooting
-Some of the issues that can occur during the installation are:
-
-- Forgetting to create and initialize the `.kube` directory in your home
- directory (As instructed by `kubeadm init --token-ttl 0`). This can manifest
- itself as the following error:
- ```
- W1017 09:25:43.403159 2233 factory_object_mapping.go:423] Failed to download OpenAPI (Get https://192.168.209.128:6443/swagger-2.0.0.pb-v1: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "kubernetes")), falling back to swagger
- Unable to connect to the server: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "kubernetes")
- ```
-- Previous installation lingering on the file system.
- `'kubeadm init --token-ttl 0` fails to initialize kubelet with one or more
- of the following error messages:
- ```
- ...
- [kubelet-check] It seems like the kubelet isn't running or healthy.
- [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10255/healthz' failed with error: Get http://localhost:10255/healthz: dial tcp [::1]:10255: getsockopt: connection refused.
- ...
- ```
-
-If you run into any of the above issues, try to clean up and reinstall as root:
-```
-sudo su
-rm -rf $HOME/.kube
-kubeadm reset
-kubeadm init --token-ttl 0
-rm -rf /var/etcd/contiv-data
-rm -rf /var/bolt/bolt.db
-```
-
-## Contiv-specific kubeadm installation on Aarch64
-Supplemental instructions apply when using Contiv-VPP for Aarch64. Most
-installation steps for Aarch64 are the same as that described earlier in this
-chapter, so you should firstly read it before you start the installation on
-Aarch64 platform.
-
-Use the [Aarch64-specific kubeadm install instructions][18] to manually install
-Kubernetes with Contiv-VPP networking on one or more bare-metals of Aarch64 platform.
-
-[1]: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
-[3]: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#initializing-your-master
-[4]: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#pod-network
-[5]: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#joining-your-nodes
-[6]: https://kubernetes.io/docs/setup/independent/install-kubeadm/
-[8]: #tearing-down-kubernetes
-[10]: https://github.com/contiv/vpp/blob/master/docs/CUSTOM_MGMT_NETWORK.md#setting-up-a-custom-management-network-on-multi-homed-nodes
-[11]: ../vagrant/README.md
-[12]: https://github.com/contiv/vpp/tree/master/docs/CUSTOM_MGMT_NETWORK.md
-[13]: https://github.com/contiv/vpp/tree/master/docs/VMWARE_FUSION_HOST.md
-[14]: https://github.com/contiv/vpp/tree/master/docs/SINGLE_NIC_SETUP.md
-[15]: https://github.com/contiv/vpp/tree/master/docs/MULTI_NIC_SETUP.md
-[16]: https://github.com/contiv/vpp/tree/master/docs/SINGLE_NIC_SETUP.md#configuring-stn-in-contiv-vpp-k8s-deployment-files
-[17]: https://github.com/contiv/vpp/tree/master/k8s/README.md#setup-node-sh
-[18]: https://github.com/contiv/vpp/blob/master/docs/arm64/MANUAL_INSTALL_ARM64.md
diff --git a/docs/usecases/contiv/MANUAL_INSTALL.rst b/docs/usecases/contiv/MANUAL_INSTALL.rst
new file mode 100644
index 0000000..4e0d0c6
--- /dev/null
+++ b/docs/usecases/contiv/MANUAL_INSTALL.rst
@@ -0,0 +1,609 @@
+Manual Installation
+===================
+
+This document describes how to clone the Contiv repository and then use
+`kubeadm <https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/>`__
+to manually install Kubernetes with Contiv-VPP networking on one or more
+bare metal or VM hosts.
+
+Clone the Contiv Repository
+---------------------------
+
+To clone the Contiv repository enter the following command:
+
+::
+
+ git clone https://github.com/contiv/vpp/<repository-name>
+
+**Note:** Replace ** with the name you want assigned to your cloned
+contiv repository.
+
+The cloned repository has important folders that contain content that
+are referenced in this Contiv documentation; those folders are noted
+below:
+
+::
+
+ vpp-contiv2$ ls
+ build build-root doxygen gmod LICENSE Makefile RELEASE.md src
+ build-data docs extras INFO.yaml MAINTAINERS README.md sphinx_venv test
+
+Preparing Your Hosts
+--------------------
+
+Host-specific Configurations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- **VmWare VMs**: the vmxnet3 driver is required on each interface that
+ will be used by VPP. Please see
+ `here <https://github.com/contiv/vpp/tree/master/docs/VMWARE_FUSION_HOST.md>`__
+ for instructions how to install the vmxnet3 driver on VmWare Fusion.
+
+Setting up Network Adapter(s)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Setting up DPDK
+^^^^^^^^^^^^^^^
+
+DPDK setup must be completed **on each node** as follows:
+
+- Load the PCI UIO driver:
+
+ ::
+
+ $ sudo modprobe uio_pci_generic
+
+- Verify that the PCI UIO driver has loaded successfully:
+
+ ::
+
+ $ lsmod | grep uio
+ uio_pci_generic 16384 0
+ uio 20480 1 uio_pci_generic
+
+ Please note that this driver needs to be loaded upon each server
+ bootup, so you may want to add ``uio_pci_generic`` into the
+ ``/etc/modules`` file, or a file in the ``/etc/modules-load.d/``
+ directory. For example, the ``/etc/modules`` file could look as
+ follows:
+
+ ::
+
+ # /etc/modules: kernel modules to load at boot time.
+ #
+ # This file contains the names of kernel modules that should be loaded
+ # at boot time, one per line. Lines beginning with "#" are ignored.
+ uio_pci_generic
+
+ .. rubric:: Determining Network Adapter PCI Addresses
+ :name: determining-network-adapter-pci-addresses
+
+ You need the PCI address of the network interface that VPP will use
+ for the multi-node pod interconnect. On Debian-based distributions,
+ you can use ``lshw``\ (*):
+
+::
+
+ $ sudo lshw -class network -businfo
+ Bus info Device Class Description
+ ====================================================
+ pci@0000:00:03.0 ens3 network Virtio network device
+ pci@0000:00:04.0 ens4 network Virtio network device
+
+**Note:** On CentOS/RedHat/Fedora distributions, ``lshw`` may not be
+available by default, install it by issuing the following command:
+``yum -y install lshw``
+
+Configuring vswitch to Use Network Adapters
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Finally, you need to set up the vswitch to use the network adapters:
+
+- `Setup on a node with a single
+ NIC <https://github.com/contiv/vpp/tree/master/docs/SINGLE_NIC_SETUP.md>`__
+- `Setup a node with multiple
+ NICs <https://github.com/contiv/vpp/tree/master/docs/MULTI_NIC_SETUP.md>`__
+
+Using a Node Setup Script
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can perform the above steps using the `node setup
+script <https://github.com/contiv/vpp/tree/master/k8s/README.md#setup-node-sh>`__.
+
+Installing Kubernetes with Contiv-VPP CNI plugin
+------------------------------------------------
+
+After the nodes you will be using in your K8s cluster are prepared, you
+can install the cluster using
+`kubeadm <https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/>`__.
+
+(1/4) Installing Kubeadm on Your Hosts
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For first-time installation, see `Installing
+kubeadm <https://kubernetes.io/docs/setup/independent/install-kubeadm/>`__.
+To update an existing installation, you should do a
+``apt-get update && apt-get upgrade`` or ``yum update`` to get the
+latest version of kubeadm.
+
+On each host with multiple NICs where the NIC that will be used for
+Kubernetes management traffic is not the one pointed to by the default
+route out of the host, a `custom management
+network <https://github.com/contiv/vpp/tree/master/docs/CUSTOM_MGMT_NETWORK.md>`__
+for Kubernetes must be configured.
+
+Using Kubernetes 1.10 and Above
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In K8s 1.10, support for huge pages in a pod has been introduced. For
+now, this feature must be either disabled or memory limit must be
+defined for vswitch container.
+
+To disable huge pages, perform the following steps as root: \* Using
+your favorite editor, disable huge pages in the kubelet configuration
+file (``/etc/systemd/system/kubelet.service.d/10-kubeadm.conf`` or
+``/etc/default/kubelet`` for version 1.11+):
+
+::
+
+ Environment="KUBELET_EXTRA_ARGS=--feature-gates HugePages=false"
+
+- Restart the kubelet daemon:
+
+::
+
+ systemctl daemon-reload
+ systemctl restart kubelet
+
+To define memory limit, append the following snippet to vswitch
+container in deployment yaml file:
+
+::
+
+ resources:
+ limits:
+ hugepages-2Mi: 1024Mi
+ memory: 1024Mi
+
+or set ``contiv.vswitch.defineMemoryLimits`` to ``true`` in `helm
+values <https://github.com/contiv/vpp/blob/master/k8s/contiv-vpp/README.md>`__.
+
+(2/4) Initializing Your Master
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before initializing the master, you may want to
+`remove <#tearing-down-kubernetes>`__ any previously installed K8s
+components. Then, proceed with master initialization as described in the
+`kubeadm
+manual <https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#initializing-your-master>`__.
+Execute the following command as root:
+
+::
+
+ kubeadm init --token-ttl 0 --pod-network-cidr=10.1.0.0/16
+
+**Note:** ``kubeadm init`` will autodetect the network interface to
+advertise the master on as the interface with the default gateway. If
+you want to use a different interface (i.e. a custom management network
+setup), specify the ``--apiserver-advertise-address=<ip-address>``
+argument to kubeadm init. For example:
+
+::
+
+ kubeadm init --token-ttl 0 --pod-network-cidr=10.1.0.0/16 --apiserver-advertise-address=192.168.56.106
+
+**Note:** The CIDR specified with the flag ``--pod-network-cidr`` is
+used by kube-proxy, and it **must include** the ``PodSubnetCIDR`` from
+the ``IPAMConfig`` section in the Contiv-vpp config map in Contiv-vpp’s
+deployment file
+`contiv-vpp.yaml <https://github.com/contiv/vpp/blob/master/k8s/contiv-vpp/values.yaml>`__.
+Pods in the host network namespace are a special case; they share their
+respective interfaces and IP addresses with the host. For proxying to
+work properly it is therefore required for services with backends
+running on the host to also **include the node management IP** within
+the ``--pod-network-cidr`` subnet. For example, with the default
+``PodSubnetCIDR=10.1.0.0/16`` and ``PodIfIPCIDR=10.2.1.0/24``, the
+subnet ``10.3.0.0/16`` could be allocated for the management network and
+``--pod-network-cidr`` could be defined as ``10.0.0.0/8``, so as to
+include IP addresses of all pods in all network namespaces:
+
+::
+
+ kubeadm init --token-ttl 0 --pod-network-cidr=10.0.0.0/8 --apiserver-advertise-address=10.3.1.1
+
+If Kubernetes was initialized successfully, it prints out this message:
+
+::
+
+ Your Kubernetes master has initialized successfully!
+
+After successful initialization, don’t forget to set up your .kube
+directory as a regular user (as instructed by ``kubeadm``):
+
+.. code:: bash
+
+ mkdir -p $HOME/.kube
+ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
+ sudo chown $(id -u):$(id -g) $HOME/.kube/config
+
+(3/4) Installing the Contiv-VPP Pod Network
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you have already used the Contiv-VPP plugin before, you may need to
+pull the most recent Docker images on each node:
+
+::
+
+ bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/pull-images.sh)
+
+Install the Contiv-VPP network for your cluster as follows:
+
+- If you do not use the STN feature, install Contiv-vpp as follows:
+
+ ::
+
+ kubectl apply -f https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml
+
+- If you use the STN feature, download the ``contiv-vpp.yaml`` file:
+
+ ::
+
+ wget https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml
+
+ Then edit the STN configuration as described
+ `here <https://github.com/contiv/vpp/tree/master/docs/SINGLE_NIC_SETUP.md#configuring-stn-in-contiv-vpp-k8s-deployment-files>`__.
+ Finally, create the Contiv-vpp deployment from the edited file:
+
+ ::
+
+ kubectl apply -f ./contiv-vpp.yaml
+
+Beware contiv-etcd data is persisted in ``/var/etcd`` by default. It has
+to be cleaned up manually after ``kubeadm reset``. Otherwise outdated
+data will be loaded by a subsequent deployment.
+
+You can also generate random subfolder, alternatively:
+
+::
+
+ curl --silent https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml | sed "s/\/var\/etcd\/contiv-data/\/var\/etcd\/contiv-data\/$RANDOM/g" | kubectl apply -f -
+
+Deployment Verification
+^^^^^^^^^^^^^^^^^^^^^^^
+
+After some time, all contiv containers should enter the running state:
+
+::
+
+ root@cvpp:/home/jan# kubectl get pods -n kube-system -o wide | grep contiv
+ NAME READY STATUS RESTARTS AGE IP NODE
+ ...
+ contiv-etcd-gwc84 1/1 Running 0 14h 192.168.56.106 cvpp
+ contiv-ksr-5c2vk 1/1 Running 2 14h 192.168.56.106 cvpp
+ contiv-vswitch-l59nv 2/2 Running 0 14h 192.168.56.106 cvpp
+
+In particular, make sure that the Contiv-VPP pod IP addresses are the
+same as the IP address specified in the
+``--apiserver-advertise-address=<ip-address>`` argument to kubeadm init.
+
+Verify that the VPP successfully grabbed the network interface specified
+in the VPP startup config (``GigabitEthernet0/4/0`` in our case):
+
+::
+
+ $ sudo vppctl
+ vpp# sh inter
+ Name Idx State Counter Count
+ GigabitEthernet0/4/0 1 up rx packets 1294
+ rx bytes 153850
+ tx packets 512
+ tx bytes 21896
+ drops 962
+ ip4 1032
+ host-40df9b44c3d42f4 3 up rx packets 126601
+ rx bytes 44628849
+ tx packets 132155
+ tx bytes 27205450
+ drops 24
+ ip4 126585
+ ip6 16
+ host-vppv2 2 up rx packets 132162
+ rx bytes 27205824
+ tx packets 126658
+ tx bytes 44634963
+ drops 15
+ ip4 132147
+ ip6 14
+ local0 0 down
+
+You should also see the interface to kube-dns (``host-40df9b44c3d42f4``)
+and to the node’s IP stack (``host-vppv2``).
+
+Master Isolation (Optional)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+By default, your cluster will not schedule pods on the master for
+security reasons. If you want to be able to schedule pods on the master,
+(e.g., for a single-machine Kubernetes cluster for development), then
+run:
+
+::
+
+ kubectl taint nodes --all node-role.kubernetes.io/master-
+
+More details about installing the pod network can be found in the
+`kubeadm
+manual <https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#pod-network>`__.
+
+(4/4) Joining Your Nodes
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To add a new node to your cluster, run as root the command that was
+output by kubeadm init. For example:
+
+::
+
+ kubeadm join --token <token> <master-ip>:<master-port> --discovery-token-ca-cert-hash sha256:<hash>
+
+More details can be found int the `kubeadm
+manual <https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#joining-your-nodes>`__.
+
+.. _deployment-verification-1:
+
+Deployment Verification
+^^^^^^^^^^^^^^^^^^^^^^^
+
+After some time, all contiv containers should enter the running state:
+
+::
+
+ root@cvpp:/home/jan# kubectl get pods -n kube-system -o wide | grep contiv
+ NAME READY STATUS RESTARTS AGE IP NODE
+ contiv-etcd-gwc84 1/1 Running 0 14h 192.168.56.106 cvpp
+ contiv-ksr-5c2vk 1/1 Running 2 14h 192.168.56.106 cvpp
+ contiv-vswitch-h6759 2/2 Running 0 14h 192.168.56.105 cvpp-slave2
+ contiv-vswitch-l59nv 2/2 Running 0 14h 192.168.56.106 cvpp
+ etcd-cvpp 1/1 Running 0 14h 192.168.56.106 cvpp
+ kube-apiserver-cvpp 1/1 Running 0 14h 192.168.56.106 cvpp
+ kube-controller-manager-cvpp 1/1 Running 0 14h 192.168.56.106 cvpp
+ kube-dns-545bc4bfd4-fr6j9 3/3 Running 0 14h 10.1.134.2 cvpp
+ kube-proxy-q8sv2 1/1 Running 0 14h 192.168.56.106 cvpp
+ kube-proxy-s8kv9 1/1 Running 0 14h 192.168.56.105 cvpp-slave2
+ kube-scheduler-cvpp 1/1 Running 0 14h 192.168.56.106 cvpp
+
+In particular, verify that a vswitch pod and a kube-proxy pod is running
+on each joined node, as shown above.
+
+On each joined node, verify that the VPP successfully grabbed the
+network interface specified in the VPP startup config
+(``GigabitEthernet0/4/0`` in our case):
+
+::
+
+ $ sudo vppctl
+ vpp# sh inter
+ Name Idx State Counter Count
+ GigabitEthernet0/4/0 1 up
+ ...
+
+From the vpp CLI on a joined node you can also ping kube-dns to verify
+node-to-node connectivity. For example:
+
+::
+
+ vpp# ping 10.1.134.2
+ 64 bytes from 10.1.134.2: icmp_seq=1 ttl=64 time=.1557 ms
+ 64 bytes from 10.1.134.2: icmp_seq=2 ttl=64 time=.1339 ms
+ 64 bytes from 10.1.134.2: icmp_seq=3 ttl=64 time=.1295 ms
+ 64 bytes from 10.1.134.2: icmp_seq=4 ttl=64 time=.1714 ms
+ 64 bytes from 10.1.134.2: icmp_seq=5 ttl=64 time=.1317 ms
+
+ Statistics: 5 sent, 5 received, 0% packet loss
+
+Deploying Example Applications
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Simple Deployment
+^^^^^^^^^^^^^^^^^
+
+You can go ahead and create a simple deployment:
+
+::
+
+ $ kubectl run nginx --image=nginx --replicas=2
+
+Use ``kubectl describe pod`` to get the IP address of a pod, e.g.:
+
+::
+
+ $ kubectl describe pod nginx | grep IP
+
+You should see two ip addresses, for example:
+
+::
+
+ IP: 10.1.1.3
+ IP: 10.1.1.4
+
+You can check the pods’ connectivity in one of the following ways: \*
+Connect to the VPP debug CLI and ping any pod:
+
+::
+
+ sudo vppctl
+ vpp# ping 10.1.1.3
+
+- Start busybox and ping any pod:
+
+::
+
+ kubectl run busybox --rm -ti --image=busybox /bin/sh
+ If you don't see a command prompt, try pressing enter.
+ / #
+ / # ping 10.1.1.3
+
+- You should be able to ping any pod from the host:
+
+::
+
+ ping 10.1.1.3
+
+Deploying Pods on Different Nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+to enable pod deployment on the master, untaint the master first:
+
+::
+
+ kubectl taint nodes --all node-role.kubernetes.io/master-
+
+In order to verify inter-node pod connectivity, we need to tell
+Kubernetes to deploy one pod on the master node and one POD on the
+worker. For this, we can use node selectors.
+
+In your deployment YAMLs, add the ``nodeSelector`` sections that refer
+to preferred node hostnames, e.g.:
+
+::
+
+ nodeSelector:
+ kubernetes.io/hostname: vm5
+
+Example of whole JSONs:
+
+::
+
+ apiVersion: v1
+ kind: Pod
+ metadata:
+ name: nginx1
+ spec:
+ nodeSelector:
+ kubernetes.io/hostname: vm5
+ containers:
+ - name: nginx
+
+ : nginx
+
+::
+
+ apiVersion: v1
+ kind: Pod
+ metadata:
+ name: nginx2
+ spec:
+ nodeSelector:
+ kubernetes.io/hostname: vm6
+ containers:
+ - name: nginx
+ image: nginx
+
+After deploying the JSONs, verify they were deployed on different hosts:
+
+::
+
+ $ kubectl get pods -o wide
+ NAME READY STATUS RESTARTS AGE IP NODE
+ nginx1 1/1 Running 0 13m 10.1.36.2 vm5
+ nginx2 1/1 Running 0 13m 10.1.219.3 vm6
+
+Now you can verify the connectivity to both nginx PODs from a busybox
+POD:
+
+::
+
+ kubectl run busybox --rm -it --image=busybox /bin/sh
+
+ / # wget 10.1.36.2
+ Connecting to 10.1.36.2 (10.1.36.2:80)
+ index.html 100% |*******************************************************************************************************************************************************************| 612 0:00:00 ETA
+
+ / # rm index.html
+
+ / # wget 10.1.219.3
+ Connecting to 10.1.219.3 (10.1.219.3:80)
+ index.html 100% |*******************************************************************************************************************************************************************| 612 0:00:00 ETA
+
+Uninstalling Contiv-VPP
+~~~~~~~~~~~~~~~~~~~~~~~
+
+To uninstall the network plugin itself, use ``kubectl``:
+
+::
+
+ kubectl delete -f https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml
+
+Tearing down Kubernetes
+~~~~~~~~~~~~~~~~~~~~~~~
+
+- First, drain the node and make sure that the node is empty before
+ shutting it down:
+
+::
+
+ kubectl drain <node name> --delete-local-data --force --ignore-daemonsets
+ kubectl delete node <node name>
+
+- Next, on the node being removed, reset all kubeadm installed state:
+
+::
+
+ rm -rf $HOME/.kube
+ sudo su
+ kubeadm reset
+
+- If you added environment variable definitions into
+ ``/etc/systemd/system/kubelet.service.d/10-kubeadm.conf``, this would
+ have been a process from the `Custom Management Network
+ file <https://github.com/contiv/vpp/blob/master/docs/CUSTOM_MGMT_NETWORK.md#setting-up-a-custom-management-network-on-multi-homed-nodes>`__,
+ then remove the definitions now.
+
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+Some of the issues that can occur during the installation are:
+
+- Forgetting to create and initialize the ``.kube`` directory in your
+ home directory (As instructed by ``kubeadm init --token-ttl 0``).
+ This can manifest itself as the following error:
+
+ ::
+
+ W1017 09:25:43.403159 2233 factory_object_mapping.go:423] Failed to download OpenAPI (Get https://192.168.209.128:6443/swagger-2.0.0.pb-v1: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "kubernetes")), falling back to swagger
+ Unable to connect to the server: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "kubernetes")
+
+- Previous installation lingering on the file system.
+ ``'kubeadm init --token-ttl 0`` fails to initialize kubelet with one
+ or more of the following error messages:
+
+ ::
+
+ ...
+ [kubelet-check] It seems like the kubelet isn't running or healthy.
+ [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10255/healthz' failed with error: Get http://localhost:10255/healthz: dial tcp [::1]:10255: getsockopt: connection refused.
+ ...
+
+If you run into any of the above issues, try to clean up and reinstall
+as root:
+
+::
+
+ sudo su
+ rm -rf $HOME/.kube
+ kubeadm reset
+ kubeadm init --token-ttl 0
+ rm -rf /var/etcd/contiv-data
+ rm -rf /var/bolt/bolt.db
+
+Contiv-specific kubeadm installation on Aarch64
+-----------------------------------------------
+
+Supplemental instructions apply when using Contiv-VPP for Aarch64. Most
+installation steps for Aarch64 are the same as that described earlier in
+this chapter, so you should firstly read it before you start the
+installation on Aarch64 platform.
+
+Use the `Aarch64-specific kubeadm install
+instructions <https://github.com/contiv/vpp/blob/master/docs/arm64/MANUAL_INSTALL_ARM64.md>`__
+to manually install Kubernetes with Contiv-VPP networking on one or more
+bare-metals of Aarch64 platform.
diff --git a/docs/usecases/contiv/MULTI_NIC_SETUP.md b/docs/usecases/contiv/MULTI_NIC_SETUP.md
deleted file mode 100644
index cacbcbb..0000000
--- a/docs/usecases/contiv/MULTI_NIC_SETUP.md
+++ /dev/null
@@ -1,21 +0,0 @@
-### Setting Up a Node with Multiple NICs
-
-* First, configure hardware interfaces in the VPP startup config, as
-described [here](https://github.com/contiv/vpp/blob/master/docs/VPP_CONFIG.md#multi-nic-configuration).
-
-* For each interface owned by Linux, you need to provide individual
- configuration for each interface used by VPP in the Node Configuration
- for the node in the `contiv-vpp.yaml`. For example, if both `ens3` and
- `ens4` are known to Linux, then put the following stanza into the node's
- NodeConfig:
-```
-...
- NodeConfig:
- - NodeName: "ubuntu-1"
- StealInterface: "ens3"
- StealInterface: "ens4"
-...
-```
- If only `ens3` is known to Linux, you only put a line for `ens3` into the
- above NodeConfig.
-
diff --git a/docs/usecases/contiv/MULTI_NIC_SETUP.rst b/docs/usecases/contiv/MULTI_NIC_SETUP.rst
new file mode 100644
index 0000000..57242d1
--- /dev/null
+++ b/docs/usecases/contiv/MULTI_NIC_SETUP.rst
@@ -0,0 +1,24 @@
+Setting Up a Node with Multiple NICs
+====================================
+
+- First, configure hardware interfaces in the VPP startup config, as
+ described
+ `here <https://github.com/contiv/vpp/blob/master/docs/VPP_CONFIG.md#multi-nic-configuration>`__.
+
+- For each interface owned by Linux, you need to provide individual
+ configuration for each interface used by VPP in the Node
+ Configuration for the node in the ``contiv-vpp.yaml``. For example,
+ if both ``ens3`` and ``ens4`` are known to Linux, then put the
+ following stanza into the node’s NodeConfig:
+
+::
+
+ ...
+ NodeConfig:
+ - NodeName: "ubuntu-1"
+ StealInterface: "ens3"
+ StealInterface: "ens4"
+ ...
+
+If only ``ens3`` is known to Linux, you only put a line for ``ens3``
+into the above NodeConfig.
diff --git a/docs/usecases/contiv/NETWORKING.md b/docs/usecases/contiv/NETWORKING.md
deleted file mode 100644
index 25ce3ce..0000000
--- a/docs/usecases/contiv/NETWORKING.md
+++ /dev/null
@@ -1,137 +0,0 @@
-# Contiv/VPP Network Operation
-
-This document describes the network operation of the Contiv/VPP k8s network plugin. It
-elaborates the operation and config options of the Contiv IPAM, as well as
-details on how the VPP gets programmed by Contiv/VPP control plane.
-
-The following picture shows 2-node k8s deployment of Contiv/VPP, with a VXLAN tunnel
-established between the nodes to forward inter-node POD traffic. The IPAM options
-are depicted on the Node 1, whereas the VPP programming is depicted on the Node 2.
-
-
-
-## Contiv/VPP IPAM (IP Address Management)
-
-IPAM in Contiv/VPP is based on the concept of **Node ID**. The Node ID is a number
-that uniquely identifies a node in the k8s cluster. The first node is assigned
-the ID of 1, the second node 2, etc. If a node leaves the cluster, its
-ID is released back to the pool and will be re-used by the next node.
-
-The Node ID is used to calculate per-node IP subnets for PODs
-and other internal subnets that need to be unique on each node. Apart from the Node ID,
-the input for IPAM calculations is a set of config knobs, which can be specified
-in the `IPAMConfig` section of the \[Contiv/VPP deployment YAML\](../../../k8s/contiv-vpp.yaml):
-
-- **PodSubnetCIDR** (default `10.1.0.0/16`): each pod gets an IP address assigned
-from this range. The size of this range (default `/16`) dictates upper limit of
-POD count for the entire k8s cluster (default 65536 PODs).
-
-- **PodNetworkPrefixLen** (default `24`): per-node dedicated podSubnet range.
-From the allocatable range defined in `PodSubnetCIDR`, this value will dictate the
-allocation for each node. With the default value (`24`) this indicates that each node
-has a `/24` slice of the `PodSubnetCIDR`. The Node ID is used to address the node.
-In case of `PodSubnetCIDR = 10.1.0.0/16`, `PodNetworkPrefixLen = 24` and `NodeID = 5`,
-the resulting POD subnet for the node would be `10.1.5.0/24`.
-
-- **PodIfIPCIDR** (default `10.2.1.0/24`): VPP-internal addresses put the VPP interfaces
-facing towards the PODs into L3 mode. This IP range will be reused
-on each node, thereby it is never externally addressable outside of the node itself.
-The only requirement is that this subnet should not collide with any other IPAM subnet.
-
-- **VPPHostSubnetCIDR** (default `172.30.0.0/16`): used for addressing
-the interconnect of VPP with the Linux network stack, within the same node.
-Since this subnet needs to be unique on each node, the Node ID is used to determine
-the actual subnet used on the node with the combination of `VPPHostNetworkPrefixLen`, `PodSubnetCIDR` and `PodNetworkPrefixLen`.
-
-- **VPPHostNetworkPrefixLen** (default `24`): used to calculate the subnet
-for addressing the interconnect of VPP with the Linux network stack, within the same node.
-With `VPPHostSubnetCIDR = 172.30.0.0/16`, `VPPHostNetworkPrefixLen = 24` and
-`NodeID = 5` the resulting subnet for the node would be `172.30.5.0/24`.
-
-- **NodeInterconnectCIDR** (default `192.168.16.0/24`): range for the addresses
-assigned to the data plane interfaces managed by VPP. Unless DHCP is used
-(`NodeInterconnectDHCP = True`), the Contiv/VPP control plane automatically assigns
-an IP address from this range to the DPDK-managed ethernet interface bound to VPP
-on each node. The actual IP address will be calculated from the Node ID (e.g., with
-`NodeInterconnectCIDR = 192.168.16.0/24` and `NodeID = 5`, the resulting IP
-address assigned to the ethernet interface on VPP will be `192.168.16.5` ).
-
-- **NodeInterconnectDHCP** (default `False`): instead of assigning the IPs
-for the data plane interfaces, which are managed by VPP from `NodeInterconnectCIDR` by the Contiv/VPP
-control plane, DHCP assigns the IP addresses. The DHCP must be running in the network where the data
-plane interface is connected, in case `NodeInterconnectDHCP = True`,
-`NodeInterconnectCIDR` is ignored.
-
-- **VxlanCIDR** (default `192.168.30.0/24`): in order to provide inter-node
-POD to POD connectivity via any underlay network (not necessarily an L2 network),
-Contiv/VPP sets up a VXLAN tunnel overlay between each of the 2 nodes within the cluster. Each node needs its unique IP address of the VXLAN BVI interface. This IP address
-is automatically calculated from the Node ID, (e.g., with `VxlanCIDR = 192.168.30.0/24`
-and `NodeID = 5`, the resulting IP address assigned to the VXLAN BVI interface will be `192.168.30.5`).
-
-## VPP Programming
-This section describes how the Contiv/VPP control plane programs VPP, based on the
-events it receives from k8s. This section is not necessarily for understanding
-basic Contiv/VPP operation, but is very useful for debugging purposes.
-
-Contiv/VPP currently uses a single VRF to forward the traffic between PODs on a node,
-PODs on different nodes, host network stack, and DPDK-managed dataplane interface. The forwarding
-between each of them is purely L3-based, even for cases of communication
-between 2 PODs within the same node.
-
-#### DPDK-Managed Data Interface
-In order to allow inter-node communication between PODs on different
-nodes and between PODs and outside world, Contiv/VPP uses data-plane interfaces
-bound to VPP using DPDK. Each node should have one "main" VPP interface,
-which is unbound from the host network stack and bound to VPP.
-The Contiv/VPP control plane automatically configures the interface either
-via DHCP, or with a statically assigned address (see `NodeInterconnectCIDR` and
-`NodeInterconnectDHCP` yaml settings).
-
-#### PODs on the Same Node
-PODs are connected to VPP using virtio-based TAP interfaces created by VPP,
-with the POD-end of the interface placed into the POD container network namespace.
-Each POD is assigned an IP address from the `PodSubnetCIDR`. The allocated IP
-is configured with the prefix length `/32`. Additionally, a static route pointing
-towards the VPP is configured in the POD network namespace.
-The prefix length `/32` means that all IP traffic will be forwarded to the
-default route - VPP. To get rid of unnecessary broadcasts between POD and VPP,
-a static ARP entry is configured for the gateway IP in the POD namespace, as well
-as for POD IP on VPP. Both ends of the TAP interface have a static (non-default)
-MAC address applied.
-
-#### PODs with hostNetwork=true
-PODs with a `hostNetwork=true` attribute are not placed into a separate network namespace, they instead use the main host Linux network namespace; therefore, they are not directly connected to the VPP. They rely on the interconnection between the VPP and the host Linux network stack,
-which is described in the next paragraph. Note, when these PODs access some service IP, their network communication will be NATed in Linux (by iptables rules programmed by kube-proxy)
-as opposed to VPP, which is the case for the PODs connected to VPP directly.
-
-#### Linux Host Network Stack
-In order to interconnect the Linux host network stack with VPP (to allow access
-to the cluster resources from the host itself, as well as for the PODs with `hostNetwork=true`),
-VPP creates a TAP interface between VPP and the main network namespace. The TAP interface is configured with IP addresses from the `VPPHostSubnetCIDR` range, with `.1` in the latest octet on the VPP side, and `.2` on the host side. The name of the host interface is `vpp1`. The host has static routes pointing to VPP configured with:
-- A route to the whole `PodSubnetCIDR` to route traffic targeting PODs towards VPP.
-- A route to `ServiceCIDR` (default `10.96.0.0/12`), to route service IP targeted traffic that has not been translated by kube-proxy for some reason towards VPP.
-- The host also has a static ARP entry configured for the IP of the VPP-end TAP interface, to get rid of unnecessary broadcasts between the main network namespace and VPP.
-
-#### VXLANs to Other Nodes
-In order to provide inter-node POD to POD connectivity via any underlay network
-(not necessarily an L2 network), Contiv/VPP sets up a VXLAN tunnel overlay between
-each 2 nodes within the cluster (full mesh).
-
-All VXLAN tunnels are terminated in one bridge domain on each VPP. The bridge domain
-has learning and flooding disabled, the l2fib of the bridge domain contains a static entry for each VXLAN tunnel. Each bridge domain has a BVI interface, which
-interconnects the bridge domain with the main VRF (L3 forwarding). This interface needs
-a unique IP address, which is assigned from the `VxlanCIDR` as describe above.
-
-The main VRF contains several static routes that point to the BVI IP addresses of other nodes.
-For each node, it is a route to PODSubnet and VppHostSubnet of the remote node, as well as a route
-to the management IP address of the remote node. For each of these routes, the next hop IP is the
-BVI interface IP of the remote node, which goes via the BVI interface of the local node.
-
-The VXLAN tunnels and the static routes pointing to them are added/deleted on each VPP,
-whenever a node is added/deleted in the k8s cluster.
-
-
-#### More Info
-Please refer to the \[Packet Flow Dev Guide\](../dev-guide/PACKET_FLOW.html) for more
-detailed description of paths traversed by request and response packets
-inside Contiv/VPP Kubernetes cluster under different situations.
\ No newline at end of file
diff --git a/docs/usecases/contiv/NETWORKING.rst b/docs/usecases/contiv/NETWORKING.rst
new file mode 100644
index 0000000..b679996
--- /dev/null
+++ b/docs/usecases/contiv/NETWORKING.rst
@@ -0,0 +1,196 @@
+Contiv/VPP Network Operation
+============================
+
+This document describes the network operation of the Contiv/VPP k8s
+network plugin. It elaborates the operation and config options of the
+Contiv IPAM, as well as details on how the VPP gets programmed by
+Contiv/VPP control plane.
+
+The following picture shows 2-node k8s deployment of Contiv/VPP, with a
+VXLAN tunnel established between the nodes to forward inter-node POD
+traffic. The IPAM options are depicted on the Node 1, whereas the VPP
+programming is depicted on the Node 2.
+
+.. figure:: /_images/contiv-networking.png
+ :alt: contiv-networking.png
+
+ Contiv/VPP Architecture
+
+Contiv/VPP IPAM (IP Address Management)
+---------------------------------------
+
+IPAM in Contiv/VPP is based on the concept of **Node ID**. The Node ID
+is a number that uniquely identifies a node in the k8s cluster. The
+first node is assigned the ID of 1, the second node 2, etc. If a node
+leaves the cluster, its ID is released back to the pool and will be
+re-used by the next node.
+
+The Node ID is used to calculate per-node IP subnets for PODs and other
+internal subnets that need to be unique on each node. Apart from the
+Node ID, the input for IPAM calculations is a set of config knobs, which
+can be specified in the ``IPAMConfig`` section of the [Contiv/VPP
+deployment YAML](../../../k8s/contiv-vpp.yaml):
+
+- **PodSubnetCIDR** (default ``10.1.0.0/16``): each pod gets an IP
+ address assigned from this range. The size of this range (default
+ ``/16``) dictates upper limit of POD count for the entire k8s cluster
+ (default 65536 PODs).
+
+- **PodNetworkPrefixLen** (default ``24``): per-node dedicated
+ podSubnet range. From the allocatable range defined in
+ ``PodSubnetCIDR``, this value will dictate the allocation for each
+ node. With the default value (``24``) this indicates that each node
+ has a ``/24`` slice of the ``PodSubnetCIDR``. The Node ID is used to
+ address the node. In case of ``PodSubnetCIDR = 10.1.0.0/16``,
+ ``PodNetworkPrefixLen = 24`` and ``NodeID = 5``, the resulting POD
+ subnet for the node would be ``10.1.5.0/24``.
+
+- **PodIfIPCIDR** (default ``10.2.1.0/24``): VPP-internal addresses put
+ the VPP interfaces facing towards the PODs into L3 mode. This IP
+ range will be reused on each node, thereby it is never externally
+ addressable outside of the node itself. The only requirement is that
+ this subnet should not collide with any other IPAM subnet.
+
+- **VPPHostSubnetCIDR** (default ``172.30.0.0/16``): used for
+ addressing the interconnect of VPP with the Linux network stack,
+ within the same node. Since this subnet needs to be unique on each
+ node, the Node ID is used to determine the actual subnet used on the
+ node with the combination of ``VPPHostNetworkPrefixLen``,
+ ``PodSubnetCIDR`` and ``PodNetworkPrefixLen``.
+
+- **VPPHostNetworkPrefixLen** (default ``24``): used to calculate the
+ subnet for addressing the interconnect of VPP with the Linux network
+ stack, within the same node. With
+ ``VPPHostSubnetCIDR = 172.30.0.0/16``,
+ ``VPPHostNetworkPrefixLen = 24`` and ``NodeID = 5`` the resulting
+ subnet for the node would be ``172.30.5.0/24``.
+
+- **NodeInterconnectCIDR** (default ``192.168.16.0/24``): range for the
+ addresses assigned to the data plane interfaces managed by VPP.
+ Unless DHCP is used (``NodeInterconnectDHCP = True``), the Contiv/VPP
+ control plane automatically assigns an IP address from this range to
+ the DPDK-managed ethernet interface bound to VPP on each node. The
+ actual IP address will be calculated from the Node ID (e.g., with
+ ``NodeInterconnectCIDR = 192.168.16.0/24`` and ``NodeID = 5``, the
+ resulting IP address assigned to the ethernet interface on VPP will
+ be ``192.168.16.5`` ).
+
+- **NodeInterconnectDHCP** (default ``False``): instead of assigning
+ the IPs for the data plane interfaces, which are managed by VPP from
+ ``NodeInterconnectCIDR`` by the Contiv/VPP control plane, DHCP
+ assigns the IP addresses. The DHCP must be running in the network
+ where the data plane interface is connected, in case
+ ``NodeInterconnectDHCP = True``, ``NodeInterconnectCIDR`` is ignored.
+
+- **VxlanCIDR** (default ``192.168.30.0/24``): in order to provide
+ inter-node POD to POD connectivity via any underlay network (not
+ necessarily an L2 network), Contiv/VPP sets up a VXLAN tunnel overlay
+ between each of the 2 nodes within the cluster. Each node needs its
+ unique IP address of the VXLAN BVI interface. This IP address is
+ automatically calculated from the Node ID, (e.g., with
+ ``VxlanCIDR = 192.168.30.0/24`` and ``NodeID = 5``, the resulting IP
+ address assigned to the VXLAN BVI interface will be
+ ``192.168.30.5``).
+
+VPP Programming
+---------------
+
+This section describes how the Contiv/VPP control plane programs VPP,
+based on the events it receives from k8s. This section is not
+necessarily for understanding basic Contiv/VPP operation, but is very
+useful for debugging purposes.
+
+Contiv/VPP currently uses a single VRF to forward the traffic between
+PODs on a node, PODs on different nodes, host network stack, and
+DPDK-managed dataplane interface. The forwarding between each of them is
+purely L3-based, even for cases of communication between 2 PODs within
+the same node.
+
+DPDK-Managed Data Interface
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to allow inter-node communication between PODs on different
+nodes and between PODs and outside world, Contiv/VPP uses data-plane
+interfaces bound to VPP using DPDK. Each node should have one “main” VPP
+interface, which is unbound from the host network stack and bound to
+VPP. The Contiv/VPP control plane automatically configures the interface
+either via DHCP, or with a statically assigned address (see
+``NodeInterconnectCIDR`` and ``NodeInterconnectDHCP`` yaml settings).
+
+PODs on the Same Node
+~~~~~~~~~~~~~~~~~~~~~
+
+PODs are connected to VPP using virtio-based TAP interfaces created by
+VPP, with the POD-end of the interface placed into the POD container
+network namespace. Each POD is assigned an IP address from the
+``PodSubnetCIDR``. The allocated IP is configured with the prefix length
+``/32``. Additionally, a static route pointing towards the VPP is
+configured in the POD network namespace. The prefix length ``/32`` means
+that all IP traffic will be forwarded to the default route - VPP. To get
+rid of unnecessary broadcasts between POD and VPP, a static ARP entry is
+configured for the gateway IP in the POD namespace, as well as for POD
+IP on VPP. Both ends of the TAP interface have a static (non-default)
+MAC address applied.
+
+PODs with hostNetwork=true
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+PODs with a ``hostNetwork=true`` attribute are not placed into a
+separate network namespace, they instead use the main host Linux network
+namespace; therefore, they are not directly connected to the VPP. They
+rely on the interconnection between the VPP and the host Linux network
+stack, which is described in the next paragraph. Note, when these PODs
+access some service IP, their network communication will be NATed in
+Linux (by iptables rules programmed by kube-proxy) as opposed to VPP,
+which is the case for the PODs connected to VPP directly.
+
+Linux Host Network Stack
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to interconnect the Linux host network stack with VPP (to allow
+access to the cluster resources from the host itself, as well as for the
+PODs with ``hostNetwork=true``), VPP creates a TAP interface between VPP
+and the main network namespace. The TAP interface is configured with IP
+addresses from the ``VPPHostSubnetCIDR`` range, with ``.1`` in the
+latest octet on the VPP side, and ``.2`` on the host side. The name of
+the host interface is ``vpp1``. The host has static routes pointing to
+VPP configured with: - A route to the whole ``PodSubnetCIDR`` to route
+traffic targeting PODs towards VPP. - A route to ``ServiceCIDR``
+(default ``10.96.0.0/12``), to route service IP targeted traffic that
+has not been translated by kube-proxy for some reason towards VPP. - The
+host also has a static ARP entry configured for the IP of the VPP-end
+TAP interface, to get rid of unnecessary broadcasts between the main
+network namespace and VPP.
+
+VXLANs to Other Nodes
+~~~~~~~~~~~~~~~~~~~~~
+
+In order to provide inter-node POD to POD connectivity via any underlay
+network (not necessarily an L2 network), Contiv/VPP sets up a VXLAN
+tunnel overlay between each 2 nodes within the cluster (full mesh).
+
+All VXLAN tunnels are terminated in one bridge domain on each VPP. The
+bridge domain has learning and flooding disabled, the l2fib of the
+bridge domain contains a static entry for each VXLAN tunnel. Each bridge
+domain has a BVI interface, which interconnects the bridge domain with
+the main VRF (L3 forwarding). This interface needs a unique IP address,
+which is assigned from the ``VxlanCIDR`` as describe above.
+
+The main VRF contains several static routes that point to the BVI IP
+addresses of other nodes. For each node, it is a route to PODSubnet and
+VppHostSubnet of the remote node, as well as a route to the management
+IP address of the remote node. For each of these routes, the next hop IP
+is the BVI interface IP of the remote node, which goes via the BVI
+interface of the local node.
+
+The VXLAN tunnels and the static routes pointing to them are
+added/deleted on each VPP, whenever a node is added/deleted in the k8s
+cluster.
+
+More Info
+~~~~~~~~~
+
+Please refer to the [Packet Flow Dev
+Guide](../dev-guide/PACKET_FLOW.html) for more detailed description of
+paths traversed by request and response packets inside Contiv/VPP
+Kubernetes cluster under different situations.
diff --git a/docs/usecases/contiv/Prometheus.md b/docs/usecases/contiv/Prometheus.md
deleted file mode 100644
index ba61be3..0000000
--- a/docs/usecases/contiv/Prometheus.md
+++ /dev/null
@@ -1,159 +0,0 @@
-# Prometheus Statistics
-
-Each contiv-agent exposes statistics in Prometheus format at port `9999` by default.
-Exposed data is split into two groups:
-- `/stats` provides statistics for VPP interfaces managed by contiv-agent
- Prometheus data is a set of counters with labels. For each interface,
- the following counters are exposed:
- * *inPackets*
- * *outPackets*
- * *inBytes*
- * *outBytes*
- * *ipv4Packets*
- * *ipv6Packets*
- * *outErrorPackets*
- * *dropPackets*
- * *inMissPackets*
- * *inNobufPackets*
- * *puntPackets*
-
- Labels let you add additional information to a counter. The *interfaceName* and *node*
- labels are specified for all counters. If an interface is associated with a particular
- pod, then the *podName* and *podNamespace* labels are also specified for its counters;
- otherwise, a placeholder value (`--`) is used (for example, for node interconnect
- interfaces).
-- `/metrics` provides general go runtime statistics
-
-To access Prometheus stats of a node you can use `curl localhost:9999/stats` from the node. The output of contiv-agent running at k8s master node looks similar to the following:
-
-```
-$ curl localhost:9999/stats
-# HELP dropPackets Number of dropped packets for interface
-# TYPE dropPackets gauge
-dropPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-dropPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 52
-dropPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 9
-dropPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 12
-dropPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP inBytes Number of received bytes for interface
-# TYPE inBytes gauge
-inBytes{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-inBytes{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 24716
-inBytes{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 726
-inBytes{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 6113
-inBytes{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP inErrorPackets Number of received packets with error for interface
-# TYPE inErrorPackets gauge
-inErrorPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-inErrorPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 0
-inErrorPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
-inErrorPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 0
-inErrorPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP inMissPackets Number of missed packets for interface
-# TYPE inMissPackets gauge
-inMissPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-inMissPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 0
-inMissPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
-inMissPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 0
-inMissPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP inNobufPackets Number of received packets ??? for interface
-# TYPE inNobufPackets gauge
-inNobufPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-inNobufPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 0
-inNobufPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
-inNobufPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 0
-inNobufPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP inPackets Number of received packets for interface
-# TYPE inPackets gauge
-inPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-inPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 97
-inPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 9
-inPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 60
-inPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP ipv4Packets Number of ipv4 packets for interface
-# TYPE ipv4Packets gauge
-ipv4Packets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-ipv4Packets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 68
-ipv4Packets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
-ipv4Packets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 52
-ipv4Packets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP ipv6Packets Number of ipv6 packets for interface
-# TYPE ipv6Packets gauge
-ipv6Packets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-ipv6Packets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 26
-ipv6Packets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 9
-ipv6Packets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 8
-ipv6Packets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP outBytes Number of transmitted bytes for interface
-# TYPE outBytes gauge
-outBytes{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-outBytes{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 5203
-outBytes{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
-outBytes{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 17504
-outBytes{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP outErrorPackets Number of transmitted packets with error for interface
-# TYPE outErrorPackets gauge
-outErrorPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-outErrorPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 0
-outErrorPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
-outErrorPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 0
-outErrorPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP outPackets Number of transmitted packets for interface
-# TYPE outPackets gauge
-outPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-outPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 49
-outPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
-outPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 45
-outPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-# HELP puntPackets Number of punt packets for interface
-# TYPE puntPackets gauge
-puntPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
-puntPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 0
-puntPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
-puntPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 0
-puntPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
-
-```
-
-
-In order to browse stats in web UI Prometheus, it must be started locally by following the information in
-the [Prometheus Getting Started Guide](https://prometheus.io/docs/prometheus/latest/getting_started/).
-
-If you start Prometheus on a node, the following sample config can be used:
-```yaml
-global:
- scrape_interval: 15s
-
-scrape_configs:
- - job_name: 'contiv_stats'
- metrics_path: '/stats'
- static_configs:
- - targets: ['localhost:9999']
- - job_name: 'contiv_agent'
- # metrics_path defaults to '/metrics'
- static_configs:
- - targets: ['localhost:9999']
-```
-
-Once Prometheus is started with the specified config, you should be able access its web UI at
-`localhost:9090`.
-```
-tester@dev:~/Downloads/prometheus-2.1.0.linux-amd64$ ./prometheus --config.file=config.yml
-```
-
-If security features are enabled for the HTTP endpoint, then the config must be adjusted:
-```yaml
- - job_name: 'contiv_secured'
-
- scheme: https
- basic_auth:
- username: user
- password: pass
- metrics_path: /stats
- tls_config:
- insecure_skip_verify: true
- # CA certificate to validate API server certificate with.
- #[ ca_file: <filename> ]
- static_configs:
- - targets: ['localhost:9191']
-```
\ No newline at end of file
diff --git a/docs/usecases/contiv/Prometheus.rst b/docs/usecases/contiv/Prometheus.rst
new file mode 100644
index 0000000..cb4c164
--- /dev/null
+++ b/docs/usecases/contiv/Prometheus.rst
@@ -0,0 +1,158 @@
+Prometheus Statistics
+=====================
+
+Each contiv-agent exposes statistics in Prometheus format at port
+``9999`` by default. Exposed data is split into two groups: - ``/stats``
+provides statistics for VPP interfaces managed by contiv-agent
+Prometheus data is a set of counters with labels. For each interface,
+the following counters are exposed: \* *inPackets* \* *outPackets* \*
+*inBytes* \* *outBytes* \* *ipv4Packets* \* *ipv6Packets* \*
+*outErrorPackets* \* *dropPackets* \* *inMissPackets* \*
+*inNobufPackets* \* *puntPackets*
+
+Labels let you add additional information to a counter. The
+*interfaceName* and *node* labels are specified for all counters. If an
+interface is associated with a particular pod, then the *podName* and
+*podNamespace* labels are also specified for its counters; otherwise, a
+placeholder value (``--``) is used (for example, for node interconnect
+interfaces). - ``/metrics`` provides general go runtime statistics
+
+To access Prometheus stats of a node you can use
+``curl localhost:9999/stats`` from the node. The output of contiv-agent
+running at k8s master node looks similar to the following:
+
+::
+
+ $ curl localhost:9999/stats
+ # HELP dropPackets Number of dropped packets for interface
+ # TYPE dropPackets gauge
+ dropPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ dropPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 52
+ dropPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 9
+ dropPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 12
+ dropPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP inBytes Number of received bytes for interface
+ # TYPE inBytes gauge
+ inBytes{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ inBytes{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 24716
+ inBytes{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 726
+ inBytes{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 6113
+ inBytes{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP inErrorPackets Number of received packets with error for interface
+ # TYPE inErrorPackets gauge
+ inErrorPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ inErrorPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 0
+ inErrorPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
+ inErrorPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 0
+ inErrorPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP inMissPackets Number of missed packets for interface
+ # TYPE inMissPackets gauge
+ inMissPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ inMissPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 0
+ inMissPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
+ inMissPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 0
+ inMissPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP inNobufPackets Number of received packets ??? for interface
+ # TYPE inNobufPackets gauge
+ inNobufPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ inNobufPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 0
+ inNobufPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
+ inNobufPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 0
+ inNobufPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP inPackets Number of received packets for interface
+ # TYPE inPackets gauge
+ inPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ inPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 97
+ inPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 9
+ inPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 60
+ inPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP ipv4Packets Number of ipv4 packets for interface
+ # TYPE ipv4Packets gauge
+ ipv4Packets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ ipv4Packets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 68
+ ipv4Packets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
+ ipv4Packets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 52
+ ipv4Packets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP ipv6Packets Number of ipv6 packets for interface
+ # TYPE ipv6Packets gauge
+ ipv6Packets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ ipv6Packets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 26
+ ipv6Packets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 9
+ ipv6Packets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 8
+ ipv6Packets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP outBytes Number of transmitted bytes for interface
+ # TYPE outBytes gauge
+ outBytes{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ outBytes{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 5203
+ outBytes{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
+ outBytes{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 17504
+ outBytes{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP outErrorPackets Number of transmitted packets with error for interface
+ # TYPE outErrorPackets gauge
+ outErrorPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ outErrorPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 0
+ outErrorPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
+ outErrorPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 0
+ outErrorPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP outPackets Number of transmitted packets for interface
+ # TYPE outPackets gauge
+ outPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ outPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 49
+ outPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
+ outPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 45
+ outPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+ # HELP puntPackets Number of punt packets for interface
+ # TYPE puntPackets gauge
+ puntPackets{interfaceName="GigabitEthernet0/9/0",node="dev",podName="--",podNamespace="--"} 0
+ puntPackets{interfaceName="tap-vpp2",node="dev",podName="--",podNamespace="--"} 0
+ puntPackets{interfaceName="tap0e6439a7a934336",node="dev",podName="web-667bdcb4d8-pxkfs",podNamespace="default"} 0
+ puntPackets{interfaceName="tap5338a3285ad6bd7",node="dev",podName="kube-dns-6f4fd4bdf-rsz9b",podNamespace="kube-system"} 0
+ puntPackets{interfaceName="vxlanBVI",node="dev",podName="--",podNamespace="--"} 0
+
+In order to browse stats in web UI Prometheus, it must be started
+locally by following the information in the `Prometheus Getting Started
+Guide <https://prometheus.io/docs/prometheus/latest/getting_started/>`__.
+
+If you start Prometheus on a node, the following sample config can be
+used:
+
+.. code:: yaml
+
+ global:
+ scrape_interval: 15s
+
+ scrape_configs:
+ - job_name: 'contiv_stats'
+ metrics_path: '/stats'
+ static_configs:
+ - targets: ['localhost:9999']
+ - job_name: 'contiv_agent'
+ # metrics_path defaults to '/metrics'
+ static_configs:
+ - targets: ['localhost:9999']
+
+Once Prometheus is started with the specified config, you should be able
+access its web UI at ``localhost:9090``.
+
+::
+
+ tester@dev:~/Downloads/prometheus-2.1.0.linux-amd64$ ./prometheus --config.file=config.yml
+
+If security features are enabled for the HTTP endpoint, then the config
+must be adjusted:
+
+.. code:: yaml
+
+ - job_name: 'contiv_secured'
+
+ scheme: https
+ basic_auth:
+ username: user
+ password: pass
+ metrics_path: /stats
+ tls_config:
+ insecure_skip_verify: true
+ # CA certificate to validate API server certificate with.
+ #[ ca_file: <filename> ]
+ static_configs:
+ - targets: ['localhost:9191']
diff --git a/docs/usecases/contiv/SECURITY.md b/docs/usecases/contiv/SECURITY.md
deleted file mode 100644
index 40c5250..0000000
--- a/docs/usecases/contiv/SECURITY.md
+++ /dev/null
@@ -1,104 +0,0 @@
-# Security
-
-There are two types of security that are utilized in Contiv, and are discussed in this section: [HTTP](#http-security) and [ETCD](#etcd-security).
-
-## HTTP Security
-
-By default, the access to endpoints (liveness, readiness probe, prometheus stats, ...) served by Contiv-vswitch and
-Contiv-ksr is open to anybody. Contiv-vswitch exposes endpoints using port `9999` and contiv-ksr uses `9191`.
-
-To secure access to the endpoints, the SSL/TLS server certificate and basic auth (username password) can be configured.
-
-In Contiv-VPP, this can be done using the Helm charts in [k8s/contiv-vpp folder](https://github.com/contiv/vpp/tree/master/k8s/contiv-vpp).
-
-To generate server certificate the approach described in [ETCD security](#etcd-security) can be leveraged.
-
-## ETCD Security
-
-By default, the access to Contiv-VPP ETCD is open to anybody. ETCD gets deployed
-on the master node, on port `12379`, and is exposed using the NodePort service
-on port `32379`, on each node.
-
-To secure access to ETCD, we recommend using the SSL/TLS certificates to authenticate
-both the client and server side, and encrypt the communication. In Contiv-VPP, this can be done using the Helm charts in [k8s/contiv-vpp folder](https://github.com/contiv/vpp/tree/master/k8s/contiv-vpp).
-
-The prerequisite for that is the generation of SSL certificates.
-
-
-### Generate Self-Signed Certificates
-In order to secure ETCD, we need to create our own certificate authority,
-and then generate the private keys and certificates for both the ETCD server and ETCD clients.
-
-This guide uses CloudFlare's [cfssl](https://github.com/cloudflare/cfssl) tools to do this job.
-It follows the steps described in this [CoreOS guide](https://github.com/coreos/docs/blob/master/os/generate-self-signed-certificates.md).
-
-Perform the following steps to generate private keys and certificates:
-
-##### 1. Install cfssl
-```
-mkdir ~/bin
-curl -s -L -o ~/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
-curl -s -L -o ~/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
-chmod +x ~/bin/{cfssl,cfssljson}
-export PATH=$PATH:~/bin
-```
-
-##### 2. Initialize a Certificate Authority
-```
-echo '{"CN":"CA","key":{"algo":"rsa","size":2048}}' | cfssl gencert -initca - | cfssljson -bare ca -
-echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","server auth","client auth"]}}}' > ca-config.json
-```
-
-##### 3. Generate Server Key + Certificate
-Replace the IP address `10.0.2.15` below with the IP address of your master node:
-```
-export ADDRESS=127.0.0.1,10.0.2.15
-export NAME=server
-echo '{"CN":"'$NAME'","hosts":[""],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -config=ca-config.json -ca=ca.pem -ca-key=ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $NAME
-```
-
-##### 4. Generate Client Key + Certificate
-```
-export ADDRESS=
-export NAME=client
-echo '{"CN":"'$NAME'","hosts":[""],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -config=ca-config.json -ca=ca.pem -ca-key=ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $NAME
-```
-
-The above commands produce the following files that will be needed in order to secure ETCD:
- - `ca.pem`: certificate of the certificate authority
- - `server.pem`: certificate of the ETCD server
- - `server-key.pem`: private key of the ETCD server
- - `client.pem`: certificate for the ETCD clients
- - `client-key.pem`: private key for the ETCD clients
-
-
-### Distribute Certificates and Generate Contiv-VPP Deployment Yaml
-There are two options for distributing the certificates to all nodes in a k8s cluster.
-You can either distribute the certificates
-[manually](#distribute-certificates-manually), or embed the certificates into the deployment yaml file and
-distribute them as [k8s secrets](https://kubernetes.io/docs/concepts/configuration/secret/).
-
-##### Distribute Certificates Manually
-In this case, you need to copy the `ca.pem`, `client.pem` and `client-key.pem` files
-into a specific folder (`/var/contiv/etcd-secrets` by default) on each worker node.
-On the master node, you also need to add the `server.pem` and `server-key.pem` into that location.
-
-Then you can generate the Contiv-VPP deployment YAML as follows:
-```
-cd k8s
-helm template --name my-release contiv-vpp --set etcd.secureTransport=True > contiv-vpp.yaml
-```
-Then you can go ahead and deploy Contiv-VPP using this yaml file.
-
-##### Embed the certificates into deployment the yaml and use k8s secret to distribute them {: #Embed-certificates }
-In this case, you need to copy all 5 generated files into the folder with helm definitions
-(`k8s/contiv-vpp`) and generate the Contiv-VPP deployment YAML as follows:
-```
-cd k8s
-helm template --name my-release contiv-vpp --set etcd.secureTransport=True --set etcd.secrets.mountFromHost=False > contiv-vpp.yaml
-```
-Then just deploy Contiv-VPP using this yaml file.
-
-Please note that the path of the mount folder with certificates, as well as the certificate
-file names can be customized using the config parameters of the Contiv-VPP chart,
-as described in [this README](https://github.com/contiv/vpp/blob/master/k8s/contiv-vpp/README.md).
\ No newline at end of file
diff --git a/docs/usecases/contiv/SECURITY.rst b/docs/usecases/contiv/SECURITY.rst
new file mode 100644
index 0000000..8e8308e
--- /dev/null
+++ b/docs/usecases/contiv/SECURITY.rst
@@ -0,0 +1,145 @@
+Security
+========
+
+There are two types of security that are utilized in Contiv, and are
+discussed in this section: `HTTP <#http-security>`__ and
+`ETCD <#etcd-security>`__.
+
+HTTP Security
+-------------
+
+By default, the access to endpoints (liveness, readiness probe,
+prometheus stats, …) served by Contiv-vswitch and Contiv-ksr is open to
+anybody. Contiv-vswitch exposes endpoints using port ``9999`` and
+contiv-ksr uses ``9191``.
+
+To secure access to the endpoints, the SSL/TLS server certificate and
+basic auth (username password) can be configured.
+
+In Contiv-VPP, this can be done using the Helm charts in `k8s/contiv-vpp
+folder <https://github.com/contiv/vpp/tree/master/k8s/contiv-vpp>`__.
+
+To generate server certificate the approach described in `ETCD
+security <#etcd-security>`__ can be leveraged.
+
+ETCD Security
+-------------
+
+By default, the access to Contiv-VPP ETCD is open to anybody. ETCD gets
+deployed on the master node, on port ``12379``, and is exposed using the
+NodePort service on port ``32379``, on each node.
+
+To secure access to ETCD, we recommend using the SSL/TLS certificates to
+authenticate both the client and server side, and encrypt the
+communication. In Contiv-VPP, this can be done using the Helm charts in
+`k8s/contiv-vpp
+folder <https://github.com/contiv/vpp/tree/master/k8s/contiv-vpp>`__.
+
+The prerequisite for that is the generation of SSL certificates.
+
+Generate Self-Signed Certificates
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to secure ETCD, we need to create our own certificate
+authority, and then generate the private keys and certificates for both
+the ETCD server and ETCD clients.
+
+This guide uses CloudFlare’s
+`cfssl <https://github.com/cloudflare/cfssl>`__ tools to do this job. It
+follows the steps described in this `CoreOS
+guide <https://github.com/coreos/docs/blob/master/os/generate-self-signed-certificates.md>`__.
+
+Perform the following steps to generate private keys and certificates:
+
+1. Install cfssl
+^^^^^^^^^^^^^^^^
+
+::
+
+ mkdir ~/bin
+ curl -s -L -o ~/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
+ curl -s -L -o ~/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
+ chmod +x ~/bin/{cfssl,cfssljson}
+ export PATH=$PATH:~/bin
+
+2. Initialize a Certificate Authority
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
+ echo '{"CN":"CA","key":{"algo":"rsa","size":2048}}' | cfssl gencert -initca - | cfssljson -bare ca -
+ echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","server auth","client auth"]}}}' > ca-config.json
+
+3. Generate Server Key + Certificate
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Replace the IP address ``10.0.2.15`` below with the IP address of your
+master node:
+
+::
+
+ export ADDRESS=127.0.0.1,10.0.2.15
+ export NAME=server
+ echo '{"CN":"'$NAME'","hosts":[""],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -config=ca-config.json -ca=ca.pem -ca-key=ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $NAME
+
+4. Generate Client Key + Certificate
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
+ export ADDRESS=
+ export NAME=client
+ echo '{"CN":"'$NAME'","hosts":[""],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -config=ca-config.json -ca=ca.pem -ca-key=ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $NAME
+
+The above commands produce the following files that will be needed in
+order to secure ETCD: - ``ca.pem``: certificate of the certificate
+authority - ``server.pem``: certificate of the ETCD server -
+``server-key.pem``: private key of the ETCD server - ``client.pem``:
+certificate for the ETCD clients - ``client-key.pem``: private key for
+the ETCD clients
+
+Distribute Certificates and Generate Contiv-VPP Deployment Yaml
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are two options for distributing the certificates to all nodes in
+a k8s cluster. You can either distribute the certificates
+`manually <#distribute-certificates-manually>`__, or embed the
+certificates into the deployment yaml file and distribute them as `k8s
+secrets <https://kubernetes.io/docs/concepts/configuration/secret/>`__.
+
+Distribute Certificates Manually
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In this case, you need to copy the ``ca.pem``, ``client.pem`` and
+``client-key.pem`` files into a specific folder
+(``/var/contiv/etcd-secrets`` by default) on each worker node. On the
+master node, you also need to add the ``server.pem`` and
+``server-key.pem`` into that location.
+
+Then you can generate the Contiv-VPP deployment YAML as follows:
+
+::
+
+ cd k8s
+ helm template --name my-release contiv-vpp --set etcd.secureTransport=True > contiv-vpp.yaml
+
+Then you can go ahead and deploy Contiv-VPP using this yaml file.
+
+Embed the certificates into deployment the yaml and use k8s secret to distribute them {: #Embed-certificates }
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In this case, you need to copy all 5 generated files into the folder
+with helm definitions (``k8s/contiv-vpp``) and generate the Contiv-VPP
+deployment YAML as follows:
+
+::
+
+ cd k8s
+ helm template --name my-release contiv-vpp --set etcd.secureTransport=True --set etcd.secrets.mountFromHost=False > contiv-vpp.yaml
+
+Then just deploy Contiv-VPP using this yaml file.
+
+Please note that the path of the mount folder with certificates, as well
+as the certificate file names can be customized using the config
+parameters of the Contiv-VPP chart, as described in `this
+README <https://github.com/contiv/vpp/blob/master/k8s/contiv-vpp/README.md>`__.
diff --git a/docs/usecases/contiv/SINGLE_NIC_SETUP.md b/docs/usecases/contiv/SINGLE_NIC_SETUP.md
deleted file mode 100644
index 83dd47d..0000000
--- a/docs/usecases/contiv/SINGLE_NIC_SETUP.md
+++ /dev/null
@@ -1,111 +0,0 @@
-### Setting up a Node with a Single NIC
-
-#### Installing the STN Daemon
-The STN (Steal the NIC) daemon must be installed on every node in the cluster that has only
-one NIC. The STN daemon installation(*) should be performed before deployment
-of the Contiv-VPP plugin.
-
-\* Docker daemon must be present when installing STN. Also, Docker must be configured to allow shared mount.
-On CentOS, this may not be the case by default. You can enable it by following the instructions at
-[https://docs.portworx.com/knowledgebase/shared-mount-propagation.html](https://docs.portworx.com/knowledgebase/shared-mount-propagation.html).
-
-
-Run as root (not using sudo):
-```
-bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/stn-install.sh)
-```
-The install script should output the following:
-```
-Installing Contiv STN daemon.
-Starting contiv-stn Docker container:
-550334308f85f05b2690f5cfb5dd945bd9c501ab9d074231f15c14d7098ef212
-```
-
-Check that the STN daemon is running:
-```
-docker ps -a
-CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
-550334308f85 contivvpp/stn "/stn" 33 seconds ago Up 33 seconds contiv-stn
-```
-
-Check that the STN daemon is operational:
-```
-docker logs contiv-stn
-```
-The expected logs would look like the following excerpt:
-```
-2018/02/23 10:08:34 Starting the STN GRPC server at port 50051
-```
-
-For more details, please read the Go documentation for [contiv-stn](https://github.com/contiv/vpp/blob/master/cmd/contiv-stn/doc.go)
-and [contiv-init](https://github.com/contiv/vpp/blob/master/cmd/contiv-init/doc.go).
-
-#### Creating a VPP Interface Configuration
-Create the VPP configuration for the hardware interface as described
-[here](https://github.com/contiv/vpp/blob/master/docs/VPP_CONFIG.md#single-nic-configuration).
-
-#### Configuring STN in Contiv-VPP K8s Deployment Files
-The STN feature is disabled by default. It needs to be enabled either globally,
-or individually for every node in the cluster.
-
-##### Global Configuration:
-Global configuration is used in homogeneous environments where all nodes in
-a given cluster have the same hardware configuration, for example only a single
-Network Adapter. To enable the STN feature globally, put the `StealFirstNIC: True`
-stanza into the \[`contiv-vpp.yaml`\]\[1\] deployment file, for example:
-```
-data:
- contiv.yaml: |-
- TCPstackDisabled: true
- ...
- StealFirstNIC: True
- ...
- IPAMConfig:
-```
-
-Setting `StealFirstNIC` to `True` will tell the STN Daemon on every node in the
-cluster to steal the first NIC from the kernel and assign it to VPP. Note that
-the Network Adapters on different nodes do not need to be of the same type. You
-still need to create the respective vswitch configurations on every node in the
-cluster, as shown \[above\](#creating-a-vpp-interface-configuration).
-
-##### Individual Configuration:
-Individual configuration is used in heterogeneous environments where each node
-in a given cluster may be configured differently. To enable the STN feature
-for a specific node in the cluster, put the following stanza into its Node
-Configuration in the \[`contiv-vpp.yaml`\]\[1\] deployment file, for example:
-```
-...
- NodeConfig:
- - NodeName: "ubuntu-1"
- StealInterface: "enp0s8"
- - NodeName: "ubuntu-2"
- StealInterface: "enp0s8"
-...
-```
-Note that you still have to create the vswitch configuration on the node as
-shown [here](#creating-a-vpp-interface-configuration).
-
-
-
-#### Uninstalling the STN Daemon
-
-Run as root (not using sudo):
-```
-bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/stn-install.sh) --uninstall
-```
-The install script should output the following:
-```
-Uninstalling Contiv STN daemon.
-Stopping contiv-stn Docker container:
-contiv-stn
-contiv-stn
-contiv-stn
-```
-Make sure that the STN daemon has been uninstalled:
-```
-docker ps -q -f name=contiv-stn
-```
-No containers should be listed.
-
-[1]: ../k8s/contiv-vpp.yaml
diff --git a/docs/usecases/contiv/SINGLE_NIC_SETUP.rst b/docs/usecases/contiv/SINGLE_NIC_SETUP.rst
new file mode 100644
index 0000000..43d4c3f
--- /dev/null
+++ b/docs/usecases/contiv/SINGLE_NIC_SETUP.rst
@@ -0,0 +1,140 @@
+Setting up a Node with a Single NIC
+===================================
+
+Installing the STN Daemon
+-------------------------
+
+The STN (Steal the NIC) daemon must be installed on every node in the
+cluster that has only one NIC. The STN daemon installation(*) should be
+performed before deployment of the Contiv-VPP plugin.
+
+\* Docker daemon must be present when installing STN. Also, Docker must
+be configured to allow shared mount. On CentOS, this may not be the case
+by default. You can enable it by following the instructions at
+https://docs.portworx.com/knowledgebase/shared-mount-propagation.html.
+
+Run as root (not using sudo):
+
+::
+
+ bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/stn-install.sh)
+
+The install script should output the following:
+
+::
+
+ Installing Contiv STN daemon.
+ Starting contiv-stn Docker container:
+ 550334308f85f05b2690f5cfb5dd945bd9c501ab9d074231f15c14d7098ef212
+
+Check that the STN daemon is running:
+
+::
+
+ docker ps -a
+ CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
+ 550334308f85 contivvpp/stn "/stn" 33 seconds ago Up 33 seconds contiv-stn
+
+Check that the STN daemon is operational:
+
+::
+
+ docker logs contiv-stn
+
+The expected logs would look like the following excerpt:
+
+::
+
+ 2018/02/23 10:08:34 Starting the STN GRPC server at port 50051
+
+For more details, please read the Go documentation for
+`contiv-stn <https://github.com/contiv/vpp/blob/master/cmd/contiv-stn/doc.go>`__
+and
+`contiv-init <https://github.com/contiv/vpp/blob/master/cmd/contiv-init/doc.go>`__.
+
+Creating a VPP Interface Configuration
+--------------------------------------
+
+Create the VPP configuration for the hardware interface as described
+`here <https://github.com/contiv/vpp/blob/master/docs/VPP_CONFIG.md#single-nic-configuration>`__.
+
+Configuring STN in Contiv-VPP K8s Deployment Files
+--------------------------------------------------
+
+The STN feature is disabled by default. It needs to be enabled either
+globally, or individually for every node in the cluster.
+
+Global Configuration:
+~~~~~~~~~~~~~~~~~~~~~
+
+Global configuration is used in homogeneous environments where all nodes
+in a given cluster have the same hardware configuration, for example
+only a single Network Adapter. To enable the STN feature globally, put
+the ``StealFirstNIC: True`` stanza into the [``contiv-vpp.yaml``][1]
+deployment file, for example:
+
+::
+
+ data:
+ contiv.yaml: |-
+ TCPstackDisabled: true
+ ...
+ StealFirstNIC: True
+ ...
+ IPAMConfig:
+
+Setting ``StealFirstNIC`` to ``True`` will tell the STN Daemon on every
+node in the cluster to steal the first NIC from the kernel and assign it
+to VPP. Note that the Network Adapters on different nodes do not need to
+be of the same type. You still need to create the respective vswitch
+configurations on every node in the cluster, as shown
+[above](#creating-a-vpp-interface-configuration).
+
+Individual Configuration:
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Individual configuration is used in heterogeneous environments where
+each node in a given cluster may be configured differently. To enable
+the STN feature for a specific node in the cluster, put the following
+stanza into its Node Configuration in the [``contiv-vpp.yaml``][1]
+deployment file, for example:
+
+::
+
+ ...
+ NodeConfig:
+ - NodeName: "ubuntu-1"
+ StealInterface: "enp0s8"
+ - NodeName: "ubuntu-2"
+ StealInterface: "enp0s8"
+ ...
+
+Note that you still have to create the vswitch configuration on the node
+as shown `here <#creating-a-vpp-interface-configuration>`__.
+
+Uninstalling the STN Daemon
+---------------------------
+
+Run as root (not using sudo):
+
+::
+
+ bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/stn-install.sh) --uninstall
+
+The install script should output the following:
+
+::
+
+ Uninstalling Contiv STN daemon.
+ Stopping contiv-stn Docker container:
+ contiv-stn
+ contiv-stn
+ contiv-stn
+
+Make sure that the STN daemon has been uninstalled:
+
+::
+
+ docker ps -q -f name=contiv-stn
+
+No containers should be listed.
diff --git a/docs/usecases/contiv/VMWARE_FUSION_HOST.md b/docs/usecases/contiv/VMWARE_FUSION_HOST.md
deleted file mode 100644
index d4e251c..0000000
--- a/docs/usecases/contiv/VMWARE_FUSION_HOST.md
+++ /dev/null
@@ -1,52 +0,0 @@
-### Preparing a VmWare Fusion Host
-The *vmxnet3 driver* is required on a GigE Network Adapter used by VPP. On VmWare
-Fusion, the default Network Adapter driver is an *Intel 82545EM (e1000)*, and there
-is no GUI to change it to *vmxnet3*. The change must be done manually in the VM's
-configuration file as follows:
-
-- Bring up the VM library window: **Window -> Virtual Machine Library**
-- Right click on the VM where you want to change the driver:
- <*VM-Name*> **-> Show in Finder**. This pops up a new Finder window with a line
- for each VM that Fusion knows about.
-- Right click on the VM where you want to change the driver:
- <*VM-Name*> **-> Show package contents**. This brings up a window with the
- contents of the package.
-- Open the file <*VM-Name*> **.vmx** with your favorite text editor.
-- For each Network Adapter that you want used by VPP, look for the
- Network Adapter's driver configuration. For example, for the VM's first
- Network Adapter look for:
- ```
- ethernet0.virtualDev = "e1000"
- ```
- Replace `e1000` with `vmxnet3`:
- ```
- ethernet0.virtualDev = "vmxnet3"
- ```
-and restart the VM.
-
-If you replaced the driver on your VM's primary Network Adapter, you will
-have to change the primary network interface configuration in Linux.
-
-First, get the new primary network interface name:
-```
-sudo lshw -class network -businfo
-
-Bus info Device Class Description
-========================================================
-pci@0000:03:00.0 ens160 network VMXNET3 Ethernet Controller
-```
-Replace the existing primary network interface name in `/etc/network/interfaces`
-with the above device name (ens160):
-```
-# This file describes the network interfaces available on your system,
-# and how to activate them. For more information, see interfaces(5).
-
-source /etc/network/interfaces.d/*
-
-# The loopback network interface
-auto lo
-iface lo inet loopback
-
-# The primary network interface
-auto ens160
-iface ens160 inet dhcp
\ No newline at end of file
diff --git a/docs/usecases/contiv/VMWARE_FUSION_HOST.rst b/docs/usecases/contiv/VMWARE_FUSION_HOST.rst
new file mode 100644
index 0000000..6ddc665
--- /dev/null
+++ b/docs/usecases/contiv/VMWARE_FUSION_HOST.rst
@@ -0,0 +1,66 @@
+Preparing a VmWare Fusion Host
+==============================
+
+The *vmxnet3 driver* is required on a GigE Network Adapter used by VPP.
+On VmWare Fusion, the default Network Adapter driver is an *Intel
+82545EM (e1000)*, and there is no GUI to change it to *vmxnet3*. The
+change must be done manually in the VM’s configuration file as follows:
+
+- Bring up the VM library window: **Window -> Virtual Machine Library**
+
+- Right click on the VM where you want to change the driver:
+ <*VM-Name*> **-> Show in Finder**. This pops up a new Finder window
+ with a line for each VM that Fusion knows about.
+
+- Right click on the VM where you want to change the driver:
+ <*VM-Name*> **-> Show package contents**. This brings up a window
+ with the contents of the package.
+
+- Open the file <*VM-Name*> **.vmx** with your favorite text editor.
+
+- For each Network Adapter that you want used by VPP, look for the
+ Network Adapter’s driver configuration. For example, for the VM’s
+ first Network Adapter look for:
+
+ ::
+
+ ethernet0.virtualDev = "e1000"
+
+ Replace ``e1000`` with ``vmxnet3``:
+
+ ::
+
+ ethernet0.virtualDev = "vmxnet3"
+
+ and restart the VM.
+
+If you replaced the driver on your VM’s primary Network Adapter, you
+will have to change the primary network interface configuration in
+Linux.
+
+First, get the new primary network interface name:
+
+::
+
+ sudo lshw -class network -businfo
+
+ Bus info Device Class Description
+ ========================================================
+ pci@0000:03:00.0 ens160 network VMXNET3 Ethernet Controller
+
+Replace the existing primary network interface name in
+``/etc/network/interfaces`` with the above device name (ens160): \``\` #
+This file describes the network interfaces available on your system, #
+and how to activate them. For more information, see interfaces(5).
+
+source /etc/network/interfaces.d/\*
+
+The loopback network interface
+==============================
+
+auto lo iface lo inet loopback
+
+The primary network interface
+=============================
+
+auto ens160 iface ens160 inet dhcp
diff --git a/docs/usecases/contiv/VPPTRACE.md b/docs/usecases/contiv/VPPTRACE.md
deleted file mode 100644
index c9d2088..0000000
--- a/docs/usecases/contiv/VPPTRACE.md
+++ /dev/null
@@ -1,95 +0,0 @@
-## Using vpptrace.sh for VPP Packet Tracing
-
-VPP allows tracing of incoming packets using CLI commands `trace add` and `show trace`
-as explained \[here\](VPP_PACKET_TRACING_K8S.html), but it is a rather cumbersome process.
-
-The buffer for captured packets is limited in size, and once it gets full the tracing stops. The user has to manually clear the buffer content, and then repeat the trace command to resume the packet capture, losing information about all packets received in the meantime.
-
-Packet filtering exposed via the CLI command `trace filter` is also quite limited in what it can do. Currently there is just one available filter, which allows you to keep only packets that include a certain node in the trace or exclude a certain node in the trace.
-It is not possible to filter the traffic by its content (e.g., by the source/destination IP address, protocol, etc.).
-
-Last but not least, it is not possible to trace packets on a selected interface
-like `tcpdump`, which allows tracing via the option `-i`. VPP is only able to capture packets
-on the *RX side* of selected *devices* (e.g., dpdk, virtio, af-packet). This means
-that interfaces based on the same device cannot be traced for incoming packets
-individually, but only all at the same time. In Contiv/VPP all pods are connected
-with VPP via the same kind of the TAP interface, meaning that it is not possible to
-capture packets incoming only from one selected pod.
-
-Contiv/VPP ships with a simple bash script [vpptrace.sh](https://github.com/contiv/vpp/blob/master/scripts/vpptrace.sh),
-which helps alleviate the aforementioned VPP limitations. The script automatically
-re-initializes buffers and traces whenever it is close to getting full, in order to
-avoid packet loss as much as possible. Next it allows you to filter packets
-by the content of the trace. There are two modes of filtering:
- - *substring mode* (default): packet trace must contain a given sub-string in order to
- be included in the output
- - *regex mode*: packet trace must match a given regex in order to be printed
-
-The script is still limited, in that capture runs only on the RX side of all interfaces that are built on top of selected devices. Using filtering, however, it is possible to limit
-*traffic by interface* simply by using the interface name as a substring to match against.
-
-#### Usage
-
-Run the script with option `-h` to get the usage printed:
-```
-Usage: ./vpptrace.sh [-i <VPP-IF-TYPE>]... [-a <VPP-ADDRESS>] [-r] [-f <REGEXP> / <SUBSTRING>]
- -i <VPP-IF-TYPE> : VPP interface *type* to run the packet capture on (e.g., dpdk-input, virtio-input, etc.)
- - available aliases:
- - af-packet-input: afpacket, af-packet, veth
- - virtio-input: tap (version determined from the VPP runtime config), tap2, tapv2
- - tapcli-rx: tap (version determined from the VPP config), tap1, tapv1
- - dpdk-input: dpdk, gbe, phys*
- - multiple interfaces can be watched at the same time - the option can be repeated with
- different values
- - default = dpdk + tap
- -a <VPP-ADDRESS> : IP address or hostname of the VPP to capture packets from
- - not supported if VPP listens on a UNIX domain socket
- - default = 127.0.0.1
- -r : apply filter string (passed with -f) as a regexp expression
- - by default the filter is NOT treated as regexp
- -f : filter string that packet must contain (without -r) or match as regexp (with -r) to be printed
- - default is no filtering
-```
-
-`VPP-IF-TYPE` is a repeated option used to select the set of devices (e.g., virtio, dpdk, etc.)
-to capture the incoming traffic. Script provides multiple aliases, which
-are much easier to remember than the device names. For `dpdk-input` one can enter
-just `dpdk`, or anything starting with `phys`, etc. For TAPs, the script is even
-smart enough to find out the TAP version used, which allows to enter just `tap`
-as the device name.
-
-If `VPP-IF-TYPE` is not specified, then the default behaviour is to capture from both
-`dpdk` (traffic entering the node from outside) and `tap` (preferred interface type
-for pod-VPP and host-VPP interconnection, receiving node-initiated traffic).
-
-vpptrace.sh can capture packets even from a VPP on a different host, provided that
-VPP-CLI listens on a port, and not on a UNIX domain socket (for security reasons IPC
-is the default communication link, see `/etc/vpp/contiv-vswitch.conf`). Enter the destination
-node IP address via the option `-a`(localhost is the default).
-
-The capture can be filtered via the `-f` option. The output will include only packets
-whose trace matches contain the given expression/sub-string.
-
-Option `-r` enables the regex mode for filtering.
-
-#### Examples
-
-- Capture all packets entering VPP via `tapcli-1` interface **AND** all packets
- leaving VPP via `tapcli-1` that were sent from a pod, or the host on the *same node*
- (sent from tap, not Gbe):
-```
-$ vpptrace.sh -i tap -f "tapcli-1"
-```
-
-- Capture all packets with source or destination IP address 10.1.1.3:
-```
-$ vpptrace.sh -i tap -i dpdk -f "10.1.1.3"
-
-Or just:
-$ vpptrace.sh "10.1.1.3"
-```
-
-- Capture all SYN-ACKs received from outside:
-```
-$ vpptrace.sh -i dpdk -f "SYN-ACK"
-```
\ No newline at end of file
diff --git a/docs/usecases/contiv/VPPTRACE.rst b/docs/usecases/contiv/VPPTRACE.rst
new file mode 100644
index 0000000..a277a57
--- /dev/null
+++ b/docs/usecases/contiv/VPPTRACE.rst
@@ -0,0 +1,120 @@
+Using vpptrace.sh for VPP Packet Tracing
+========================================
+
+VPP allows tracing of incoming packets using CLI commands ``trace add``
+and ``show trace`` as explained [here](VPP_PACKET_TRACING_K8S.html), but
+it is a rather cumbersome process.
+
+The buffer for captured packets is limited in size, and once it gets
+full the tracing stops. The user has to manually clear the buffer
+content, and then repeat the trace command to resume the packet capture,
+losing information about all packets received in the meantime.
+
+Packet filtering exposed via the CLI command ``trace filter`` is also
+quite limited in what it can do. Currently there is just one available
+filter, which allows you to keep only packets that include a certain
+node in the trace or exclude a certain node in the trace. It is not
+possible to filter the traffic by its content (e.g., by the
+source/destination IP address, protocol, etc.).
+
+Last but not least, it is not possible to trace packets on a selected
+interface like ``tcpdump``, which allows tracing via the option ``-i``.
+VPP is only able to capture packets on the *RX side* of selected
+*devices* (e.g., dpdk, virtio, af-packet). This means that interfaces
+based on the same device cannot be traced for incoming packets
+individually, but only all at the same time. In Contiv/VPP all pods are
+connected with VPP via the same kind of the TAP interface, meaning that
+it is not possible to capture packets incoming only from one selected
+pod.
+
+Contiv/VPP ships with a simple bash script
+`vpptrace.sh <https://github.com/contiv/vpp/blob/master/scripts/vpptrace.sh>`__,
+which helps alleviate the aforementioned VPP limitations. The script
+automatically re-initializes buffers and traces whenever it is close to
+getting full, in order to avoid packet loss as much as possible. Next it
+allows you to filter packets by the content of the trace. There are two
+modes of filtering: - *substring mode* (default): packet trace must
+contain a given sub-string in order to be included in the output -
+*regex mode*: packet trace must match a given regex in order to be
+printed
+
+The script is still limited, in that capture runs only on the RX side of
+all interfaces that are built on top of selected devices. Using
+filtering, however, it is possible to limit *traffic by interface*
+simply by using the interface name as a substring to match against.
+
+Usage
+-----
+
+Run the script with option ``-h`` to get the usage printed:
+
+::
+
+ Usage: ./vpptrace.sh [-i <VPP-IF-TYPE>]... [-a <VPP-ADDRESS>] [-r] [-f <REGEXP> / <SUBSTRING>]
+ -i <VPP-IF-TYPE> : VPP interface *type* to run the packet capture on (e.g., dpdk-input, virtio-input, etc.)
+ - available aliases:
+ - af-packet-input: afpacket, af-packet, veth
+ - virtio-input: tap (version determined from the VPP runtime config), tap2, tapv2
+ - tapcli-rx: tap (version determined from the VPP config), tap1, tapv1
+ - dpdk-input: dpdk, gbe, phys*
+ - multiple interfaces can be watched at the same time - the option can be repeated with
+ different values
+ - default = dpdk + tap
+ -a <VPP-ADDRESS> : IP address or hostname of the VPP to capture packets from
+ - not supported if VPP listens on a UNIX domain socket
+ - default = 127.0.0.1
+ -r : apply filter string (passed with -f) as a regexp expression
+ - by default the filter is NOT treated as regexp
+ -f : filter string that packet must contain (without -r) or match as regexp (with -r) to be printed
+ - default is no filtering
+
+``VPP-IF-TYPE`` is a repeated option used to select the set of devices
+(e.g., virtio, dpdk, etc.) to capture the incoming traffic. Script
+provides multiple aliases, which are much easier to remember than the
+device names. For ``dpdk-input`` one can enter just ``dpdk``, or
+anything starting with ``phys``, etc. For TAPs, the script is even smart
+enough to find out the TAP version used, which allows to enter just
+``tap`` as the device name.
+
+If ``VPP-IF-TYPE`` is not specified, then the default behaviour is to
+capture from both ``dpdk`` (traffic entering the node from outside) and
+``tap`` (preferred interface type for pod-VPP and host-VPP
+interconnection, receiving node-initiated traffic).
+
+vpptrace.sh can capture packets even from a VPP on a different host,
+provided that VPP-CLI listens on a port, and not on a UNIX domain socket
+(for security reasons IPC is the default communication link, see
+``/etc/vpp/contiv-vswitch.conf``). Enter the destination node IP address
+via the option ``-a``\ (localhost is the default).
+
+The capture can be filtered via the ``-f`` option. The output will
+include only packets whose trace matches contain the given
+expression/sub-string.
+
+Option ``-r`` enables the regex mode for filtering.
+
+Examples
+--------
+
+- Capture all packets entering VPP via ``tapcli-1`` interface **AND**
+ all packets leaving VPP via ``tapcli-1`` that were sent from a pod,
+ or the host on the *same node* (sent from tap, not Gbe):
+
+::
+
+ $ vpptrace.sh -i tap -f "tapcli-1"
+
+ - Capture all packets with source or destination IP address 10.1.1.3:
+
+::
+
+ $ vpptrace.sh -i tap -i dpdk -f "10.1.1.3"
+
+ Or just:
+ $ vpptrace.sh "10.1.1.3"
+
+- Capture all SYN-ACKs received from outside:
+
+::
+
+ $ vpptrace.sh -i dpdk -f "SYN-ACK"
diff --git a/docs/usecases/contiv/VPP_CONFIG.md b/docs/usecases/contiv/VPP_CONFIG.md
deleted file mode 100644
index 0d05593..0000000
--- a/docs/usecases/contiv/VPP_CONFIG.md
+++ /dev/null
@@ -1,153 +0,0 @@
-## Creating VPP Startup Configuration
-This document describes how to create the VPP startup configuration
-file located at `/etc/vpp/contiv-vswitch.conf`.
-
-### Hardware Interface Configuration
-#### Single-NIC Configuration
-You need to configure hardware interfaces for use by VPP. First, find out the PCI address of the host's network interface. On
-Debian-based distributions, you can use `lshw`:
-
-```
-sudo lshw -class network -businfo
-Bus info Device Class Description
-========================================================
-pci@0000:03:00.0 ens160 network VMXNET3 Ethernet Controller
-```
-
-In our case, it would be the `ens3` interface with the PCI address
-`0000:00:03.0`
-
-Now, add or modify the VPP startup config file (`/etc/vpp/contiv-vswitch.conf`)
-to contain the proper PCI address:
-```
-unix {
- nodaemon
- cli-listen /run/vpp/cli.sock
- cli-no-pager
- coredump-size unlimited
- full-coredump
- poll-sleep-usec 100
-}
-nat {
- endpoint-dependent
-}
-dpdk {
- dev 0000:00:03.0
-}
-api-trace {
- on
- nitems 500
-}
-```
-#### Multi-NIC Configuration
-Similar to the single-NIC configuration, use command *lshw* to find the PCI
-addresses of all the NICs in the system, for example:
-
-```
-$ sudo lshw -class network -businfo
-Bus info Device Class Description
-====================================================
-pci@0000:00:03.0 ens3 network Virtio network device
-pci@0000:00:04.0 ens4 network Virtio network device
-```
-
-In the example above, `ens3` would be the primary interface and `ens4` would
-be the interface that would be used by VPP. The PCI address of the `ens4`
-interface would be `0000:00:04.0`.
-
-Make sure the selected interface is *shut down*, otherwise VPP
-will not grab it:
-```
-sudo ip link set ens4 down
-```
-
-Now, add or modify the VPP startup config file in `/etc/vpp/contiv-vswitch.conf`
-to contain the proper PCI address:
-```
-unix {
- nodaemon
- cli-listen /run/vpp/cli.sock
- cli-no-pager
- coredump-size unlimited
- full-coredump
- poll-sleep-usec 100
-}
-nat {
- endpoint-dependent
-}
-dpdk {
- dev 0000:00:04.0
-}
-api-trace {
- on
- nitems 500
-}
-```
-If assigning multiple NICs to VPP you will need to include each NIC's PCI address
-in the dpdk stanza in `/etc/vpp/contiv-vswitch.conf`.
-
-##### Assigning all NICs to VPP
-On a multi-NIC node, it is also possible to assign all NICs from the kernel for
-use by VPP. First, you need to install the STN daemon, as described [here][1],
-since you will want the NICs to revert to the kernel if VPP crashes.
-
-You also need to configure the NICs in the VPP startup config file
-in `/etc/vpp/contiv-vswitch.conf`. For example, to use both the primary and
-secondary NIC, in a two-NIC node, your VPP startup config file would look
-something like this:
-
-```
-unix {
- nodaemon
- cli-listen /run/vpp/cli.sock
- cli-no-pager
- coredump-size unlimited
- full-coredump
- poll-sleep-usec 100
-}
-nat {
- endpoint-dependent
-}
-dpdk {
- dev 0000:00:03.0
- dev 0000:00:04.0
-}
-api-trace {
- on
- nitems 500
-}
-```
-
-#### Installing `lshw` on CentOS/RedHat/Fedora
-Note: On CentOS/RedHat/Fedora distributions, `lshw` may not be available
-by default, install it by
-```
-sudo yum -y install lshw
-```
-
-### Power-saving Mode
-In regular operation, VPP takes 100% of one CPU core at all times (poll loop).
-If high performance and low latency is not required you can "slow-down"
-the poll-loop and drastically reduce CPU utilization by adding the following
-stanza to the `unix` section of the VPP startup config file:
-```
-unix {
- ...
- poll-sleep-usec 100
- ...
-}
-```
-The power-saving mode is especially useful in VM-based development environments
-running on laptops or less powerful servers.
-
-### VPP API Trace
-To troubleshoot VPP configuration issues in production environments, it is
-strongly recommended to configure VPP API trace. This is done by adding the
-following stanza to the VPP startup config file:
-```
-api-trace {
- on
- nitems 500
-}
-```
-You can set the size of the trace buffer with the <nitems> attribute.
diff --git a/docs/usecases/contiv/VPP_CONFIG.rst b/docs/usecases/contiv/VPP_CONFIG.rst
new file mode 100644
index 0000000..5dcb6dd
--- /dev/null
+++ b/docs/usecases/contiv/VPP_CONFIG.rst
@@ -0,0 +1,183 @@
+Creating VPP Startup Configuration
+==================================
+
+This document describes how to create the VPP startup configuration file
+located at ``/etc/vpp/contiv-vswitch.conf``.
+
+Hardware Interface Configuration
+--------------------------------
+
+Single-NIC Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+You need to configure hardware interfaces for use by VPP. First, find
+out the PCI address of the host’s network interface. On Debian-based
+distributions, you can use ``lshw``:
+
+::
+
+ sudo lshw -class network -businfo
+ Bus info Device Class Description
+ ========================================================
+ pci@0000:03:00.0 ens160 network VMXNET3 Ethernet Controller
+
+In our case, it would be the ``ens3`` interface with the PCI address
+``0000:00:03.0``
+
+Now, add or modify the VPP startup config file
+(``/etc/vpp/contiv-vswitch.conf``) to contain the proper PCI address:
+
+::
+
+ unix {
+ nodaemon
+ cli-listen /run/vpp/cli.sock
+ cli-no-pager
+ coredump-size unlimited
+ full-coredump
+ poll-sleep-usec 100
+ }
+ nat {
+ endpoint-dependent
+ }
+ dpdk {
+ dev 0000:00:03.0
+ }
+ api-trace {
+ on
+ nitems 500
+ }
+
+Multi-NIC Configuration
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Similar to the single-NIC configuration, use command *lshw* to find the
+PCI addresses of all the NICs in the system, for example:
+
+::
+
+ $ sudo lshw -class network -businfo
+ Bus info Device Class Description
+ ====================================================
+ pci@0000:00:03.0 ens3 network Virtio network device
+ pci@0000:00:04.0 ens4 network Virtio network device
+
+In the example above, ``ens3`` would be the primary interface and
+``ens4`` would be the interface that would be used by VPP. The PCI
+address of the ``ens4`` interface would be ``0000:00:04.0``.
+
+Make sure the selected interface is *shut down*, otherwise VPP will not
+grab it:
+
+::
+
+ sudo ip link set ens4 down
+
+Now, add or modify the VPP startup config file in
+``/etc/vpp/contiv-vswitch.conf`` to contain the proper PCI address:
+
+::
+
+ unix {
+ nodaemon
+ cli-listen /run/vpp/cli.sock
+ cli-no-pager
+ coredump-size unlimited
+ full-coredump
+ poll-sleep-usec 100
+ }
+ nat {
+ endpoint-dependent
+ }
+ dpdk {
+ dev 0000:00:04.0
+ }
+ api-trace {
+ on
+ nitems 500
+ }
+
+If assigning multiple NICs to VPP you will need to include each NIC’s
+PCI address in the dpdk stanza in ``/etc/vpp/contiv-vswitch.conf``.
+
+Assigning all NICs to VPP
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+On a multi-NIC node, it is also possible to assign all NICs from the
+kernel for use by VPP. First, you need to install the STN daemon, as
+described [here][1], since you will want the NICs to revert to the
+kernel if VPP crashes.
+
+You also need to configure the NICs in the VPP startup config file in
+``/etc/vpp/contiv-vswitch.conf``. For example, to use both the primary
+and secondary NIC, in a two-NIC node, your VPP startup config file would
+look something like this:
+
+::
+
+ unix {
+ nodaemon
+ cli-listen /run/vpp/cli.sock
+ cli-no-pager
+ coredump-size unlimited
+ full-coredump
+ poll-sleep-usec 100
+ }
+ nat {
+ endpoint-dependent
+ }
+ dpdk {
+ dev 0000:00:03.0
+ dev 0000:00:04.0
+ }
+ api-trace {
+ on
+ nitems 500
+ }
+
+Installing ``lshw`` on CentOS/RedHat/Fedora
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Note: On CentOS/RedHat/Fedora distributions, ``lshw`` may not be
+available by default, install it by
+
+::
+
+ sudo yum -y install lshw
+
+Power-saving Mode
+-----------------
+
+In regular operation, VPP takes 100% of one CPU core at all times (poll
+loop). If high performance and low latency is not required you can
+“slow-down” the poll-loop and drastically reduce CPU utilization by
+adding the following stanza to the ``unix`` section of the VPP startup
+config file:
+
+::
+
+ unix {
+ ...
+ poll-sleep-usec 100
+ ...
+ }
+
+The power-saving mode is especially useful in VM-based development
+environments running on laptops or less powerful servers.
+
+VPP API Trace
+-------------
+
+To troubleshoot VPP configuration issues in production environments, it
+is strongly recommended to configure VPP API trace. This is done by
+adding the following stanza to the VPP startup config file:
+
+::
+
+ api-trace {
+ on
+ nitems 500
+ }
+
+You can set the size of the trace buffer with the attribute.
+
diff --git a/docs/usecases/contiv/VPP_PACKET_TRACING_K8S.md b/docs/usecases/contiv/VPP_PACKET_TRACING_K8S.md
deleted file mode 100644
index 7409181..0000000
--- a/docs/usecases/contiv/VPP_PACKET_TRACING_K8S.md
+++ /dev/null
@@ -1,510 +0,0 @@
-## How to do VPP Packet Tracing in Kubernetes
-
-This document describes the steps to do *manual* packet tracing (capture) using
-VPP in Kubernetes. Contiv/VPP also ships with a simple bash script
-[vpptrace.sh](https://github.com/contiv/vpp/blob/master/scripts/vpptrace.sh),
-which allows to *continuously* trace and
-*filter* packets incoming through a given set of interface types.
-Documentation for vpptrace.sh is available [here](https://github.com/contiv/vpp/blob/master/docs/VPPTRACE.md).
-
-
-More information about VPP packet tracing is in:
-
-* <https://wiki.fd.io/view/VPP/Command-line_Interface_(CLI)_Guide#packet_tracer>
-* <https://wiki.fd.io/view/VPP/How_To_Use_The_Packet_Generator_and_Packet_Tracer>
-* <https://wiki.fd.io/view/VPP/Tutorial_Routing_and_Switching>
-
-#### SSH into the Node
-Perform the following commands to SSH into the node:
-
-```
-cd vpp/vagrant/vagrant-scripts/
-vagrant ssh k8s-worker1
-```
-
-#### Check the VPP Graph Nodes (Input and Output Queues)
-
-The following content shows what is running on VPP, via the `show run` command
-
-```
-vagrant@k8s-worker1:~$ sudo vppctl
- _______ _ _ _____ ___
- __/ __/ _ \ (_)__ | | / / _ \/ _ \
- _/ _// // / / / _ \ | |/ / ___/ ___/
- /_/ /____(_)_/\___/ |___/_/ /_/
-
-vpp# show run
-Time 1026791.9, average vectors/node 1.12, last 128 main loops 0.00 per node 0.00
- vector rates in 1.6459e-4, out 1.5485e-4, drop 1.3635e-5, punt 0.0000e0
- Name State Calls Vectors Suspends Clocks Vectors/Call
-GigabitEthernet0/8/0-output active 56 69 0 1.34e3 1.23
-GigabitEthernet0/8/0-tx active 54 67 0 8.09e5 1.24
-acl-plugin-fa-cleaner-process event wait 0 0 1 2.84e4 0.00
-admin-up-down-process event wait 0 0 1 4.59e3 0.00
-api-rx-from-ring any wait 0 0 3316292 1.24e5 0.00
-arp-input active 3 3 0 2.53e5 1.00
-bfd-process event wait 0 0 1 5.94e3 0.00
-cdp-process any wait 0 0 145916 1.36e4 0.00
-dhcp-client-process any wait 0 0 10268 3.65e4 0.00
-dns-resolver-process any wait 0 0 1027 5.86e4 0.00
-dpdk-input polling 8211032318951 93 0 1.48e13 0.00
-dpdk-ipsec-process done 1 0 0 2.10e5 0.00
-dpdk-process any wait 0 0 342233 9.86e6 0.00
-error-drop active 12 14 0 6.67e3 1.17
-ethernet-input active 60 74 0 5.81e3 1.23
-fib-walk any wait 0 0 513322 1.59e4 0.00
-flow-report-process any wait 0 0 1 1.45e3 0.00
-flowprobe-timer-process any wait 0 0 1 6.34e3 0.00
-ikev2-manager-process any wait 0 0 1026484 1.18e4 0.00
-interface-output active 2 2 0 3.23e3 1.00
-ioam-export-process any wait 0 0 1 1.98e3 0.00
-ip-route-resolver-process any wait 0 0 10268 3.02e4 0.00
-ip4-arp active 1 1 0 1.49e4 1.00
-ip4-input active 223 248 0 3.39e3 1.11
-ip4-load-balance active 106 132 0 5.34e3 1.25
-ip4-local active 86 92 0 2.46e3 1.07
-ip4-local-end-of-arc active 86 92 0 1.00e3 1.07
-ip4-lookup active 223 248 0 3.31e3 1.11
-ip4-rewrite active 190 222 0 1.92e3 1.17
-ip4-udp-lookup active 86 92 0 3.76e3 1.07
-ip6-drop active 6 7 0 2.29e3 1.17
-ip6-icmp-neighbor-discovery-ev any wait 0 0 1026484 1.13e4 0.00
-ip6-input active 6 7 0 3.33e3 1.17
-l2-flood active 2 2 0 4.42e3 1.00
-l2-fwd active 138 157 0 2.13e3 1.14
-l2-input active 140 159 0 2.41e3 1.14
-l2-learn active 86 92 0 3.64e4 1.07
-l2-output active 54 67 0 3.05e3 1.24
-l2fib-mac-age-scanner-process event wait 0 0 85 5.01e4 0.00
-lisp-retry-service any wait 0 0 513322 1.62e4 0.00
-lldp-process event wait 0 0 1 5.02e4 0.00
-loop0-output active 54 67 0 1.66e3 1.24
-loop0-tx active 54 0 0 2.49e3 0.00
-memif-process event wait 0 0 1 1.70e4 0.00
-nat-det-expire-walk done 1 0 0 3.79e3 0.00
-nat44-classify active 171 183 0 2.49e3 1.07
-nat44-hairpinning active 86 92 0 1.80e3 1.07
-nat44-in2out active 171 183 0 4.45e3 1.07
-nat44-in2out-slowpath active 171 183 0 3.98e3 1.07
-nat44-out2in active 52 65 0 1.28e4 1.25
-nat64-expire-walk any wait 0 0 102677 5.95e4 0.00
-nat64-expire-worker-walk interrupt wa 102676 0 0 7.39e3 0.00
-send-garp-na-process event wait 0 0 1 1.28e3 0.00
-startup-config-process done 1 0 1 4.19e3 0.00
-tapcli-0-output active 1 1 0 6.97e3 1.00
-tapcli-0-tx active 1 1 0 7.32e4 1.00
-tapcli-1-output active 57 63 0 1.66e3 1.11
-tapcli-1-tx active 57 63 0 1.35e5 1.11
-tapcli-2-output active 28 28 0 3.26e3 1.00
-tapcli-2-tx active 28 28 0 4.06e5 1.00
-tapcli-rx interrupt wa 62 76 0 6.58e4 1.23
-udp-ping-process any wait 0 0 1 1.79e4 0.00
-unix-cli-127.0.0.1:43282 active 2 0 455 1.26e15 0.00
-unix-epoll-input polling 8010763239 0 0 8.17e2 0.00
-vhost-user-process any wait 0 0 1 1.96e3 0.00
-vhost-user-send-interrupt-proc any wait 0 0 1 3.85e3 0.00
-vpe-link-state-process event wait 0 0 8 9.79e4 0.00
-vpe-oam-process any wait 0 0 503263 1.21e4 0.00
-vxlan-gpe-ioam-export-process any wait 0 0 1 2.91e3 0.00
-vxlan4-encap active 54 67 0 3.55e3 1.24
-vxlan4-input active 86 92 0 3.79e3 1.07
-wildcard-ip4-arp-publisher-pro event wait 0 0 1 6.44e3 0.00
-```
-
-`tapcli-rx` above is the node-level input queue for incoming packets into all the pods on the node. There is one `tapcli-rx` input queue for every node.
-
-The following are the input and output queues for each pod and the node:
-
-```
-tapcli-0-output
-tapcli-0-tx
-tapcli-1-output
-tapcli-1-tx
-tapcli-2-output
-tapcli-2-tx
-```
-
-Each pod and node has two queues, one for rx (`tapcli-X-output`), and one for tx (`tapcli-X-tx`). The above output is with two `nginx` pods in kubernetes.
-
-#### Clear Existing VPP Packet Trace
-Enter the following command:
-```
-vpp# clear trace
-```
-
-#### How to Turn on VPP Packet Tracing
-Enter the following commands:
-
-```
-vpp# trace add <input or output queue name> <number of packets to capture>
-
-vpp# trace add dpdk-input 1000
-
-vpp# trace add tapcli-rx 1000
-```
-
-#### Send Traffic to the Pods
-
-Open another terminal, SSH into the master node, refer the documentation in `vpp/vagrant/README.md` and send traffic to the two `nginx` pods using `wget`.
-
-```
-cd vpp/vagrant/vagrant-scripts/
-vagrant ssh k8s-master
-
-vagrant@k8s-master:~$ kubectl get pods -o wide
-NAME READY STATUS RESTARTS AGE IP NODE
-nginx-8586cf59-768qw 1/1 Running 0 11d 10.1.2.3 k8s-worker1
-nginx-8586cf59-d27h2 1/1 Running 0 11d 10.1.2.2 k8s-worker1
-
-vagrant@k8s-master:~$ wget 10.1.2.2
---2018-02-08 16:46:01-- http://10.1.2.2/
-Connecting to 10.1.2.2:80... connected.
-HTTP request sent, awaiting response... 200 OK
-Length: 612 [text/html]
-Saving to: ‘index.html’
-index.html 100%[=========================================================>] 612 --.-KB/s in 0.004s
-2018-02-08 16:46:01 (162 KB/s) - ‘index.html’ saved [612/612]
-
-vagrant@k8s-master:~$ wget 10.1.2.3
---2018-02-08 16:46:02-- http://10.1.2.3/
-Connecting to 10.1.2.3:80... connected.
-HTTP request sent, awaiting response... 200 OK
-Length: 612 [text/html]
-Saving to: ‘index.html.1’
-index.html.1 100%[=========================================================>] 612 --.-KB/s in 0.004s
-2018-02-08 16:46:02 (143 KB/s) - ‘index.html.1’ saved [612/612]
-```
-
-#### Check the Packets Captured by VPP
-
-Back in the first terminal, check the packets captured by VPP.
-
-```
-vpp# show trace
-...
-...
-Packet 33
-
-21:34:51:476110: tapcli-rx
- tapcli-2
-21:34:51:476115: ethernet-input
- IP4: 00:00:00:00:00:02 -> 02:fe:72:95:66:c7
-21:34:51:476117: ip4-input
- TCP: 10.1.2.3 -> 172.30.1.2
- tos 0x00, ttl 64, length 52, checksum 0x6fb4
- fragment id 0x11ec, flags DONT_FRAGMENT
- TCP: 80 -> 58430
- seq. 0x5db741c8 ack 0x709defa7
- flags 0x11 FIN ACK, tcp header: 32 bytes
- window 235, checksum 0x55c3
-21:34:51:476118: nat44-out2in
- NAT44_OUT2IN: sw_if_index 6, next index 1, session index -1
-21:34:51:476120: ip4-lookup
- fib 0 dpo-idx 23 flow hash: 0x00000000
- TCP: 10.1.2.3 -> 172.30.1.2
- tos 0x00, ttl 64, length 52, checksum 0x6fb4
- fragment id 0x11ec, flags DONT_FRAGMENT
- TCP: 80 -> 58430
- seq. 0x5db741c8 ack 0x709defa7
- flags 0x11 FIN ACK, tcp header: 32 bytes
- window 235, checksum 0x55c3
-21:34:51:476121: ip4-load-balance
- fib 0 dpo-idx 23 flow hash: 0x00000000
- TCP: 10.1.2.3 -> 172.30.1.2
- tos 0x00, ttl 64, length 52, checksum 0x6fb4
- fragment id 0x11ec, flags DONT_FRAGMENT
- TCP: 80 -> 58430
- seq. 0x5db741c8 ack 0x709defa7
- flags 0x11 FIN ACK, tcp header: 32 bytes
- window 235, checksum 0x55c3
-21:34:51:476122: ip4-rewrite
- tx_sw_if_index 3 dpo-idx 5 : ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800 flow hash: 0x00000000
- 00000000: 1a2b3c4d5e011a2b3c4d5e0208004500003411ec40003f0670b40a010203ac1e
- 00000020: 01020050e43e5db741c8709defa7801100eb55c300000101080a0f4b
-21:34:51:476123: loop0-output
- loop0
- IP4: 1a:2b:3c:4d:5e:02 -> 1a:2b:3c:4d:5e:01
- TCP: 10.1.2.3 -> 172.30.1.2
- tos 0x00, ttl 63, length 52, checksum 0x70b4
- fragment id 0x11ec, flags DONT_FRAGMENT
- TCP: 80 -> 58430
- seq. 0x5db741c8 ack 0x709defa7
- flags 0x11 FIN ACK, tcp header: 32 bytes
- window 235, checksum 0x55c3
-21:34:51:476124: l2-input
- l2-input: sw_if_index 3 dst 1a:2b:3c:4d:5e:01 src 1a:2b:3c:4d:5e:02
-21:34:51:476125: l2-fwd
- l2-fwd: sw_if_index 3 dst 1a:2b:3c:4d:5e:01 src 1a:2b:3c:4d:5e:02 bd_index 1
-21:34:51:476125: l2-output
- l2-output: sw_if_index 4 dst 1a:2b:3c:4d:5e:01 src 1a:2b:3c:4d:5e:02 data 08 00 45 00 00 34 11 ec 40 00 3f 06
-21:34:51:476126: vxlan4-encap
- VXLAN encap to vxlan_tunnel0 vni 10
-21:34:51:476126: ip4-load-balance
- fib 4 dpo-idx 22 flow hash: 0x00000103
- UDP: 192.168.16.2 -> 192.168.16.1
- tos 0x00, ttl 254, length 102, checksum 0x1b33
- fragment id 0x0000
- UDP: 24320 -> 4789
- length 82, checksum 0x0000
-21:34:51:476127: ip4-rewrite
- tx_sw_if_index 1 dpo-idx 4 : ipv4 via 192.168.16.1 GigabitEthernet0/8/0: 080027b2610908002733fb6f0800 flow hash: 0x00000103
- 00000000: 080027b2610908002733fb6f08004500006600000000fd111c33c0a81002c0a8
- 00000020: 10015f0012b5005200000800000000000a001a2b3c4d5e011a2b3c4d
-21:34:51:476127: GigabitEthernet0/8/0-output
- GigabitEthernet0/8/0
- IP4: 08:00:27:33:fb:6f -> 08:00:27:b2:61:09
- UDP: 192.168.16.2 -> 192.168.16.1
- tos 0x00, ttl 253, length 102, checksum 0x1c33
- fragment id 0x0000
- UDP: 24320 -> 4789
- length 82, checksum 0x0000
-21:34:51:476128: GigabitEthernet0/8/0-tx
- GigabitEthernet0/8/0 tx queue 0
- buffer 0xfa7f: current data -50, length 116, free-list 0, clone-count 0, totlen-nifb 0, trace 0x20
- l2-hdr-offset 0 l3-hdr-offset 14
- PKT MBUF: port 255, nb_segs 1, pkt_len 116
- buf_len 2176, data_len 116, ol_flags 0x0, data_off 78, phys_addr 0x569ea040
- packet_type 0x0 l2_len 0 l3_len 0 outer_l2_len 0 outer_l3_len 0
- IP4: 08:00:27:33:fb:6f -> 08:00:27:b2:61:09
- UDP: 192.168.16.2 -> 192.168.16.1
- tos 0x00, ttl 253, length 102, checksum 0x1c33
- fragment id 0x0000
- UDP: 24320 -> 4789
- length 82, checksum 0x0000
-```
-
-In the above captured packet, we can see:
-
-* Input queue name `tapcli-rx`
-* Pod's IP address `10.1.2.3`
-* IP address of the master node `172.30.1.2`, which sent the `wget` traffic to the two pods
-* HTTP port `80`, destination port and TCP protocol (`TCP: 80 -> 58430`)
-* NAT queue name `nat44-out2in`
-* VXLAN VNI ID `VXLAN encap to vxlan_tunnel0 vni 10`
-* VXLAN UDP port `4789`
-* IP address of `GigabitEthernet0/8/0` interface (`192.168.16.2`)
-* Packet on the outgoing queue `GigabitEthernet0/8/0-tx`
-
-#### Find IP Addresses of GigabitEthernet and the Tap Interfaces
-Enter the following commands to find the IP addresses and Tap interfaces:
-
-```
-vpp# show int address
-GigabitEthernet0/8/0 (up):
- L3 192.168.16.2/24
-local0 (dn):
-loop0 (up):
- L2 bridge bd-id 1 idx 1 shg 0 bvi
- L3 192.168.30.2/24
-tapcli-0 (up):
- L3 172.30.2.1/24
-tapcli-1 (up):
- L3 10.2.1.2/32
-tapcli-2 (up):
- L3 10.2.1.3/32
-vxlan_tunnel0 (up):
- L2 bridge bd-id 1 idx 1 shg 0
-```
-
-#### Other Useful VPP CLIs
-
-Enter the following commands to see additional information about VPP:
-
-```
-vpp# show int
- Name Idx State Counter Count
-GigabitEthernet0/8/0 1 up rx packets 138
- rx bytes 18681
- tx packets 100
- tx bytes 29658
- drops 1
- ip4 137
- tx-error 2
-local0 0 down drops 1
-loop0 3 up rx packets 137
- rx bytes 9853
- tx packets 200
- tx bytes 49380
- drops 1
- ip4 136
-tapcli-0 2 up rx packets 8
- rx bytes 600
- tx packets 1
- tx bytes 42
- drops 9
- ip6 7
-tapcli-1 5 up rx packets 56
- rx bytes 13746
- tx packets 78
- tx bytes 6733
- drops 1
- ip4 56
-tapcli-2 6 up rx packets 42
- rx bytes 10860
- tx packets 58
- tx bytes 4996
- drops 1
- ip4 42
-vxlan_tunnel0 4 up rx packets 137
- rx bytes 11771
- tx packets 100
- tx bytes 28290
-
-vpp# show hardware
- Name Idx Link Hardware
-GigabitEthernet0/8/0 1 up GigabitEthernet0/8/0
- Ethernet address 08:00:27:33:fb:6f
- Intel 82540EM (e1000)
- carrier up full duplex speed 1000 mtu 9216
- rx queues 1, rx desc 1024, tx queues 1, tx desc 1024
- cpu socket 0
-
- tx frames ok 100
- tx bytes ok 29658
- rx frames ok 138
- rx bytes ok 19233
- extended stats:
- rx good packets 138
- tx good packets 100
- rx good bytes 19233
- tx good bytes 29658
-local0 0 down local0
- local
-loop0 3 up loop0
- Ethernet address 1a:2b:3c:4d:5e:02
-tapcli-0 2 up tapcli-0
- Ethernet address 02:fe:95:07:df:9c
-tapcli-1 5 up tapcli-1
- Ethernet address 02:fe:3f:5f:0f:9a
-tapcli-2 6 up tapcli-2
- Ethernet address 02:fe:72:95:66:c7
-vxlan_tunnel0 4 up vxlan_tunnel0
- VXLAN
-
-vpp# show bridge-domain
- BD-ID Index BSN Age(min) Learning U-Forwrd UU-Flood Flooding ARP-Term BVI-Intf
- 1 1 1 off on on on on off loop0
-
-vpp# show bridge-domain 1 detail
- BD-ID Index BSN Age(min) Learning U-Forwrd UU-Flood Flooding ARP-Term BVI-Intf
- 1 1 1 off on on on on off loop0
-
- Interface If-idx ISN SHG BVI TxFlood VLAN-Tag-Rewrite
- loop0 3 3 0 * * none
- vxlan_tunnel0 4 1 0 - * none
-
-vpp# show l2fib verbose
- Mac-Address BD-Idx If-Idx BSN-ISN Age(min) static filter bvi Interface-Name
- 1a:2b:3c:4d:5e:02 1 3 0/0 - * - * loop0
- 1a:2b:3c:4d:5e:01 1 4 1/1 - - - - vxlan_tunnel0
-L2FIB total/learned entries: 2/1 Last scan time: 0.0000e0sec Learn limit: 4194304
-
-vpp# show ip fib
-ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] locks:[src:(nil):2, src:adjacency:3, src:default-route:1, ]
-0.0.0.0/0
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:1 buckets:1 uRPF:21 to:[0:0]]
- [0] [@5]: ipv4 via 172.30.2.2 tapcli-0: def35b93961902fe9507df9c0800
-0.0.0.0/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:2 buckets:1 uRPF:1 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-10.1.1.0/24
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:24 buckets:1 uRPF:29 to:[0:0]]
- [0] [@10]: dpo-load-balance: [proto:ip4 index:23 buckets:1 uRPF:28 to:[0:0] via:[98:23234]]
- [0] [@5]: ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800
-10.1.2.2/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:27 buckets:1 uRPF:12 to:[78:5641]]
- [0] [@5]: ipv4 via 10.1.2.2 tapcli-1: 00000000000202fe3f5f0f9a0800
-10.1.2.3/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:29 buckets:1 uRPF:32 to:[58:4184]]
- [0] [@5]: ipv4 via 10.1.2.3 tapcli-2: 00000000000202fe729566c70800
-10.2.1.2/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:26 buckets:1 uRPF:31 to:[0:0]]
- [0] [@2]: dpo-receive: 10.2.1.2 on tapcli-1
-10.2.1.3/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:28 buckets:1 uRPF:33 to:[0:0]]
- [0] [@2]: dpo-receive: 10.2.1.3 on tapcli-2
-172.30.1.0/24
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:25 buckets:1 uRPF:29 to:[98:23234]]
- [0] [@10]: dpo-load-balance: [proto:ip4 index:23 buckets:1 uRPF:28 to:[0:0] via:[98:23234]]
- [0] [@5]: ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800
-172.30.2.0/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:14 buckets:1 uRPF:15 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-172.30.2.0/24
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:13 buckets:1 uRPF:14 to:[0:0]]
- [0] [@4]: ipv4-glean: tapcli-0
-172.30.2.1/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:16 buckets:1 uRPF:19 to:[0:0]]
- [0] [@2]: dpo-receive: 172.30.2.1 on tapcli-0
-172.30.2.2/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:17 buckets:1 uRPF:18 to:[0:0]]
- [0] [@5]: ipv4 via 172.30.2.2 tapcli-0: def35b93961902fe9507df9c0800
-172.30.2.255/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:15 buckets:1 uRPF:17 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-192.168.16.0/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:10 buckets:1 uRPF:9 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-192.168.16.1/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:22 buckets:1 uRPF:34 to:[0:0] via:[100:28290]]
- [0] [@5]: ipv4 via 192.168.16.1 GigabitEthernet0/8/0: 080027b2610908002733fb6f0800
-192.168.16.0/24
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:9 buckets:1 uRPF:30 to:[0:0]]
- [0] [@4]: ipv4-glean: GigabitEthernet0/8/0
-192.168.16.2/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:12 buckets:1 uRPF:13 to:[137:16703]]
- [0] [@2]: dpo-receive: 192.168.16.2 on GigabitEthernet0/8/0
-192.168.16.255/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:11 buckets:1 uRPF:11 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-192.168.30.0/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:19 buckets:1 uRPF:23 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-192.168.30.1/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:23 buckets:1 uRPF:28 to:[0:0] via:[98:23234]]
- [0] [@5]: ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800
-192.168.30.0/24
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:18 buckets:1 uRPF:22 to:[0:0]]
- [0] [@4]: ipv4-glean: loop0
-192.168.30.2/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:21 buckets:1 uRPF:27 to:[0:0]]
- [0] [@2]: dpo-receive: 192.168.30.2 on loop0
-192.168.30.255/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:20 buckets:1 uRPF:25 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-224.0.0.0/4
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:4 buckets:1 uRPF:3 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-240.0.0.0/4
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:3 buckets:1 uRPF:2 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-255.255.255.255/32
- unicast-ip4-chain
- [@0]: dpo-load-balance: [proto:ip4 index:5 buckets:1 uRPF:4 to:[0:0]]
- [0] [@0]: dpo-drop ip4
-```
diff --git a/docs/usecases/contiv/VPP_PACKET_TRACING_K8S.rst b/docs/usecases/contiv/VPP_PACKET_TRACING_K8S.rst
new file mode 100644
index 0000000..765d657
--- /dev/null
+++ b/docs/usecases/contiv/VPP_PACKET_TRACING_K8S.rst
@@ -0,0 +1,535 @@
+How to do VPP Packet Tracing in Kubernetes
+==========================================
+
+This document describes the steps to do *manual* packet tracing
+(capture) using VPP in Kubernetes. Contiv/VPP also ships with a simple
+bash script
+`vpptrace.sh <https://github.com/contiv/vpp/blob/master/scripts/vpptrace.sh>`__,
+which allows to *continuously* trace and *filter* packets incoming
+through a given set of interface types. Documentation for vpptrace.sh is
+available
+`here <https://github.com/contiv/vpp/blob/master/docs/VPPTRACE.md>`__.
+
+More information about VPP packet tracing is in:
+
+- https://wiki.fd.io/view/VPP/Command-line_Interface_(CLI)_Guide#packet_tracer
+- https://wiki.fd.io/view/VPP/How_To_Use_The_Packet_Generator_and_Packet_Tracer
+- https://wiki.fd.io/view/VPP/Tutorial_Routing_and_Switching
+
+SSH into the Node
+-----------------
+
+Perform the following commands to SSH into the node:
+
+::
+
+ cd vpp/vagrant/vagrant-scripts/
+ vagrant ssh k8s-worker1
+
+Check the VPP Graph Nodes (Input and Output Queues)
+---------------------------------------------------
+
+The following content shows what is running on VPP, via the ``show run``
+command
+
+::
+
+ vagrant@k8s-worker1:~$ sudo vppctl
+ _______ _ _ _____ ___
+ __/ __/ _ \ (_)__ | | / / _ \/ _ \
+ _/ _// // / / / _ \ | |/ / ___/ ___/
+ /_/ /____(_)_/\___/ |___/_/ /_/
+
+ vpp# show run
+ Time 1026791.9, average vectors/node 1.12, last 128 main loops 0.00 per node 0.00
+ vector rates in 1.6459e-4, out 1.5485e-4, drop 1.3635e-5, punt 0.0000e0
+ Name State Calls Vectors Suspends Clocks Vectors/Call
+ GigabitEthernet0/8/0-output active 56 69 0 1.34e3 1.23
+ GigabitEthernet0/8/0-tx active 54 67 0 8.09e5 1.24
+ acl-plugin-fa-cleaner-process event wait 0 0 1 2.84e4 0.00
+ admin-up-down-process event wait 0 0 1 4.59e3 0.00
+ api-rx-from-ring any wait 0 0 3316292 1.24e5 0.00
+ arp-input active 3 3 0 2.53e5 1.00
+ bfd-process event wait 0 0 1 5.94e3 0.00
+ cdp-process any wait 0 0 145916 1.36e4 0.00
+ dhcp-client-process any wait 0 0 10268 3.65e4 0.00
+ dns-resolver-process any wait 0 0 1027 5.86e4 0.00
+ dpdk-input polling 8211032318951 93 0 1.48e13 0.00
+ dpdk-ipsec-process done 1 0 0 2.10e5 0.00
+ dpdk-process any wait 0 0 342233 9.86e6 0.00
+ error-drop active 12 14 0 6.67e3 1.17
+ ethernet-input active 60 74 0 5.81e3 1.23
+ fib-walk any wait 0 0 513322 1.59e4 0.00
+ flow-report-process any wait 0 0 1 1.45e3 0.00
+ flowprobe-timer-process any wait 0 0 1 6.34e3 0.00
+ ikev2-manager-process any wait 0 0 1026484 1.18e4 0.00
+ interface-output active 2 2 0 3.23e3 1.00
+ ioam-export-process any wait 0 0 1 1.98e3 0.00
+ ip-route-resolver-process any wait 0 0 10268 3.02e4 0.00
+ ip4-arp active 1 1 0 1.49e4 1.00
+ ip4-input active 223 248 0 3.39e3 1.11
+ ip4-load-balance active 106 132 0 5.34e3 1.25
+ ip4-local active 86 92 0 2.46e3 1.07
+ ip4-local-end-of-arc active 86 92 0 1.00e3 1.07
+ ip4-lookup active 223 248 0 3.31e3 1.11
+ ip4-rewrite active 190 222 0 1.92e3 1.17
+ ip4-udp-lookup active 86 92 0 3.76e3 1.07
+ ip6-drop active 6 7 0 2.29e3 1.17
+ ip6-icmp-neighbor-discovery-ev any wait 0 0 1026484 1.13e4 0.00
+ ip6-input active 6 7 0 3.33e3 1.17
+ l2-flood active 2 2 0 4.42e3 1.00
+ l2-fwd active 138 157 0 2.13e3 1.14
+ l2-input active 140 159 0 2.41e3 1.14
+ l2-learn active 86 92 0 3.64e4 1.07
+ l2-output active 54 67 0 3.05e3 1.24
+ l2fib-mac-age-scanner-process event wait 0 0 85 5.01e4 0.00
+ lisp-retry-service any wait 0 0 513322 1.62e4 0.00
+ lldp-process event wait 0 0 1 5.02e4 0.00
+ loop0-output active 54 67 0 1.66e3 1.24
+ loop0-tx active 54 0 0 2.49e3 0.00
+ memif-process event wait 0 0 1 1.70e4 0.00
+ nat-det-expire-walk done 1 0 0 3.79e3 0.00
+ nat44-classify active 171 183 0 2.49e3 1.07
+ nat44-hairpinning active 86 92 0 1.80e3 1.07
+ nat44-in2out active 171 183 0 4.45e3 1.07
+ nat44-in2out-slowpath active 171 183 0 3.98e3 1.07
+ nat44-out2in active 52 65 0 1.28e4 1.25
+ nat64-expire-walk any wait 0 0 102677 5.95e4 0.00
+ nat64-expire-worker-walk interrupt wa 102676 0 0 7.39e3 0.00
+ send-garp-na-process event wait 0 0 1 1.28e3 0.00
+ startup-config-process done 1 0 1 4.19e3 0.00
+ tapcli-0-output active 1 1 0 6.97e3 1.00
+ tapcli-0-tx active 1 1 0 7.32e4 1.00
+ tapcli-1-output active 57 63 0 1.66e3 1.11
+ tapcli-1-tx active 57 63 0 1.35e5 1.11
+ tapcli-2-output active 28 28 0 3.26e3 1.00
+ tapcli-2-tx active 28 28 0 4.06e5 1.00
+ tapcli-rx interrupt wa 62 76 0 6.58e4 1.23
+ udp-ping-process any wait 0 0 1 1.79e4 0.00
+ unix-cli-127.0.0.1:43282 active 2 0 455 1.26e15 0.00
+ unix-epoll-input polling 8010763239 0 0 8.17e2 0.00
+ vhost-user-process any wait 0 0 1 1.96e3 0.00
+ vhost-user-send-interrupt-proc any wait 0 0 1 3.85e3 0.00
+ vpe-link-state-process event wait 0 0 8 9.79e4 0.00
+ vpe-oam-process any wait 0 0 503263 1.21e4 0.00
+ vxlan-gpe-ioam-export-process any wait 0 0 1 2.91e3 0.00
+ vxlan4-encap active 54 67 0 3.55e3 1.24
+ vxlan4-input active 86 92 0 3.79e3 1.07
+ wildcard-ip4-arp-publisher-pro event wait 0 0 1 6.44e3 0.00
+
+``tapcli-rx`` above is the node-level input queue for incoming packets
+into all the pods on the node. There is one ``tapcli-rx`` input queue
+for every node.
+
+The following are the input and output queues for each pod and the node:
+
+::
+
+ tapcli-0-output
+ tapcli-0-tx
+ tapcli-1-output
+ tapcli-1-tx
+ tapcli-2-output
+ tapcli-2-tx
+
+Each pod and node has two queues, one for rx (``tapcli-X-output``), and
+one for tx (``tapcli-X-tx``). The above output is with two ``nginx``
+pods in kubernetes.
+
+Clear Existing VPP Packet Trace
+-------------------------------
+
+Enter the following command:
+
+::
+
+ vpp# clear trace
+
+How to Turn on VPP Packet Tracing
+---------------------------------
+
+Enter the following commands:
+
+::
+
+ vpp# trace add <input or output queue name> <number of packets to capture>
+
+ vpp# trace add dpdk-input 1000
+
+ vpp# trace add tapcli-rx 1000
+
+Send Traffic to the Pods
+------------------------
+
+Open another terminal, SSH into the master node, refer the documentation
+in ``vpp/vagrant/README.md`` and send traffic to the two ``nginx`` pods
+using ``wget``.
+
+::
+
+ cd vpp/vagrant/vagrant-scripts/
+ vagrant ssh k8s-master
+
+ vagrant@k8s-master:~$ kubectl get pods -o wide
+ NAME READY STATUS RESTARTS AGE IP NODE
+ nginx-8586cf59-768qw 1/1 Running 0 11d 10.1.2.3 k8s-worker1
+ nginx-8586cf59-d27h2 1/1 Running 0 11d 10.1.2.2 k8s-worker1
+
+ vagrant@k8s-master:~$ wget 10.1.2.2
+ --2018-02-08 16:46:01-- http://10.1.2.2/
+ Connecting to 10.1.2.2:80... connected.
+ HTTP request sent, awaiting response... 200 OK
+ Length: 612 [text/html]
+ Saving to: ‘index.html’
+ index.html 100%[=========================================================>] 612 --.-KB/s in 0.004s
+ 2018-02-08 16:46:01 (162 KB/s) - ‘index.html’ saved [612/612]
+
+ vagrant@k8s-master:~$ wget 10.1.2.3
+ --2018-02-08 16:46:02-- http://10.1.2.3/
+ Connecting to 10.1.2.3:80... connected.
+ HTTP request sent, awaiting response... 200 OK
+ Length: 612 [text/html]
+ Saving to: ‘index.html.1’
+ index.html.1 100%[=========================================================>] 612 --.-KB/s in 0.004s
+ 2018-02-08 16:46:02 (143 KB/s) - ‘index.html.1’ saved [612/612]
+
+Check the Packets Captured by VPP
+---------------------------------
+
+Back in the first terminal, check the packets captured by VPP.
+
+::
+
+ vpp# show trace
+ ...
+ ...
+ Packet 33
+
+ 21:34:51:476110: tapcli-rx
+ tapcli-2
+ 21:34:51:476115: ethernet-input
+ IP4: 00:00:00:00:00:02 -> 02:fe:72:95:66:c7
+ 21:34:51:476117: ip4-input
+ TCP: 10.1.2.3 -> 172.30.1.2
+ tos 0x00, ttl 64, length 52, checksum 0x6fb4
+ fragment id 0x11ec, flags DONT_FRAGMENT
+ TCP: 80 -> 58430
+ seq. 0x5db741c8 ack 0x709defa7
+ flags 0x11 FIN ACK, tcp header: 32 bytes
+ window 235, checksum 0x55c3
+ 21:34:51:476118: nat44-out2in
+ NAT44_OUT2IN: sw_if_index 6, next index 1, session index -1
+ 21:34:51:476120: ip4-lookup
+ fib 0 dpo-idx 23 flow hash: 0x00000000
+ TCP: 10.1.2.3 -> 172.30.1.2
+ tos 0x00, ttl 64, length 52, checksum 0x6fb4
+ fragment id 0x11ec, flags DONT_FRAGMENT
+ TCP: 80 -> 58430
+ seq. 0x5db741c8 ack 0x709defa7
+ flags 0x11 FIN ACK, tcp header: 32 bytes
+ window 235, checksum 0x55c3
+ 21:34:51:476121: ip4-load-balance
+ fib 0 dpo-idx 23 flow hash: 0x00000000
+ TCP: 10.1.2.3 -> 172.30.1.2
+ tos 0x00, ttl 64, length 52, checksum 0x6fb4
+ fragment id 0x11ec, flags DONT_FRAGMENT
+ TCP: 80 -> 58430
+ seq. 0x5db741c8 ack 0x709defa7
+ flags 0x11 FIN ACK, tcp header: 32 bytes
+ window 235, checksum 0x55c3
+ 21:34:51:476122: ip4-rewrite
+ tx_sw_if_index 3 dpo-idx 5 : ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800 flow hash: 0x00000000
+ 00000000: 1a2b3c4d5e011a2b3c4d5e0208004500003411ec40003f0670b40a010203ac1e
+ 00000020: 01020050e43e5db741c8709defa7801100eb55c300000101080a0f4b
+ 21:34:51:476123: loop0-output
+ loop0
+ IP4: 1a:2b:3c:4d:5e:02 -> 1a:2b:3c:4d:5e:01
+ TCP: 10.1.2.3 -> 172.30.1.2
+ tos 0x00, ttl 63, length 52, checksum 0x70b4
+ fragment id 0x11ec, flags DONT_FRAGMENT
+ TCP: 80 -> 58430
+ seq. 0x5db741c8 ack 0x709defa7
+ flags 0x11 FIN ACK, tcp header: 32 bytes
+ window 235, checksum 0x55c3
+ 21:34:51:476124: l2-input
+ l2-input: sw_if_index 3 dst 1a:2b:3c:4d:5e:01 src 1a:2b:3c:4d:5e:02
+ 21:34:51:476125: l2-fwd
+ l2-fwd: sw_if_index 3 dst 1a:2b:3c:4d:5e:01 src 1a:2b:3c:4d:5e:02 bd_index 1
+ 21:34:51:476125: l2-output
+ l2-output: sw_if_index 4 dst 1a:2b:3c:4d:5e:01 src 1a:2b:3c:4d:5e:02 data 08 00 45 00 00 34 11 ec 40 00 3f 06
+ 21:34:51:476126: vxlan4-encap
+ VXLAN encap to vxlan_tunnel0 vni 10
+ 21:34:51:476126: ip4-load-balance
+ fib 4 dpo-idx 22 flow hash: 0x00000103
+ UDP: 192.168.16.2 -> 192.168.16.1
+ tos 0x00, ttl 254, length 102, checksum 0x1b33
+ fragment id 0x0000
+ UDP: 24320 -> 4789
+ length 82, checksum 0x0000
+ 21:34:51:476127: ip4-rewrite
+ tx_sw_if_index 1 dpo-idx 4 : ipv4 via 192.168.16.1 GigabitEthernet0/8/0: 080027b2610908002733fb6f0800 flow hash: 0x00000103
+ 00000000: 080027b2610908002733fb6f08004500006600000000fd111c33c0a81002c0a8
+ 00000020: 10015f0012b5005200000800000000000a001a2b3c4d5e011a2b3c4d
+ 21:34:51:476127: GigabitEthernet0/8/0-output
+ GigabitEthernet0/8/0
+ IP4: 08:00:27:33:fb:6f -> 08:00:27:b2:61:09
+ UDP: 192.168.16.2 -> 192.168.16.1
+ tos 0x00, ttl 253, length 102, checksum 0x1c33
+ fragment id 0x0000
+ UDP: 24320 -> 4789
+ length 82, checksum 0x0000
+ 21:34:51:476128: GigabitEthernet0/8/0-tx
+ GigabitEthernet0/8/0 tx queue 0
+ buffer 0xfa7f: current data -50, length 116, free-list 0, clone-count 0, totlen-nifb 0, trace 0x20
+ l2-hdr-offset 0 l3-hdr-offset 14
+ PKT MBUF: port 255, nb_segs 1, pkt_len 116
+ buf_len 2176, data_len 116, ol_flags 0x0, data_off 78, phys_addr 0x569ea040
+ packet_type 0x0 l2_len 0 l3_len 0 outer_l2_len 0 outer_l3_len 0
+ IP4: 08:00:27:33:fb:6f -> 08:00:27:b2:61:09
+ UDP: 192.168.16.2 -> 192.168.16.1
+ tos 0x00, ttl 253, length 102, checksum 0x1c33
+ fragment id 0x0000
+ UDP: 24320 -> 4789
+ length 82, checksum 0x0000
+
+In the above captured packet, we can see:
+
+- Input queue name ``tapcli-rx``
+- Pod’s IP address ``10.1.2.3``
+- IP address of the master node ``172.30.1.2``, which sent the ``wget``
+ traffic to the two pods
+- HTTP port ``80``, destination port and TCP protocol
+ (``TCP: 80 -> 58430``)
+- NAT queue name ``nat44-out2in``
+- VXLAN VNI ID ``VXLAN encap to vxlan_tunnel0 vni 10``
+- VXLAN UDP port ``4789``
+- IP address of ``GigabitEthernet0/8/0`` interface (``192.168.16.2``)
+- Packet on the outgoing queue ``GigabitEthernet0/8/0-tx``
+
+Find IP Addresses of GigabitEthernet and the Tap Interfaces
+-----------------------------------------------------------
+
+Enter the following commands to find the IP addresses and Tap
+interfaces:
+
+::
+
+ vpp# show int address
+ GigabitEthernet0/8/0 (up):
+ L3 192.168.16.2/24
+ local0 (dn):
+ loop0 (up):
+ L2 bridge bd-id 1 idx 1 shg 0 bvi
+ L3 192.168.30.2/24
+ tapcli-0 (up):
+ L3 172.30.2.1/24
+ tapcli-1 (up):
+ L3 10.2.1.2/32
+ tapcli-2 (up):
+ L3 10.2.1.3/32
+ vxlan_tunnel0 (up):
+ L2 bridge bd-id 1 idx 1 shg 0
+
+Other Useful VPP CLIs
+---------------------
+
+Enter the following commands to see additional information about VPP:
+
+::
+
+ vpp# show int
+ Name Idx State Counter Count
+ GigabitEthernet0/8/0 1 up rx packets 138
+ rx bytes 18681
+ tx packets 100
+ tx bytes 29658
+ drops 1
+ ip4 137
+ tx-error 2
+ local0 0 down drops 1
+ loop0 3 up rx packets 137
+ rx bytes 9853
+ tx packets 200
+ tx bytes 49380
+ drops 1
+ ip4 136
+ tapcli-0 2 up rx packets 8
+ rx bytes 600
+ tx packets 1
+ tx bytes 42
+ drops 9
+ ip6 7
+ tapcli-1 5 up rx packets 56
+ rx bytes 13746
+ tx packets 78
+ tx bytes 6733
+ drops 1
+ ip4 56
+ tapcli-2 6 up rx packets 42
+ rx bytes 10860
+ tx packets 58
+ tx bytes 4996
+ drops 1
+ ip4 42
+ vxlan_tunnel0 4 up rx packets 137
+ rx bytes 11771
+ tx packets 100
+ tx bytes 28290
+
+ vpp# show hardware
+ Name Idx Link Hardware
+ GigabitEthernet0/8/0 1 up GigabitEthernet0/8/0
+ Ethernet address 08:00:27:33:fb:6f
+ Intel 82540EM (e1000)
+ carrier up full duplex speed 1000 mtu 9216
+ rx queues 1, rx desc 1024, tx queues 1, tx desc 1024
+ cpu socket 0
+
+ tx frames ok 100
+ tx bytes ok 29658
+ rx frames ok 138
+ rx bytes ok 19233
+ extended stats:
+ rx good packets 138
+ tx good packets 100
+ rx good bytes 19233
+ tx good bytes 29658
+ local0 0 down local0
+ local
+ loop0 3 up loop0
+ Ethernet address 1a:2b:3c:4d:5e:02
+ tapcli-0 2 up tapcli-0
+ Ethernet address 02:fe:95:07:df:9c
+ tapcli-1 5 up tapcli-1
+ Ethernet address 02:fe:3f:5f:0f:9a
+ tapcli-2 6 up tapcli-2
+ Ethernet address 02:fe:72:95:66:c7
+ vxlan_tunnel0 4 up vxlan_tunnel0
+ VXLAN
+
+ vpp# show bridge-domain
+ BD-ID Index BSN Age(min) Learning U-Forwrd UU-Flood Flooding ARP-Term BVI-Intf
+ 1 1 1 off on on on on off loop0
+
+ vpp# show bridge-domain 1 detail
+ BD-ID Index BSN Age(min) Learning U-Forwrd UU-Flood Flooding ARP-Term BVI-Intf
+ 1 1 1 off on on on on off loop0
+
+ Interface If-idx ISN SHG BVI TxFlood VLAN-Tag-Rewrite
+ loop0 3 3 0 * * none
+ vxlan_tunnel0 4 1 0 - * none
+
+ vpp# show l2fib verbose
+ Mac-Address BD-Idx If-Idx BSN-ISN Age(min) static filter bvi Interface-Name
+ 1a:2b:3c:4d:5e:02 1 3 0/0 - * - * loop0
+ 1a:2b:3c:4d:5e:01 1 4 1/1 - - - - vxlan_tunnel0
+ L2FIB total/learned entries: 2/1 Last scan time: 0.0000e0sec Learn limit: 4194304
+
+ vpp# show ip fib
+ ipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport proto ] locks:[src:(nil):2, src:adjacency:3, src:default-route:1, ]
+ 0.0.0.0/0
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:1 buckets:1 uRPF:21 to:[0:0]]
+ [0] [@5]: ipv4 via 172.30.2.2 tapcli-0: def35b93961902fe9507df9c0800
+ 0.0.0.0/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:2 buckets:1 uRPF:1 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+ 10.1.1.0/24
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:24 buckets:1 uRPF:29 to:[0:0]]
+ [0] [@10]: dpo-load-balance: [proto:ip4 index:23 buckets:1 uRPF:28 to:[0:0] via:[98:23234]]
+ [0] [@5]: ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800
+ 10.1.2.2/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:27 buckets:1 uRPF:12 to:[78:5641]]
+ [0] [@5]: ipv4 via 10.1.2.2 tapcli-1: 00000000000202fe3f5f0f9a0800
+ 10.1.2.3/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:29 buckets:1 uRPF:32 to:[58:4184]]
+ [0] [@5]: ipv4 via 10.1.2.3 tapcli-2: 00000000000202fe729566c70800
+ 10.2.1.2/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:26 buckets:1 uRPF:31 to:[0:0]]
+ [0] [@2]: dpo-receive: 10.2.1.2 on tapcli-1
+ 10.2.1.3/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:28 buckets:1 uRPF:33 to:[0:0]]
+ [0] [@2]: dpo-receive: 10.2.1.3 on tapcli-2
+ 172.30.1.0/24
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:25 buckets:1 uRPF:29 to:[98:23234]]
+ [0] [@10]: dpo-load-balance: [proto:ip4 index:23 buckets:1 uRPF:28 to:[0:0] via:[98:23234]]
+ [0] [@5]: ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800
+ 172.30.2.0/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:14 buckets:1 uRPF:15 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+ 172.30.2.0/24
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:13 buckets:1 uRPF:14 to:[0:0]]
+ [0] [@4]: ipv4-glean: tapcli-0
+ 172.30.2.1/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:16 buckets:1 uRPF:19 to:[0:0]]
+ [0] [@2]: dpo-receive: 172.30.2.1 on tapcli-0
+ 172.30.2.2/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:17 buckets:1 uRPF:18 to:[0:0]]
+ [0] [@5]: ipv4 via 172.30.2.2 tapcli-0: def35b93961902fe9507df9c0800
+ 172.30.2.255/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:15 buckets:1 uRPF:17 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+ 192.168.16.0/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:10 buckets:1 uRPF:9 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+ 192.168.16.1/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:22 buckets:1 uRPF:34 to:[0:0] via:[100:28290]]
+ [0] [@5]: ipv4 via 192.168.16.1 GigabitEthernet0/8/0: 080027b2610908002733fb6f0800
+ 192.168.16.0/24
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:9 buckets:1 uRPF:30 to:[0:0]]
+ [0] [@4]: ipv4-glean: GigabitEthernet0/8/0
+ 192.168.16.2/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:12 buckets:1 uRPF:13 to:[137:16703]]
+ [0] [@2]: dpo-receive: 192.168.16.2 on GigabitEthernet0/8/0
+ 192.168.16.255/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:11 buckets:1 uRPF:11 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+ 192.168.30.0/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:19 buckets:1 uRPF:23 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+ 192.168.30.1/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:23 buckets:1 uRPF:28 to:[0:0] via:[98:23234]]
+ [0] [@5]: ipv4 via 192.168.30.1 loop0: 1a2b3c4d5e011a2b3c4d5e020800
+ 192.168.30.0/24
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:18 buckets:1 uRPF:22 to:[0:0]]
+ [0] [@4]: ipv4-glean: loop0
+ 192.168.30.2/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:21 buckets:1 uRPF:27 to:[0:0]]
+ [0] [@2]: dpo-receive: 192.168.30.2 on loop0
+ 192.168.30.255/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:20 buckets:1 uRPF:25 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+ 224.0.0.0/4
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:4 buckets:1 uRPF:3 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+ 240.0.0.0/4
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:3 buckets:1 uRPF:2 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
+ 255.255.255.255/32
+ unicast-ip4-chain
+ [@0]: dpo-load-balance: [proto:ip4 index:5 buckets:1 uRPF:4 to:[0:0]]
+ [0] [@0]: dpo-drop ip4
diff --git a/docs/usecases/contiv/Vagrant.md b/docs/usecases/contiv/Vagrant.md
deleted file mode 100644
index a9040a6..0000000
--- a/docs/usecases/contiv/Vagrant.md
+++ /dev/null
@@ -1,250 +0,0 @@
-## Contiv-VPP Vagrant Installation
-
-### Prerequisites
-The following items are prerequisites before installing vagrant:
-- Vagrant 2.0.1 or later
-- Hypervisors:
- - VirtualBox 5.2.8 or later
- - VMWare Fusion 10.1.0 or later or VmWare Workstation 14
- - For VmWare Fusion, you will need the [Vagrant VmWare Fusion plugin](https://www.vagrantup.com/vmware/index.html)
-- Laptop or server with at least 4 CPU cores and 16 Gig of RAM
-
-### Creating / Shutting Down / Destroying the Cluster
-This folder contains the Vagrant file that is used to create a single or multi-node
-Kubernetes cluster using Contiv-VPP as a Network Plugin.
-
-The folder is organized into two subfolders:
-
- - (config) - contains the files that share cluster information, which are used
- during the provisioning stage (master IP address, Certificates, hash-keys).
- **CAUTION:** Editing is not recommended!
- - (vagrant) - contains scripts that are used for creating, destroying, rebooting
- and shutting down the VMs that host the K8s cluster.
-
-To create and run a K8s cluster with a *contiv-vpp CNI* plugin, run the
-`vagrant-start` script, located in the [vagrant folder](https://github.com/contiv/vpp/tree/master/vagrant). The `vagrant-start`
-script prompts the user to select the number of worker nodes for the kubernetes cluster.
-Zero (0) worker nodes mean that a single-node cluster (with one kubernetes master node) will be deployed.
-
-Next, the user is prompted to select either the *production environment* or the *development environment*.
-Instructions on how to build the development *contiv/vpp-vswitch* image can be found below in the
-[development environment](#building-and-deploying-the-dev-contiv-vswitch-image) command section.
-
-The last option asks the user to select either *Without StealTheNIC* or *With StealTheNIC*.
-Using option *With StealTheNIC* has the plugin "steal" interfaces owned by Linux and uses their configuration in VPP.
-
-For the production environment, enter the following commands:
-```
-| => ./vagrant-start
-Please provide the number of workers for the Kubernetes cluster (0-50) or enter [Q/q] to exit: 1
-
-Please choose Kubernetes environment:
-1) Production
-2) Development
-3) Quit
---> 1
-You chose Development environment
-
-Please choose deployment scenario:
-1) Without StealTheNIC
-2) With StealTheNIC
-3) Quit
---> 1
-You chose deployment without StealTheNIC
-
-Creating a production environment, without STN and 1 worker node(s)
-```
-
-For the development environment, enter the following commands:
-```
-| => ./vagrant-start
-Please provide the number of workers for the Kubernetes cluster (0-50) or enter [Q/q] to exit: 1
-
-Please choose Kubernetes environment:
-1) Production
-2) Development
-3) Quit
---> 2
-You chose Development environment
-
-Please choose deployment scenario:
-1) Without StealTheNIC
-2) With StealTheNIC
-3) Quit
---> 1
-You chose deployment without StealTheNIC
-
-Creating a development environment, without STN and 1 worker node(s)
-```
-
-To destroy and clean-up the cluster, run the *vagrant-cleanup* script, located
-[inside the vagrant folder](https://github.com/contiv/vpp/tree/master/vagrant):
-```
-cd vagrant/
-./vagrant-cleanup
-```
-
-To shutdown the cluster, run the *vagrant-shutdown* script, located [inside the vagrant folder](https://github.com/contiv/vpp/tree/master/vagrant):
-```
-cd vagrant/
-./vagrant-shutdown
-```
-
-- To reboot the cluster, run the *vagrant-reload* script, located [inside the vagrant folder](https://github.com/contiv/vpp/tree/master/vagrant):
-```
-cd vagrant/
-./vagrant-reload
-```
-
-- From a suspended state, or after a reboot of the host machine, the cluster
-can be brought up by running the *vagrant-up* script.
-
-
-### Building and Deploying the dev-contiv-vswitch Image
-If you chose the optional development-environment-deployment option, then perform the
-following instructions on how to build a modified *contivvpp/vswitch* image:
-
-- Make sure changes in the code have been saved. From the k8s-master node,
- build the new *contivvpp/vswitch* image (run as sudo):
-
-```
-vagrant ssh k8s-master
-cd /vagrant/config
-sudo ./save-dev-image
-```
-
-- The newly built *contivvpp/vswitch* image is now tagged as *latest*. Verify the
-build with `sudo docker images`; the *contivvpp/vswitch* should have been created a few
-seconds ago. The new image with all the changes must become available to all
-the nodes in the K8s cluster. To make the changes available to all, load the docker image into the running
-worker nodes (run as sudo):
-
-```
-vagrant ssh k8s-worker1
-cd /vagrant/config
-sudo ./load-dev-image
-```
-
-- Verify with `sudo docker images`; the old *contivvpp/vswitch* should now be tagged as
-`<none>` and the latest tagged *contivvpp/vswitch* should have been created a
-few seconds ago.
-
-### Exploring the Cluster
-Once the cluster is up, perform the following steps:
-- Log into the master:
-```
-cd vagrant
-
-vagrant ssh k8s-master
-
-Welcome to Ubuntu 16.04 LTS (GNU/Linux 4.4.0-21-generic x86_64)
-
- * Documentation: https://help.ubuntu.com/
-vagrant@k8s-master:~$
-```
-- Verify the Kubernetes/Contiv-VPP installation. First, verify the nodes
-in the cluster:
-
-```
-vagrant@k8s-master:~$ kubectl get nodes -o wide
-
-NAME STATUS ROLES AGE VERSION EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
-k8s-master Ready master 22m v1.9.2 <none> Ubuntu 16.04 LTS 4.4.0-21-generic docker://17.12.0-ce
-k8s-worker1 Ready <none> 15m v1.9.2 <none> Ubuntu 16.04 LTS 4.4.0-21-generic docker://17.12.0-ce
-```
-
-- Next, verify that all pods are running correctly:
-
-```
-vagrant@k8s-master:~$ kubectl get pods -n kube-system -o wide
-
-NAME READY STATUS RESTARTS AGE IP NODE
-contiv-etcd-2ngdc 1/1 Running 0 17m 192.169.1.10 k8s-master
-contiv-ksr-x7gsq 1/1 Running 3 17m 192.169.1.10 k8s-master
-contiv-vswitch-9bql6 2/2 Running 0 17m 192.169.1.10 k8s-master
-contiv-vswitch-hpt2x 2/2 Running 0 10m 192.169.1.11 k8s-worker1
-etcd-k8s-master 1/1 Running 0 16m 192.169.1.10 k8s-master
-kube-apiserver-k8s-master 1/1 Running 0 16m 192.169.1.10 k8s-master
-kube-controller-manager-k8s-master 1/1 Running 0 15m 192.169.1.10 k8s-master
-kube-dns-6f4fd4bdf-62rv4 2/3 CrashLoopBackOff 14 17m 10.1.1.2 k8s-master
-kube-proxy-bvr74 1/1 Running 0 10m 192.169.1.11 k8s-worker1
-kube-proxy-v4fzq 1/1 Running 0 17m 192.169.1.10 k8s-master
-kube-scheduler-k8s-master 1/1 Running 0 16m 192.169.1.10 k8s-master
-```
-
-- If you want your pods to be scheduled on both the master and the workers,
-you have to untaint the master node:
-```
-
-```
-
-- Check VPP and its interfaces:
-```
-vagrant@k8s-master:~$ sudo vppctl
- _______ _ _ _____ ___
- __/ __/ _ \ (_)__ | | / / _ \/ _ \
- _/ _// // / / / _ \ | |/ / ___/ ___/
- /_/ /____(_)_/\___/ |___/_/ /_/
-
-vpp# sh interface
- Name Idx State Counter Count
-GigabitEthernet0/8/0 1 up rx packets 14
- rx bytes 3906
- tx packets 18
- tx bytes 2128
- drops 3
- ip4 13
-...
-
-```
-- Make sure that `GigabitEthernet0/8/0` is listed and that its status is `up`.
-
-- Next, create an example deployment of nginx pods:
-```
-vagrant@k8s-master:~$ kubectl run nginx --image=nginx --replicas=2
-deployment "nginx" created
-```
-- Check the status of the deployment:
-
-```
-vagrant@k8s-master:~$ kubectl get deploy -o wide
-
-NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR
-nginx 2 2 2 2 2h nginx nginx run=nginx
-```
-
-- Verify that the pods in the deployment are up and running:
-```
-vagrant@k8s-master:~$ kubectl get pods -o wide
-
-NAME READY STATUS RESTARTS AGE IP NODE
-nginx-8586cf59-6kx2m 1/1 Running 1 1h 10.1.2.3 k8s-worker1
-nginx-8586cf59-j5vf9 1/1 Running 1 1h 10.1.2.2 k8s-worker1
-```
-
-- Issue an HTTP GET request to a pod in the deployment:
-
-```
-vagrant@k8s-master:~$ wget 10.1.2.2
-
---2018-01-19 12:34:08-- http://10.1.2.2/
-Connecting to 10.1.2.2:80... connected.
-HTTP request sent, awaiting response... 200 OK
-Length: 612 [text/html]
-Saving to: ‘index.html.1’
-
-index.html.1 100%[=========================================>] 612 --.-KB/s in 0s
-
-2018-01-19 12:34:08 (1.78 MB/s) - ‘index.html.1’ saved [612/612]
-```
-
-#### How to SSH into k8s Worker Node
-To SSH into k8s Worker Node, perform the following steps:
-
-```
-cd vagrant
-
-vagrant status
-
-vagrant ssh k8s-worker1
-```
diff --git a/docs/usecases/contiv/Vagrant.rst b/docs/usecases/contiv/Vagrant.rst
new file mode 100644
index 0000000..035dd09
--- /dev/null
+++ b/docs/usecases/contiv/Vagrant.rst
@@ -0,0 +1,284 @@
+Contiv-VPP Vagrant Installation
+===============================
+
+Prerequisites
+-------------
+
+The following items are prerequisites before installing vagrant: -
+Vagrant 2.0.1 or later - Hypervisors: - VirtualBox 5.2.8 or later -
+VMWare Fusion 10.1.0 or later or VmWare Workstation 14 - For VmWare
+Fusion, you will need the `Vagrant VmWare Fusion
+plugin <https://www.vagrantup.com/vmware/index.html>`__ - Laptop or
+server with at least 4 CPU cores and 16 Gig of RAM
+
+Creating / Shutting Down / Destroying the Cluster
+-------------------------------------------------
+
+This folder contains the Vagrant file that is used to create a single or
+multi-node Kubernetes cluster using Contiv-VPP as a Network Plugin.
+
+The folder is organized into two subfolders:
+
+- (config) - contains the files that share cluster information, which
+ are used during the provisioning stage (master IP address,
+ Certificates, hash-keys). **CAUTION:** Editing is not recommended!
+- (vagrant) - contains scripts that are used for creating, destroying,
+ rebooting and shutting down the VMs that host the K8s cluster.
+
+To create and run a K8s cluster with a *contiv-vpp CNI* plugin, run the
+``vagrant-start`` script, located in the `vagrant
+folder <https://github.com/contiv/vpp/tree/master/vagrant>`__. The
+``vagrant-start`` script prompts the user to select the number of worker
+nodes for the kubernetes cluster. Zero (0) worker nodes mean that a
+single-node cluster (with one kubernetes master node) will be deployed.
+
+Next, the user is prompted to select either the *production environment*
+or the *development environment*. Instructions on how to build the
+development *contiv/vpp-vswitch* image can be found below in the
+`development
+environment <#building-and-deploying-the-dev-contiv-vswitch-image>`__
+command section.
+
+The last option asks the user to select either *Without StealTheNIC* or
+*With StealTheNIC*. Using option *With StealTheNIC* has the plugin
+“steal” interfaces owned by Linux and uses their configuration in VPP.
+
+For the production environment, enter the following commands:
+
+::
+
+ | => ./vagrant-start
+ Please provide the number of workers for the Kubernetes cluster (0-50) or enter [Q/q] to exit: 1
+
+ Please choose Kubernetes environment:
+ 1) Production
+ 2) Development
+ 3) Quit
+ --> 1
+ You chose Development environment
+
+ Please choose deployment scenario:
+ 1) Without StealTheNIC
+ 2) With StealTheNIC
+ 3) Quit
+ --> 1
+ You chose deployment without StealTheNIC
+
+ Creating a production environment, without STN and 1 worker node(s)
+
+For the development environment, enter the following commands:
+
+::
+
+ | => ./vagrant-start
+ Please provide the number of workers for the Kubernetes cluster (0-50) or enter [Q/q] to exit: 1
+
+ Please choose Kubernetes environment:
+ 1) Production
+ 2) Development
+ 3) Quit
+ --> 2
+ You chose Development environment
+
+ Please choose deployment scenario:
+ 1) Without StealTheNIC
+ 2) With StealTheNIC
+ 3) Quit
+ --> 1
+ You chose deployment without StealTheNIC
+
+ Creating a development environment, without STN and 1 worker node(s)
+
+To destroy and clean-up the cluster, run the *vagrant-cleanup* script,
+located `inside the vagrant
+folder <https://github.com/contiv/vpp/tree/master/vagrant>`__:
+
+::
+
+ cd vagrant/
+ ./vagrant-cleanup
+
+To shutdown the cluster, run the *vagrant-shutdown* script, located
+`inside the vagrant
+folder <https://github.com/contiv/vpp/tree/master/vagrant>`__:
+
+::
+
+ cd vagrant/
+ ./vagrant-shutdown
+
+- To reboot the cluster, run the *vagrant-reload* script, located
+ `inside the vagrant
+ folder <https://github.com/contiv/vpp/tree/master/vagrant>`__:
+
+::
+
+ cd vagrant/
+ ./vagrant-reload
+
+- From a suspended state, or after a reboot of the host machine, the
+ cluster can be brought up by running the *vagrant-up* script.
+
+Building and Deploying the dev-contiv-vswitch Image
+---------------------------------------------------
+
+If you chose the optional development-environment-deployment option,
+then perform the following instructions on how to build a modified
+*contivvpp/vswitch* image:
+
+- Make sure changes in the code have been saved. From the k8s-master
+ node, build the new *contivvpp/vswitch* image (run as sudo):
+
+::
+
+ vagrant ssh k8s-master
+ cd /vagrant/config
+ sudo ./save-dev-image
+
+- The newly built *contivvpp/vswitch* image is now tagged as *latest*.
+ Verify the build with ``sudo docker images``; the *contivvpp/vswitch*
+ should have been created a few seconds ago. The new image with all
+ the changes must become available to all the nodes in the K8s
+ cluster. To make the changes available to all, load the docker image
+ into the running worker nodes (run as sudo):
+
+::
+
+ vagrant ssh k8s-worker1
+ cd /vagrant/config
+ sudo ./load-dev-image
+
+- Verify with ``sudo docker images``; the old *contivvpp/vswitch*
+ should now be tagged as ``<none>`` and the latest tagged
+ *contivvpp/vswitch* should have been created a few seconds ago.
+
+Exploring the Cluster
+---------------------
+
+Once the cluster is up, perform the following steps: - Log into the
+master:
+
+::
+
+ cd vagrant
+
+ vagrant ssh k8s-master
+
+ Welcome to Ubuntu 16.04 LTS (GNU/Linux 4.4.0-21-generic x86_64)
+
+ * Documentation: https://help.ubuntu.com/
+ vagrant@k8s-master:~$
+
+- Verify the Kubernetes/Contiv-VPP installation. First, verify the
+ nodes in the cluster:
+
+::
+
+ vagrant@k8s-master:~$ kubectl get nodes -o wide
+
+ NAME STATUS ROLES AGE VERSION EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
+ k8s-master Ready master 22m v1.9.2 <none> Ubuntu 16.04 LTS 4.4.0-21-generic docker://17.12.0-ce
+ k8s-worker1 Ready <none> 15m v1.9.2 <none> Ubuntu 16.04 LTS 4.4.0-21-generic docker://17.12.0-ce
+
+- Next, verify that all pods are running correctly:
+
+::
+
+ vagrant@k8s-master:~$ kubectl get pods -n kube-system -o wide
+
+ NAME READY STATUS RESTARTS AGE IP NODE
+ contiv-etcd-2ngdc 1/1 Running 0 17m 192.169.1.10 k8s-master
+ contiv-ksr-x7gsq 1/1 Running 3 17m 192.169.1.10 k8s-master
+ contiv-vswitch-9bql6 2/2 Running 0 17m 192.169.1.10 k8s-master
+ contiv-vswitch-hpt2x 2/2 Running 0 10m 192.169.1.11 k8s-worker1
+ etcd-k8s-master 1/1 Running 0 16m 192.169.1.10 k8s-master
+ kube-apiserver-k8s-master 1/1 Running 0 16m 192.169.1.10 k8s-master
+ kube-controller-manager-k8s-master 1/1 Running 0 15m 192.169.1.10 k8s-master
+ kube-dns-6f4fd4bdf-62rv4 2/3 CrashLoopBackOff 14 17m 10.1.1.2 k8s-master
+ kube-proxy-bvr74 1/1 Running 0 10m 192.169.1.11 k8s-worker1
+ kube-proxy-v4fzq 1/1 Running 0 17m 192.169.1.10 k8s-master
+ kube-scheduler-k8s-master 1/1 Running 0 16m 192.169.1.10 k8s-master
+
+- If you want your pods to be scheduled on both the master and the
+ workers, you have to untaint the master node:
+
+::
+
+- Check VPP and its interfaces:
+
+::
+
+ vagrant@k8s-master:~$ sudo vppctl
+ _______ _ _ _____ ___
+ __/ __/ _ \ (_)__ | | / / _ \/ _ \
+ _/ _// // / / / _ \ | |/ / ___/ ___/
+ /_/ /____(_)_/\___/ |___/_/ /_/
+
+ vpp# sh interface
+ Name Idx State Counter Count
+ GigabitEthernet0/8/0 1 up rx packets 14
+ rx bytes 3906
+ tx packets 18
+ tx bytes 2128
+ drops 3
+ ip4 13
+ ...
+
+
+- Make sure that ``GigabitEthernet0/8/0`` is listed and that its status
+ is ``up``.
+
+- Next, create an example deployment of nginx pods:
+
+::
+
+ vagrant@k8s-master:~$ kubectl run nginx --image=nginx --replicas=2
+ deployment "nginx" created
+
+- Check the status of the deployment:
+
+::
+
+ vagrant@k8s-master:~$ kubectl get deploy -o wide
+
+ NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR
+ nginx 2 2 2 2 2h nginx nginx run=nginx
+
+- Verify that the pods in the deployment are up and running:
+
+::
+
+ vagrant@k8s-master:~$ kubectl get pods -o wide
+
+ NAME READY STATUS RESTARTS AGE IP NODE
+ nginx-8586cf59-6kx2m 1/1 Running 1 1h 10.1.2.3 k8s-worker1
+ nginx-8586cf59-j5vf9 1/1 Running 1 1h 10.1.2.2 k8s-worker1
+
+- Issue an HTTP GET request to a pod in the deployment:
+
+::
+
+ vagrant@k8s-master:~$ wget 10.1.2.2
+
+ --2018-01-19 12:34:08-- http://10.1.2.2/
+ Connecting to 10.1.2.2:80... connected.
+ HTTP request sent, awaiting response... 200 OK
+ Length: 612 [text/html]
+ Saving to: ‘index.html.1’
+
+ index.html.1 100%[=========================================>] 612 --.-KB/s in 0s
+
+ 2018-01-19 12:34:08 (1.78 MB/s) - ‘index.html.1’ saved [612/612]
+
+How to SSH into k8s Worker Node
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To SSH into k8s Worker Node, perform the following steps:
+
+::
+
+ cd vagrant
+
+ vagrant status
+
+ vagrant ssh k8s-worker1
diff --git a/docs/usecases/contiv/index.rst b/docs/usecases/contiv/index.rst
index bc52e61..45d5095 100644
--- a/docs/usecases/contiv/index.rst
+++ b/docs/usecases/contiv/index.rst
@@ -1,26 +1,26 @@
-.. _contiv:
-
-##########
-Contiv/VPP
-##########
-
-This section provides the following information about the Contiv function:
-
-.. toctree::
- :maxdepth: 2
-
- K8s_Overview
- SECURITY
- Vagrant
- MANUAL_INSTALL
- VPP_CONFIG
- VMWARE_FUSION_HOST
- NETWORKING
- SINGLE_NIC_SETUP
- MULTI_NIC_SETUP
- CUSTOM_MGMT_NETWORK
- Prometheus
- VPP_PACKET_TRACING_K8S
- VPPTRACE
- CORE_FILES
- BUG_REPORTS
+.. _contiv:
+
+=====================================
+VPP in kubernetes (Contiv/Deprecated)
+=====================================
+
+This section provides the following information about the Contiv function:
+
+.. toctree::
+ :maxdepth: 2
+
+ K8s_Overview
+ SECURITY
+ Vagrant
+ MANUAL_INSTALL
+ VPP_CONFIG
+ VMWARE_FUSION_HOST
+ NETWORKING
+ SINGLE_NIC_SETUP
+ MULTI_NIC_SETUP
+ CUSTOM_MGMT_NETWORK
+ Prometheus
+ VPP_PACKET_TRACING_K8S
+ VPPTRACE
+ CORE_FILES
+ BUG_REPORTS
diff --git a/docs/usecases/hgw.md b/docs/usecases/hgw.md
deleted file mode 100644
index 0b659e9..0000000
--- a/docs/usecases/hgw.md
+++ /dev/null
@@ -1,497 +0,0 @@
-Using VPP as a Home Gateway
-===========================
-
-Vpp running on a small system (with appropriate NICs) makes a fine
-home gateway. The resulting system performs far in excess of
-requirements: a debug image runs at a vector size of \~1.2 terminating
-a 150-mbit down / 10-mbit up cable modem connection.
-
-At a minimum, install sshd and the isc-dhcp-server. If you prefer, you
-can use dnsmasq.
-
-System configuration files
---------------------------
-
-/etc/vpp/startup.conf:
-
- unix {
- nodaemon
- log /var/log/vpp/vpp.log
- full-coredump
- cli-listen /run/vpp/cli.sock
- startup-config /setup.gate
- poll-sleep-usec 100
- gid vpp
- }
- api-segment {
- gid vpp
- }
- dpdk {
- dev 0000:03:00.0
- dev 0000:14:00.0
- etc.
- }
-
- plugins {
- ## Disable all plugins, selectively enable specific plugins
- ## YMMV, you may wish to enable other plugins (acl, etc.)
- plugin default { disable }
- plugin dpdk_plugin.so { enable }
- plugin nat_plugin.so { enable }
- ## if you plan to use the time-based MAC filter
- plugin mactime_plugin.so { enable }
- }
-
-/etc/dhcp/dhcpd.conf:
-
- subnet 192.168.1.0 netmask 255.255.255.0 {
- range 192.168.1.10 192.168.1.99;
- option routers 192.168.1.1;
- option domain-name-servers 8.8.8.8;
- }
-
-If you decide to enable the vpp dns name resolver, substitute
-192.168.1.2 for 8.8.8.8 in the dhcp server configuration.
-
-/etc/default/isc-dhcp-server:
-
- # On which interfaces should the DHCP server (dhcpd) serve DHCP requests?
- # Separate multiple interfaces with spaces, e.g. "eth0 eth1".
- INTERFACESv4="lstack"
- INTERFACESv6=""
-
-/etc/ssh/sshd\_config:
-
- # What ports, IPs and protocols we listen for
- Port <REDACTED-high-number-port>
- # Change to no to disable tunnelled clear text passwords
- PasswordAuthentication no
-
-For your own comfort and safety, do NOT allow password authentication
-and do not answer ssh requests on port 22. Experience shows several hack
-attempts per hour on port 22, but none (ever) on random high-number
-ports.
-
-Systemd configuration
----------------------
-
-In a typical home-gateway use-case, vpp owns the one-and-only WAN link
-with a prayer of reaching the public internet. Simple things like
-updating distro software requires use of the \"lstack\" interface
-created above, and configuring a plausible upstream DNS name resolver.
-
-Configure /etc/systemd/resolved.conf as follows.
-
-/etc/systemd/resolved.conf:
-
- [Resolve]
- DNS=8.8.8.8
- #FallbackDNS=
- #Domains=
- #LLMNR=no
- #MulticastDNS=no
- #DNSSEC=no
- #Cache=yes
- #DNSStubListener=yes
-
-Netplan configuration
----------------------
-
-If you want to configure a static IP address on one of your home-gateway
-Ethernet ports on Ubuntu 18.04, you\'ll need to configure netplan.
-Netplan is relatively new. It and the network manager GUI and can be
-cranky. In the configuration shown below,
-s/enp4s0/\<your-interface\>/\...
-
-/etc/netplan-01-netcfg.yaml:
-
- # This file describes the network interfaces available on your system
- # For more information, see netplan(5).
- network:
- version: 2
- renderer: networkd
- ethernets:
- enp4s0:
- dhcp4: no
- addresses: [192.168.2.254/24]
- gateway4: 192.168.2.100
- nameservers:
- search: [my.local]
- addresses: [8.8.8.8]
-
-/etc/systemd/network-10.enp4s0.network:
-
- [Match]
- Name=enp4s0
-
- [Link]
- RequiredForOnline=no
-
- [Network]
- ConfigureWithoutCarrier=true
- Address=192.168.2.254/24
-
-Note that we\'ve picked an IP address for the home gateway which is on
-an independent unrouteable subnet. This is handy for installing (and
-possibly reverting) new vpp software.
-
-VPP Configuration Files
------------------------
-
-Here we see a nice use-case for the vpp debug CLI macro expander:
-
-/setup.gate:
-
- define HOSTNAME vpp1
- define TRUNK GigabitEthernet3/0/0
-
- comment { Specific MAC address yields a constant IP address }
- define TRUNK_MACADDR 48:f8:b3:00:01:01
- define BVI_MACADDR 48:f8:b3:01:01:02
-
- comment { inside subnet 192.168.<inside_subnet>.0/24 }
- define INSIDE_SUBNET 1
-
- define INSIDE_PORT1 GigabitEthernet6/0/0
- define INSIDE_PORT2 GigabitEthernet6/0/1
- define INSIDE_PORT3 GigabitEthernet8/0/0
- define INSIDE_PORT4 GigabitEthernet8/0/1
-
- comment { feature selections }
- define FEATURE_NAT44 comment
- define FEATURE_CNAT uncomment
- define FEATURE_DNS comment
- define FEATURE_IP6 comment
- define FEATURE_MACTIME uncomment
-
- exec /setup.tmpl
-
-/setup.tmpl:
-
- show macro
-
- set int mac address $(TRUNK) $(TRUNK_MACADDR)
- set dhcp client intfc $(TRUNK) hostname $(HOSTNAME)
- set int state $(TRUNK) up
-
- bvi create instance 0
- set int mac address bvi0 $(BVI_MACADDR)
- set int l2 bridge bvi0 1 bvi
- set int ip address bvi0 192.168.$(INSIDE_SUBNET).1/24
- set int state bvi0 up
-
- set int l2 bridge $(INSIDE_PORT1) 1
- set int state $(INSIDE_PORT1) up
- set int l2 bridge $(INSIDE_PORT2) 1
- set int state $(INSIDE_PORT2) up
- set int l2 bridge $(INSIDE_PORT3) 1
- set int state $(INSIDE_PORT3) up
- set int l2 bridge $(INSIDE_PORT4) 1
- set int state $(INSIDE_PORT4) up
-
- comment { dhcp server and host-stack access }
- create tap host-if-name lstack host-ip4-addr 192.168.$(INSIDE_SUBNET).2/24 host-ip4-gw 192.168.$(INSIDE_SUBNET).1
- set int l2 bridge tap0 1
- set int state tap0 up
-
- service restart isc-dhcp-server
-
- $(FEATURE_NAT44) { nat44 enable users 50 user-sessions 750 sessions 63000 }
- $(FEATURE_NAT44) { nat44 add interface address $(TRUNK) }
- $(FEATURE_NAT44) { set interface nat44 in bvi0 out $(TRUNK) }
-
- $(FEATURE_NAT44) { nat44 add static mapping local 192.168.$(INSIDE_SUBNET).2 22432 external $(TRUNK) 22432 tcp }
-
- $(FEATURE_CNAT) { cnat snat with $(TRUNK) }
- $(FEATURE_CNAT) { set interface feature bvi0 ip4-cnat-snat arc ip4-unicast }
- $(FEATURE_CNAT) { cnat translation add proto tcp real $(TRUNK) 22432 to -> 192.168.$(INSIDE_SUBNET).2 22432 }
- $(FEATURE_CNAT) { $(FEATURE_DNS) { cnat translation add proto udp real $(TRUNK) 53053 to -> 192.168.$(INSIDE_SUBNET).1 53053 } }
-
- $(FEATURE_DNS) { $(FEATURE_NAT44) { nat44 add identity mapping external $(TRUNK) udp 53053 } }
- $(FEATURE_DNS) { bin dns_name_server_add_del 8.8.8.8 }
- $(FEATURE_DNS) { bin dns_enable_disable }
-
- comment { set ct6 inside $(TRUNK) }
- comment { set ct6 outside $(TRUNK) }
-
- $(FEATURE_IP6) { set int ip6 table $(TRUNK) 0 }
- $(FEATURE_IP6) { ip6 nd address autoconfig $(TRUNK) default-route }
- $(FEATURE_IP6) { dhcp6 client $(TRUNK) }
- $(FEATURE_IP6) { dhcp6 pd client $(TRUNK) prefix group hgw }
- $(FEATURE_IP6) { set ip6 address bvi0 prefix group hgw ::1/64 }
- $(FEATURE_IP6) { ip6 nd address autoconfig bvi0 default-route }
- comment { iPhones seem to need lots of RA messages... }
- $(FEATURE_IP6) { ip6 nd bvi0 ra-managed-config-flag ra-other-config-flag ra-interval 5 3 ra-lifetime 180 }
- comment { ip6 nd bvi0 prefix 0::0/0 ra-lifetime 100000 }
-
-
- $(FEATURE_MACTIME) { bin mactime_add_del_range name cisco-vpn mac a8:b4:56:e1:b8:3e allow-static }
- $(FEATURE_MACTIME) { bin mactime_add_del_range name old-mac mac <redacted> allow-static }
- $(FEATURE_MACTIME) { bin mactime_add_del_range name roku mac <redacted> allow-static }
- $(FEATURE_MACTIME) { bin mactime_enable_disable $(INSIDE_PORT1) }
- $(FEATURE_MACTIME) { bin mactime_enable_disable $(INSIDE_PORT2) }
- $(FEATURE_MACTIME) { bin mactime_enable_disable $(INSIDE_PORT3) }
- $(FEATURE_MACTIME) { bin mactime_enable_disable $(INSIDE_PORT4) }
-
-Installing new vpp software
----------------------------
-
-If you\'re **sure** that a given set of vpp Debian packages will install
-and work properly, you can install them while logged into the gateway
-via the lstack / nat path. This procedure is a bit like standing on a
-rug and yanking it. If all goes well, a perfect back-flip occurs. If
-not, you may wish that you\'d configured a static IP address on a
-reserved Ethernet interface as described above.
-
-Installing a new vpp image via ssh to 192.168.1.2:
-
- # nohup dpkg -i *.deb >/dev/null 2>&1 &
-
-Within a few seconds, the inbound ssh connection SHOULD begin to respond
-again. If it does not, you\'ll have to debug the issue(s).
-
-Reasonably Robust Remote Software Installation
-----------------------------------------------
-
-Here are a couple of scripts which yield a reasonably robust software
-installation scheme.
-
-### Build-host script
-
- #!/bin/bash
-
- buildroot=/scratch/vpp-workspace/build-root
- if [ $1x = "testx" ] ; then
- subdir="test"
- ipaddr="192.168.2.48"
- elif [ $1x = "foox" ] ; then
- subdir="foo"
- ipaddr="foo.some.net"
- elif [ $1x = "barx" ] ; then
- subdir="bar"
- ipaddr="bar.some.net"
- else
- subdir="test"
- ipaddr="192.168.2.48"
- fi
-
- echo Save current software...
- ssh -p 22432 $ipaddr "rm -rf /gate_debians.prev"
- ssh -p 22432 $ipaddr "mv /gate_debians /gate_debians.prev"
- ssh -p 22432 $ipaddr "mkdir /gate_debians"
- echo Copy new software to the gateway...
- scp -P 22432 $buildroot/*.deb $ipaddr:/gate_debians
- echo Install new software...
- ssh -p 22432 $ipaddr "nohup /usr/local/bin/vpp-swupdate > /dev/null 2>&1 &"
-
- for i in 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1
- do
- echo Wait for $i seconds...
- sleep 1
- done
-
- echo Try to access the device...
-
- ssh -p 22432 -o ConnectTimeout=10 $ipaddr "tail -20 /var/log/syslog | grep Ping"
- if [ $? == 0 ] ; then
- echo Access test OK...
- else
- echo Access failed, wait for configuration restoration...
- for i in 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1
- do
- echo Wait for $i seconds...
- sleep 1
- done
- echo Retry access test
- ssh -p 22432 -o ConnectTimeout=10 $ipaddr "tail -20 /var/log/syslog | grep Ping"
- if [ $? == 0 ] ; then
- echo Access test OK, check syslog on the device
- exit 1
- else
- echo Access test still fails, manual intervention required.
- exit 2
- fi
- fi
-
- exit 0
-
-### Target script
-
- #!/bin/bash
-
- logger "About to update vpp software..."
- cd /gate_debians
- service vpp stop
- sudo dpkg -i *.deb >/dev/null 2>&1 &
- sleep 20
- logger "Ping connectivity test..."
- for i in 1 2 3 4 5 6 7 8 9 10
- do
- ping -4 -c 1 yahoo.com
- if [ $? == 0 ] ; then
- logger "Ping test OK..."
- exit 0
- fi
- done
-
- logger "Ping test NOT OK, restore old software..."
- rm -rf /gate_debians
- mv /gate_debians.prev /gate_debians
- cd /gate_debians
- nohup sudo dpkg -i *.deb >/dev/null 2>&1 &
- sleep 20
- logger "Repeat connectivity test..."
- for i in 1 2 3 4 5 6 7 8 9 10
- do
- ping -4 -c 1 yahoo.com
- if [ $? == 0 ] ; then
- logger "Ping test OK after restoring old software..."
- exit 0
- fi
- done
-
- logger "Ping test FAIL after restoring software, manual intervention required"
- exit 2
-
-Note that the target script **requires** that the userid which invokes
-it will manage to "sudo dpkg ..." without further authentication. If
-you're uncomfortable with the security implications of that
-requirement, you'll need to solve the problem a different
-way. Strongly suggest configuring sshd as described above to minimize
-risk.
-
-
-Testing new software
---------------------
-
-If you frequently test new home gateway software, it may be handy to set
-up a test gateway behind your production gateway. This testing
-methodology reduces complaints from family members, to name one benefit.
-
-Change the inside network (dhcp) subnet from 192.168.1.0/24 to
-192.168.3.0/24, change the (dhcp) advertised router to 192.168.3.1,
-reconfigure the vpp tap interface addresses onto the 192.168.3.0/24
-subnet, and you should be all set.
-
-This scenario nats traffic twice: first, from the 192.168.3.0/24 network
-onto the 192.168.1.0/24 network. Next, from the 192.168.1.0/24 network
-onto the public internet.
-
-Patches
--------
-
-You\'ll want this addition to src/vpp/vnet/main.c to add the \"service
-restart isc-dhcp-server" and \"service restart vpp\" commands:
-
- #include <sys/types.h>
- #include <sys/wait.h>
-
- static int
- mysystem (char *cmd)
- {
- int rv = 0;
-
- if (fork())
- wait (&rv);
- else
- execl("/bin/sh", "sh", "-c", cmd);
-
- if (rv != 0)
- clib_unix_warning ("('%s') child process returned %d", cmd, rv);
- return rv;
- }
-
- static clib_error_t *
- restart_isc_dhcp_server_command_fn (vlib_main_t * vm,
- unformat_input_t * input,
- vlib_cli_command_t * cmd)
- {
- int rv;
-
- /* Wait a while... */
- vlib_process_suspend (vm, 2.0);
-
- rv = mysystem("/usr/sbin/service isc-dhcp-server restart");
-
- vlib_cli_output (vm, "Restarted the isc-dhcp-server, status %d...", rv);
- return 0;
- }
-
- /* *INDENT-OFF* */
- VLIB_CLI_COMMAND (restart_isc_dhcp_server_command, static) =
- {
- .path = "service restart isc-dhcp-server",
- .short_help = "restarts the isc-dhcp-server",
- .function = restart_isc_dhcp_server_command_fn,
- };
- /* *INDENT-ON* */
-
- static clib_error_t *
- restart_dora_tunnels_command_fn (vlib_main_t * vm,
- unformat_input_t * input,
- vlib_cli_command_t * cmd)
- {
- int rv;
-
- /* Wait three seconds... */
- vlib_process_suspend (vm, 3.0);
-
- rv = mysystem ("/usr/sbin/service dora restart");
-
- vlib_cli_output (vm, "Restarted the dora tunnel service, status %d...", rv);
- return 0;
- }
-
- /* *INDENT-OFF* */
- VLIB_CLI_COMMAND (restart_dora_tunnels_command, static) =
- {
- .path = "service restart dora",
- .short_help = "restarts the dora tunnel service",
- .function = restart_dora_tunnels_command_fn,
- };
- /* *INDENT-ON* */
-
- static clib_error_t *
- restart_vpp_service_command_fn (vlib_main_t * vm,
- unformat_input_t * input,
- vlib_cli_command_t * cmd)
- {
- (void) mysystem ("/usr/sbin/service vpp restart");
- return 0;
- }
-
- /* *INDENT-OFF* */
- VLIB_CLI_COMMAND (restart_vpp_service_command, static) =
- {
- .path = "service restart vpp",
- .short_help = "restarts the vpp service, be careful what you wish for",
- .function = restart_vpp_service_command_fn,
- };
- /* *INDENT-ON* */
-
-Using the time-based mac filter plugin
---------------------------------------
-
-If you need to restrict network access for certain devices to specific
-daily time ranges, configure the \"mactime\" plugin. Add it to the list
-of enabled plugins in /etc/vpp/startup.conf, then enable the feature on
-the NAT \"inside\" interfaces:
-
- bin mactime_enable_disable GigabitEthernet0/14/0
- bin mactime_enable_disable GigabitEthernet0/14/1
- ...
-
-Create the required src-mac-address rule database. There are 4 rule
-entry types:
-
-- allow-static - pass traffic from this mac address
-- drop-static - drop traffic from this mac address
-- allow-range - pass traffic from this mac address at specific times
-- drop-range - drop traffic from this mac address at specific times
-
-Here are some examples:
-
- bin mactime_add_del_range name alarm-system mac 00:de:ad:be:ef:00 allow-static
- bin mactime_add_del_range name unwelcome mac 00:de:ad:be:ef:01 drop-static
- bin mactime_add_del_range name not-during-business-hours mac <mac> drop-range Mon - Fri 7:59 - 18:01
- bin mactime_add_del_range name monday-busines-hours mac <mac> allow-range Mon 7:59 - 18:01
diff --git a/docs/usecases/home_gateway.rst b/docs/usecases/home_gateway.rst
new file mode 100644
index 0000000..90d25cf
--- /dev/null
+++ b/docs/usecases/home_gateway.rst
@@ -0,0 +1,520 @@
+VPP as a Home Gateway
+=====================
+
+Vpp running on a small system (with appropriate NICs) makes a fine home
+gateway. The resulting system performs far in excess of requirements: a
+debug image runs at a vector size of ~1.2 terminating a 150-mbit down /
+10-mbit up cable modem connection.
+
+At a minimum, install sshd and the isc-dhcp-server. If you prefer, you
+can use dnsmasq.
+
+System configuration files
+--------------------------
+
+/etc/vpp/startup.conf:
+
+.. code-block:: c
+
+ unix {
+ nodaemon
+ log /var/log/vpp/vpp.log
+ full-coredump
+ cli-listen /run/vpp/cli.sock
+ startup-config /setup.gate
+ poll-sleep-usec 100
+ gid vpp
+ }
+ api-segment {
+ gid vpp
+ }
+ dpdk {
+ dev 0000:03:00.0
+ dev 0000:14:00.0
+ etc.
+ }
+
+ plugins {
+ ## Disable all plugins, selectively enable specific plugins
+ ## YMMV, you may wish to enable other plugins (acl, etc.)
+ plugin default { disable }
+ plugin dpdk_plugin.so { enable }
+ plugin nat_plugin.so { enable }
+ ## if you plan to use the time-based MAC filter
+ plugin mactime_plugin.so { enable }
+ }
+
+/etc/dhcp/dhcpd.conf:
+
+.. code-block:: c
+
+ subnet 192.168.1.0 netmask 255.255.255.0 {
+ range 192.168.1.10 192.168.1.99;
+ option routers 192.168.1.1;
+ option domain-name-servers 8.8.8.8;
+ }
+
+If you decide to enable the vpp dns name resolver, substitute
+192.168.1.2 for 8.8.8.8 in the dhcp server configuration.
+
+/etc/default/isc-dhcp-server:
+
+.. code-block:: c
+
+ # On which interfaces should the DHCP server (dhcpd) serve DHCP requests?
+ # Separate multiple interfaces with spaces, e.g. "eth0 eth1".
+ INTERFACESv4="lstack"
+ INTERFACESv6=""
+
+/etc/ssh/sshd_config:
+
+.. code-block:: c
+
+ # What ports, IPs and protocols we listen for
+ Port <REDACTED-high-number-port>
+ # Change to no to disable tunnelled clear text passwords
+ PasswordAuthentication no
+
+For your own comfort and safety, do NOT allow password authentication
+and do not answer ssh requests on port 22. Experience shows several hack
+attempts per hour on port 22, but none (ever) on random high-number
+ports.
+
+Systemd configuration
+---------------------
+
+In a typical home-gateway use-case, vpp owns the one-and-only WAN link
+with a prayer of reaching the public internet. Simple things like
+updating distro software requires use of the "lstack" interface created
+above, and configuring a plausible upstream DNS name resolver.
+
+Configure /etc/systemd/resolved.conf as follows.
+
+/etc/systemd/resolved.conf:
+
+.. code-block:: c
+
+ [Resolve]
+ DNS=8.8.8.8
+ #FallbackDNS=
+ #Domains=
+ #LLMNR=no
+ #MulticastDNS=no
+ #DNSSEC=no
+ #Cache=yes
+ #DNSStubListener=yes
+
+Netplan configuration
+---------------------
+
+If you want to configure a static IP address on one of your home-gateway
+Ethernet ports on Ubuntu 18.04, you'll need to configure netplan.
+Netplan is relatively new. It and the network manager GUI and can be
+cranky. In the configuration shown below, s/enp4s0/<your-interface>/...
+
+/etc/netplan-01-netcfg.yaml:
+
+.. code-block:: c
+
+ # This file describes the network interfaces available on your system
+ # For more information, see netplan(5).
+ network:
+ version: 2
+ renderer: networkd
+ ethernets:
+ enp4s0:
+ dhcp4: no
+ addresses: [192.168.2.254/24]
+ gateway4: 192.168.2.100
+ nameservers:
+ search: [my.local]
+ addresses: [8.8.8.8]
+
+/etc/systemd/network-10.enp4s0.network:
+
+.. code-block:: c
+
+ [Match]
+ Name=enp4s0
+
+ [Link]
+ RequiredForOnline=no
+
+ [Network]
+ ConfigureWithoutCarrier=true
+ Address=192.168.2.254/24
+
+Note that we've picked an IP address for the home gateway which is on an
+independent unrouteable subnet. This is handy for installing (and
+possibly reverting) new vpp software.
+
+VPP Configuration Files
+-----------------------
+
+Here we see a nice use-case for the vpp debug CLI macro expander:
+
+/setup.gate:
+
+.. code-block:: c
+
+ define HOSTNAME vpp1
+ define TRUNK GigabitEthernet3/0/0
+
+ comment { Specific MAC address yields a constant IP address }
+ define TRUNK_MACADDR 48:f8:b3:00:01:01
+ define BVI_MACADDR 48:f8:b3:01:01:02
+
+ comment { inside subnet 192.168.<inside_subnet>.0/24 }
+ define INSIDE_SUBNET 1
+
+ define INSIDE_PORT1 GigabitEthernet6/0/0
+ define INSIDE_PORT2 GigabitEthernet6/0/1
+ define INSIDE_PORT3 GigabitEthernet8/0/0
+ define INSIDE_PORT4 GigabitEthernet8/0/1
+
+ comment { feature selections }
+ define FEATURE_NAT44 comment
+ define FEATURE_CNAT uncomment
+ define FEATURE_DNS comment
+ define FEATURE_IP6 comment
+ define FEATURE_MACTIME uncomment
+
+ exec /setup.tmpl
+
+/setup.tmpl:
+
+.. code-block:: c
+
+ show macro
+
+ set int mac address $(TRUNK) $(TRUNK_MACADDR)
+ set dhcp client intfc $(TRUNK) hostname $(HOSTNAME)
+ set int state $(TRUNK) up
+
+ bvi create instance 0
+ set int mac address bvi0 $(BVI_MACADDR)
+ set int l2 bridge bvi0 1 bvi
+ set int ip address bvi0 192.168.$(INSIDE_SUBNET).1/24
+ set int state bvi0 up
+
+ set int l2 bridge $(INSIDE_PORT1) 1
+ set int state $(INSIDE_PORT1) up
+ set int l2 bridge $(INSIDE_PORT2) 1
+ set int state $(INSIDE_PORT2) up
+ set int l2 bridge $(INSIDE_PORT3) 1
+ set int state $(INSIDE_PORT3) up
+ set int l2 bridge $(INSIDE_PORT4) 1
+ set int state $(INSIDE_PORT4) up
+
+ comment { dhcp server and host-stack access }
+ create tap host-if-name lstack host-ip4-addr 192.168.$(INSIDE_SUBNET).2/24 host-ip4-gw 192.168.$(INSIDE_SUBNET).1
+ set int l2 bridge tap0 1
+ set int state tap0 up
+
+ service restart isc-dhcp-server
+
+ $(FEATURE_NAT44) { nat44 enable users 50 user-sessions 750 sessions 63000 }
+ $(FEATURE_NAT44) { nat44 add interface address $(TRUNK) }
+ $(FEATURE_NAT44) { set interface nat44 in bvi0 out $(TRUNK) }
+
+ $(FEATURE_NAT44) { nat44 add static mapping local 192.168.$(INSIDE_SUBNET).2 22432 external $(TRUNK) 22432 tcp }
+
+ $(FEATURE_CNAT) { cnat snat with $(TRUNK) }
+ $(FEATURE_CNAT) { set interface feature bvi0 ip4-cnat-snat arc ip4-unicast }
+ $(FEATURE_CNAT) { cnat translation add proto tcp real $(TRUNK) 22432 to -> 192.168.$(INSIDE_SUBNET).2 22432 }
+ $(FEATURE_CNAT) { $(FEATURE_DNS) { cnat translation add proto udp real $(TRUNK) 53053 to -> 192.168.$(INSIDE_SUBNET).1 53053 } }
+
+ $(FEATURE_DNS) { $(FEATURE_NAT44) { nat44 add identity mapping external $(TRUNK) udp 53053 } }
+ $(FEATURE_DNS) { bin dns_name_server_add_del 8.8.8.8 }
+ $(FEATURE_DNS) { bin dns_enable_disable }
+
+ comment { set ct6 inside $(TRUNK) }
+ comment { set ct6 outside $(TRUNK) }
+
+ $(FEATURE_IP6) { set int ip6 table $(TRUNK) 0 }
+ $(FEATURE_IP6) { ip6 nd address autoconfig $(TRUNK) default-route }
+ $(FEATURE_IP6) { dhcp6 client $(TRUNK) }
+ $(FEATURE_IP6) { dhcp6 pd client $(TRUNK) prefix group hgw }
+ $(FEATURE_IP6) { set ip6 address bvi0 prefix group hgw ::1/64 }
+ $(FEATURE_IP6) { ip6 nd address autoconfig bvi0 default-route }
+ comment { iPhones seem to need lots of RA messages... }
+ $(FEATURE_IP6) { ip6 nd bvi0 ra-managed-config-flag ra-other-config-flag ra-interval 5 3 ra-lifetime 180 }
+ comment { ip6 nd bvi0 prefix 0::0/0 ra-lifetime 100000 }
+
+
+ $(FEATURE_MACTIME) { bin mactime_add_del_range name cisco-vpn mac a8:b4:56:e1:b8:3e allow-static }
+ $(FEATURE_MACTIME) { bin mactime_add_del_range name old-mac mac <redacted> allow-static }
+ $(FEATURE_MACTIME) { bin mactime_add_del_range name roku mac <redacted> allow-static }
+ $(FEATURE_MACTIME) { bin mactime_enable_disable $(INSIDE_PORT1) }
+ $(FEATURE_MACTIME) { bin mactime_enable_disable $(INSIDE_PORT2) }
+ $(FEATURE_MACTIME) { bin mactime_enable_disable $(INSIDE_PORT3) }
+ $(FEATURE_MACTIME) { bin mactime_enable_disable $(INSIDE_PORT4) }
+
+Installing new vpp software
+---------------------------
+
+If you're **sure** that a given set of vpp Debian packages will install
+and work properly, you can install them while logged into the gateway
+via the lstack / nat path. This procedure is a bit like standing on a
+rug and yanking it. If all goes well, a perfect back-flip occurs. If
+not, you may wish that you'd configured a static IP address on a
+reserved Ethernet interface as described above.
+
+Installing a new vpp image via ssh to 192.168.1.2:
+
+.. code-block:: c
+
+ # nohup dpkg -i *.deb >/dev/null 2>&1 &
+
+Within a few seconds, the inbound ssh connection SHOULD begin to respond
+again. If it does not, you'll have to debug the issue(s).
+
+Reasonably Robust Remote Software Installation
+----------------------------------------------
+
+Here are a couple of scripts which yield a reasonably robust software
+installation scheme.
+
+Build-host script
+~~~~~~~~~~~~~~~~~
+
+.. code-block:: c
+
+ #!/bin/bash
+
+ buildroot=/scratch/vpp-workspace/build-root
+ if [ $1x = "testx" ] ; then
+ subdir="test"
+ ipaddr="192.168.2.48"
+ elif [ $1x = "foox" ] ; then
+ subdir="foo"
+ ipaddr="foo.some.net"
+ elif [ $1x = "barx" ] ; then
+ subdir="bar"
+ ipaddr="bar.some.net"
+ else
+ subdir="test"
+ ipaddr="192.168.2.48"
+ fi
+
+ echo Save current software...
+ ssh -p 22432 $ipaddr "rm -rf /gate_debians.prev"
+ ssh -p 22432 $ipaddr "mv /gate_debians /gate_debians.prev"
+ ssh -p 22432 $ipaddr "mkdir /gate_debians"
+ echo Copy new software to the gateway...
+ scp -P 22432 $buildroot/*.deb $ipaddr:/gate_debians
+ echo Install new software...
+ ssh -p 22432 $ipaddr "nohup /usr/local/bin/vpp-swupdate > /dev/null 2>&1 &"
+
+ for i in 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1
+ do
+ echo Wait for $i seconds...
+ sleep 1
+ done
+
+ echo Try to access the device...
+
+ ssh -p 22432 -o ConnectTimeout=10 $ipaddr "tail -20 /var/log/syslog | grep Ping"
+ if [ $? == 0 ] ; then
+ echo Access test OK...
+ else
+ echo Access failed, wait for configuration restoration...
+ for i in 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1
+ do
+ echo Wait for $i seconds...
+ sleep 1
+ done
+ echo Retry access test
+ ssh -p 22432 -o ConnectTimeout=10 $ipaddr "tail -20 /var/log/syslog | grep Ping"
+ if [ $? == 0 ] ; then
+ echo Access test OK, check syslog on the device
+ exit 1
+ else
+ echo Access test still fails, manual intervention required.
+ exit 2
+ fi
+ fi
+
+ exit 0
+
+Target script
+~~~~~~~~~~~~~
+
+.. code-block:: c
+
+ #!/bin/bash
+
+ logger "About to update vpp software..."
+ cd /gate_debians
+ service vpp stop
+ sudo dpkg -i *.deb >/dev/null 2>&1 &
+ sleep 20
+ logger "Ping connectivity test..."
+ for i in 1 2 3 4 5 6 7 8 9 10
+ do
+ ping -4 -c 1 yahoo.com
+ if [ $? == 0 ] ; then
+ logger "Ping test OK..."
+ exit 0
+ fi
+ done
+
+ logger "Ping test NOT OK, restore old software..."
+ rm -rf /gate_debians
+ mv /gate_debians.prev /gate_debians
+ cd /gate_debians
+ nohup sudo dpkg -i *.deb >/dev/null 2>&1 &
+ sleep 20
+ logger "Repeat connectivity test..."
+ for i in 1 2 3 4 5 6 7 8 9 10
+ do
+ ping -4 -c 1 yahoo.com
+ if [ $? == 0 ] ; then
+ logger "Ping test OK after restoring old software..."
+ exit 0
+ fi
+ done
+
+ logger "Ping test FAIL after restoring software, manual intervention required"
+ exit 2
+
+Note that the target script **requires** that the user id which invokes
+it will manage to “sudo dpkg …” without further authentication. If
+you’re uncomfortable with the security implications of that requirement,
+you’ll need to solve the problem a different way. Strongly suggest
+configuring sshd as described above to minimize risk.
+
+Testing new software
+--------------------
+
+If you frequently test new home gateway software, it may be handy to set
+up a test gateway behind your production gateway. This testing
+methodology reduces complaints from family members, to name one benefit.
+
+Change the inside network (dhcp) subnet from 192.168.1.0/24 to
+192.168.3.0/24, change the (dhcp) advertised router to 192.168.3.1,
+reconfigure the vpp tap interface addresses onto the 192.168.3.0/24
+subnet, and you should be all set.
+
+This scenario nats traffic twice: first, from the 192.168.3.0/24 network
+onto the 192.168.1.0/24 network. Next, from the 192.168.1.0/24 network
+onto the public internet.
+
+Patches
+-------
+
+You'll want this addition to src/vpp/vnet/main.c to add the "service
+restart isc-dhcp-server” and "service restart vpp" commands:
+
+.. code-block:: c
+
+ #include <sys/types.h>
+ #include <sys/wait.h>
+
+ static int
+ mysystem (char *cmd)
+ {
+ int rv = 0;
+
+ if (fork())
+ wait (&rv);
+ else
+ execl("/bin/sh", "sh", "-c", cmd);
+
+ if (rv != 0)
+ clib_unix_warning ("('%s') child process returned %d", cmd, rv);
+ return rv;
+ }
+
+ static clib_error_t *
+ restart_isc_dhcp_server_command_fn (vlib_main_t * vm,
+ unformat_input_t * input,
+ vlib_cli_command_t * cmd)
+ {
+ int rv;
+
+ /* Wait a while... */
+ vlib_process_suspend (vm, 2.0);
+
+ rv = mysystem("/usr/sbin/service isc-dhcp-server restart");
+
+ vlib_cli_output (vm, "Restarted the isc-dhcp-server, status %d...", rv);
+ return 0;
+ }
+
+ VLIB_CLI_COMMAND (restart_isc_dhcp_server_command, static) =
+ {
+ .path = "service restart isc-dhcp-server",
+ .short_help = "restarts the isc-dhcp-server",
+ .function = restart_isc_dhcp_server_command_fn,
+ };
+
+ static clib_error_t *
+ restart_dora_tunnels_command_fn (vlib_main_t * vm,
+ unformat_input_t * input,
+ vlib_cli_command_t * cmd)
+ {
+ int rv;
+
+ /* Wait three seconds... */
+ vlib_process_suspend (vm, 3.0);
+
+ rv = mysystem ("/usr/sbin/service dora restart");
+
+ vlib_cli_output (vm, "Restarted the dora tunnel service, status %d...", rv);
+ return 0;
+ }
+
+ VLIB_CLI_COMMAND (restart_dora_tunnels_command, static) =
+ {
+ .path = "service restart dora",
+ .short_help = "restarts the dora tunnel service",
+ .function = restart_dora_tunnels_command_fn,
+ };
+
+ static clib_error_t *
+ restart_vpp_service_command_fn (vlib_main_t * vm,
+ unformat_input_t * input,
+ vlib_cli_command_t * cmd)
+ {
+ (void) mysystem ("/usr/sbin/service vpp restart");
+ return 0;
+ }
+
+ VLIB_CLI_COMMAND (restart_vpp_service_command, static) =
+ {
+ .path = "service restart vpp",
+ .short_help = "restarts the vpp service, be careful what you wish for",
+ .function = restart_vpp_service_command_fn,
+ };
+
+Using the time-based mac filter plugin
+--------------------------------------
+
+If you need to restrict network access for certain devices to specific
+daily time ranges, configure the "mactime" plugin. Add it to the list of
+enabled plugins in /etc/vpp/startup.conf, then enable the feature on the
+NAT "inside" interfaces:
+
+.. code-block:: c
+
+ bin mactime_enable_disable GigabitEthernet0/14/0
+ bin mactime_enable_disable GigabitEthernet0/14/1
+ ...
+
+Create the required src-mac-address rule database. There are 4 rule
+entry types:
+
+- allow-static - pass traffic from this mac address
+- drop-static - drop traffic from this mac address
+- allow-range - pass traffic from this mac address at specific times
+- drop-range - drop traffic from this mac address at specific times
+
+Here are some examples:
+
+.. code-block:: c
+
+ bin mactime_add_del_range name alarm-system mac 00:de:ad:be:ef:00 allow-static
+ bin mactime_add_del_range name unwelcome mac 00:de:ad:be:ef:01 drop-static
+ bin mactime_add_del_range name not-during-business-hours mac <mac> drop-range Mon - Fri 7:59 - 18:01
+ bin mactime_add_del_range name monday-busines-hours mac <mac> allow-range Mon 7:59 - 18:01
diff --git a/docs/usecases/ikev2/2_vpp.rst b/docs/usecases/ikev2/2_vpp.rst
new file mode 100644
index 0000000..2c6fe6b
--- /dev/null
+++ b/docs/usecases/ikev2/2_vpp.rst
@@ -0,0 +1,128 @@
+How to connect VPP instances using IKEv2
+========================================
+
+This section describes how to initiate IKEv2 session between two VPP
+instances using Linux veth interfaces and namespaces.
+
+Create veth interfaces and namespaces and configure it:
+
+::
+
+ sudo ip link add ifresp type veth peer name ifinit
+ sudo ip link set dev ifresp up
+ sudo ip link set dev ifinit up
+
+ sudo ip netns add clientns
+ sudo ip netns add serverns
+ sudo ip link add veth_client type veth peer name client
+ sudo ip link add veth_server type veth peer name server
+ sudo ip link set dev veth_client up netns clientns
+ sudo ip link set dev veth_server up netns serverns
+
+ sudo ip netns exec clientns \
+ bash -c "
+ ip link set dev lo up
+ ip addr add 192.168.5.2/24 dev veth_client
+ ip addr add fec5::2/16 dev veth_client
+ ip route add 192.168.3.0/24 via 192.168.5.1
+ ip route add fec3::0/16 via fec5::1
+ "
+
+ sudo ip netns exec serverns \
+ bash -c "
+ ip link set dev lo up
+ ip addr add 192.168.3.2/24 dev veth_server
+ ip addr add fec3::2/16 dev veth_server
+ ip route add 192.168.5.0/24 via 192.168.3.1
+ ip route add fec5::0/16 via fec3::1
+ "
+
+Run responder VPP:
+
+::
+
+ sudo /usr/bin/vpp unix { \
+ cli-listen /tmp/vpp_resp.sock \
+ gid $(id -g) } \
+ api-segment { prefix vpp } \
+ plugins { plugin dpdk_plugin.so { disable } }
+
+Configure the responder
+
+::
+
+ create host-interface name ifresp
+ set interface ip addr host-ifresp 192.168.10.2/24
+ set interface state host-ifresp up
+
+ create host-interface name server
+ set interface ip addr host-server 192.168.3.1/24
+ set interface state host-server up
+
+ ikev2 profile add pr1
+ ikev2 profile set pr1 auth shared-key-mic string Vpp123
+ ikev2 profile set pr1 id local ipv4 192.168.10.2
+ ikev2 profile set pr1 id remote ipv4 192.168.10.1
+
+ ikev2 profile set pr1 traffic-selector local ip-range 192.168.3.0 - 192.168.3.255 port-range 0 - 65535 protocol 0
+ ikev2 profile set pr1 traffic-selector remote ip-range 192.168.5.0 - 192.168.5.255 port-range 0 - 65535 protocol 0
+
+ create ipip tunnel src 192.168.10.2 dst 192.168.10.1
+ ikev2 profile set pr1 tunnel ipip0
+ ip route add 192.168.5.0/24 via 192.168.10.1 ipip0
+ set interface unnumbered ipip0 use host-ifresp
+
+Run initiator VPP:
+
+::
+
+ sudo /usr/bin/vpp unix { \
+ cli-listen /tmp/vpp_init.sock \
+ gid $(id -g) } \
+ api-segment { prefix vpp } \
+ plugins { plugin dpdk_plugin.so { disable } }
+
+Configure initiator:
+
+::
+
+ create host-interface name ifinit
+ set interface ip addr host-ifinit 192.168.10.1/24
+ set interface state host-ifinit up
+
+ create host-interface name client
+ set interface ip addr host-client 192.168.5.1/24
+ set interface state host-client up
+
+ ikev2 profile add pr1
+ ikev2 profile set pr1 auth shared-key-mic string Vpp123
+ ikev2 profile set pr1 id local ipv4 192.168.10.1
+ ikev2 profile set pr1 id remote ipv4 192.168.10.2
+
+ ikev2 profile set pr1 traffic-selector remote ip-range 192.168.3.0 - 192.168.3.255 port-range 0 - 65535 protocol 0
+ ikev2 profile set pr1 traffic-selector local ip-range 192.168.5.0 - 192.168.5.255 port-range 0 - 65535 protocol 0
+
+ ikev2 profile set pr1 responder host-ifinit 192.168.10.2
+ ikev2 profile set pr1 ike-crypto-alg aes-gcm-16 256 ike-dh modp-2048
+ ikev2 profile set pr1 esp-crypto-alg aes-gcm-16 256
+
+ create ipip tunnel src 192.168.10.1 dst 192.168.10.2
+ ikev2 profile set pr1 tunnel ipip0
+ ip route add 192.168.3.0/24 via 192.168.10.2 ipip0
+ set interface unnumbered ipip0 use host-ifinit
+
+Initiate the IKEv2 connection:
+
+::
+
+ vpp# ikev2 initiate sa-init pr1
+
+Responder’s and initiator’s private networks are now connected with
+IPSEC tunnel:
+
+::
+
+ $ sudo ip netns exec clientns ping 192.168.3.1
+ PING 192.168.3.1 (192.168.3.1) 56(84) bytes of data.
+ 64 bytes from 192.168.3.1: icmp_seq=1 ttl=63 time=1.64 ms
+ 64 bytes from 192.168.3.1: icmp_seq=2 ttl=63 time=7.24 ms
diff --git a/docs/usecases/ikev2.rst b/docs/usecases/ikev2/index.rst
similarity index 90%
rename from docs/usecases/ikev2.rst
rename to docs/usecases/ikev2/index.rst
index 853b22e..c9829b4 100644
--- a/docs/usecases/ikev2.rst
+++ b/docs/usecases/ikev2/index.rst
@@ -1,7 +1,7 @@
.. _ikev2:
-IKEv2 in VPP
-============
+IKEv2 with VPP
+==============
This sections describes some of the ways to establish IKEv2 connection
between two VPP instances or VPP and strongSwan. It covers scenarios in
diff --git a/docs/usecases/ikev2/vpp_init_sswan_resp.rst b/docs/usecases/ikev2/vpp_init_sswan_resp.rst
new file mode 100644
index 0000000..9386218
--- /dev/null
+++ b/docs/usecases/ikev2/vpp_init_sswan_resp.rst
@@ -0,0 +1,202 @@
+VPP as IKEv2 initiator and strongSwan as responder
+==================================================
+
+Prerequisites
+-------------
+
+To make the examples easier to configure ``docker`` it is required to
+pull strongSwan docker image. The networking is done using Linux’ veth
+interfaces and namespaces.
+
+Setup
+-----
+
+First a topology:
+
+::
+
+ 192.168.3.2 192.168.5.2
+ + loopback
+ | +
+ +----+----+ 192.168.10.2 +-----+----+
+ | VPP | |strongSwan|
+ |initiator+----------------------+responder |
+ +---------+ +----------+
+ 192.168.10.1
+
+Create veth interfaces and namespaces and configure them:
+
+::
+
+ sudo ip link add gw type veth peer name swanif
+ sudo ip link set dev gw up
+
+ sudo ip netns add ns
+ sudo ip link add veth_priv type veth peer name priv
+ sudo ip link set dev priv up
+ sudo ip link set dev veth_priv up netns ns
+
+ sudo ip netns exec ns \
+ bash -c "
+ ip link set dev lo up
+ ip addr add 192.168.3.2/24 dev veth_priv
+ ip route add 192.168.5.0/24 via 192.168.3.1"
+
+Create directory with strongswan configs that will be mounted to the
+docker container
+
+::
+
+ mkdir /tmp/sswan
+
+Create the ``ipsec.conf`` file in the ``/tmp/sswan`` directory with
+following content:
+
+::
+
+ config setup
+ strictcrlpolicy=no
+
+ conn initiator
+ mobike=no
+ auto=add
+ type=tunnel
+ keyexchange=ikev2
+ ike=aes256gcm16-prfsha256-modp2048!
+ esp=aes256gcm16-esn!
+
+ # local:
+ leftauth=psk
+ leftid=@sswan.vpn.example.com
+ leftsubnet=192.168.5.0/24
+
+ # remote: (gateway)
+ rightid=@roadwarrior.vpp
+ right=192.168.10.2
+ rightauth=psk
+ rightsubnet=192.168.3.0/24
+
+``/tmp/sswan/ipsec.secrets``
+
+::
+
+ : PSK 'Vpp123'
+
+``/tmp/sswan/strongswan.conf``
+
+::
+
+ charon {
+ load_modular = yes
+ plugins {
+ include strongswan.d/charon/*.conf
+ }
+ filelog {
+ /tmp/charon.log {
+ time_format = %b %e %T
+ ike_name = yes
+ append = no
+ default = 2
+ flush_line = yes
+ }
+ }
+ }
+ include strongswan.d/*.conf
+
+Start docker container with strongSwan:
+
+::
+
+ docker run --name sswan -d --privileged --rm --net=none \
+ -v /tmp/sswan:/conf -v /tmp/sswan:/etc/ipsec.d philplckthun/strongswan
+
+Finish configuration of initiator’s private network:
+
+::
+
+ pid=$(docker inspect --format "{{.State.Pid}}" sswan)
+ sudo ip link set netns $pid dev swanif
+
+ sudo nsenter -t $pid -n ip addr add 192.168.10.1/24 dev swanif
+ sudo nsenter -t $pid -n ip link set dev swanif up
+
+ sudo nsenter -t $pid -n ip addr add 192.168.5.2/32 dev lo
+ sudo nsenter -t $pid -n ip link set dev lo up
+
+Start VPP …
+
+::
+
+ sudo /usr/bin/vpp unix { \
+ cli-listen /tmp/vpp.sock \
+ gid $(id -g) } \
+ api-segment { prefix vpp } \
+ plugins { plugin dpdk_plugin.so { disable } }
+
+… and configure it:
+
+::
+
+ create host-interface name gw
+ set interface ip addr host-gw 192.168.10.2/24
+ set interface state host-gw up
+
+ create host-interface name priv
+ set interface ip addr host-priv 192.168.3.1/24
+ set interface state host-priv up
+
+ ikev2 profile add pr1
+ ikev2 profile set pr1 auth shared-key-mic string Vpp123
+ ikev2 profile set pr1 id local fqdn roadwarrior.vpp
+ ikev2 profile set pr1 id remote fqdn sswan.vpn.example.com
+
+ ikev2 profile set pr1 traffic-selector local ip-range 192.168.3.0 - 192.168.3.255 port-range 0 - 65535 protocol 0
+ ikev2 profile set pr1 traffic-selector remote ip-range 192.168.5.0 - 192.168.5.255 port-range 0 - 65535 protocol 0
+
+ ikev2 profile set pr1 responder host-gw 192.168.10.1
+ ikev2 profile set pr1 ike-crypto-alg aes-gcm-16 256 ike-dh modp-2048
+ ikev2 profile set pr1 esp-crypto-alg aes-gcm-16 256
+
+ create ipip tunnel src 192.168.10.2 dst 192.168.10.1
+ ikev2 profile set pr1 tunnel ipip0
+ ip route add 192.168.5.0/24 via 192.168.10.1 ipip0
+ set interface unnumbered ipip0 use host-gw
+
+Initiate the IKEv2 connection:
+
+::
+
+ vpp# ikev2 initiate sa-init pr1
+
+::
+
+ vpp# show ikev2 sa details
+ iip 192.168.10.2 ispi f717b0cbd17e27c3 rip 192.168.10.1 rspi e9b7af7fc9b13361
+ encr:aes-gcm-16 prf:hmac-sha2-256 dh-group:modp-2048
+ nonce i:eb0354613b268c6372061bbdaab13deca37c8a625b1f65c073d25df2ecfe672e
+ r:70e1248ac09943047064f6a2135fa2a424778ba03038ab9c4c2af8aba179ed84
+ SK_d 96bd4feb59be2edf1930a12a3a5d22e30195ee9f56ea203c5fb6cba5dd2bb80f
+ SK_e i:00000000: 5b75b9d808c8467fd00a0923c06efee2a4eb1d033c57532e05f9316ed9c56fe9
+ 00000020: c4db9114
+ r:00000000: 95121b63372d20b83558dc3e209b9affef042816cf071c86a53543677b40c15b
+ 00000020: f169ab67
+ SK_p i:fb40d1114c347ddc3228ba004d4759d58f9c1ae6f1746833f908d39444ef92b1
+ r:aa049828240cb242e1d5aa625cd5914dc8f8e980a74de8e06883623d19384902
+ identifier (i) id-type fqdn data roadwarrior.vpp
+ identifier (r) id-type fqdn data sswan.vpn.example.com
+ child sa 0:encr:aes-gcm-16 esn:yes
+ spi(i) 9dffd57a spi(r) c4e0ef53
+ SK_e i:290c681694f130b33d511335dd257e78721635b7e8aa87930dd77bb1d6dd3f42
+ r:0a09fa18cf1cf65c6324df02b46dcc998b84e5397cf911b63e0c096053946c2e
+ traffic selectors (i):0 type 7 protocol_id 0 addr 192.168.3.0 - 192.168.3.255 port 0 - 65535
+ traffic selectors (r):0 type 7 protocol_id 0 addr 192.168.5.0 - 192.168.5.255 port 0 - 65535
+
+Now we can generate some traffic between responder’s and initiator’s
+private networks and see it works.
+
+::
+
+ $ sudo ip netns exec ns ping 192.168.5.2
+ PING 192.168.5.2 (192.168.5.2) 56(84) bytes of data.
+ 64 bytes from 192.168.5.2: icmp_seq=1 ttl=63 time=0.450 ms
+ 64 bytes from 192.168.5.2: icmp_seq=2 ttl=63 time=0.630 ms
diff --git a/docs/usecases/ikev2/vpp_resp_sswan_init.rst b/docs/usecases/ikev2/vpp_resp_sswan_init.rst
new file mode 100644
index 0000000..9f3c7e7
--- /dev/null
+++ b/docs/usecases/ikev2/vpp_resp_sswan_init.rst
@@ -0,0 +1,203 @@
+VPP as IKEv2 responder and strongSwan as initiator
+==================================================
+
+Prerequisites
+-------------
+
+To make the examples easier to configure ``docker`` it is required to
+pull strongSwan docker image. The networking is done using Linux’ veth
+interfaces and namespaces.
+
+Setup
+-----
+
+First a topology:
+
+::
+
+ 192.168.3.2 192.168.5.2
+ + loopback
+ | +
+ +----+----+ 192.168.10.2 +-----+----+
+ | VPP | |initiator |
+ |responder+----------------------+strongSwan|
+ +---------+ +----------+
+ 192.168.10.1
+
+Create veth interfaces and namespaces and configure them:
+
+::
+
+ sudo ip link add gw type veth peer name swanif
+ sudo ip link set dev gw up
+
+ sudo ip netns add ns
+ sudo ip link add veth_priv type veth peer name priv
+ sudo ip link set dev priv up
+ sudo ip link set dev veth_priv up netns ns
+
+ sudo ip netns exec ns \
+ bash -c "
+ ip link set dev lo up
+ ip addr add 192.168.3.2/24 dev veth_priv
+ ip route add 192.168.5.0/24 via 192.168.3.1"
+
+Create directory with strongswan configs that will be mounted to the
+docker container
+
+::
+
+ mkdir /tmp/sswan
+
+Create the ``ipsec.conf`` file in the ``/tmp/sswan`` directory with
+following content:
+
+::
+
+ config setup
+ strictcrlpolicy=no
+
+ conn initiator
+ mobike=no
+ auto=add
+ type=tunnel
+ keyexchange=ikev2
+ ike=aes256gcm16-prfsha256-modp2048!
+ esp=aes256gcm16-esn!
+
+ # local:
+ leftauth=psk
+ leftid=@roadwarrior.vpn.example.com
+ leftsubnet=192.168.5.0/24
+
+ # remote: (vpp gateway)
+ rightid=@vpp.home
+ right=192.168.10.2
+ rightauth=psk
+ rightsubnet=192.168.3.0/24
+
+``/tmp/sswan/ipsec.secrets``
+
+::
+
+ : PSK 'Vpp123'
+
+``/tmp/sswan/strongswan.conf``
+
+::
+
+ charon {
+ load_modular = yes
+ plugins {
+ include strongswan.d/charon/*.conf
+ }
+ filelog {
+ /tmp/charon.log {
+ time_format = %b %e %T
+ ike_name = yes
+ append = no
+ default = 2
+ flush_line = yes
+ }
+ }
+ }
+ include strongswan.d/*.conf
+
+Start docker container with strongSwan:
+
+::
+
+ docker run --name sswan -d --privileged --rm --net=none \
+ -v /tmp/sswan:/conf -v /tmp/sswan:/etc/ipsec.d philplckthun/strongswan
+
+Finish configuration of initiator’s private network:
+
+::
+
+ pid=$(docker inspect --format "{{.State.Pid}}" sswan)
+ sudo ip link set netns $pid dev swanif
+
+ sudo nsenter -t $pid -n ip addr add 192.168.10.1/24 dev swanif
+ sudo nsenter -t $pid -n ip link set dev swanif up
+
+ sudo nsenter -t $pid -n ip addr add 192.168.5.2/32 dev lo
+ sudo nsenter -t $pid -n ip link set dev lo up
+
+Start VPP …
+
+::
+
+ sudo /usr/bin/vpp unix { \
+ cli-listen /tmp/vpp.sock \
+ gid $(id -g) } \
+ api-segment { prefix vpp } \
+ plugins { plugin dpdk_plugin.so { disable } }
+
+… and configure it:
+
+::
+
+ create host-interface name gw
+ set interface ip addr host-gw 192.168.10.2/24
+ set interface state host-gw up
+
+ create host-interface name priv
+ set interface ip addr host-priv 192.168.3.1/24
+ set interface state host-priv up
+
+ ikev2 profile add pr1
+ ikev2 profile set pr1 auth shared-key-mic string Vpp123
+ ikev2 profile set pr1 id local fqdn vpp.home
+ ikev2 profile set pr1 id remote fqdn roadwarrior.vpn.example.com
+
+ ikev2 profile set pr1 traffic-selector local ip-range 192.168.3.0 - 192.168.3.255 port-range 0 - 65535 protocol 0
+ ikev2 profile set pr1 traffic-selector remote ip-range 192.168.5.0 - 192.168.5.255 port-range 0 - 65535 protocol 0
+
+ create ipip tunnel src 192.168.10.2 dst 192.168.10.1
+ ikev2 profile set pr1 tunnel ipip0
+ ip route add 192.168.5.0/24 via 192.168.10.1 ipip0
+ set interface unnumbered ipip0 use host-gw
+
+Initiate the IKEv2 connection:
+
+::
+
+ $ sudo docker exec sswan ipsec up initiator
+
+ ...
+ CHILD_SA initiator{1} established with SPIs c320c95f_i 213932c2_o and TS 192.168.5.0/24 === 192.168.3.0/24
+ connection 'initiator' established successfully
+
+::
+
+ vpp# show ikev2 sa details
+
+ iip 192.168.10.1 ispi 7849021d9f655f1b rip 192.168.10.2 rspi 5a9ca7469a035205
+ encr:aes-gcm-16 prf:hmac-sha2-256 dh-group:modp-2048
+ nonce i:692ce8fd8f1c1934f63bfa2b167c4de2cff25640dffe938cdfe01a5d7f6820e6
+ r:3ed84a14ea8526063e5aa762312be225d33e866d7152b9ce23e50f0ededca9e3
+ SK_d 9a9b896ed6c35c78134fcd6e966c04868b6ecacf6d5088b4b2aee8b05d30fdda
+ SK_e i:00000000: 1b1619788d8c812ca5916c07e635bda860f15293099f3bf43e8d88e52074b006
+ 00000020: 72c8e3e3
+ r:00000000: 89165ceb2cef6a6b3319f437386292d9ef2e96d8bdb21eeb0cb0d3b92733de03
+ 00000020: bbc29c50
+ SK_p i:fe35fca30985ee75e7c8bc0d7bc04db7a0e1655e997c0f5974c31458826b6fef
+ r:0dd318662a96a25fcdf4998d8c6e4180c67c03586cf91dab26ed43aeda250272
+ identifier (i) id-type fqdn data roadwarrior.vpn.example.com
+ identifier (r) id-type fqdn data vpp.home
+ child sa 0:encr:aes-gcm-16 esn:yes
+ spi(i) c320c95f spi(r) 213932c2
+ SK_e i:2a6c9eae9dbed202c0ae6ccc001621aba5bb0b01623d4de4d14fd27bd5185435
+ r:15e2913d39f809040ca40a02efd27da298b6de05f67bd8f10210da5e6ae606fb
+ traffic selectors (i):0 type 7 protocol_id 0 addr 192.168.5.0 - 192.168.5.255 port 0 - 65535
+ traffic selectors (r):0 type 7 protocol_id 0 addr 192.168.3.0 - 192.168.3.255 port 0 - 65535
+
+Now we can generate some traffic between responder’s and initiator’s
+private networks and see it works.
+
+::
+
+ $ sudo ip netns exec ns ping 192.168.5.2
+ PING 192.168.5.2 (192.168.5.2) 56(84) bytes of data.
+ 64 bytes from 192.168.5.2: icmp_seq=1 ttl=63 time=1.02 ms
+ 64 bytes from 192.168.5.2: icmp_seq=2 ttl=63 time=0.599 ms
diff --git a/docs/usecases/index.rst b/docs/usecases/index.rst
deleted file mode 100644
index d33e9c5..0000000
--- a/docs/usecases/index.rst
+++ /dev/null
@@ -1,24 +0,0 @@
-.. _usecases:
-
-
-Use Cases
-==========
-
-This chapter contains a sample of the many ways FD.io VPP can be used. It is by no means an
-extensive list, but should give a sampling of the many features contained in FD.io VPP.
-
-.. toctree::
-
- containers
- simpleperf/index.rst
- vhost/index.rst
- vmxnet3
- acls
- vppcloud
- hgw
- contiv/index.rst
- networksim
- webapp
- container_test
- trafficgen
- ikev2
diff --git a/docs/usecases/networksim.md b/docs/usecases/networksim.md
deleted file mode 100644
index 817ddf8..0000000
--- a/docs/usecases/networksim.md
+++ /dev/null
@@ -1,90 +0,0 @@
-Network Simulator Plugin
-========================
-
-Vpp includes a fairly capable network simulator plugin, which can
-simulate real-world round-trip times and a configurable network packet
-loss rate. It's perfect for evaluating the performance of a TCP stack
-under specified delay/bandwidth/loss conditions.
-
-The "nsim" plugin cross-connects two physical interfaces at layer 2,
-introducing the specified delay and network loss
-parameters. Reconfiguration on the fly is OK, with the proviso that
-packets held in the network simulator scheduling wheel will be lost.
-
-Configuration
--------------
-
-Configuration by debug CLI is simple. First, specify the simulator
-configuration: unidirectional delay (half of the desired RTT), the
-link bandwidth, and the expected average packet size. These parameters
-allow the network simulator allocate the right amount of buffering to
-produce the requested delay/bandwidth product.
-
-```
- set nsim delay 25.0 ms bandwidth 10 gbit packet-size 128
-```
-
-To simulate network packet drops, add either "packets-per-drop <nnnnn>" or
-"drop-fraction [0.0 ... 1.0]" parameters:
-
-```
- set nsim delay 25.0 ms bandwidth 10 gbit packet-size 128 packets-per-drop 10000
-```
-Remember to configure the layer-2 cross-connect:
-
-```
- nsim enable-disable <interface-1> <interface-2>
-```
-
-Packet Generator Configuration
-------------------------------
-
-Here's a unit-test configuration for the vpp packet generator:
-
-```
- loop cre
- set int ip address loop0 11.22.33.1/24
- set int state loop0 up
-
- loop cre
- set int ip address loop1 11.22.34.1/24
- set int state loop1 up
-
- set nsim delay 1.0 ms bandwidth 10 gbit packet-size 128 packets-per-drop 1000
- nsim enable-disable loop0 loop1
-
- packet-generator new {
- name s0
- limit 10000
- size 128-128
- interface loop0
- node ethernet-input
- data { IP4: 1.2.3 -> 4.5.6
- UDP: 11.22.33.44 -> 11.22.34.44
- UDP: 1234 -> 2345
- incrementing 114
- }
- }
-```
-
-For extra realism, the network simulator drops any specific packet
-with the specified probability. In this example, we see that slight
-variation from run to run occurs as it should.
-
-```
- DBGvpp# pa en
- DBGvpp# sh err
- Count Node Reason
- 9991 nsim Packets buffered
- 9 nsim Network loss simulation drop packets
- 9991 ethernet-input l3 mac mismatch
-
- DBGvpp# clear err
- DBGvpp# pa en
- DBGvpp# sh err
- sh err
- Count Node Reason
- 9993 nsim Packets buffered
- 7 nsim Network loss simulation drop packets
- 9993 ethernet-input l3 mac mismatch
-```
diff --git a/docs/usecases/networksim.rst b/docs/usecases/networksim.rst
new file mode 100644
index 0000000..9324c99
--- /dev/null
+++ b/docs/usecases/networksim.rst
@@ -0,0 +1,91 @@
+Generating traffic with VPP
+===========================
+
+Vpp includes a fairly capable network simulator plugin, which can
+simulate real-world round-trip times and a configurable network packet
+loss rate. It’s perfect for evaluating the performance of a TCP stack
+under specified delay/bandwidth/loss conditions.
+
+The “nsim” plugin cross-connects two physical interfaces at layer 2,
+introducing the specified delay and network loss parameters.
+Reconfiguration on the fly is OK, with the proviso that packets held in
+the network simulator scheduling wheel will be lost.
+
+Configuration
+-------------
+
+Configuration by debug CLI is simple. First, specify the simulator
+configuration: unidirectional delay (half of the desired RTT), the link
+bandwidth, and the expected average packet size. These parameters allow
+the network simulator allocate the right amount of buffering to produce
+the requested delay/bandwidth product.
+
+::
+
+ set nsim delay 25.0 ms bandwidth 10 gbit packet-size 128
+
+To simulate network packet drops, add either “packets-per-drop ” or
+“drop-fraction [0.0 … 1.0]” parameters:
+
+::
+
+ set nsim delay 25.0 ms bandwidth 10 gbit packet-size 128 packets-per-drop 10000
+
+Remember to configure the layer-2 cross-connect:
+
+::
+
+ nsim enable-disable <interface-1> <interface-2>
+
+Packet Generator Configuration
+------------------------------
+
+Here’s a unit-test configuration for the vpp packet generator:
+
+::
+
+ loop cre
+ set int ip address loop0 11.22.33.1/24
+ set int state loop0 up
+
+ loop cre
+ set int ip address loop1 11.22.34.1/24
+ set int state loop1 up
+
+ set nsim delay 1.0 ms bandwidth 10 gbit packet-size 128 packets-per-drop 1000
+ nsim enable-disable loop0 loop1
+
+ packet-generator new {
+ name s0
+ limit 10000
+ size 128-128
+ interface loop0
+ node ethernet-input
+ data { IP4: 1.2.3 -> 4.5.6
+ UDP: 11.22.33.44 -> 11.22.34.44
+ UDP: 1234 -> 2345
+ incrementing 114
+ }
+ }
+
+For extra realism, the network simulator drops any specific packet with
+the specified probability. In this example, we see that slight variation
+from run to run occurs as it should.
+
+::
+
+ DBGvpp# pa en
+ DBGvpp# sh err
+ Count Node Reason
+ 9991 nsim Packets buffered
+ 9 nsim Network loss simulation drop packets
+ 9991 ethernet-input l3 mac mismatch
+
+ DBGvpp# clear err
+ DBGvpp# pa en
+ DBGvpp# sh err
+ sh err
+ Count Node Reason
+ 9993 nsim Packets buffered
+ 7 nsim Network loss simulation drop packets
+ 9993 ethernet-input l3 mac mismatch
diff --git a/docs/usecases/simpleperf/iperf3.rst b/docs/usecases/simpleperf/iperf3.rst
index 6f5d345..d485a5e 100644
--- a/docs/usecases/simpleperf/iperf3.rst
+++ b/docs/usecases/simpleperf/iperf3.rst
@@ -60,7 +60,7 @@
csp2s22c03$ sudo ip link set dev ens802f0 up
csp2s22c03$ sudo ip addr add 10.10.2.1/24 dev ens802f1
csp2s22c03$ sudo ip link set dev ens802f1 up
-
+
List the route table:
.. code-block:: console
@@ -123,7 +123,7 @@
TX packets:1179 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:262230 (262.2 KB) TX bytes:139975 (139.9 KB)
-
+
ens802 Link encap:Ethernet HWaddr 68:05:ca:2e:76:e0
inet addr:10.10.1.2 Bcast:0.0.0.0 Mask:255.255.255.0
inet6 addr: fe80::6a05:caff:fe2e:76e0/64 Scope:Link
@@ -132,7 +132,7 @@
TX packets:40 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 B) TX bytes:5480 (5.4 KB)
-
+
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
@@ -233,5 +233,5 @@
[ ID] Interval Transfer Bandwidth Retr
[ 4] 0.00-10.00 sec 9.45 GBytes 8.12 Gbits/sec 16474 sender
[ 4] 0.00-10.00 sec 9.44 GBytes 8.11 Gbits/sec receiver
-
+
iperf Done.
diff --git a/docs/usecases/simpleperf/iperf31.rst b/docs/usecases/simpleperf/iperf31.rst
index 50abfdf..dbd0e72 100644
--- a/docs/usecases/simpleperf/iperf31.rst
+++ b/docs/usecases/simpleperf/iperf31.rst
@@ -22,38 +22,38 @@
.. code-block:: console
csp2s22c03$ ./install-vpp-native/dpdk/sbin/dpdk-devbind -s
-
+
Network devices using DPDK-compatible driver
============================================
<none>
-
+
Network devices using kernel driver
===================================
0000:03:00.0 'Ethernet Controller 10-Gigabit X540-AT2' if=enp3s0f0 drv=ixgbe unused=vfio-pci,uio_pci_generic *Active*
0000:03:00.1 'Ethernet Controller 10-Gigabit X540-AT2' if=enp3s0f1 drv=ixgbe unused=vfio-pci,uio_pci_generic *Active*
- 0000:82:00.0 'Ethernet Controller XL710 for 40GbE QSFP+' if=ens802f0d1,ens802f0 drv=i40e unused=uio_pci_generic
- 0000:82:00.1 'Ethernet Controller XL710 for 40GbE QSFP+' if=ens802f1d1,ens802f1 drv=i40e unused=uio_pci_generic
-
+ 0000:82:00.0 'Ethernet Controller XL710 for 40GbE QSFP+' if=ens802f0d1,ens802f0 drv=i40e unused=uio_pci_generic
+ 0000:82:00.1 'Ethernet Controller XL710 for 40GbE QSFP+' if=ens802f1d1,ens802f1 drv=i40e unused=uio_pci_generic
+
Other network devices
=====================
<none>
-
+
csp2s22c03$ sudo modprobe uio_pci_generic
csp2s22c03$ sudo ./install-vpp-native/dpdk/sbin/dpdk-devbind --bind uio_pci_generic 82:00.0
csp2s22c03$ sudo ./install-vpp-native/dpdk/sbin/dpdk-devbind --bind uio_pci_generic 82:00.1
csp2s22c03$ sudo ./install-vpp-native/dpdk/sbin/dpdk-devbind -s
-
+
Network devices using DPDK-compatible driver
============================================
0000:82:00.0 'Ethernet Controller XL710 for 40GbE QSFP+' drv=uio_pci_generic unused=i40e,vfio-pci
0000:82:00.1 'Ethernet Controller XL710 for 40GbE QSFP+' drv=uio_pci_generic unused=i40e,vfio-pci
-
+
Network devices using kernel driver
===================================
0000:03:00.0 'Ethernet Controller 10-Gigabit X540-AT2' if=enp3s0f0 drv=ixgbe unused=vfio-pci,uio_pci_generic *Active*
0000:03:00.1 'Ethernet Controller 10-Gigabit X540-AT2' if=enp3s0f1 drv=ixgbe unused=vfio-pci,uio_pci_generic *Active*
-
+
Start the VPP service, and verify that VPP is running:
.. code-block:: console
@@ -63,7 +63,7 @@
root 105655 1 98 17:34 ? 00:00:02 /usr/bin/vpp -c /etc/vpp/startup.conf
:w
105675 105512 0 17:34 pts/4 00:00:00 grep --color=auto vpp
-
+
To access the VPP CLI, issue the command sudo vppctl . From the VPP interface, list
all interfaces that are bound to DPDK using the command show interface:
@@ -109,7 +109,7 @@
[ ID] Interval Transfer Bandwidth Retr
[ 4] 0.00-10.00 sec 23.7 GBytes 20.3 Gbits/sec 13434 sender
[ 4] 0.00-10.00 sec 23.7 GBytes 20.3 Gbits/sec receiver
-
+
iperf Done.
The **show run** command displays the graph runtime statistics. Observe that the
diff --git a/docs/usecases/simpleperf/trex.rst b/docs/usecases/simpleperf/trex.rst
index 996ed15..6d38ce5 100644
--- a/docs/usecases/simpleperf/trex.rst
+++ b/docs/usecases/simpleperf/trex.rst
@@ -65,7 +65,7 @@
default_gw: 10.10.2.1
- ip: 10.10.1.2
default_gw: 10.10.1.1
-
+
Stop the previous VPP session and start it again in order to add a route for new
IP addresses 16.0.0.0/8 and 48.0.0.0/8, according to Figure 2. Those IP addresses
are needed because TRex generates packets that use these addresses. Refer to the
@@ -81,13 +81,13 @@
__/ __/ _ \ (_)__ | | / / _ \/ _ \
_/ _// // / / / _ \ | |/ / ___/ ___/
/_/ /____(_)_/\___/ |___/_/ /_/
-
+
vpp# sho int
Name Idx State Counter Count
FortyGigabitEthernet82/0/0 1 down
FortyGigabitEthernet82/0/1 2 down
local0 0 down
-
+
vpp#
vpp# set interface ip address FortyGigabitEthernet82/0/0 10.10.1.1/24
vpp# set interface ip address FortyGigabitEthernet82/0/1 10.10.2.1/24
@@ -109,7 +109,7 @@
Total-tx-bytes : 166886 bytes
Total-tx-sw-bytes : 166716 bytes
Total-rx-bytes : 166886 byte
-
+
Total-tx-pkt : 2528 pkts
Total-rx-pkt : 2528 pkts
Total-sw-tx-pkt : 2526 pkts
diff --git a/docs/usecases/simpleperf/trex1.rst b/docs/usecases/simpleperf/trex1.rst
index 0daa57f..1704b3f 100644
--- a/docs/usecases/simpleperf/trex1.rst
+++ b/docs/usecases/simpleperf/trex1.rst
@@ -15,7 +15,7 @@
Total-tx-bytes : 251062132504 bytes
Total-tx-sw-bytes : 21426636 bytes
Total-rx-bytes : 251040139922 byte
-
+
Total-tx-pkt : 430598064 pkts
Total-rx-pkt : 430554755 pkts
Total-sw-tx-pkt : 324646 pkts
diff --git a/docs/usecases/simpleperf/trex2.rst b/docs/usecases/simpleperf/trex2.rst
index 590bfd0..e1ff98f 100644
--- a/docs/usecases/simpleperf/trex2.rst
+++ b/docs/usecases/simpleperf/trex2.rst
@@ -18,36 +18,36 @@
# cd v2.46/
# ./trex -i
- -Per port stats table
- ports | 0 | 1 | 2 | 3
+ -Per port stats table
+ ports | 0 | 1 | 2 | 3
-----------------------------------------------------------------------------------------
- opackets | 0 | 0 | 0 | 0
- obytes | 0 | 0 | 0 | 0
- ipackets | 6 | 6 | 5 | 5
- ibytes | 384 | 384 | 320 | 320
- ierrors | 0 | 0 | 0 | 0
- oerrors | 0 | 0 | 0 | 0
- Tx Bw | 0.00 bps | 0.00 bps | 0.00 bps | 0.00 bps
-
- -Global stats enabled
+ opackets | 0 | 0 | 0 | 0
+ obytes | 0 | 0 | 0 | 0
+ ipackets | 6 | 6 | 5 | 5
+ ibytes | 384 | 384 | 320 | 320
+ ierrors | 0 | 0 | 0 | 0
+ oerrors | 0 | 0 | 0 | 0
+ Tx Bw | 0.00 bps | 0.00 bps | 0.00 bps | 0.00 bps
+
+ -Global stats enabled
Cpu Utilization : 0.0 %
- Platform_factor : 1.0
- Total-Tx : 0.00 bps
- Total-Rx : 238.30 bps
- Total-PPS : 0.00 pps
- Total-CPS : 0.00 cps
-
- Expected-PPS : 0.00 pps
- Expected-CPS : 0.00 cps
- Expected-BPS : 0.00 bps
-
- Active-flows : 0 Clients : 0 Socket-util : 0.0000 %
- Open-flows : 0 Servers : 0 Socket : 0 Socket/Clients : -nan
- drop-rate : 0.00 bps
- current time : 21.4 sec
- test duration : 0.0 sec
+ Platform_factor : 1.0
+ Total-Tx : 0.00 bps
+ Total-Rx : 238.30 bps
+ Total-PPS : 0.00 pps
+ Total-CPS : 0.00 cps
+
+ Expected-PPS : 0.00 pps
+ Expected-CPS : 0.00 cps
+ Expected-BPS : 0.00 bps
+
+ Active-flows : 0 Clients : 0 Socket-util : 0.0000 %
+ Open-flows : 0 Servers : 0 Socket : 0 Socket/Clients : -nan
+ drop-rate : 0.00 bps
+ current time : 21.4 sec
+ test duration : 0.0 sec
*** TRex is shutting down - cause: 'CTRL + C detected'
- All cores stopped !!
+ All cores stopped !!
In the other terminal start the TRex console. With this console we will execute the TRex commands.
@@ -55,28 +55,28 @@
# cd v2.46/
# ./trex -console
-
+
Using 'python' as Python interpreter
-
-
+
+
Connecting to RPC server on localhost:4501 [SUCCESS]
-
-
+
+
Connecting to publisher server on localhost:4500 [SUCCESS]
-
-
+
+
Acquiring ports [0, 1, 2, 3]: [SUCCESS]
-
-
+
+
Server Info:
-
+
Server version: v2.46 @ STL
Server mode: Stateless
Server CPU: 2 x Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz
- Ports count: 4 x 10Gbps @ VMXNET3 Ethernet Controller
-
+ Ports count: 4 x 10Gbps @ VMXNET3 Ethernet Controller
+
-=TRex Console v3.0=-
-
+
Type 'help' or '?' for supported actions
trex>
@@ -85,26 +85,26 @@
.. code-block:: console
trex>start -f ./stl/imix.py -p 0 1 2 3 -m 9475mbps
-
+
Removing all streams from port(s) [0, 1, 2, 3]: [SUCCESS]
-
-
+
+
Attaching 3 streams to port(s) [0]: [SUCCESS]
-
-
+
+
Attaching 3 streams to port(s) [1]: [SUCCESS]
-
-
+
+
Attaching 3 streams to port(s) [2]: [SUCCESS]
-
-
+
+
Attaching 3 streams to port(s) [3]: [SUCCESS]
-
-
+
+
Starting traffic on port(s) [0, 1, 2, 3]: [SUCCESS]
-
+
80.94 [ms]
-
+
trex>
The **-f ./stl/imix.py** argument specifies the file that is used to create the
@@ -116,77 +116,77 @@
.. code-block:: console
- -Per port stats table
+ -Per port stats table
ports | 0 | 1 | 2 | 3
-----------------------------------------------------------------------------------------
- opackets | 789907304 | 789894738 | 790017701 | 790017132
- obytes | 285397726750 | 285392406754 | 285406864578 | 285405883070
- ipackets | 1563501970 | 45 | 1563504693 | 44
- ibytes | 564870783050 | 2880 | 564873491682 | 2816
- ierrors | 15728759 | 0 | 15732451 | 0
- oerrors | 0 | 0 | 0 | 0
- Tx Bw | 606.55 Mbps | 606.19 Mbps | 606.25 Mbps | 606.51 Mbps
+ opackets | 789907304 | 789894738 | 790017701 | 790017132
+ obytes | 285397726750 | 285392406754 | 285406864578 | 285405883070
+ ipackets | 1563501970 | 45 | 1563504693 | 44
+ ibytes | 564870783050 | 2880 | 564873491682 | 2816
+ ierrors | 15728759 | 0 | 15732451 | 0
+ oerrors | 0 | 0 | 0 | 0
+ Tx Bw | 606.55 Mbps | 606.19 Mbps | 606.25 Mbps | 606.51 Mbps
- -Global stats enabled
- Cpu Utilization : 100.0 % 2.4 Gb/core
- Platform_factor : 1.0
- Total-Tx : 2.43 Gbps
- Total-Rx : 2.40 Gbps
- Total-PPS : 841.44 Kpps
- Total-CPS : 0.00 cps
-
- Expected-PPS : 0.00 pps
- Expected-CPS : 0.00 cps
- Expected-BPS : 0.00 bps
-
- Active-flows : 0 Clients : 0 Socket-util : 0.0000 %
- Open-flows : 0 Servers : 0 Socket : 0 Socket/Clients : -nan
- Total_queue_full : 6529970196
- drop-rate : 0.00 bps
- current time : 4016.8 sec
- test duration : 0.0 sec
-
+ -Global stats enabled
+ Cpu Utilization : 100.0 % 2.4 Gb/core
+ Platform_factor : 1.0
+ Total-Tx : 2.43 Gbps
+ Total-Rx : 2.40 Gbps
+ Total-PPS : 841.44 Kpps
+ Total-CPS : 0.00 cps
+
+ Expected-PPS : 0.00 pps
+ Expected-CPS : 0.00 cps
+ Expected-BPS : 0.00 bps
+
+ Active-flows : 0 Clients : 0 Socket-util : 0.0000 %
+ Open-flows : 0 Servers : 0 Socket : 0 Socket/Clients : -nan
+ Total_queue_full : 6529970196
+ drop-rate : 0.00 bps
+ current time : 4016.8 sec
+ test duration : 0.0 sec
+
More statistics can be displayed on the TRex console using the **tui** command.
.. code-block:: console
trex>tui
-
+
Global Statistics
-
- connection : localhost, Port 4501 total_tx_L2 : 2.45 Gb/sec
- version : STL @ v2.46 total_tx_L1 : 2.59 Gb/sec
- cpu_util. : 99.89% @ 2 cores (1 per port) total_rx : 2.42 Gb/sec
- rx_cpu_util. : 4.03% / 837.39 Kpkt/sec total_pps : 846.96 Kpkt/sec
- async_util. : 0.05% / 1.76 KB/sec drop_rate : 0 b/sec
- queue_full : 42,750,771 pkts
-
+
+ connection : localhost, Port 4501 total_tx_L2 : 2.45 Gb/sec
+ version : STL @ v2.46 total_tx_L1 : 2.59 Gb/sec
+ cpu_util. : 99.89% @ 2 cores (1 per port) total_rx : 2.42 Gb/sec
+ rx_cpu_util. : 4.03% / 837.39 Kpkt/sec total_pps : 846.96 Kpkt/sec
+ async_util. : 0.05% / 1.76 KB/sec drop_rate : 0 b/sec
+ queue_full : 42,750,771 pkts
+
Port Statistics
-
- port | 0 | 1 | 2 | 3 | total
+
+ port | 0 | 1 | 2 | 3 | total
-----------+-------------------+-------------------+-------------------+-------------------+------------------
- owner | root | root | root | root |
- link | UP | UP | UP | UP |
- state | TRANSMITTING | TRANSMITTING | TRANSMITTING | TRANSMITTING |
- speed | 10 Gb/s | 10 Gb/s | 10 Gb/s | 10 Gb/s |
- CPU util. | 99.89% | 99.89% | 99.89% | 99.89% |
- -- | | | | |
- Tx bps L2 | 612.76 Mbps | 613.07 Mbps | 612.52 Mbps | 612.77 Mbps | 2.45 Gbps
- Tx bps L1 | 646.64 Mbps | 646.96 Mbps | 646.4 Mbps | 646.64 Mbps | 2.59 Gbps
- Tx pps | 211.72 Kpps | 211.8 Kpps | 211.73 Kpps | 211.71 Kpps | 846.96 Kpps
- Line Util. | 6.47 % | 6.47 % | 6.46 % | 6.47 % |
- --- | | | | |
- Rx bps | 1.21 Gbps | \u25bc\u25bc\u25bc 23.03 bps | 1.21 Gbps | 5.94 bps | 2.42 G bps
- Rx pps | 418.59 Kpps | 0.04 pps | 418.77 Kpps | 0.01 pps | 837.36 Kpps
- ---- | | | | |
- opackets | 5227126 | 5227271 | 5432528 | 5432354 | 21319279
- ipackets | 10526000 | 5 | 10527054 | 4 | 21053063
- obytes | 1890829910 | 1891039152 | 1965259162 | 1965124338 | 7712252562
- ibytes | 3807894454 | 320 | 3808149896 | 256 | 7616044926
- tx-pkts | 5.23 Mpkts | 5.23 Mpkts | 5.43 Mpkts | 5.43 Mpkts | 21.32 Mpkts
- rx-pkts | 10.53 Mpkts | 5 pkts | 10.53 Mpkts | 4 pkts | 21.05 Mpkts
- tx-bytes | 1.89 GB | 1.89 GB | 1.97 GB | 1.97 GB | 7.71 GB
- rx-bytes | 3.81 GB | 320 B | 3.81 GB | 256 B | 7.62 GB
- ----- | | | | |
- oerrors | 0 | 0 | 0 | 0 | 0
- ierrors | 133,370 | 0 | 132,529 | 0 | 265,899
+ owner | root | root | root | root |
+ link | UP | UP | UP | UP |
+ state | TRANSMITTING | TRANSMITTING | TRANSMITTING | TRANSMITTING |
+ speed | 10 Gb/s | 10 Gb/s | 10 Gb/s | 10 Gb/s |
+ CPU util. | 99.89% | 99.89% | 99.89% | 99.89% |
+ -- | | | | |
+ Tx bps L2 | 612.76 Mbps | 613.07 Mbps | 612.52 Mbps | 612.77 Mbps | 2.45 Gbps
+ Tx bps L1 | 646.64 Mbps | 646.96 Mbps | 646.4 Mbps | 646.64 Mbps | 2.59 Gbps
+ Tx pps | 211.72 Kpps | 211.8 Kpps | 211.73 Kpps | 211.71 Kpps | 846.96 Kpps
+ Line Util. | 6.47 % | 6.47 % | 6.46 % | 6.47 % |
+ --- | | | | |
+ Rx bps | 1.21 Gbps | \u25bc\u25bc\u25bc 23.03 bps | 1.21 Gbps | 5.94 bps | 2.42 G bps
+ Rx pps | 418.59 Kpps | 0.04 pps | 418.77 Kpps | 0.01 pps | 837.36 Kpps
+ ---- | | | | |
+ opackets | 5227126 | 5227271 | 5432528 | 5432354 | 21319279
+ ipackets | 10526000 | 5 | 10527054 | 4 | 21053063
+ obytes | 1890829910 | 1891039152 | 1965259162 | 1965124338 | 7712252562
+ ibytes | 3807894454 | 320 | 3808149896 | 256 | 7616044926
+ tx-pkts | 5.23 Mpkts | 5.23 Mpkts | 5.43 Mpkts | 5.43 Mpkts | 21.32 Mpkts
+ rx-pkts | 10.53 Mpkts | 5 pkts | 10.53 Mpkts | 4 pkts | 21.05 Mpkts
+ tx-bytes | 1.89 GB | 1.89 GB | 1.97 GB | 1.97 GB | 7.71 GB
+ rx-bytes | 3.81 GB | 320 B | 3.81 GB | 256 B | 7.62 GB
+ ----- | | | | |
+ oerrors | 0 | 0 | 0 | 0 | 0
+ ierrors | 133,370 | 0 | 132,529 | 0 | 265,899
diff --git a/docs/usecases/trafficgen.md b/docs/usecases/trafficgen.md
deleted file mode 100644
index fe3d4c9..0000000
--- a/docs/usecases/trafficgen.md
+++ /dev/null
@@ -1,105 +0,0 @@
-Vpp Stateless Traffic Generation
-================================
-
-It's simple to configure vpp as a high-performance stateless traffic
-generator. A couple of vpp worker threads running on an older system
-can easily generate 20 MPPS' worth of traffic.
-
-In the configurations shown below, we connect a vpp traffic generator
-and a vpp UUT using two 40 gigabit ethernet ports on each system:
-
-```
- +-------------------+ +-------------------+
- | traffic generator | | UUT |
- | port 0 | <=======> | port 0 |
- | 192.168.40.2/24 | | 192.168.40.1/24 |
- +-------------------+ +-------------------+
-
- +-------------------+ +-------------------+
- | traffic generator | | UUT |
- | port 1 | <=======> | port 1 |
- | 192.168.41.2/24 | | 192.168.41.1/24 |
- +-------------------+ +-------------------+
-```
-
-Traffic Generator Setup Script
-------------------------------
-
-```
- set int ip address FortyGigabitEthernet2/0/0 192.168.40.2/24
- set int ip address FortyGigabitEthernet2/0/1 192.168.41.2/24
- set int state FortyGigabitEthernet2/0/0 up
- set int state FortyGigabitEthernet2/0/1 up
-
- comment { send traffic to the VPP UUT }
-
- packet-generator new {
- name worker0
- worker 0
- limit 0
- rate 1.2e7
- size 128-128
- tx-interface FortyGigabitEthernet2/0/0
- node FortyGigabitEthernet2/0/0-output
- data { IP4: 1.2.40 -> 3cfd.fed0.b6c8
- UDP: 192.168.40.10 -> 192.168.50.10
- UDP: 1234 -> 2345
- incrementing 114
- }
- }
-
- packet-generator new {
- name worker1
- worker 1
- limit 0
- rate 1.2e7
- size 128-128
- tx-interface FortyGigabitEthernet2/0/1
- node FortyGigabitEthernet2/0/1-output
- data { IP4: 1.2.4 -> 3cfd.fed0.b6c9
- UDP: 192.168.41.10 -> 192.168.51.10
- UDP: 1234 -> 2345
- incrementing 114
- }
- }
-
- comment { delete return traffic on sight }
-
- ip route add 192.168.50.0/24 via drop
- ip route add 192.168.51.0/24 via drop
-```
-
-Note 1: the destination MAC addresses shown in the configuration (e.g.
-3cfd.fed0.b6c8 and 3cfd.fed0.b6c9) **must** match the vpp UUT port MAC
-addresses.
-
-Note 2: this script assumes that /etc/vpp/startup.conf and/or the
-command-line in use specifies (at least) two worker threads. Uncomment
-"workers 2" in the cpu configuration section of /etc/vpp/startup.conf:
-
-```
- ## Specify a number of workers to be created
- ## Workers are pinned to N consecutive CPU cores while skipping "skip-cores" CPU core(s)
- ## and main thread's CPU core
- workers 2
-```
-
-Any plausible packet generator script - including one which replays
-pcap captures - can be used.
-
-
-UUT Setup Script
-----------------
-
-The vpp UUT uses a couple of static routes to forward traffic back to
-the traffic generator:
-
-```
- set int ip address FortyGigabitEthernet2/0/0 192.168.40.1/24
- set int ip address FortyGigabitEthernet2/0/1 192.168.41.1/24
- set int state FortyGigabitEthernet2/0/0 up
- set int state FortyGigabitEthernet2/0/1 up
-
- ip route add 192.168.50.10/32 via 192.168.41.2
- ip route add 192.168.51.10/32 via 192.168.40.2
-```
diff --git a/docs/usecases/trafficgen.rst b/docs/usecases/trafficgen.rst
new file mode 100644
index 0000000..82dba96
--- /dev/null
+++ b/docs/usecases/trafficgen.rst
@@ -0,0 +1,104 @@
+Stateless Traffic Gen with VPP
+==============================
+
+It’s simple to configure vpp as a high-performance stateless traffic
+generator. A couple of vpp worker threads running on an older system can
+easily generate 20 MPPS’ worth of traffic.
+
+In the configurations shown below, we connect a vpp traffic generator
+and a vpp UUT using two 40 gigabit ethernet ports on each system:
+
+::
+
+ +-------------------+ +-------------------+
+ | traffic generator | | UUT |
+ | port 0 | <=======> | port 0 |
+ | 192.168.40.2/24 | | 192.168.40.1/24 |
+ +-------------------+ +-------------------+
+
+ +-------------------+ +-------------------+
+ | traffic generator | | UUT |
+ | port 1 | <=======> | port 1 |
+ | 192.168.41.2/24 | | 192.168.41.1/24 |
+ +-------------------+ +-------------------+
+
+Traffic Generator Setup Script
+------------------------------
+
+::
+
+ set int ip address FortyGigabitEthernet2/0/0 192.168.40.2/24
+ set int ip address FortyGigabitEthernet2/0/1 192.168.41.2/24
+ set int state FortyGigabitEthernet2/0/0 up
+ set int state FortyGigabitEthernet2/0/1 up
+
+ comment { send traffic to the VPP UUT }
+
+ packet-generator new {
+ name worker0
+ worker 0
+ limit 0
+ rate 1.2e7
+ size 128-128
+ tx-interface FortyGigabitEthernet2/0/0
+ node FortyGigabitEthernet2/0/0-output
+ data { IP4: 1.2.40 -> 3cfd.fed0.b6c8
+ UDP: 192.168.40.10 -> 192.168.50.10
+ UDP: 1234 -> 2345
+ incrementing 114
+ }
+ }
+
+ packet-generator new {
+ name worker1
+ worker 1
+ limit 0
+ rate 1.2e7
+ size 128-128
+ tx-interface FortyGigabitEthernet2/0/1
+ node FortyGigabitEthernet2/0/1-output
+ data { IP4: 1.2.4 -> 3cfd.fed0.b6c9
+ UDP: 192.168.41.10 -> 192.168.51.10
+ UDP: 1234 -> 2345
+ incrementing 114
+ }
+ }
+
+ comment { delete return traffic on sight }
+
+ ip route add 192.168.50.0/24 via drop
+ ip route add 192.168.51.0/24 via drop
+
+Note 1: the destination MAC addresses shown in the configuration (e.g.
+3cfd.fed0.b6c8 and 3cfd.fed0.b6c9) **must** match the vpp UUT port MAC
+addresses.
+
+Note 2: this script assumes that /etc/vpp/startup.conf and/or the
+command-line in use specifies (at least) two worker threads. Uncomment
+“workers 2” in the cpu configuration section of /etc/vpp/startup.conf:
+
+::
+
+ ## Specify a number of workers to be created
+ ## Workers are pinned to N consecutive CPU cores while skipping "skip-cores" CPU core(s)
+ ## and main thread's CPU core
+ workers 2
+
+Any plausible packet generator script - including one which replays pcap
+captures - can be used.
+
+UUT Setup Script
+----------------
+
+The vpp UUT uses a couple of static routes to forward traffic back to
+the traffic generator:
+
+::
+
+ set int ip address FortyGigabitEthernet2/0/0 192.168.40.1/24
+ set int ip address FortyGigabitEthernet2/0/1 192.168.41.1/24
+ set int state FortyGigabitEthernet2/0/0 up
+ set int state FortyGigabitEthernet2/0/1 up
+
+ ip route add 192.168.50.10/32 via 192.168.41.2
+ ip route add 192.168.51.10/32 via 192.168.40.2
diff --git a/docs/usecases/vhost/index.rst b/docs/usecases/vhost/index.rst
index 002ebc1..dd189b4 100644
--- a/docs/usecases/vhost/index.rst
+++ b/docs/usecases/vhost/index.rst
@@ -1,7 +1,7 @@
.. _vhost:
-FD.io VPP with Virtual Machines
-===============================
+VPP with Virtual Machines
+=========================
This chapter will describe how to use FD.io VPP with virtual machines. We describe
how to create Vhost port with VPP and how to connect them to VPP. We will also discuss
some limitations of Vhost.
diff --git a/docs/usecases/vhost/vhost.rst b/docs/usecases/vhost/vhost.rst
index f62faad..e5ff591 100644
--- a/docs/usecases/vhost/vhost.rst
+++ b/docs/usecases/vhost/vhost.rst
@@ -12,11 +12,11 @@
The image that we use is based on an Ubuntu cloud image downloaded from:
`Ubuntu Cloud Images <https://cloud-images.ubuntu.com/xenial/current>`_.
-All FD.io VPP commands are being run from a su shell.
+All FD.io VPP commands are being run from a su shell.
.. _vhosttopo:
-Topology
+Topology
---------
In this case we will use 2 systems. One system we will be running standard linux, the other will
@@ -45,7 +45,7 @@
__/ __/ _ \ (_)__ | | / / _ \/ _ \
_/ _// // / / / _ \ | |/ / ___/ ___/
/_/ /____(_)_/\___/ |___/_/ /_/
-
+
vpp# clear interfaces
vpp# show int
Name Idx State Counter Count
@@ -56,7 +56,7 @@
For more information on the interface commands refer to: :ref:`intcommands`
-The next step will be to create the virtual port using the :ref:`createvhostuser` command.
+The next step will be to create the virtual port using the ``createvhostuser`` command.
This command will create the virtual port in VPP and create a linux socket that the VM will
use to connect to VPP.
@@ -79,7 +79,7 @@
Notice the interface **VirtualEthernet0/0/0**. In this example we created the virtual interface as
a client.
-We can get more detail on the vhost connection with the :ref:`showvhost` command.
+We can get more detail on the vhost connection with the ``showvhost`` command.
.. code-block:: console
@@ -100,14 +100,14 @@
protocol features (0x3)
VHOST_USER_PROTOCOL_F_MQ (0)
VHOST_USER_PROTOCOL_F_LOG_SHMFD (1)
-
+
socket filename /tmp/vm00.sock type client errno "No such file or directory"
-
+
rx placement:
tx placement: spin-lock
thread 0 on vring 0
thread 1 on vring 0
-
+
Memory regions (total 0)
Notice **No such file or directory** and **Memory regions (total 0)**. This is because the
diff --git a/docs/usecases/vhost/vhost02.rst b/docs/usecases/vhost/vhost02.rst
index d316463..17bbafc 100644
--- a/docs/usecases/vhost/vhost02.rst
+++ b/docs/usecases/vhost/vhost02.rst
@@ -3,8 +3,7 @@
Creating the Virtual Machine
----------------------------
-We will now create the virtual machine. We use the "virsh create command". For the complete file we
-use refer to :ref:`xmlexample`.
+We will now create the virtual machine. We use the "virsh create command".
It is important to note that in the XML file we specify the socket path that is used to connect to
FD.io VPP.
@@ -85,26 +84,26 @@
protocol features (0x3)
VHOST_USER_PROTOCOL_F_MQ (0)
VHOST_USER_PROTOCOL_F_LOG_SHMFD (1)
-
+
socket filename /tmp/vm00.sock type client errno "Success"
-
+
rx placement:
thread 1 on vring 1, polling
tx placement: spin-lock
thread 0 on vring 0
thread 1 on vring 0
-
+
Memory regions (total 2)
region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr
====== ===== ================== ================== ================== ================== =============== ===
0 31 0x0000000000000000 0x00000000000a0000 0x00007f1db9c00000 0x0000000000000000 0x00007f7db0400 000
1 32 0x00000000000c0000 0x000000000ff40000 0x00007f1db9cc0000 0x00000000000c0000 0x00007f7d94ec0 000
-
+
Virtqueue 0 (TX)
qsz 1024 last_avail_idx 0 last_used_idx 0
avail.flags 0 avail.idx 256 used.flags 1 used.idx 0
kickfd 33 callfd 34 errfd -1
-
+
Virtqueue 1 (RX)
qsz 1024 last_avail_idx 8 last_used_idx 8
avail.flags 0 avail.idx 8 used.flags 1 used.idx 8
diff --git a/docs/usecases/vhost/vhost03.rst b/docs/usecases/vhost/vhost03.rst
index ed58334..05d1fd1 100644
--- a/docs/usecases/vhost/vhost03.rst
+++ b/docs/usecases/vhost/vhost03.rst
@@ -17,7 +17,7 @@
vpp# show bridge 100 det
BD-ID Index BSN Age(min) Learning U-Forwrd UU-Flood Flooding ARP-Term BVI-Intf
100 1 0 off on on on on off N/A
-
+
Interface If-idx ISN SHG BVI TxFlood VLAN-Tag-Rewrite
VirtualEthernet0/0/0 3 1 0 - * none
TenGigabitEthernet86/0/0 1 1 0 - * none
@@ -85,4 +85,4 @@
tx packets 16
tx bytes 1476
local0 0 down
- vpp#
+ vpp#
diff --git a/docs/usecases/vhost/vhost04.rst b/docs/usecases/vhost/vhost04.rst
index 256c0b8..02e05bc 100644
--- a/docs/usecases/vhost/vhost04.rst
+++ b/docs/usecases/vhost/vhost04.rst
@@ -11,7 +11,7 @@
Id Name State
----------------------------------------------------
65 iperf-server3 running
-
+
cto@tf-ucs-3:~$ virsh destroy iperf-server3
Domain iperf-server3 destroyed
diff --git a/docs/usecases/vhost/vhost05.rst b/docs/usecases/vhost/vhost05.rst
index 4eba6e1..c7c80d5 100644
--- a/docs/usecases/vhost/vhost05.rst
+++ b/docs/usecases/vhost/vhost05.rst
@@ -11,7 +11,7 @@
shows with 2 threads, 2 cores and a Queue size of 1024 the maximum NDR throughput was about 7.5 Mpps.
This is about the limit at this time.
-For all the details on the CSIT VM vhost connection refer to the
+For all the details on the CSIT VM vhost connection refer to the
`CSIT VM vHost performance tests <https://docs.fd.io/csit/rls1804/report/vpp_performance_tests/packet_throughput_graphs/vm_vhost.html>`_.
diff --git a/docs/usecases/vmxnet3.rst b/docs/usecases/vmxnet3.rst
index 427ebba..3cf730c 100644
--- a/docs/usecases/vmxnet3.rst
+++ b/docs/usecases/vmxnet3.rst
@@ -124,10 +124,10 @@
Network devices using DPDK-compatible driver
============================================
<none>
-
+
Network devices using kernel driver
===================================
- 0000:03:00.0 'VMXNET3 Ethernet Controller' if=ens160 drv=vmxnet3 unused=vfio-pci,uio_pci_generic
+ 0000:03:00.0 'VMXNET3 Ethernet Controller' if=ens160 drv=vmxnet3 unused=vfio-pci,uio_pci_generic
0000:0b:00.0 'VMXNET3 Ethernet Controller' drv=vfio-pci unused=vmxnet3,uio_pci_generic
0000:13:00.0 'VMXNET3 Ethernet Controller' drv=vfio-pci unused=vmxnet3,uio_pci_generic
.....
diff --git a/docs/usecases/vpp_init_sswan_resp.md b/docs/usecases/vpp_init_sswan_resp.md
deleted file mode 100644
index 40da137..0000000
--- a/docs/usecases/vpp_init_sswan_resp.md
+++ /dev/null
@@ -1,195 +0,0 @@
-VPP as IKEv2 initiator and strongSwan as responder
-==================================================
-
-Prerequisites
--------------
-
-To make the examples easier to configure ``docker`` it is required to pull strongSwan docker image. The networking is done using Linux' veth interfaces and namespaces.
-
-Setup
------
-
-First a topology:
-
-```
-192.168.3.2 192.168.5.2
- + loopback
- | +
-+----+----+ 192.168.10.2 +-----+----+
-| VPP | |strongSwan|
-|initiator+----------------------+responder |
-+---------+ +----------+
- 192.168.10.1
-```
-
-Create veth interfaces and namespaces and configure them:
-
-```
-sudo ip link add gw type veth peer name swanif
-sudo ip link set dev gw up
-
-sudo ip netns add ns
-sudo ip link add veth_priv type veth peer name priv
-sudo ip link set dev priv up
-sudo ip link set dev veth_priv up netns ns
-
-sudo ip netns exec ns \
- bash -c "
- ip link set dev lo up
- ip addr add 192.168.3.2/24 dev veth_priv
- ip route add 192.168.5.0/24 via 192.168.3.1"
-```
-
-
-Create directory with strongswan configs that will be mounted to the docker container
-```
-mkdir /tmp/sswan
-```
-
-Create the ``ipsec.conf`` file in the ``/tmp/sswan`` directory with following content:
-
-```
-config setup
- strictcrlpolicy=no
-
-conn initiator
- mobike=no
- auto=add
- type=tunnel
- keyexchange=ikev2
- ike=aes256gcm16-prfsha256-modp2048!
- esp=aes256gcm16-esn!
-
-# local:
- leftauth=psk
- leftid=@sswan.vpn.example.com
- leftsubnet=192.168.5.0/24
-
-# remote: (gateway)
- rightid=@roadwarrior.vpp
- right=192.168.10.2
- rightauth=psk
- rightsubnet=192.168.3.0/24
-```
-
-``/tmp/sswan/ipsec.secrets``
-```
-: PSK 'Vpp123'
-```
-
-``/tmp/sswan/strongswan.conf``
-```
-charon {
- load_modular = yes
- plugins {
- include strongswan.d/charon/*.conf
- }
- filelog {
- /tmp/charon.log {
- time_format = %b %e %T
- ike_name = yes
- append = no
- default = 2
- flush_line = yes
- }
- }
-}
-include strongswan.d/*.conf
-```
-
-Start docker container with strongSwan:
-
-```
- docker run --name sswan -d --privileged --rm --net=none \
- -v /tmp/sswan:/conf -v /tmp/sswan:/etc/ipsec.d philplckthun/strongswan
-```
-
-Finish configuration of initiator's private network:
-
-```
-pid=$(docker inspect --format "{{.State.Pid}}" sswan)
-sudo ip link set netns $pid dev swanif
-
-sudo nsenter -t $pid -n ip addr add 192.168.10.1/24 dev swanif
-sudo nsenter -t $pid -n ip link set dev swanif up
-
-sudo nsenter -t $pid -n ip addr add 192.168.5.2/32 dev lo
-sudo nsenter -t $pid -n ip link set dev lo up
-```
-
-Start VPP ...
-
-```
-sudo /usr/bin/vpp unix { \
- cli-listen /tmp/vpp.sock \
- gid $(id -g) } \
- api-segment { prefix vpp } \
- plugins { plugin dpdk_plugin.so { disable } }
-```
-
-... and configure it:
-
-```
-create host-interface name gw
-set interface ip addr host-gw 192.168.10.2/24
-set interface state host-gw up
-
-create host-interface name priv
-set interface ip addr host-priv 192.168.3.1/24
-set interface state host-priv up
-
-ikev2 profile add pr1
-ikev2 profile set pr1 auth shared-key-mic string Vpp123
-ikev2 profile set pr1 id local fqdn roadwarrior.vpp
-ikev2 profile set pr1 id remote fqdn sswan.vpn.example.com
-
-ikev2 profile set pr1 traffic-selector local ip-range 192.168.3.0 - 192.168.3.255 port-range 0 - 65535 protocol 0
-ikev2 profile set pr1 traffic-selector remote ip-range 192.168.5.0 - 192.168.5.255 port-range 0 - 65535 protocol 0
-
-ikev2 profile set pr1 responder host-gw 192.168.10.1
-ikev2 profile set pr1 ike-crypto-alg aes-gcm-16 256 ike-dh modp-2048
-ikev2 profile set pr1 esp-crypto-alg aes-gcm-16 256
-
-create ipip tunnel src 192.168.10.2 dst 192.168.10.1
-ikev2 profile set pr1 tunnel ipip0
-ip route add 192.168.5.0/24 via 192.168.10.1 ipip0
-set interface unnumbered ipip0 use host-gw
-```
-
-Initiate the IKEv2 connection:
-
-```
-vpp# ikev2 initiate sa-init pr1
-```
-
-```
-vpp# show ikev2 sa details
- iip 192.168.10.2 ispi f717b0cbd17e27c3 rip 192.168.10.1 rspi e9b7af7fc9b13361
- encr:aes-gcm-16 prf:hmac-sha2-256 dh-group:modp-2048
- nonce i:eb0354613b268c6372061bbdaab13deca37c8a625b1f65c073d25df2ecfe672e
- r:70e1248ac09943047064f6a2135fa2a424778ba03038ab9c4c2af8aba179ed84
- SK_d 96bd4feb59be2edf1930a12a3a5d22e30195ee9f56ea203c5fb6cba5dd2bb80f
- SK_e i:00000000: 5b75b9d808c8467fd00a0923c06efee2a4eb1d033c57532e05f9316ed9c56fe9
- 00000020: c4db9114
- r:00000000: 95121b63372d20b83558dc3e209b9affef042816cf071c86a53543677b40c15b
- 00000020: f169ab67
- SK_p i:fb40d1114c347ddc3228ba004d4759d58f9c1ae6f1746833f908d39444ef92b1
- r:aa049828240cb242e1d5aa625cd5914dc8f8e980a74de8e06883623d19384902
- identifier (i) id-type fqdn data roadwarrior.vpp
- identifier (r) id-type fqdn data sswan.vpn.example.com
- child sa 0:encr:aes-gcm-16 esn:yes
- spi(i) 9dffd57a spi(r) c4e0ef53
- SK_e i:290c681694f130b33d511335dd257e78721635b7e8aa87930dd77bb1d6dd3f42
- r:0a09fa18cf1cf65c6324df02b46dcc998b84e5397cf911b63e0c096053946c2e
- traffic selectors (i):0 type 7 protocol_id 0 addr 192.168.3.0 - 192.168.3.255 port 0 - 65535
- traffic selectors (r):0 type 7 protocol_id 0 addr 192.168.5.0 - 192.168.5.255 port 0 - 65535
-```
-
-Now we can generate some traffic between responder's and initiator's private networks and see it works.
-
-```
-$ sudo ip netns exec ns ping 192.168.5.2
-PING 192.168.5.2 (192.168.5.2) 56(84) bytes of data.
-64 bytes from 192.168.5.2: icmp_seq=1 ttl=63 time=0.450 ms
-64 bytes from 192.168.5.2: icmp_seq=2 ttl=63 time=0.630 ms
-```
diff --git a/docs/usecases/vpp_resp_sswan_init.md b/docs/usecases/vpp_resp_sswan_init.md
deleted file mode 100644
index 613a4b6..0000000
--- a/docs/usecases/vpp_resp_sswan_init.md
+++ /dev/null
@@ -1,197 +0,0 @@
-VPP as IKEv2 responder and strongSwan as initiator
-==================================================
-
-
-Prerequisites
--------------
-
-To make the examples easier to configure ``docker`` it is required to pull strongSwan docker image. The networking is done using Linux' veth interfaces and namespaces.
-
-Setup
------
-
-First a topology:
-
-```
-192.168.3.2 192.168.5.2
- + loopback
- | +
-+----+----+ 192.168.10.2 +-----+----+
-| VPP | |initiator |
-|responder+----------------------+strongSwan|
-+---------+ +----------+
- 192.168.10.1
-```
-
-Create veth interfaces and namespaces and configure them:
-
-```
-sudo ip link add gw type veth peer name swanif
-sudo ip link set dev gw up
-
-sudo ip netns add ns
-sudo ip link add veth_priv type veth peer name priv
-sudo ip link set dev priv up
-sudo ip link set dev veth_priv up netns ns
-
-sudo ip netns exec ns \
- bash -c "
- ip link set dev lo up
- ip addr add 192.168.3.2/24 dev veth_priv
- ip route add 192.168.5.0/24 via 192.168.3.1"
-```
-
-
-Create directory with strongswan configs that will be mounted to the docker container
-```
-mkdir /tmp/sswan
-```
-
-Create the ``ipsec.conf`` file in the ``/tmp/sswan`` directory with following content:
-```
-config setup
- strictcrlpolicy=no
-
-conn initiator
- mobike=no
- auto=add
- type=tunnel
- keyexchange=ikev2
- ike=aes256gcm16-prfsha256-modp2048!
- esp=aes256gcm16-esn!
-
- # local:
- leftauth=psk
- leftid=@roadwarrior.vpn.example.com
- leftsubnet=192.168.5.0/24
-
- # remote: (vpp gateway)
- rightid=@vpp.home
- right=192.168.10.2
- rightauth=psk
- rightsubnet=192.168.3.0/24
-```
-
-``/tmp/sswan/ipsec.secrets``
-```
-: PSK 'Vpp123'
-```
-
-``/tmp/sswan/strongswan.conf``
-```
-charon {
- load_modular = yes
- plugins {
- include strongswan.d/charon/*.conf
- }
- filelog {
- /tmp/charon.log {
- time_format = %b %e %T
- ike_name = yes
- append = no
- default = 2
- flush_line = yes
- }
- }
-}
-include strongswan.d/*.conf
-```
-
-Start docker container with strongSwan:
-
-```
- docker run --name sswan -d --privileged --rm --net=none \
- -v /tmp/sswan:/conf -v /tmp/sswan:/etc/ipsec.d philplckthun/strongswan
-```
-
-Finish configuration of initiator's private network:
-
-```
-pid=$(docker inspect --format "{{.State.Pid}}" sswan)
-sudo ip link set netns $pid dev swanif
-
-sudo nsenter -t $pid -n ip addr add 192.168.10.1/24 dev swanif
-sudo nsenter -t $pid -n ip link set dev swanif up
-
-sudo nsenter -t $pid -n ip addr add 192.168.5.2/32 dev lo
-sudo nsenter -t $pid -n ip link set dev lo up
-```
-
-Start VPP ...
-
-```
-sudo /usr/bin/vpp unix { \
- cli-listen /tmp/vpp.sock \
- gid $(id -g) } \
- api-segment { prefix vpp } \
- plugins { plugin dpdk_plugin.so { disable } }
-```
-
-... and configure it:
-
-```
-create host-interface name gw
-set interface ip addr host-gw 192.168.10.2/24
-set interface state host-gw up
-
-create host-interface name priv
-set interface ip addr host-priv 192.168.3.1/24
-set interface state host-priv up
-
-ikev2 profile add pr1
-ikev2 profile set pr1 auth shared-key-mic string Vpp123
-ikev2 profile set pr1 id local fqdn vpp.home
-ikev2 profile set pr1 id remote fqdn roadwarrior.vpn.example.com
-
-ikev2 profile set pr1 traffic-selector local ip-range 192.168.3.0 - 192.168.3.255 port-range 0 - 65535 protocol 0
-ikev2 profile set pr1 traffic-selector remote ip-range 192.168.5.0 - 192.168.5.255 port-range 0 - 65535 protocol 0
-
-create ipip tunnel src 192.168.10.2 dst 192.168.10.1
-ikev2 profile set pr1 tunnel ipip0
-ip route add 192.168.5.0/24 via 192.168.10.1 ipip0
-set interface unnumbered ipip0 use host-gw
-```
-
-Initiate the IKEv2 connection:
-
-```
-$ sudo docker exec sswan ipsec up initiator
-
-...
-CHILD_SA initiator{1} established with SPIs c320c95f_i 213932c2_o and TS 192.168.5.0/24 === 192.168.3.0/24
-connection 'initiator' established successfully
-```
-
-```
-vpp# show ikev2 sa details
-
-iip 192.168.10.1 ispi 7849021d9f655f1b rip 192.168.10.2 rspi 5a9ca7469a035205
- encr:aes-gcm-16 prf:hmac-sha2-256 dh-group:modp-2048
- nonce i:692ce8fd8f1c1934f63bfa2b167c4de2cff25640dffe938cdfe01a5d7f6820e6
- r:3ed84a14ea8526063e5aa762312be225d33e866d7152b9ce23e50f0ededca9e3
- SK_d 9a9b896ed6c35c78134fcd6e966c04868b6ecacf6d5088b4b2aee8b05d30fdda
- SK_e i:00000000: 1b1619788d8c812ca5916c07e635bda860f15293099f3bf43e8d88e52074b006
- 00000020: 72c8e3e3
- r:00000000: 89165ceb2cef6a6b3319f437386292d9ef2e96d8bdb21eeb0cb0d3b92733de03
- 00000020: bbc29c50
- SK_p i:fe35fca30985ee75e7c8bc0d7bc04db7a0e1655e997c0f5974c31458826b6fef
- r:0dd318662a96a25fcdf4998d8c6e4180c67c03586cf91dab26ed43aeda250272
- identifier (i) id-type fqdn data roadwarrior.vpn.example.com
- identifier (r) id-type fqdn data vpp.home
- child sa 0:encr:aes-gcm-16 esn:yes
- spi(i) c320c95f spi(r) 213932c2
- SK_e i:2a6c9eae9dbed202c0ae6ccc001621aba5bb0b01623d4de4d14fd27bd5185435
- r:15e2913d39f809040ca40a02efd27da298b6de05f67bd8f10210da5e6ae606fb
- traffic selectors (i):0 type 7 protocol_id 0 addr 192.168.5.0 - 192.168.5.255 port 0 - 65535
- traffic selectors (r):0 type 7 protocol_id 0 addr 192.168.3.0 - 192.168.3.255 port 0 - 65535
-
-```
-
-Now we can generate some traffic between responder's and initiator's private networks and see it works.
-
-```
-$ sudo ip netns exec ns ping 192.168.5.2
-PING 192.168.5.2 (192.168.5.2) 56(84) bytes of data.
-64 bytes from 192.168.5.2: icmp_seq=1 ttl=63 time=1.02 ms
-64 bytes from 192.168.5.2: icmp_seq=2 ttl=63 time=0.599 ms
-```
diff --git a/docs/usecases/ConnectingVPC.rst b/docs/usecases/vppcloud/ConnectingVPC.rst
similarity index 100%
rename from docs/usecases/ConnectingVPC.rst
rename to docs/usecases/vppcloud/ConnectingVPC.rst
diff --git a/docs/usecases/automatingthedeployment.rst b/docs/usecases/vppcloud/automatingthedeployment.rst
similarity index 100%
rename from docs/usecases/automatingthedeployment.rst
rename to docs/usecases/vppcloud/automatingthedeployment.rst
diff --git a/docs/usecases/vppcloud.rst b/docs/usecases/vppcloud/index.rst
similarity index 91%
rename from docs/usecases/vppcloud.rst
rename to docs/usecases/vppcloud/index.rst
index 22bb984..728eb2e 100644
--- a/docs/usecases/vppcloud.rst
+++ b/docs/usecases/vppcloud/index.rst
@@ -1,7 +1,7 @@
.. _vppcloud:
-VPP inside the Cloud
-====================
+VPP in the Cloud
+================
This section will cover the VPP deployment inside two different Public Cloud Provider: Amazon AWS and Microsoft Azure. Furthermore, we describe how to interconnect several Public Cloud Regions together with Segment Routing per IPv6 and we show some Performance Evaluation. Finally, we make our Terraform scripts available to the community, which could help in automating the VPP deployment inside the Cloud.
diff --git a/docs/usecases/vppinaws.rst b/docs/usecases/vppcloud/vppinaws.rst
similarity index 84%
rename from docs/usecases/vppinaws.rst
rename to docs/usecases/vppcloud/vppinaws.rst
index 468915a..8d5662d 100644
--- a/docs/usecases/vppinaws.rst
+++ b/docs/usecases/vppcloud/vppinaws.rst
@@ -3,21 +3,21 @@
.. toctree::
VPP in AWS
-___________________
+==========
Warning: before starting this guide you should have a minimum knowledge on how `AWS works <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html>`_!
-First of all, you should log into your Virtual Machine inside AWS (we suggest to create an instance with Ubuntu 16.04 on a m5 type) and download some useful packages to make VPP installation as smooth as possible:
+First of all, you should log into your Virtual Machine inside AWS (we suggest to create an instance with Ubuntu 16.04 on a m5 type) and download some useful packages to make VPP installation as smooth as possible:
.. code-block:: console
- $ sudo apt-get update
- $ sudo apt-get upgrade
- $ sudo apt-get install build-essential
- $ sudo apt-get install python-pip
- $ sudo apt-get install libnuma-dev
- $ sudo apt-get install make
- $ sudo apt install libelf-dev
+ $ sudo apt-get update
+ $ sudo apt-get upgrade
+ $ sudo apt-get install build-essential
+ $ sudo apt-get install python-pip
+ $ sudo apt-get install libnuma-dev
+ $ sudo apt-get install make
+ $ sudo apt install libelf-dev
@@ -30,14 +30,14 @@
-In this case we downloaded VPP version 18.07 but actually you can use any VPP version available. Then, you can install VPP with all of its plugins:
+In this case we downloaded VPP version 18.07 but actually you can use any VPP version available. Then, you can install VPP with all of its plugins:
.. code-block:: console
- $ sudo apt-get update
- $ sudo apt-get install vpp
- $ sudo apt-get install vpp-plugins vpp-dbg vpp-dev vpp-api-java vpp-api-python vpp-api-lua
+ $ sudo apt-get update
+ $ sudo apt-get install vpp
+ $ sudo apt-get install vpp-plugins vpp-dbg vpp-dev vpp-api-java vpp-api-python vpp-api-lua
diff --git a/docs/usecases/vppinazure.rst b/docs/usecases/vppcloud/vppinazure.rst
similarity index 60%
rename from docs/usecases/vppinazure.rst
rename to docs/usecases/vppcloud/vppinazure.rst
index f64e2a7..f1bfa42 100644
--- a/docs/usecases/vppinazure.rst
+++ b/docs/usecases/vppcloud/vppinazure.rst
@@ -3,14 +3,14 @@
.. toctree::
VPP in Azure
-___________________
+============
Before starting, a few notes:
-* in our configuration we use only DPDK 18.02, since with the newer versions, such as DPDK 18.05, we obtained several problems during VPP installation (mostly related with MLX4 PMD Drivers).
+* in our configuration we use only DPDK 18.02, since with the newer versions, such as DPDK 18.05, we obtained several problems during VPP installation (mostly related with MLX4 PMD Drivers).
* Some of the commands are taken from `Azure’s DPDK page <https://docs.microsoft.com/en-us/azure/virtual-network/setup-dpdk>`_.
@@ -20,9 +20,9 @@
.. code-block:: console
- $ sudo add-apt-repository ppa:canonical-server/dpdk-azure –y
- $ sudo apt-get update
- $ sudo apt-get install -y librdmacm-dev librdmacm1 build-essential libnuma-dev libmnl-dev
+ $ sudo add-apt-repository ppa:canonical-server/dpdk-azure –y
+ $ sudo apt-get update
+ $ sudo apt-get install -y librdmacm-dev librdmacm1 build-essential libnuma-dev libmnl-dev
Then, we download DPDK 18.02:
@@ -37,11 +37,11 @@
.. code-block:: console
- CONFIG_RTE_BUILD_SHARED_LIB=n
- CONFIG_RTE_LIBRTE_MLX4_PMD=y
- CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS=y
- CONFIG_RTE_LIBRTE_TAP_PMD=y
- CONFIG_RTE_LIBRTE_FAILSAFE_PMD=y
+ CONFIG_RTE_BUILD_SHARED_LIB=n
+ CONFIG_RTE_LIBRTE_MLX4_PMD=y
+ CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS=y
+ CONFIG_RTE_LIBRTE_TAP_PMD=y
+ CONFIG_RTE_LIBRTE_FAILSAFE_PMD=y
and then:
@@ -67,16 +67,16 @@
.. code-block:: console
- $ echo 1024 | sudo tee /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages
- $ mkdir /mnt/huge
- $ sudo mount -t hugetlbfs nodev /mnt/huge
- $ grep Huge /proc/meminfo
- $ modprobe -a ib_uverbs
- $ cd x86_64-native-linuxapp-gcc/
- $ ls
- $ cd lib/
- $ ls
- $ sudo cp librte_pmd_mlx4_glue.so.18.02.0 /usr/lib
+ $ echo 1024 | sudo tee /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages
+ $ mkdir /mnt/huge
+ $ sudo mount -t hugetlbfs nodev /mnt/huge
+ $ grep Huge /proc/meminfo
+ $ modprobe -a ib_uverbs
+ $ cd x86_64-native-linuxapp-gcc/
+ $ ls
+ $ cd lib/
+ $ ls
+ $ sudo cp librte_pmd_mlx4_glue.so.18.02.0 /usr/lib
**Now we focus on VPP installation:**
@@ -88,8 +88,8 @@
.. code-block:: console
- $ git clone https://gerrit.fd.io/r/vpp
- $ git checkout v18.07
+ $ git clone https://gerrit.fd.io/r/vpp
+ $ git checkout v18.07
Then, we build VPP, using the external DPDK configuration we previously made:
@@ -97,10 +97,10 @@
.. code-block:: console
- $ build-data/platforms/vpp.mk
- $ vpp_uses_external_dpdk = yes
- $ vpp_dpdk_inc_dir = <PATH_TO_DESTDIR_NAME_FROM_ABOVE>/include/dpdk/
- $ vpp_dpdk_lib_dir =<PATH_TO_DESTDIR_NAME_FROM_ABOVE>/lib
+ $ build-data/platforms/vpp.mk
+ $ vpp_uses_external_dpdk = yes
+ $ vpp_dpdk_inc_dir = <PATH_TO_DESTDIR_NAME_FROM_ABOVE>/include/dpdk/
+ $ vpp_dpdk_lib_dir =<PATH_TO_DESTDIR_NAME_FROM_ABOVE>/lib
<PATH_TO_DESTDIR_NAME_FROM_ABOVE> is whatever the path used when compiling DPDK above. These paths have to be absolute path in order for it to work.
@@ -108,34 +108,34 @@
.. code-block:: console
- vpp_uses_dpdk_mlx4_pmd = yes
+ vpp_uses_dpdk_mlx4_pmd = yes
.. code-block:: console
- $ make build
- $ cd build-root/
- $ make V=0 PLATFORM=vpp TAG=vpp install-deb
- $ sudo dpkg -i *.deb
+ $ make build
+ $ cd build-root/
+ $ make V=0 PLATFORM=vpp TAG=vpp install-deb
+ $ sudo dpkg -i *.deb
Finally, we modify the startup.conf file:
.. code-block:: console
- $ cd /etc/vpp
- $ sudo nano startup.conf
+ $ cd /etc/vpp
+ $ sudo nano startup.conf
Inside the DPDK block, the following commands:
.. code-block:: console
- ## Whitelist specific interface by specifying PCI address
- dev 000X:00:0X.0
- dev 000X:00:0X.0
-
- # Running failsafe
- vdev net_vdev_netvsc0,iface=eth1
- vdev net_vdev_netvsc1,iface=eth2
+ ## Whitelist specific interface by specifying PCI address
+ dev 000X:00:0X.0
+ dev 000X:00:0X.0
+
+ # Running failsafe
+ vdev net_vdev_netvsc0,iface=eth1
+ vdev net_vdev_netvsc1,iface=eth2
*Please refer to Azure DPDK document to pick the right iface to use for failsafe vdev.*
diff --git a/docs/usecases/webapp.md b/docs/usecases/webapp.md
deleted file mode 100644
index 3b1a3b7..0000000
--- a/docs/usecases/webapp.md
+++ /dev/null
@@ -1,274 +0,0 @@
-Building VPP web applications
-=============================
-
-Vpp includes a versatile http/https "static" server plugin. We quote
-the word static in the previous sentence because the server is easily
-extended. This note describes how to build a Hugo site which includes
-both monitoring and control functions.
-
-Let's assume that we have a vpp data-plane plugin which needs a
-monitoring and control web application. Here's how to build one.
-
-Step 1: Add URL handlers
-------------------------
-
-Individual URL handlers are pretty straightforward. You can
-return just about anything you like, but as we work through
-the example you'll see why returning data in .json format
-tends to work out pretty well.
-
-```
- static int
- handle_get_status (http_builtin_method_type_t reqtype,
- u8 * request, http_session_t * hs)
- {
- my_main_t *mm = &my_main;
- u8 *s = 0;
-
- /* Construct a .json reply */
- s = format (s, "{\"status\": {");
- s = format (s, " \"thing1\": \"%s\",", mm->thing1_value_string);
- s = format (s, " \"thing2\": \"%s\",", mm->thing2_value_string);
- /* ... etc ... */
- s = format (s, " \"lastthing\": \"%s\"", mm->last_value_string);
- s = format (s, "}}");
-
- /* And tell the static server plugin how to send the results */
- hs->data = s;
- hs->data_offset = 0;
- hs->cache_pool_index = ~0;
- hs->free_data = 1; /* free s when done with it, in the framework */
- return 0;
- }
-```
-
-Words to the Wise: Chrome has a very nice set of debugging
-tools. Select "More Tools -> Developer Tools". Right-hand sidebar
-appears with html source code, a javascript debugger, network results
-including .json objects, and so on.
-
-Note: .json object format is **intolerant** of both missing and extra
-commas, missing and extra curly-braces. It's easy to waste a
-considerable amount of time debugging .json bugs.
-
-Step 2: Register URL handlers with the server
----------------------------------------------
-
-Call http_static_server_register_builtin_handler() as shown. It's
-likely but not guaranteed that the static server plugin will be
-available.
-
-
-```
- int
- plugin_url_init (vlib_main_t * vm)
- {
- void (*fp) (void *, char *, int);
-
- /* Look up the builtin URL registration handler */
- fp = vlib_get_plugin_symbol ("http_static_plugin.so",
- "http_static_server_register_builtin_handler");
-
- if (fp == 0)
- {
- clib_warning ("http_static_plugin.so not loaded...");
- return -1;
- }
-
- (*fp) (handle_get_status, "status.json", HTTP_BUILTIN_METHOD_GET);
- (*fp) (handle_get_run, "run.json", HTTP_BUILTIN_METHOD_GET);
- (*fp) (handle_get_reset, "reset.json", HTTP_BUILTIN_METHOD_GET);
- (*fp) (handle_get_stop, "stop.json", HTTP_BUILTIN_METHOD_GET);
- return 0;
- }
-```
-
-Make sure to start the http static server **before** calling
-plugin_url_init(...), or the registrations will disappear.
-
-Step 3: Install Hugo, pick a theme, and create a site
------------------------------------------------------
-
-Please refer to the Hugo documentation.
-
-See [the Hugo Quick Start
-Page](https://gohugo.io/getting-started/quick-start). Prebuilt binary
-artifacts for many different environments are available on
-[the Hugo release page](https://github.com/gohugoio/hugo/releases).
-
-To pick a theme, visit [the Hugo Theme
-site](https://themes.gohugo.io). Decide what you need your site to
-look like. Stay away from complex themes unless you're prepared to
-spend considerable time tweaking and tuning.
-
-The "Introduction" theme is a good choice for a simple site, YMMV.
-
-Step 4: Create a "rawhtml" shortcode
-------------------------------------
-
-Once you've initialized your new site, create the directory
-<site-root>/layouts/shortcodes. Create the file "rawhtml.html" in that
-directory, with the following contents:
-
-```
- <!-- raw html -->
- {{.Inner}}
-```
-This is a key trick which allows a static Hugo site to include
-javascript code.
-
-Step 5: create Hugo content which interacts with vpp
-----------------------------------------------------
-
-Now it's time to do some web front-end coding in javascript. Of
-course, you can create static text, images, etc. as described in the
-Hugo documentation. Nothing changes in that respect.
-
-To include dynamically-generated data in your Hugo pages, splat down
-some <div> HTML tags, and define a few buttons:
-
-```
- {{< rawhtml >}}
- <div id="Thing1"></div>
- <div id="Thing2"></div>
- <div id="Lastthing"></div>
- <input type="button" value="Run" onclick="runButtonClick()">
- <input type="button" value="Reset" onclick="resetButtonClick()">
- <input type="button" value="Stop" onclick="stopButtonClick()">
- <div id="Message"></div>
- {{< /rawhtml >}}
-```
-
-Time for some javascript code to interact with vpp:
-
- {{< rawhtml >}}
- <script>
- async function getStatusJson() {
- pump_url = location.href + "status.json";
- const json = await fetch(pump_url, {
- method: 'GET',
- mode: 'no-cors',
- cache: 'no-cache',
- headers: {
- 'Content-Type': 'application/json',
- },
- })
- .then((response) => response.json())
- .catch(function(error) {
- console.log(error);
- });
-
- return json.status;
- };
-
- async function sendButton(which) {
- my_url = location.href + which + ".json";
- const json = await fetch(my_url, {
- method: 'GET',
- mode: 'no-cors',
- cache: 'no-cache',
- headers: {
- 'Content-Type': 'application/json',
- },
- })
- .then((response) => response.json())
- .catch(function(error) {
- console.log(error);
- });
- return json.message;
- };
-
- async function getStatus() {
- const status = await getStatusJson();
-
- document.getElementById("Thing1").innerHTML = status.thing1;
- document.getElementById("Thing2").innerHTML = status.thing2;
- document.getElementById("Lastthing").innerHTML = status.lastthing;
- };
-
- async function runButtonClick() {
- const json = await sendButton("run");
- document.getElementById("Message").innerHTML = json.Message;
- }
-
- async function resetButtonClick() {
- const json = await sendButton("reset");
- document.getElementById("Message").innerHTML = json.Message;
- }
- async function stopButtonClick() {
- const json = await sendButton("stop");
- document.getElementById("Message").innerHTML = json.Message;
- }
-
- getStatus();
-
- </script>
- {{< /rawhtml >}}
-
-At this level, javascript coding is pretty simple. Unless you know
-exactly what you're doing, please follow the async function / await
-pattern shown above.
-
-Step 6: compile the website
----------------------------
-
-At the top of the website workspace, simply type "hugo". The compiled
-website lands in the "public" subdirectory.
-
-You can use the Hugo static server - with suitable stub javascript
-code - to see what your site will eventually look like. To start the
-hugo static server, type "hugo server". Browse to
-"http://localhost:1313".
-
-Step 7: configure vpp
----------------------
-
-In terms of command-line args: you may wish to use poll-sleep-usec 100
-to keep the load average low. Totally appropriate if vpp won't be
-processing a lot of packets or handling high-rate http/https traffic.
-
-```
- unix {
- ...
- poll-sleep-usec 100
- startup-config ... see below ...
- ...
- }
-```
-
-If you wish to provide an https site, configure tls. The simplest tls
-configuration uses a built-in test certificate - which will annoy
-Chrome / Firefox - but it's sufficient for testing:
-
-```
- tls {
- use-test-cert-in-ca
- }
-```
-
-
-
-
-### vpp startup configuration
-
-Enable the vpp static server by way of the startup config mentioned above:
-
-```
- http static server www-root /myhugosite/public uri tcp://0.0.0.0/2345 cache-size 5m fifo-size 8192
-```
-
-The www-root must be specified, and must correctly name the compiled
-hugo site root. If your Hugo site is located at /myhugosite, specify
-"www-root /myhugosite/public" in the "http static server" stanza. The
-uri shown above binds to TCP port 2345.
-
-If you're using https, use a uri like "tls://0.0.0.0/443" instead of
-the uri shown above.
-
-You may want to add a Linux host interface to view the full-up site locally:
-
-```
- create tap host-if-name lstack host-ip4-addr 192.168.10.2/24
- set int ip address tap0 192.168.10.1/24
- set int state tap0 up
-```
diff --git a/docs/usecases/webapp.rst b/docs/usecases/webapp.rst
new file mode 100644
index 0000000..f76fd5b
--- /dev/null
+++ b/docs/usecases/webapp.rst
@@ -0,0 +1,279 @@
+Web applications with VPP
+=========================
+
+Vpp includes a versatile http/https “static” server plugin. We quote the
+word static in the previous sentence because the server is easily
+extended. This note describes how to build a Hugo site which includes
+both monitoring and control functions.
+
+Let’s assume that we have a vpp data-plane plugin which needs a
+monitoring and control web application. Here’s how to build one.
+
+Step 1: Add URL handlers
+------------------------
+
+Individual URL handlers are pretty straightforward. You can return just
+about anything you like, but as we work through the example you’ll see
+why returning data in .json format tends to work out pretty well.
+
+::
+
+ static int
+ handle_get_status (http_builtin_method_type_t reqtype,
+ u8 * request, http_session_t * hs)
+ {
+ my_main_t *mm = &my_main;
+ u8 *s = 0;
+
+ /* Construct a .json reply */
+ s = format (s, "{\"status\": {");
+ s = format (s, " \"thing1\": \"%s\",", mm->thing1_value_string);
+ s = format (s, " \"thing2\": \"%s\",", mm->thing2_value_string);
+ /* ... etc ... */
+ s = format (s, " \"lastthing\": \"%s\"", mm->last_value_string);
+ s = format (s, "}}");
+
+ /* And tell the static server plugin how to send the results */
+ hs->data = s;
+ hs->data_offset = 0;
+ hs->cache_pool_index = ~0;
+ hs->free_data = 1; /* free s when done with it, in the framework */
+ return 0;
+ }
+
+Words to the Wise: Chrome has a very nice set of debugging tools. Select
+“More Tools -> Developer Tools”. Right-hand sidebar appears with html
+source code, a javascript debugger, network results including .json
+objects, and so on.
+
+Note: .json object format is **intolerant** of both missing and extra
+commas, missing and extra curly-braces. It’s easy to waste a
+considerable amount of time debugging .json bugs.
+
+Step 2: Register URL handlers with the server
+---------------------------------------------
+
+Call http_static_server_register_builtin_handler() as shown. It’s likely
+but not guaranteed that the static server plugin will be available.
+
+::
+
+ int
+ plugin_url_init (vlib_main_t * vm)
+ {
+ void (*fp) (void *, char *, int);
+
+ /* Look up the builtin URL registration handler */
+ fp = vlib_get_plugin_symbol ("http_static_plugin.so",
+ "http_static_server_register_builtin_handler");
+
+ if (fp == 0)
+ {
+ clib_warning ("http_static_plugin.so not loaded...");
+ return -1;
+ }
+
+ (*fp) (handle_get_status, "status.json", HTTP_BUILTIN_METHOD_GET);
+ (*fp) (handle_get_run, "run.json", HTTP_BUILTIN_METHOD_GET);
+ (*fp) (handle_get_reset, "reset.json", HTTP_BUILTIN_METHOD_GET);
+ (*fp) (handle_get_stop, "stop.json", HTTP_BUILTIN_METHOD_GET);
+ return 0;
+ }
+
+Make sure to start the http static server **before** calling
+plugin_url_init(…), or the registrations will disappear.
+
+Step 3: Install Hugo, pick a theme, and create a site
+-----------------------------------------------------
+
+Please refer to the Hugo documentation.
+
+See `the Hugo Quick Start
+Page <https://gohugo.io/getting-started/quick-start>`__. Prebuilt binary
+artifacts for many different environments are available on `the Hugo
+release page <https://github.com/gohugoio/hugo/releases>`__.
+
+To pick a theme, visit `the Hugo Theme
+site <https://themes.gohugo.io>`__. Decide what you need your site to
+look like. Stay away from complex themes unless you’re prepared to spend
+considerable time tweaking and tuning.
+
+The “Introduction” theme is a good choice for a simple site, YMMV.
+
+Step 4: Create a “rawhtml” shortcode
+------------------------------------
+
+Once you’ve initialized your new site, create the directory
+/layouts/shortcodes. Create the file “rawhtml.html” in that directory,
+with the following contents:
+
+::
+
+ <!-- raw html -->
+ {{.Inner}}
+
+This is a key trick which allows a static Hugo site to include
+javascript code.
+
+Step 5: create Hugo content which interacts with vpp
+----------------------------------------------------
+
+Now it’s time to do some web front-end coding in javascript. Of course,
+you can create static text, images, etc. as described in the Hugo
+documentation. Nothing changes in that respect.
+
+To include dynamically-generated data in your Hugo pages, splat down
+some
+
+.. raw:: html
+
+ <div>
+
+HTML tags, and define a few buttons:
+
+::
+
+ {{< rawhtml >}}
+ <div id="Thing1"></div>
+ <div id="Thing2"></div>
+ <div id="Lastthing"></div>
+ <input type="button" value="Run" onclick="runButtonClick()">
+ <input type="button" value="Reset" onclick="resetButtonClick()">
+ <input type="button" value="Stop" onclick="stopButtonClick()">
+ <div id="Message"></div>
+ {{< /rawhtml >}}
+
+Time for some javascript code to interact with vpp:
+
+::
+
+ {{< rawhtml >}}
+ <script>
+ async function getStatusJson() {
+ pump_url = location.href + "status.json";
+ const json = await fetch(pump_url, {
+ method: 'GET',
+ mode: 'no-cors',
+ cache: 'no-cache',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ })
+ .then((response) => response.json())
+ .catch(function(error) {
+ console.log(error);
+ });
+
+ return json.status;
+ };
+
+ async function sendButton(which) {
+ my_url = location.href + which + ".json";
+ const json = await fetch(my_url, {
+ method: 'GET',
+ mode: 'no-cors',
+ cache: 'no-cache',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ })
+ .then((response) => response.json())
+ .catch(function(error) {
+ console.log(error);
+ });
+ return json.message;
+ };
+
+ async function getStatus() {
+ const status = await getStatusJson();
+
+ document.getElementById("Thing1").innerHTML = status.thing1;
+ document.getElementById("Thing2").innerHTML = status.thing2;
+ document.getElementById("Lastthing").innerHTML = status.lastthing;
+ };
+
+ async function runButtonClick() {
+ const json = await sendButton("run");
+ document.getElementById("Message").innerHTML = json.Message;
+ }
+
+ async function resetButtonClick() {
+ const json = await sendButton("reset");
+ document.getElementById("Message").innerHTML = json.Message;
+ }
+ async function stopButtonClick() {
+ const json = await sendButton("stop");
+ document.getElementById("Message").innerHTML = json.Message;
+ }
+
+ getStatus();
+
+ </script>
+ {{< /rawhtml >}}
+
+At this level, javascript coding is pretty simple. Unless you know
+exactly what you’re doing, please follow the async function / await
+pattern shown above.
+
+Step 6: compile the website
+---------------------------
+
+At the top of the website workspace, simply type “hugo”. The compiled
+website lands in the “public” subdirectory.
+
+You can use the Hugo static server - with suitable stub javascript code
+- to see what your site will eventually look like. To start the hugo
+static server, type “hugo server”. Browse to “http://localhost:1313”.
+
+Step 7: configure vpp
+---------------------
+
+In terms of command-line args: you may wish to use poll-sleep-usec 100
+to keep the load average low. Totally appropriate if vpp won’t be
+processing a lot of packets or handling high-rate http/https traffic.
+
+::
+
+ unix {
+ ...
+ poll-sleep-usec 100
+ startup-config ... see below ...
+ ...
+ }
+
+If you wish to provide an https site, configure tls. The simplest tls
+configuration uses a built-in test certificate - which will annoy Chrome
+/ Firefox - but it’s sufficient for testing:
+
+::
+
+ tls {
+ use-test-cert-in-ca
+ }
+
+vpp startup configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Enable the vpp static server by way of the startup config mentioned
+above:
+
+::
+
+ http static server www-root /myhugosite/public uri tcp://0.0.0.0/2345 cache-size 5m fifo-size 8192
+
+The www-root must be specified, and must correctly name the compiled
+hugo site root. If your Hugo site is located at /myhugosite, specify
+“www-root /myhugosite/public” in the “http static server” stanza. The
+uri shown above binds to TCP port 2345.
+
+If you’re using https, use a uri like “tls://0.0.0.0/443” instead of the
+uri shown above.
+
+You may want to add a Linux host interface to view the full-up site
+locally:
+
+::
+
+ create tap host-if-name lstack host-ip4-addr 192.168.10.2/24
+ set int ip address tap0 192.168.10.1/24
+ set int state tap0 up
diff --git a/docs/whatisvpp/index.rst b/docs/whatisvpp/index.rst
deleted file mode 100644
index 464119c..0000000
--- a/docs/whatisvpp/index.rst
+++ /dev/null
@@ -1,37 +0,0 @@
-.. _whatisvpp:
-
-=================================
-The Vector Packet Processor (VPP)
-=================================
-
-This section describes some of the core concepts and features of FD.io VPP.
-
-To start with FD.io VPP uses a technique called Vector Packet Processing.
-This gives FD.io VPP a siginficant performance improvement over packet
-processing applications that use scalar processing.
-
-Also, At the heart of Fd.io VPP's modular design is a 'Packet Processing Graph'.
-This makes FD.io VPP scalable and easily extensible.
-
-The FD.io software also includes a feature rich network stack. This includes
-a TCP host stack that utilizes VPP’s graph based forwarding model and vectorized
-packet processing.
-
-FD.io VPP is tested nightly for functionality and performance with the
-CSIT project.
-
-For more information on any of these features click on the links below or
-press next.
-
-.. toctree::
- :maxdepth: 1
-
- scalar-vs-vector-packet-processing.rst
- extensible.rst
- networkstack.rst
- hoststack.rst
- developer.rst
- supported.rst
- performance.rst
-
-Press next for more about Scalar/Vector Packet processing.
