Update protobuf mininet p4testgen (#518)

* Support negative-valued action parameters
by converting them in the Python helper code into a positive value
whose bit pattern is the 2's complement representation of the negative
value.

* Update versions of protobuf and grpc installed, install p4testgen
Update protobuf from v3.6.1 to v3.18.1
Update grpc from v1.17.2 to v1.43.2

Both of those versions have been long tested in the p4-guide repo
install scripts.

Update BMv2, PI, p4c, and ptf to latest versions as of 2023-Jun-24

Enable compilation of p4testgen

* Change method to install ptf
I found that the former way that I am replacing did not seem to install
it correctly.

* Make clean.sh remove sources for protobuf and grpc
Saves disk space for the image.  protobuf and grpc source code is easily
obtainable from public sources if the user wishes.

vm-ubuntu-20.04/Vagrantfile

@@ -16,7 +16,6 @@ Vagrant.configure(2) do |config|
     end
     dev.vm.provision "file", source: "py3localpath.py", destination: "/home/vagrant/py3localpath.py"
     dev.vm.provision "shell", inline: "chmod 755 /home/vagrant/py3localpath.py"
-    dev.vm.provision "file", source: "patches/disable-Wno-error-and-other-small-changes.diff", destination: "/home/vagrant/patches/disable-Wno-error-and-other-small-changes.diff"
     dev.vm.provision "file", source: "patches/behavioral-model-use-correct-libssl-pkg.patch", destination: "/home/vagrant/patches/behavioral-model-use-correct-libssl-pkg.patch"
     dev.vm.provision "file", source: "patches/mininet-patch-for-2023-jun.patch", destination: "/home/vagrant/patches/mininet-patch-for-2023-jun.patch"
     dev.vm.provision "file", source: "clean.sh", destination: "/home/vagrant/clean.sh"

vm-ubuntu-20.04/clean.sh

@@ -11,13 +11,9 @@
 DF1_BEFORE=`df -h .`
 DF2_BEFORE=`df -BM .`
 
-cd protobuf
+# Remove protobuf and grpc sources completely, since they are large
-make clean
+# and easily downloadable.
-cd ..
+/bin/rm -fr protobuf grpc
-
-cd grpc
-make clean
-cd ..
 
 cd behavioral-model
 make clean

vm-ubuntu-20.04/patches/disable-Wno-error-and-other-small-changes.diff

@@ -1,78 +0,0 @@
-diff --git a/Makefile b/Makefile
-index 736583fd93..472d91337d 100644
---- a/Makefile
-+++ b/Makefile
-@@ -300,37 +300,37 @@ else
- TMPOUT = `mktemp /tmp/test-out-XXXXXX`
- endif
- 
--CHECK_NO_CXX14_COMPAT_WORKS_CMD = $(CC) -std=c++11 -Werror -Wno-c++14-compat -o $(TMPOUT) -c test/build/no-c++14-compat.cc
-+CHECK_NO_CXX14_COMPAT_WORKS_CMD = $(CC) -std=c++11 -Wno-c++14-compat -o $(TMPOUT) -c test/build/no-c++14-compat.cc
- HAS_WORKING_NO_CXX14_COMPAT = $(shell $(CHECK_NO_CXX14_COMPAT_WORKS_CMD) 2> /dev/null && echo true || echo false)
- ifeq ($(HAS_WORKING_NO_CXX14_COMPAT),true)
- W_NO_CXX14_COMPAT=-Wno-c++14-compat
- endif
- 
--CHECK_SHADOW_WORKS_CMD = $(CC) -std=c99 -Werror -Wshadow -o $(TMPOUT) -c test/build/shadow.c
-+CHECK_SHADOW_WORKS_CMD = $(CC) -std=c99 -Wshadow -o $(TMPOUT) -c test/build/shadow.c
- HAS_WORKING_SHADOW = $(shell $(CHECK_SHADOW_WORKS_CMD) 2> /dev/null && echo true || echo false)
- ifeq ($(HAS_WORKING_SHADOW),true)
- W_SHADOW=-Wshadow
- NO_W_SHADOW=-Wno-shadow
- endif
--CHECK_EXTRA_SEMI_WORKS_CMD = $(CC) -std=c99 -Werror -Wextra-semi -o $(TMPOUT) -c test/build/extra-semi.c
-+CHECK_EXTRA_SEMI_WORKS_CMD = $(CC) -std=c99 -Wextra-semi -o $(TMPOUT) -c test/build/extra-semi.c
- HAS_WORKING_EXTRA_SEMI = $(shell $(CHECK_EXTRA_SEMI_WORKS_CMD) 2> /dev/null && echo true || echo false)
- ifeq ($(HAS_WORKING_EXTRA_SEMI),true)
- W_EXTRA_SEMI=-Wextra-semi
- NO_W_EXTRA_SEMI=-Wno-extra-semi
- endif
--CHECK_NO_SHIFT_NEGATIVE_VALUE_WORKS_CMD = $(CC) -std=c99 -Werror -Wno-shift-negative-value -o $(TMPOUT) -c test/build/no-shift-negative-value.c
-+CHECK_NO_SHIFT_NEGATIVE_VALUE_WORKS_CMD = $(CC) -std=c99 -Wno-shift-negative-value -o $(TMPOUT) -c test/build/no-shift-negative-value.c
- HAS_WORKING_NO_SHIFT_NEGATIVE_VALUE = $(shell $(CHECK_NO_SHIFT_NEGATIVE_VALUE_WORKS_CMD) 2> /dev/null && echo true || echo false)
- ifeq ($(HAS_WORKING_NO_SHIFT_NEGATIVE_VALUE),true)
- W_NO_SHIFT_NEGATIVE_VALUE=-Wno-shift-negative-value
- NO_W_NO_SHIFT_NEGATIVE_VALUE=-Wshift-negative-value
- endif
--CHECK_NO_UNUSED_BUT_SET_VARIABLE_WORKS_CMD = $(CC) -std=c99 -Werror -Wno-unused-but-set-variable -o $(TMPOUT) -c test/build/no-unused-but-set-variable.c
-+CHECK_NO_UNUSED_BUT_SET_VARIABLE_WORKS_CMD = $(CC) -std=c99 -Wno-unused-but-set-variable -o $(TMPOUT) -c test/build/no-unused-but-set-variable.c
- HAS_WORKING_NO_UNUSED_BUT_SET_VARIABLE = $(shell $(CHECK_NO_UNUSED_BUT_SET_VARIABLE_WORKS_CMD) 2> /dev/null && echo true || echo false)
- ifeq ($(HAS_WORKING_NO_UNUSED_BUT_SET_VARIABLE),true)
- W_NO_UNUSED_BUT_SET_VARIABLE=-Wno-unused-but-set-variable
- NO_W_NO_UNUSED_BUT_SET_VARIABLE=-Wunused-but-set-variable
- endif
--CHECK_NO_MAYBE_UNINITIALIZED_WORKS_CMD = $(CC) -std=c99 -Werror -Wno-maybe-uninitialized -o $(TMPOUT) -c test/build/no-maybe-uninitialized.c
-+CHECK_NO_MAYBE_UNINITIALIZED_WORKS_CMD = $(CC) -std=c99 -Wno-maybe-uninitialized -o $(TMPOUT) -c test/build/no-maybe-uninitialized.c
- HAS_WORKING_NO_MAYBE_UNINITIALIZED = $(shell $(CHECK_NO_MAYBE_UNINITIALIZED_WORKS_CMD) 2> /dev/null && echo true || echo false)
- ifeq ($(HAS_WORKING_NO_MAYBE_UNINITIALIZED),true)
- W_NO_MAYBE_UNINITIALIZED=-Wno-maybe-uninitialized
-@@ -353,7 +353,7 @@ ifeq ($(SYSTEM),Darwin)
- CXXFLAGS += -stdlib=libc++
- endif
- CXXFLAGS += -Wnon-virtual-dtor
--CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 -Wno-deprecated-declarations -Ithird_party/nanopb -DPB_FIELD_32BIT
-+CPPFLAGS += -g -Wall -Wextra -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 -Wno-deprecated-declarations -Ithird_party/nanopb -DPB_FIELD_32BIT
- COREFLAGS += -fno-rtti -fno-exceptions
- LDFLAGS += -g
- 
-@@ -10180,7 +10180,7 @@ PUBLIC_HEADERS_C += \
- LIBARES_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBARES_SRC))))
- 
- $(LIBARES_OBJS): CPPFLAGS += -Ithird_party/cares -Ithird_party/cares/cares -fvisibility=hidden -D_GNU_SOURCE $(if $(subst Darwin,,$(SYSTEM)),,-Ithird_party/cares/config_darwin) $(if $(subst FreeBSD,,$(SYSTEM)),,-Ithird_party/cares/config_freebsd) $(if $(subst Linux,,$(SYSTEM)),,-Ithird_party/cares/config_linux) $(if $(subst OpenBSD,,$(SYSTEM)),,-Ithird_party/cares/config_openbsd) -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX $(if $(subst MINGW32,,$(SYSTEM)),-DHAVE_CONFIG_H,)
--$(LIBARES_OBJS): CFLAGS += -Wno-sign-conversion $(if $(subst Darwin,,$(SYSTEM)),,-Wno-shorten-64-to-32) $(if $(subst MINGW32,,$(SYSTEM)),-Wno-invalid-source-encoding,)
-+$(LIBARES_OBJS): CFLAGS += -Wno-sign-conversion $(if $(subst Darwin,,$(SYSTEM)),,-Wno-shorten-64-to-32) $(if $(subst MINGW32,,$(SYSTEM)),)
- 
- $(LIBDIR)/$(CONFIG)/libares.a:  $(LIBARES_OBJS) 
- 	$(E) "[AR]      Creating $@"
-diff --git a/src/core/lib/gpr/log_linux.cc b/src/core/lib/gpr/log_linux.cc
-index 561276f0c2..59bc0fee95 100644
---- a/src/core/lib/gpr/log_linux.cc
-+++ b/src/core/lib/gpr/log_linux.cc
-@@ -40,7 +40,6 @@
- #include <time.h>
- #include <unistd.h>
- 
--static long gettid(void) { return syscall(__NR_gettid); }
- 
- void gpr_log(const char* file, int line, gpr_log_severity severity,
-              const char* format, ...) {

vm-ubuntu-20.04/root-dev-bootstrap.sh

@@ -72,11 +72,6 @@ apt-get install -y --no-install-recommends --fix-missing\
   xcscope-el \
   xterm
 
-# TBD: Should these packages be installed via apt-get ?  They are in
-# my install-p4dev-v4.sh script, but they might not be needed, either.
-
-# zlib1g-dev18
-
 # On a freshly installed Ubuntu 20.04.1 or 18.04.5 system, desktop
 # amd64 minimal installation, the Debian package python3-protobuf is
 # installed.  This is depended upon by another package called
@@ -115,7 +110,7 @@ apt-get install -y --no-install-recommends --fix-missing\
 # At that point, attempting to import any of the 3 modules above gave NO error.
 
 sudo apt-get purge -y python3-protobuf || echo "Failed to remove python3-protobuf, probably because there was no such package installed"
-sudo pip3 install protobuf==3.6.1
+sudo pip3 install protobuf==3.18.1
 
 # Starting in 2019-Nov, Python3 version of Scapy is needed for `cd
 # p4c/build ; make check` to succeed.

vm-ubuntu-20.04/user-dev-bootstrap.sh

@@ -4,12 +4,12 @@
 set -xe
 
 #Src
-BMV2_COMMIT="f16d0de3486aa7fb2e1fe554aac7d237cc1adc33"  # 2022-May-01
+BMV2_COMMIT="1c44f6d79c4d99e9c68789973b5ea08a1580ae7d"  # 2023-Jun-24
-PI_COMMIT="f547455a260b710706bef82afab4cb9937bac416"    # 2022-May-01
+PI_COMMIT="25e218b1b91cb083d51039118c9df469b842e8f9"    # 2023-Jun-24
-P4C_COMMIT="1471fdd22b683e1946b7730d83c877d94daba683"   # 2022-May-01
+P4C_COMMIT="0d3dcd626ddbfdf1f5618ff84ecb1e34346c7134"   # 2023-Jun-24
-PTF_COMMIT="405513bcad2eae3092b0ac4ceb31e8dec5e32311"   # 2022-May-01
+PTF_COMMIT="d2e2d8ad005a451ad11f9d21af50079a0552921a"   # 2023-Jun-24
-PROTOBUF_COMMIT="v3.6.1"
+PROTOBUF_COMMIT="v3.18.1"
-GRPC_COMMIT="tags/v1.17.2"
+GRPC_COMMIT="tags/v1.43.2"
 
 #Get the number of cores to speed up the compilation process
 NUM_CORES=`grep -c ^processor /proc/cpuinfo`
@@ -96,11 +96,12 @@ move_usr_local_lib_python3_from_site_packages_to_dist_packages() {
 find /usr/lib /usr/local $HOME/.local | sort > $HOME/usr-local-1-before-protobuf.txt
 
 # --- Protobuf --- #
-git clone https://github.com/google/protobuf.git
+git clone https://github.com/protocolbuffers/protobuf
 cd protobuf
 git checkout ${PROTOBUF_COMMIT}
+git submodule update --init --recursive
 ./autogen.sh
-# install-p4dev-v4.sh script doesn't have --prefix=/usr option here.
+# install-p4dev-v6.sh script doesn't have --prefix=/usr option here.
 ./configure --prefix=/usr
 make -j${NUM_CORES}
 sudo make install
@@ -118,11 +119,9 @@ git clone https://github.com/grpc/grpc.git
 cd grpc
 git checkout ${GRPC_COMMIT}
 git submodule update --init --recursive
-# Apply patch that seems to be necessary in order for grpc v1.17.2 to
+mkdir -p cmake/build
-# compile and install successfully on an Ubuntu 19.10 and later
+cd cmake/build
-# system.
+cmake ../..
-PATCH_DIR="${HOME}/patches"
-patch -p1 < "${PATCH_DIR}/disable-Wno-error-and-other-small-changes.diff" || echo "Errors while attempting to patch grpc, but continuing anyway ..."
 make -j${NUM_CORES}
 sudo make install
 # I believe the following 2 commands, adapted from similar commands in
@@ -130,6 +129,7 @@ sudo make install
 # grpc.
 find /usr/lib /usr/local $HOME/.local | sort > $HOME/usr-local-2b-before-grpc-pip3.txt
 pip3 list | tee $HOME/pip3-list-2b-before-grpc-pip3.txt
+cd ../..
 sudo pip3 install -rrequirements.txt
 GRPC_PYTHON_BUILD_WITH_CYTHON=1 sudo pip3 install .
 sudo ldconfig
@@ -151,15 +151,12 @@ cd PI
 git checkout ${PI_COMMIT}
 git submodule update --init --recursive
 ./autogen.sh
-# install-p4dev-v4.sh adds more --without-* options to the configure
+# install-p4dev-v6.sh adds more --without-* options to the configure
 # script here.  I suppose without those, this script will cause
 # building PI code to include more features?
 ./configure --with-proto
 make -j${NUM_CORES}
 sudo make install
-# install-p4dev-v4.sh at this point does these things, which might be
-# useful in this script, too:
-# Save about 0.25G of storage by cleaning up PI build
 make clean
 move_usr_local_lib_python3_from_site_packages_to_dist_packages
 sudo ldconfig
@@ -177,7 +174,7 @@ git checkout ${BMV2_COMMIT}
 make -j${NUM_CORES}
 sudo make install-strip
 sudo ldconfig
-# install-p4dev-v4.sh script does this here:
+# install-p4dev-v6.sh script does this here:
 move_usr_local_lib_python3_from_site_packages_to_dist_packages
 cd ..
 
@@ -190,7 +187,7 @@ git checkout ${P4C_COMMIT}
 git submodule update --init --recursive
 mkdir -p build
 cd build
-cmake ..
+cmake .. -DENABLE_TEST_TOOLS=ON
 # The command 'make -j${NUM_CORES}' works fine for the others, but
 # with 2 GB of RAM for the VM, there are parts of the p4c build where
 # running 2 simultaneous C++ compiler runs requires more than that
@@ -207,7 +204,7 @@ find /usr/lib /usr/local $HOME/.local | sort > $HOME/usr-local-6-after-p4c.txt
 git clone https://github.com/p4lang/ptf
 cd ptf
 git checkout ${PTF_COMMIT}
-sudo python3 setup.py install
+sudo pip3 install .
 cd ..
 
 find /usr/lib /usr/local $HOME/.local | sort > $HOME/usr-local-8-after-ptf-install.txt