From 6c82635445227d95c6c747fef4076c6674ffbe8b Mon Sep 17 00:00:00 2001 From: Andy Fingerhut Date: Sat, 24 Jun 2023 21:21:43 -0400 Subject: [PATCH] Update protobuf mininet p4testgen (#518) * Support negative-valued action parameters by converting them in the Python helper code into a positive value whose bit pattern is the 2's complement representation of the negative value. * Update versions of protobuf and grpc installed, install p4testgen Update protobuf from v3.6.1 to v3.18.1 Update grpc from v1.17.2 to v1.43.2 Both of those versions have been long tested in the p4-guide repo install scripts. Update BMv2, PI, p4c, and ptf to latest versions as of 2023-Jun-24 Enable compilation of p4testgen * Change method to install ptf I found that the former way that I am replacing did not seem to install it correctly. * Make clean.sh remove sources for protobuf and grpc Saves disk space for the image. protobuf and grpc source code is easily obtainable from public sources if the user wishes. --- vm-ubuntu-20.04/Vagrantfile | 1 - vm-ubuntu-20.04/clean.sh | 10 +-- ...ble-Wno-error-and-other-small-changes.diff | 78 ------------------- vm-ubuntu-20.04/root-dev-bootstrap.sh | 7 +- vm-ubuntu-20.04/user-dev-bootstrap.sh | 37 ++++----- 5 files changed, 21 insertions(+), 112 deletions(-) delete mode 100644 vm-ubuntu-20.04/patches/disable-Wno-error-and-other-small-changes.diff diff --git a/vm-ubuntu-20.04/Vagrantfile b/vm-ubuntu-20.04/Vagrantfile index 95cdaae..58ec797 100644 --- a/vm-ubuntu-20.04/Vagrantfile +++ b/vm-ubuntu-20.04/Vagrantfile @@ -16,7 +16,6 @@ Vagrant.configure(2) do |config| end dev.vm.provision "file", source: "py3localpath.py", destination: "/home/vagrant/py3localpath.py" dev.vm.provision "shell", inline: "chmod 755 /home/vagrant/py3localpath.py" - dev.vm.provision "file", source: "patches/disable-Wno-error-and-other-small-changes.diff", destination: "/home/vagrant/patches/disable-Wno-error-and-other-small-changes.diff" dev.vm.provision "file", source: "patches/behavioral-model-use-correct-libssl-pkg.patch", destination: "/home/vagrant/patches/behavioral-model-use-correct-libssl-pkg.patch" dev.vm.provision "file", source: "patches/mininet-patch-for-2023-jun.patch", destination: "/home/vagrant/patches/mininet-patch-for-2023-jun.patch" dev.vm.provision "file", source: "clean.sh", destination: "/home/vagrant/clean.sh" diff --git a/vm-ubuntu-20.04/clean.sh b/vm-ubuntu-20.04/clean.sh index 25e6de7..b339b0f 100755 --- a/vm-ubuntu-20.04/clean.sh +++ b/vm-ubuntu-20.04/clean.sh @@ -11,13 +11,9 @@ DF1_BEFORE=`df -h .` DF2_BEFORE=`df -BM .` -cd protobuf -make clean -cd .. - -cd grpc -make clean -cd .. +# Remove protobuf and grpc sources completely, since they are large +# and easily downloadable. +/bin/rm -fr protobuf grpc cd behavioral-model make clean diff --git a/vm-ubuntu-20.04/patches/disable-Wno-error-and-other-small-changes.diff b/vm-ubuntu-20.04/patches/disable-Wno-error-and-other-small-changes.diff deleted file mode 100644 index 3a6158e..0000000 --- a/vm-ubuntu-20.04/patches/disable-Wno-error-and-other-small-changes.diff +++ /dev/null @@ -1,78 +0,0 @@ -diff --git a/Makefile b/Makefile -index 736583fd93..472d91337d 100644 ---- a/Makefile -+++ b/Makefile -@@ -300,37 +300,37 @@ else - TMPOUT = `mktemp /tmp/test-out-XXXXXX` - endif - --CHECK_NO_CXX14_COMPAT_WORKS_CMD = $(CC) -std=c++11 -Werror -Wno-c++14-compat -o $(TMPOUT) -c test/build/no-c++14-compat.cc -+CHECK_NO_CXX14_COMPAT_WORKS_CMD = $(CC) -std=c++11 -Wno-c++14-compat -o $(TMPOUT) -c test/build/no-c++14-compat.cc - HAS_WORKING_NO_CXX14_COMPAT = $(shell $(CHECK_NO_CXX14_COMPAT_WORKS_CMD) 2> /dev/null && echo true || echo false) - ifeq ($(HAS_WORKING_NO_CXX14_COMPAT),true) - W_NO_CXX14_COMPAT=-Wno-c++14-compat - endif - --CHECK_SHADOW_WORKS_CMD = $(CC) -std=c99 -Werror -Wshadow -o $(TMPOUT) -c test/build/shadow.c -+CHECK_SHADOW_WORKS_CMD = $(CC) -std=c99 -Wshadow -o $(TMPOUT) -c test/build/shadow.c - HAS_WORKING_SHADOW = $(shell $(CHECK_SHADOW_WORKS_CMD) 2> /dev/null && echo true || echo false) - ifeq ($(HAS_WORKING_SHADOW),true) - W_SHADOW=-Wshadow - NO_W_SHADOW=-Wno-shadow - endif --CHECK_EXTRA_SEMI_WORKS_CMD = $(CC) -std=c99 -Werror -Wextra-semi -o $(TMPOUT) -c test/build/extra-semi.c -+CHECK_EXTRA_SEMI_WORKS_CMD = $(CC) -std=c99 -Wextra-semi -o $(TMPOUT) -c test/build/extra-semi.c - HAS_WORKING_EXTRA_SEMI = $(shell $(CHECK_EXTRA_SEMI_WORKS_CMD) 2> /dev/null && echo true || echo false) - ifeq ($(HAS_WORKING_EXTRA_SEMI),true) - W_EXTRA_SEMI=-Wextra-semi - NO_W_EXTRA_SEMI=-Wno-extra-semi - endif --CHECK_NO_SHIFT_NEGATIVE_VALUE_WORKS_CMD = $(CC) -std=c99 -Werror -Wno-shift-negative-value -o $(TMPOUT) -c test/build/no-shift-negative-value.c -+CHECK_NO_SHIFT_NEGATIVE_VALUE_WORKS_CMD = $(CC) -std=c99 -Wno-shift-negative-value -o $(TMPOUT) -c test/build/no-shift-negative-value.c - HAS_WORKING_NO_SHIFT_NEGATIVE_VALUE = $(shell $(CHECK_NO_SHIFT_NEGATIVE_VALUE_WORKS_CMD) 2> /dev/null && echo true || echo false) - ifeq ($(HAS_WORKING_NO_SHIFT_NEGATIVE_VALUE),true) - W_NO_SHIFT_NEGATIVE_VALUE=-Wno-shift-negative-value - NO_W_NO_SHIFT_NEGATIVE_VALUE=-Wshift-negative-value - endif --CHECK_NO_UNUSED_BUT_SET_VARIABLE_WORKS_CMD = $(CC) -std=c99 -Werror -Wno-unused-but-set-variable -o $(TMPOUT) -c test/build/no-unused-but-set-variable.c -+CHECK_NO_UNUSED_BUT_SET_VARIABLE_WORKS_CMD = $(CC) -std=c99 -Wno-unused-but-set-variable -o $(TMPOUT) -c test/build/no-unused-but-set-variable.c - HAS_WORKING_NO_UNUSED_BUT_SET_VARIABLE = $(shell $(CHECK_NO_UNUSED_BUT_SET_VARIABLE_WORKS_CMD) 2> /dev/null && echo true || echo false) - ifeq ($(HAS_WORKING_NO_UNUSED_BUT_SET_VARIABLE),true) - W_NO_UNUSED_BUT_SET_VARIABLE=-Wno-unused-but-set-variable - NO_W_NO_UNUSED_BUT_SET_VARIABLE=-Wunused-but-set-variable - endif --CHECK_NO_MAYBE_UNINITIALIZED_WORKS_CMD = $(CC) -std=c99 -Werror -Wno-maybe-uninitialized -o $(TMPOUT) -c test/build/no-maybe-uninitialized.c -+CHECK_NO_MAYBE_UNINITIALIZED_WORKS_CMD = $(CC) -std=c99 -Wno-maybe-uninitialized -o $(TMPOUT) -c test/build/no-maybe-uninitialized.c - HAS_WORKING_NO_MAYBE_UNINITIALIZED = $(shell $(CHECK_NO_MAYBE_UNINITIALIZED_WORKS_CMD) 2> /dev/null && echo true || echo false) - ifeq ($(HAS_WORKING_NO_MAYBE_UNINITIALIZED),true) - W_NO_MAYBE_UNINITIALIZED=-Wno-maybe-uninitialized -@@ -353,7 +353,7 @@ ifeq ($(SYSTEM),Darwin) - CXXFLAGS += -stdlib=libc++ - endif - CXXFLAGS += -Wnon-virtual-dtor --CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 -Wno-deprecated-declarations -Ithird_party/nanopb -DPB_FIELD_32BIT -+CPPFLAGS += -g -Wall -Wextra -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 -Wno-deprecated-declarations -Ithird_party/nanopb -DPB_FIELD_32BIT - COREFLAGS += -fno-rtti -fno-exceptions - LDFLAGS += -g - -@@ -10180,7 +10180,7 @@ PUBLIC_HEADERS_C += \ - LIBARES_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBARES_SRC)))) - - $(LIBARES_OBJS): CPPFLAGS += -Ithird_party/cares -Ithird_party/cares/cares -fvisibility=hidden -D_GNU_SOURCE $(if $(subst Darwin,,$(SYSTEM)),,-Ithird_party/cares/config_darwin) $(if $(subst FreeBSD,,$(SYSTEM)),,-Ithird_party/cares/config_freebsd) $(if $(subst Linux,,$(SYSTEM)),,-Ithird_party/cares/config_linux) $(if $(subst OpenBSD,,$(SYSTEM)),,-Ithird_party/cares/config_openbsd) -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX $(if $(subst MINGW32,,$(SYSTEM)),-DHAVE_CONFIG_H,) --$(LIBARES_OBJS): CFLAGS += -Wno-sign-conversion $(if $(subst Darwin,,$(SYSTEM)),,-Wno-shorten-64-to-32) $(if $(subst MINGW32,,$(SYSTEM)),-Wno-invalid-source-encoding,) -+$(LIBARES_OBJS): CFLAGS += -Wno-sign-conversion $(if $(subst Darwin,,$(SYSTEM)),,-Wno-shorten-64-to-32) $(if $(subst MINGW32,,$(SYSTEM)),) - - $(LIBDIR)/$(CONFIG)/libares.a: $(LIBARES_OBJS) - $(E) "[AR] Creating $@" -diff --git a/src/core/lib/gpr/log_linux.cc b/src/core/lib/gpr/log_linux.cc -index 561276f0c2..59bc0fee95 100644 ---- a/src/core/lib/gpr/log_linux.cc -+++ b/src/core/lib/gpr/log_linux.cc -@@ -40,7 +40,6 @@ - #include - #include - --static long gettid(void) { return syscall(__NR_gettid); } - - void gpr_log(const char* file, int line, gpr_log_severity severity, - const char* format, ...) { diff --git a/vm-ubuntu-20.04/root-dev-bootstrap.sh b/vm-ubuntu-20.04/root-dev-bootstrap.sh index 4d70427..7f08759 100755 --- a/vm-ubuntu-20.04/root-dev-bootstrap.sh +++ b/vm-ubuntu-20.04/root-dev-bootstrap.sh @@ -72,11 +72,6 @@ apt-get install -y --no-install-recommends --fix-missing\ xcscope-el \ xterm -# TBD: Should these packages be installed via apt-get ? They are in -# my install-p4dev-v4.sh script, but they might not be needed, either. - -# zlib1g-dev18 - # On a freshly installed Ubuntu 20.04.1 or 18.04.5 system, desktop # amd64 minimal installation, the Debian package python3-protobuf is # installed. This is depended upon by another package called @@ -115,7 +110,7 @@ apt-get install -y --no-install-recommends --fix-missing\ # At that point, attempting to import any of the 3 modules above gave NO error. sudo apt-get purge -y python3-protobuf || echo "Failed to remove python3-protobuf, probably because there was no such package installed" -sudo pip3 install protobuf==3.6.1 +sudo pip3 install protobuf==3.18.1 # Starting in 2019-Nov, Python3 version of Scapy is needed for `cd # p4c/build ; make check` to succeed. diff --git a/vm-ubuntu-20.04/user-dev-bootstrap.sh b/vm-ubuntu-20.04/user-dev-bootstrap.sh index 3f91652..209d13d 100755 --- a/vm-ubuntu-20.04/user-dev-bootstrap.sh +++ b/vm-ubuntu-20.04/user-dev-bootstrap.sh @@ -4,12 +4,12 @@ set -xe #Src -BMV2_COMMIT="f16d0de3486aa7fb2e1fe554aac7d237cc1adc33" # 2022-May-01 -PI_COMMIT="f547455a260b710706bef82afab4cb9937bac416" # 2022-May-01 -P4C_COMMIT="1471fdd22b683e1946b7730d83c877d94daba683" # 2022-May-01 -PTF_COMMIT="405513bcad2eae3092b0ac4ceb31e8dec5e32311" # 2022-May-01 -PROTOBUF_COMMIT="v3.6.1" -GRPC_COMMIT="tags/v1.17.2" +BMV2_COMMIT="1c44f6d79c4d99e9c68789973b5ea08a1580ae7d" # 2023-Jun-24 +PI_COMMIT="25e218b1b91cb083d51039118c9df469b842e8f9" # 2023-Jun-24 +P4C_COMMIT="0d3dcd626ddbfdf1f5618ff84ecb1e34346c7134" # 2023-Jun-24 +PTF_COMMIT="d2e2d8ad005a451ad11f9d21af50079a0552921a" # 2023-Jun-24 +PROTOBUF_COMMIT="v3.18.1" +GRPC_COMMIT="tags/v1.43.2" #Get the number of cores to speed up the compilation process NUM_CORES=`grep -c ^processor /proc/cpuinfo` @@ -96,11 +96,12 @@ move_usr_local_lib_python3_from_site_packages_to_dist_packages() { find /usr/lib /usr/local $HOME/.local | sort > $HOME/usr-local-1-before-protobuf.txt # --- Protobuf --- # -git clone https://github.com/google/protobuf.git +git clone https://github.com/protocolbuffers/protobuf cd protobuf git checkout ${PROTOBUF_COMMIT} +git submodule update --init --recursive ./autogen.sh -# install-p4dev-v4.sh script doesn't have --prefix=/usr option here. +# install-p4dev-v6.sh script doesn't have --prefix=/usr option here. ./configure --prefix=/usr make -j${NUM_CORES} sudo make install @@ -118,11 +119,9 @@ git clone https://github.com/grpc/grpc.git cd grpc git checkout ${GRPC_COMMIT} git submodule update --init --recursive -# Apply patch that seems to be necessary in order for grpc v1.17.2 to -# compile and install successfully on an Ubuntu 19.10 and later -# system. -PATCH_DIR="${HOME}/patches" -patch -p1 < "${PATCH_DIR}/disable-Wno-error-and-other-small-changes.diff" || echo "Errors while attempting to patch grpc, but continuing anyway ..." +mkdir -p cmake/build +cd cmake/build +cmake ../.. make -j${NUM_CORES} sudo make install # I believe the following 2 commands, adapted from similar commands in @@ -130,6 +129,7 @@ sudo make install # grpc. find /usr/lib /usr/local $HOME/.local | sort > $HOME/usr-local-2b-before-grpc-pip3.txt pip3 list | tee $HOME/pip3-list-2b-before-grpc-pip3.txt +cd ../.. sudo pip3 install -rrequirements.txt GRPC_PYTHON_BUILD_WITH_CYTHON=1 sudo pip3 install . sudo ldconfig @@ -151,15 +151,12 @@ cd PI git checkout ${PI_COMMIT} git submodule update --init --recursive ./autogen.sh -# install-p4dev-v4.sh adds more --without-* options to the configure +# install-p4dev-v6.sh adds more --without-* options to the configure # script here. I suppose without those, this script will cause # building PI code to include more features? ./configure --with-proto make -j${NUM_CORES} sudo make install -# install-p4dev-v4.sh at this point does these things, which might be -# useful in this script, too: -# Save about 0.25G of storage by cleaning up PI build make clean move_usr_local_lib_python3_from_site_packages_to_dist_packages sudo ldconfig @@ -177,7 +174,7 @@ git checkout ${BMV2_COMMIT} make -j${NUM_CORES} sudo make install-strip sudo ldconfig -# install-p4dev-v4.sh script does this here: +# install-p4dev-v6.sh script does this here: move_usr_local_lib_python3_from_site_packages_to_dist_packages cd .. @@ -190,7 +187,7 @@ git checkout ${P4C_COMMIT} git submodule update --init --recursive mkdir -p build cd build -cmake .. +cmake .. -DENABLE_TEST_TOOLS=ON # The command 'make -j${NUM_CORES}' works fine for the others, but # with 2 GB of RAM for the VM, there are parts of the p4c build where # running 2 simultaneous C++ compiler runs requires more than that @@ -207,7 +204,7 @@ find /usr/lib /usr/local $HOME/.local | sort > $HOME/usr-local-6-after-p4c.txt git clone https://github.com/p4lang/ptf cd ptf git checkout ${PTF_COMMIT} -sudo python3 setup.py install +sudo pip3 install . cd .. find /usr/lib /usr/local $HOME/.local | sort > $HOME/usr-local-8-after-ptf-install.txt