travis: handle reduced max run time by splitting autobahn tests across two instances

2025-03-09 00:00:04 +01:00 · 2018-05-25 21:45:27 +08:00 · 2018-05-25 21:45:27 +08:00 · 2c2969cdac
commit 2c2969cdac
parent ad078faaaf
5 changed files with 131 additions and 58 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -5,6 +5,7 @@ env:
    - secure: "KhAdQ9ja+LBObWNQTYO7Df5J4DyOih6S+eerDMu8UPSO+CoWV2pWoQzbOfocjyOscGOwC+2PrrHDNZyGfqkCLDXg1BxynXPCFerHC1yc2IajvKpGXmAAygNIvp4KACDfGv/dkXrViqIzr/CdcNaU4vIMHSVb5xkeLi0W1dPnQOI="
  matrix:
    - LWS_METHOD=lwsws CMAKE_ARGS="-DLWS_WITH_LWSWS=ON -DLWS_WITHOUT_EXTENSIONS=0 -DLWS_WITH_HTTP2=1 -DLWS_WITH_ACME=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG"
+    - LWS_METHOD=lwsws2 CMAKE_ARGS="-DLWS_WITH_LWSWS=ON -DLWS_WITHOUT_EXTENSIONS=0 -DLWS_WITH_HTTP2=1 -DLWS_WITH_ACME=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG"
    - LWS_METHOD=default CMAKE_ARGS="-DLWS_WITH_MINIMAL_EXAMPLES=1"
    - LWS_METHOD=mbedtls CMAKE_ARGS="-DLWS_WITH_MBEDTLS=1 -DLWS_WITH_HTTP2=1 -DLWS_WITH_LWSWS=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG"
    - LWS_METHOD=noserver CMAKE_ARGS="-DLWS_WITHOUT_SERVER=ON -DLWS_WITH_MINIMAL_EXAMPLES=1"
--- a/scripts/autobahn-test-client.sh
+++ b/scripts/autobahn-test-client.sh
@ -142,49 +142,6 @@ else
 	echo
 fi

-# 2) lws-as-server tests
-
-echo
-echo "----------------------------------------------"
-echo "-------   tests: autobahn as server"
-echo
-
-$SERV -p 9001 -d7 &
-wstest -m fuzzingclient
-R=$?
-echo "Autobahn client exit $R"
-
-killall lws-minimal-ws-server-echo
-sleep 1s
-
-# repeat the client results
-
-R=`cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
-echo -n "AUTOBAHN SERVER / LWS CLIENT: Total tests: " `cat /tmp/ji | wc -l` " : "
-if [ "$R" == "0" ] ;then
-	echo "All pass"
-else
-	RESULT=1
-	echo -n "$R FAIL : "
-	cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
-	echo
-fi
-
-# and then the server results
-
-cat reports/servers/index.json | tr '\n' '!' | sed "s|\},\!|\n|g" | tr '!' ' ' | tr -s ' ' > /tmp/jis
-R=`cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
-
-echo -n "AUTOBAHN CLIENT / LWS SERVER: Total tests: " `cat /tmp/jis | wc -l` " : "
-if [ "$R" == "0" ] ;then
-	echo "All pass"
-else
-	RESULT=$(( $RESULT + 2 ))
-	echo -n "$R FAIL : "
-	cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
-	echo
-fi
-
 echo $RESULT
 exit $RESULT

--- a/scripts/autobahn-test-server.sh
+++ b/scripts/autobahn-test-server.sh
@ -0,0 +1,107 @@
+#!/bin/bash
+#
+# Requires pip install autobahntestsuite
+#
+# you should run this from ./build, after building with
+# cmake .. -DLWS_WITH_MINIMAL_EXAMPLES=1
+#
+# It will use the minimal echo client and server to run
+# autobahn ws tests as both client and server.
+
+set -u
+
+PARALLEL=8
+N=1
+OS=`uname`
+
+CLIE=bin/lws-minimal-ws-client-echo
+SERV=bin/lws-minimal-ws-server-echo
+
+RESULT=0
+
+which wstest 2>/dev/null
+if [ $? -ne 0 ]; then
+	echo "wstest is not installed"
+	exit 8
+fi
+
+killall wstest 2>/dev/null
+
+#
+# 2.10 / 2.11:      There is no requirement to handle multiple PING / PONG
+#                   in flight in RFC6455.  lws doesn't waste memory on it
+#                   since it is useless.
+#
+# 12.3.1 / 12.3.2
+# 12.4.* / 12.5.*:  Autobahn has been broken for these tests since Aug 2017
+#                   https://github.com/crossbario/autobahn-testsuite/issues/71
+
+
+cat << EOF >fuzzingserver.json
+{
+   "url": "ws://127.0.0.1:9001",
+   "outdir": "./reports/clients",
+   "cases": ["*"],
+   "exclude-cases": [ "2.10", "2.11", "12.3.1", "12.3.2", "12.4.*", "12.5.*"],
+   "exclude-agent-cases": {}
+}
+EOF
+
+cat << EOF >fuzzingclient.json
+{ 
+   "outdir": "./reports/servers",
+   "servers": [
+      {
+         "url": "ws://127.0.0.1:9001"
+      }
+   ],
+   "cases": ["*"],
+   "exclude-cases": ["2.10", "2.11", "12.3.1", "12.3.2", "12.4.*", "12.5.*" ],
+   "exclude-agent-cases": {}
+}
+EOF
+
+echo
+echo "----------------------------------------------"
+echo "-------   tests: autobahn as server"
+echo
+
+$SERV -p 9001 -d7 &
+wstest -m fuzzingclient
+R=$?
+echo "Autobahn client exit $R"
+
+killall lws-minimal-ws-server-echo
+sleep 1s
+
+# repeat the client results
+
+R=`cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
+echo -n "AUTOBAHN SERVER / LWS CLIENT: Total tests: " `cat /tmp/ji | wc -l` " : "
+if [ "$R" == "0" ] ;then
+	echo "All pass"
+else
+	RESULT=1
+	echo -n "$R FAIL : "
+	cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
+	echo
+fi
+
+# and then the server results
+
+cat reports/servers/index.json | tr '\n' '!' | sed "s|\},\!|\n|g" | tr '!' ' ' | tr -s ' ' > /tmp/jis
+R=`cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
+
+echo -n "AUTOBAHN CLIENT / LWS SERVER: Total tests: " `cat /tmp/jis | wc -l` " : "
+if [ "$R" == "0" ] ;then
+	echo "All pass"
+else
+	RESULT=$(( $RESULT + 2 ))
+	echo -n "$R FAIL : "
+	cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
+	echo
+fi
+
+echo $RESULT
+exit $RESULT
+
--- a/scripts/travis_control.sh
+++ b/scripts/travis_control.sh
@ -18,25 +18,33 @@ else
 			../scripts/h2spec.sh &&
 			../scripts/attack.sh &&
 			../scripts/h2load.sh &&
-			../scripts/autobahn-test.sh
+			../scripts/autobahn-test-client.sh
 		else
-			if [ "$LWS_METHOD" = "smp" ] ; then
+			if [ "$LWS_METHOD" = "lwsws2" ] ; then
 				cmake -DLWS_OPENSSL_LIBRARIES="/usr/local/lib/libssl.so;/usr/local/lib/libcrypto.so" \
 				      -DLWS_OPENSSL_INCLUDE_DIRS="/usr/local/include/openssl" $CMAKE_ARGS .. &&
 				cmake --build . &&
-				../scripts/h2load-smp.sh
+				sudo make install &&
+				../scripts/autobahn-test-server.sh
 			else
-				if [ "$LWS_METHOD" = "mbedtls" ] ; then
-					cmake $CMAKE_ARGS .. &&
+				if [ "$LWS_METHOD" = "smp" ] ; then
+					cmake -DLWS_OPENSSL_LIBRARIES="/usr/local/lib/libssl.so;/usr/local/lib/libcrypto.so" \
+					      -DLWS_OPENSSL_INCLUDE_DIRS="/usr/local/include/openssl" $CMAKE_ARGS .. &&
 					cmake --build . &&
-					sudo make install &&
-					../minimal-examples/selftests.sh &&
-					../scripts/h2spec.sh &&
-					../scripts/h2load.sh &&
-					../scripts/attack.sh
+					../scripts/h2load-smp.sh
 				else
-					cmake $CMAKE_ARGS .. &&
-					cmake --build .
+					if [ "$LWS_METHOD" = "mbedtls" ] ; then
+						cmake $CMAKE_ARGS .. &&
+						cmake --build . &&
+						sudo make install &&
+						../minimal-examples/selftests.sh &&
+						../scripts/h2spec.sh &&
+						../scripts/h2load.sh &&
+						../scripts/attack.sh
+					else
+						cmake $CMAKE_ARGS .. &&
+						cmake --build .
+					fi
 				fi
 			fi
 		fi
--- a/scripts/travis_install.sh
+++ b/scripts/travis_install.sh
@ -6,7 +6,7 @@ if [ "$TRAVIS_OS_NAME" == "linux" ];
 then
 	sudo apt-get update -qq

-	if [ "$LWS_METHOD" == "lwsws" ];
+	if [ "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ];
 	then
 		sudo apt-get install -y -qq realpath libjemalloc1 libev4 libuv-dev
 		sudo apt-get remove python-six
@ -46,7 +46,7 @@ then
 		sudo apt-get install -y -qq libev-dev;
 	fi

-	if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" ];
+	if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ];
 	then
 		sudo apt-get install -y -qq libuv-dev;
 #libuv1 libuv1-dev;
@ -62,7 +62,7 @@ then
 		brew install libev;
 	fi

-	if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" ];
+	if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ];
 	then
 		brew update;
 		brew install libuv;