diff --git a/Doxyfile b/Doxyfile index c9c623918..704354980 100644 --- a/Doxyfile +++ b/Doxyfile @@ -1030,7 +1030,7 @@ HTML_OUTPUT = html # The default value is: .html. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_FILE_EXTENSION = .xhtml +HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a user-defined HTML header file for # each generated HTML page. If the tag is left blank doxygen will generate a diff --git a/README.txt b/README.txt new file mode 100644 index 000000000..87c68447e --- /dev/null +++ b/README.txt @@ -0,0 +1,17 @@ +This is the S2SS, a Simulator 2 Simulator Server +to connect realtime simulators of TCP/UDP/IP or raw 801.3 Ethernet. + +############################################################################### +# # +# Use Doxygen to build the documentation by calling "doxygen" in this dir. # +# If you are at the EONERC, you can use the prebuild documentation at: # +# https://134.130.169.2/doc/s2ss/ # +# # +############################################################################### + +Institute for Automation of Complex Power Systems (ACS) +EON Energy Research Center (EONERC) +RWTH University Aachen, Germany + +Author: Steffen Vogel +Date: February 2015 \ No newline at end of file diff --git a/clients/README.txt b/clients/README.txt new file mode 100644 index 000000000..619e5fd7c --- /dev/null +++ b/clients/README.txt @@ -0,0 +1,27 @@ +This directory contains code and projects to connect +various simulators and tools to the S2SS server. + +Author: Steffen Vogel +Date: Mid 2014 - End 2015 + +- ml50x_cpld + A slightly modified configuration of the CPLD on the ML507 board. + It is based on the reference design provided by Xilinx. + The modification redirects the PCIe reset signal directly to the Virtex 5. + +- ml507_gtfpga_pcie + A Xilinx ISE 12.4 project to directly access RTDS registers via PCIe memory. + This is a WIP and should allow improve the latency of by directly accessing RTDS + registers through the S2SS server which runs on the same machine. + +- ml507_ppc440_udp + A Xilinx XPS 12.4 project which allows access to the RTDS registers via S2SS's + UDP protocol over the ML507 Ethernet interface. + This is working! + +- opal + Contains the implementation of an asynchronous process block for RT-LAB. + This block allows exchanging messages with an S2SS server over UDP/TCP. + +- rscad + Some example RSCAD drafts to test the S2SS <-> RTDS interface. diff --git a/clients/opal/.project b/clients/opal_udp/.project similarity index 100% rename from clients/opal/.project rename to clients/opal_udp/.project diff --git a/clients/opal/.settings/com.opalrt.rtlab.ui.application.prefs b/clients/opal_udp/.settings/com.opalrt.rtlab.ui.application.prefs similarity index 100% rename from clients/opal/.settings/com.opalrt.rtlab.ui.application.prefs rename to clients/opal_udp/.settings/com.opalrt.rtlab.ui.application.prefs diff --git a/clients/opal/models/send_receive/include/config.h b/clients/opal_udp/models/send_receive/include/config.h similarity index 100% rename from clients/opal/models/send_receive/include/config.h rename to clients/opal_udp/models/send_receive/include/config.h diff --git a/clients/opal/models/send_receive/include/msg.h b/clients/opal_udp/models/send_receive/include/msg.h similarity index 100% rename from clients/opal/models/send_receive/include/msg.h rename to clients/opal_udp/models/send_receive/include/msg.h diff --git a/clients/opal/models/send_receive/include/msg_format.h b/clients/opal_udp/models/send_receive/include/msg_format.h similarity index 100% rename from clients/opal/models/send_receive/include/msg_format.h rename to clients/opal_udp/models/send_receive/include/msg_format.h diff --git a/clients/opal/models/send_receive/include/socket.h b/clients/opal_udp/models/send_receive/include/socket.h similarity index 100% rename from clients/opal/models/send_receive/include/socket.h rename to clients/opal_udp/models/send_receive/include/socket.h diff --git a/clients/opal/models/send_receive/include/utils.h b/clients/opal_udp/models/send_receive/include/utils.h similarity index 100% rename from clients/opal/models/send_receive/include/utils.h rename to clients/opal_udp/models/send_receive/include/utils.h diff --git a/clients/opal/models/send_receive/s2ss.mk b/clients/opal_udp/models/send_receive/s2ss.mk similarity index 100% rename from clients/opal/models/send_receive/s2ss.mk rename to clients/opal_udp/models/send_receive/s2ss.mk diff --git a/clients/opal/models/send_receive/send_receive.llm b/clients/opal_udp/models/send_receive/send_receive.llm similarity index 100% rename from clients/opal/models/send_receive/send_receive.llm rename to clients/opal_udp/models/send_receive/send_receive.llm diff --git a/clients/opal/models/send_receive/send_receive.mdl b/clients/opal_udp/models/send_receive/send_receive.mdl similarity index 100% rename from clients/opal/models/send_receive/send_receive.mdl rename to clients/opal_udp/models/send_receive/send_receive.mdl diff --git a/clients/opal/models/send_receive/src/msg.c b/clients/opal_udp/models/send_receive/src/msg.c similarity index 100% rename from clients/opal/models/send_receive/src/msg.c rename to clients/opal_udp/models/send_receive/src/msg.c diff --git a/clients/opal/models/send_receive/src/s2ss.c b/clients/opal_udp/models/send_receive/src/s2ss.c similarity index 91% rename from clients/opal/models/send_receive/src/s2ss.c rename to clients/opal_udp/models/send_receive/src/s2ss.c index c373ca5eb..41d4d3cd2 100644 --- a/clients/opal/models/send_receive/src/s2ss.c +++ b/clients/opal_udp/models/send_receive/src/s2ss.c @@ -50,9 +50,28 @@ #define ASYNC_SHMEM_SIZE atoi(argv[2]) #define PRINT_SHMEM_NAME argv[3] -#ifdef DEBUG // TODO: workaround +#ifdef _DEBUG // TODO: workaround +#define CPU_TICKS 3466948000 struct msg *msg_send = NULL; -#endif /* DEBUG */ + +void Tick(int sig, siginfo_t *si, void *ptr) +{ + Opal_GenAsyncParam_Ctrl *IconCtrlStruct; + unsigned long long CpuTime, CpuTimeStart; + double ModelTime; + + if (!msg_send) + return; + + IconCtrlStruct = (Opal_GenAsyncParam_Ctrl*) si->si_value.sival_ptr; + + OpalGetAsyncStartExecCpuTime(IconCtrlStruct, &CpuTimeStart); + OpalGetAsyncModelTime(IconCtrlStruct, &CpuTime, &ModelTime); + + OpalPrint("%s: CpuTime: %llu\tModelTime: %.3f\tSequence: %hu\tValue: %.2f\n", + PROGNAME, (CpuTime - CpuTimeStart) / CPU_TICKS, ModelTime, ntohs(msg_send->sequence), msg_send->data[0].f); +} +#endif /* _DEBUG */ static void *SendToIPPort(void *arg) { @@ -70,9 +89,9 @@ static void *SendToIPPort(void *arg) struct msg msg = MSG_INIT(0); int msg_size; -#ifdef DEBUG // TODO: workaround +#ifdef _DEBUG // TODO: workaround msg_send = &msg; -#endif /* DEBUG */ +#endif /* _DEBUG */ OpalPrint("%s: SendToIPPort thread started\n", PROGNAME); @@ -245,23 +264,6 @@ static void *RecvFromIPPort(void *arg) return NULL; } -void Tick(int sig, siginfo_t *si, void *ptr) -{ - Opal_GenAsyncParam_Ctrl *IconCtrlStruct; - unsigned long long CpuTime; - double ModelTime; - - if (!msg_send) - return; - - IconCtrlStruct = (Opal_GenAsyncParam_Ctrl*) si->si_value.sival_ptr; - - OpalGetAsyncModelTime(IconCtrlStruct, &CpuTime, &ModelTime) - - OpalPrint("%s: TICK! CpuTime: %llu\tModelTime: %f\tSequence: %hu\tValue: %f\n", - PROGNAME, CpuTime, ModelTime, msg_send->sequence, msg_send->data[0].f); -} - int main(int argc, char *argv[]) { int err; @@ -308,7 +310,7 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } -#ifdef DEBUG +#ifdef _DEBUG /* Setup signals */ struct sigaction sa_tick = { .sa_flags = SA_SIGINFO, @@ -316,13 +318,13 @@ int main(int argc, char *argv[]) }; sigemptyset(&sa_tick.sa_mask); - sigaction(SIG, &sa_tick, NULL); + sigaction(SIGUSR1, &sa_tick, NULL); /* Setup timer */ timer_t t; struct sigevent sev = { .sigev_notify = SIGEV_SIGNAL, - .sigev_signo = SIG, + .sigev_signo = SIGUSR1, .sigev_value.sival_ptr = &IconCtrlStruct }; @@ -333,7 +335,7 @@ int main(int argc, char *argv[]) timer_create(CLOCK_REALTIME, &sev, &t); timer_settime(t, 0, &its, NULL); -#endif /* DEBUG */ +#endif /* _DEBUG */ /* Start send/receive threads */ if ((pthread_create(&tid_send, NULL, SendToIPPort, NULL)) == -1) @@ -352,9 +354,9 @@ int main(int argc, char *argv[]) OpalCloseAsyncMem (ASYNC_SHMEM_SIZE, ASYNC_SHMEM_NAME); OpalSystemCtrl_UnRegister(PRINT_SHMEM_NAME); -#ifdef DEBUG +#ifdef _DEBUG timer_delete(t); -#endif /* DEBUG */ +#endif /* _DEBUG */ return 0; } diff --git a/clients/opal/models/send_receive/src/socket.c b/clients/opal_udp/models/send_receive/src/socket.c similarity index 100% rename from clients/opal/models/send_receive/src/socket.c rename to clients/opal_udp/models/send_receive/src/socket.c diff --git a/clients/opal/models/send_receive/src/utils.c b/clients/opal_udp/models/send_receive/src/utils.c similarity index 100% rename from clients/opal/models/send_receive/src/utils.c rename to clients/opal_udp/models/send_receive/src/utils.c diff --git a/clients/opal/s2ss_tests.llp b/clients/opal_udp/s2ss_tests.llp similarity index 100% rename from clients/opal/s2ss_tests.llp rename to clients/opal_udp/s2ss_tests.llp diff --git a/contrib/liveusb/chroot.sh b/contrib/liveusb/chroot.sh new file mode 100755 index 000000000..6e02c1bed --- /dev/null +++ b/contrib/liveusb/chroot.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +mount /dev/sdb1 /media/usb + +for part in dev sys proc; do + umount /media/usb/$part + mount -o bind /$part /media/usb/$part +done + +chroot /media/usb diff --git a/contrib/liveusb/etc/default/grub b/contrib/liveusb/etc/default/grub new file mode 100644 index 000000000..d3bd7ac6b --- /dev/null +++ b/contrib/liveusb/etc/default/grub @@ -0,0 +1,7 @@ +GRUB_TIMEOUT=5 +GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)" +GRUB_DEFAULT=1 +GRUB_DISABLE_SUBMENU=false +GRUB_TERMINAL_OUTPUT="console" +GRUB_CMDLINE_LINUX="isolcpus=6,7 selinux=0 audit=0" +GRUB_DISABLE_RECOVERY=true diff --git a/contrib/liveusb/etc/hostname b/contrib/liveusb/etc/hostname new file mode 100644 index 000000000..da598be9b --- /dev/null +++ b/contrib/liveusb/etc/hostname @@ -0,0 +1 @@ +unknown-s2ss diff --git a/contrib/liveusb/etc/hosts b/contrib/liveusb/etc/hosts new file mode 100644 index 000000000..05fc89de2 --- /dev/null +++ b/contrib/liveusb/etc/hosts @@ -0,0 +1,10 @@ +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +# Orchestrator + +# ACS hosts +134.130.169.31 acs-s2ss +134.130.169.32 acs-gtfpga +137.226.160.69 acs-opal +137.226.160.115 acs-workstation diff --git a/contrib/liveusb/etc/systemd/system/dhclient.service b/contrib/liveusb/etc/systemd/system/dhclient.service new file mode 100644 index 000000000..002f7ad7d --- /dev/null +++ b/contrib/liveusb/etc/systemd/system/dhclient.service @@ -0,0 +1,11 @@ +[Unit] +Description=dhclient on all interfaces +Wants=network.target +Before=network.target + +[Service] +Type=forking +ExecStart=/sbin/dhclient -4 + +[Install] +WantedBy=network.target diff --git a/contrib/liveusb/etc/systemd/system/mongoose.service b/contrib/liveusb/etc/systemd/system/mongoose.service new file mode 100644 index 000000000..2ff7070d7 --- /dev/null +++ b/contrib/liveusb/etc/systemd/system/mongoose.service @@ -0,0 +1,15 @@ +[Unit] +Description=The mongoose Web server +After=network.target + +[Service] +Type=simple +User=nobody +Group=nobody +Restart=always +ExecStart=/usr/bin/mongoose -p 80 -r /var/www/ +StandardOutput=syslog +SyslogIdentifier=mongoose + +[Install] +WantedBy=multi-user.target diff --git a/contrib/liveusb/etc/systemd/system/setup.service b/contrib/liveusb/etc/systemd/system/setup.service new file mode 100644 index 000000000..50206d3cc --- /dev/null +++ b/contrib/liveusb/etc/systemd/system/setup.service @@ -0,0 +1,17 @@ +[Unit] +Description=S2SS LiveUSB Image setup +Requires=dhclient.service network.service +After=dhclient.service + +[Service] +Type=simple +ExecStart=/s2ss/contrib/liveusb/setup.sh +RemainAfterExit=yes + +TimeoutSec=120 + +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/contrib/liveusb/etc/tuned/active_profile b/contrib/liveusb/etc/tuned/active_profile new file mode 100644 index 000000000..239b12e08 --- /dev/null +++ b/contrib/liveusb/etc/tuned/active_profile @@ -0,0 +1 @@ +latency-performance diff --git a/contrib/liveusb/prepare.sh b/contrib/liveusb/prepare.sh new file mode 100644 index 000000000..eb7a9a755 --- /dev/null +++ b/contrib/liveusb/prepare.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +rpm -Uvh http://ccrma.stanford.edu/planetccrma/mirror/fedora/linux/planetccrma/20/i386/planetccrma-repo-1.1-3.fc20.ccrma.noarch.rpm + +yum update + +yum install planetccrma-core + +source update_boot.sh diff --git a/contrib/liveusb/setup.sh b/contrib/liveusb/setup.sh new file mode 100755 index 000000000..cb42fa00a --- /dev/null +++ b/contrib/liveusb/setup.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -e + +RECIPIENTS="stvogel@eonerc.rwth-aachen.de,mstevic@eonerc.rwth-aachen.de" + +SERVER=tux.0l.de +USER=acs + +PORT=$(shuf -i 60000-65535 -n 1) + +IP=$(curl -s http://canihazip.com/s) + +# check if system has net connectivity. otherwise die... +ssh -q -o ConnectTimeout=2 $USER@$SERVER + +# setup SSH tunnel for mail notification +ssh -f -N -L 25:localhost:25 $USER@$SERVER +# setup SSH reverse tunnel for remote administration +ssh -f -N -R $PORT:localhost:22 $USER@$SERVER + +# send mail with notification about new node +sendmail "$RECIPIENTS" < +To: $RECIPIENTS + +There's a new host with the S2SS LiveUSB Image running: + +Reverse SSH tunnel port: $PORT +Internet IP: $IP +Hostname: $HOSTNAME + +Latency: +$(ping -qc 5 $SERVER) + +Traceroute: +$(traceroute $SERVER) + +Interfaces: +$(ip addr) + +Hardware: +$(lshw) + +EOF diff --git a/contrib/liveusb/update_boot.sh b/contrib/liveusb/update_boot.sh new file mode 100755 index 000000000..f0c87ca0e --- /dev/null +++ b/contrib/liveusb/update_boot.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# author: Christian Berendt + +set -x + +for kernel in $(find /boot/vmlinuz*); do + version=$(basename $kernel) + version=${version#*-} + if [ ! -e /boot/initramfs-$version.img ]; then + sudo /usr/bin/dracut /boot/initramfs-$version.img $version + fi +done + +for image in $(find /boot/initramfs*); do + version=${image%.img} + version=${version#*initramfs-} + if [ ! -e /boot/vmlinuz-$version ]; then + sudo rm $image + fi +done + +/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg + diff --git a/contrib/liveusb/yum-packages.txt b/contrib/liveusb/yum-packages.txt new file mode 100644 index 000000000..0a5117f19 --- /dev/null +++ b/contrib/liveusb/yum-packages.txt @@ -0,0 +1,56 @@ +authconfig-6.2.6-4.fc20.x86_64 +automake-1.13.4-6.fc20.noarch +bash-completion-2.1-3.fc20.noarch +bind-utils-9.9.4-18.P2.fc20.x86_64 +biosdevname-0.5.0-2.fc20.x86_64 +bzip2-1.0.6-9.fc20.x86_64 +dhclient-4.2.7-2.fc20.x86_64 +dosfstools-3.0.27-1.fc20.x86_64 +dracut-config-rescue-037-11.git20140402.fc20.x86_64 +e2fsprogs-1.42.12-3.fc20.x86_64 +efibootmgr-0.11.0-1.fc20.x86_64 +ftp-0.17-65.fc20.x86_64 +gcc-4.8.3-7.fc20.x86_64 +gdb-7.7.1-21.fc20.x86_64 +gdisk-0.8.10-2.fc20.x86_64 +git-svn-1.9.3-2.fc20.x86_64 +grub2-2.00-27.fc20.x86_64 +htop-1.0.3-3.fc20.x86_64 +iptables-services-1.4.19.1-1.fc20.x86_64 +kbd-1.15.5-12.fc20.x86_64 +kernel-modules-extra-3.18.9-100.fc20.x86_64 +kernel-rt-modules-extra-3.14.29-200.rt26.1.fc20.ccrma.x86_64 +libconfig-1.4.9-5.fc20.x86_64 +lshw-B.02.17-2.fc20.x86_64 +lzo-devel-2.08-1.fc20.x86_64 +mailx-12.5-11.fc20.x86_64 +man-db-2.6.5-6.fc20.x86_64 +minicom-2.6.2-4.fc20.x86_64 +nano-2.3.2-5.fc20.x86_64 +nmap-6.45-1.fc20.x86_64 +ntp-4.2.6p5-20.fc20.x86_64 +numactl-2.0.9-2.fc20.x86_64 +openssh-server-6.4p1-8.fc20.x86_64 +openssl-devel-1.0.1e-41.fc20.x86_64 +parted-3.1-13.fc20.x86_64 +passwd-0.79-2.fc20.x86_64 +patch-2.7.1-7.fc20.x86_64 +pciutils-devel-3.3.0-1.fc20.x86_64 +planetccrma-repo-1.1-3.fc20.ccrma.noarch +policycoreutils-2.2.5-4.fc20.x86_64 +psmisc-22.20-3.fc20.x86_64 +readline-devel-6.2-10.fc20.x86_64 +rootfiles-8.1-16.fc20.noarch +schedtool-1.3.0-9.fc20.x86_64 +screen-4.1.0-0.19.20120314git3c2946.fc20.x86_64 +setserial-2.17-34.fc20.x86_64 +socat-1.7.2.4-1.fc20.x86_64 +ssmtp-2.64-14.fc20.x86_64 +sudo-1.8.12-1.fc20.x86_64 +tar-1.26-31.fc20.x86_64 +tcpdump-4.5.1-3.fc20.x86_64 +texinfo-5.1-4.fc20.x86_64 +traceroute-2.0.20-1.fc20.x86_64 +tuned-2.4.1-3.fc20.noarch +wget-1.16.1-2.fc20.x86_64 +yum-utils-1.1.31-27.fc20.noarch diff --git a/contrib/tests.sh b/contrib/tests.sh new file mode 100755 index 000000000..fdb36faff --- /dev/null +++ b/contrib/tests.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -e + +# Configuration + +HOST=$(hostname -s) + +#LOCAL_IP=10.10.12.2 +LOCAL_IP=127.0.0.1 +LOCAL_PORT=12001 + +#REMOTE_IP=10.10.12.3 +REMOTE_IP=127.0.0.1 +REMOTE_PORT=12002 + +#LOCAL_DIR=/s2ss/server/ +#REMOTE_DIR=/s2ss/server/ +LOCAL_DIR=/home/stv0g/workspace/rwth/acs/s2ss/server +REMOTE_DIR=$LOCAL_DIR +######################### End of Configuration ################################ +# There's no need to change something below this line + +SSH="ssh -T $REMOTE_IP" + +TEST_CONFIG=" + affinity = 0x02; # Mask of cores the server should run on + priority = 50; # Scheduler priority for the server + debug = 10; # The level of verbosity for debug messages + stats = 3; # The interval in seconds fo path statistics + + nodes = { + remote = { + type = \"udp\"; + local = \"$LOCAL_IP:$LOCAL_PORT\"; + remote = \"$REMOTE_IP:$REMOTE_PORT\"; + } + }; +" + + +REMOTE_CONFIG=" + affinity = 0x02; # Mask of cores the server should run on + priority = 50; # Scheduler priority for the server + debug = 0; # The level of verbosity for debug messages + stats = 0; # The interval in seconds fo path statistics + + nodes = { + remote = { + type = \"udp\"; + local = \"$REMOTE_IP:$REMOTE_PORT\"; + remote = \"$LOCAL_IP:$LOCAL_PORT\"; + }, + }; + + paths = ({ + in = \"remote\"; + out = \"remote\"; + }); +" + +# Start remote server +$SSH -n "echo '$REMOTE_CONFIG' | sudo $REMOTE_DIR/server -" & +REMOTE_PID=$! +echo "Started remote server with pid: $REMOTE_PID" + +# Start tests +echo "$TEST_CONFIG" | $LOCAL_DIR/test - rtt -f 3 -c 100000 3>> test_output + +# Stop remote server +$SSH sudo kill $REMOTE_PID +echo "Stopped remote server with pid: $REMOTE_PID" \ No newline at end of file diff --git a/contrib/update_docs.sh b/contrib/update_docs.sh new file mode 100755 index 000000000..f517e1e5b --- /dev/null +++ b/contrib/update_docs.sh @@ -0,0 +1,12 @@ +#/bin/bash + +cd $( cd "$( dirname "$0" )/.." && pwd ) + +LASTREV=$(git rev-parse HEAD) +git pull --all +NEWREV=$(git rev-parse HEAD) + +if [ "$LASTREV" != "$NEWREV" ]; then + echo "There's a new version. Running doxygen" + doxygen +fi diff --git a/server/.gitignore b/server/.gitignore index f929d8dbc..5772b93b7 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -1,3 +1,5 @@ +logs/ + *.d *.o *~ diff --git a/server/Makefile b/server/Makefile index b71f16836..88f35adbb 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,40 +1,52 @@ TARGETS = server send random receive test -SRCS = server.c send.c receive.c random.c node.c path.c utils.c socket.c msg.c cfg.c if.c tc.c -# Default target: build everything -all: $(TARGETS) - -COMMON = socket.o if.o utils.o msg.o node.o cfg.o tc.o hooks.o - -# Dependencies for individual binaries -server: $(COMMON) path.o -send: $(COMMON) -receive: $(COMMON) -random: utils.o msg.o -test: $(COMMON) +# Common dependencies for all binaries +OBJS = socket.o if.o utils.o msg.o node.o cfg.o tc.o hooks.o list.o path.o hist.o log.o VPATH = src # Default debug level V ?= 2 -# Some details about the compiled version - # Compiler and linker flags -LDFLAGS = -pthread -lrt -lm -lconfig -CFLAGS = -std=c99 -Iinclude/ -MMD -Wall +LDLIBS = -pthread -lrt -lm -lconfig +CFLAGS = -std=gnu99 -Iinclude/ -MMD -Wall CFLAGS += -D_XOPEN_SOURCE=500 -D_GNU_SOURCE -DV=$(V) -CFLAGS += -D__GIT_REV__='"-$(shell git rev-parse --short HEAD)"' -# Conditional flags +# Add git commit hash +ifneq (,$(shell which git)) + CFLAGS += -D_GIT_REV='"$(shell git rev-parse --short HEAD)"' +endif + +# Conditional debug flags ifdef DEBUG - CFLAGS += -g + CFLAGS += -O0 -g else CFLAGS += -O3 endif +# Enable OPAL-RT Asynchronous Process support +OPALDIR = /usr/opalrt/common +#OPALDIR = ../opal +ifneq (,$(wildcard $(OPALDIR)/include_target/AsyncApi.h)) + CFLAGS += -m32 -DENABLE_OPAL_ASYNC -I$(OPALDIR)/include_target + LDFLAGS += -m32 + LDLIBS += $(addprefix $(OPALDIR)/lib/redhawk/, libOpalAsyncApiCore.a libOpalCore.a libOpalUtils.a libirc.a) + OBJS += opal.o +endif + .PHONY: all clean +# Default target: build everything +all: $(TARGETS) + +# Dependencies for individual binaries +server: server.o $(OBJS) +send: send.o $(OBJS) +receive: receive.o $(OBJS) +random: random.o $(OBJS) +test: test.o $(OBJS) + clean: $(RM) *~ *.o *.d $(RM) $(TARGETS) diff --git a/server/etc/example.conf b/server/etc/example.conf index c8871fca2..b7e1a64c3 100644 --- a/server/etc/example.conf +++ b/server/etc/example.conf @@ -7,12 +7,12 @@ stats = 3; # The interval in seconds fo path statistics nodes = { acs = { - type = "opal", # server, workstation, opal, rtds or dsp + type = "udp", # server, workstation, opal, rtds or dsp local = "127.0.0.1:12001", # Local ip:port, use '*' for random port remote = "127.0.0.1:12000" }, sintef = { - type = "rtds", + type = "udp", local = "127.0.0.1:12002", remote = "127.0.0.1:12003", diff --git a/server/etc/opal-shmem.conf b/server/etc/opal-shmem.conf new file mode 100644 index 000000000..aa816dc0b --- /dev/null +++ b/server/etc/opal-shmem.conf @@ -0,0 +1,28 @@ +# Example configuration file for the s2ss server + +affinity = 0x01; # Mask of cores the server should run on +priority = 99; # Scheduler priority for the server +debug = 5; # The level of verbosity for debug messages +stats = 1; # The interval in seconds for path statistics + +nodes = { + opal = { + type = "opal"; + send_id = 1; + recv_id = 1; + }, + acs-s2ss = { + type = "udp"; + local = "*:12000"; + remote = "134.130.169.31:12000"; + } +}; + +paths = ( + { + in = "opal"; + out = "acs-s2ss"; + reverse = true; + hook = "print"; + } +); diff --git a/server/include/cfg.h b/server/include/cfg.h index 5e93c42aa..8f097c749 100644 --- a/server/include/cfg.h +++ b/server/include/cfg.h @@ -14,6 +14,8 @@ #include +/* Forward declarations */ +struct list; struct node; struct path; struct interface; @@ -49,7 +51,7 @@ struct settings { * @retval <0 Error. Something went wrong. */ int config_parse(const char *filename, config_t *cfg, struct settings *set, - struct node **nodes, struct path **paths); + struct list *nodes, struct list *paths); /** Parse the global section of a configuration file. * @@ -69,21 +71,26 @@ int config_parse_global(config_setting_t *cfg, struct settings *set); * @retval <0 Error. Something went wrong. */ int config_parse_path(config_setting_t *cfg, - struct path **paths, struct node **nodes); + struct list *paths, struct list *nodes); + +int config_parse_nodelist(config_setting_t *cfg, struct list *nodes, struct list *all); + + +int config_parse_hooks(config_setting_t *cfg, struct list *hooks); /** Parse a single node and add it to the global configuration. * - * @param cfg A libconfig object pointing to the node - * @param nodes Add new nodes to this linked list + * @param cfg A libconfig object pointing to the node. + * @param nodes Add new nodes to this linked list. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int config_parse_node(config_setting_t *cfg, - struct node **nodes); +int config_parse_node(config_setting_t *cfg, struct list *nodes); /** Parse node connection details for OPAL type * - * @param cfg A libconfig object pointing to the node + * @param cfg A libconfig object pointing to the node. + * @param nodes Add new nodes to this linked list. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ @@ -91,7 +98,8 @@ int config_parse_opal(config_setting_t *cfg, struct node *n); /** Parse node connection details for GTFPGA type * - * @param cfg A libconfig object pointing to the node + * @param cfg A libconfig object pointing to the node. + * @param n A pointer to the node structure which should be parsed. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ @@ -99,7 +107,8 @@ int config_parse_gtfpga(config_setting_t *cfg, struct node *n); /** Parse node connection details for SOCKET type * - * @param cfg A libconfig object pointing to the node + * @param cfg A libconfig object pointing to the node. + * @param n A pointer to the node structure which should be parsed. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ @@ -107,8 +116,8 @@ int config_parse_socket(config_setting_t *cfg, struct node *n); /** Parse network emulator (netem) settings. * - * @param cfg A libconfig object containing the settings - * @param em A pointer to the settings + * @param cfg A libconfig object containing the settings. + * @param em A pointer to the netem settings structure (part of the path structure). * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ diff --git a/server/include/config.h b/server/include/config.h index c6100ea30..7fb0cda5b 100644 --- a/server/include/config.h +++ b/server/include/config.h @@ -11,19 +11,19 @@ #ifndef _CONFIG_H_ #define _CONFIG_H_ +#ifndef _GIT_REV + #define _GIT_REV "nogit" +#endif + /** The version number of the s2ss server */ -#define VERSION "v0.4" __GIT_REV__ +#define VERSION "v0.4-" _GIT_REV /** Maximum number of double values in a struct msg */ -#define MAX_VALUES 64 +#define MAX_VALUES 16 /** Socket priority */ #define SOCKET_PRIO 7 -/* Some parameters for histogram statistics */ -#define HIST_HEIGHT 50 -#define HIST_SEQ 17 - /* Protocol numbers */ #define IPPROTO_S2SS 137 #define ETH_P_S2SS 0xBABE diff --git a/server/include/hist.h b/server/include/hist.h new file mode 100644 index 000000000..af26154e2 --- /dev/null +++ b/server/include/hist.h @@ -0,0 +1,79 @@ +/** Histogram functions. + * + * @author Steffen Vogel + * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC + */ + +#ifndef _HIST_H_ +#define _HIST_H_ + +#include + +#define HIST_HEIGHT 50 +#define HIST_SEQ 17 + +typedef unsigned hist_cnt_t; + +/** Histogram structure used to collect statistics. */ +struct hist { + /** The distance between two adjacent buckets. */ + double resolution; + + /** The value of the highest bucket. */ + double high; + /** The value of the lowest bucket. */ + double low; + + /** The highest value observed (may be higher than #high). */ + double highest; + /** The lowest value observed (may be lower than #low). */ + double lowest; + + /** The number of buckets in #data. */ + int length; + + /** Total number of counted values between #low and #high. */ + hist_cnt_t total; + /** The number of values which are higher than #high. */ + hist_cnt_t higher; + /** The number of values which are lower than #low. */ + hist_cnt_t lower; + + /** Pointer to dynamically allocated array of size length. */ + hist_cnt_t *data; +}; + +/** Initialize struct hist with supplied values and allocate memory for buckets. */ +void hist_create(struct hist *h, double start, double end, double resolution); + +/** Free the dynamically allocated memory. */ +void hist_destroy(struct hist *h); + +/** Reset all counters and values back to zero. */ +void hist_reset(struct hist *h); + +/** Count a value within its corresponding bucket. */ +void hist_put(struct hist *h, double value); + +/** Calcluate the variance of all counted values. */ +double hist_var(struct hist *h); + +/** Calculate the mean average of all counted values. */ +double hist_mean(struct hist *h); + +/** Calculate the standard derivation of all counted values. */ +double hist_stddev(struct hist *h); + +/** Print all statistical properties of distribution including a graphilcal plot of the histogram. */ +void hist_print(struct hist *h); + +/** Print ASCII style plot of histogram */ +void hist_plot(struct hist *h); + +/** Dump histogram data in Matlab format to buf */ +void hist_dump(struct hist *h, char *buf, int len); + +/** Prints Matlab struct containing all infos to file. */ +void hist_matlab(struct hist *h, FILE *f); + +#endif /* _HIST_H_ */ diff --git a/server/include/hooks.h b/server/include/hooks.h index ca949c4fc..b245cc108 100644 --- a/server/include/hooks.h +++ b/server/include/hooks.h @@ -48,13 +48,28 @@ hook_cb_t hook_lookup(const char *name); /** Example hook: Print the message. */ int hook_print(struct msg *m, struct path *p); -/** Example hook: Filter the message on some criteria. */ -int hook_filter(struct msg *m, struct path *p); +/** Example hook: Log messages to a logfile in /tmp */ +int hook_log(struct msg *m, struct path *p); + +#define HOOK_LOG_MODE "w+" +#define HOOK_LOG_TEMPLATE "logs/s2ss-%Y_%m_%d-%H_%M_%S.log" + +/** Example hook: Drop messages. */ +int hook_decimate(struct msg *m, struct path *p); + +#define HOOK_DECIMATE_RATIO 10 /** Example hook: Convert the message values to fixed precision. */ int hook_tofixed(struct msg *m, struct path *p); -/** Example hook: Chain multiple hooks */ -int hook_multiple(struct msg *m, struct path *p); +/** Example hook: add timestamp to message. */ +int hook_ts(struct msg *m, struct path *p); + +#define HOOK_TS_INDEX -1 // last message + +/** Example hook: Finite-Impulse-Response (FIR) filter. */ +int hook_fir(struct msg *m, struct path *p); + +#define HOOK_FIR_INDEX 1 #endif /* _HOOKS_H_ */ diff --git a/server/include/if.h b/server/include/if.h index a236e6bd3..931f20234 100644 --- a/server/include/if.h +++ b/server/include/if.h @@ -14,9 +14,15 @@ #include #include +#include "list.h" + #define IF_NAME_MAX IFNAMSIZ /**< Maximum length of an interface name */ #define IF_IRQ_MAX 3 /**< Maxmimal number of IRQs of an interface */ +#ifndef SO_MARK + #define SO_MARK 36 /**< Workaround: add missing constant for OPAL-RT Redhawk target */ +#endif + struct socket; /** Interface data structure */ @@ -31,9 +37,7 @@ struct interface { char irqs[IF_IRQ_MAX]; /** Linked list of associated sockets */ - struct socket *sockets; - /** Linked list pointer */ - struct interface *next; + struct list sockets; }; /** Add a new interface to the global list and lookup name, irqs... @@ -44,6 +48,13 @@ struct interface { */ struct interface * if_create(int index); + +/** Destroy interface by freeing dynamically allocated memory. + * + * @param i A pointer to the interface structure. + */ +void if_destroy(struct interface *i); + /** Start interface. * * This setups traffic controls queue discs, network emulation and @@ -102,7 +113,7 @@ int if_getirqs(struct interface *i); */ int if_setaffinity(struct interface *i, int affinity); -/** Search the list of interfaces for a given index. +/** Search the global list of interfaces for a given index. * * @param index The interface index to search for * @param interfaces A linked list of all interfaces diff --git a/server/include/list.h b/server/include/list.h new file mode 100644 index 000000000..4c9f5d373 --- /dev/null +++ b/server/include/list.h @@ -0,0 +1,78 @@ +/** A generic linked list + * + * Linked lists a used for several data structures in the code. + * + * @author Steffen Vogel + * @copyright 2015, Institute for Automation of Complex Power Systems, EONERC + * @file + */ + +#ifndef _LIST_H_ +#define _LIST_H_ + +#include + +#include "hooks.h" + +/* Forward declarations */ +struct list_elm; +struct node; +struct path; +struct interface; + +/** Static list initialization */ +#define LIST_INIT { \ + .head = NULL, \ + .tail = NULL, \ + .count = 0, \ + .lock = PTHREAD_MUTEX_INITIALIZER \ +} + +#define FOREACH(list, elm) \ + for ( struct list_elm *elm = (list)->head; \ + elm; elm = elm->next ) + +#define FOREACH_R(list, elm) \ + for ( struct list_elm *elm = (list)->tail; \ + elm; elm = elm->prev ) + +#define list_first(list) ((list)->head) +#define list_last(list) ((list)->head) +#define list_length(list) ((list)->count) + +/** Callback to destroy list elements. + * + * @param data A pointer to the data which should be freed. + */ +typedef void (*dtor_cb_t)(void *data); + +struct list { + struct list_elm *head, *tail; + int count; + + dtor_cb_t destructor; + pthread_mutex_t lock; +}; + +struct list_elm { + union { + void *ptr; + struct node *node; + struct path *path; + struct interface *interface; + struct socket *socket; + hook_cb_t hook; + } /* anonymous */; + + struct list_elm *prev, *next; +}; + +void list_init(struct list *l, dtor_cb_t dtor); + +void list_destroy(struct list *l); + +void list_push(struct list *l, void *p); + +struct list_elm * list_search(struct list *l, int (*cmp)(void *)); + +#endif /* _LIST_H_ */ diff --git a/server/include/log.h b/server/include/log.h new file mode 100644 index 000000000..ecde7b582 --- /dev/null +++ b/server/include/log.h @@ -0,0 +1,77 @@ +/** Logging and debugging routines + * + * @author Steffen Vogel + * @copyright 2015, Institute for Automation of Complex Power Systems, EONERC + * @file + */ + + #ifndef _LOG_H_ +#define _LOG_H_ + +#ifdef __GNUC__ + #define INDENT int __attribute__ ((__cleanup__(log_outdent), unused)) _old_indent = log_indent(1); +#else + #define INDENT ; +#endif + +/** Global debug level used by the debug() macro. + * It defaults to V (defined by the Makefile) and can be + * overwritten by the 'debug' setting in the config file. + */ +extern int _debug; + +/** The log level which is passed as first argument to print() */ +enum log_level { + DEBUG, + INFO, + WARN, + ERROR +}; + +int log_indent(int levels); + +void log_outdent(int *); + +/** Reset the wallclock of debugging outputs */ +void log_reset(); + +/** Logs variadic messages to stdout. + * + * @param lvl The log level + * @param fmt The format string (printf alike) + */ +void log_print(enum log_level lvl, const char *fmt, ...) + __attribute__ ((format(printf, 2, 3))); + +/** Printf alike debug message with level. */ +#define debug(lvl, msg, ...) do if (lvl <= _debug) log_print(DEBUG, msg, ##__VA_ARGS__); while (0) + +/** Printf alike info message. */ +#define info(msg, ...) do log_print(INFO, msg, ##__VA_ARGS__); while (0) + +/** Printf alike warning message. */ +#define warn(msg, ...) do log_print(WARN, msg, ##__VA_ARGS__); while (0) + +/** Print error and exit. */ +#define error(msg, ...) do { \ + log_print(ERROR, msg, ##__VA_ARGS__); \ + die(); \ + } while (0) + +/** Print error and strerror(errno). */ +#define serror(msg, ...) do { \ + log_print(ERROR, msg ": %s", ##__VA_ARGS__, strerror(errno)); \ + die(); \ + } while (0) + +/** Print configuration error and exit. */ +#define cerror(c, msg, ...) do { \ + log_print(ERROR, msg " in %s:%u", ##__VA_ARGS__, \ + (config_setting_source_file(c)) ? \ + config_setting_source_file(c) : "(stdio)", \ + config_setting_source_line(c)); \ + die(); \ + } while (0) + +#endif /* _LOG_H_ */ + diff --git a/server/include/msg_format.h b/server/include/msg_format.h index 40c21453b..c3c188721 100644 --- a/server/include/msg_format.h +++ b/server/include/msg_format.h @@ -14,7 +14,7 @@ #define _BSD_SOURCE 1 #include #elif defined(__PPC__) /* Xilinx toolchain */ - #include + #include #endif #include "config.h" @@ -30,8 +30,8 @@ #define MSG_TYPE_START 1 /**< Message marks the beginning of a new simulation case */ #define MSG_TYPE_STOP 2 /**< Message marks the end of a simulation case */ -#define MSG_ENDIAN_LITTLE 0 /**< Message values are in little endian format (float too!) */ -#define MSG_ENDIAN_BIG 1 /**< Message values are in bit endian format */ +#define MSG_ENDIAN_LITTLE 0 /**< Message values are in little endian format (float too!) */ +#define MSG_ENDIAN_BIG 1 /**< Message values are in bit endian format */ #if BYTE_ORDER == LITTLE_ENDIAN #define MSG_ENDIAN_HOST MSG_ENDIAN_LITTLE diff --git a/server/include/node.h b/server/include/node.h index fe31fdf2e..fe5ac91c9 100644 --- a/server/include/node.h +++ b/server/include/node.h @@ -20,6 +20,7 @@ #include "msg.h" #include "tc.h" +#include "list.h" /** Static node initialization */ #define NODE_INIT(n) { \ @@ -39,12 +40,12 @@ enum node_type { UDP, /* BSD socket: AF_INET SOCK_DGRAM */ TCPD, /* BSD socket: AF_INET SOCK_STREAM bind + listen + accept */ TCP, /* BSD socket: AF_INET SOCK_STREAM bind + connect */ -// OPAL_ASYNC, /* OPAL-RT AsyncApi */ + OPAL_ASYNC, /* OPAL-RT Asynchronous Process Api */ // GTFPGA, /* Xilinx ML507 GTFPGA card */ INVALID }; -/** C++ like vtable construct for socket_types */ +/** C++ like vtable construct for node_types */ struct node_vtable { enum node_type type; const char *name; @@ -75,15 +76,12 @@ struct node /** Virtual data (used by vtable functions) */ union { struct socket *socket; - struct opal *opal; + struct opal *opal; struct gtfpga *gtfpga; }; /** A pointer to the libconfig object which instantiated this node */ config_setting_t *cfg; - - /** Linked list pointer */ - struct node *next; }; /** Connect and bind the UDP socket of this node. @@ -118,8 +116,8 @@ int node_stop(struct node *n); /** Lookup string representation of socket type * - * @param type A string describing the socket type. This must be one of: tcp, tcpd, udp, ip, ieee802.3 - * @return An enumeration value or INVALID (0) + * @param str A string describing the socket type. This must be one of: tcp, tcpd, udp, ip, ieee802.3 or opal + * @return A pointer to the vtable, or NULL if there is no socket type / vtable with this id. */ struct node_vtable const * node_lookup_vtable(const char *str); @@ -129,6 +127,21 @@ struct node_vtable const * node_lookup_vtable(const char *str); * @param nodes A linked list of all nodes * @return A pointer to the node or NULL if not found */ -struct node* node_lookup_name(const char *str, struct node *nodes); +struct node * node_lookup_name(const char *str, struct list *nodes); + +/** Reverse local and remote socket address. + * This is usefull for the helper programs: send, receive, test + * because they usually use the same configuration file as the + * server and therefore the direction needs to be swapped. */ +void node_reverse(struct node *n); + +/** Create a node by allocating dynamic memory. */ +struct node * node_create(); + +/** Destroy node by freeing dynamically allocated memory. + * + * @param i A pointer to the interface structure. + */ +void node_destroy(struct node *n); #endif /* _NODE_H_ */ diff --git a/server/include/opal.h b/server/include/opal.h index 7b0d74c7b..4fbf74b8c 100644 --- a/server/include/opal.h +++ b/server/include/opal.h @@ -1,4 +1,4 @@ -/** Node type: OPAL (AsyncApi) +/** Node type: OPAL (libOpalAsync API) * * This file implements the opal subtype for nodes. * @@ -9,8 +9,78 @@ #ifndef _OPAL_H_ #define _OPAL_H_ -struct opal { +#include +#include "node.h" +#include "msg.h" + +/* Define RTLAB before including OpalPrint.h for messages to be sent + * to the OpalDisplay. Otherwise stdout will be used. */ +#define RTLAB +#include "OpalPrint.h" +#include "AsyncApi.h" +#include "OpalGenAsyncParamCtrl.h" + +/** This global structure holds libOpalAsync related information. + * It's only used once in the code. */ +struct opal_global { + /** Shared Memory identifiers and size, provided via argv. */ + char *async_shmem_name, *print_shmem_name; + int async_shmem_size; + + /** Number of send blocks used in the running OPAL model. */ + int send_icons, recv_icons; + /** A dynamically allocated array of SendIDs. */ + int *send_ids, *recv_ids; + + /** String and Float parameters, provided by the OPAL AsyncProcess block. */ + Opal_GenAsyncParam_Ctrl params; + + /** Big Global Lock for libOpalAsync API */ + pthread_mutex_t lock; }; +struct opal { + int reply; + int mode; + + int send_id; + int recv_id; + + int seq_no; + + struct opal_global *global; + + Opal_SendAsyncParam send_params; + Opal_RecvAsyncParam recv_params; +}; + +/** Initialize global OPAL settings and maps shared memory regions. + * + * @param argc The number of CLI arguments, provided to main(). + * @param argv The CLI argument list, provided to main(). + * @retval 0 On success. + * @retval <0 On failure. + */ +int opal_init(int argc, char *argv[]); + +/** Free global OPAL settings and unmaps shared memory regions. + * + * @retval 0 On success. + * @retval <0 On failure. + */ +int opal_deinit(); + +int opal_print(struct node *n, char *buf, int len); + +int opal_print_global(struct opal_global *g); + +int opal_open(struct node *n); + +int opal_close(struct node *n); + +int opal_read(struct node *n, struct msg *m); + +int opal_write(struct node *n, struct msg *m); + #endif /* _OPAL_H_ */ diff --git a/server/include/path.h b/server/include/path.h index 7f71ce8f2..4195855c7 100644 --- a/server/include/path.h +++ b/server/include/path.h @@ -11,7 +11,9 @@ #include #include +#include "list.h" #include "config.h" +#include "hist.h" #include "node.h" #include "msg.h" #include "hooks.h" @@ -24,22 +26,27 @@ struct path { /** Pointer to the incoming node */ struct node *in; - /** Pointer to the outgoing node */ + /** Pointer to the first outgoing node. + * Usually this is only a pointer to the first list element of path::destinations. */ struct node *out; - - /** Function pointer of the hook */ - hook_cb_t hook; + + /** List of all outgoing nodes */ + struct list destinations; + /** List of function pointers to hooks */ + struct list hooks; /** Send messages with a fixed rate over this path */ double rate; /** A pointer to the last received message */ - struct msg *last; + struct msg *current; + /** A pointer to the previously received message */ + struct msg *previous; + + /** Counter for received messages according to their sequence no displacement */ + struct hist histogram; - /** Last known message number */ - unsigned int sequence; - - /** Counter for sent messages to all outgoing nodes*/ + /** Counter for sent messages to all outgoing nodes */ unsigned int sent; /** Counter for received messages from all incoming nodes */ unsigned int received; @@ -49,25 +56,31 @@ struct path unsigned int skipped; /** Counter for dropped messages due to reordering */ unsigned int dropped; - /** Counter for received messages according to their sequence no displacement */ - unsigned int histogram[HIST_SEQ]; + /** A timer used for fixed rate transmission. */ + timer_t timer; /** The thread id for this path */ pthread_t recv_tid; /** A second thread id for fixed rate sending thread */ pthread_t sent_tid; /** A pointer to the libconfig object which instantiated this path */ config_setting_t *cfg; - - /** Linked list pointer */ - struct path *next; }; +/** Create a path by allocating dynamic memory. */ +struct path * path_create(); + +/** Destroy path by freeing dynamically allocated memory. + * + * @param i A pointer to the path structure. + */ +void path_destroy(struct path *p); + /** Start a path. * * Start a new pthread for receiving/sending messages over this path. * - * @param p A pointer to the path struct + * @param p A pointer to the path structure. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ @@ -75,7 +88,7 @@ int path_start(struct path *p); /** Stop a path. * - * @param p A pointer to the path struct + * @param p A pointer to the path structure. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ @@ -83,8 +96,18 @@ int path_stop(struct path *p); /** Show some basic statistics for a path. * - * @param p A pointer to the path struct + * @param p A pointer to the path structure. */ -void path_stats(struct path *p); +void path_print_stats(struct path *p); + +/** Fills the provided buffer with a string representation of the path. + * + * Format: source => [ dest1 dest2 dest3 ] + * + * @param p A pointer to the path structure. + * @param buf A pointer to the buffer which should be filled. + * @param len The length of buf in bytes. + */ +int path_print(struct path *p, char *buf, int len); #endif /* _PATH_H_ */ diff --git a/server/include/socket.h b/server/include/socket.h index baa5e3e0a..0b8b63bf3 100644 --- a/server/include/socket.h +++ b/server/include/socket.h @@ -16,6 +16,8 @@ struct socket { /** The socket descriptor */ int sd; + /** The socket descriptor for an established TCP connection */ + int sd2; /** Socket mark for netem, routing and filtering */ int mark; diff --git a/server/include/utils.h b/server/include/utils.h index 5763086e4..fce60c599 100644 --- a/server/include/utils.h +++ b/server/include/utils.h @@ -12,6 +12,10 @@ #include #include #include +#include +#include + +#include "log.h" #ifdef __GNUC__ #define EXPECT(x, v) __builtin_expect(x, v) @@ -20,52 +24,73 @@ #endif /* Some color escape codes for pretty log messages */ -#define GRY(str) "\e[30m" str "\e[0m" /**< Print str in gray */ -#define RED(str) "\e[31m" str "\e[0m" /**< Print str in red */ -#define GRN(str) "\e[32m" str "\e[0m" /**< Print str in green */ -#define YEL(str) "\e[33m" str "\e[0m" /**< Print str in yellow */ -#define BLU(str) "\e[34m" str "\e[0m" /**< Print str in blue */ -#define MAG(str) "\e[35m" str "\e[0m" /**< Print str in magenta */ -#define CYN(str) "\e[36m" str "\e[0m" /**< Print str in cyan */ -#define WHT(str) "\e[37m" str "\e[0m" /**< Print str in white */ -#define BLD(str) "\e[1m" str "\e[0m" /**< Print str in bold */ +#ifndef ENABLE_OPAL_ASYNC + #define GRY(str) "\e[30m" str "\e[0m" /**< Print str in gray */ + #define RED(str) "\e[31m" str "\e[0m" /**< Print str in red */ + #define GRN(str) "\e[32m" str "\e[0m" /**< Print str in green */ + #define YEL(str) "\e[33m" str "\e[0m" /**< Print str in yellow */ + #define BLU(str) "\e[34m" str "\e[0m" /**< Print str in blue */ + #define MAG(str) "\e[35m" str "\e[0m" /**< Print str in magenta */ + #define CYN(str) "\e[36m" str "\e[0m" /**< Print str in cyan */ + #define WHT(str) "\e[37m" str "\e[0m" /**< Print str in white */ + #define BLD(str) "\e[1m" str "\e[0m" /**< Print str in bold */ -#define GFX(chr) "\e(0" chr "\e(B" -#define UP(n) "\e[" ## n ## "A" -#define DOWN(n) "\e[" ## n ## "B" -#define RIGHT(n) "\e[" ## n ## "C" -#define LEFT(n) "\e[" ## n ## "D" + #define GFX(chr) "\e(0" chr "\e(B" + #define UP(n) "\e[" ## n ## "A" + #define DOWN(n) "\e[" ## n ## "B" + #define RIGHT(n) "\e[" ## n ## "C" + #define LEFT(n) "\e[" ## n ## "D" +#else + #define GRY(str) str + #define RED(str) str + #define GRN(str) str + #define YEL(str) str + #define BLU(str) str + #define MAG(str) str + #define CYN(str) str + #define WHT(str) str + #define BLD(str) str -#define ARRAY_LEN(a) ( sizeof a / sizeof a[0] ) + #define GFX(chr) " " + #define UP(n) "" + #define DOWN(n) "" + #define RIGHT(n) "" + #define LEFT(n) "" +#endif -/** The log level which is passed as first argument to print() */ -enum log_level { DEBUG, INFO, WARN, ERROR }; +/* CPP stringification */ +#define XSTR(x) STR(x) +#define STR(x) #x + +/** Calculate the number of elements in an array. */ +#define ARRAY_LEN(a) ( sizeof (a) / sizeof (a)[0] ) + +/** Swap two values by using a local third one. */ +#define SWAP(a, b) do { \ + __typeof__(a) tmp = a; \ + a = b; \ + b = tmp; \ + } while(0) /* Forward declarations */ struct settings; struct timespec; -/* These global variables allow changing the output style and verbosity */ -extern int _debug; -extern int _indent; - -void outdent(int *old); - -#ifdef __GNUC__ - #define INDENT int __attribute__ ((__cleanup__(outdent), unused)) _old_indent = _indent++; -#else - #define INDENT ; -#endif - -/** Reset the wallclock of debugging outputs */ -void epoch_reset(); - -/** Logs variadic messages to stdout. - * - * @param lvl The log level - * @param fmt The format string (printf alike) +/** The main thread id. + * This is used to notify the main thread about + * the program termination. + * See error() macros. */ -void print(enum log_level lvl, const char *fmt, ...); +extern pthread_t _mtid; + +/** Safely append a format string to an existing string. + * + * This function is similar to strlcat() from BSD. + */ +int strap(char *dest, size_t size, const char *fmt, ...); + +/** Variable arguments (stdarg) version of strap() */ +int vstrap(char *dest, size_t size, const char *fmt, va_list va); /** Convert integer to cpu_set_t. * @@ -74,26 +99,20 @@ void print(enum log_level lvl, const char *fmt, ...); */ cpu_set_t to_cpu_set(int set); +/** Allocate and initialize memory. */ +void * alloc(size_t bytes); + /** Get delta between two timespec structs */ double timespec_delta(struct timespec *start, struct timespec *end); /** Get period as timespec from rate */ struct timespec timespec_rate(double rate); -/** Print ASCII style plot of histogram */ -void hist_plot(unsigned *hist, int length); - -/** Dump histogram data in Matlab format */ -void hist_dump(unsigned *hist, int length); - -/** A system(2) emulator with popen/pclose(2) and proper output handling */ +/** A system(2) emulator with popen / pclose(2) and proper output handling */ int system2(const char* cmd, ...); -/** Append an element to a single linked list */ -#define list_add(list, elm) do { \ - elm->next = list; \ - list = elm; \ - } while (0) +/** Call quit() in the main thread. */ +void die(); /** Check assertion and exit if failed. */ #define assert(exp) do { \ @@ -103,41 +122,5 @@ int system2(const char* cmd, ...); exit(EXIT_FAILURE); \ } } while (0) -/** Printf alike debug message with level. */ -#define debug(lvl, msg, ...) do { \ - if (lvl <= _debug) \ - print(DEBUG, msg, ##__VA_ARGS__); \ - } while (0) - -/** Printf alike info message. */ -#define info(msg, ...) do { \ - print(INFO, msg, ##__VA_ARGS__); \ - } while (0) - -/** Printf alike warning message. */ -#define warn(msg, ...) do { \ - print(WARN, msg, ##__VA_ARGS__); \ - } while (0) - -/** Print error and exit. */ -#define error(msg, ...) do { \ - print(ERROR, msg, ##__VA_ARGS__); \ - exit(EXIT_FAILURE); \ - } while (0) - -/** Print error and strerror(errno). */ -#define perror(msg, ...) do { \ - print(ERROR, msg ": %s", ##__VA_ARGS__, \ - strerror(errno)); \ - exit(EXIT_FAILURE); \ - } while (0) - -/** Print configuration error and exit. */ -#define cerror(c, msg, ...) do { \ - print(ERROR, msg " in %s:%u", ##__VA_ARGS__, \ - config_setting_source_file(c), \ - config_setting_source_line(c)); \ - exit(EXIT_FAILURE); \ - } while (0) - #endif /* _UTILS_H_ */ + diff --git a/server/src/cfg.c b/server/src/cfg.c index de173b4a4..6a3cf53ba 100644 --- a/server/src/cfg.c +++ b/server/src/cfg.c @@ -4,33 +4,41 @@ * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC */ +#include #include #include #include +#include "utils.h" +#include "list.h" #include "if.h" #include "tc.h" #include "cfg.h" #include "node.h" #include "path.h" -#include "utils.h" #include "hooks.h" #include "socket.h" #include "gtfpga.h" +#ifdef ENABLE_OPAL_ASYNC #include "opal.h" +#endif int config_parse(const char *filename, config_t *cfg, struct settings *set, - struct node **nodes, struct path **paths) + struct list *nodes, struct list *paths) { config_set_auto_convert(cfg, 1); - if (!config_read_file(cfg, filename)) + int ret = strcmp("-", filename) ? config_read_file(cfg, filename) + : config_read(cfg, stdin); + + if (ret != CONFIG_TRUE) error("Failed to parse configuration: %s in %s:%d", - config_error_text(cfg), filename, + config_error_text(cfg), + config_error_file(cfg) ? config_error_file(cfg) : filename, config_error_line(cfg) ); - + config_setting_t *cfg_root = config_root_setting(cfg); /* Parse global settings */ @@ -65,7 +73,7 @@ int config_parse(const char *filename, config_t *cfg, struct settings *set, } } - return CONFIG_TRUE; + return 0; } int config_parse_global(config_setting_t *cfg, struct settings *set) @@ -79,44 +87,42 @@ int config_parse_global(config_setting_t *cfg, struct settings *set) set->cfg = cfg; - return CONFIG_TRUE; + return 0; } int config_parse_path(config_setting_t *cfg, - struct path **paths, struct node **nodes) + struct list *paths, struct list *nodes) { - const char *in, *out, *hook; + const char *in; int enabled = 1; int reverse = 0; + + struct path *p = alloc(sizeof(struct path)); - struct path *p = (struct path *) malloc(sizeof(struct path)); - if (!p) - error("Failed to allocate memory for path"); + /* Input node */ + struct config_setting_t *cfg_in = config_setting_get_member(cfg, "in"); + if (!cfg_in || config_setting_type(cfg_in) != CONFIG_TYPE_STRING) + cerror(cfg, "Invalid input node for path"); + + in = config_setting_get_string(cfg_in); + p->in = node_lookup_name(in, nodes); + if (!p->in) + cerror(cfg_in, "Invalid input node '%s'", in); + + /* Output node(s) */ + struct config_setting_t *cfg_out = config_setting_get_member(cfg, "out"); + if (cfg_out) + config_parse_nodelist(cfg_out, &p->destinations, nodes); + + if (list_length(&p->destinations) >= 1) + p->out = list_first(&p->destinations)->node; else - memset(p, 0, sizeof(struct path)); - - /* Required settings */ - if (!config_setting_lookup_string(cfg, "in", &in)) - cerror(cfg, "Missing input node for path"); - - if (!config_setting_lookup_string(cfg, "out", &out)) cerror(cfg, "Missing output node for path"); - p->in = node_lookup_name(in, *nodes); - if (!p->in) - cerror(cfg, "Invalid input node '%s'", in); - - p->out = node_lookup_name(out, *nodes); - if (!p->out) - cerror(cfg, "Invalid output node '%s'", out); - /* Optional settings */ - if (config_setting_lookup_string(cfg, "hook", &hook)) { - p->hook = hook_lookup(hook); - - if (!p->hook) - cerror(cfg, "Failed to lookup hook function. Not registred?"); - } + struct config_setting_t *cfg_hook = config_setting_get_member(cfg, "hook"); + if (cfg_hook) + config_parse_hooks(cfg_hook, &p->hooks); config_setting_lookup_bool(cfg, "enabled", &enabled); config_setting_lookup_bool(cfg, "reverse", &reverse); @@ -126,45 +132,110 @@ int config_parse_path(config_setting_t *cfg, if (enabled) { p->in->refcnt++; - p->out->refcnt++; - - list_add(*paths, p); - + FOREACH(&p->destinations, it) + it->node->refcnt++; + if (reverse) { - struct path *rev = (struct path *) malloc(sizeof(struct path)); - if (!rev) - error("Failed to allocate memory for path"); - else - memcpy(rev, p, sizeof(struct path)); + if (list_length(&p->destinations) > 1) + warn("Using first destination '%s' as source for reverse path. " + "Ignoring remaining nodes", p->out->name); - rev->in = p->out; /* Swap in/out */ - rev->out = p->in; + struct path *r = path_create(); - rev->in->refcnt++; - rev->out->refcnt++; + r->in = p->out; /* Swap in/out */ + r->out = p->in; + + list_push(&r->destinations, r->out); - list_add(*paths, rev); + r->in->refcnt++; + r->out->refcnt++; + + list_push(paths, r); } + + list_push(paths, p); } else { - warn("Path '%s' => '%s' is not enabled", p->in->name, p->out->name); - free(p); + char buf[33]; + path_print(p, buf, sizeof(buf)); + + warn("Path %s is not enabled", buf); + path_destroy(p); } return 0; } -int config_parse_node(config_setting_t *cfg, struct node **nodes) +int config_parse_nodelist(config_setting_t *cfg, struct list *nodes, struct list *all) { + const char *str; + struct node *node; + + switch (config_setting_type(cfg)) { + case CONFIG_TYPE_STRING: + str = config_setting_get_string(cfg); + node = node_lookup_name(str, all); + if (!node) + cerror(cfg, "Invalid outgoing node '%s'", str); + + list_push(nodes, node); + break; + + case CONFIG_TYPE_ARRAY: + for (int i=0; icfg = cfg; @@ -173,41 +244,71 @@ int config_parse_node(config_setting_t *cfg, struct node **nodes) cerror(cfg, "Missing node name"); if (!config_setting_lookup_string(cfg, "type", &type)) - cerror(cfg, "Missing node type"); - + cerror(cfg, "Missing node name"); + n->vt = node_lookup_vtable(type); if (!n->vt) - cerror(cfg, "Invalid node type"); + cerror(cfg, "Invalid type for node '%s'", n->name); ret = n->vt->parse(cfg, n); - - list_add(*nodes, n); + if (!ret) + list_push(nodes, n); return ret; } -/** @todo Implement */ +#ifdef ENABLE_OPAL_ASYNC +/** @todo: Remove this global variable. */ +extern struct opal_global *og; + int config_parse_opal(config_setting_t *cfg, struct node *n) -{ +{ + if (!og) { + warn("Skipping node '%s', because this server is not running as an OPAL Async process!", n->name); + return -1; + } + + struct opal *o = alloc(sizeof(struct opal)); + + config_setting_lookup_int(cfg, "send_id", &o->send_id); + config_setting_lookup_int(cfg, "recv_id", &o->recv_id); + config_setting_lookup_bool(cfg, "reply", &o->reply); + + /* Search for valid send and recv ids */ + int sfound = 0, rfound = 0; + for (int i=0; isend_icons; i++) + sfound += og->send_ids[i] == o->send_id; + for (int i=0; isend_icons; i++) + rfound += og->send_ids[i] == o->send_id; + + if (!sfound) + cerror(config_setting_get_member(cfg, "send_id"), "Invalid send_id '%u' for node '%s'", o->send_id, n->name); + if (!rfound) + cerror(config_setting_get_member(cfg, "recv_id"), "Invalid recv_id '%u' for node '%s'", o->recv_id, n->name); + + n->opal = o; + n->opal->global = og; + n->cfg = cfg; + return 0; } +#endif /* ENABLE_OPAL_ASYNC */ + +#ifdef ENABLE_GTFPGA /** @todo Implement */ int config_parse_gtfpga(config_setting_t *cfg, struct node *n) { return 0; } +#endif /* ENABLE_GTFPGA */ int config_parse_socket(config_setting_t *cfg, struct node *n) { const char *local, *remote; int ret; - struct socket *s = (struct socket *) malloc(sizeof(struct socket)); - if (!s) - perror("Failed to allocate memory"); - - memset(s, 0, sizeof(struct socket)); + struct socket *s = alloc(sizeof(struct socket)); if (!config_setting_lookup_string(cfg, "remote", &remote)) cerror(cfg, "Missing remote address for node '%s'", n->name); @@ -228,13 +329,14 @@ int config_parse_socket(config_setting_t *cfg, struct node *n) /** @todo Netem settings are not usable AF_UNIX */ config_setting_t *cfg_netem = config_setting_get_member(cfg, "netem"); if (cfg_netem) { - s->netem = (struct netem *) malloc(sizeof(struct netem)); + s->netem = alloc(sizeof(struct netem)); + config_parse_netem(cfg_netem, s->netem); } n->socket = s; - return CONFIG_TRUE; + return 0; } int config_parse_netem(config_setting_t *cfg, struct netem *em) @@ -256,5 +358,5 @@ int config_parse_netem(config_setting_t *cfg, struct netem *em) /** @todo Validate netem config values */ - return CONFIG_TRUE; + return 0; } diff --git a/server/src/hist.c b/server/src/hist.c new file mode 100644 index 000000000..361375c41 --- /dev/null +++ b/server/src/hist.c @@ -0,0 +1,169 @@ +/** Histogram functions. + * + * @author Steffen Vogel + * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC + */ + +#include +#include +#include +#include +#include +#include + +#include "utils.h" +#include "hist.h" + +#define VAL(h, i) ((h)->low + (i) * (h)->resolution) +#define INDEX(h, v) round((v - (h)->low) / (h)->resolution) + +void hist_create(struct hist *h, double low, double high, double resolution) +{ + h->low = low; + h->high = high; + h->resolution = resolution; + h->length = (high - low) / resolution; + h->data = alloc(h->length * sizeof(unsigned)); + + hist_reset(h); +} + +void hist_destroy(struct hist *h) +{ + free(h->data); +} + +void hist_put(struct hist *h, double value) +{ + int idx = INDEX(h, value); + + /* Update min/max */ + if (value > h->highest) + h->highest = value; + if (value < h->lowest) + h->lowest = value; + + /* Check bounds and increment */ + if (idx >= h->length) + h->higher++; + else if (idx < 0) + h->lower++; + else { + h->total++; + h->data[idx]++; + } +} + +void hist_reset(struct hist *h) +{ + h->total = 0; + h->higher = 0; + h->lower = 0; + + h->highest = DBL_MIN; + h->lowest = DBL_MAX; + + memset(h->data, 0, h->length * sizeof(unsigned)); +} + +double hist_mean(struct hist *h) +{ + double mean = 0; + + for (int i = 0; i < h->length; i++) + mean += VAL(h, i) * h->data[i]; + + return mean / h->total; +} + +double hist_var(struct hist *h) +{ + double mean_x2 = 0; + double mean = hist_mean(h); + + for (int i = 0; i < h->length; i++) + mean_x2 += pow(VAL(h, i), 2) * h->data[i]; + + /* Var[X] = E[X^2] - E^2[X] */ + return mean_x2 / h->total - pow(mean, 2); +} + +double hist_stddev(struct hist *h) +{ + return sqrt(hist_var(h)); +} + +void hist_print(struct hist *h) +{ INDENT + info("Total: %u values between %f and %f", h->total, h->low, h->high); + info("Missed: %u (above), %u (below) ", h->higher, h->lower); + info("Highest value: %f, lowest %f", h->highest, h->lowest); + info("Mean: %f", hist_mean(h)); + info("Variance: %f", hist_var(h)); + info("Standard derivation: %f", hist_stddev(h)); + + hist_plot(h); + + char buf[h->length * 8]; + hist_dump(h, buf, sizeof(buf)); + + info("hist = %s", buf); +} + +void hist_plot(struct hist *h) +{ + unsigned min = UINT_MAX; + unsigned max = 0; + + /* Get max, first & last */ + for (int i = 0; i < h->length; i++) { + if (h->data[i] > max) + max = h->data[i]; + if (h->data[i] < min) + min = h->data[i]; + } + + char buf[HIST_HEIGHT]; + memset(buf, '#', HIST_HEIGHT); + + /* Print plot */ + info("%9s | %5s | %s", "Value", "Occur", "Histogram Plot:"); + for (int i = 0; i < h->length; i++) { + int bar = HIST_HEIGHT * ((double) h->data[i] / max); + + if (h->data[i] == min) info("%+5.2e | " GRN("%5u") " | %.*s", VAL(h, i), h->data[i], bar, buf); + else if (h->data[i] == max) info("%+5.2e | " RED("%5u") " | %.*s", VAL(h, i), h->data[i], bar, buf); + else info("%+5.2e | " "%5u" " | %.*s", VAL(h, i), h->data[i], bar, buf); + } +} + +void hist_dump(struct hist *h, char *buf, int len) +{ + char tok[8]; + memset(buf, 0, len); + + strncat(buf, "[ ", len); + + for (int i = 0; i < h->length; i++) { + snprintf(tok, sizeof(tok), "%u ", h->data[i]); + strncat(buf, tok, len - strlen(buf)); + } + + strncat(buf, "]", len - strlen(buf)); +} + +void hist_matlab(struct hist *h, FILE *f) +{ + char buf[h->length * 8]; + hist_dump(h, buf, sizeof(buf)); + + fprintf(f, "%lu = struct( ", time(NULL)); + fprintf(f, "'min', %f, 'max', %f, ", h->low, h->high); + fprintf(f, "'ok', %u, too_high', %u, 'too_low', %u, ", h->total, h->higher, h->lower); + fprintf(f, "'highest', %f, 'lowest', %f, ", h->highest, h->lowest); + fprintf(f, "'mean', %f, ", hist_mean(h)); + fprintf(f, "'var', %f, ", hist_var(h)); + fprintf(f, "'stddev', %f, ", hist_stddev(h)); + fprintf(f, "'hist', %s ", buf); + fprintf(f, "),\n"); +} diff --git a/server/src/hooks.c b/server/src/hooks.c index 3ee6bfbf4..637328ce9 100644 --- a/server/src/hooks.c +++ b/server/src/hooks.c @@ -9,18 +9,26 @@ * @author Steffen Vogel * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC */ - + +#include +#include #include +#include +#include #include "msg.h" #include "hooks.h" +#include "path.h" +#include "utils.h" /** @todo Make const */ static struct hook_id hook_list[] = { { hook_print, "print" }, - { hook_filter, "filter" }, + { hook_log, "log" }, + { hook_decimate, "decimate" }, { hook_tofixed, "tofixed" }, - { hook_multiple, "multiple" }, + { hook_ts, "ts" }, + { hook_fir, "fir" }, { NULL } }; @@ -43,13 +51,39 @@ int hook_print(struct msg *m, struct path *p) return 0; } -int hook_filter(struct msg *m, struct path *p) +int hook_log(struct msg *m, struct path *p) { - /* Drop every 10th message */ - if (m->sequence % 10 == 0) - return -1; - else - return 0; + static pthread_key_t pkey; + FILE *file = pthread_getspecific(pkey); + + if (!file) { + char fstr[64], pstr[33]; + path_print(p, pstr, sizeof(pstr)); + + struct tm tm; + time_t ts = time(NULL); + localtime_r(&ts, &tm); + strftime(fstr, sizeof(fstr), HOOK_LOG_TEMPLATE, &tm); + + + + file = fopen(fstr, HOOK_LOG_MODE); + if (file) + debug(5, "Opened log file for path %s: %s", pstr, fstr); + + pthread_key_create(&pkey, (dtor_cb_t) fclose); + pthread_setspecific(pkey, file); + } + + msg_fprint(file, m); + + return 0; +} + +int hook_decimate(struct msg *m, struct path *p) +{ + /* Drop every HOOK_DECIMATE_RATIO'th message */ + return (m->sequence % HOOK_DECIMATE_RATIO == 0) ? -1 : 0; } int hook_tofixed(struct msg *m, struct path *p) @@ -61,12 +95,50 @@ int hook_tofixed(struct msg *m, struct path *p) return 0; } -int hook_multiple(struct msg *m, struct path *p) +int hook_ts(struct msg *m, struct path *p) { - if (hook_print(m, p)) - return -1; - else if (hook_tofixed(m, p)) - return -1; - else - return 0; + struct timespec *ts = (struct timespec *) &m->data[HOOK_TS_INDEX]; + + clock_gettime(CLOCK_REALTIME, ts); + + return 0; +} + +/** Simple FIR-LP: F_s = 1kHz, F_pass = 100 Hz, F_block = 300 + * Tip: Use MATLAB's filter design tool and export coefficients + * with the integrated C-Header export */ +static const double hook_fir_coeffs[] = { -0.003658148158728, -0.008882653268281, 0.008001024183003, + 0.08090485991761, 0.2035239551043, 0.3040703593515, + 0.3040703593515, 0.2035239551043, 0.08090485991761, + 0.008001024183003, -0.008882653268281,-0.003658148158728 }; + +/** @todo: test */ +int hook_fir(struct msg *m, struct path *p) +{ + static pthread_key_t pkey; + float *history = pthread_getspecific(pkey); + + /** Length of impulse response */ + int len = ARRAY_LEN(hook_fir_coeffs); + /** Current index in circular history buffer */ + int cur = m->sequence % len; + /* Accumulator */ + double sum = 0; + + /* Create thread local storage for circular history buffer */ + if (!history) { + history = alloc(len * sizeof(float)); + pthread_key_create(&pkey, free); + pthread_setspecific(pkey, history); + } + + /* Update circular buffer */ + history[cur] = m->data[HOOK_FIR_INDEX].f; + + for (int i=0; idata[HOOK_FIR_INDEX].f = sum; + + return 0; } diff --git a/server/src/if.c b/server/src/if.c index aaba33ea7..01eb104df 100644 --- a/server/src/if.c +++ b/server/src/if.c @@ -22,26 +22,31 @@ #include "socket.h" #include "utils.h" -/** Linked list of interfaces */ -struct interface *interfaces; +/** Linked list of interfaces. */ +struct list interfaces; struct interface * if_create(int index) { - struct interface *i = malloc(sizeof(struct interface)); - if (!i) - error("Failed to allocate memory for interface"); - else - memset(i, 0, sizeof(struct interface)); + struct interface *i = alloc(sizeof(struct interface)); i->index = index; if_indextoname(index, i->name); - debug(3, "Created interface '%s'", i->name, i->index, i->refcnt); + debug(3, "Created interface '%s' (index=%u)", i->name, i->index); - list_add(interfaces, i); + list_init(&i->sockets, NULL); + list_push(&interfaces, i); return i; } +void if_destroy(struct interface *i) +{ + /* List members are freed by their belonging nodes. */ + list_destroy(&i->sockets); + + free(i); +} + int if_start(struct interface *i, int affinity) { INDENT if (!i->refcnt) { @@ -53,13 +58,15 @@ int if_start(struct interface *i, int affinity) { INDENT int mark = 0; - for (struct socket *s = i->sockets; s; s = s->next) { + FOREACH(&i->sockets, it) { + struct socket *s = it->socket; + if (s->netem) { s->mark = 1 + mark++; /* Set fwmark for outgoing packets */ if (setsockopt(s->sd, SOL_SOCKET, SO_MARK, &s->mark, sizeof(s->mark))) - perror("Failed to set fwmark for outgoing packets"); + serror("Failed to set fwmark for outgoing packets"); else debug(4, "Set fwmark for socket->sd = %u to %u", s->sd, s->mark); @@ -82,7 +89,7 @@ int if_start(struct interface *i, int affinity) int if_stop(struct interface *i) { INDENT - info("Stopping interface '%s'", i->name); + info("Stopping interface '%s'", i->name); { INDENT if_setaffinity(i, -1L); @@ -175,10 +182,9 @@ int if_setaffinity(struct interface *i, int affinity) struct interface * if_lookup_index(int index) { - for (struct interface *i = interfaces; i; i = i->next) { - if (i->index == index) { - return i; - } + FOREACH(&interfaces, it) { + if (it->interface->index == index) + return it->interface; } return NULL; diff --git a/server/src/list.c b/server/src/list.c new file mode 100644 index 000000000..bfc5882cc --- /dev/null +++ b/server/src/list.c @@ -0,0 +1,74 @@ +/** A generic linked list + * + * Linked lists a used for several data structures in the code. + * + * @author Steffen Vogel + * @copyright 2015, Institute for Automation of Complex Power Systems, EONERC + * @file + */ + +#include "utils.h" +#include "list.h" + +void list_init(struct list *l, dtor_cb_t dtor) +{ + pthread_mutex_init(&l->lock, NULL); + + l->destructor = dtor; + l->count = 0; + l->head = NULL; + l->tail = NULL; +} + +void list_destroy(struct list *l) +{ + pthread_mutex_lock(&l->lock); + + struct list_elm *elm = l->head; + while (elm) { + struct list_elm *tmp = elm; + elm = elm->next; + + if (l->destructor) + l->destructor(tmp->ptr); + + free(tmp); + } + + pthread_mutex_unlock(&l->lock); + pthread_mutex_destroy(&l->lock); +} + +void list_push(struct list *l, void *p) +{ + struct list_elm *e = alloc(sizeof(struct list_elm)); + + pthread_mutex_lock(&l->lock); + + e->ptr = p; + e->prev = l->tail; + e->next = NULL; + + if (l->tail) + l->tail->next = e; + if (l->head) + l->head->prev = e; + else + l->head = e; + + l->tail = e; + + l->count++; + + pthread_mutex_unlock(&l->lock); +} + +struct list_elm * list_search(struct list *l, int (*cmp)(void *)) +{ + FOREACH(l, it) { + if (!cmp(it->ptr)) + return it; + } + + return NULL; +} diff --git a/server/src/log.c b/server/src/log.c new file mode 100644 index 000000000..61a30abee --- /dev/null +++ b/server/src/log.c @@ -0,0 +1,75 @@ +/** Logging and debugging routines + * + * @author Steffen Vogel + * @copyright 2015, Institute for Automation of Complex Power Systems, EONERC + */ + + #include +#include + +#include "log.h" +#include "utils.h" + +int _debug = V; + +static struct timespec epoch; + +#ifdef __GNUC__ +static __thread int indent = 0; + +/** Get thread-specific pointer to indent level */ +int log_indent(int levels) +{ + int old = indent; + indent += levels; + return old; +} + +void log_outdent(int *old) +{ + indent = *old; +} +#endif + +void log_reset() +{ + clock_gettime(CLOCK_REALTIME, &epoch); +} + +void log_print(enum log_level lvl, const char *fmt, ...) +{ + struct timespec ts; + char buf[512] = ""; + + va_list ap; + + /* Timestamp */ + clock_gettime(CLOCK_REALTIME, &ts); + strap(buf, sizeof(buf), "%8.3f ", timespec_delta(&epoch, &ts)); + + /* Severity */ + switch (lvl) { + case DEBUG: strap(buf, sizeof(buf), BLD("%-5s "), GRY("Debug")); break; + case INFO: strap(buf, sizeof(buf), BLD("%-5s "), " " ); break; + case WARN: strap(buf, sizeof(buf), BLD("%-5s "), YEL(" Warn")); break; + case ERROR: strap(buf, sizeof(buf), BLD("%-5s "), RED("Error")); break; + } + + /* Indention */ +#ifdef __GNUC__ + for (int i = 0; i < indent; i++) + strap(buf, sizeof(buf), GFX("\x78") " "); + strap(buf, sizeof(buf), GFX("\x74") " "); +#endif + + /* Format String */ + va_start(ap, fmt); + vstrap(buf, sizeof(buf), fmt, ap); + va_end(ap); + + /* Output */ +#ifdef ENABLE_OPAL_ASYNC + OpalPrint("S2SS: %s\n", buf); +#endif + fprintf(stderr, "\r%s\n", buf); +} diff --git a/server/src/node.c b/server/src/node.c index e3aa6789b..2a6e71db4 100644 --- a/server/src/node.c +++ b/server/src/node.c @@ -13,7 +13,9 @@ /* Node types */ #include "socket.h" #include "gtfpga.h" +#ifdef ENABLE_OPAL_ASYNC #include "opal.h" +#endif #define VTABLE(type, name, fnc) { type, name, config_parse_ ## fnc, \ fnc ## _print, \ @@ -24,23 +26,24 @@ /** Vtable for virtual node sub types */ static const struct node_vtable vtables[] = { +#ifdef ENABLE_OPAL_ASYNC + VTABLE(OPAL_ASYNC, "opal", opal), +#endif VTABLE(IEEE_802_3, "ieee802.3", socket), VTABLE(IP, "ip", socket), VTABLE(UDP, "udp", socket), VTABLE(TCP, "tcp", socket), - VTABLE(TCPD, "tcpd", socket), - //VTABLE(OPAL, "opal", opal ), - //VTABLE(GTFPGA, "gtfpga", gtfpga), + VTABLE(TCPD, "tcpd", socket) }; -/** Linked list of nodes */ -struct node *nodes; +/** Linked list of nodes. */ +struct list nodes; -struct node * node_lookup_name(const char *str, struct node *nodes) +struct node * node_lookup_name(const char *str, struct list *nodes) { - for (struct node *n = nodes; n; n = n->next) { - if (!strcmp(str, n->name)) - return n; + FOREACH(nodes, it) { + if (!strcmp(str, it->node->name)) + return it->node; } return NULL; @@ -57,8 +60,11 @@ struct node_vtable const * node_lookup_vtable(const char *str) } int node_start(struct node *n) -{ - int ret; +{ INDENT + if (!n->refcnt) { + warn("Node '%s' is unused. Skipping...", n->name); + return -1; + } char str[256]; node_print(n, str, sizeof(str)); @@ -66,38 +72,33 @@ int node_start(struct node *n) debug(1, "Starting node '%s' of type '%s' (%s)", n->name, n->vt->name, str); { INDENT - if (!n->refcnt) - warn("Node '%s' is not used by an active path", n->name); - - ret = n->vt->open(n); + return n->vt->open(n); } - - return ret; } int node_start_defer(struct node *n) { - switch (node_type(n)) { - case TCPD: - info("Wait for incoming TCP connection from node '%s'...", n->name); - listen(n->socket->sd, 1); - n->socket->sd = accept(n->socket->sd, NULL, NULL); - break; - - case TCP: - info("Connect with TCP to remote node '%s'", n->name); - connect(n->socket->sd, (struct sockaddr *) &n->socket->remote, sizeof(n->socket->remote)); - break; + int ret; - default: - break; + if (node_type(n) == TCPD) { + info("Wait for incoming TCP connection from node '%s'...", n->name); + + ret = listen(n->socket->sd2, 1); + if (ret < 0) + serror("Failed to listen on socket for node '%s'", n->name); + + ret = accept(n->socket->sd2, NULL, NULL); + if (ret < 0) + serror("Failed to accept on socket for node '%s'", n->name); + + n->socket->sd = ret; } return 0; } int node_stop(struct node *n) -{ +{ INDENT int ret; info("Stopping node '%s'", n->name); @@ -107,3 +108,36 @@ int node_stop(struct node *n) return ret; } + +void node_reverse(struct node *n) +{ + switch (n->vt->type) { + case IEEE_802_3: + case IP: + case UDP: + case TCP: + SWAP(n->socket->remote, n->socket->local); + break; + default: { } + } +} + +struct node * node_create() +{ + return alloc(sizeof(struct node)); +} + +void node_destroy(struct node *n) +{ + switch (n->vt->type) { + case IEEE_802_3: + case IP: + case UDP: + case TCP: + free(n->socket->netem); + default: { } + } + + free(n->socket); + free(n); +} diff --git a/server/src/opal.c b/server/src/opal.c index 8539723c0..b7783aa05 100644 --- a/server/src/opal.c +++ b/server/src/opal.c @@ -6,4 +6,228 @@ * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC */ +#include +#include + #include "opal.h" +#include "utils.h" + +/** @todo: delcare statice */ +struct opal_global *og = NULL; + +int opal_init(int argc, char *argv[]) +{ + int err; + + if (argc != 4) + return -1; + + struct opal_global *g = alloc(sizeof(struct opal_global)); + + pthread_mutex_init(&g->lock, NULL); + + g->async_shmem_name = argv[1]; + g->async_shmem_size = atoi(argv[2]); + g->print_shmem_name = argv[3]; + + /* Enable the OpalPrint function. This prints to the OpalDisplay. */ + if ((err = OpalSystemCtrl_Register(g->print_shmem_name)) != EOK) + error("OpalPrint() access not available (%d)", err); + + /* Open Share Memory created by the model. */ + if ((err = OpalOpenAsyncMem(g->async_shmem_size, g->async_shmem_name)) != EOK) + error("Model shared memory not available (%d)", err); + + if ((err = OpalGetAsyncCtrlParameters(&g->params, sizeof(Opal_GenAsyncParam_Ctrl))) != EOK) + error("Could not get OPAL controller parameters (%d)", err); + + /* Get list of Send and RecvIDs */ + if ((err = OpalGetNbAsyncSendIcon(&g->send_icons)) != EOK) + error("Failed to get number of send blocks (%d)", err); + if ((err = OpalGetNbAsyncRecvIcon(&g->recv_icons)) != EOK) + error("Failed to get number of recv blocks (%d)", err); + + g->send_ids = alloc(g->send_icons * sizeof(int)); + g->recv_ids = alloc(g->recv_icons * sizeof(int)); + + if ((err = OpalGetAsyncSendIDList(g->send_ids, g->send_icons * sizeof(int))) != EOK) + error("Failed to get list of send ids (%d)", err); + if ((err = OpalGetAsyncRecvIDList(g->recv_ids, g->recv_icons * sizeof(int))) != EOK) + error("Failed to get list of recv ids (%d)", err); + + info("Started as OPAL Asynchronous process"); + info("This is Simulator2Simulator Server (S2SS) %s (built on %s, %s, debug=%d)", + VERSION, __DATE__, __TIME__, _debug); + opal_print_global(g); + + og = g; + + return 0; +} + +int opal_deinit() +{ + int err; + + if (!og) + return 0; + + if ((err = OpalCloseAsyncMem(og->async_shmem_size, og->async_shmem_name)) != EOK) + error("Failed to close shared memory area (%d)", err); + + debug(4, "Closing OPAL shared memory mapping"); + + if ((err = OpalSystemCtrl_UnRegister(og->print_shmem_name)) != EOK) + error("Failed to close shared memory for system control (%d)", err); + + free(og->send_ids); + free(og->recv_ids); + free(og); + + og = NULL; + + return 0; +} + +int opal_print_global(struct opal_global *g) +{ INDENT + char sbuf[512] = ""; + char rbuf[512] = ""; + + for (int i=0; isend_icons; i++) + strap(sbuf, sizeof(sbuf), "%u ", g->send_ids[i]); + for (int i=0; irecv_icons; i++) + strap(rbuf, sizeof(rbuf), "%u ", g->recv_ids[i]); + + debug(2, "Controller ID: %u", g->params.controllerID); + debug(2, "Send Blocks: %s", sbuf); + debug(2, "Receive Blocks: %s", rbuf); + + debug(2, "Control Block Parameters:"); + for (int i=0; iparams.FloatParam[i]); + for (int i=0; iparams.StringParam[i]); + + return 0; +} + +int opal_print(struct node *n, char *buf, int len) +{ + struct opal *o = n->opal; + + /** @todo: Print send_params, recv_params */ + + return snprintf(buf, len, "send_id=%u, recv_id=%u, reply=%u", + o->send_id, o->recv_id, o->reply); +} + +int opal_open(struct node *n) +{ + struct opal *o = n->opal; + + OpalGetAsyncSendIconMode(&o->mode, o->send_id); + OpalGetAsyncSendParameters(&o->send_params, sizeof(Opal_SendAsyncParam), o->send_id); + OpalGetAsyncRecvParameters(&o->recv_params, sizeof(Opal_RecvAsyncParam), o->recv_id); + + return 0; +} + +int opal_close(struct node *n) +{ + return 0; +} + +int opal_read(struct node *n, struct msg *m) +{ + struct opal *o = n->opal; + + int state, len, ret; + unsigned id; + + double data[MSG_VALUES]; + + /* This call unblocks when the 'Data Ready' line of a send icon is asserted. */ + do { + if ((ret = OpalWaitForAsyncSendRequest(&id)) != EOK) { + state = OpalGetAsyncModelState(); + if ((state == STATE_RESET) || (state == STATE_STOP)) { + warn("OpalGetAsyncModelState(): Model stopped or resetted!"); + die(); + } + + return -1; // FIXME: correct return value + } + } while (id != o->send_id); + + /* No errors encountered yet */ + OpalSetAsyncSendIconError(0, o->send_id); + + /* Get the size of the data being sent by the unblocking SendID */ + OpalGetAsyncSendIconDataLength(&len, o->send_id); + if (len > sizeof(data)) { + warn("Ignoring the last %u of %u values for OPAL node '%s' (send_id=%u).", + len / sizeof(double) - MSG_VALUES, len / sizeof(double), n->name, o->send_id); + + len = sizeof(data); + } + + /* Read data from the model */ + OpalGetAsyncSendIconData(data, len, o->send_id); + + m->sequence = htons(o->seq_no++); + m->length = len / sizeof(double); + + for (int i = 0; i < m->length; i++) + m->data[i].f = (float) data[i]; // casting to float! + + /* This next call allows the execution of the "asynchronous" process + * to actually be synchronous with the model. To achieve this, you + * should set the "Sending Mode" in the Async_Send block to + * NEED_REPLY_BEFORE_NEXT_SEND or NEED_REPLY_NOW. This will force + * the model to wait for this process to call this + * OpalAsyncSendRequestDone function before continuing. */ + if (o->reply) + OpalAsyncSendRequestDone(o->send_id); + + /* Before continuing, we make sure that the real-time model + * has not been stopped. If it has, we quit. */ + state = OpalGetAsyncModelState(); + if ((state == STATE_RESET) || (state == STATE_STOP)) { + warn("OpalGetAsyncModelState(): Model stopped or resetted!"); + die(); + } + + return 0; +} + +int opal_write(struct node *n, struct msg *m) +{ + struct opal *o = n->opal; + + int state; + int len; + + double data[MSG_VALUES] = { NAN }; + + state = OpalGetAsyncModelState(); + if ((state == STATE_RESET) || (state == STATE_STOP)) { + warn("OpalGetAsyncModelState(): Model stopped or resetted!"); + die(); + } + + OpalSetAsyncRecvIconStatus(m->sequence, o->recv_id); /* Set the Status to the message ID */ + OpalSetAsyncRecvIconError(0, o->recv_id); /* Set the Error to 0 */ + + /* Get the number of signals to send back to the model */ + OpalGetAsyncRecvIconDataLength(&len, o->recv_id); + if (len > sizeof(data)) + error("Receive Block of OPAL node '%s' is expecting more signals than"); + + for (int i = 0; i < m->length; i++) + data[i] = (double) m->data[i].f; + + OpalSetAsyncRecvIconData(data, len, o->recv_id); + + return 0; +} diff --git a/server/src/path.c b/server/src/path.c index e5ee226eb..c08810433 100644 --- a/server/src/path.c +++ b/server/src/path.c @@ -16,17 +16,19 @@ #include "utils.h" #include "path.h" -#define sigev_notify_thread_id _sigev_un._tid +#ifndef sigev_notify_thread_id + #define sigev_notify_thread_id _sigev_un._tid +#endif -/** Linked list of paths */ -struct path *paths; +/** Linked list of paths. */ +struct list paths; -/** Send messages */ +/** Send messages asynchronously */ static void * path_send(void *arg) { + struct path *p = arg; + int sig; - struct path *p = (struct path *) arg; - timer_t tmr; sigset_t set; struct sigevent sev = { @@ -43,21 +45,21 @@ static void * path_send(void *arg) sigemptyset(&set); sigaddset(&set, SIGALRM); if(pthread_sigmask(SIG_BLOCK, &set, NULL)) - perror("Set signal mask"); + serror("Set signal mask"); - if (timer_create(CLOCK_REALTIME, &sev, &tmr)) - perror("Failed to create timer"); + if (timer_create(CLOCK_REALTIME, &sev, &p->timer)) + serror("Failed to create timer"); - if (timer_settime(tmr, 0, &its, NULL)) - perror("Failed to start timer"); + if (timer_settime(p->timer, 0, &its, NULL)) + serror("Failed to start timer"); while (1) { sigwait(&set, &sig); /* blocking wait for next timer tick */ - if (p->last) { - node_write(p->out, p->last); - p->last = NULL; - p->sent++; - } + + FOREACH(&p->destinations, it) + node_write(it->node, p->current); + + p->sent++; } return NULL; @@ -66,44 +68,48 @@ static void * path_send(void *arg) /** Receive messages */ static void * path_run(void *arg) { - struct path *p = (struct path *) arg; - struct msg *m = malloc(sizeof(struct msg)); - if (!m) - error("Failed to allocate memory for message!"); + struct path *p = arg; + + p->previous = alloc(sizeof(struct msg)); + p->current = alloc(sizeof(struct msg)); + char buf[33]; + /* Open deferred TCP connection */ node_start_defer(p->in); - node_start_defer(p->out); + // FIXME: node_start_defer(p->out); + /* Main thread loop */ while (1) { - node_read(p->in, m); /* Receive message */ + node_read(p->in, p->current); /* Receive message */ p->received++; /* Check header fields */ - if (m->version != MSG_VERSION || - m->type != MSG_TYPE_DATA) { + if (p->current->version != MSG_VERSION || + p->current->type != MSG_TYPE_DATA) { p->invalid++; continue; } /* Update histogram */ - int dist = (UINT16_MAX + m->sequence - p->sequence) % UINT16_MAX; + int dist = (UINT16_MAX + p->current->sequence - p->previous->sequence) % UINT16_MAX; if (dist > UINT16_MAX / 2) dist -= UINT16_MAX; - int idx = HIST_SEQ / 2 + dist; - if (idx < HIST_SEQ && idx >= 0) - p->histogram[idx]++; + hist_put(&p->histogram, dist); /* Handle simulation restart */ - if (m->sequence == 0 && abs(dist) > 16) { - path_stats(p); - warn("Simulation for path %s " MAG("=>") " %s " - "restarted (p->seq=%u, m->seq=%u, dist=%d)", - p->in->name, p->out->name, - p->sequence, m->sequence, dist); + if (p->current->sequence == 0 && abs(dist) >= 1) { + if (p->received) { + path_print_stats(p); + hist_print(&p->histogram); + } + + path_print(p, buf, sizeof(buf)); + warn("Simulation for path %s restarted (prev->seq=%u, current->seq=%u, dist=%d)", + buf, p->previous->sequence, p->current->sequence, dist); /* Reset counters */ p->sent = 0; @@ -112,8 +118,7 @@ static void * path_run(void *arg) p->skipped = 0; p->dropped = 0; - /* Reset sequence no tracking */ - memset(p->histogram, 0, sizeof(p->histogram)); + hist_reset(&p->histogram); } else if (dist <= 0 && p->received > 1) { p->dropped++; @@ -121,41 +126,47 @@ static void * path_run(void *arg) } /* Call hook callbacks */ - if (p->hook && p->hook(m, p)) { - p->skipped++; - continue; + FOREACH(&p->hooks, it) { + if (it->hook(p->current, p)) { + p->skipped++; + continue; + } } - /* Update last known sequence number */ - p->sequence = m->sequence; - p->last = m; - /* At fixed rate mode, messages are send by another thread */ if (!p->rate) { - node_write(p->out, m); /* Send message */ + FOREACH(&p->destinations, it) + node_write(it->node, p->current); + p->sent++; } - } - free(m); + SWAP(p->previous, p->current); + } return NULL; } int path_start(struct path *p) { INDENT - info("Starting path: %12s " GRN("=>") " %-12s", p->in->name, p->out->name); + char buf[33]; + path_print(p, buf, sizeof(buf)); + + info("Starting path: %s", buf); /* At fixed rate mode, we start another thread for sending */ if (p->rate) - pthread_create(&p->sent_tid, NULL, &path_send, (void *) p); + pthread_create(&p->sent_tid, NULL, &path_send, p); - return pthread_create(&p->recv_tid, NULL, &path_run, (void *) p); + return pthread_create(&p->recv_tid, NULL, &path_run, p); } int path_stop(struct path *p) { INDENT - info("Stopping path: %12s " RED("=>") " %-12s", p->in->name, p->out->name); + char buf[33]; + path_print(p, buf, sizeof(buf)); + + info("Stopping path: %s", buf); pthread_cancel(p->recv_tid); pthread_join(p->recv_tid, NULL); @@ -163,21 +174,60 @@ int path_stop(struct path *p) if (p->rate) { pthread_cancel(p->sent_tid); pthread_join(p->sent_tid, NULL); + + timer_delete(p->timer); } - if (p->received) { - path_stats(p); - hist_plot(p->histogram, HIST_SEQ); - hist_dump(p->histogram, HIST_SEQ); - } + if (p->received) + hist_print(&p->histogram); return 0; } -void path_stats(struct path *p) +void path_print_stats(struct path *p) { - info("%12s " MAG("=>") " %-12s: %-8u %-8u %-8u %-8u %-8u", - p->in->name, p->out->name, - p->sent, p->received, p->dropped, p->skipped, p->invalid - ); + char buf[33]; + path_print(p, buf, sizeof(buf)); + + info("%-32s : %-8u %-8u %-8u %-8u %-8u", buf, + p->sent, p->received, p->dropped, p->skipped, p->invalid); +} + +int path_print(struct path *p, char *buf, int len) +{ + *buf = 0; + + if (list_length(&p->destinations) > 1) { + strap(buf, len, "%s " MAG("=>") " [", p->in->name); + FOREACH(&p->destinations, it) + strap(buf, len, " %s", it->node->name); + strap(buf, len, " ]"); + } + else + strap(buf, len, "%s " MAG("=>") " %s", p->in->name, p->out->name); + + return 0; +} + +struct path * path_create() +{ + struct path *p = alloc(sizeof(struct path)); + + list_init(&p->destinations, NULL); + list_init(&p->hooks, NULL); + + hist_create(&p->histogram, -HIST_SEQ, +HIST_SEQ, 1); + + return p; +} + +void path_destroy(struct path *p) +{ + list_destroy(&p->destinations); + list_destroy(&p->hooks); + hist_destroy(&p->histogram); + + free(p->current); + free(p->previous); + free(p); } diff --git a/server/src/random.c b/server/src/random.c index 7e3849cd6..4d08c0844 100644 --- a/server/src/random.c +++ b/server/src/random.c @@ -22,7 +22,7 @@ void tick(int sig, siginfo_t *si, void *ptr) { - struct msg *m = (struct msg*) si->si_value.sival_ptr; + struct msg *m = (struct msg *) si->si_value.sival_ptr; msg_random(m); msg_fprint(stdout, m); @@ -37,9 +37,12 @@ int main(int argc, char *argv[]) printf("Usage: %s VALUES RATE\n", argv[0]); printf(" VALUES is the number of values a message contains\n"); printf(" RATE how many messages per second\n\n"); - printf("Simulator2Simulator Server %s (built on %s %s)\n", BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); - printf(" Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); - printf(" Steffen Vogel \n"); + + printf("Simulator2Simulator Server %s (built on %s %s)\n", + BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); + printf(" Copyright 2015, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Steffen Vogel \n"); + exit(EXIT_FAILURE); } diff --git a/server/src/receive.c b/server/src/receive.c index e96c5e286..378311073 100644 --- a/server/src/receive.c +++ b/server/src/receive.c @@ -24,7 +24,7 @@ static struct settings set; static struct msg msg = MSG_INIT(0); -extern struct node *nodes; +extern struct list nodes; static struct node *node; void quit(int sig, siginfo_t *si, void *ptr) @@ -33,21 +33,39 @@ void quit(int sig, siginfo_t *si, void *ptr) exit(EXIT_SUCCESS); } +void usage(char *name) +{ + printf("Usage: %s [-r] CONFIG NODE\n", name); + printf(" -r swap local / remote address of socket based nodes)\n\n"); + printf(" CONFIG path to a configuration file\n"); + printf(" NODE name of the node which shoud be used\n\n"); + + printf("Simulator2Simulator Server %s (built on %s %s)\n", + BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); + printf(" Copyright 2015, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Steffen Vogel \n"); + + exit(EXIT_FAILURE); +} + int main(int argc, char *argv[]) { + char c; + int reverse = 0; + struct config_t config; - - if (argc != 2) { - printf("Usage: %s CONFIG NODE\n", argv[0]); - printf(" CONFIG path to a configuration file\n"); - printf(" NODE name of the node which shoud be used\n\n"); - printf("Simulator2Simulator Server %s (built on %s %s)\n", - BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); - printf(" Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); - printf(" Steffen Vogel \n"); - exit(EXIT_FAILURE); + + while ((c = getopt(argc, argv, "hr")) != -1) { + switch (c) { + case 'r': reverse = 1; break; + case 'h': + case '?': usage(argv[0]); + } } + if (argc - optind != 2) + usage(argv[0]); + /* Setup signals */ struct sigaction sa_quit = { .sa_flags = SA_SIGINFO, @@ -59,11 +77,16 @@ int main(int argc, char *argv[]) sigaction(SIGINT, &sa_quit, NULL); config_init(&config); - config_parse(argv[1], &config, &set, &nodes, NULL); + config_parse(argv[optind], &config, &set, &nodes, NULL); - node = node_lookup_name(argv[2], nodes); + node = node_lookup_name(argv[optind+1], &nodes); if (!node) - error("There's no node with the name '%s'", argv[2]); + error("There's no node with the name '%s'", argv[optind+1]); + + node->refcnt++; + + if (reverse) + node_reverse(node); node_start(node); node_start_defer(node); diff --git a/server/src/send.c b/server/src/send.c index 51db9cc99..a4f841e7a 100644 --- a/server/src/send.c +++ b/server/src/send.c @@ -27,7 +27,7 @@ static struct settings set; static struct msg msg = MSG_INIT(0); static struct node *node; -extern struct node *nodes; +extern struct list nodes; void quit(int sig, siginfo_t *si, void *ptr) { @@ -35,20 +35,38 @@ void quit(int sig, siginfo_t *si, void *ptr) exit(EXIT_SUCCESS); } +void usage(char *name) +{ + printf("Usage: %s [-r] CONFIG NODE\n", name); + printf(" -r swap local / remote address of socket based nodes)\n\n"); + printf(" CONFIG path to a configuration file\n"); + printf(" NODE name of the node which shoud be used\n"); + + printf("Simulator2Simulator Server %s (built on %s %s)\n", + BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); + printf(" Copyright 2015, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Steffen Vogel \n"); + + exit(EXIT_FAILURE); +} + int main(int argc, char *argv[]) { + char c; + int reverse = 0; + struct config_t config; - - if (argc != 3) { - printf("Usage: %s CONFIG NODE\n", argv[0]); - printf(" CONFIG path to a configuration file\n"); - printf(" NODE name of the node which shoud be used\n\n"); - printf("Simulator2Simulator Server %s (built on %s %s)\n", - BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); - printf(" Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); - printf(" Steffen Vogel \n"); - exit(EXIT_FAILURE); + + while ((c = getopt(argc, argv, "hr")) != -1) { + switch (c) { + case 'r': reverse = 1; break; + case 'h': + case '?': usage(argv[0]); + } } + + if (argc - optind != 2) + usage(argv[0]); /* Setup signals */ struct sigaction sa_quit = { @@ -61,11 +79,16 @@ int main(int argc, char *argv[]) sigaction(SIGINT, &sa_quit, NULL); config_init(&config); - config_parse(argv[1], &config, &set, &nodes, NULL); + config_parse(argv[optind], &config, &set, &nodes, NULL); - node = node_lookup_name(argv[2], nodes); + node = node_lookup_name(argv[optind+1], &nodes); if (!node) - error("There's no node with the name '%s'", argv[2]); + error("There's no node with the name '%s'", argv[optind+1]); + + node->refcnt++; + + if (reverse) + node_reverse(node); node_start(node); node_start_defer(node); diff --git a/server/src/server.c b/server/src/server.c index fb9d77485..ccd1e1018 100644 --- a/server/src/server.c +++ b/server/src/server.c @@ -22,55 +22,57 @@ #include "path.h" #include "node.h" +#ifdef ENABLE_OPAL_ASYNC +#include "opal.h" +#endif + /** Linked list of nodes */ -extern struct node *nodes; +extern struct list nodes; /** Linked list of paths */ -extern struct path *paths; +extern struct list paths; /** Linked list of interfaces */ -extern struct interface *interfaces; +extern struct list interfaces; /** The global configuration */ -struct settings settings; -config_t config; +static struct settings settings; +static config_t config; static void quit() -{ _indent = 0; +{ info("Stopping paths:"); - for (struct path *p = paths; p; p = p->next) { INDENT - path_stop(p); - } + FOREACH(&paths, it) + path_stop(it->path); info("Stopping nodes:"); - for (struct node *n = nodes; n; n = n->next) { INDENT - node_stop(n); - } + FOREACH(&nodes, it) + node_stop(it->node); info("Stopping interfaces:"); - for (struct interface *i = interfaces; i; i = i->next) { INDENT - if_stop(i); - } + FOREACH(&interfaces, it) + if_stop(it->interface); - /** @todo Free nodes and paths */ +#ifdef ENABLE_OPAL_ASYNC + opal_deinit(); +#endif + /* Freeing dynamically allocated memory */ + list_destroy(&paths); + list_destroy(&nodes); + list_destroy(&interfaces); config_destroy(&config); + info("Goodbye!"); + _exit(EXIT_SUCCESS); } void realtime_init() { INDENT - /* Check for realtime kernel patch */ - struct stat st; - if (stat("/sys/kernel/realtime", &st)) - warn("Use a RT-preempt patched Linux for lower latencies!"); - else - info("Server is running on a realtime patched kernel"); - - /* Use FIFO scheduler with realtime priority */ + /* Use FIFO scheduler with real time priority */ if (settings.priority) { struct sched_param param = { .sched_priority = settings.priority }; if (sched_setscheduler(0, SCHED_FIFO, ¶m)) - perror("Failed to set realtime priority"); + serror("Failed to set real time priority"); else debug(3, "Set task priority to %u", settings.priority); } @@ -79,7 +81,7 @@ void realtime_init() if (settings.affinity) { cpu_set_t cset = to_cpu_set(settings.affinity); if (sched_setaffinity(0, sizeof(cset), &cset)) - perror("Failed to set CPU affinity to '%#x'", settings.affinity); + serror("Failed to set CPU affinity to '%#x'", settings.affinity); else debug(3, "Set affinity to %#x", settings.affinity); } @@ -94,83 +96,105 @@ void signals_init() }; sigemptyset(&sa_quit.sa_mask); + sigaction(SIGQUIT, &sa_quit, NULL); sigaction(SIGTERM, &sa_quit, NULL); sigaction(SIGINT, &sa_quit, NULL); - atexit(&quit); } void usage(const char *name) { printf("Usage: %s CONFIG\n", name); printf(" CONFIG is a required path to a configuration file\n\n"); - printf("Simulator2Simulator Server %s (built on %s, %s)\n", +#ifdef ENABLE_OPAL_ASYNC + printf("Usage: %s OPAL_ASYNC_SHMEM_NAME OPAL_ASYNC_SHMEM_SIZE OPAL_PRINT_SHMEM_NAME\n", name); + printf(" This type of invocation is used by OPAL-RT Asynchronous processes.\n"); + printf(" See in the RT-LAB User Guide for more information.\n\n"); +#endif + printf("Simulator2Simulator Server %s (built on %s %s)\n", BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); - + printf(" Copyright 2015, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Steffen Vogel \n"); + exit(EXIT_FAILURE); } int main(int argc, char *argv[]) { - epoch_reset(); - info("This is Simulator2Simulator Server (S2SS) %s (built on %s, %s)", - BLD(YEL(VERSION)), BLD(MAG(__DATE__)), BLD(MAG(__TIME__))); + _mtid = pthread_self(); /* Check arguments */ +#ifdef ENABLE_OPAL_ASYNC + if (argc != 2 && argc != 4) +#else if (argc != 2) +#endif usage(argv[0]); + info("This is Simulator2Simulator Server (S2SS) %s (built on %s, %s, debug=%d)", + BLD(YEL(VERSION)), BLD(MAG(__DATE__)), BLD(MAG(__TIME__)), _debug); + /* Check priviledges */ if (getuid() != 0) error("The server requires superuser privileges!"); - /* Start initialization */ + /* Initialize lists */ + list_init(&nodes, (dtor_cb_t) node_destroy); + list_init(&paths, (dtor_cb_t) path_destroy); + list_init(&interfaces, (dtor_cb_t) if_destroy); + + info("Initialize realtime system:"); realtime_init(); + info("Setup signals:"); signals_init(); + info("Parsing configuration:"); config_init(&config); - /* Parse configuration and create nodes/paths */ - config_parse(argv[1], &config, &settings, &nodes, &paths); +#ifdef ENABLE_OPAL_ASYNC + /* Check if called we are called as an asynchronous process from RT-LAB. */ + opal_init(argc, argv); + /* @todo: look in predefined locations for a file */ + char *configfile = "opal-shmem.conf"; +#else + char *configfile = argv[1]; +#endif + + /* Parse configuration and create nodes/paths */ + config_parse(configfile, &config, &settings, &nodes, &paths); /* Connect all nodes and start one thread per path */ info("Starting nodes:"); - for (struct node *n = nodes; n; n = n->next) { INDENT - node_start(n); - } + FOREACH(&nodes, it) + node_start(it->node); info("Starting interfaces:"); - for (struct interface *i = interfaces; i; i = i->next) { INDENT - if_start(i, settings.affinity); - } + FOREACH(&interfaces, it) + if_start(it->interface, settings.affinity); - info("Starting pathes:"); - for (struct path *p = paths; p; p = p->next) { INDENT - path_start(p); - } + info("Starting paths:"); + FOREACH(&paths, it) + path_start(it->path); /* Run! */ if (settings.stats > 0) { - struct path *p = paths; - info("Runtime Statistics:"); - info("%12s " MAG("=>") " %-12s: %-8s %-8s %-8s %-8s %-8s", - "Source", "Destination", "#Sent", "#Recv", "#Drop", "#Skip", "#Inval"); + info("%-32s : %-8s %-8s %-8s %-8s %-8s", + "Source " MAG("=>") " Destination", "#Sent", "#Recv", "#Drop", "#Skip", "#Inval"); info("---------------------------------------------------------------------------"); - while (1) { + do { FOREACH(&paths, it) { usleep(settings.stats * 1e6); - path_stats(p); + path_print_stats(it->path); + } } while (1); - p = (p->next) ? p->next : paths; - } } else pause(); - /* Note: quit() is called by exit handler! */ + quit(); return 0; } diff --git a/server/src/socket.c b/server/src/socket.c index 96dc7acab..3e4fc85d6 100644 --- a/server/src/socket.c +++ b/server/src/socket.c @@ -34,8 +34,8 @@ int socket_print(struct node *n, char *buf, int len) char local[INET6_ADDRSTRLEN + 16]; char remote[INET6_ADDRSTRLEN + 16]; - socket_print_addr(local, sizeof(local), (struct sockaddr*) &s->local); - socket_print_addr(remote, sizeof(remote), (struct sockaddr*) &s->remote); + socket_print_addr(local, sizeof(local), (struct sockaddr *) &s->local); + socket_print_addr(remote, sizeof(remote), (struct sockaddr *) &s->remote); return snprintf(buf, len, "local=%s, remote=%s", local, remote); } @@ -43,25 +43,39 @@ int socket_print(struct node *n, char *buf, int len) int socket_open(struct node *n) { struct socket *s = n->socket; - int af = s->local.ss_family; - + struct sockaddr_in *sin = (struct sockaddr_in *) &s->local; + struct sockaddr_ll *sll = (struct sockaddr_ll *) &s->local; + int ret; + /* Create socket */ switch (node_type(n)) { case TCPD: - case TCP: s->sd = socket(af, SOCK_STREAM, 0); break; - case UDP: s->sd = socket(af, SOCK_DGRAM, 0); break; - case IP: s->sd = socket(af, SOCK_RAW, IPPROTO_S2SS); break; - case IEEE_802_3:s->sd = socket(af, SOCK_DGRAM, ETH_P_S2SS); break; + case TCP: s->sd = socket(sin->sin_family, SOCK_STREAM, IPPROTO_TCP); break; + case UDP: s->sd = socket(sin->sin_family, SOCK_DGRAM, IPPROTO_UDP); break; + case IP: s->sd = socket(sin->sin_family, SOCK_RAW, ntohs(sin->sin_port)); break; + case IEEE_802_3:s->sd = socket(sin->sin_family, SOCK_DGRAM, sll->sll_protocol); break; default: error("Invalid socket type!"); } if (s->sd < 0) - perror("Failed to create socket"); + serror("Failed to create socket"); /* Bind socket for receiving */ - if (bind(s->sd, (struct sockaddr *) &s->local, sizeof(s->local))) - perror("Failed to bind to socket"); + ret = bind(s->sd, (struct sockaddr *) &s->local, sizeof(s->local)); + if (ret < 0) + serror("Failed to bind socket"); + + /* Connect socket for sending */ + if (node_type(n) == TCPD) { + /* Listening TCP sockets will be connected later by calling accept() */ + s->sd2 = s->sd; + } + else if (node_type(n) != IEEE_802_3) { + ret = connect(s->sd, (struct sockaddr *) &s->remote, sizeof(s->remote)); + if (ret < 0) + serror("Failed to connect socket"); + } /* Determine outgoing interface */ int index = if_getegress((struct sockaddr *) &s->remote); @@ -72,7 +86,7 @@ int socket_open(struct node *n) if (!i) i = if_create(index); - list_add(i->sockets, s); + list_push(&i->sockets, s); i->refcnt++; /* Set socket priority, QoS or TOS IP options */ @@ -84,7 +98,7 @@ int socket_open(struct node *n) case IP: prio = IPTOS_LOWDELAY; if (setsockopt(s->sd, IPPROTO_IP, IP_TOS, &prio, sizeof(prio))) - perror("Failed to set type of service (QoS)"); + serror("Failed to set type of service (QoS)"); else debug(4, "Set QoS/TOS IP option for node '%s' to %#x", n->name, prio); break; @@ -92,7 +106,7 @@ int socket_open(struct node *n) default: prio = SOCKET_PRIO; if (setsockopt(s->sd, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio))) - perror("Failed to set socket priority"); + serror("Failed to set socket priority"); else debug(4, "Set socket priority for node '%s' to %u", n->name, prio); break; @@ -103,20 +117,25 @@ int socket_open(struct node *n) int socket_close(struct node *n) { - return close(n->socket->sd); + struct socket *s = n->socket; + + if (s->sd >= 0) + close(s->sd); + + if (s->sd2 >= 0) + close(s->sd2); + + return 0; } -int socket_read(struct node* n, struct msg *m) +int socket_read(struct node *n, struct msg *m) { - /** @todo Fix this for multiple paths calling msg_recv. */ - /* Receive message from socket */ - if (recv(n->socket->sd, m, sizeof(struct msg), 0) < 0) { - if (errno == EINTR) - return -EINTR; - - perror("Failed recv"); - } + int ret = recv(n->socket->sd, m, sizeof(struct msg), 0); + if (ret == 0) + error("Remote node '%s' closed the connection", n->name); + else if (ret < 0) + serror("Failed recv"); /* Convert headers to host byte order */ m->sequence = ntohs(m->sequence); @@ -131,15 +150,21 @@ int socket_read(struct node* n, struct msg *m) return 0; } -int socket_write(struct node* n, struct msg *m) +int socket_write(struct node *n, struct msg *m) { + struct socket *s = n->socket; + int ret; + /* Convert headers to network byte order */ m->sequence = htons(m->sequence); - if (sendto(n->socket->sd, m, MSG_LEN(m->length), 0, - (struct sockaddr *) &n->socket->remote, - sizeof(struct sockaddr_in)) < 0) - perror("Failed sendto"); + if (node_type(n) == IEEE_802_3) + ret = sendto(s->sd, m, MSG_LEN(m->length), 0, (struct sockaddr *) &s->remote, sizeof(s->remote)); + else + ret = send(s->sd, m, MSG_LEN(m->length), 0); + + if (ret < 0) + serror("Failed send(to)"); debug(10, "Message sent to node '%s': version=%u, type=%u, endian=%u, length=%u, sequence=%u", n->name, m->version, m->type, m->endian, m->length, ntohs(m->sequence)); @@ -160,14 +185,14 @@ int socket_print_addr(char *buf, int len, struct sockaddr *sa) struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; char ifname[IF_NAMESIZE]; - return snprintf(buf, len, "%s%%%s:%hu", + return snprintf(buf, len, "%s%%%s:%#hx", ether_ntoa((struct ether_addr *) &sll->sll_addr), if_indextoname(sll->sll_ifindex, ifname), ntohs(sll->sll_protocol)); } default: - error("Unsupported address family"); + error("Unsupported address family: %u", sa->sa_family); } return 0; @@ -185,7 +210,8 @@ int socket_parse_addr(const char *addr, struct sockaddr *sa, enum node_type type /* Split string */ char *node = strtok(copy, "%"); - char *ifname = strtok(NULL, "\0"); + char *ifname = strtok(NULL, ":"); + char *proto = strtok(NULL, "\0"); /* Parse link layer (MAC) address */ struct ether_addr *mac = ether_aton(node); @@ -194,18 +220,17 @@ int socket_parse_addr(const char *addr, struct sockaddr *sa, enum node_type type memcpy(&sll->sll_addr, &mac->ether_addr_octet, 6); - sll->sll_protocol = ETH_P_S2SS; + sll->sll_protocol = htons((proto) ? strtol(proto, NULL, 0) : ETH_P_S2SS); sll->sll_halen = 6; sll->sll_family = AF_PACKET; sll->sll_ifindex = if_nametoindex(ifname); ret = 0; } - else { - //struct sockaddr_in *sin = (struct sockaddr_in *) sa; + else { /* Format: "192.168.0.10:12001" */ struct addrinfo hint = { .ai_flags = flags, - .ai_family = AF_UNSPEC + .ai_family = AF_INET }; /* Split string */ @@ -220,8 +245,9 @@ int socket_parse_addr(const char *addr, struct sockaddr *sa, enum node_type type switch (type) { case IP: - hint.ai_socktype = 0; - hint.ai_protocol = IPPROTO_S2SS; + hint.ai_socktype = SOCK_RAW; + hint.ai_protocol = (service) ? strtol(service, NULL, 0) : IPPROTO_S2SS; + hint.ai_flags |= AI_NUMERICSERV; break; case TCPD: @@ -235,16 +261,23 @@ int socket_parse_addr(const char *addr, struct sockaddr *sa, enum node_type type hint.ai_protocol = IPPROTO_UDP; break; - case INVALID: default: error("Invalid address type"); } /* Lookup address */ struct addrinfo *result; - ret = getaddrinfo(node, service, &hint, &result); + ret = getaddrinfo(node, (type == IP) ? NULL : service, &hint, &result); if (!ret) { - memcpy(sa, result->ai_addr, result->ai_addrlen); + + if (type == IP) { + /* We mis-use the sin_port field to store the IP protocol number on RAW sockets */ + struct sockaddr_in *sin = (struct sockaddr_in *) result->ai_addr; + sin->sin_port = htons(result->ai_protocol); + } + + memcpy(sa, result->ai_addr, result->ai_addrlen); + freeaddrinfo(result); } } diff --git a/server/src/test.c b/server/src/test.c index 9fa826c89..2deec58bf 100644 --- a/server/src/test.c +++ b/server/src/test.c @@ -13,25 +13,45 @@ #include #include #include +#include +#include +#include #include "config.h" #include "cfg.h" #include "msg.h" #include "node.h" #include "utils.h" +#include "hist.h" static struct settings set; static struct node *node; -extern struct node *nodes; +extern struct list nodes; +/* Test options */ int running = 1; -#define CLOCK_ID CLOCK_MONOTONIC_RAW +/** Amount of messages which should be sent (default: -1 for unlimited) */ +int count = -1; -#define RTT_MIN 20 -#define RTT_MAX 100 -#define RTT_RESOLUTION 2 -#define RTT_HIST (int) ((RTT_MAX - RTT_MIN) / RTT_RESOLUTION) +/** File descriptor for Matlab results. + * This allows you to write Matlab results in a seperate log file: + * + * ./test etc/example.conf rtt -f 3 3>> measurement_results.m + */ +int fd = STDOUT_FILENO; + +/** Lowest value in histogram. */ +double low = 0; +/** Highest value in histogram. */ +double high = 2e-4; +/** Histogram resolution. */ +double res = 1e-5; + +#define CLOCK_ID CLOCK_MONOTONIC + +/* Prototypes */ +void test_rtt(); void quit(int sig, siginfo_t *si, void *ptr) { @@ -42,14 +62,17 @@ int main(int argc, char *argv[]) { config_t config; - if (argc != 4) { - printf("Usage: %s CONFIG NODE\n", argv[0]); + if (argc < 4) { + printf("Usage: %s CONFIG TEST NODE [ARGS]\n", argv[0]); printf(" CONFIG path to a configuration file\n"); + printf(" TEST the name of the test to execute: 'rtt'\n"); printf(" NODE name of the node which shoud be used\n\n"); + printf("Simulator2Simulator Server %s (built on %s %s)\n", BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); - printf(" Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); - printf(" Steffen Vogel \n"); + printf(" Copyright 2015, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Steffen Vogel \n"); + exit(EXIT_FAILURE); } @@ -66,70 +89,119 @@ int main(int argc, char *argv[]) config_init(&config); config_parse(argv[1], &config, &set, &nodes, NULL); - node = node_lookup_name(argv[2], nodes); + node = node_lookup_name(argv[3], &nodes); if (!node) - error("There's no node with the name '%s'", argv[2]); + error("There's no node with the name '%s'", argv[3]); + node->refcnt++; node_start(node); node_start_defer(node); - - if (!strcmp(argv[1], "rtt")) { - struct msg m = MSG_INIT(sizeof(struct timespec) / sizeof(float)); - struct timespec *ts1 = (struct timespec *) &m.data; - struct timespec *ts2 = malloc(sizeof(struct timespec)); - - double rtt; - double rtt_max = LLONG_MIN; - double rtt_min = LLONG_MAX; - double avg = 0; - int bar; - unsigned hist[RTT_HIST]; - - - memset(hist, 0, RTT_HIST * sizeof(unsigned)); - -#if 1 /* Print header */ - fprintf(stdout, "%17s", "timestamp"); -#endif - fprintf(stdout, "%5s%10s%10s%10s%10s\n", "seq", "rtt", "min", "max", "avg"); - - while (running) { - clock_gettime(CLOCK_ID, ts1); - node_write(node, &m); - node_read(node, &m); - clock_gettime(CLOCK_ID, ts2); - - rtt = timespec_delta(ts1, ts2); - - if (rtt < 0) continue; - if (rtt > rtt_max) rtt_max = rtt; - if (rtt < rtt_min) rtt_min = rtt; - - avg += rtt; - - /* Update histogram */ - bar = (rtt * 1000 / RTT_RESOLUTION) - (RTT_MIN / RTT_RESOLUTION); - if (bar < RTT_HIST) - hist[bar]++; - -#if 1 - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - fprintf(stdout, "%17.6f", ts.tv_sec + ts.tv_nsec / 1e9); -#endif - m.sequence++; - - fprintf(stdout, "%5u%10.3f%10.3f%10.3f%10.3f\n", m.sequence, - 1e3 * rtt, 1e3 * rtt_min, 1e3 * rtt_max, 1e3 * avg / m.sequence); + + /* Parse Arguments */ + char c; + char *endptr; + while ((c = getopt (argc-3, argv+3, "l:h:r:f:c:")) != -1) { + switch (c) { + case 'c': + count = strtoul(optarg, &endptr, 10); + goto check; + case 'f': + fd = strtoul(optarg, &endptr, 10); + goto check; + case 'l': + low = strtod(optarg, &endptr); + goto check; + case 'h': + high = strtod(optarg, &endptr); + goto check; + case 'r': + res = strtod(optarg, &endptr); + goto check; + case '?': + if (optopt == 'c') + error("Option -%c requires an argument.", optopt); + else if (isprint(optopt)) + error("Unknown option '-%c'.", optopt); + else + error("Unknown option character '\\x%x'.", optopt); + exit(EXIT_FAILURE); + default: + abort(); } - - free(ts2); - - hist_plot(hist, RTT_HIST); - hist_dump(hist, RTT_HIST); + + continue; +check: + if (optarg == endptr) + error("Failed to parse parse option argument '-%c %s'", c, optarg); } + if (!strcmp(argv[2], "rtt")) + test_rtt(); + else + error("Unknown test: '%s'", argv[2]); + node_stop(node); + config_destroy(&config); return 0; } + +void test_rtt() { + struct msg m = MSG_INIT(sizeof(struct timespec) / sizeof(float)); + struct timespec *ts1 = (struct timespec *) &m.data; + struct timespec *ts2 = alloc(sizeof(struct timespec)); + + double rtt; + double rtt_max = LLONG_MIN; + double rtt_min = LLONG_MAX; + double avg = 0; + + struct hist histogram; + hist_create(&histogram, low, high, res); + +#if 1 /* Print header */ + fprintf(stdout, "%17s", "timestamp"); +#endif + fprintf(stdout, "%5s%10s%10s%10s%10s\n", "seq", "rtt", "min", "max", "avg"); + + while (running && (count < 0 || count--)) { + clock_gettime(CLOCK_ID, ts1); + node_write(node, &m); + node_read(node, &m); + clock_gettime(CLOCK_ID, ts2); + + rtt = timespec_delta(ts1, ts2); + + if (rtt < 0) continue; + if (rtt > rtt_max) rtt_max = rtt; + if (rtt < rtt_min) rtt_min = rtt; + + avg += rtt; + + hist_put(&histogram, rtt); + +#if 1 + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + fprintf(stdout, "%17.6f", ts.tv_sec + ts.tv_nsec / 1e9); +#endif + m.sequence++; + + fprintf(stdout, "%5u%10.3f%10.3f%10.3f%10.3f\n", m.sequence, + 1e3 * rtt, 1e3 * rtt_min, 1e3 * rtt_max, 1e3 * avg / m.sequence); + } + + free(ts2); + + hist_print(&histogram); + + struct stat st; + if (!fstat(fd, &st)) { + FILE *f = fdopen(fd, "w"); + hist_matlab(&histogram, f); + } + else + error("Invalid file descriptor: %u", fd); + + hist_destroy(&histogram); +} diff --git a/server/src/utils.c b/server/src/utils.c index 0bbf3ff25..d30dd2503 100644 --- a/server/src/utils.c +++ b/server/src/utils.c @@ -12,58 +12,42 @@ #include #include #include -#include #include +#include + +#ifdef ENABLE_OPAL_ASYNC +#define RTLAB +#include +#endif #include "config.h" #include "cfg.h" #include "utils.h" -/* This global variable contains the debug level for debug() and assert() macros */ -int _debug = V; -int _indent = 0; +pthread_t _mtid; -struct timespec epoch; - -void outdent(int *old) +void die() { - _indent = *old; + pthread_kill(_mtid, SIGINT); } -void epoch_reset() +int strap(char *dest, size_t size, const char *fmt, ...) { - clock_gettime(CLOCK_REALTIME, &epoch); -} - -void print(enum log_level lvl, const char *fmt, ...) -{ - struct timespec ts; - + int ret; + va_list ap; va_start(ap, fmt); - - /* Timestamp */ - clock_gettime(CLOCK_REALTIME, &ts); - fprintf(stderr, "%8.3f ", timespec_delta(&epoch, &ts)); - - switch (lvl) { - case DEBUG: fprintf(stderr, BLD("%-5s "), GRY("Debug")); break; - case INFO: fprintf(stderr, BLD("%-5s "), " " ); break; - case WARN: fprintf(stderr, BLD("%-5s "), YEL(" Warn")); break; - case ERROR: fprintf(stderr, BLD("%-5s "), RED("Error")); break; - } - - if (_indent) { - for (int i = 0; i < _indent-1; i++) - fprintf(stderr, GFX("\x78") " "); - - fprintf(stderr, GFX("\x74") " "); - } - - vfprintf(stderr, fmt, ap); - fprintf(stderr, "\n"); - + ret = vstrap(dest, size, fmt, ap); va_end(ap); + + return ret; +} + +int vstrap(char *dest, size_t size, const char *fmt, va_list ap) +{ + int len = strlen(dest); + + return vsnprintf(dest + len, size - len, fmt, ap); } cpu_set_t to_cpu_set(int set) @@ -80,6 +64,17 @@ cpu_set_t to_cpu_set(int set) return cset; } +void * alloc(size_t bytes) +{ + void *p = malloc(bytes); + if (!p) + error("Failed to allocate memory"); + + memset(p, 0, bytes); + + return p; +} + double timespec_delta(struct timespec *start, struct timespec *end) { double sec = end->tv_sec - start->tv_sec; @@ -103,49 +98,6 @@ struct timespec timespec_rate(double rate) return ts; } -void hist_plot(unsigned *hist, int length) -{ - char buf[HIST_HEIGHT + 32]; - int bar; - int max = 0; - - /* Get max, first & last */ - for (int i = 0; i < length; i++) { - if (hist[i] > hist[max]) - max = i; - } - - /* Print header */ - info("%2s | %5s | %s", "Id", "Value", "Histogram Plot:"); - - /* Print plot */ - memset(buf, '#', sizeof(buf)); - for (int i = 0; i < length; i++) { - bar = HIST_HEIGHT * (float) hist[i] / hist[max]; - if (hist[i] == 0) - info("%2u | " GRN("%5u") " | " , i, hist[i]); - else if (hist[i] == hist[max]) - info("%2u | " RED("%5u") " | " BLD("%.*s"), i, hist[i], bar, buf); - else - info("%2u | " "%5u" " | " "%.*s", i, hist[i], bar, buf); - } -} - -void hist_dump(unsigned *hist, int length) -{ - char tok[16]; - char buf[length * sizeof(tok)]; - memset(buf, 0, sizeof(buf)); - - /* Print in Matlab vector format */ - for (int i = 0; i < length; i++) { - snprintf(tok, sizeof(tok), "%u ", hist[i]); - strncat(buf, tok, sizeof(buf)-strlen(buf)); - } - - info("Matlab: hist = [ %s]", buf); -} - /** @todo: Proper way: create additional pipe for stderr in child process */ int system2(const char *cmd, ...) { @@ -163,7 +115,7 @@ int system2(const char *cmd, ...) FILE *f = popen(buf, "r"); if (f == NULL) - perror("Failed to execute: '%s'", cmd); + serror("Failed to execute: '%s'", cmd); while (!feof(f) && fgets(buf, sizeof(buf), f) != NULL) { INDENT strtok(buf, "\n"); /* strip trailing newline */