diff --git a/server/Makefile b/server/Makefile index bf9752125..0ca0dab1c 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,26 +1,15 @@ TARGETS = server send random receive test -# Default target: build everything -all: $(TARGETS) - # Common dependencies for all binaries OBJS = socket.o if.o utils.o msg.o node.o cfg.o tc.o hooks.o list.o path.o hist.o -# Dependencies for individual binaries -server: server.o $(OBJS) -send: send.o $(OBJS) -receive: receive.o $(OBJS) -random: random.o $(OBJS) -test: test.o $(OBJS) - -# Search path for source files VPATH = src # Default debug level V ?= 2 # Compiler and linker flags -LDFLAGS = -pthread -lrt -lm -lconfig +LDLIBS = -pthread -lrt -lm -lconfig CFLAGS = -std=c99 -Iinclude/ -MMD -Wall CFLAGS += -D_XOPEN_SOURCE=500 -D_GNU_SOURCE -DV=$(V) CFLAGS += -D__GIT_REV__='"-$(shell git rev-parse --short HEAD)"' @@ -32,8 +21,28 @@ else CFLAGS += -O3 endif +# Enable OPAL-RT Asynchronous Process support +#OPALDIR = /usr/opalrt/common +OPALDIR = ../opal +ifneq (,$(wildcard $(OPALDIR)/include_target/AsyncApi.h)) + CFLAGS += -m32 -DENABLE_OPAL_ASYNC -I$(OPALDIR)/include_target + LDFLAGS += -m32 + LDLIBS += $(addprefix $(OPALDIR)/lib/redhawk/, libOpalAsyncApiCore.a libOpalCore.a libOpalUtils.a libirc.a) + COMMON += opal.o +endif + .PHONY: all clean +# Default target: build everything +all: $(TARGETS) + +# Dependencies for individual binaries +server: server.o $(OBJS) +send: send.o $(OBJS) +receive: receive.o $(OBJS) +random: random.o $(OBJS) +test: test.o $(OBJS) + clean: $(RM) *~ *.o *.d $(RM) $(TARGETS) diff --git a/server/include/cfg.h b/server/include/cfg.h index a60c493bf..8201489ff 100644 --- a/server/include/cfg.h +++ b/server/include/cfg.h @@ -80,8 +80,8 @@ int config_parse_hooks(config_setting_t *cfg, struct list *hooks); /** Parse a single node and add it to the global configuration. * - * @param cfg A libconfig object pointing to the node - * @param nodes Add new nodes to this linked list + * @param cfg A libconfig object pointing to the node. + * @param nodes Add new nodes to this linked list. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ @@ -89,7 +89,8 @@ int config_parse_node(config_setting_t *cfg, struct node **nodes); /** Parse node connection details for OPAL type * - * @param cfg A libconfig object pointing to the node + * @param cfg A libconfig object pointing to the node. + * @param nodes Add new nodes to this linked list. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ @@ -97,7 +98,8 @@ int config_parse_opal(config_setting_t *cfg, struct node *n); /** Parse node connection details for GTFPGA type * - * @param cfg A libconfig object pointing to the node + * @param cfg A libconfig object pointing to the node. + * @param n A pointer to the node structure which should be parsed. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ @@ -105,7 +107,8 @@ int config_parse_gtfpga(config_setting_t *cfg, struct node *n); /** Parse node connection details for SOCKET type * - * @param cfg A libconfig object pointing to the node + * @param cfg A libconfig object pointing to the node. + * @param n A pointer to the node structure which should be parsed. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ @@ -113,8 +116,8 @@ int config_parse_socket(config_setting_t *cfg, struct node *n); /** Parse network emulator (netem) settings. * - * @param cfg A libconfig object containing the settings - * @param em A pointer to the settings + * @param cfg A libconfig object containing the settings. + * @param em A pointer to the netem settings structure (part of the path structure). * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ diff --git a/server/include/node.h b/server/include/node.h index 2b97294be..d3c186504 100644 --- a/server/include/node.h +++ b/server/include/node.h @@ -118,8 +118,8 @@ int node_stop(struct node *n); /** Lookup string representation of socket type * - * @param type A string describing the socket type. This must be one of: tcp, tcpd, udp, ip, ieee802.3 - * @return An enumeration value or INVALID (0) + * @param str A string describing the socket type. This must be one of: tcp, tcpd, udp, ip, ieee802.3 or opal + * @return A pointer to the vtable, or NULL if there is no socket type / vtable with this id. */ struct node_vtable const * node_lookup_vtable(const char *str); diff --git a/server/include/opal.h b/server/include/opal.h index 7b0d74c7b..4fbf74b8c 100644 --- a/server/include/opal.h +++ b/server/include/opal.h @@ -1,4 +1,4 @@ -/** Node type: OPAL (AsyncApi) +/** Node type: OPAL (libOpalAsync API) * * This file implements the opal subtype for nodes. * @@ -9,8 +9,78 @@ #ifndef _OPAL_H_ #define _OPAL_H_ -struct opal { +#include +#include "node.h" +#include "msg.h" + +/* Define RTLAB before including OpalPrint.h for messages to be sent + * to the OpalDisplay. Otherwise stdout will be used. */ +#define RTLAB +#include "OpalPrint.h" +#include "AsyncApi.h" +#include "OpalGenAsyncParamCtrl.h" + +/** This global structure holds libOpalAsync related information. + * It's only used once in the code. */ +struct opal_global { + /** Shared Memory identifiers and size, provided via argv. */ + char *async_shmem_name, *print_shmem_name; + int async_shmem_size; + + /** Number of send blocks used in the running OPAL model. */ + int send_icons, recv_icons; + /** A dynamically allocated array of SendIDs. */ + int *send_ids, *recv_ids; + + /** String and Float parameters, provided by the OPAL AsyncProcess block. */ + Opal_GenAsyncParam_Ctrl params; + + /** Big Global Lock for libOpalAsync API */ + pthread_mutex_t lock; }; +struct opal { + int reply; + int mode; + + int send_id; + int recv_id; + + int seq_no; + + struct opal_global *global; + + Opal_SendAsyncParam send_params; + Opal_RecvAsyncParam recv_params; +}; + +/** Initialize global OPAL settings and maps shared memory regions. + * + * @param argc The number of CLI arguments, provided to main(). + * @param argv The CLI argument list, provided to main(). + * @retval 0 On success. + * @retval <0 On failure. + */ +int opal_init(int argc, char *argv[]); + +/** Free global OPAL settings and unmaps shared memory regions. + * + * @retval 0 On success. + * @retval <0 On failure. + */ +int opal_deinit(); + +int opal_print(struct node *n, char *buf, int len); + +int opal_print_global(struct opal_global *g); + +int opal_open(struct node *n); + +int opal_close(struct node *n); + +int opal_read(struct node *n, struct msg *m); + +int opal_write(struct node *n, struct msg *m); + #endif /* _OPAL_H_ */ diff --git a/server/src/cfg.c b/server/src/cfg.c index 6e01e5139..3060b3a56 100644 --- a/server/src/cfg.c +++ b/server/src/cfg.c @@ -248,6 +248,9 @@ int config_parse_node(config_setting_t *cfg, struct node **nodes) n->vt = node_lookup_vtable(type); if (!n->vt) cerror(cfg, "Invalid type for node '%s'", n->name); + + if (!n->vt->parse) + cerror(cfg, "Node type '%s' is not allowed in the config", type); } else n->vt = node_lookup_vtable("udp"); @@ -259,9 +262,42 @@ int config_parse_node(config_setting_t *cfg, struct node **nodes) return ret; } -/** @todo Implement */ +/** @todo: Remove this global variable. */ +extern struct opal_global *og; + int config_parse_opal(config_setting_t *cfg, struct node *n) -{ +{ + if (!og) { + warn("Skipping this node, because this server is not running as an OPAL Async process!"); + return -1; + } + + struct opal *o = (struct opal *) malloc(sizeof(struct opal)); + if (!o) + error("Failed to allocate memory for opal settings"); + + memset(o, 0, sizeof(struct opal)); + + config_setting_lookup_int(cfg, "send_id", &o->send_id); + config_setting_lookup_int(cfg, "recv_id", &o->send_id); + config_setting_lookup_bool(cfg, "reply", &o->reply); + + /* Search for valid send and recv ids */ + int sfound = 0, rfound = 0; + for (int i=0; isend_icons; i++) + sfound += og->send_ids[i] == o->send_id; + for (int i=0; isend_icons; i++) + rfound += og->send_ids[i] == o->send_id; + + if (!sfound) + cerror(config_setting_get_member(cfg, "send_id"), "Invalid send_id '%u' for node '%s'", o->send_id, n->name); + if (!rfound) + cerror(config_setting_get_member(cfg, "send_id"), "Invalid send_id '%u' for node '%s'", o->send_id, n->name); + + n->opal = o; + n->opal->global = og; + n->cfg = cfg; + return 0; } diff --git a/server/src/node.c b/server/src/node.c index 216e200e2..28a5204c8 100644 --- a/server/src/node.c +++ b/server/src/node.c @@ -24,13 +24,14 @@ /** Vtable for virtual node sub types */ static const struct node_vtable vtables[] = { +#ifdef ENABLE_OPAL_ASYNC + VTABLE(OPAL_ASYNC, "opal", opal), +#endif VTABLE(IEEE_802_3, "ieee802.3", socket), VTABLE(IP, "ip", socket), VTABLE(UDP, "udp", socket), VTABLE(TCP, "tcp", socket), - VTABLE(TCPD, "tcpd", socket), - //VTABLE(OPAL, "opal", opal ), - //VTABLE(GTFPGA, "gtfpga", gtfpga), + VTABLE(TCPD, "tcpd", socket) }; /** Linked list of nodes */ diff --git a/server/src/opal.c b/server/src/opal.c index 8539723c0..44fb41a4a 100644 --- a/server/src/opal.c +++ b/server/src/opal.c @@ -6,4 +6,224 @@ * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC */ +#include +#include + #include "opal.h" +#include "utils.h" + +/** @todo: delcare statice */ +struct opal_global *og = NULL; + +int opal_init(int argc, char *argv[]) +{ + int err; + + if (argc != 4) + return -1; + + struct opal_global *g = (struct opal_global *) malloc(sizeof(struct opal_global)); + if (!g) + error("Failed to allocate memory for global OPAL settings"); + + memset(g, 0, sizeof(struct opal_global)); + + pthread_mutex_init(&g->lock, NULL); + + g->async_shmem_name = argv[1]; + g->async_shmem_size = atoi(argv[2]); + g->print_shmem_name = argv[3]; + + /* Enable the OpalPrint function. This prints to the OpalDisplay. */ + if ((err = OpalSystemCtrl_Register(g->print_shmem_name)) != EOK) + error("OpalPrint() access not available (%d)", err); + + /* Open Share Memory created by the model. */ + if ((err = OpalOpenAsyncMem(g->async_shmem_size, g->async_shmem_name)) != EOK) + error("Model shared memory not available (%d)", err); + + if ((err = OpalGetAsyncCtrlParameters(&g->params, sizeof(Opal_GenAsyncParam_Ctrl))) != EOK) + error("Could not get OPAL controller parameters (%d)", err); + + /* Get list of Send and RecvIDs */ + if ((err = OpalGetNbAsyncSendIcon(&g->send_icons)) != EOK) + error("Failed to get number of send blocks (%d)", err); + if ((err = OpalGetNbAsyncRecvIcon(&g->recv_icons)) != EOK); + error("Failed to get number of recv blocks (%d)", err); + + g->send_ids = (int *) malloc(g->send_icons * sizeof(int)); + g->recv_ids = (int *) malloc(g->recv_icons * sizeof(int)); + if (!g->send_ids || !g->recv_ids) + error("Failed to allocate memory for OPAL AsyncApi ID list."); + + if ((err = OpalGetAsyncSendIDList(g->send_ids, g->send_icons)) != EOK) + error("Failed to get list of send ids (%d)", err); + if ((err = OpalGetAsyncRecvIDList(g->recv_ids, g->recv_icons)) != EOK) + error("Failed to get list of recv ids (%d)", err); + + info("Started as OPAL async process:"); + opal_print_global(g); + + og = g; + + return 0; +} + +int opal_deinit() +{ + int err; + + if (og) { + if ((err = OpalCloseAsyncMem(og->async_shmem_size, og->async_shmem_name)) != EOK) + error("Failed to close shared memory area (%d)", err); + if ((err = OpalSystemCtrl_UnRegister(og->print_shmem_name)) != EOK) + error("Failed to close shared memory for system control (%d)", err); + + free(og->send_ids); + free(og->recv_ids); + free(og); + + og = NULL; + } + + return 0; +} + +int opal_print_global(struct opal_global *g) +{ INDENT + char sbuf[512] = ""; + char rbuf[512] = ""; + + for (int i=0; isend_icons; i++) + strap(sbuf, sizeof(sbuf), "%u ", g->send_ids[i]); + for (int i=0; irecv_icons; i++) + strap(rbuf, sizeof(rbuf), "%u ", g->recv_ids[i]); + + debug(4, "Controller ID: %u", g->params.controllerID); + debug(4, "Send Blocks: %s", sbuf); + debug(4, "Receive Blocks: %s", rbuf); + + debug(4, "Control Block Parameters:"); + for (int i=0; iparams.FloatParam[i]); + for (int i=0; iparams.StringParam[i]); + + return 0; +} + +int opal_print(struct node *n, char *buf, int len) +{ + struct opal *o = n->opal; + + /** @todo: Print send_params, recv_params */ + + return snprintf(buf, len, "send_id=%u, recv_id=%u, reply=%u", + o->send_id, o->recv_id, o->reply); +} + +int opal_open(struct node *n) +{ + struct opal *o = n->opal; + + OpalGetAsyncSendIconMode(&o->mode, o->send_id); + OpalGetAsyncSendParameters(&o->send_params, sizeof(Opal_SendAsyncParam), o->send_id); + OpalGetAsyncRecvParameters(&o->recv_params, sizeof(Opal_RecvAsyncParam), o->recv_id); + + return 0; +} + +int opal_close(struct node *n) +{ + return 0; +} + +int opal_read(struct node *n, struct msg *m) +{ + struct opal *o = n->opal; + + int state, len, ret; + unsigned id; + + double data[MSG_VALUES]; + + /* This call unblocks when the 'Data Ready' line of a send icon is asserted. */ + do { + if ((ret = OpalWaitForAsyncSendRequest(&id)) != EOK) { + state = OpalGetAsyncModelState(); + if ((state != STATE_RESET) && (state != STATE_STOP)) { + OpalSetAsyncSendIconError(ret, id); + info("OpalWaitForAsyncSendRequest(), errno %d", ret); + } + + return -1; // FIXME: correct return value + } + } while (id != o->send_id); + + /* No errors encountered yet */ + OpalSetAsyncSendIconError(0, o->send_id); + + /* Get the size of the data being sent by the unblocking SendID */ + OpalGetAsyncSendIconDataLength(&len, o->send_id); + if (len > sizeof(data)) { + warn("Ignoring the last %u of %u values for OPAL node '%s' (send_id=%u).", + len / sizeof(double) - MSG_VALUES, len / sizeof(double), n->name, o->send_id); + + len = sizeof(data); + } + + /* Read data from the model */ + OpalGetAsyncSendIconData(data, len, o->send_id); + + m->sequence = htons(o->seq_no++); + m->length = len / sizeof(double); + + for (int i = 0; i < m->length; i++) + m->data[i].f = (float) data[i]; // casting to float! + + /* This next call allows the execution of the "asynchronous" process + * to actually be synchronous with the model. To achieve this, you + * should set the "Sending Mode" in the Async_Send block to + * NEED_REPLY_BEFORE_NEXT_SEND or NEED_REPLY_NOW. This will force + * the model to wait for this process to call this + * OpalAsyncSendRequestDone function before continuing. */ + if (o->reply) + OpalAsyncSendRequestDone(o->send_id); + + /* Before continuing, we make sure that the real-time model + * has not been stopped. If it has, we quit. */ + state = OpalGetAsyncModelState(); + if ((state == STATE_RESET) || (state == STATE_STOP)) + error("OpalGetAsyncModelState(): Model stopped or resetted!"); // TODO: fixme + + return 0; +} + +int opal_write(struct node *n, struct msg *m) +{ + struct opal *o = n->opal; + + int state; + int len; + + double data[MSG_VALUES] = { NAN }; + + state = OpalGetAsyncModelState(); + if ((state == STATE_RESET) || (state == STATE_STOP)) + return -1; + + OpalSetAsyncRecvIconStatus(m->sequence, o->recv_id); /* Set the Status to the message ID */ + OpalSetAsyncRecvIconError(0, o->recv_id); /* Set the Error to 0 */ + + /* Get the number of signals to send back to the model */ + OpalGetAsyncRecvIconDataLength(&len, o->recv_id); + if (len > sizeof(data)) + error("Receive Block of OPAL node '%s' is expecting more signals than"); + + for (int i = 0; i < m->length; i++) + data[i] = (double) m->data[i].f; + + OpalSetAsyncRecvIconData(data, len, o->recv_id); + + return 0; +} diff --git a/server/src/server.c b/server/src/server.c index f9ba46162..9513ca02e 100644 --- a/server/src/server.c +++ b/server/src/server.c @@ -22,6 +22,10 @@ #include "path.h" #include "node.h" +#ifdef ENABLE_OPAL_ASYNC +#include "opal.h" +#endif + /** Linked list of nodes */ extern struct node *nodes; /** Linked list of paths */ @@ -104,6 +108,11 @@ void usage(const char *name) { printf("Usage: %s CONFIG\n", name); printf(" CONFIG is a required path to a configuration file\n\n"); +#ifdef ENABLE_OPAL_ASYNC + printf("Usage: %s OPAL_ASYNC_SHMEM_NAME OPAL_ASYNC_SHMEM_SIZE OPAL_PRINT_SHMEM_NAME\n", name); + printf(" This type of invocation is used by OPAL-RT Asynchronous processes.\n"); + printf(" See in the RT-LAB User Guide for more information.\n\n"); +#endif printf("Simulator2Simulator Server %s (built on %s, %s)\n", BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); @@ -113,13 +122,19 @@ void usage(const char *name) int main(int argc, char *argv[]) { /* Check arguments */ +#ifdef ENABLE_OPAL_ASYNC + if (argc != 2 && argc != 4) +#else if (argc != 2) +#endif usage(argv[0]); epoch_reset(); info("This is Simulator2Simulator Server (S2SS) %s (built on %s, %s)", BLD(YEL(VERSION)), BLD(MAG(__DATE__)), BLD(MAG(__TIME__))); + char *configfile = argv[1]; + /* Check priviledges */ if (getuid() != 0) error("The server requires superuser privileges!"); @@ -132,8 +147,13 @@ int main(int argc, char *argv[]) info("Parsing configuration:"); config_init(&config); +#ifdef ENABLE_OPAL_ASYNC + /* Check if called as asynchronous process from RT-LAB */ + opal_init(argc, argv); +#endif + /* Parse configuration and create nodes/paths */ - config_parse(argv[1], &config, &settings, &nodes, &paths); + config_parse(configfile, &config, &settings, &nodes, &paths); /* Connect all nodes and start one thread per path */ info("Starting nodes:"); diff --git a/server/src/utils.c b/server/src/utils.c index 1cd554765..cfdbf0486 100644 --- a/server/src/utils.c +++ b/server/src/utils.c @@ -15,6 +15,10 @@ #include #include +#ifdef ENABLE_OPAL_ASYNC +#include +#endif + #include "config.h" #include "cfg.h" #include "utils.h" @@ -57,32 +61,38 @@ int vstrap(char *dest, size_t size, const char *fmt, va_list ap) void print(enum log_level lvl, const char *fmt, ...) { struct timespec ts; + char buf[512] = ""; va_list ap; - va_start(ap, fmt); /* Timestamp */ clock_gettime(CLOCK_REALTIME, &ts); - fprintf(stderr, "%8.3f ", timespec_delta(&epoch, &ts)); + strap(buf, sizeof(buf), "%8.3f ", timespec_delta(&epoch, &ts)); + /* Severity */ switch (lvl) { - case DEBUG: fprintf(stderr, BLD("%-5s "), GRY("Debug")); break; - case INFO: fprintf(stderr, BLD("%-5s "), " " ); break; - case WARN: fprintf(stderr, BLD("%-5s "), YEL(" Warn")); break; - case ERROR: fprintf(stderr, BLD("%-5s "), RED("Error")); break; + case DEBUG: strap(buf, sizeof(buf), BLD("%-5s "), GRY("Debug")); break; + case INFO: strap(buf, sizeof(buf), BLD("%-5s "), " " ); break; + case WARN: strap(buf, sizeof(buf), BLD("%-5s "), YEL(" Warn")); break; + case ERROR: strap(buf, sizeof(buf), BLD("%-5s "), RED("Error")); break; } - if (_indent) { - for (int i = 0; i < _indent-1; i++) - fprintf(stderr, GFX("\x78") " "); - - fprintf(stderr, GFX("\x74") " "); - } - - vfprintf(stderr, fmt, ap); - fprintf(stderr, "\n"); + /* Indention */ + for (int i = 0; i < _indent-1; i++) + strap(buf, sizeof(buf), GFX("\x78") " "); + strap(buf, sizeof(buf), GFX("\x74") " "); + /* Format String */ + va_start(ap, fmt); + vstrap(buf, sizeof(buf), fmt, ap); va_end(ap); + + /* Output */ +#ifdef ENABLE_OPAL_ASYNC + OpalPrint("%s\n", buf); +#else + fprintf(stderr, "%s\n", buf); +#endif } cpu_set_t to_cpu_set(int set)