python: added first version of VILLASnode package

python/README.md

@@ -0,0 +1,52 @@
+# VILLASnode Python Support
+
+## Merge two files and filter the output based on timestamps
+
+```
+villas-file-merge testfile.dat testfile2.dat | villas-file-filter 3 5 > output.dat
+```
+
+## Documentation
+
+User documentation is available here: <https://villas.fein-aachen.org/doc/node.html>
+
+## Copyright
+
+2018, Institute for Automation of Complex Power Systems, EONERC  
+
+## License
+
+This project is released under the terms of the [GPL version 3](COPYING.md).
+
+We kindly ask all academic publications employing components of VILLASframework to cite one of the following papers:
+
+- A. Monti et al., "[A Global Real-Time Superlab: Enabling High Penetration of Power Electronics in the Electric Grid](https://ieeexplore.ieee.org/document/8458285/)," in IEEE Power Electronics Magazine, vol. 5, no. 3, pp. 35-44, Sept. 2018.
+- S. Vogel, M. Mirz, L. Razik and A. Monti, "[An open solution for next-generation real-time power system simulation](http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=8245739&isnumber=8244404)," 2017 IEEE Conference on Energy Internet and Energy System Integration (EI2), Beijing, 2017, pp. 1-6.
+
+```
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+```
+
+For other licensing options please consult [Prof. Antonello Monti](mailto:amonti@eonerc.rwth-aachen.de).
+
+## Contact
+
+[![EONERC ACS Logo](doc/pictures/eonerc_logo.png)](http://www.acs.eonerc.rwth-aachen.de)
+
+- Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
+- Marija Stevic <mstevic@eonerc.rwth-aachen.de>
+
+[Institute for Automation of Complex Power Systems (ACS)](http://www.acs.eonerc.rwth-aachen.de)  
+[EON Energy Research Center (EONERC)](http://www.eonerc.rwth-aachen.de)  
+[RWTH University Aachen, Germany](http://www.rwth-aachen.de)  

python/bin/villas-cumulative-dist

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+
+import csv
+import sys
+import numpy as np
+import matplotlib.pyplot as plt
+import re
+
+# check if called correctly
+if len(sys.argv) < 2:
+	sys.exit('Usage: %s FILE1 FILE2' % sys.argv[0])
+
+plt.figure(figsize=(8,4))
+
+for fn in sys.argv[1:]:
+
+#	m = re.match('[a-zA-Z-]+[-_](\d+)[-_](\d+).', fn)
+#	rate = m.group(1)
+#	values = m.group(2)
+#	print 'Processing file %s (rate=%s, values=%s)' % (fn, rate, values)
+
+	# read data from file
+	data = [ ]
+	with open(fn) as f:
+		reader = csv.reader(f, delimiter='\t')
+
+		for row in reader:
+			offset = float(row[0])
+
+#			if fn != 'nrel-test1_offset.log':
+#				offset = offset * 0.001
+
+			if offset > 100:
+				continue
+
+			data.append(offset)
+
+	# evaluate the histogram
+	values, base = np.histogram(data, bins='fd')
+
+	# evaluate the cumulative
+	cumulative = np.cumsum(values)
+	cumscaled = [ float(x) / len(data) for x in cumulative ]
+
+	# plot the cumulative function
+	plt.plot(base[:-1], cumscaled, label=fn, linewidth=1)
+
+	# plot the distribution
+	#valscaled = [ float(x) / len(data) for x in values ]
+	#plt.plot(base[:-1], valscaled, label=fn, linewidth=1)
+
+plt.xlabel('RTT (s)')
+plt.ylabel('Cum. Probability')
+plt.grid(color='0.75')
+
+#plt.yscale('log')
+
+#plt.ylim([0, 1.03])
+#plt.xlim([0.025, 0.05])
+
+lgd = plt.legend(title='Rate (p/s)', loc='center left', bbox_to_anchor=(1, 0.5))
+
+plt.show()
+
+#plt.savefig('cumdist.png', dpi=600, bbox_extra_artists=(lgd,), bbox_inches='tight')

python/bin/villas-extract-rtt

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+import csv
+import sys
+import villas.human as vh
+
+# check if called correctly
+if len(sys.argv) != 1:
+	sys.exit('Usage: %s < IN_FILE > OUT_FILE' % sys.argv[0])
+
+with sys.stdin as f:
+	reader = csv.reader(f, delimiter='\t')
+
+	for row in reader:
+		m = vh.Message.parse(row)
+
+		if m.ts == None or m.ts.offset == None:
+			 continue
+
+		print(m.ts.offset)

python/bin/villas-file-filter

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+import sys
+import villas.human as vh
+
+if len(sys.argv) != 3:
+	print("Usage: %s from to" % (sys.argv[0]))
+	sys.exit(-1)
+
+start = vh.Timestamp.parse(sys.argv[1])
+end = vh.Timestamp.parse(sys.argv[2])
+
+for line in sys.stdin:
+	msg = vh.Message.parse(line)
+
+	if start <= msg.ts <= end:
+		print msg

python/bin/villas-file-merge

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+import sys
+import villas.human
+
+files = sys.argv[1:]
+
+all = [ ]
+last = { }
+
+for file in files:
+	handle = sys.stdin if file == '-' else open(file, "r")
+
+	msgs = [ ]
+	for line in handle.xreadlines():
+		msgs.append(vh.Message.parse(line, file))
+
+	all += msgs
+	last[file] = vh.Message(vh.Timestamp(), [0] * len(msgs[0].values), file)
+
+all.sort()
+for msg in all:
+	last[msg.source] = msg
+
+	values = [ ]
+	for file in files:
+		values += last[file].values
+
+	print(vh.Message(msg.ts, values, ""))

python/setup.py

@@ -0,0 +1,52 @@
+import os, re
+from setuptools import setup
+from glob import glob
+
+def cleanhtml(raw_html):
+    cleanr = re.compile('<.*?>')
+    cleantext = re.sub(cleanr, '', raw_html)
+    return cleantext
+
+def read(fname):
+    dname = os.path.dirname(__file__)
+    fname = os.path.join(dname, fname)
+
+    with open(fname) as f:
+        contents = f.read()
+        sanatized = cleanhtml(contents)
+
+        try:
+            from m2r import M2R
+            m2r = M2R()
+            return m2r(sanatized)
+        except:
+            return sanatized
+
+setup(
+    name = 'villas-node',
+    version = '0.6.4',
+    author = 'Steffen Vogel',
+    author_email = 'acs-software@eonerc.rwth-aachen.de',
+    description = 'Python-support for VILLASnode simulation-data gateway',
+    license = 'GPL-3.0',
+    keywords = 'simulation power system real-time villas',
+    url = 'https://git.rwth-aachen.de/acs/public/villas/dataprocessing',
+    packages = [ 'villas.node' ],
+    long_description = read('README.md'),
+    classifiers = [
+        'Development Status :: 4 - Beta',
+        'Topic :: Scientific/Engineering',
+        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
+        'Operating System :: MacOS :: MacOS X',
+        'Operating System :: Microsoft :: Windows',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python :: 3'
+    ],
+    install_requires = [
+    ],
+    setup_requires = [
+        'm2r'
+    ],
+    scripts = glob('bin/*')
+)
+

python/villas/human/__init__.py

@@ -0,0 +1,2 @@
+from message import Message
+from timestamp import Timestamp

python/villas/human/message.py

@@ -0,0 +1,24 @@
+import .timestamp
+
+class Message:
+	"""Parsing a VILLASnode sample from a file (not a UDP package!!)"""
+
+	def __init__(self, ts, values, source = None):
+		self.source = source
+		self.ts = ts
+		self.values = values
+
+	@classmethod
+	def parse(self, line, source = None):
+		csv = line.split()
+
+		t = ts.Timestamp.parse(csv[0])
+		v = map(float, csv[1:])
+
+		return Message(t, v, source)
+
+	def __str__(self):
+		return '%s %s' % (self.ts, " ".join(map(str, self.values)))
+
+	def __cmp__(self, other):
+		return cmp(self.ts, other.ts)

python/villas/human/timestamp.py

@@ -0,0 +1,46 @@
+import re
+
+class Timestamp:
+	"""Parsing the VILLASnode human-readable timestamp format"""
+
+	def __init__(self, seconds = 0, nanoseconds = None, offset = None, sequence = None):
+		self.seconds = seconds
+		self.nanoseconds = nanoseconds
+		self.offset = offset
+		self.sequence = sequence
+
+	@classmethod
+	def parse(self, ts):
+		m = re.match('(\d+)(?:\.(\d+))?([-+]\d+(?:\.\d+)?(?:e[+-]?\d+)?)?(?:\((\d+)\))?', ts)
+
+		seconds		= int(m.group(1)); # Mandatory
+		nanoseconds	= int(m.group(2))   if m.group(2) else None
+		offset		= float(m.group(3)) if m.group(3) else None
+		sequence	= int(m.group(4))   if m.group(4) else None
+
+		return Timestamp(seconds, nanoseconds, offset, sequence)
+
+	def __str__(self):
+		str = "%u" % (self.seconds)
+
+		if self.nanoseconds is not None:
+			str += ".%09u" % self.nanoseconds
+		if self.offset is not None:
+			str += "+%u" % self.offset
+		if self.sequence is not None:
+			str += "(%u)" % self.sequence
+
+		return str
+
+	def __float__(self):
+		sum = float(self.seconds)
+
+		if self.nanoseconds is not None:
+			sum += self.nanoseconds * 1e-9
+		if self.offset is not None:
+			sum += self.offset
+
+		return sum
+
+	def __cmp__(self, other):
+		return cmp(float(self), float(other))

python/villas/node/__init__.py

@@ -0,0 +1 @@
+import .node

python/villas/node/node.py

@@ -0,0 +1,26 @@
+import json
+import tempfile
+import subprocess
+import logging
+
+LOGGER = logging.getLogger('villas.node')
+
+class Node(object):
+
+    def __init__(self, cfg):
+        self.config = cfg
+
+    def start(self):
+        self.config_file = tempfile.NamedTemporaryFile(mode='w+', suffix='.json')
+
+        json.dump(self.config, self.config_file)
+        self.config_file.flush()
+
+        LOGGER.info("Starting VILLASnode with config: %s", self.config_file.name)
+
+        self.child = subprocess.Popen(['villas-node', self.config_file.name])
+
+    def stop(self):
+        self.child.kill()
+        self.child.wait()
+