diff --git a/python/README.md b/python/README.md new file mode 100644 index 000000000..65c819272 --- /dev/null +++ b/python/README.md @@ -0,0 +1,52 @@ +# VILLASnode Python Support + +## Merge two files and filter the output based on timestamps + +``` +villas-file-merge testfile.dat testfile2.dat | villas-file-filter 3 5 > output.dat +``` + +## Documentation + +User documentation is available here: + +## Copyright + +2018, Institute for Automation of Complex Power Systems, EONERC + +## License + +This project is released under the terms of the [GPL version 3](COPYING.md). + +We kindly ask all academic publications employing components of VILLASframework to cite one of the following papers: + +- A. Monti et al., "[A Global Real-Time Superlab: Enabling High Penetration of Power Electronics in the Electric Grid](https://ieeexplore.ieee.org/document/8458285/)," in IEEE Power Electronics Magazine, vol. 5, no. 3, pp. 35-44, Sept. 2018. +- S. Vogel, M. Mirz, L. Razik and A. Monti, "[An open solution for next-generation real-time power system simulation](http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=8245739&isnumber=8244404)," 2017 IEEE Conference on Energy Internet and Energy System Integration (EI2), Beijing, 2017, pp. 1-6. + +``` +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +``` + +For other licensing options please consult [Prof. Antonello Monti](mailto:amonti@eonerc.rwth-aachen.de). + +## Contact + +[![EONERC ACS Logo](doc/pictures/eonerc_logo.png)](http://www.acs.eonerc.rwth-aachen.de) + +- Steffen Vogel +- Marija Stevic + +[Institute for Automation of Complex Power Systems (ACS)](http://www.acs.eonerc.rwth-aachen.de) +[EON Energy Research Center (EONERC)](http://www.eonerc.rwth-aachen.de) +[RWTH University Aachen, Germany](http://www.rwth-aachen.de) diff --git a/python/bin/villas-cumulative-dist b/python/bin/villas-cumulative-dist new file mode 100755 index 000000000..acdf6a39f --- /dev/null +++ b/python/bin/villas-cumulative-dist @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +import csv +import sys +import numpy as np +import matplotlib.pyplot as plt +import re + +# check if called correctly +if len(sys.argv) < 2: + sys.exit('Usage: %s FILE1 FILE2' % sys.argv[0]) + +plt.figure(figsize=(8,4)) + +for fn in sys.argv[1:]: + +# m = re.match('[a-zA-Z-]+[-_](\d+)[-_](\d+).', fn) +# rate = m.group(1) +# values = m.group(2) +# print 'Processing file %s (rate=%s, values=%s)' % (fn, rate, values) + + # read data from file + data = [ ] + with open(fn) as f: + reader = csv.reader(f, delimiter='\t') + + for row in reader: + offset = float(row[0]) + +# if fn != 'nrel-test1_offset.log': +# offset = offset * 0.001 + + if offset > 100: + continue + + data.append(offset) + + # evaluate the histogram + values, base = np.histogram(data, bins='fd') + + # evaluate the cumulative + cumulative = np.cumsum(values) + cumscaled = [ float(x) / len(data) for x in cumulative ] + + # plot the cumulative function + plt.plot(base[:-1], cumscaled, label=fn, linewidth=1) + + # plot the distribution + #valscaled = [ float(x) / len(data) for x in values ] + #plt.plot(base[:-1], valscaled, label=fn, linewidth=1) + +plt.xlabel('RTT (s)') +plt.ylabel('Cum. Probability') +plt.grid(color='0.75') + +#plt.yscale('log') + +#plt.ylim([0, 1.03]) +#plt.xlim([0.025, 0.05]) + +lgd = plt.legend(title='Rate (p/s)', loc='center left', bbox_to_anchor=(1, 0.5)) + +plt.show() + +#plt.savefig('cumdist.png', dpi=600, bbox_extra_artists=(lgd,), bbox_inches='tight') diff --git a/python/bin/villas-extract-rtt b/python/bin/villas-extract-rtt new file mode 100755 index 000000000..f4556c9a3 --- /dev/null +++ b/python/bin/villas-extract-rtt @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +import csv +import sys +import villas.human as vh + +# check if called correctly +if len(sys.argv) != 1: + sys.exit('Usage: %s < IN_FILE > OUT_FILE' % sys.argv[0]) + +with sys.stdin as f: + reader = csv.reader(f, delimiter='\t') + + for row in reader: + m = vh.Message.parse(row) + + if m.ts == None or m.ts.offset == None: + continue + + print(m.ts.offset) diff --git a/python/bin/villas-file-filter b/python/bin/villas-file-filter new file mode 100755 index 000000000..900aa64f1 --- /dev/null +++ b/python/bin/villas-file-filter @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +import sys +import villas.human as vh + +if len(sys.argv) != 3: + print("Usage: %s from to" % (sys.argv[0])) + sys.exit(-1) + +start = vh.Timestamp.parse(sys.argv[1]) +end = vh.Timestamp.parse(sys.argv[2]) + +for line in sys.stdin: + msg = vh.Message.parse(line) + + if start <= msg.ts <= end: + print msg diff --git a/python/bin/villas-file-merge b/python/bin/villas-file-merge new file mode 100755 index 000000000..da792d899 --- /dev/null +++ b/python/bin/villas-file-merge @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +import sys +import villas.human + +files = sys.argv[1:] + +all = [ ] +last = { } + +for file in files: + handle = sys.stdin if file == '-' else open(file, "r") + + msgs = [ ] + for line in handle.xreadlines(): + msgs.append(vh.Message.parse(line, file)) + + all += msgs + last[file] = vh.Message(vh.Timestamp(), [0] * len(msgs[0].values), file) + +all.sort() +for msg in all: + last[msg.source] = msg + + values = [ ] + for file in files: + values += last[file].values + + print(vh.Message(msg.ts, values, "")) diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 000000000..caa89fe7a --- /dev/null +++ b/python/setup.py @@ -0,0 +1,52 @@ +import os, re +from setuptools import setup +from glob import glob + +def cleanhtml(raw_html): + cleanr = re.compile('<.*?>') + cleantext = re.sub(cleanr, '', raw_html) + return cleantext + +def read(fname): + dname = os.path.dirname(__file__) + fname = os.path.join(dname, fname) + + with open(fname) as f: + contents = f.read() + sanatized = cleanhtml(contents) + + try: + from m2r import M2R + m2r = M2R() + return m2r(sanatized) + except: + return sanatized + +setup( + name = 'villas-node', + version = '0.6.4', + author = 'Steffen Vogel', + author_email = 'acs-software@eonerc.rwth-aachen.de', + description = 'Python-support for VILLASnode simulation-data gateway', + license = 'GPL-3.0', + keywords = 'simulation power system real-time villas', + url = 'https://git.rwth-aachen.de/acs/public/villas/dataprocessing', + packages = [ 'villas.node' ], + long_description = read('README.md'), + classifiers = [ + 'Development Status :: 4 - Beta', + 'Topic :: Scientific/Engineering', + 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3' + ], + install_requires = [ + ], + setup_requires = [ + 'm2r' + ], + scripts = glob('bin/*') +) + diff --git a/python/villas/human/__init__.py b/python/villas/human/__init__.py new file mode 100644 index 000000000..77a4ac0a5 --- /dev/null +++ b/python/villas/human/__init__.py @@ -0,0 +1,2 @@ +from message import Message +from timestamp import Timestamp diff --git a/python/villas/human/message.py b/python/villas/human/message.py new file mode 100644 index 000000000..baae5627e --- /dev/null +++ b/python/villas/human/message.py @@ -0,0 +1,24 @@ +import .timestamp + +class Message: + """Parsing a VILLASnode sample from a file (not a UDP package!!)""" + + def __init__(self, ts, values, source = None): + self.source = source + self.ts = ts + self.values = values + + @classmethod + def parse(self, line, source = None): + csv = line.split() + + t = ts.Timestamp.parse(csv[0]) + v = map(float, csv[1:]) + + return Message(t, v, source) + + def __str__(self): + return '%s %s' % (self.ts, " ".join(map(str, self.values))) + + def __cmp__(self, other): + return cmp(self.ts, other.ts) diff --git a/python/villas/human/timestamp.py b/python/villas/human/timestamp.py new file mode 100644 index 000000000..96df86812 --- /dev/null +++ b/python/villas/human/timestamp.py @@ -0,0 +1,46 @@ +import re + +class Timestamp: + """Parsing the VILLASnode human-readable timestamp format""" + + def __init__(self, seconds = 0, nanoseconds = None, offset = None, sequence = None): + self.seconds = seconds + self.nanoseconds = nanoseconds + self.offset = offset + self.sequence = sequence + + @classmethod + def parse(self, ts): + m = re.match('(\d+)(?:\.(\d+))?([-+]\d+(?:\.\d+)?(?:e[+-]?\d+)?)?(?:\((\d+)\))?', ts) + + seconds = int(m.group(1)); # Mandatory + nanoseconds = int(m.group(2)) if m.group(2) else None + offset = float(m.group(3)) if m.group(3) else None + sequence = int(m.group(4)) if m.group(4) else None + + return Timestamp(seconds, nanoseconds, offset, sequence) + + def __str__(self): + str = "%u" % (self.seconds) + + if self.nanoseconds is not None: + str += ".%09u" % self.nanoseconds + if self.offset is not None: + str += "+%u" % self.offset + if self.sequence is not None: + str += "(%u)" % self.sequence + + return str + + def __float__(self): + sum = float(self.seconds) + + if self.nanoseconds is not None: + sum += self.nanoseconds * 1e-9 + if self.offset is not None: + sum += self.offset + + return sum + + def __cmp__(self, other): + return cmp(float(self), float(other)) diff --git a/python/villas/node/__init__.py b/python/villas/node/__init__.py new file mode 100644 index 000000000..997d5c906 --- /dev/null +++ b/python/villas/node/__init__.py @@ -0,0 +1 @@ +import .node diff --git a/python/villas/node/node.py b/python/villas/node/node.py new file mode 100644 index 000000000..9bc1fda9b --- /dev/null +++ b/python/villas/node/node.py @@ -0,0 +1,26 @@ +import json +import tempfile +import subprocess +import logging + +LOGGER = logging.getLogger('villas.node') + +class Node(object): + + def __init__(self, cfg): + self.config = cfg + + def start(self): + self.config_file = tempfile.NamedTemporaryFile(mode='w+', suffix='.json') + + json.dump(self.config, self.config_file) + self.config_file.flush() + + LOGGER.info("Starting VILLASnode with config: %s", self.config_file.name) + + self.child = subprocess.Popen(['villas-node', self.config_file.name]) + + def stop(self): + self.child.kill() + self.child.wait() +