diff --git a/python/accelerated-stats/Dockerfile b/python/accelerated-stats/Dockerfile new file mode 100644 index 0000000..f93a9ac --- /dev/null +++ b/python/accelerated-stats/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.8-slim + +RUN mkdir /app +ADD . /app + +RUN pip3 install /app + +ENTRYPOINT [ "/usr/local/bin/accelerated_exporter" ] diff --git a/python/accelerated-stats/README b/python/accelerated-stats/README new file mode 100644 index 0000000..c9efcd0 --- /dev/null +++ b/python/accelerated-stats/README @@ -0,0 +1,12 @@ +# Accelerated Server Stats Exporter for Prometheus + +## Examples + +```bash +export KDNUMMER=10642 +export PASSWORD=$(pass providers/interface.datafabrik.de | head -n1) + +# python3 accelerated_stats $* usage +python3 accelerated_stats $* usage_sum +# python3 accelerated_stats $* -f raw -F usage_avg stats +``` diff --git a/python/accelerated-stats/accelerated_stats/__init__.py b/python/accelerated-stats/accelerated_stats/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/accelerated-stats/accelerated_stats/exporter.py b/python/accelerated-stats/accelerated_stats/exporter.py new file mode 100644 index 0000000..a8f4bf6 --- /dev/null +++ b/python/accelerated-stats/accelerated_stats/exporter.py @@ -0,0 +1,44 @@ +from flask import Flask, Response + +from accelerated_stats.tenant import Tenant +from accelerated_stats.utils import parse_arguments + +args = parse_arguments(with_subcommands=False) +args_dict = vars(args) + +acc = Tenant(**args_dict) +app = Flask(__name__) + +app.config['DEBUG'] = args.debug + +if args.server: + servers = [ args.server ] +else: + servers = acc.get_servers() + +@app.route('/healthz') +def healthz(): + return Response('OK', mimetype='text/plain') + +@app.route('/metrics') +def metrics(): + metrics = [] + + for server in servers: + stats = server.get_stats() + + for k, v in stats.items(): + if k == 'port': + continue + + t = [f'{p}="{q}"' for p, q in server.as_dict().items()] + t = ','.join(t) + + metrics.append(f'{args.prefix}{k}{{{t}}} {v}') + + resp = '\n'.join(metrics) + + return Response(resp, mimetype='text/plain') + +def main(): + app.run(host='::') diff --git a/python/accelerated-stats/accelerated_stats/stats.py b/python/accelerated-stats/accelerated_stats/stats.py new file mode 100644 index 0000000..fc84840 --- /dev/null +++ b/python/accelerated-stats/accelerated_stats/stats.py @@ -0,0 +1,74 @@ +#!/bin/env python3 + +import os +import sys +import json +from datetime import datetime + +from accelerated_stats import utils +from accelerated_stats.tenant import Tenant + +def converter(o): + if isinstance(o, datetime): + return o.strftime('%Y-%m-%d') + + +def main(): + + args = utils.parse_arguments() + + args.coerce = not args.no_coerce + + args_dict = vars(args) + + acc = Tenant(**args_dict) + + servers = acc.get_servers() + if args.server: + servers = filter(lambda s: s.id == args.server, servers) + + if args.cmd == 'servers': + out = [s.as_dict() for s in servers] + else: + out = [] + + for server in servers: + if args.cmd == 'stats': + sout = server.get_stats() + elif args.cmd == 'usage': + sout = { + 'usage': server.get_usage(args.date) + } + elif args.cmd == 'usage_sum': + usage = server.get_usage(args.date) + + usage_sum = { + k: sum([ d[k] for d in usage ]) for k in ['in', 'out'] + } + + usage_sum['total'] = usage_sum['in'] + usage_sum['out'] + + sout = { + 'usage_sum': usage_sum + } + + out.append({ + **server.as_dict(), + **sout + }) + + if args.format == 'json': + json.dump(out, sys.stdout, indent=4, default=converter) + sys.stdout.write('\n') + elif args.format == 'raw': + if out is list: + out.keys().join(',') + for l in out: + l.values().join(',') + + elif out is dict: + if args.field: + print(out[args.field]) + else: + for k, v in out.enumerate(): + print(f'{k}: {v}') diff --git a/python/accelerated-stats/accelerated_stats/tenant.py b/python/accelerated-stats/accelerated_stats/tenant.py new file mode 100644 index 0000000..cc1321b --- /dev/null +++ b/python/accelerated-stats/accelerated_stats/tenant.py @@ -0,0 +1,228 @@ +import requests +from datetime import datetime +from lxml import etree +from pint import UnitRegistry +import re +import io + +ureg = UnitRegistry() +ureg.define('MBit = Mbit') +ureg.define('KBit = kbit') +ureg.define('MB = megabyte') +ureg.define('KB = kilobyte') +ureg.define('GB = gigabyte') + +class Server: + + def __init__(self, tenant, id, switch=None, port=None, name=None, hostname=None): + self.tenant = tenant + self.id = id + self.switch = switch + self.port = port + self.name = name + self.hostname = hostname + + def get_usage(self, date): + return self.tenant._get_usage(self.id, date) + + def get_stats(self): + return self.tenant._get_stats(self.id) + + def as_dict(self): + srv = { + 'id': self.id, + 'tenant': int(self.tenant.kdnummer) + } + + if self.switch: + srv['switch'] = self.switch + + if self.port: + srv['port'] = self.port + + if self.name: + srv['name'] = self.name + + if self.hostname: + srv['hostname'] = self.hostname + + return srv + + +class Tenant: + + STATS_FIELDS = { + # 'port': 1, + 'max_speed': 2, + 'switch_uptime': 3, + 'incoming': 4, + 'outgoing': 5, + 'sum': 6, + 'usage_95perc': 7, + 'usage_avg': 8, + 'current_in': 10, + 'current_out': 11 + } + + XPATH_CONTENT = '//*[@id="accelerated-layout-container-content"]' + + XPATH_USAGE_TABLE = XPATH_CONTENT + '/table' + XPATH_SERVER_TABLE = XPATH_USAGE_TABLE + XPATH_STATS_TABLE = XPATH_CONTENT + '/table[3]/tr[1]/td/table' + XPATH_SERVER_ROWS = XPATH_SERVER_TABLE + '/tr[position() > 2 and position() < last() and position() mod 2]' + XPATH_FIELDS = { k: f'tr[{i}]/td[2]' for k, i in STATS_FIELDS.items() } + + def __init__(self, **kwargs): + self.sess = requests.Session() + + self.coerce = kwargs.get('coerce', True) + self.url = kwargs.get('url') + self.kdnummer = kwargs.get('kdnummer') + self.password = kwargs.get('password') + + self.unit_volume = kwargs.get('unit_volume', 'TiB') + self.unit_speed = kwargs.get('unit_speed', 'MBit/s') + self.unit_time = kwargs.get('unit_time', 's') + + self.do_login() + + def as_dict(self): + return { + 'kdnummer': self.kdnummer + } + + @property + def login_url(self): + return f'{self.url}/verify.php' + + @property + def server_url(self): + return f'{self.url}/CServer.php' + + def usage_url(self, server, date): + d = date.strftime('%Y.%m') + return f'{self.server_url}?action=detailUsage&id={server}&date={d}' + + def stats_url(self, server): + return f'{self.server_url}?action=stats&id={server}' + + def do_login(self): + + payload = { + 'kdnummer': self.kdnummer, + 'passwort': self.password, + 'Login': 'Login', + 'url': '' + } + + r = self.sess.post(self.login_url, data=payload) + + + def get_servers(self): + r = self.sess.get(self.server_url + '?switchPort=show') + + parser = etree.HTMLParser() + root = etree.parse(io.StringIO(r.text), parser) + table = root.xpath(self.XPATH_USAGE_TABLE)[0] + rows = root.xpath(self.XPATH_SERVER_ROWS) + + servers = [] + + for row in rows: + anchor = row.xpath('td[3]/a')[0] + href = anchor.get('href') + + match = re.match('CServer.php\?action=stats&id=([0-9]+)', href) + if match: + server_id = int(match.group(1)) + + server = { + 'id': server_id + } + + name = row.xpath('td[2]') + if name: + server['name'] = re.sub(r'\s+|\|', ' ', name[0].text).strip() + + hostname = row.xpath('td[2]/u/font') + if len(hostname) > 0: + server['hostname'] = hostname[0].text.strip() + + swport_row = row.getnext() + if swport_row is not None: + swport = swport_row.xpath('td[2]/table/tr/td[2]/font') + if len(swport) > 0: + switch, port = swport[0].text.strip().split(' -> ') + server['port'] = port + server['switch'] = switch + + servers.append(Server(self, **server)) + + return servers + + def _get_usage(self, server_id, date): + r = self.sess.get(self.usage_url(server_id, date)) + + parser = etree.HTMLParser() + root = etree.parse(io.StringIO(r.text), parser) + table = root.xpath(self.XPATH_USAGE_TABLE)[0] + rows = table.xpath('tr') + + data = [] + + for row in rows[1:]: + columns = row.xpath('td') + + data_row = { + 'date': datetime.strptime(columns[2].xpath('b')[0].text.strip(), '%d.%m.%Y'), + 'in': columns[3].text.strip(), + 'out': columns[4].text.strip() + } + + if self.coerce: + target_unit = ureg.parse_expression(self.unit_volume) + + for f in [ 'in', 'out' ]: + d = data_row[f] + d = ureg.parse_expression(d) + d = d.to(target_unit).magnitude + + data_row[f] = d + + data.append(data_row) + + return data + + def _get_stats(self, server_id): + r = self.sess.get(self.stats_url(server_id)) + + parser = etree.HTMLParser() + root = etree.parse(io.StringIO(r.text), parser) + table = root.xpath(self.XPATH_STATS_TABLE)[0] + + data = { k: table.xpath(p)[0].text for k, p in self.XPATH_FIELDS.items() } + + if self.coerce: + target_units = { + 'switch_uptime': ureg.parse_expression(self.unit_time), + 'incoming': ureg.parse_expression(self.unit_volume), + 'outgoing': ureg.parse_expression(self.unit_volume), + 'sum': ureg.parse_expression(self.unit_volume), + 'max_speed': ureg.parse_expression(self.unit_speed), + 'usage_95perc': ureg.parse_expression(self.unit_speed), + 'usage_avg': ureg.parse_expression(self.unit_speed), + 'current_in': ureg.parse_expression(self.unit_speed), + 'current_out': ureg.parse_expression(self.unit_speed) + } + + for f in [ 'incoming', 'outgoing', 'sum' ]: + data[f] = re.sub(r"(K|M|G|T|)B$", r"\1iB", data[f]) + + data['switch_uptime'] = re.sub(r"(\d+) days, (\d+):(\d+):(\d+).(\d+)", r"\1 days + \2 hours + \3 minutes + \4 seconds + \5 centiseconds", data['switch_uptime']) + + coerced_data = { k: ureg.parse_expression(v) for k, v in data.items() if k != 'port' } + converted_data = { k: coerced_data[k].to(target_units[k]).magnitude for k, v in coerced_data.items() } + + data = { **data, **converted_data } + + return data diff --git a/python/accelerated-stats/accelerated_stats/utils.py b/python/accelerated-stats/accelerated_stats/utils.py new file mode 100644 index 0000000..acd4e04 --- /dev/null +++ b/python/accelerated-stats/accelerated_stats/utils.py @@ -0,0 +1,32 @@ +import argparse +from datetime import datetime +import os + +def parse_arguments(with_subcommands=True): + def valid_date(s): + try: + return datetime.strptime(s, "%Y-%m") + except ValueError: + msg = "Not a valid date: '{0}'.".format(s) + raise argparse.ArgumentTypeError(msg) + + parser = argparse.ArgumentParser('accelerated_stats', + description='Get port stats from Accelerated Customer Interface (e.g. interface.datafabrik.de)') + parser.add_argument('--debug', '-d', type=bool, default=False) + parser.add_argument('--format', '-f', choices=['raw', 'json'], default='json') + parser.add_argument('--field', '-F', type=str) + parser.add_argument('--no-coerce', '-c', action='store_true', default=False) + parser.add_argument('--unit-volume', type=str, default='B') + parser.add_argument('--unit-speed', type=str, default='bit/s') + parser.add_argument('--unit-time', type=str, default='s') + parser.add_argument('--kdnummer', '-u', type=str, default=os.environ.get('KDNUMMER')) + parser.add_argument('--password', '-p', type=str, default=os.environ.get('PASSWORD')) + parser.add_argument('--url', '-U', type=str, default='https://interface.datafabrik.de/') + parser.add_argument('--server', '-s', type=int) + parser.add_argument('--date', '-D', type=valid_date, default=datetime.now()) + parser.add_argument('--prefix', '-P', type=str, default='accelerated_') + + if with_subcommands: + parser.add_argument('cmd', metavar='CMD', choices=['stats', 'usage', 'usage_sum', 'servers']) + + return parser.parse_args() diff --git a/python/accelerated-stats/gunicorn_config.py b/python/accelerated-stats/gunicorn_config.py new file mode 100644 index 0000000..1fd20db --- /dev/null +++ b/python/accelerated-stats/gunicorn_config.py @@ -0,0 +1,4 @@ +bind = "[::]:5000" +workers = 4 +threads = 4 +timeout = 120 diff --git a/python/accelerated-stats/k8s.yaml b/python/accelerated-stats/k8s.yaml new file mode 100644 index 0000000..50cdf74 --- /dev/null +++ b/python/accelerated-stats/k8s.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: accelerated-stats +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: accelerated-stats + namespace: accelerated-stats + labels: + app: accelerated-stats +spec: + selector: + matchLabels: + app: accelerated-stats + template: + metadata: + labels: + app: accelerated-stats + spec: + containers: + - name: accelerated-stats + image: stv0g/accelerated-stats + imagePullPolicy: Always + ports: + - name: http-metrics + containerPort: 5000 + envFrom: + - secretRef: + name: accelerated-stats + resource: +--- +apiVersion: v1 +kind: Service +metadata: + name: accelerated-stats + namespace: accelerated-stats + labels: + app: accelerated-stats +spec: + selector: + app: accelerated-stats + ports: + - protocol: TCP + port: 80 + name: http-metrics + targetPort: http-metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: accelerated-stats + namespace: accelerated-stats + labels: + app: accelerated-stats +spec: + selector: + matchLabels: + app: accelerated-stats + namespaceSelector: + matchNames: + - accelerated-stats + endpoints: + - port: http-metrics + interval: 1m + +# Create secret: +# kubectl --namespace=accelerated-stats create secret generic accelerated-stats \ +# --from-literal=KDNUMMER=$(pass providers/interface.datafabrik.de | sed -nE 's/^User: (.*)/\1/p') \ +# --from-literal=PASSWORD=$(pass providers/interface.datafabrik.de | sed -n 1p) diff --git a/python/accelerated-stats/setup.py b/python/accelerated-stats/setup.py new file mode 100644 index 0000000..89d5c03 --- /dev/null +++ b/python/accelerated-stats/setup.py @@ -0,0 +1,36 @@ +import os +from setuptools import setup, find_packages + +def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + +setup( + name = 'accelerated_stats', + version = '0.1.0', + author = 'Steffen Vogel', + author_email = 'post@steffenvogel.de', + description = ('Fetch and export status and bandwidth ' + 'for servers hosted by Accelerated'), + license = 'GPL-3.0', + keywords = 'accelerated promtheus exporter', + url = 'http://packages.python.org/an_example_pypi_project', + packages=find_packages(), + long_description=read('README'), + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Topic :: Utilities', + 'License :: OSI Approved :: BSD License', + ], + install_requires=[ + 'pint', + 'flask', + 'requests', + 'lxml' + ], + entry_points={ + 'console_scripts': [ + 'accelerated_stats = accelerated_stats.stats:main', + 'accelerated_exporter = accelerated_stats.exporter:main', + ] + } +)