stv0g/bird-lg
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge remote-tracking branch 'gh/dsx/master'

Browse source 
Conflicts:
	.gitignore
	lg.cfg
	lg.py
	lgproxy.cfg
	lgproxy.py
	toolbox.py
This commit is contained in:
Steffen Vogel 2015-04-04 15:48:30 +02:00
commit 2962e4b7ed
4 changed files with 243 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.gitignore vendored
View file

@ -1,3 +1,7 @@
*.pyc
*.pyo
*.py[co]
*.egg
*.egg-info
eggs
*.cfg
*.log

157
lg.py
View file

@ -20,22 +20,23 @@
#
###
import memcache
import subprocess
import logging
from collections import defaultdict
from logging.handlers import TimedRotatingFileHandler
import re
from urllib2 import urlopen
from urllib import quote, unquote
from urllib2 import urlopen
import json
import logging
import memcache
import random
import re
import subprocess
from toolbox import mask_is_valid, ipv6_is_valid, ipv4_is_valid, resolve, save_cache_pickle, load_cache_pickle, get_asname_from_whois, unescape
#from xml.sax.saxutils import escape
import pydot
from dns.resolver import NXDOMAIN
from flask import Flask, render_template, jsonify, redirect, session, request, abort, Response, Markup
import pydot
app = Flask(__name__)
app.config.from_pyfile('lg.cfg')
@ -47,9 +48,19 @@ file_handler.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper()))
app.logger.addHandler(file_handler)
memcache_server = app.config.get("MEMCACHE_SERVER", "127.0.0.1:11211")
memcache_expiration = int(app.config.get("MEMCACHE_EXPIRATION", "1296000")) # 15 days by default
memcache_expiration = int(app.config.get("MEMCACHE_EXPIRATION", "1296000")) # 15 days by default
mc = memcache.Client([memcache_server])
def get_asn_from_as(n):
asn_zone = app.config.get("ASN_ZONE", "asn.cymru.com")
try:
data = resolve("AS%s.%s" % (n, asn_zone), "TXT").replace("'", "").replace('"', '')
except:
return " " * 5
return [field.strip() for field in data.split("|")]
def add_links(text):
"""Browser a string and replace ipv4, ipv6, as number, with a
whois link """
@ -60,8 +71,7 @@ def add_links(text):
ret_text = []
for line in text:
# Some heuristic to create link
if line.strip().startswith("BGP.as_path:") or \
line.strip().startswith("Neighbor AS:"):
if line.strip().startswith("BGP.as_path:") or line.strip().startswith("Neighbor AS:"):
ret_text.append(re.sub(r'(\d+)', r'<a href="/whois?q=\1" class="whois">\1</a>', line))
else:
line = re.sub(r'([a-zA-Z0-9\-]*\.([a-zA-Z]{2,3}){1,2})(\s|$)', r'<a href="/whois?q=\1" class="whois">\1</a>\3', line)
@ -102,7 +112,7 @@ def set_session(request_type, hosts, proto, request_args):
def whois_command(query):
server = []
if app.config.get("WHOIS_SERVER", ""):
server = [ "-h", app.config.get("WHOIS_SERVER") ]
server = ["-h", app.config.get("WHOIS_SERVER")]
return subprocess.Popen(['whois'] + server + [query], stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore')
@ -134,7 +144,7 @@ def bird_proxy(host, proto, service, query):
elif not path:
return False, 'Proto "%s" invalid' % proto
else:
url = "http://%s.%s:%d/%s?q=%s" % (host, app.config["DOMAIN"], port, path, quote(query))
url = 'http://{}:{}/{}?q={}'.format(app.config['ROUTER_IP'][host][0], port, path, quote(query))
try:
f = urlopen(url)
resultat = f.read()
@ -189,10 +199,12 @@ def incorrect_request(e):
def page_not_found(e):
return render_template('error.html', warnings=["The requested URL was not found on the server."]), 404
def get_query():
q = unquote(request.args.get('q', '').strip())
return q
@app.route("/whois")
def whois():
query = get_query()
@ -313,7 +325,6 @@ def traceroute(hosts, proto):
errors.append("%s" % resultat)
continue
infos[host] = add_links(resultat)
return render_template('traceroute.html', infos=infos, errors=errors)
@ -406,7 +417,7 @@ def show_bgpmap():
def add_node(_as, **kwargs):
if _as not in nodes:
kwargs["label"] = '<<TABLE CELLBORDER="0" BORDER="0" CELLPADDING="0" CELLSPACING="0"><TR><TD ALIGN="CENTER">' + escape(kwargs.get("label", get_as_name(_as))).replace("\r","<BR/>") + "</TD></TR></TABLE>>"
kwargs["label"] = '<<TABLE CELLBORDER="0" BORDER="0" CELLPADDING="0" CELLSPACING="0"><TR><TD ALIGN="CENTER">' + escape(kwargs.get("label", get_as_name(_as))).replace("\r", "<BR/>") + "</TD></TR></TABLE>>"
nodes[_as] = pydot.Node(_as, style="filled", fontsize="10", **kwargs)
graph.add_node(nodes[_as])
return nodes[_as]
@ -426,15 +437,15 @@ def show_bgpmap():
label_without_star = kwargs["label"].replace("*", "")
labels = e.get_label().split("\r")
if "%s*" % label_without_star not in labels:
labels = [ kwargs["label"] ] + [ l for l in labels if not l.startswith(label_without_star) ]
labels = sorted(labels, cmp=lambda x,y: x.endswith("*") and -1 or 1)
labels = [kwargs["label"]] + [l for l in labels if not l.startswith(label_without_star)]
labels = sorted(labels, cmp=lambda x, y: x.endswith("*") and -1 or 1)
label = escape("\r".join(labels))
e.set_label(label)
return edges[edge_tuple]
for host, asmaps in data.iteritems():
add_node(host, label= "%s\r%s" % (host.upper(), app.config["DOMAIN"].upper()), shape="box", fillcolor="#F5A9A9")
add_node(host, label="%s\r%s" % (host.upper(), app.config["DOMAIN"].upper()), shape="box", fillcolor="#F5A9A9")
as_number = app.config["AS_NUMBER"].get(host, None)
if as_number:
@ -443,7 +454,7 @@ def show_bgpmap():
edge.set_color("red")
edge.set_style("bold")
#colors = [ "#009e23", "#1a6ec1" , "#d05701", "#6f879f", "#939a0e", "#0e9a93", "#9a0e85", "#56d8e1" ]
# colors = [ "#009e23", "#1a6ec1" , "#d05701", "#6f879f", "#939a0e", "#0e9a93", "#9a0e85", "#56d8e1" ]
previous_as = None
hosts = data.keys()
for host, asmaps in data.iteritems():
@ -459,8 +470,11 @@ def show_bgpmap():
continue
if not hop:
if app.config.get('BIRD_HAS_FULL_VIEW', False):
hop = True
if _as not in hosts:
hop_label = ''
continue
elif _as not in hosts:
hop_label = _as
if first:
hop_label = hop_label + "*"
@ -468,7 +482,6 @@ def show_bgpmap():
else:
hop_label = ""
add_node(_as, fillcolor=(first and "#F5A9A9" or "white"))
if hop_label:
edge = add_edge(nodes[previous_as], nodes[_as], label=hop_label, fontsize="7")
@ -491,7 +504,7 @@ def show_bgpmap():
node = add_node(previous_as)
node.set_shape("box")
#return Response("<pre>" + graph.create_dot() + "</pre>")
# return Response("<pre>" + graph.create_dot() + "</pre>")
return Response(graph.create_png(), mimetype='image/png')
@ -501,10 +514,14 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
path = None
paths = []
net_dest = None
re_via = re.compile(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on.*\[(\w+)\s+')
re_unreachable = re.compile(r'(.*)unreachable\s+\[(\w+)\s+')
for line in text:
line = line.strip()
expr = re.search(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on.*\[(\w+)\s+', line)
expr = re_via.search(line)
if expr:
if path:
path.append(net_dest)
@ -524,10 +541,10 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
break
else:
# ugly hack for good printing
path = [ peer_protocol_name ]
# path = ["%s\r%s" % (peer_protocol_name, get_as_name(get_as_number_from_protocol_name(host, proto, peer_protocol_name)))]
path = [peer_protocol_name]
# path = ["%s\r%s" % (peer_protocol_name, get_as_name(get_as_number_from_protocol_name(host, proto, peer_protocol_name)))]
expr2 = re.search(r'(.*)unreachable\s+\[(\w+)\s+', line)
expr2 = re_unreachable.search(line)
if expr2:
if path:
path.append(net_dest)
@ -547,6 +564,93 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text):
return paths
def build_as_tree_from_full_view(host, proto, res):
re_chunk_start = re.compile(r'(.*)unreachable\s+\[(.*)\s+.*\s+from\s+(.*)\].*\(.*\)\s\[.*\]')
dest_subnet = None
raw = defaultdict(dict)
for line in res:
line = line.strip()
expr = re_chunk_start.search(line)
if expr:
# Beginning of the BGP reply chunk
if not dest_subnet:
dest_subnet = expr.group(1).strip()
router_tag = expr.group(2).strip()
router_ip = expr.group(3).strip()
try:
router_ip = resolve_ptr(router_ip)
except NXDOMAIN:
# If PTR record can't be found, IP will do too
pass
elif line.startswith('BGP.as_path:'):
# BGP AS path
line = line.replace('BGP.as_path:', '')
line = line.strip()
path = [router_tag, ]
for as_num in line.split(' '):
if as_num:
path.append(as_num)
path_tag = '+'.join(path[1:])
if path_tag not in raw:
raw[path_tag] = list()
raw[path_tag].append(dict(router_tag=router_tag, router_ip=router_ip, path=path))
elif line.startswith('BGP.community:'):
# BGP community
line = line.replace('BGP.community:', '')
line = line.strip()
raw[path_tag][-1]['community'] = line.split(' ')
elif line.startswith('BGP.cluster_list:'):
# BGP cluster size
line = line.replace('BGP.cluster_list:', '')
line = line.strip()
raw[path_tag][-1]['cluster_size'] = len(line.split(' '))
for path_tag in raw:
raw[path_tag] = iter(raw[path_tag])
result = defaultdict(list)
exhausted_tags = set()
existing_paths_num = len(raw)
if len(raw) > app.config.get('MAX_PATHS', 10):
max_paths = existing_paths_num
else:
max_paths = app.config.get('MAX_PATHS', 10)
path_count = 0
while path_count < max_paths:
for path_tag in sorted(raw, key=lambda x: x.count('+')):
if path_tag in exhausted_tags:
continue
try:
path = next(raw[path_tag])
except StopIteration:
exhausted_tags.add(path_tag)
continue
result[path['router_ip']].append(path['path'])
result[path['router_ip']][-1].append(dest_subnet)
path_count += 1
if path_count == max_paths:
break
if path_count == max_paths or len(exhausted_tags) == existing_paths_num:
break
return result
def show_route(request_type, hosts, proto):
expression = get_query()
if not expression:
@ -609,6 +713,9 @@ def show_route(request_type, hosts, proto):
continue
if bgpmap:
if app.config['BIRD_HAS_FULL_VIEW']:
detail = build_as_tree_from_full_view(host, proto, res)
else:
detail[host] = build_as_tree_from_raw_bird_ouput(host, proto, res)
else:
detail[host] = add_links(res)

50
lgproxy.py
View file

@ -39,71 +39,81 @@ file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="m
app.logger.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper()))
app.logger.addHandler(file_handler)
@app.before_request
def access_log_before(*args, **kwargs):
app.logger.info("[%s] request %s, %s", request.remote_addr, request.url, "|".join(["%s:%s"%(k,v) for k,v in request.headers.items()]))
app.logger.info("[%s] request %s, %s", request.remote_addr, request.url, "|".join(["%s:%s" % (k, v) for k, v in request.headers.items()]))
@app.after_request
def access_log_after(response, *args, **kwargs):
app.logger.info("[%s] reponse %s, %s", request.remote_addr, request.url, response.status_code)
return response
def check_accesslist():
if app.config["ACCESS_LIST"] and request.remote_addr not in app.config["ACCESS_LIST"]:
abort(401)
def check_features():
features = app.config.get('FEATURES', [])
if request.endpoint not in features:
abort(401)
@app.route("/traceroute")
@app.route("/traceroute6")
def traceroute():
check_accesslist()
check_features()
if sys.platform.startswith('freebsd') or sys.platform.startswith('netbsd') or sys.platform.startswith('openbsd'):
traceroute4 = [ 'traceroute' ]
traceroute6 = [ 'traceroute6' ]
traceroute4 = ['traceroute']
traceroute6 = ['traceroute6']
else: # For Linux
traceroute4 = [ 'traceroute', '-4' ]
traceroute6 = [ 'traceroute', '-6' ]
traceroute4 = ['traceroute', '-4']
traceroute6 = ['traceroute', '-6']
src = []
if request.path == '/traceroute6':
traceroute = traceroute6
if app.config.get("IPV6_SOURCE",""):
src = [ "-s", app.config.get("IPV6_SOURCE") ]
if app.config.get("IPV6_SOURCE", ""):
src = ["-s", app.config.get("IPV6_SOURCE")]
else:
traceroute = traceroute4
if app.config.get("IPV4_SOURCE",""):
src = [ "-s", app.config.get("IPV4_SOURCE") ]
if app.config.get("IPV4_SOURCE", ""):
src = ["-s", app.config.get("IPV4_SOURCE")]
query = request.args.get("q","")
query = request.args.get("q", "")
query = unquote(query)
if sys.platform.startswith('freebsd') or sys.platform.startswith('netbsd'):
options = [ '-a', '-q1', '-w1', '-m15' ]
options = ['-a', '-q1', '-w1', '-m15']
elif sys.platform.startswith('openbsd'):
options = [ '-A', '-q1', '-w1', '-m15' ]
options = ['-A', '-q1', '-w1', '-m15']
else: # For Linux
options = [ '-A', '-q1', '-N32', '-w1', '-m15' ]
command = traceroute + src + options + [ query ]
result = subprocess.Popen( command , stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore').replace("\n","<br>")
options = ['-A', '-q1', '-N32', '-w1', '-m15']
command = traceroute + src + options + [query]
result = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore').replace("\n", "<br>")
return result
@app.route("/bird")
@app.route("/bird6")
def bird():
check_accesslist()
check_features()
if request.path == "/bird":
b = BirdSocket(file=app.config.get("BIRD_SOCKET", "/var/run/bird.ctl"))
b = BirdSocket(file=app.config.get('BIRD_SOCKET'))
elif request.path == "/bird6":
b = BirdSocket(file=app.config.get("BIRD6_SOCKET", "/var/run/bird6.ctl"))
b = BirdSocket(file=app.config.get('BIRD6_SOCKET'))
else:
return "No bird socket selected"
query = request.args.get("q","")
query = request.args.get("q", "")
query = unquote(query)
status, result = b.cmd(query)

27
toolbox.py
View file

@ -19,18 +19,32 @@
#
###
from dns import resolver
from dns import resolver, reversename
import socket
import pickle
import xml.parsers.expat
import re
from flask import Flask
resolv = resolver.Resolver()
resolv.timeout = 0.5
resolv.lifetime = 1
app = Flask(__name__)
app.config.from_pyfile('lg.cfg')
def resolve(n, q):
return str(resolv.query(n,q)[0])
return str(resolv.query(n, q)[0])
def resolve_ptr(ip):
ptr = str(resolve(reversename.from_address(ip), 'PTR')).lower()
ptr = ptr.replace(app.config.get('ROUTER_NAME_REMOVE', ''), '')
return ptr
asname_regex = re.compile("(ASName|as-name):\s+(?P<name>\S+)")
@ -45,10 +59,11 @@ def mask_is_valid(n):
return True
try:
mask = int(n)
return ( mask >= 1 and mask <= 128)
return (mask >= 1 and mask <= 128)
except:
return False
def ipv4_is_valid(n):
try:
socket.inet_pton(socket.AF_INET, n)
@ -56,6 +71,7 @@ def ipv4_is_valid(n):
except socket.error:
return False
def ipv6_is_valid(n):
try:
socket.inet_pton(socket.AF_INET6, n)
@ -63,12 +79,14 @@ def ipv6_is_valid(n):
except socket.error:
return False
def save_cache_pickle(filename, data):
output = open(filename, 'wb')
pickle.dump(data, output)
output.close()
def load_cache_pickle(filename, default = None):
def load_cache_pickle(filename, default=None):
try:
pkl_file = open(filename, 'rb')
except IOError:
@ -80,6 +98,7 @@ def load_cache_pickle(filename, default = None):
pkl_file.close()
return data
def unescape(s):
want_unicode = False
if isinstance(s, unicode):