diff --git a/.gitignore b/.gitignore index a8c5f5a..fca2509 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ -*.pyc -*.pyo +*.py[co] +*.egg +*.egg-info +eggs + *.cfg +*.log diff --git a/lg.py b/lg.py index cc561d2..3991495 100644 --- a/lg.py +++ b/lg.py @@ -20,22 +20,23 @@ # ### -import memcache -import subprocess -import logging +from collections import defaultdict from logging.handlers import TimedRotatingFileHandler -import re -from urllib2 import urlopen from urllib import quote, unquote +from urllib2 import urlopen import json +import logging +import memcache import random +import re +import subprocess from toolbox import mask_is_valid, ipv6_is_valid, ipv4_is_valid, resolve, save_cache_pickle, load_cache_pickle, get_asname_from_whois, unescape -#from xml.sax.saxutils import escape -import pydot +from dns.resolver import NXDOMAIN from flask import Flask, render_template, jsonify, redirect, session, request, abort, Response, Markup +import pydot app = Flask(__name__) app.config.from_pyfile('lg.cfg') @@ -47,9 +48,19 @@ file_handler.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper())) app.logger.addHandler(file_handler) memcache_server = app.config.get("MEMCACHE_SERVER", "127.0.0.1:11211") -memcache_expiration = int(app.config.get("MEMCACHE_EXPIRATION", "1296000")) # 15 days by default +memcache_expiration = int(app.config.get("MEMCACHE_EXPIRATION", "1296000")) # 15 days by default mc = memcache.Client([memcache_server]) + +def get_asn_from_as(n): + asn_zone = app.config.get("ASN_ZONE", "asn.cymru.com") + try: + data = resolve("AS%s.%s" % (n, asn_zone), "TXT").replace("'", "").replace('"', '') + except: + return " " * 5 + return [field.strip() for field in data.split("|")] + + def add_links(text): """Browser a string and replace ipv4, ipv6, as number, with a whois link """ @@ -60,8 +71,7 @@ def add_links(text): ret_text = [] for line in text: # Some heuristic to create link - if line.strip().startswith("BGP.as_path:") or \ - line.strip().startswith("Neighbor AS:"): + if line.strip().startswith("BGP.as_path:") or line.strip().startswith("Neighbor AS:"): ret_text.append(re.sub(r'(\d+)', r'\1', line)) else: line = re.sub(r'([a-zA-Z0-9\-]*\.([a-zA-Z]{2,3}){1,2})(\s|$)', r'\1\3', line) @@ -102,7 +112,7 @@ def set_session(request_type, hosts, proto, request_args): def whois_command(query): server = [] if app.config.get("WHOIS_SERVER", ""): - server = [ "-h", app.config.get("WHOIS_SERVER") ] + server = ["-h", app.config.get("WHOIS_SERVER")] return subprocess.Popen(['whois'] + server + [query], stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore') @@ -134,7 +144,7 @@ def bird_proxy(host, proto, service, query): elif not path: return False, 'Proto "%s" invalid' % proto else: - url = "http://%s.%s:%d/%s?q=%s" % (host, app.config["DOMAIN"], port, path, quote(query)) + url = 'http://{}:{}/{}?q={}'.format(app.config['ROUTER_IP'][host][0], port, path, quote(query)) try: f = urlopen(url) resultat = f.read() @@ -148,18 +158,18 @@ def bird_proxy(host, proto, service, query): @app.context_processor def inject_commands(): commands = [ - ("traceroute", "traceroute ..."), - ("summary", "show protocols"), - ("detail", "show protocols ... all"), - ("prefix", "show route for ..."), - ("prefix_detail", "show route for ... all"), - ("prefix_bgpmap", "show route for ... (bgpmap)"), - ("where", "show route where net ~ [ ... ]"), - ("where_detail", "show route where net ~ [ ... ] all"), - ("where_bgpmap", "show route where net ~ [ ... ] (bgpmap)"), - ("adv", "show route ..."), - ("adv_bgpmap", "show route ... (bgpmap)"), - ] + ("traceroute", "traceroute ..."), + ("summary", "show protocols"), + ("detail", "show protocols ... all"), + ("prefix", "show route for ..."), + ("prefix_detail", "show route for ... all"), + ("prefix_bgpmap", "show route for ... (bgpmap)"), + ("where", "show route where net ~ [ ... ]"), + ("where_detail", "show route where net ~ [ ... ] all"), + ("where_bgpmap", "show route where net ~ [ ... ] (bgpmap)"), + ("adv", "show route ..."), + ("adv_bgpmap", "show route ... (bgpmap)"), + ] commands_dict = {} for id, text in commands: commands_dict[id] = text @@ -189,10 +199,12 @@ def incorrect_request(e): def page_not_found(e): return render_template('error.html', warnings=["The requested URL was not found on the server."]), 404 + def get_query(): q = unquote(request.args.get('q', '').strip()) return q + @app.route("/whois") def whois(): query = get_query() @@ -313,7 +325,6 @@ def traceroute(hosts, proto): errors.append("%s" % resultat) continue - infos[host] = add_links(resultat) return render_template('traceroute.html', infos=infos, errors=errors) @@ -406,7 +417,7 @@ def show_bgpmap(): def add_node(_as, **kwargs): if _as not in nodes: - kwargs["label"] = '<
' + escape(kwargs.get("label", get_as_name(_as))).replace("\r"," ") + " |
' + escape(kwargs.get("label", get_as_name(_as))).replace("\r", " ") + " |
" + graph.create_dot() + "") + # return Response("
" + graph.create_dot() + "") return Response(graph.create_png(), mimetype='image/png') @@ -501,10 +514,14 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text): path = None paths = [] net_dest = None + + re_via = re.compile(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on.*\[(\w+)\s+') + re_unreachable = re.compile(r'(.*)unreachable\s+\[(\w+)\s+') + for line in text: line = line.strip() - expr = re.search(r'(.*)via\s+([0-9a-fA-F:\.]+)\s+on.*\[(\w+)\s+', line) + expr = re_via.search(line) if expr: if path: path.append(net_dest) @@ -524,10 +541,10 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text): break else: # ugly hack for good printing - path = [ peer_protocol_name ] -# path = ["%s\r%s" % (peer_protocol_name, get_as_name(get_as_number_from_protocol_name(host, proto, peer_protocol_name)))] - - expr2 = re.search(r'(.*)unreachable\s+\[(\w+)\s+', line) + path = [peer_protocol_name] + # path = ["%s\r%s" % (peer_protocol_name, get_as_name(get_as_number_from_protocol_name(host, proto, peer_protocol_name)))] + + expr2 = re_unreachable.search(line) if expr2: if path: path.append(net_dest) @@ -539,7 +556,7 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text): if line.startswith("BGP.as_path:"): path.extend(line.replace("BGP.as_path:", "").strip().split(" ")) - + if path: path.append(net_dest) paths.append(path) @@ -547,6 +564,93 @@ def build_as_tree_from_raw_bird_ouput(host, proto, text): return paths +def build_as_tree_from_full_view(host, proto, res): + re_chunk_start = re.compile(r'(.*)unreachable\s+\[(.*)\s+.*\s+from\s+(.*)\].*\(.*\)\s\[.*\]') + dest_subnet = None + raw = defaultdict(dict) + + for line in res: + line = line.strip() + expr = re_chunk_start.search(line) + + if expr: + # Beginning of the BGP reply chunk + if not dest_subnet: + dest_subnet = expr.group(1).strip() + + router_tag = expr.group(2).strip() + router_ip = expr.group(3).strip() + + try: + router_ip = resolve_ptr(router_ip) + except NXDOMAIN: + # If PTR record can't be found, IP will do too + pass + + elif line.startswith('BGP.as_path:'): + # BGP AS path + line = line.replace('BGP.as_path:', '') + line = line.strip() + path = [router_tag, ] + for as_num in line.split(' '): + if as_num: + path.append(as_num) + + path_tag = '+'.join(path[1:]) + + if path_tag not in raw: + raw[path_tag] = list() + + raw[path_tag].append(dict(router_tag=router_tag, router_ip=router_ip, path=path)) + + elif line.startswith('BGP.community:'): + # BGP community + line = line.replace('BGP.community:', '') + line = line.strip() + raw[path_tag][-1]['community'] = line.split(' ') + + elif line.startswith('BGP.cluster_list:'): + # BGP cluster size + line = line.replace('BGP.cluster_list:', '') + line = line.strip() + raw[path_tag][-1]['cluster_size'] = len(line.split(' ')) + + for path_tag in raw: + raw[path_tag] = iter(raw[path_tag]) + + result = defaultdict(list) + exhausted_tags = set() + existing_paths_num = len(raw) + if len(raw) > app.config.get('MAX_PATHS', 10): + max_paths = existing_paths_num + else: + max_paths = app.config.get('MAX_PATHS', 10) + path_count = 0 + + while path_count < max_paths: + for path_tag in sorted(raw, key=lambda x: x.count('+')): + if path_tag in exhausted_tags: + continue + + try: + path = next(raw[path_tag]) + except StopIteration: + exhausted_tags.add(path_tag) + continue + + result[path['router_ip']].append(path['path']) + result[path['router_ip']][-1].append(dest_subnet) + + path_count += 1 + if path_count == max_paths: + break + + if path_count == max_paths or len(exhausted_tags) == existing_paths_num: + break + + return result + + def show_route(request_type, hosts, proto): expression = get_query() if not expression: @@ -609,7 +713,10 @@ def show_route(request_type, hosts, proto): continue if bgpmap: - detail[host] = build_as_tree_from_raw_bird_ouput(host, proto, res) + if app.config['BIRD_HAS_FULL_VIEW']: + detail = build_as_tree_from_full_view(host, proto, res) + else: + detail[host] = build_as_tree_from_raw_bird_ouput(host, proto, res) else: detail[host] = add_links(res) diff --git a/lgproxy.py b/lgproxy.py index a6f7aac..dc81509 100644 --- a/lgproxy.py +++ b/lgproxy.py @@ -35,82 +35,92 @@ app = Flask(__name__) app.debug = app.config["DEBUG"] app.config.from_pyfile('lgproxy.cfg') -file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight") +file_handler = TimedRotatingFileHandler(filename=app.config["LOG_FILE"], when="midnight") app.logger.setLevel(getattr(logging, app.config["LOG_LEVEL"].upper())) app.logger.addHandler(file_handler) + @app.before_request def access_log_before(*args, **kwargs): - app.logger.info("[%s] request %s, %s", request.remote_addr, request.url, "|".join(["%s:%s"%(k,v) for k,v in request.headers.items()])) + app.logger.info("[%s] request %s, %s", request.remote_addr, request.url, "|".join(["%s:%s" % (k, v) for k, v in request.headers.items()])) + @app.after_request def access_log_after(response, *args, **kwargs): - app.logger.info("[%s] reponse %s, %s", request.remote_addr, request.url, response.status_code) + app.logger.info("[%s] reponse %s, %s", request.remote_addr, request.url, response.status_code) return response + def check_accesslist(): - if app.config["ACCESS_LIST"] and request.remote_addr not in app.config["ACCESS_LIST"]: + if app.config["ACCESS_LIST"] and request.remote_addr not in app.config["ACCESS_LIST"]: abort(401) + +def check_features(): + features = app.config.get('FEATURES', []) + if request.endpoint not in features: + abort(401) + + @app.route("/traceroute") @app.route("/traceroute6") def traceroute(): check_accesslist() - + check_features() + if sys.platform.startswith('freebsd') or sys.platform.startswith('netbsd') or sys.platform.startswith('openbsd'): - traceroute4 = [ 'traceroute' ] - traceroute6 = [ 'traceroute6' ] - else: # For Linux - traceroute4 = [ 'traceroute', '-4' ] - traceroute6 = [ 'traceroute', '-6' ] + traceroute4 = ['traceroute'] + traceroute6 = ['traceroute6'] + else: # For Linux + traceroute4 = ['traceroute', '-4'] + traceroute6 = ['traceroute', '-6'] src = [] - if request.path == '/traceroute6': - traceroute = traceroute6 - if app.config.get("IPV6_SOURCE",""): - src = [ "-s", app.config.get("IPV6_SOURCE") ] + if request.path == '/traceroute6': + traceroute = traceroute6 + if app.config.get("IPV6_SOURCE", ""): + src = ["-s", app.config.get("IPV6_SOURCE")] + else: + traceroute = traceroute4 + if app.config.get("IPV4_SOURCE", ""): + src = ["-s", app.config.get("IPV4_SOURCE")] - else: - traceroute = traceroute4 - if app.config.get("IPV4_SOURCE",""): - src = [ "-s", app.config.get("IPV4_SOURCE") ] - - query = request.args.get("q","") + query = request.args.get("q", "") query = unquote(query) if sys.platform.startswith('freebsd') or sys.platform.startswith('netbsd'): - options = [ '-a', '-q1', '-w1', '-m15' ] + options = ['-a', '-q1', '-w1', '-m15'] elif sys.platform.startswith('openbsd'): - options = [ '-A', '-q1', '-w1', '-m15' ] - else: # For Linux - options = [ '-A', '-q1', '-N32', '-w1', '-m15' ] - command = traceroute + src + options + [ query ] - result = subprocess.Popen( command , stdout=subprocess.PIPE).communicate()[0].decode('utf-8', 'ignore').replace("\n","