#
# Hack to make the script directory is also threated as a module search path.
import sys
import os
sys.path.append(os.path.dirname(__file__))
SVN = next(filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn')))
SVNVERSION = next(filter(os.path.isfile, ('/usr/local/bin/svnversion', '/usr/bin/svnversion')))
import argparse
import cgi
import cgitb
import codecs
import copy
import glob
import make_network_kml
import math
import pyproj
import random
import re
import socket
import string
import subprocess
import textwrap
import time
import traceback
import urllib
from pprint import pprint
from collections import defaultdict, OrderedDict
from operator import itemgetter, attrgetter
from sys import stderr
try:
import yaml
except ImportError as e:
print(e)
print("[ERROR] Please install the python-yaml or devel/py-yaml package")
exit(1)
try:
from yaml import CLoader as Loader
from yaml import CDumper as Dumper
except ImportError:
from yaml import Loader, Dumper
from jinja2 import Environment, Template
def yesorno(value):
return "YES" if bool(value) else "NO"
env = Environment()
env.filters['yesorno'] = yesorno
def render_template(datadump, template):
result = env.from_string(template).render(datadump)
# Make it look pretty to the naked eye, as jinja templates are not so
# friendly when it comes to whitespace formatting
## Remove extra whitespace at end of line lstrip() style.
result = re.sub(r'\n[\ ]+','\n', result)
## Include only a single newline between an definition and a comment
result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
## Remove extra newlines after single comment
result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
return result
import logging
logging.basicConfig(format='# %(levelname)s: %(message)s' )
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
if os.environ.get('CONFIGROOT', None):
NODE_DIR = os.environ['CONFIGROOT']
else:
NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
__version__ = '$Id: gformat.py 14374 2019-06-11 10:41:16Z rick $'
CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
files = [
'authorized_keys',
'dnsmasq.conf',
'dhcpd.conf',
'rc.conf.local',
'resolv.conf',
'motd',
'ntp.conf',
'pf.hybrid.conf.local',
'unbound.wleiden.conf',
'wleiden.yaml',
]
# Global variables uses
OK = 10
DOWN = 20
UNKNOWN = 90
ileiden_proxies = OrderedDict()
normal_proxies = []
datadump_cache = {}
interface_list_cache = {}
rc_conf_local_cache = {}
nameservers_cache = []
relations_cache = None
NO_DHCP = 0
DHCP_CLIENT = 10
DHCP_SERVER = 20
def dhcp_type(item):
if not 'dhcp' in item:
return NO_DHCP
elif not item['dhcp']:
return NO_DHCP
elif item['dhcp'].lower() == 'client':
return DHCP_CLIENT
else:
# Validation Checks
begin,end = map(int,item['dhcp'].split('-'))
if begin >= end:
raise ValueError("DHCP Start >= DHCP End")
return DHCP_SERVER
def etrs2rd(lat, lon):
p1 = pyproj.Proj(proj='latlon',datum='WGS84')
p2 = pyproj.Proj(init='EPSG:28992')
RDx, RDy = pyproj.transform(p1,p2,lon, lat)
return (RDx, RDy)
def rd2etrs(RDx, RDy):
p1 = pyproj.Proj(init='EPSG:28992')
p2 = pyproj.Proj(proj='latlon',datum='WGS84')
lon, lat = pyproj.transform(p1,p2, RDx, RDy)
return (lat, lon)
def get_yaml(item,add_version_info=True):
try:
""" Get configuration yaml for 'item'"""
if item in datadump_cache:
return datadump_cache[item].copy()
gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
# Default values
datadump = {
'autogen_revision' : 'NOTFOUND',
'autogen_gfile' : gfile,
'service_proxy_ileiden' : False,
'publicnat' : ['http', 'https'],
}
f = open(gfile, 'r')
datadump.update(yaml.load(f,Loader=Loader))
datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
if datadump['nodetype'] == 'Hybrid':
# Some values are defined implicitly
if 'rdr_host' in datadump and datadump['rdr_host'] and not 'service_incoming_rdr' in datadump:
datadump['service_incoming_rdr'] = True
# Use some boring defaults
defaults = {
'service_proxy_normal' : False,
'service_accesspoint' : True,
'service_incoming_rdr' : False,
'service_concentrator' : False,
'monitoring_group' : 'wleiden',
}
for (key,value) in defaults.items():
if not key in datadump:
datadump[key] = value
f.close()
# Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
if add_version_info:
p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True)
lines = p.communicate()[0].split('\n')
if p.returncode == 0:
for line in lines:
if line:
(key, value) = line.split(': ')
datadump["autogen_" + key.lower().replace(' ','_')] = value
# Preformat certain needed variables for formatting and push those into special object
datadump['autogen_iface_keys'] = get_interface_keys(datadump)
wlan_count=0
try:
for key in get_interface_keys(datadump, True):
datadump[key]['autogen_ifbase'] = key.split('_')[1]
datadump[key]['autogen_vlan'] = False
datadump[key]['autogen_vlan_alias'] = False
datadump[key]['autogen_bridge_member'] = 'parent' in datadump[key]
datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
if 'parent' in datadump[key]:
if 'ip' in datadump[key]:
raise ValueError("Interface bridge member cannot have IP assigned")
if 'dhcp' in datadump[key] and datadump[key]['dhcp'] != False:
raise ValueError("Interface bridge member cannot have DHCP set")
if 'ip' in datadump[key]:
datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
wlan_count += 1
else:
datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
datadump[key]['autogen_if_dhcp'] = '.'.join(key.split('_')[1:3])
datadump[key]['autogen_vlan'] = key.split('_')[2]
datadump[key]['autogen_vlan_alias'] = '_alias' in key
datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
else:
datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
except Exception as exc:
exc.args = ("# Error while processing interface %s" % key,) + exc.args
raise
dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
if dhcp_type(datadump[key]) == DHCP_SERVER]
datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
datadump['autogen_item'] = item
datadump['autogen_domain'] = datadump['domain'] if 'domain' in datadump else 'wleiden.net.'
datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
datadump_cache[item] = datadump.copy()
except Exception as exc:
exc.args = ("# Error while processing %s" % item,) + exc.args
raise
return datadump
def store_yaml(datadump, header=False):
""" Store configuration yaml for 'item'"""
item = datadump['autogen_item']
gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
output = generate_wleiden_yaml(datadump, header)
f = open(gfile, 'w')
f.write(output)
f.close()
def network(ip):
addr, mask = ip.split('/')
# Not parsing of these folks please
addr = parseaddr(addr)
mask = int(mask)
network = addr & ~((1 << (32 - mask)) - 1)
return network
def make_relations():
""" Process _ALL_ yaml files to get connection relations """
global relations_cache
if relations_cache:
return relations_cache
errors = []
poel = defaultdict(list)
for host in get_hostlist():
datadump = get_yaml(host)
try:
for iface_key in get_interface_keys(datadump):
# Bridge members has no IP assigned
if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
continue
net_addr = network(datadump[iface_key]['ip'])
poel[net_addr] += [(host,datadump[iface_key].copy())]
except (KeyError, ValueError) as e:
errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
continue
relations_cache = (poel, errors)
return relations_cache
def valid_addr(addr):
""" Show which address is valid in which are not """
return str(addr).startswith('172.')
def get_hostlist():
""" Combined hosts and proxy list"""
return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
def angle_between_points(lat1,lat2,long1,long2):
"""
Return Angle in radians between two GPS coordinates
See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
"""
dy = lat2 - lat1
dx = math.cos(lat1)*(long2 - long1)
angle = math.atan2(dy,dx)
return angle
def angle_to_cd(angle):
""" Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
# For easy conversion get positive degree
degrees = math.degrees(angle)
abs_degrees = 360 + degrees if degrees < 0 else degrees
# Numbers can be confusing calculate from the 4 main directions
p = 22.5
if abs_degrees < p:
cd = "n"
elif abs_degrees < (90 - p):
cd = "no"
elif abs_degrees < (90 + p):
cd = "o"
elif abs_degrees < (180 - p):
cd = "zo"
elif abs_degrees < (180 + p):
cd = "z"
elif abs_degrees < (270 - p):
cd = "zw"
elif abs_degrees < (270 + p):
cd = "w"
elif abs_degrees < (360 - p):
cd = "nw"
else:
cd = "n"
return cd
def cd_between_hosts(hostA, hostB, datadumps):
# Using RDNAP coordinates
dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
return angle_to_cd(math.atan2(dx,dy))
# GPS coordinates seems to fail somehow
#latA = float(datadumps[hostA]['latitude'])
#latB = float(datadumps[hostB]['latitude'])
#lonA = float(datadumps[hostA]['longitude'])
#lonB = float(datadumps[hostB]['longitude'])
#return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
def generate_title(nodelist):
""" Main overview page """
items = { \
'root' : ".",
'version' : subprocess.Popen([SVNVERSION, "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
}
def fl(spaces, line):
return (' ' * spaces) + line + '\n'
output = """
Wireless leiden Configurator - GFormat
Wireless Leiden Configurator - Revision %(version)s
""" % items
for node in nodelist:
items['node'] = node
output += fl(5, '') + fl(7,'%(node)s | ' % items)
for config in files:
items['config'] = config
output += fl(7,'%(config)s | ' % items)
output += fl(5, "
")
output += """
%s
""" % __version__
return output
def generate_node(node):
""" Print overview of all files available for node """
return "\n".join(files)
def generate_node_overview(host, datadump=False):
""" Print overview of all files available for node """
if not datadump:
datadump = get_yaml(host)
params = { 'host' : host }
output = "Back to overview
"
output += "Available files:
"
for cf in files:
params['cf'] = cf
output += '- %(cf)s
\n' % params
output += "
"
# Generate and connection listing
output += "Connected To:
"
(poel, errors) = make_relations()
for network, hosts in poel.items():
if host in [x[0] for x in hosts]:
if len(hosts) == 1:
# Single not connected interface
continue
for remote,ifacedump in hosts:
if remote == host:
# This side of the interface
continue
params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
output += '- %(remote)s -- %(remote_ip)s
\n' % params
output += "
"
output += "MOTD details:
" + generate_motd(datadump) + "
"
output += "
Back to overview"
return output
def generate_header(datadump, ctag="#"):
return """\
%(ctag)s
%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
%(ctag)s
""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
def parseaddr(s):
""" Process IPv4 CIDR notation addr to a (binary) number """
f = list(map(int, s.split('.')))
return ((f[0] << 24) + (f[1] << 16) + (f[2] << 8) + (f[3]))
def showaddr(a):
""" Display IPv4 addr in (dotted) CIDR notation """
return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
def is_member(ip, mask, canidate):
""" Return True if canidate is part of ip/mask block"""
ip_addr = parseaddr(ip)
ip_canidate = parseaddr(canidate)
mask = int(mask)
ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
return ip_addr == ip_canidate
def cidr2netmask(netmask):
""" Given a 'netmask' return corresponding CIDR """
return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
def get_network(addr, mask):
return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
def generate_dhcpd_conf(datadump):
""" Generate config file '/usr/local/etc/dhcpd.conf """
# Redundency support, in cause local DNS server is not running/responding.
datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
output = generate_header(datadump)
output += Template("""\
# option definitions common to all supported networks...
option domain-name "dhcp.{{ autogen_fqdn }}";
default-lease-time 600;
max-lease-time 7200;
# Use this to enble / disable dynamic dns updates globally.
#ddns-update-style none;
# If this DHCP server is the official DHCP server for the local
# network, the authoritative directive should be uncommented.
authoritative;
# Use this to send dhcp log messages to a different log file (you also
# have to hack syslog.conf to complete the redirection).
log-facility local7;
# UniFi Discovery Support
option space ubnt;
option ubnt.unifi-address code 1 = ip-address;
#
class "ubnt" {
match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
option vendor-class-identifier "ubnt";
vendor-option-space ubnt;
}
#
# Interface definitions
#
\n\n""").render(datadump)
# TODO: Use textwrap.fill instead
def indent(text, count):
return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
# Process DHCP blocks
dhcp_out = defaultdict(list)
for iface_key in get_interface_keys(datadump):
ifname = datadump[iface_key]['autogen_ifbase']
groupif = datadump[iface_key]['autogen_if_dhcp']
if not 'comment' in datadump[iface_key]:
datadump[iface_key]['comment'] = None
if not 'ip' in datadump[iface_key]:
continue
dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
(addr, mask) = datadump[iface_key]['ip'].split('/')
datadump[iface_key]['autogen_addr'] = addr
datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
dhcp_out[groupif].append(textwrap.dedent("""\
subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
### not autoritive
}
""" % datadump[iface_key]))
continue
(dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
dhcp_part = ".".join(addr.split('.')[0:3])
datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
# Assume the first 10 IPs could be used for static entries
if 'no_portal' in datadump:
fixed = 5
for mac in datadump['no_portal']:
dhcp_out[groupif].append(textwrap.dedent("""\
host fixed-%(ifname)s-%(fixed)s {
hardware ethernet %(mac)s;
fixed-address %(prefix)s.%(fixed)s;
}
""" % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
fixed += 1
if 'dhcp_fixed' in datadump[iface_key]:
for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
dhcp_out[groupif].append(textwrap.dedent("""\
host fixed-%(host)s {
hardware ethernet %(mac)s;
fixed-address %(addr)s;
}
""" % { 'host' : host, 'mac' : mac, 'addr' : addr}))
dhcp_out[groupif].append(textwrap.dedent("""\
subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
option routers %(autogen_addr)s;
option domain-name-servers %(autogen_dns_servers)s;
option ubnt.unifi-address 172.17.107.10;
}
""" % datadump[iface_key]))
# Output the blocks in groups
for ifname,value in sorted(dhcp_out.items()):
output += ("shared-network \"%s\" {\n" % ifname) + indent(''.join(value), 2).rstrip() + '\n}\n\n'
return output
def generate_dnsmasq_conf(datadump):
""" Generate configuration file '/usr/local/etc/dnsmasq.conf' """
output = generate_header(datadump)
output += Template("""\
# DHCP server options
dhcp-authoritative
dhcp-fqdn
domain=dhcp.{{ autogen_fqdn }}
domain-needed
expand-hosts
log-async=100
# Low memory footprint
cache-size=10000
\n""").render(datadump)
for iface_key in get_interface_keys(datadump):
if not 'comment' in datadump[iface_key]:
datadump[iface_key]['comment'] = None
output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
output += "# not autoritive\n\n"
continue
(dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
(ip, cidr) = datadump[iface_key]['ip'].split('/')
datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
dhcp_part = ".".join(ip.split('.')[0:3])
datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
return output
class AutoVivification(dict):
"""Implementation of perl's autovivification feature."""
def __getitem__(self, item):
try:
return dict.__getitem__(self, item)
except KeyError:
value = self[item] = type(self)()
return value
def make_interface_list(datadump):
if datadump['autogen_item'] in interface_list_cache:
return (interface_list_cache[datadump['autogen_item']])
# lo0 configuration:
# - 172.32.255.1/32 is the proxy.wleiden.net deflector
# - masterip is special as it needs to be assigned to at
# least one interface, so if not used assign to lo0
addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
vlan_list = defaultdict(list)
bridge_list = defaultdict(list)
flags_if = AutoVivification()
dhclient_if = {'lo0' : False}
# XXX: Find some way of send this output nicely
output = ''
masterip_used = False
for iface_key in get_interface_keys(datadump):
if 'ip' in datadump[iface_key] and datadump[iface_key]['ip'].startswith(datadump['masterip']):
masterip_used = True
break
if not masterip_used:
addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
if 'serviceid' in datadump:
addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
for iface_key in get_interface_keys(datadump):
ifacedump = datadump[iface_key]
if ifacedump['autogen_bridge_alias']:
ifname = ifacedump['autogen_ifbase']
else:
ifname = ifacedump['autogen_ifname']
# If defined as vlan interface
if ifacedump['autogen_vlan']:
vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
# If defined as bridge interface
if ifacedump['autogen_bridge_member']:
bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
# Flag dhclient is possible
if not ifname in dhclient_if or dhclient_if[ifname] == False:
dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
# Ethernet address
if 'ether' in ifacedump:
flags_if[ifname]['ether'] = ifacedump['ether']
# Handle special interface states
flags_if[ifname]['status'] = ifacedump['status']
# Add interface IP to list
if 'ip' in ifacedump:
item = (ifacedump['ip'], ifacedump['comment'])
if ifname in addrs_list:
addrs_list[ifname].append(item)
else:
addrs_list[ifname] = [item]
# Alias only needs IP assignment for now, this might change if we
# are going to use virtual accesspoints
if "alias" in iface_key:
continue
# XXX: Might want to deduct type directly from interface name
if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
# Default to station (client) mode
ifacedump['autogen_wlanmode'] = "sta"
if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
ifacedump['autogen_wlanmode'] = "ap"
if not 'channel' in ifacedump:
if ifacedump['type'] == '11a':
ifacedump['channel'] = 36
else:
ifacedump['channel'] = 1
# Allow special hacks at the back like wds and stuff
if not 'extra' in ifacedump:
ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
else:
ifacedump['autogen_extra'] = ifacedump['extra']
ifacedump['autogen_ssid_hex'] = '0x' + codecs.encode(ifacedump['ssid'].encode('ascii'), 'hex').decode('ascii')
output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
"%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
output += "\n"
elif ifacedump['type'] in ['ethernet', 'eth']:
# No special config needed besides IP
pass
elif ifacedump['type'] in ['vlan']:
# VLAN member has no special configuration
pass
else:
assert False, "Unknown type " + ifacedump['type']
store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
interface_list_cache[datadump['autogen_item']] = store
return(store)
def create_proxies_list():
if not ileiden_proxies or not normal_proxies:
# Placeholder for to-be-installed proxies, this will avoid updating the all
# nodes to include this new machine, yet due to an unbound issue, this list
# has to be kept small.
for i in range(1,20):
ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
for host in get_hostlist():
hostdump = get_yaml(host)
if hostdump['status'] == 'up':
if hostdump['service_proxy_ileiden']:
ileiden_proxies[hostdump['serviceid']] = hostdump
if hostdump['service_proxy_normal']:
normal_proxies.append(hostdump)
def generate_rc_conf_local(datadump):
""" Generate configuration file '/etc/rc.conf.local' """
item = datadump['autogen_item']
if item in rc_conf_local_cache:
return rc_conf_local_cache[item]
if not 'ileiden' in datadump:
datadump['autogen_ileiden_enable'] = False
else:
datadump['autogen_ileiden_enable'] = datadump['ileiden']
datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
create_proxies_list()
datadump['autogen_ileiden_proxies'] = ileiden_proxies
datadump['autogen_normal_proxies'] = normal_proxies
datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
output = generate_header(datadump, "#");
output += render_template(datadump, """\
hostname='{{ autogen_fqdn }}'
location='{{ location }}'
nodetype="{{ nodetype }}"
#
# Configured listings
#
captive_portal_whitelist=""
{% if nodetype == "Proxy" %}
#
# Proxy Configuration
#
{% if gateway and service_proxy_ileiden -%}
defaultrouter="{{ gateway }}"
{% else -%}
#defaultrouter="NOTSET"
{% endif -%}
internalif="{{ internalif }}"
ileiden_enable="{{ autogen_ileiden_enable }}"
gateway_enable="{{ autogen_ileiden_enable }}"
pf_enable="yes"
pf_rules="/etc/pf.conf"
{% if autogen_ileiden_enable -%}
pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={{ publicnat|join(',') }} -D ileiden_ports={{ publicnat|join(',') }}"
lvrouted_enable="{{ autogen_ileiden_enable }}"
lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
{% else -%}
pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
{% endif -%}
{% if internalroute -%}
static_routes="wleiden"
route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
{% endif -%}
{% elif nodetype == "Hybrid" %}
#
# Hybrid Configuration
#
list_ileiden_proxies="
{% for serviceid,item in autogen_ileiden_proxies.items() -%}
{{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
{% endfor -%}
"
list_normal_proxies="
{% for item in autogen_normal_proxies -%}
{{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
{% endfor -%}
"
{% if autogen_dhcp_interfaces -%}
captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
{% else %}
captive_portal_interfaces="dummy"
{% endif %}
externalif="{{ externalif|default('vr0', true) }}"
masterip="{{ masterip }}"
{% if gateway and service_proxy_ileiden %}
defaultrouter="{{ gateway }}"
{% else %}
#defaultrouter="NOTSET"
{% endif %}
#
# Defined services
#
service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
service_proxy_normal="{{ service_proxy_normal|yesorno }}"
service_accesspoint="{{ service_accesspoint|yesorno }}"
service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
service_concentrator="{{ service_concentrator|yesorno }}"
{% if service_proxy_ileiden %}
pf_rules="/etc/pf.hybrid.conf"
{% if service_concentrator %}
pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
{% else %}
pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
{% endif %}
pf_flags="$pf_flags -D publicnat={{ publicnat|join(',') }}"
lvrouted_flags="$lvrouted_flags -g"
{% elif service_proxy_normal or service_incoming_rdr %}
pf_rules="/etc/pf.hybrid.conf"
pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
pf_flags="$pf_flags -D publicnat=0"
lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
named_setfib="1"
tinyproxy_setfib="1"
dnsmasq_setfib="1"
sshd_setfib="1"
{% else %}
named_auto_forward_only="YES"
pf_rules="/etc/pf.node.conf"
pf_flags="-D ileiden_ports={{ publicnat|join(',') }}"
lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
{% endif %}
{% if service_concentrator %}
# Do mind installing certificates is NOT done automatically for security reasons
openvpn_enable="YES"
openvpn_configfile="/usr/local/etc/openvpn/client.conf"
{% endif %}
{% if service_proxy_normal %}
tinyproxy_enable="yes"
{% else %}
pen_wrapper_enable="yes"
{% endif %}
{% if service_accesspoint %}
pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
{% endif %}
{% if board == "ALIX2" or board == "net4801" %}
#
# ''Fat'' configuration, board has 256MB RAM
#
dnsmasq_enable="NO"
named_enable="YES"
unbound_enable="YES"
{% if autogen_dhcp_interfaces -%}
dhcpd_enable="YES"
dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
{% endif -%}
{% elif board == "apu1d" %}
#
# ''Fat'' configuration, board has 1024MB RAM
#
dnsmasq_enable="NO"
unbound_enable="YES"
{% if autogen_dhcp_interfaces -%}
dhcpd_enable="YES"
dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
{% endif -%}
{% endif -%}
{% endif %}
#
# Script variables
#
attached_devices="{{ autogen_attached_devices|join(' ') }}"
neighbours="{{ autogen_neighbours|join(' ') }}"
#
# Interface definitions
#\n
""")
(addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
for iface, vlans in sorted(vlan_list.items()):
output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
# VLAN Parent interfaces not containing a configuration should be marked active explcitly.
for iface in sorted(vlan_list.keys()):
if not iface in addrs_list.keys():
output += "ifconfig_%s='up'\n" % iface
output += "\n"
# Bridge configuration:
if bridge_list.keys():
output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
for iface in bridge_list.keys():
output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
# Bridge member interfaces not containing a configuration should be marked active explcitly.
for _,members in bridge_list.items():
for iface in members:
if not iface in addrs_list.keys():
output += "ifconfig_%s='up'\n" % iface.replace('.','_')
output += "\n"
# Details like SSID
if extra_ouput:
output += extra_ouput.strip() + "\n"
# Print IP address which needs to be assigned over here
output += "\n"
for iface,addrs in sorted(addrs_list.items()):
for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
output += "# %s || %s || %s\n" % (iface, addr, comment)
prefix = ''
if flags_if[iface]['status'] == 'broken':
output += "# Interface %s disabled since status=broken\n" % iface
prefix = '#'
# Write DHCLIENT entry
if iface in dhclient_if and dhclient_if[iface]:
output += prefix + "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
continue
# Make sure the external address is always first as this is needed in the
# firewall setup
addrs_tmp = []
for addr in addrs:
if addr[0].startswith('172'):
addrs_tmp.insert(0, addr)
else:
addrs_tmp.append(addr)
addrs = addrs_tmp
idx_offset = 0
# Set MAC is required
if 'ether' in flags_if[iface]:
output += prefix + "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
output += prefix + "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
idx_offset += 1
elif iface in bridge_list:
output += prefix + "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
idx_offset += 1
else:
output += prefix + "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
for idx, addr in enumerate(addrs[1:]):
output += prefix + "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
output += "\n"
rc_conf_local_cache[datadump['autogen_item']] = output
return output
def get_all_configs():
""" Get dict with key 'host' with all configs present """
configs = dict()
for host in get_hostlist():
datadump = get_yaml(host)
configs[host] = datadump
return configs
def get_interface_keys(config, extra=False):
""" Quick hack to get all interface keys, later stage convert this to a iterator """
elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
if extra == False:
return filter(lambda x: not "extra" in x, elems)
else:
return elems
def get_used_ips(configs):
""" Return array of all IPs used in config files"""
ip_list = []
for config in configs:
ip_list.append(config['masterip'])
if 'serviceid' in config:
ip_list.append(config['serviceid'])
for iface_key in get_interface_keys(config, True):
l = config[iface_key]['ip']
addr, mask = l.split('/')
# Special case do not process
if valid_addr(addr):
ip_list.append(addr)
else:
logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
return sorted(ip_list)
def get_nameservers(max_servers=None):
if nameservers_cache:
return nameservers_cache[0:max_servers]
for host in get_hostlist():
hostdump = get_yaml(host)
if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
return nameservers_cache[0:max_servers]
def get_neighbours(datadump):
(addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
(poel, errors) = make_relations()
table = []
for iface,addrs in sorted(addrs_list.items()):
if iface in ['lo0']:
continue
for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
if not addr.startswith('172.'):
# Avoid listing internet connections as pool
continue
for neighbour in poel[network(addr)]:
if neighbour[0] != datadump['autogen_item']:
table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
return table
def get_attached_devices(datadump, url=False):
table = []
for iface_key in get_interface_keys(datadump, True):
ifacedump = datadump[iface_key]
if not 'ns_ip' in ifacedump:
continue
x_ip = ifacedump['ns_ip'].split('/')[0]
if 'mode' in ifacedump:
x_mode = ifacedump['mode']
else:
x_mode = 'unknown'
if 'bridge_type' in ifacedump:
device_type = ifacedump['bridge_type']
else:
device_type = 'Unknown'
table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
return table
def generate_resolv_conf(datadump):
""" Generate configuration file '/etc/resolv.conf' """
# XXX: This should properly going to be an datastructure soon
datadump['autogen_header'] = generate_header(datadump, "#")
datadump['autogen_edge_nameservers'] = ''
for masterip,realname in get_nameservers():
datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
return Template("""\
{{ autogen_header }}
search wleiden.net
# Try local (cache) first
nameserver 127.0.0.1
{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
nameserver 8.8.8.8 # Google Public NameServer
nameserver 64.6.64.6 # Verisign Public NameServer
{% else -%}
# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
{{ autogen_edge_nameservers }}
{% endif -%}
""").render(datadump)
def generate_ntp_conf(datadump):
""" Generate configuration file '/etc/ntp.conf' """
# XXX: This should properly going to be an datastructure soon
datadump['autogen_header'] = generate_header(datadump, "#")
datadump['autogen_ntp_servers'] = ''
for host in get_hostlist():
hostdump = get_yaml(host)
if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
return Template("""\
{{ autogen_header }}
{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
# Machine hooked to internet.
server 0.nl.pool.ntp.org iburst maxpoll 9
server 1.nl.pool.ntp.org iburst maxpoll 9
server 2.nl.pool.ntp.org iburst maxpoll 9
server 3.nl.pool.ntp.org iburst maxpoll 9
{% else -%}
# Local Wireless Leiden NTP Servers.
server 0.pool.ntp.wleiden.net iburst maxpoll 9
server 1.pool.ntp.wleiden.net iburst maxpoll 9
server 2.pool.ntp.wleiden.net iburst maxpoll 9
server 3.pool.ntp.wleiden.net iburst maxpoll 9
# All the configured NTP servers
{{ autogen_ntp_servers }}
{% endif %}
# If a server loses sync with all upstream servers, NTP clients
# no longer follow that server. The local clock can be configured
# to provide a time source when this happens, but it should usually
# be configured on just one server on a network. For more details see
# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
# The use of Orphan Mode may be preferable.
#
server 127.127.1.0
fudge 127.127.1.0 stratum 10
""").render(datadump)
def generate_pf_hybrid_conf_local(datadump):
""" Generate configuration file '/etc/pf.hybrid.conf.local' """
datadump['autogen_header'] = generate_header(datadump, "#")
if datadump['service_incoming_rdr']:
datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
return Template("""\
{{ autogen_header }}
# Redirect some internal facing services outside (7)
# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
{% endfor -%}
# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
{% endfor -%}
""").render(datadump)
def generate_unbound_wleiden_conf(datadump):
""" Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
datadump['autogen_header'] = generate_header(datadump, "#")
autogen_ips = []
(addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
for iface,addrs in sorted(addrs_list.items()):
for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
if addr.startswith('172'):
autogen_ips.append((addr.split('/')[0], comment))
datadump['autogen_ips'] = autogen_ips
create_proxies_list()
datadump['autogen_ileiden_proxies'] = ileiden_proxies
return Template("""\
{{ autogen_header }}
server:
## Static definitions fail to start on systems with broken ue(4) interfaces
{%- for ip,comment in autogen_ips %}
# interface: {{ "%-16s"|format(ip) }} # {{ comment }}
{%- endfor %}
## Enabling wildcard matching as work-around
interface: 0.0.0.0
interface: ::0
forward-zone:
name: '.'
{%- if service_proxy_ileiden %}
forward-addr: 8.8.8.8 # Google DNS A
forward-addr: 8.8.4.4 # Google DNS B
forward-addr: 208.67.222.222 # OpenDNS DNS A
forward-addr: 208.67.220.220 # OpenDNS DNS B
{% else -%}
{%- for serviceid,item in autogen_ileiden_proxies.items() %}
{%- if loop.index <= 5 %}
forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
{%- endif %}
{%- endfor %}
{%- endif %}
""").render(datadump)
def generate_motd(datadump):
""" Generate configuration file '/etc/motd' """
output = Template("""\
FreeBSD run ``service motd onestart'' to make me look normal
WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
Loc: {{ location }}
Services:
{% if board == "ALIX2" or board == "net4801" -%}
{{" -"}} Core Node ({{ board }})
{% else -%}
{{" -"}} Hulp Node ({{ board }})
{% endif -%}
{% if service_proxy_normal -%}
{{" -"}} Normal Proxy
{% endif -%}
{% if service_proxy_ileiden -%}
{{" -"}} iLeiden Proxy
{% endif -%}
{% if service_incoming_rdr -%}
{{" -"}} Incoming port redirects
{% endif %}
Interlinks:\n
""").render(datadump)
def make_table(table):
if not table:
return " - none\n"
else:
lines = ""
col_width = [max(len(x) for x in col) for col in zip(*table)]
for row in table:
# replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
return lines
(addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
table = []
for iface,addrs in sorted(addrs_list.items()):
if iface in ['lo0']:
continue
for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
table.append((iface, addr, comment))
output += make_table(table)
output += '\n'
output += """\
Attached devices:
"""
output += make_table(get_attached_devices(datadump, url=True))
output += '\n'
output += """\
Available neighbours:
"""
output += make_table(get_neighbours(datadump))
return output
def format_yaml_value(value):
""" Get yaml value in right syntax for outputting """
if isinstance(value,str):
output = '"%s"' % value
else:
output = value
return output
def format_wleiden_yaml(datadump):
""" Special formatting to ensure it is editable"""
output = "# Genesis config yaml style\n"
output += "# vim:ts=2:et:sw=2:ai\n"
output += "#\n"
iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
for key in sorted(set(datadump.keys()) - set(iface_keys)):
if key == 'rdr_rules':
output += '%-10s:\n' % 'rdr_rules'
for rdr_rule in datadump[key]:
output += '- %s\n' % rdr_rule
else:
output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
output += "\n\n"
# Format (key, required)
key_order = (
('comment', True),
('parent', False),
('ip', False),
('ipv6', False),
('ether', False),
('desc', True),
('sdesc', True),
('mode', True),
('type', True),
('extra_type', False),
('channel', False),
('ssid', False),
('wlan_mac', False),
('dhcp', True),
('dhcp_fixed', False),
('compass', False),
('distance', False),
('ns_ip', False),
('repeater_ip', False),
('bullet2_ip', False),
('ns_mac', False),
('bullet2_mac', False),
('ns_type', False),
('bridge_type', False),
('encrypted', False),
('status', True),
)
for iface_key in sorted(iface_keys):
try:
remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
if remainder:
raise KeyError("invalid keys: %s" % remainder)
output += "%s:\n" % iface_key
for key,required in key_order:
if key in datadump[iface_key]:
output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
output += "\n\n"
except Exception as exc:
exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
raise
return output
def generate_wleiden_yaml(datadump, header=True):
""" Generate (petty) version of wleiden.yaml"""
output = generate_header(datadump, "#") if header else ''
for key in list(datadump.keys()):
if key.startswith('autogen_'):
del datadump[key]
# Interface autogen cleanups
elif type(datadump[key]) == dict:
for key2 in list(datadump[key].keys()):
if key2.startswith('autogen_'):
del datadump[key][key2]
output += format_wleiden_yaml(datadump)
return output
def generate_nanostation_config(datadump, iface, ns_type):
#TODO(rvdz): Make sure the proper nanostation IP and subnet is set
datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
datadump.update(datadump['iface_%s' % iface])
return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
def generate_yaml(datadump):
return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
def generate_config(node, config, datadump=None):
""" Print configuration file 'config' of 'node' """
output = ""
try:
# Load config file
if datadump == None:
datadump = get_yaml(node)
if config == 'wleiden.yaml':
output += generate_wleiden_yaml(datadump)
elif config == 'authorized_keys':
f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
output += f.read()
node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
# Fetch local keys if existing
if os.path.exists(node_keys):
output += open(node_keys, 'r').read()
f.close()
elif config == 'dnsmasq.conf':
output += generate_dnsmasq_conf(datadump)
elif config == 'dhcpd.conf':
output += generate_dhcpd_conf(datadump)
elif config == 'rc.conf.local':
output += generate_rc_conf_local(datadump)
elif config == 'resolv.conf':
output += generate_resolv_conf(datadump)
elif config == 'ntp.conf':
output += generate_ntp_conf(datadump)
elif config == 'motd':
output += generate_motd(datadump)
elif config == 'pf.hybrid.conf.local':
output += generate_pf_hybrid_conf_local(datadump)
elif config == 'unbound.wleiden.conf':
output += generate_unbound_wleiden_conf(datadump)
elif config.startswith('vr'):
interface, ns_type = config.strip('.yaml').split('-')
output += generate_nanostation_config(datadump, interface, ns_type)
else:
assert False, "Config not found!"
except IOError as e:
output += "[ERROR] Config file not found"
return output
def generate_static(output_dir, logging=True):
items = {'output_dir' : output_dir}
for node in get_hostlist():
items['node'] = node
items['wdir'] = "%(output_dir)s/%(node)s" % items
if not os.path.isdir(items['wdir']):
os.makedirs(items['wdir'])
datadump = get_yaml(node)
f = open("%(wdir)s/index.html" % items, "w")
f.write(generate_node_overview(items['node'], datadump))
f.close()
for config in files:
items['config'] = config
if logging: logger.info("## Generating %(node)s %(config)s" % items)
f = open("%(wdir)s/%(config)s" % items, "w")
f.write(generate_config(node, config, datadump))
f.close()
def process_cgi_request(environ=os.environ):
""" When calling from CGI """
response_headers = []
content_type = 'text/plain'
# Update repository if requested
form = urllib.parse.urlparse.parse_qs(environ['QUERY_STRING']) if QUERY_STRING in environ else None
if form and 'action' in form and 'update' in form['action']:
refresh_rate = 5
output = "[INFO] Updating subverion, please wait...\n"
old_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
new_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
if old_version != new_version or ('QUERY_STRING' in environ and 'force' in environ['QUERY_STRING']):
try:
generate_static(CACHE_DIR, False)
except:
output += traceback.format_exc()
refresh_rate = 120
pass
output += "[INFO] All done, redirecting in %s seconds" % refresh_rate
response_headers += [
('Refresh', '%s; url=.' % refresh_rate),
]
else:
# Bootstap cache directory if none exists
if not glob.glob(os.path.join(CACHE_DIR,'*','index.html')):
generate_static(CACHE_DIR, False)
# URL must be of format /config/
base_uri = environ['REQUEST_URI']
uri = filter(None, base_uri.split('/config/')[1].strip('/').split('/'))
output = "Template Holder"
if base_uri.endswith('/create/network.kml'):
content_type='application/vnd.google-earth.kml+xml'
output = make_network_kml.make_graph()
elif base_uri.endswith('/api/get/nodeplanner.json'):
content_type='application/json'
output = make_network_kml.make_nodeplanner_json()
elif not uri:
if is_text_request(environ):
output = '\n'.join(get_hostlist())
else:
content_type = 'text/html'
output = generate_title(get_hostlist())
elif len(uri) == 1:
if is_text_request(environ):
output = generate_node(uri[0])
else:
content_type = 'text/html'
output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
elif len(uri) == 2:
output = generate_config(uri[0], uri[1])
else:
assert False, "Invalid option"
# Return response
response_headers += [
('Content-type', content_type),
('Content-Length', str(len(output))),
]
return(response_headers, str(output))
def make_dns(output_dir = 'dns', external = False):
items = dict()
# hostname is key, IP is value
wleiden_zone = defaultdict(list)
wleiden_cname = dict()
pool = dict()
for node in get_hostlist():
datadump = get_yaml(node)
fqdn = datadump['nodename']
if 'rdr_host' in datadump:
remote_target = datadump['rdr_host']
elif 'remote_access' in datadump and datadump['remote_access']:
remote_target = datadump['remote_access'].split(':')[0]
else:
remote_target = None
if remote_target:
try:
parseaddr(remote_target)
wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
except (IndexError, ValueError):
wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
wleiden_zone[fqdn].append((datadump['masterip'], True))
# Hacking to get proper DHCP IPs and hostnames
for iface_key in get_interface_keys(datadump):
iface_name = iface_key.replace('_','-')
if 'ip' in datadump[iface_key]:
(ip, cidr) = datadump[iface_key]['ip'].split('/')
try:
(dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
dhcp_part = ".".join(ip.split('.')[0:3])
if ip != datadump['masterip']:
wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
for i in range(int(dhcp_start), int(dhcp_stop) + 1):
wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
except (AttributeError, ValueError, KeyError):
# First push it into a pool, to indentify the counter-part later on
addr = parseaddr(ip)
cidr = int(cidr)
addr = addr & ~((1 << (32 - cidr)) - 1)
if addr in pool:
pool[addr] += [(iface_name, fqdn, ip)]
else:
pool[addr] = [(iface_name, fqdn, ip)]
continue
# WL uses an /29 to configure an interface. IP's are ordered like this:
# MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
sn = lambda x: re.sub(r'(?i)^cnode','',x)
# Automatic naming convention of interlinks namely 2 + remote.lower()
for (key,value) in pool.items():
# Make sure they are sorted from low-ip to high-ip
value = sorted(value, key=lambda x: parseaddr(x[2]))
if len(value) == 1:
(iface_name, fqdn, ip) = value[0]
wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
# Device DNS names
if 'cnode' in fqdn.lower():
wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
elif len(value) == 2:
(a_iface_name, a_fqdn, a_ip) = value[0]
(b_iface_name, b_fqdn, b_ip) = value[1]
wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
# Device DNS names
if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
else:
pool_members = [k[1] for k in value]
for item in value:
(iface_name, fqdn, ip) = item
wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
# Include static DNS entries
# XXX: Should they override the autogenerated results?
# XXX: Convert input to yaml more useable.
# Format:
##; this is a comment
## roomburgh=Roomburgh1
## apkerk1.Vosko=172.17.176.8 ;this as well
dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
# Hack to allow special entries, for development
wleiden_raw = {}
for line in dns_list:
reverse = False
k, items = line.items()[0]
if type(items) == dict:
if 'reverse' in items:
reverse = items['reverse']
items = items['a']
else:
items = items['cname']
items = [items] if type(items) != list else items
for item in items:
if item.startswith('IN '):
wleiden_raw[k] = item
elif valid_addr(item):
wleiden_zone[k].append((item, reverse))
else:
wleiden_cname[k] = item
# Hack to get dynamic pool listing
def chunks(l, n):
return [l[i:i+n] for i in range(0, len(l), n)]
ntp_servers = [x[0] for x in get_nameservers()]
for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
for ntp_server in chunk:
wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
details = dict()
# 24 updates a day allowed
details['serial'] = time.strftime('%Y%m%d%H')
if external:
dns_masters = ['ns1.vanderzwet.net', 'ns1.anywi.com']
else:
dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
details['master'] = dns_masters[0]
details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
dns_header = '''
$TTL 3h
%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 3h )
; Serial, Refresh, Retry, Expire, Neg. cache TTL
%(ns_servers)s
\n'''
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
details['zone'] = 'wleiden.net'
f = open(os.path.join(output_dir,"db." + details['zone']), "w")
f.write(dns_header % details)
for host,items in wleiden_zone.items():
for ip,reverse in items:
if ip not in ['0.0.0.0']:
f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
for source,dest in wleiden_cname.items():
dest = dest if dest.endswith('.') else dest + ".wleiden.net."
f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
for source, dest in wleiden_raw.items():
f.write("%s.wleiden.net. %s\n" % (source, dest))
f.close()
# Create whole bunch of specific sub arpa zones. To keep it compliant
for s in range(16,32):
details['zone'] = '%i.172.in-addr.arpa' % s
f = open(os.path.join(output_dir,"db." + details['zone']), "w")
f.write(dns_header % details)
#XXX: Not effient, fix to proper data structure and do checks at other
# stages
for host,items in wleiden_zone.items():
for ip,reverse in items:
if not reverse:
continue
if valid_addr(ip):
if valid_addr(ip):
if int(ip.split('.')[1]) == s:
rev_ip = '.'.join(reversed(ip.split('.')))
f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
f.close()
def usage():
print("""Usage: %(prog)s
Argument:
\tcleanup = Cleanup all YAML files to specified format
\tstandalone [port] = Run configurator webserver [8000]
\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
\tfull-export = Generate yaml export script for heatmap.
\tstatic [outputdir] = Generate all config files and store on disk
\t with format .//%%NODE%%/%%FILE%% [./static]
\ttest [] = Receive output for certain node [all files].
\ttest-cgi = Receive output of CGI script [all files].
\tlist = List systems which have certain status
\tcreate network.kml = Create Network KML file for use in Google Earth
Arguments:
\t = NodeName (example: HybridRick)
\t = %(files)s
\t = all|up|down|planned
\t = systems|nodes|proxies
NOTE FOR DEVELOPERS; you can test your changes like this:
BEFORE any changes in this code:
$ ./gformat.py static /tmp/pre
AFTER the changes:
$ ./gformat.py static /tmp/post
VIEW differences and VERIFY all are OK:
$ diff -urI 'Generated' -r /tmp/pre /tmp/post
""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) })
exit(0)
def is_text_request(environ=os.environ):
""" Find out whether we are calling from the CLI or any text based CLI utility """
if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
return True
if 'HTTP_USER_AGENT' in environ:
return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
else:
return False
def switchFormat(setting):
if setting:
return "YES"
else:
return "NO"
def rlinput(prompt, prefill=''):
import readline
readline.set_startup_hook(lambda: readline.insert_text(prefill))
try:
return raw_input(prompt)
finally:
readline.set_startup_hook()
def fix_conflict(left, right, default='i'):
while True:
print("## %-30s | %-30s" % (left, right))
c = raw_input("## Solve Conflict (h for help) [%s]: " % default)
if not c:
c = default
if c in ['l','1']:
return left
elif c in ['r','2']:
return right
elif c in ['e', '3']:
return rlinput("Edit: ", "%30s | %30s" % (left, right))
elif c in ['i', '4']:
return None
else:
print("#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c)
def print_cgi_response(response_headers, output):
for header in response_headers:
print("%s: %s" % header)
print("")
print(output)
def main():
"""Hard working sub"""
# Allow easy hacking using the CLI
if not os.environ.get('REQUEST_URI', None):
if len(sys.argv) < 2:
usage()
if sys.argv[1] == "standalone":
import SocketServer
import CGIHTTPServer
# Hop to the right working directory.
os.chdir(os.path.dirname(__file__))
try:
PORT = int(sys.argv[2])
except (IndexError,ValueError):
PORT = 8000
class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
""" Serve this CGI from the root of the webserver """
def is_cgi(self):
if "favicon" in self.path:
return False
self.cgi_info = (os.path.basename(__file__), self.path)
self.path = ''
return True
handler = MyCGIHTTPRequestHandler
SocketServer.TCPServer.allow_reuse_address = True
httpd = SocketServer.TCPServer(("", PORT), handler)
httpd.server_name = 'localhost'
httpd.server_port = PORT
logger.info("serving at port %s", PORT)
try:
httpd.serve_forever()
except KeyboardInterrupt:
httpd.shutdown()
logger.info("All done goodbye")
elif sys.argv[1] == "test":
# Basic argument validation
try:
node = sys.argv[2]
except IndexError:
print("Invalid argument")
exit(1)
except IOError as e:
print(e)
exit(1)
datadump = get_yaml(node)
# Get files to generate
gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
# Actual config generation
for config in gen_files:
logger.info("## Generating %s %s", node, config)
print(generate_config(node, config, datadump))
elif sys.argv[1] == "test-cgi":
os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
os.environ['SCRIPT_NAME'] = __file__
response_headers, output = process_cgi_request()
print_cgi_response(response_headers, output)
elif sys.argv[1] == "static":
generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
elif sys.argv[1] == "wind-export":
items = dict()
for node in get_hostlist():
datadump = get_yaml(node)
sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
VALUES (
(SELECT id FROM users WHERE username = 'rvdzwet'),
(SELECT id FROM nodes WHERE name = '%(nodename)s'),
'Y');""" % datadump
#for config in files:
# items['config'] = config
# print "## Generating %(node)s %(config)s" % items
# f = open("%(wdir)s/%(config)s" % items, "w")
# f.write(generate_config(node, config, datadump))
# f.close()
for node in get_hostlist():
datadump = get_yaml(node)
for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
ifacedump = datadump[iface_key]
if 'mode' in ifacedump and ifacedump['mode'] == 'ap-wds':
ifacedump['nodename'] = datadump['nodename']
if not 'channel' in ifacedump or not ifacedump['channel']:
ifacedump['channel'] = 0
sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
'%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
elif sys.argv[1] == "smokeping-export":
for host in get_hostlist():
datadump = get_yaml(host)
if datadump.get('service_proxy_normal', False) or datadump.get('service_proxy_ileiden', False):
print(textwrap.dedent("""\
++ wleiden-gw-%(nodename)s
menu = %(nodename)s.gw
title = Wireless Leiden gateway %(nodename)s.gw.wleiden.net.
host = %(nodename)s.gw.wleiden.net.
""" % datadump))
elif sys.argv[1] == "nagios-export":
try:
heavy_load = (sys.argv[2] == "--heavy-load")
except IndexError:
heavy_load = False
hostgroup_details = {
'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
'walphen' : 'Stichting Wireless Alphen',
'westeinder' : 'Westeinder Plassen',
}
# Convert IP to Host
ip2host = {'root' : 'root'}
for host in get_hostlist():
datadump = get_yaml(host)
ip2host[datadump['masterip']] = datadump['autogen_fqdn']
for iface in get_interface_keys(datadump):
if 'autogen_gateway' in datadump[iface]:
ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
# Find dependency tree based on output of lvrouted.mytree of nearest node
parents = defaultdict(list)
stack = ['root']
prev_depth = 0
if os.path.isfile('lvrouted.mytree'):
for line in open('lvrouted.mytree').readlines():
depth = line.count('\t')
ip = line.strip().split()[0]
if prev_depth < depth:
try:
parents[ip2host[ip]].append(ip2host[stack[-1]])
except KeyError as e:
print("# Unable to find %s in configuration files" % e.args[0])
stack.append(ip)
elif prev_depth > depth:
stack = stack[:(depth - prev_depth)]
elif prev_depth == depth:
try:
parents[ip2host[ip]].append(ip2host[stack[-1]])
except KeyError as e:
print("# Unable to find %s in configuration files" % e.args[0])
prev_depth = depth
# Observe that some nodes has themself as parent or multiple parents
# for now take only the first parent, other behaviour is yet to be explained
params = {
'check_interval' : 5 if heavy_load else 120,
'retry_interval' : 1 if heavy_load else 10,
'max_check_attempts' : 10 if heavy_load else 6,
'notification_interval': 120 if heavy_load else 240,
}
print('''\
define host {
name wleiden-node ; Default Node Template
use generic-host ; Use the standard template as initial starting point
check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
check_interval %(check_interval)s ; Actively check the host every 5 minutes
retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
notification_interval %(notification_interval)s
max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
check_command check-host-alive ; Default command to check FreeBSD hosts
register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
}
define service {
name wleiden-service ; Default Service Template
use generic-service ; Use the standard template as initial starting point
check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
check_interval %(check_interval)s ; Actively check the host every 5 minutes
retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
notification_interval %(notification_interval)s
max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
}
# Please make sure to install:
# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
#
# Recompile net-mgmt/nagios-plugins to support check_snmp
# make -C /usr/ports/net-mgmt/nagios-plugins
#
# Install net/bind-tools to allow v2/check_dns_wl to work:
# pkg install bind-tools
#
define command{
command_name check_snmp_disk
command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
}
define command{
command_name check_netsnmp_load
command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
}
define command{
command_name check_netsnmp_proc
command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
}
define command{
command_name check_by_ssh
command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
}
define command{
command_name check_dns_wl
command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
}
define command{
command_name check_snmp_uptime
command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
}
# TDB: dhcp leases
# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
# TDB: internet status
# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
# TDB: Advanced local passive checks
# /usr/local/libexec/nagios/check_by_ssh
''' % params)
print('''\
# Service Group, not displayed by default
define hostgroup {
hostgroup_name srv_hybrid
alias All Hybrid Nodes
register 0
}
define service {
use wleiden-service
hostgroup_name srv_hybrid
service_description SSH
check_command check_ssh
}
define service {
use wleiden-service,service-pnp
hostgroup_name srv_hybrid
service_description HTTP
check_command check_http
}
define service {
use wleiden-service
hostgroup_name srv_hybrid
service_description DNS
check_command check_dns_wl!"www.wirelessleiden.nl"
}
''')
if heavy_load:
print('''\
define service {
use wleiden-service
hostgroup_name srv_hybrid
service_description UPTIME
check_command check_snmp_uptime
}
#define service {
# use wleiden-service
# hostgroup_name srv_hybrid
# service_description NTP
# check_command check_ntp_peer
#}
define service {
use wleiden-service
hostgroup_name srv_hybrid
service_description LOAD
check_command check_netsnmp_load
}
define service {
use wleiden-service
hostgroup_name srv_hybrid
service_description PROC
check_command check_netsnmp_proc
}
define service {
use wleiden-service
hostgroup_name srv_hybrid
service_description DISK
check_command check_snmp_disk
}
''')
for node in get_hostlist():
datadump = get_yaml(node)
if not datadump['status'] == 'up':
continue
if not datadump['monitoring_group'] in hostgroup_details:
hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
print('''\
define host {
use wleiden-node,host-pnp
contact_groups admins
host_name %(autogen_fqdn)s
address %(masterip)s
hostgroups srv_hybrid,%(monitoring_group)s\
''' % datadump)
if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
print('''\
parents %(parents)s\
''' % { 'parents' : parents[datadump['autogen_fqdn']][0] })
print('''\
}
''')
for name,alias in hostgroup_details.items():
print('''\
define hostgroup {
hostgroup_name %s
alias %s
} ''' % (name, alias))
elif sys.argv[1] == "full-export":
hosts = {}
for node in get_hostlist():
datadump = get_yaml(node)
hosts[datadump['nodename']] = datadump
print(yaml.dump(hosts))
elif sys.argv[1] == "dns":
make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
elif sys.argv[1] == "cleanup":
# First generate all datadumps
datadumps = dict()
ssid_to_node = dict()
for host in get_hostlist():
logger.info("# Processing: %s", host)
# Set some boring default values
datadump = { 'board' : 'UNKNOWN' }
datadump.update(get_yaml(host))
datadumps[datadump['nodename']] = datadump
(poel, errors) = make_relations()
print("\n".join(["# WARNING: %s" % x for x in errors]))
for host,datadump in datadumps.items():
try:
# Convert all yes and no to boolean values
def fix_boolean(dump):
for key in dump.keys():
if type(dump[key]) == dict:
dump[key] = fix_boolean(dump[key])
elif str(dump[key]).lower() in ["yes", "true"]:
dump[key] = True
elif str(dump[key]).lower() in ["no", "false"]:
# Compass richting no (Noord Oost) is valid input
if key != "compass": dump[key] = False
return dump
datadump = fix_boolean(datadump)
if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
if not 'latitude' in datadump and not 'longitude' in datadump:
datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
elif 'latitude' in datadump and 'longitude' in datadump:
if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
# TODO: Compare outcome of both coordinate systems and validate against each-other
if datadump['nodename'].startswith('Proxy'):
datadump['nodename'] = datadump['nodename'].lower()
for iface_key in get_interface_keys(datadump):
try:
# All our normal wireless cards are normal APs now
if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
datadump[iface_key]['mode'] = 'ap'
# Wireless Leiden SSID have an consistent lowercase/uppercase
if 'ssid' in datadump[iface_key]:
ssid = datadump[iface_key]['ssid']
prefix = 'ap-WirelessLeiden-'
if ssid.lower().startswith(prefix.lower()):
datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
if 'ns_ip' in datadump[iface_key] and not 'mode' in datadump[iface_key]:
datadump[iface_key]['mode'] = 'autogen-FIXME'
if not 'comment' in datadump[iface_key]:
datadump[iface_key]['comment'] = 'autogen-FIXME'
if 'ns_mac' in datadump[iface_key]:
datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
if 'comment' in datadump[iface_key] and datadump[iface_key]['comment'].startswith('autogen-'):
datadump[iface_key] = datadump[iface_key]['desc']
# We are not using 802.11b anymore. OFDM is preferred over DSSS
# due to better collision avoidance.
if datadump[iface_key]['type'] == '11b':
datadump[iface_key]['type'] = '11g'
# Setting 802.11g channels to de-facto standards, to avoid
# un-detected sharing with other overlapping channels
#
# Technically we could also use channel 13 in NL, but this is not
# recommended as foreign devices might not be able to select this
# channel. Secondly using 1,5,9,13 instead is going to clash with
# the de-facto usage of 1,6,11.
#
# See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
channels_at_2400Mhz = (1,6,11)
if datadump[iface_key]['type'] == '11g' and 'channel' in datadump[iface_key]:
datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
# Mandatory interface keys
if not 'status' in datadump[iface_key]:
datadump[iface_key]['status'] = 'planned'
x = datadump[iface_key]['comment']
datadump[iface_key]['comment'] = x[0].upper() + x[1:]
# Fixing bridge_type if none is found
if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
if not 'bridge_type' in datadump[iface_key]:
datadump[iface_key]['bridge_type'] = 'NanoStation M5'
# Making sure description works
if 'desc' in datadump[iface_key]:
if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
del datadump[iface_key]['desc']
else:
print("# ERROR: At %s - %s" % (datadump['nodename'], iface_key))
response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
if response:
datadump[iface_key]['comment'] = response
del datadump[iface_key]['desc']
# Check DHCP configuration
dhcp_type(datadump[iface_key])
# Set the compass value based on the angle between the poels
if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
my_pool = poel[network(datadump[iface_key]['ip'])]
remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
if remote_hosts:
compass_target = remote_hosts[0]
datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
# TODO: Compass wanted and actual direction might differ
# Monitoring Group default
if not 'monitoring_group' in datadump:
datadump['monitoring_group'] = 'wleiden'
except Exception as exc:
exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
raise
store_yaml(datadump)
except Exception as exc:
exc.args = ("# Error while processing %s" % host,) + exc.args
raise
elif sys.argv[1] == "list":
use_fqdn = False
if len(sys.argv) < 4:
usage()
if not sys.argv[2] in ["up", "down", "planned", "all"]:
usage()
if not sys.argv[3] in ["nodes","proxies","systems"]:
usage()
if len(sys.argv) > 4:
if sys.argv[4] == "fqdn":
use_fqdn = True
else:
usage()
for system in get_hostlist():
datadump = get_yaml(system)
if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
continue
output = datadump['autogen_fqdn'] if use_fqdn else system
if sys.argv[2] == "all":
print(output)
elif datadump['status'] == sys.argv[2]:
print(output)
elif sys.argv[1] == "create":
if sys.argv[2] == "network.kml":
print(make_network_kml.make_graph())
elif sys.argv[2] == "host-ips.txt":
for system in get_hostlist():
datadump = get_yaml(system)
ips = [datadump['masterip']]
for ifkey in get_interface_keys(datadump):
ips.append(datadump[ifkey]['ip'].split('/')[0])
print(system, ' '.join(ips))
elif sys.argv[2] == "host-pos.txt":
for system in get_hostlist():
datadump = get_yaml(system)
print(system, datadump['rdnap_x'], datadump['rdnap_y'])
elif sys.argv[2] == "nodeplanner.json":
print(make_network_kml.make_nodeplanner_json())
elif sys.argv[2] == 'ssh_config':
print('''
Host *.wleiden.net
User root
Host 172.16.*.*
User root
''')
for system in get_hostlist():
datadump = get_yaml(system)
print('''\
Host %s
User root
Host %s
User root
Host %s
User root
Host %s
User root
''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower()))
else:
usage()
else:
usage()
else:
# Do not enable debugging for config requests as it highly clutters the output
if not is_text_request():
cgitb.enable()
try:
response_headers, output = process_cgi_request()
except:
print('')
print('')
raise
print_cgi_response(response_headers, output)
if __name__ == "__main__":
main()