source: genesis/tools/gformat.py@ 10525

#!/usr/bin/env python
#
# vim:ts=2:et:sw=2:ai
# Wireless Leiden configuration generator, based on yaml files'
#
# XXX: This should be rewritten to make use of the ipaddr.py library.
#
# Sample apache configuration (mind the AcceptPathInfo!)
#  ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
#  <Directory /usr/local/www/genesis>
#    Allow from all
#    AcceptPathInfo On
#  </Directory>
#
# Rick van der Zwet <info@rickvanderzwet.nl>
#

# Hack to make the script directory is also threated as a module search path.
import sys
import os
import re
sys.path.append(os.path.dirname(__file__))

import cgi
import cgitb
import copy
import glob
import socket
import string
import subprocess
import time
import rdnap
import make_network_kml
from pprint import pprint
from collections import defaultdict
try:
  import yaml
except ImportError, e:
  print e
  print "[ERROR] Please install the python-yaml or devel/py-yaml package"
  exit(1)

try:
    from yaml import CLoader as Loader
    from yaml import CDumper as Dumper
except ImportError:
    from yaml import Loader, Dumper

from jinja2 import Template

import logging
logging.basicConfig(format='# %(levelname)s: %(message)s' )
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)


if os.environ.has_key('CONFIGROOT'):
  NODE_DIR = os.environ['CONFIGROOT']
else:
  NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
__version__ = '$Id: gformat.py 10524 2012-04-17 08:48:39Z rick $'


files = [
    'authorized_keys',
    'dnsmasq.conf',
    'dhcpd.conf',
    'rc.conf.local',
    'resolv.conf',
    'motd',
    'wleiden.yaml',
    ]

# Global variables uses
OK = 10
DOWN = 20
UNKNOWN = 90

def get_yaml(item):
  """ Get configuration yaml for 'item'"""
  gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')

  # Use some boring defaults
  datadump = { 'service_proxy' : False, 'service_ileiden' : False }
  f = open(gfile, 'r')
  datadump.update(yaml.load(f,Loader=Loader))
  f.close()

  # Preformat certain needed variables for formatting and push those into special object
  datadump['autogen_iface_keys'] = get_interface_keys(datadump)

  wlan_count=0
  for key in datadump['autogen_iface_keys']:
    if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
      datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
      wlan_count += 1
    else:
      datadump[key]['autogen_ifname'] = datadump[key]['interface'].split(':')[0]

  dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] if datadump[key]['dhcp']]
  datadump['autogen_dhcp_interfaces'] = ','.join(dhcp_interfaces)
  datadump['autogen_item'] = item

  datadump['autogen_realname'] = get_realname(datadump)
  datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
  datadump['autogen_fqdn'] = datadump['autogen_realname'] + '.' + datadump['autogen_domain']
  return datadump


def store_yaml(datadump, header=False):
  """ Store configuration yaml for 'item'"""
  item = datadump['autogen_item'] 
  gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')

  f = open(gfile, 'w')
  f.write(generate_wleiden_yaml(datadump, header))
  f.close()



def make_relations():
  """ Process _ALL_ yaml files to get connection relations """
  errors = ""
  poel = defaultdict(list)
  for host in get_hostlist():
    try:
      datadump = get_yaml(host)
      for iface_key in datadump['autogen_iface_keys']:
        l = datadump[iface_key]['ip']
        addr, mask = l.split('/')

        # Not parsing of these folks please
        if not valid_addr(addr):
          continue

        addr = parseaddr(addr)
        mask = int(mask)
        network = addr & ~((1 << (32 - mask)) - 1)
        poel[network] += [(host,datadump[iface_key])]
    except (KeyError, ValueError), e:
      errors += "[FOUT] in '%s' interface '%s'" % (host,iface_key) 
      errors += e
      continue
  return (poel, errors)


def get_proxylist():
  """Get all available proxies proxyX sorting based on X number"""
  proxylist = sorted([os.path.basename(x) for x in glob.glob("%s/proxy*" % NODE_DIR)],
                key=lambda name: int(''.join([c for c in name if c in string.digits])),
                cmp=lambda x,y: x - y) + sorted([os.path.basename(x) for x in glob.glob("%s/Proxy*" % NODE_DIR)])
  return proxylist

def get_hybridlist():
  """Get all available hybrid nodes/proxies"""
  hybridlist = sorted([os.path.basename(x) for x in glob.glob("%s/Hybrid*" % NODE_DIR)])
  return hybridlist


def valid_addr(addr):
  """ Show which address is valid in which are not """
  return str(addr).startswith('172.')


def get_nodelist():
  """ Get all available nodes - sorted """
  nodelist = sorted([os.path.basename(x) for x in glob.glob("%s/CNode*" % NODE_DIR)])
  return nodelist

def get_hostlist():
  """ Combined hosts and proxy list"""
  return get_nodelist() + get_proxylist() + get_hybridlist()

def angle_between_points(lat1,lat2,long1,long2):
  """
  Return Angle in radians between two GPS coordinates
  See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
  """
  dy = lat2 - lat1
  dx = math.cos(math.pi/180*lat1)*(long2 - long1)
  angle = math.atan2(dy,dx)
  return angle

def angle_to_cd(angle):
  """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """

  # For easy conversion get positive degree
  degrees = math.degrees(angle)
  if degrees < 0:
    360 - abs(degrees)

  # Numbers can be confusing calculate from the 4 main directions
  p = 22.5
  if degrees < p:
    return "n"
  elif degrees < (90 - p):
    return "no"
  elif degrees < (90 + p):
    return "o"
  elif degrees < (180 - p):
    return "zo"
  elif degrees < (180 + p):
    return "z"
  elif degrees < (270 - p):
    return "zw"
  elif degrees < (270 + p):
    return "w"
  elif degrees < (360 - p):
    return "nw"
  else:
    return "n"


def generate_title(nodelist):
  """ Main overview page """
  items = {'root' : "." }
  output = """
<html>
  <head>
    <title>Wireless leiden Configurator - GFormat</title>
    <style type="text/css">
     th        {background-color: #999999}
     tr:nth-child(odd)    {background-color: #cccccc}
     tr:nth-child(even)   {background-color: #ffffff}
     th, td    {padding: 0.1em 1em}
    </style>
  </head>
  <body>
    <center>
    <form type="GET" action="%(root)s">
      <input type="hidden" name="action" value="update">
      <input type="submit" value="Update Configuration Database (SVN)">
    </form>
    <table>
     <caption><h3>Wireless Leiden Configurator</h3></caption>
  """ % items

  for node in nodelist:
    items['node'] = node
    output += '<tr><td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items
    for config in files:
      items['config'] = config
      output += '<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items
    output += "</tr>"
  output += """
    </table>
    <hr />
    <em>%s</em>
    </center>
  </body>
</html>
  """ % __version__

  return output



def generate_node(node):
  """ Print overview of all files available for node """
  return "\n".join(files)

def generate_node_overview(host):
  """ Print overview of all files available for node """
  datadump = get_yaml(host)
  params = { 'host' : host }
  output = "<em><a href='..'>Back to overview</a></em><hr />"
  output += "<h2>Available files:</h2><ul>"
  for cf in files:
    params['cf'] = cf
    output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
  output += "</ul>"

  # Generate and connection listing
  output += "<h2>Connected To:</h2><ul>"
  (poel, errors) = make_relations()
  for network, hosts in poel.iteritems():
    if host in [x[0] for x in hosts]:
      if len(hosts) == 1:
        # Single not connected interface
        continue
      for remote,ifacedump in hosts:
        if remote == host: 
          # This side of the interface
          continue
        params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
        output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
  output += "</ul>"
  output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"

  output += "<hr /><em><a href='..'>Back to overview</a></em>"
  return output


def generate_header(ctag="#"):
  return """\
%(ctag)s
%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
%(ctag)s Generated at %(date)s by %(host)s
%(ctag)s
""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname() }



def parseaddr(s):
  """ Process IPv4 CIDR notation addr to a (binary) number """
  f = s.split('.')
  return (long(f[0]) << 24L) + \
    (long(f[1]) << 16L) + \
    (long(f[2]) << 8L) + \
    long(f[3])



def showaddr(a):
  """ Display IPv4 addr in (dotted) CIDR notation """
  return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)


def is_member(ip, mask, canidate):
  """ Return True if canidate is part of ip/mask block"""
  ip_addr = gformat.parseaddr(ip)
  ip_canidate = gformat.parseaddr(canidate)
  mask = int(mask)
  ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
  ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
  return ip_addr == ip_canidate



def cidr2netmask(netmask):
  """ Given a 'netmask' return corresponding CIDR """
  return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))

def get_network(addr, mask):
  return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))


def generate_dhcpd_conf(datadump):
  """ Generate config file '/usr/local/etc/dhcpd.conf """
  output = generate_header()
  output += Template("""\
# option definitions common to all supported networks...
option domain-name "dhcp.{{ autogen_fqdn }}";

default-lease-time 600;
max-lease-time 7200;

# Use this to enble / disable dynamic dns updates globally.
#ddns-update-style none;

# If this DHCP server is the official DHCP server for the local
# network, the authoritative directive should be uncommented.
authoritative;

# Use this to send dhcp log messages to a different log file (you also
# have to hack syslog.conf to complete the redirection).
log-facility local7;

#
# Interface definitions
#
\n""").render(datadump)

  for iface_key in datadump['autogen_iface_keys']:
    if not datadump[iface_key].has_key('comment'):
      datadump[iface_key]['comment'] = None
    output += "## %(interface)s - %(desc)s - %(comment)s\n" % datadump[iface_key]

    (addr, mask) = datadump[iface_key]['ip'].split('/')
    datadump[iface_key]['addr'] = addr
    datadump[iface_key]['netmask'] = cidr2netmask(mask)
    datadump[iface_key]['subnet'] = get_network(addr, mask)
    try:
      (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
    except (AttributeError, ValueError):
      output += "subnet %(subnet)s netmask %(netmask)s {\n  ### not autoritive\n}\n\n" % datadump[iface_key]
      continue

    dhcp_part = ".".join(addr.split('.')[0:3])
    datadump[iface_key]['dhcp_start'] = dhcp_part + "." + dhcp_start
    datadump[iface_key]['dhcp_stop'] =  dhcp_part + "." + dhcp_stop
    output += """\
subnet %(subnet)s netmask %(netmask)s {
  range %(dhcp_start)s %(dhcp_stop)s;
  option routers %(addr)s;
  option domain-name-servers %(addr)s;
}
\n""" % datadump[iface_key]

  return output



def generate_dnsmasq_conf(datadump):
  """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
  output = generate_header()
  output += Template("""\
# DHCP server options
dhcp-authoritative
dhcp-fqdn
domain=dhcp.{{ autogen_fqdn }}
domain-needed
expand-hosts
log-async=100

# Low memory footprint
cache-size=10000

\n""").render(datadump)

  for iface_key in datadump['autogen_iface_keys']:
    if not datadump[iface_key].has_key('comment'):
      datadump[iface_key]['comment'] = None
    output += "## %(interface)s - %(desc)s - %(comment)s\n" % datadump[iface_key]

    try:
      (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
      (ip, cidr) = datadump[iface_key]['ip'].split('/')
      datadump[iface_key]['netmask'] = cidr2netmask(cidr)
    except (AttributeError, ValueError):
      output += "# not autoritive\n\n"
      continue

    dhcp_part = ".".join(ip.split('.')[0:3])
    datadump[iface_key]['dhcp_start'] = dhcp_part + "." + dhcp_start
    datadump[iface_key]['dhcp_stop'] =  dhcp_part + "." + dhcp_stop
    output += "dhcp-range=%(interface)s,%(dhcp_start)s,%(dhcp_stop)s,%(netmask)s,24h\n\n" % datadump[iface_key]

  return output



def generate_rc_conf_local(datadump):
  """ Generate configuration file '/etc/rc.conf.local' """
  if not datadump.has_key('ileiden'):
    datadump['autogen_ileiden_enable'] = False
  else:
    datadump['autogen_ileiden_enable'] = datadump['ileiden']

  ileiden_proxies = []
  normal_proxies = []
  for proxy in get_proxylist():
    proxydump = get_yaml(proxy)
    if proxydump['ileiden']:
      ileiden_proxies.append(proxydump)
    else:
      normal_proxies.append(proxydump)
  for host in get_hybridlist():
    hostdump = get_yaml(host)
    if hostdump['service_ileiden']:
      ileiden_proxies.append(hostdump)
    if hostdump['service_proxy']:
      normal_proxies.append(hostdump)

  datadump['autogen_ileiden_proxies'] = ','.join([x['masterip'] for x in ileiden_proxies])
  datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
  datadump['autogen_normal_proxies'] = ','.join([x['masterip'] for x in normal_proxies])
  datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])

  output = generate_header("#");
  output += Template("""\
hostname='{{ autogen_fqdn }}'
location='{{ location }}'
nodetype="{{ nodetype }}"

#
# Configured listings
#
captive_portal_whitelist=""
captive_portal_interfaces="{{ autogen_dhcp_interfaces }}"
# iLeiden Proxies {{ autogen_ileiden_proxies_names }}
list_ileiden_proxies="{{ autogen_ileiden_proxies }}"
# normal Proxies {{ autogen_normal_proxies_names }}
list_normal_proxies="{{ autogen_normal_proxies }}"

{% if nodetype == "Proxy" %}
#
# Proxy Configuration
#
{% if gateway -%}
defaultrouter="{{ gateway }}"
{% else -%}
#defaultrouter="NOTSET"
{% endif -%}
internalif="{{ internalif }}"
ileiden_enable="{{ autogen_ileiden_enable }}"
gateway_enable="{{ autogen_ileiden_enable }}"
pf_enable="yes"
pf_rules="/etc/pf.conf"
{% if autogen_ileiden_enable -%}
pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
lvrouted_enable="{{ autogen_ileiden_enable }}"
lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
{% else -%}
pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
{% endif -%}
{% if internalroute -%}
static_routes="wleiden"
route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
{% endif -%}
{% endif -%}

{% if nodetype == "Hybrid" %}
#
# Hybrid Configuration
#
captive_portal_interfaces="{{ autogen_dhcp_interfaces|default('none', true) }}"
externalif="{{ externalif|default('vr0', true) }}"
masterip="{{ masterip }}"

pf_rules="/etc/pf.hybrid.conf"
pf_flags_hybrid="-D ext_if=$externalif -D ext_if_net=$externalif:network -D captive_portal_interfaces=$captive_portal_interfaces -D masterip=$masterip"
{% if service_ileiden -%}
# Service iLeiden
service_ileiden="yes"
pf_flags="$pf_flags_hybrid -D publicnat=80,443"
{% else -%}
pf_flags="$pf_flags_hybrid -D publicnat=0"
{% endif -%}

{% if service_proxy %}
# Service Proxy
service_proxy="yes"
tinyproxy_enable="yes"
{% else -%}
service_proxy="no"
pen_wrapper_enable="yes"
{% endif -%}

{% if board == "ALIX2" %}
# ''Fat'' configuration, board has 256MB RAM
dnsmasq_enable="NO"
named_enable="YES"
dhcpd_enable="YES"
{% endif -%}

{% if gateway -%}
defaultrouter="{{ gateway }}"
{% endif -%}
#
# END Hybrid Configuration
#
{% endif -%}


{% if nodetype == "CNode" %}
#
# NODE iLeiden Configuration
#

{% if tproxy -%}
tproxy_enable='YES'
tproxy_range='{{ tproxy }}'
{% else -%}
tproxy_enable='NO'
{% endif -%}

lvrouted_flags="-u -s s00p3rs3kr3t -m 28 -z $list_ileiden_proxies"
{% endif %}
\n
""").render(datadump)

  # lo0 configuration:
  # - 172.32.255.1/32 is the proxy.wleiden.net deflector
  # - masterip is special as it needs to be assigned to at
  #   least one interface, so if not used assign to lo0
  addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
  iface_map = {'lo0' : 'lo0'}
  dhclient_if = {'lo0' : False}

  masterip_used = False
  for iface_key in datadump['autogen_iface_keys']:
    if datadump[iface_key]['ip'].startswith(datadump['masterip']):
      masterip_used = True
      break
  if not masterip_used:
    addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))

  for iface_key in datadump['autogen_iface_keys']:
    ifacedump = datadump[iface_key]
    ifname = ifacedump['autogen_ifname']

    # Flag dhclient is possible
    dhclient_if[ifname] = ifacedump.has_key('dhcpclient') and ifacedump['dhcpclient']

    # Add interface IP to list
    item = (ifacedump['ip'], ifacedump['desc'])
    if addrs_list.has_key(ifname):
      addrs_list[ifname].append(item)
    else:
      addrs_list[ifname] = [item]

    # Alias only needs IP assignment for now, this might change if we
    # are going to use virtual accesspoints
    if "alias" in iface_key:
      continue

    # XXX: Might want to deduct type directly from interface name
    if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
      # Default to station (client) mode
      ifacedump['wlanmode'] = "sta"
      if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
        ifacedump['wlanmode'] = "ap"
      # Default to 802.11b mode
      ifacedump['mode'] = '11b'
      if ifacedump['type'] in ['11a', '11b' '11g']:
        ifacedump['mode'] = ifacedump['type']

      if not ifacedump.has_key('channel'):
        if ifacedump['type'] == '11a':
          ifacedump['channel'] = 36
        else:
          ifacedump['channel'] = 1

      # Allow special hacks at the back like wds and stuff
      if not ifacedump.has_key('extra'):
        ifacedump['extra'] = 'regdomain ETSI country NL'

      output += "wlans_%(interface)s='%(autogen_ifname)s'\n" % ifacedump
      output += ("create_args_%(autogen_ifname)s='wlanmode %(wlanmode)s mode " +\
        "%(mode)s ssid %(ssid)s %(extra)s channel %(channel)s'\n") % ifacedump

    elif ifacedump['type'] in ['ethernet', 'eth']:
      # No special config needed besides IP
      pass
    else:
      assert False, "Unknown type " + ifacedump['type']

  # Print IP address which needs to be assigned over here
  output += "\n"
  for iface,addrs in sorted(addrs_list.iteritems()):
    for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
      output += "# %s || %s || %s\n" % (iface, addr, comment)

    # Write DHCLIENT entry
    if dhclient_if[iface]:
      output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
    else:
      output += "ipv4_addrs_%s='%s'\n\n" % (iface, " ".join([x[0] for x in addrs]))

  return output




def get_all_configs():
  """ Get dict with key 'host' with all configs present """
  configs = dict()
  for host in get_hostlist():
    datadump = get_yaml(host)
    configs[host] = datadump
  return configs


def get_interface_keys(config):
  """ Quick hack to get all interface keys, later stage convert this to a iterator """
  return sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])


def get_used_ips(configs):
    """ Return array of all IPs used in config files"""
    ip_list = []
    for config in configs:
      ip_list.append(config['masterip'])
      for iface_key in get_interface_keys(config):
        l = config[iface_key]['ip']
        addr, mask = l.split('/')
        # Special case do not process
        if valid_addr(addr):
          ip_list.append(addr)
        else:
          logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
    return sorted(ip_list)



def generate_resolv_conf(datadump):
  """ Generate configuration file '/etc/resolv.conf' """
  # XXX: This should properly going to be an datastructure soon
  datadump['autogen_header'] = generate_header("#")
  datadump['autogen_edge_nameservers'] = ''
  for host in get_proxylist():
    hostdump = get_yaml(host)
    datadump['autogen_edge_nameservers'] += "nameserver %(masterip)-15s # %(autogen_realname)s\n" % hostdump
  for host in get_hybridlist():
    hostdump = get_yaml(host)
    if hostdump['service_proxy'] or hostdump['service_ileiden']:
      datadump['autogen_edge_nameservers'] += "nameserver %(masterip)-15s # %(autogen_realname)s\n" % hostdump

  return Template("""\
{{ autogen_header }}
search wleiden.net

# Try local (cache) first
nameserver 127.0.0.1

{% if service_proxy or service_ileiden or nodetype == 'Proxy' -%}
nameserver 8.8.8.8   # Google Public NameServer
nameserver 8.8.4.4   # Google Public NameServer
{% else -%}
{{ autogen_edge_nameservers }}
{% endif -%}
""").render(datadump)



def generate_motd(datadump):
  """ Generate configuration file '/etc/motd' """
  output = """\
FreeBSD 9.0-RELEASE (kernel.wleiden) #0 r230587: Sun Jan 29 17:09:57 CET 2012

  WWW: %(autogen_fqdn)s - http://www.wirelessleiden.nl
  Loc: %(location)s

Interlinks:
""" % datadump

  # XXX: This is a hacky way to get the required data
  for line in generate_rc_conf_local(datadump).split('\n'):
    if '||' in line and not line[1:].split()[0] in ['lo0', 'ath0'] :
      output += " - %s \n" % line[1:] 
  output += """\
Attached bridges:
"""
  for iface_key in datadump['autogen_iface_keys']:
    ifacedump = datadump[iface_key]
    if ifacedump.has_key('ns_ip'):
      output += " - %(interface)s || %(mode)s || %(ns_ip)s\n" % ifacedump

  return output


def format_yaml_value(value):
  """ Get yaml value in right syntax for outputting """
  if isinstance(value,str):
    output = '"%s"' % value
  else:
    output = value
  return output



def format_wleiden_yaml(datadump):
  """ Special formatting to ensure it is editable"""
  output = "# Genesis config yaml style\n"
  output += "# vim:ts=2:et:sw=2:ai\n"
  output += "#\n"
  iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
  for key in sorted(set(datadump.keys()) - set(iface_keys)):
    output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))

  output += "\n\n"

  key_order = [ 'comment', 'interface', 'ip', 'desc', 'sdesc', 'mode', 'type',
    'extra_type', 'channel', 'ssid', 'dhcp' ]

  for iface_key in sorted(iface_keys):
    output += "%s:\n" % iface_key
    for key in key_order + list(sorted(set(datadump[iface_key].keys()) - set(key_order))):
      if datadump[iface_key].has_key(key):
        output += "  %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
    output += "\n\n"

  return output



def generate_wleiden_yaml(datadump, header=True):
  """ Generate (petty) version of wleiden.yaml"""
  for key in datadump.keys():
    if key.startswith('autogen_'):
      del datadump[key]
    # Interface autogen cleanups
    elif type(datadump[key]) == dict:
      for key2 in datadump[key].keys():
        if key2.startswith('autogen_'):
          del datadump[key][key2]
      
  output = generate_header("#") if header else ''
  output += format_wleiden_yaml(datadump)
  return output


def generate_yaml(datadump):
  return generate_config(datadump['nodename'], "wleiden.yaml", datadump)



def generate_config(node, config, datadump=None):
  """ Print configuration file 'config' of 'node' """
  output = ""
  try:
    # Load config file
    if datadump == None:
      datadump = get_yaml(node)

    if config == 'wleiden.yaml':
      output += generate_wleiden_yaml(datadump)
    elif config == 'authorized_keys':
      f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
      output += f.read()
      f.close()
    elif config == 'dnsmasq.conf':
      output += generate_dnsmasq_conf(datadump)
    elif config == 'dhcpd.conf':
      output += generate_dhcpd_conf(datadump)
    elif config == 'rc.conf.local':
      output += generate_rc_conf_local(datadump)
    elif config == 'resolv.conf':
      output += generate_resolv_conf(datadump)
    elif config == 'motd':
      output += generate_motd(datadump)
    else:
      assert False, "Config not found!"
  except IOError, e:
    output += "[ERROR] Config file not found"
  return output



def process_cgi_request():
  """ When calling from CGI """
  # Update repository if requested
  form = cgi.FieldStorage()
  if form.getvalue("action") == "update":
    print "Refresh: 5; url=."
    print "Content-type:text/plain\r\n\r\n",
    print "[INFO] Updating subverion, please wait..."
    print subprocess.Popen(['svn', 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0],
    print subprocess.Popen(['svn', 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0],
    print "[INFO] All done, redirecting in 5 seconds"
    sys.exit(0)


  base_uri = os.environ['PATH_INFO']
  uri = base_uri.strip('/').split('/')

  output = ""
  if base_uri.endswith('/create/network.kml'):
      output += "Content-type:application/vnd.google-earth.kml+xml\r\n\r\n"
      output += make_network_kml.make_graph()
  elif not uri[0]:
    if is_text_request():
      output += "Content-type:text/plain\r\n\r\n"
      output += '\n'.join(get_hostlist())
    else:
      output += "Content-type:text/html\r\n\r\n"
      output += generate_title(get_hostlist())
  elif len(uri) == 1:
    if is_text_request():
      output += "Content-type:text/plain\r\n\r\n"
      output += generate_node(uri[0])
    else:
      output += "Content-type:text/html\r\n\r\n"
      output += generate_node_overview(uri[0])
  elif len(uri) == 2:
    output += "Content-type:text/plain\r\n\r\n"
    output += generate_config(uri[0], uri[1])
  else:
    assert False, "Invalid option"
  print output

def get_realname(datadump):
  # Proxy naming convention is special, as the proxy name is also included in
  # the nodename, when it comes to the numbered proxies.
  if datadump['nodetype'] == 'Proxy':
    realname = datadump['nodetype'] + datadump['nodename'].replace('proxy','')
  else:
    # By default the full name is listed and also a shortname CNAME for easy use.
    realname = datadump['nodetype'] + datadump['nodename']
  return(realname)



def make_dns(output_dir = 'dns', external = False):
  items = dict()

  # hostname is key, IP is value
  wleiden_zone = dict()
  wleiden_cname = dict()

  pool = dict()
  for node in get_hostlist():
    logger.info("Processing host %s", node)
    datadump = get_yaml(node)

    # Proxy naming convention is special
    fqdn = datadump['autogen_realname']
    if datadump['nodetype'] in ['CNode', 'Hybrid']:
      wleiden_cname[datadump['nodename']] = fqdn

    wleiden_zone[fqdn] = datadump['masterip']

    # Hacking to get proper DHCP IPs and hostnames
    for iface_key in get_interface_keys(datadump):
      iface_name = datadump[iface_key]['interface'].replace(':',"-alias-")
      (ip, cidr) = datadump[iface_key]['ip'].split('/')
      try:
        (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
        datadump[iface_key]['netmask'] = cidr2netmask(cidr)
        dhcp_part = ".".join(ip.split('.')[0:3])
        if ip != datadump['masterip']:
          wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)] = ip
        for i in range(int(dhcp_start), int(dhcp_stop) + 1):
          wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)] = "%s.%s" % (dhcp_part, i)
      except (AttributeError, ValueError):
        # First push it into a pool, to indentify the counter-part later on
        addr = parseaddr(ip)
        cidr = int(cidr)
        addr = addr & ~((1 << (32 - cidr)) - 1)
        if pool.has_key(addr):
          pool[addr] += [(iface_name, fqdn, ip)]
        else:
          pool[addr] = [(iface_name, fqdn, ip)]
        continue


  def pool_to_name(node, pool_members):
    """Convert the joined name to a usable pool name"""

    # Get rid of the own entry
    pool_members = list(set(pool_members) - set([fqdn]))

    target = oldname = ''
    for node in sorted(pool_members):
      (name, number) = re.match('^([A-Za-z]+)([0-9]*)$',node).group(1,2)
      target += "-" + number if name == oldname else "-" + node if target else node
      oldname = name

    return target


  #  WL uses an /29 to configure an interface. IP's are ordered like this: 
  #     MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4) 
    
  sn = lambda x: re.sub(r'(?i)^cnode','',x)

  # Automatic naming convention of interlinks namely 2 + remote.lower()
  for (key,value) in pool.iteritems():
    # Make sure they are sorted from low-ip to high-ip
    value = sorted(value, key=lambda x: parseaddr(x[2]))

    if len(value) == 1:
      (iface_name, fqdn, ip) = value[0]
      wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)] = ip

      # Device DNS names
      if 'cnode' in fqdn.lower():
        wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)] = showaddr(parseaddr(ip) + 1)
        wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % (iface_name, fqdn)

    elif len(value) == 2:
      (a_iface_name, a_fqdn, a_ip) = value[0]
      (b_iface_name, b_fqdn, b_ip) = value[1]
      wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)] = a_ip
      wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)] = b_ip

      # Device DNS names
      if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
        wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)] = showaddr(parseaddr(a_ip) + 1)
        wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)] = showaddr(parseaddr(b_ip) - 1)
        wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
        wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
        wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
        wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)

    else:
      pool_members = [k[1] for k in value]
      for item in value:
        (iface_name, fqdn, ip) = item
        pool_name = "2pool-" + showaddr(key).replace('.','-') + "-" + pool_to_name(fqdn,pool_members)
        wleiden_zone["%s.%s" % (pool_name, fqdn)] = ip

  # Include static DNS entries
  # XXX: Should they override the autogenerated results?
  # XXX: Convert input to yaml more useable.
  # Format:
  ##; this is a comment
  ## roomburgh=CNodeRoomburgh1
  ## apkerk1.CNodeVosko=172.17.176.8 ;this as well
  dns = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))

  # Hack to allow special entries, for development
  wleiden_raw = dns['raw']
  del dns['raw']

  for comment, block in dns.iteritems():
    for k,v in block.iteritems():
      if valid_addr(v):
        wleiden_zone[k] = v
      else:
        wleiden_cname[k] = v

  details = dict()
  # 24 updates a day allowed
  details['serial'] = time.strftime('%Y%m%d%H')

  if external:
    dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
  else:
    dns_masters = ['sunny.wleiden.net']

  details['master'] = dns_masters[0]
  details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])

  dns_header = '''
$TTL 3h
%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 1d 12h 1w 3h )
        ; Serial, Refresh, Retry, Expire, Neg. cache TTL

%(ns_servers)s
  \n'''


  if not os.path.isdir(output_dir):
    os.makedirs(output_dir)
  details['zone'] = 'wleiden.net'
  f = open(os.path.join(output_dir,"db." + details['zone']), "w")
  f.write(dns_header % details)

  for host,ip in wleiden_zone.iteritems():
    if valid_addr(ip):
      f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
  for source,dest in wleiden_cname.iteritems():
    f.write("%s.wleiden.net. IN CNAME %s.wleiden.net.\n" % (source.lower(), dest.lower()))
  for source, dest in wleiden_raw.iteritems():
    f.write("%s.wleiden.net. %s\n" % (source, dest))
  f.close()

  # Create whole bunch of specific sub arpa zones. To keep it compliant
  for s in range(16,32):
    details['zone'] = '%i.172.in-addr.arpa' % s
    f = open(os.path.join(output_dir,"db." + details['zone']), "w")
    f.write(dns_header % details)

    #XXX: Not effient, fix to proper data structure and do checks at other
    # stages
    for host,ip in wleiden_zone.iteritems():
      if valid_addr(ip):
        if int(ip.split('.')[1]) == s:
          rev_ip = '.'.join(reversed(ip.split('.')))
          f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
    f.close()


def usage():
  print """Usage: %s <standalone [port] |test [test arguments]|static|dns>
Examples:
\tdns [outputdir]              =  Generate BIND compliant zone files in dns.
\tstandalone                   =  Run configurator webserver [default port=8000]
\twind-export                  =  Generate SQL import scripts for WIND database
\tfull-export                  =  Generate yaml export script for heatmap.
\tstatic                       =  Generate all config files and store on disk
\t                                with format ./static/%%NODE%%/%%FILE%%
\ttest CNodeRick dnsmasq.conf  =  Receive output of CGI script
\t                                for arguments CNodeRick/dnsmasq.conf
\tlist <all|nodes|proxies>     =  List systems which marked up.
"""
  exit(0)


def is_text_request():
  """ Find out whether we are calling from the CLI or any text based CLI utility """
  try:
    return os.environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
  except KeyError:
    return True

def main():
  """Hard working sub"""
  # Allow easy hacking using the CLI
  if not os.environ.has_key('PATH_INFO'):
    if len(sys.argv) < 2:
      usage()

    if sys.argv[1] == "standalone":
      import SocketServer
      import CGIHTTPServer
      # Hop to the right working directory.
      os.chdir(os.path.dirname(__file__))
      try:
        PORT = int(sys.argv[2])
      except (IndexError,ValueError):
        PORT = 8000

      class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
        """ Serve this CGI from the root of the webserver """
        def is_cgi(self):
          if "favicon" in self.path:
            return False

          self.cgi_info = (os.path.basename(__file__), self.path)
          self.path = ''
          return True
      handler = MyCGIHTTPRequestHandler
      SocketServer.TCPServer.allow_reuse_address = True
      httpd = SocketServer.TCPServer(("", PORT), handler)
      httpd.server_name = 'localhost'
      httpd.server_port = PORT

      logger.info("serving at port %s", PORT)
      try:
        httpd.serve_forever()
      except KeyboardInterrupt:
        httpd.shutdown()
        logger.info("All done goodbye")
    elif sys.argv[1] == "test":
      os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
      os.environ['SCRIPT_NAME'] = __file__
      process_cgi_request()
    elif sys.argv[1] == "unit-test":
      os.environ['SCRIPT_NAME'] = __file__
      for host in get_hostlist():
        for outfile in files:
          os.environ['PATH_INFO'] = "/".join([host,outfile])
          try:
            process_cgi_request()
          except Exception:
            print "# ERROR: %s" % os.environ['PATH_INFO']
            raise
            
           
    elif sys.argv[1] == "static":
      items = dict()
      for node in get_hostlist():
        items['node'] = node
        items['wdir'] = "./static/%(node)s" % items
        if not os.path.isdir(items['wdir']):
          os.makedirs(items['wdir'])
        datadump = get_yaml(node)
        for config in files:
          items['config'] = config
          logger.info("## Generating %(node)s %(config)s" % items)
          f = open("%(wdir)s/%(config)s" % items, "w")
          f.write(generate_config(node, config, datadump))
          f.close()
    elif sys.argv[1] == "wind-export":
      items = dict()
      for node in get_hostlist():
        datadump = get_yaml(node)
        sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
        VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
        sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
        VALUES (
          (SELECT id FROM users WHERE username = 'rvdzwet'),
          (SELECT id FROM nodes WHERE name = '%(nodename)s'),
          'Y');""" % datadump
        #for config in files:
        #  items['config'] = config
        #  print "## Generating %(node)s %(config)s" % items
        #  f = open("%(wdir)s/%(config)s" % items, "w")
        #  f.write(generate_config(node, config, datadump))
        #  f.close()
      for node in get_hostlist():
        datadump = get_yaml(node)
        for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
          ifacedump = datadump[iface_key]
          if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
            ifacedump['nodename'] = datadump['nodename']
            if not ifacedump.has_key('channel') or not ifacedump['channel']:
              ifacedump['channel'] = 0
            sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
              VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
              '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
    elif sys.argv[1] == "full-export":
      hosts = {}
      for node in get_hostlist():
        datadump = get_yaml(node)
        hosts[datadump['nodename']] = datadump
      print yaml.dump(hosts)

    elif sys.argv[1] == "dns":
      make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
    elif sys.argv[1] == "cleanup":
      # First generate all datadumps
      datadumps = dict()
      for host in get_hostlist():
        logger.info("# Processing: %s", host)
        # Set some boring default values
        datadump = { 'board' : 'UNKNOWN' }
        datadump.update(get_yaml(host))
        datadumps[datadump['autogen_realname']] = datadump


      for host,datadump in datadumps.iteritems():
        # Convert all yes and no to boolean values
        def fix_boolean(dump):
          for key in dump.keys():
            if type(dump[key]) == dict:
              dump[key] = fix_boolean(dump[key])
            elif str(dump[key]).lower() in ["yes", "true"]:
              dump[key] = True
            elif str(dump[key]).lower() in ["no", "false"]:
              # Compass richting no (Noord Oost) is valid input
              if key != "compass": dump[key] = False
          return dump
        datadump = fix_boolean(datadump)

        if datadump['rdnap_x'] and datadump['rdnap_y']:
          datadump['latitude'], datadump['longitude'] = rdnap.rd2etrs(datadump['rdnap_x'], datadump['rdnap_y'])
        elif datadump['latitude'] and datadump['longitude']:
          datadump['rdnap_x'], datadump['rdnap_y'] = rdnap.etrs2rd(datadump['latitude'], datadump['longitude'])

        if datadump['nodename'].startswith('Proxy'):
          datadump['nodename'] = datadump['nodename'].lower()

        for iface_key in datadump['autogen_iface_keys']:
          # Wireless Leiden SSID have an consistent lowercase/uppercase
          if datadump[iface_key].has_key('ssid'):
            ssid = datadump[iface_key]['ssid']
            prefix = 'ap-WirelessLeiden-'
            if ssid.lower().startswith(prefix.lower()):
              datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:] 
          if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
            datadump[iface_key]['mode'] = 'autogen-FIXME'
          if not datadump[iface_key].has_key('desc'):
            datadump[iface_key]['desc'] = 'autogen-FIXME'
        store_yaml(datadump)
    elif sys.argv[1] == "list":
      if sys.argv[2] == "nodes":
        systems = get_nodelist()
      elif sys.argv[2] == "proxies":
        systems = get_proxylist()
      elif sys.argv[2] == "all":
        systems = get_hostlist()
      else:
        usage()
      for system in systems:
       datadump = get_yaml(system)
       if datadump['status'] == "up":
         print system
    elif sys.argv[1] == "create":
      if sys.argv[2] == "network.kml":
        print make_network_kml.make_graph()
      else:
       usage()
      usage()
  else:
    # Do not enable debugging for config requests as it highly clutters the output
    if not is_text_request():
      cgitb.enable()
    process_cgi_request()


if __name__ == "__main__":
  main()