source: genesis/tools/gformat.py@ 13029

#!/usr/bin/env python
#
# vim:ts=2:et:sw=2:ai
# Wireless Leiden configuration generator, based on yaml files'
#
# XXX: This should be rewritten to make use of the ipaddr.py library.
#
# Sample apache configuration (mind the AcceptPathInfo!)
#  ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
#  <Directory /usr/local/www/genesis>
#    Allow from all
#    AcceptPathInfo On
#  </Directory>
#
# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
# the heavy template lifting all the time.
#
# WSGIDaemonProcess gformat threads=25
# WSGISocketPrefix run/wsgi
# 
# <Directory /var/www/cgi-bin>
#   WSGIProcessGroup gformat
# </Directory>
# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
#
# Package dependencies list:
# yum install python-yaml pyproj proj-epsg python-jinja2
#
# Rick van der Zwet <info@rickvanderzwet.nl>
#

# Hack to make the script directory is also threated as a module search path.
import sys
import os
sys.path.append(os.path.dirname(__file__))

SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]

import argparse
import cgi
import cgitb
import copy
import glob
import make_network_kml
import math
import pyproj
import random
import re
import socket
import string
import subprocess
import textwrap
import time
import urlparse

from pprint import pprint
from collections import defaultdict
try:
  import yaml
except ImportError, e:
  print e
  print "[ERROR] Please install the python-yaml or devel/py-yaml package"
  exit(1)

try:
    from yaml import CLoader as Loader
    from yaml import CDumper as Dumper
except ImportError:
    from yaml import Loader, Dumper

from jinja2 import Environment, Template
def yesorno(value):
  return "YES" if bool(value) else "NO"
env = Environment()
env.filters['yesorno'] = yesorno
def render_template(datadump, template):
  result = env.from_string(template).render(datadump)
  # Make it look pretty to the naked eye, as jinja templates are not so
  # friendly when it comes to whitespace formatting
  ## Remove extra whitespace at end of line lstrip() style.
  result = re.sub(r'\n[\ ]+','\n', result)
  ## Include only a single newline between an definition and a comment
  result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
  ## Remove extra newlines after single comment
  result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
  return result

import logging
logging.basicConfig(format='# %(levelname)s: %(message)s' )
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)


if os.environ.has_key('CONFIGROOT'):
  NODE_DIR = os.environ['CONFIGROOT']
else:
  NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
__version__ = '$Id: gformat.py 12787 2014-05-18 16:07:23Z rick $'

files = [
    'authorized_keys',
    'dnsmasq.conf',
    'dhcpd.conf',
    'rc.conf.local',
    'resolv.conf',
    'motd',
    'ntp.conf',
    'pf.hybrid.conf.local',
    'wleiden.yaml',
    ]

# Global variables uses
OK = 10
DOWN = 20
UNKNOWN = 90


ileiden_proxies = []
normal_proxies = []
datadump_cache = {}
interface_list_cache = {}
rc_conf_local_cache = {}
nameservers_cache = []
def clear_cache():
  ''' Poor mans cache implementation '''
  global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
  datadump_cache = {}
  interface_list_cache = {}
  rc_conf_local_cache = {}
  ileiden_proxies = []
  normal_proxies = []
  nameservers_cache = []

NO_DHCP = 0
DHCP_CLIENT = 10
DHCP_SERVER = 20
def dhcp_type(item):
  if not item.has_key('dhcp'):
    return NO_DHCP
  elif not item['dhcp']:
    return NO_DHCP
  elif item['dhcp'].lower() == 'client':
    return DHCP_CLIENT
  else:
    # Validation Checks
    begin,end = map(int,item['dhcp'].split('-'))
    if begin >= end:
      raise ValueError("DHCP Start >= DHCP End")
    return DHCP_SERVER

def etrs2rd(lat, lon):
  p1 = pyproj.Proj(proj='latlon',datum='WGS84')
  p2 = pyproj.Proj(init='EPSG:28992')
  RDx, RDy = pyproj.transform(p1,p2,lon, lat)
  return (RDx, RDy)

def rd2etrs(RDx, RDy):
  p1 = pyproj.Proj(init='EPSG:28992')
  p2 = pyproj.Proj(proj='latlon',datum='WGS84')
  lon, lat = pyproj.transform(p1,p2, RDx, RDy)
  return (lat, lon)

def get_yaml(item,add_version_info=True):
  try:
    """ Get configuration yaml for 'item'"""
    if datadump_cache.has_key(item):
      return datadump_cache[item].copy()

    gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')

    # Default values
    datadump = { 
      'autogen_revision' : 'NOTFOUND',
      'autogen_gfile' : gfile,
    }
    f = open(gfile, 'r')
    datadump.update(yaml.load(f,Loader=Loader))
    if datadump['nodetype'] == 'Hybrid':
      # Some values are defined implicitly
      if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
        datadump['service_incoming_rdr'] = True
      # Use some boring defaults
      defaults = { 
        'service_proxy_normal' : False, 
        'service_proxy_ileiden' : False, 
        'service_accesspoint' : True, 
        'service_incoming_rdr' : False, 
        'service_concentrator' : False,
        'monitoring_group' : 'wleiden',
      }
      for (key,value) in defaults.iteritems():
        if not datadump.has_key(key):
          datadump[key] = value
    f.close()

    # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
    if add_version_info:
      p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
      lines = p.communicate()[0].split('\n')
      if p.returncode == 0:
        for line in lines:
          if line:
            (key, value) = line.split(': ')
            datadump["autogen_" + key.lower().replace(' ','_')] = value

    # Preformat certain needed variables for formatting and push those into special object
    datadump['autogen_iface_keys'] = get_interface_keys(datadump)

    wlan_count=0
    try:
      for key in datadump['autogen_iface_keys']:
        datadump[key]['autogen_ifbase'] = key.split('_')[1]
        datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
        if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
          datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
          wlan_count += 1
        else:
          datadump[key]['autogen_ifname'] = datadump[key]['autogen_ifbase']
    except Exception as e:
      print "# Error while processing interface %s" % key
      raise

    dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
      if dhcp_type(datadump[key]) == DHCP_SERVER]

    datadump['autogen_dhcp_interfaces'] = dhcp_interfaces
    datadump['autogen_item'] = item

    datadump['autogen_realname'] = get_realname(datadump)
    datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
    datadump['autogen_fqdn'] = datadump['autogen_realname'] + '.' + datadump['autogen_domain']
    datadump_cache[item] = datadump.copy()
  except Exception as e:
    print "# Error while processing %s" % item
    raise
  return datadump


def store_yaml(datadump, header=False):
  """ Store configuration yaml for 'item'"""
  item = datadump['autogen_item'] 
  gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')

  output = generate_wleiden_yaml(datadump, header)

  f = open(gfile, 'w')
  f.write(output)
  f.close()


def network(ip):
  addr, mask = ip.split('/')
  # Not parsing of these folks please
  addr = parseaddr(addr)
  mask = int(mask)
  network = addr & ~((1 << (32 - mask)) - 1)
  return network
  


def make_relations(datadumps=dict()):
  """ Process _ALL_ yaml files to get connection relations """
  errors = []
  poel = defaultdict(list)

  if not datadumps:
    for host in get_hostlist():
      datadumps[host] = get_yaml(host)

  for host, datadump in datadumps.iteritems():
    try:
      for iface_key in datadump['autogen_iface_keys']:
        net_addr = network(datadump[iface_key]['ip'])
        poel[net_addr] += [(host,datadump[iface_key])]
    except (KeyError, ValueError), e:
      errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
      continue
  return (poel, errors)



def valid_addr(addr):
  """ Show which address is valid in which are not """
  return str(addr).startswith('172.')

def get_system_list(prefix):
  return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/%s*/wleiden.yaml" % (NODE_DIR, prefix))])

get_hybridlist = lambda: get_system_list("Hybrid")
get_nodelist = lambda: get_system_list("CNode")
get_proxylist = lambda: get_system_list("Proxy")

def get_hostlist():
  """ Combined hosts and proxy list"""
  return get_nodelist() + get_proxylist() + get_hybridlist()

def angle_between_points(lat1,lat2,long1,long2):
  """
  Return Angle in radians between two GPS coordinates
  See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
  """
  dy = lat2 - lat1
  dx = math.cos(lat1)*(long2 - long1)
  angle = math.atan2(dy,dx)
  return angle



def angle_to_cd(angle):
  """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """

  # For easy conversion get positive degree
  degrees = math.degrees(angle)
  abs_degrees = 360 + degrees if degrees < 0 else degrees

  # Numbers can be confusing calculate from the 4 main directions
  p = 22.5
  if abs_degrees < p:
    cd = "n"
  elif abs_degrees < (90 - p):
    cd = "no"
  elif abs_degrees < (90 + p):
    cd = "o"
  elif abs_degrees < (180 - p):
    cd = "zo"
  elif abs_degrees < (180 + p):
    cd = "z"
  elif abs_degrees < (270 - p):
    cd = "zw"
  elif abs_degrees < (270 + p):
    cd = "w"
  elif abs_degrees < (360 - p):
    cd = "nw"
  else:
    cd = "n"
  return cd



def cd_between_hosts(hostA, hostB, datadumps):
  # Using RDNAP coordinates
  dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
  dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
  return angle_to_cd(math.atan2(dx,dy))

  # GPS coordinates seems to fail somehow
  #latA = float(datadumps[hostA]['latitude'])
  #latB = float(datadumps[hostB]['latitude'])
  #lonA = float(datadumps[hostA]['longitude'])
  #lonB = float(datadumps[hostB]['longitude'])
  #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))


def generate_title(nodelist):
  """ Main overview page """
  items = {'root' : "." }
  def fl(spaces, line):
    return (' ' * spaces) + line + '\n'

  output = """
<html>
  <head>
    <title>Wireless leiden Configurator - GFormat</title>
    <style type="text/css">
     th        {background-color: #999999}
     tr:nth-child(odd)    {background-color: #cccccc}
     tr:nth-child(even)   {background-color: #ffffff}
     th, td    {padding: 0.1em 1em}
    </style>
  </head>
  <body>
    <center>
    <form type="GET" action="%(root)s">
      <input type="hidden" name="action" value="update">
      <input type="submit" value="Update Configuration Database (SVN)">
    </form>
    <table>
      <caption><h3>Wireless Leiden Configurator</h3></caption>
  """ % items

  for node in nodelist:
    items['node'] = node
    output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
    for config in files:
      items['config'] = config
      output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
    output += fl(5, "</tr>")
  output += """
    </table>
    <hr />
    <em>%s</em>
    </center>
  </body>
</html>
  """ % __version__

  return output



def generate_node(node):
  """ Print overview of all files available for node """
  return "\n".join(files)

def generate_node_overview(host):
  """ Print overview of all files available for node """
  datadump = get_yaml(host)
  params = { 'host' : host }
  output = "<em><a href='..'>Back to overview</a></em><hr />"
  output += "<h2>Available files:</h2><ul>"
  for cf in files:
    params['cf'] = cf
    output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
  output += "</ul>"

  # Generate and connection listing
  output += "<h2>Connected To:</h2><ul>"
  (poel, errors) = make_relations()
  for network, hosts in poel.iteritems():
    if host in [x[0] for x in hosts]:
      if len(hosts) == 1:
        # Single not connected interface
        continue
      for remote,ifacedump in hosts:
        if remote == host: 
          # This side of the interface
          continue
        params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
        output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
  output += "</ul>"
  output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"

  output += "<hr /><em><a href='..'>Back to overview</a></em>"
  return output


def generate_header(datadump, ctag="#"):
  return """\
%(ctag)s
%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
%(ctag)s Generated at %(date)s by %(host)s from r%(revision)s
%(ctag)s
""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }



def parseaddr(s):
  """ Process IPv4 CIDR notation addr to a (binary) number """
  f = s.split('.')
  return (long(f[0]) << 24L) + \
    (long(f[1]) << 16L) + \
    (long(f[2]) << 8L) + \
    long(f[3])



def showaddr(a):
  """ Display IPv4 addr in (dotted) CIDR notation """
  return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)


def is_member(ip, mask, canidate):
  """ Return True if canidate is part of ip/mask block"""
  ip_addr = parseaddr(ip)
  ip_canidate = parseaddr(canidate)
  mask = int(mask)
  ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
  ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
  return ip_addr == ip_canidate



def cidr2netmask(netmask):
  """ Given a 'netmask' return corresponding CIDR """
  return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))

def get_network(addr, mask):
  return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))


def generate_dhcpd_conf(datadump):
  """ Generate config file '/usr/local/etc/dhcpd.conf """
  output = generate_header(datadump)
  output += Template("""\
# option definitions common to all supported networks...
option domain-name "dhcp.{{ autogen_fqdn }}";

default-lease-time 600;
max-lease-time 7200;

# Use this to enble / disable dynamic dns updates globally.
#ddns-update-style none;

# If this DHCP server is the official DHCP server for the local
# network, the authoritative directive should be uncommented.
authoritative;

# Use this to send dhcp log messages to a different log file (you also
# have to hack syslog.conf to complete the redirection).
log-facility local7;

#
# Interface definitions
#
\n""").render(datadump)

  dhcp_out = defaultdict(list)
  for iface_key in datadump['autogen_iface_keys']:
    ifname = datadump[iface_key]['autogen_ifname']
    if not datadump[iface_key].has_key('comment'):
      datadump[iface_key]['comment'] = None
    dhcp_out[ifname].append("  ## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key])

    (addr, mask) = datadump[iface_key]['ip'].split('/')
    datadump[iface_key]['autogen_addr'] = addr
    datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
    datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
    if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
      dhcp_out[ifname].append("  subnet %(autogen_subnet)s netmask %(autogen_netmask)s {\n    ### not autoritive\n  }\n" % \
        datadump[iface_key])
      continue

    (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
    dhcp_part = ".".join(addr.split('.')[0:3])
    datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
    datadump[iface_key]['autogen_dhcp_stop'] =  dhcp_part + "." + dhcp_stop

    # Assume the first 10 IPs could be used for static entries
    if 'no_portal' in datadump:
      fixed = 5
      for mac in datadump['no_portal']:
        dhcp_out[ifname].append("""\
   host fixed-%(ifname)s-%(fixed)s {
     hardware ethernet %(mac)s;
     fixed-address %(prefix)s.%(fixed)s;
   }
""" % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed })
      fixed += 1

    dhcp_out[ifname].append("""\
   subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
     range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
     option routers %(autogen_addr)s;
     option domain-name-servers %(autogen_addr)s;
   }
""" % datadump[iface_key])

  for ifname,value in dhcp_out.iteritems():
    output += ("shared-network %s {\n" % ifname) + ''.join(value)  + '}\n\n'
  return output



def generate_dnsmasq_conf(datadump):
  """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
  output = generate_header(datadump)
  output += Template("""\
# DHCP server options
dhcp-authoritative
dhcp-fqdn
domain=dhcp.{{ autogen_fqdn }}
domain-needed
expand-hosts
log-async=100

# Low memory footprint
cache-size=10000

\n""").render(datadump)

  for iface_key in datadump['autogen_iface_keys']:
    if not datadump[iface_key].has_key('comment'):
      datadump[iface_key]['comment'] = None
    output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]

    if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
      output += "# not autoritive\n\n"
      continue

    (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
    (ip, cidr) = datadump[iface_key]['ip'].split('/')
    datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)

    dhcp_part = ".".join(ip.split('.')[0:3])
    datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
    datadump[iface_key]['autogen_dhcp_stop'] =  dhcp_part + "." + dhcp_stop
    output += "dhcp-range=%(autogen_ifname)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]

  return output


def make_interface_list(datadump):
  if interface_list_cache.has_key(datadump['autogen_item']):
    return (interface_list_cache[datadump['autogen_item']])
  # lo0 configuration:
  # - 172.32.255.1/32 is the proxy.wleiden.net deflector
  # - masterip is special as it needs to be assigned to at
  #   least one interface, so if not used assign to lo0
  addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
  dhclient_if = {'lo0' : False}

  # XXX: Find some way of send this output nicely
  output = ''

  masterip_used = False
  for iface_key in datadump['autogen_iface_keys']:
    if datadump[iface_key]['ip'].startswith(datadump['masterip']):
      masterip_used = True
      break
  if not masterip_used:
    addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))

  for iface_key in datadump['autogen_iface_keys']:
    ifacedump = datadump[iface_key]
    ifname = ifacedump['autogen_ifname']

    # Flag dhclient is possible
    if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
      dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT

    # Add interface IP to list
    item = (ifacedump['ip'], ifacedump['comment'])
    if addrs_list.has_key(ifname):
      addrs_list[ifname].append(item)
    else:
      addrs_list[ifname] = [item]

    # Alias only needs IP assignment for now, this might change if we
    # are going to use virtual accesspoints
    if "alias" in iface_key:
      continue

    # XXX: Might want to deduct type directly from interface name
    if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
      # Default to station (client) mode
      ifacedump['autogen_wlanmode'] = "sta"
      if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
        ifacedump['autogen_wlanmode'] = "ap"

      if not ifacedump.has_key('channel'):
        if ifacedump['type'] == '11a':
          ifacedump['channel'] = 36
        else:
          ifacedump['channel'] = 1

      # Allow special hacks at the back like wds and stuff
      if not ifacedump.has_key('extra'):
        ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
      else:
        ifacedump['autogen_extra'] = ifacedump['extra']
        
      output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
      output += ("create_args_%(autogen_ifname)s='wlanmode %(autogen_wlanmode)s mode " +\
        "%(type)s ssid %(ssid)s %(autogen_extra)s channel %(channel)s'\n") % ifacedump

    elif ifacedump['type'] in ['ethernet', 'eth']:
      # No special config needed besides IP
      pass
    else:
      assert False, "Unknown type " + ifacedump['type']

  store = (addrs_list, dhclient_if, output)
  interface_list_cache[datadump['autogen_item']] = store
  return(store)



def generate_rc_conf_local(datadump):
  """ Generate configuration file '/etc/rc.conf.local' """
  item = datadump['autogen_item']
  if rc_conf_local_cache.has_key(item):
    return rc_conf_local_cache[item]

  if not datadump.has_key('ileiden'):
    datadump['autogen_ileiden_enable'] = False
  else:
    datadump['autogen_ileiden_enable'] = datadump['ileiden']

  datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])

  if not ileiden_proxies or not normal_proxies:
    for proxy in get_proxylist():
      proxydump = get_yaml(proxy)
      if proxydump['ileiden']:
        ileiden_proxies.append(proxydump)
      else:
        normal_proxies.append(proxydump)
    for host in get_hybridlist():
      hostdump = get_yaml(host)
      if hostdump['service_proxy_ileiden']:
        ileiden_proxies.append(hostdump)
      if hostdump['service_proxy_normal']:
        normal_proxies.append(hostdump)

  datadump['autogen_ileiden_proxies'] = ileiden_proxies
  datadump['autogen_normal_proxies'] = normal_proxies
  datadump['autogen_ileiden_proxies_ips'] = ','.join([x['masterip'] for x in ileiden_proxies])
  datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
  datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
  datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])

  output = generate_header(datadump, "#");
  output += render_template(datadump, """\
hostname='{{ autogen_fqdn }}'
location='{{ location }}'
nodetype="{{ nodetype }}"

#
# Configured listings
#
captive_portal_whitelist=""
{% if nodetype == "Proxy" %}
#
# Proxy Configuration
#
{% if gateway -%}
defaultrouter="{{ gateway }}"
{% else -%}
#defaultrouter="NOTSET"
{% endif -%}
internalif="{{ internalif }}"
ileiden_enable="{{ autogen_ileiden_enable }}"
gateway_enable="{{ autogen_ileiden_enable }}"
pf_enable="yes"
pf_rules="/etc/pf.conf"
{% if autogen_ileiden_enable -%}
pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
lvrouted_enable="{{ autogen_ileiden_enable }}"
lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
{% else -%}
pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
{% endif -%}
{% if internalroute -%}
static_routes="wleiden"
route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
{% endif -%}

{% elif nodetype == "Hybrid" %}
  #
  # Hybrid Configuration
  #
  list_ileiden_proxies="
  {% for item in autogen_ileiden_proxies -%}
    {{ "%-16s"|format(item.masterip) }} # {{ item.autogen_realname }}
  {% endfor -%}
  "
  list_normal_proxies="
  {% for item in autogen_normal_proxies -%}
    {{ "%-16s"|format(item.masterip) }} # {{ item.autogen_realname }}
  {% endfor -%}
  "

  captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',')|default('none', true) }}"
  externalif="{{ externalif|default('vr0', true) }}"
  masterip="{{ masterip }}"
  
  # Defined services 
  service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
  service_proxy_normal="{{ service_proxy_normal|yesorno }}"
  service_accesspoint="{{ service_accesspoint|yesorno }}"
  service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
  service_concentrator="{{ service_concentrator|yesorno }}"
  #

  {% if service_proxy_ileiden %}
    pf_rules="/etc/pf.hybrid.conf"
    {% if service_concentrator %}
      pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
    {% else %}
      pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
    {% endif %}
    pf_flags="$pf_flags -D publicnat=80,443"
    lvrouted_flags="$lvrouted_flags -g"
  {% elif service_proxy_normal or service_incoming_rdr %}
    pf_rules="/etc/pf.hybrid.conf"
    pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
    pf_flags="$pf_flags -D publicnat=0"
    lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
    named_setfib="1"
    tinyproxy_setfib="1"
    dnsmasq_setfib="1"
    sshd_setfib="1"
  {% else %}
    named_auto_forward_only="YES"
    pf_rules="/etc/pf.node.conf"
    pf_flags=""
    lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
  {% endif %}
  {% if service_concentrator %}
    # Do mind installing certificates is NOT done automatically for security reasons
    openvpn_enable="YES"
    openvpn_configfile="/usr/local/etc/openvpn/client.conf"
  {% endif %}

  {% if service_proxy_normal %}
    tinyproxy_enable="yes"
  {% else %}
    pen_wrapper_enable="yes"
  {% endif %}

  {% if service_accesspoint %}
    pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
  {% endif %}

  {% if board == "ALIX2" %}
    #
    # ''Fat'' configuration, board has 256MB RAM
    #
    dnsmasq_enable="NO"
    named_enable="YES"
    {% if autogen_dhcp_interfaces -%}
    dhcpd_enable="YES"
    dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
    {% endif -%}
  {% endif -%}

  {% if gateway %}
    defaultrouter="{{ gateway }}"
  {% endif %}
{% elif nodetype == "CNode" %}
#
# NODE iLeiden Configuration
#

# iLeiden Proxies {{ autogen_ileiden_proxies_names }}
list_ileiden_proxies="{{ autogen_ileiden_proxies_ips }}"
# normal Proxies {{ autogen_normal_proxies_names }}
list_normal_proxies="{{ autogen_normal_proxies_ips }}"

captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"

lvrouted_flags="-u -s s00p3rs3kr3t -m 28 -z $list_ileiden_proxies"
{% endif %}

#
# Interface definitions
#\n
""")

  (addrs_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
  output += extra_ouput

  # Print IP address which needs to be assigned over here
  output += "\n"
  for iface,addrs in sorted(addrs_list.iteritems()):
    for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
      output += "# %s || %s || %s\n" % (iface, addr, comment)

    # Write DHCLIENT entry
    if dhclient_if[iface]:
      output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
    
    # Make sure the external address is always first as this is needed in the
    # firewall setup
    addrs = sorted(
        [x for x in addrs if not '0.0.0.0' in x[0]],
        key=lambda x: x[0].split('.')[0], 
        cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
      )
    addr_str = " ".join([x[0] for x in addrs])
    output += "ipv4_addrs_%s='%s'\n\n" % (iface, addr_str)

  rc_conf_local_cache[datadump['autogen_item']] = output
  return output




def get_all_configs():
  """ Get dict with key 'host' with all configs present """
  configs = dict()
  for host in get_hostlist():
    datadump = get_yaml(host)
    configs[host] = datadump
  return configs


def get_interface_keys(config):
  """ Quick hack to get all interface keys, later stage convert this to a iterator """
  return sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])


def get_used_ips(configs):
    """ Return array of all IPs used in config files"""
    ip_list = []
    for config in configs:
      ip_list.append(config['masterip'])
      for iface_key in get_interface_keys(config):
        l = config[iface_key]['ip']
        addr, mask = l.split('/')
        # Special case do not process
        if valid_addr(addr):
          ip_list.append(addr)
        else:
          logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
    return sorted(ip_list)



def get_nameservers(max_servers=None):
  if nameservers_cache:
    return nameservers_cache[0:max_servers]

  for host in get_hybridlist():
    hostdump = get_yaml(host)
    if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
      nameservers_cache.append((hostdump['masterip'], hostdump['autogen_realname']))
  for host in get_proxylist():
    hostdump = get_yaml(host)
    if hostdump['status'] == 'up':
      nameservers_cache.append((hostdump['masterip'], hostdump['autogen_realname']))

  return nameservers_cache[0:max_servers]


def generate_resolv_conf(datadump):
  """ Generate configuration file '/etc/resolv.conf' """
  # XXX: This should properly going to be an datastructure soon
  datadump['autogen_header'] = generate_header(datadump, "#")
  datadump['autogen_edge_nameservers'] = ''


  for masterip,realname in get_nameservers():
    datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)

  return Template("""\
{{ autogen_header }}
search wleiden.net

# Try local (cache) first
nameserver 127.0.0.1

{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
nameserver 8.8.8.8   # Google Public NameServer
nameserver 8.8.4.4   # Google Public NameServer
{% else -%}
# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
{{ autogen_edge_nameservers }}
{% endif -%}
""").render(datadump)



def generate_ntp_conf(datadump):
  """ Generate configuration file '/etc/ntp.conf' """
  # XXX: This should properly going to be an datastructure soon

  datadump['autogen_header'] = generate_header(datadump, "#")
  datadump['autogen_ntp_servers'] = ''
  for host in get_proxylist():
    hostdump = get_yaml(host)
    datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(autogen_realname)s\n" % hostdump
  for host in get_hybridlist():
    hostdump = get_yaml(host)
    if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
      datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(autogen_realname)s\n" % hostdump

  return Template("""\
{{ autogen_header }}

{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
# Machine hooked to internet.
server 0.nl.pool.ntp.org iburst maxpoll 9
server 1.nl.pool.ntp.org iburst maxpoll 9
server 2.nl.pool.ntp.org iburst maxpoll 9
server 3.nl.pool.ntp.org iburst maxpoll 9
{% else -%}
# Local Wireless Leiden NTP Servers.
server 0.pool.ntp.wleiden.net iburst maxpoll 9
server 1.pool.ntp.wleiden.net iburst maxpoll 9
server 2.pool.ntp.wleiden.net iburst maxpoll 9
server 3.pool.ntp.wleiden.net iburst maxpoll 9

# All the configured NTP servers
{{ autogen_ntp_servers }}
{% endif %}

# If a server loses sync with all upstream servers, NTP clients
# no longer follow that server. The local clock can be configured
# to provide a time source when this happens, but it should usually
# be configured on just one server on a network. For more details see
# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
# The use of Orphan Mode may be preferable.
#
server 127.127.1.0
fudge 127.127.1.0 stratum 10
""").render(datadump)


def generate_pf_hybrid_conf_local(datadump):
  """ Generate configuration file '/etc/pf.hybrid.conf.local' """
  datadump['autogen_header'] = generate_header(datadump, "#")
  return Template("""\
{{ autogen_header }}

# Redirect some internal facing services outside (7)
# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
{% endfor -%}
""").render(datadump)

def generate_motd(datadump):
  """ Generate configuration file '/etc/motd' """
  output = Template("""\
FreeBSD run ``service motd onestart'' to make me look normal

  WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
  Loc: {{ location }}

Services:
{% if board == "ALIX2" -%}
{{" -"}} Core Node ({{ board }})
{% else -%}
{{" -"}} Hulp Node ({{ board }})
{% endif -%}
{% if service_proxy_normal -%}
{{" -"}} Normal Proxy
{% endif -%}
{% if service_proxy_ileiden -%} 
{{" -"}} iLeiden Proxy
{% endif -%}
{% if service_incoming_rdr -%} 
{{" -"}} Incoming port redirects
{% endif %}
Interlinks:\n
""").render(datadump)

  (addrs_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
  # Just nasty hack to make the formatting looks nice
  iface_len = max(map(len,addrs_list.keys()))
  addr_len = max(map(len,[x[0] for x in [x[0] for x in addrs_list.values()]]))
  for iface,addrs in sorted(addrs_list.iteritems()):
    if iface in ['lo0']:
      continue
    for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
      output += " - %s || %s || %s\n" % (iface.ljust(iface_len), addr.ljust(addr_len), comment)

  output += '\n'
  output += """\
Attached bridges:
"""
  has_item = False
  for iface_key in datadump['autogen_iface_keys']:
    ifacedump = datadump[iface_key]
    if ifacedump.has_key('ns_ip'):
      ifacedump['autogen_ns_ip'] = ifacedump['ns_ip'].split('/')[0]
      has_item = True
      output += " - %(autogen_ifname)s || %(mode)s || https://%(autogen_ns_ip)s\n" % ifacedump
  if not has_item:
    output += " - none\n"

  return output


def format_yaml_value(value):
  """ Get yaml value in right syntax for outputting """
  if isinstance(value,str):
    output = '"%s"' % value
  else:
    output = value
  return output



def format_wleiden_yaml(datadump):
  """ Special formatting to ensure it is editable"""
  output = "# Genesis config yaml style\n"
  output += "# vim:ts=2:et:sw=2:ai\n"
  output += "#\n"
  iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
  for key in sorted(set(datadump.keys()) - set(iface_keys)):
    if key == 'rdr_rules':
      output += '%-10s:\n' % 'rdr_rules'
      for rdr_rule in datadump[key]:
        output += '- %s\n' % rdr_rule
    else:
      output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))

  output += "\n\n"

  # Format (key, required)
  key_order = (
    ('comment', True),
    ('ip', True),
    ('desc', True),
    ('sdesc', True),
    ('mode', True),
    ('type', True),
    ('extra_type', False), 
    ('channel', False),
    ('ssid', False),
    ('dhcp', True),
    ('compass', False),
    ('distance', False),
    ('ns_ip', False),
    ('bullet2_ip', False),
    ('ns_mac', False),
    ('bullet2_mac', False),
    ('ns_type', False),
    ('bridge_type', False),
    ('status', True),
  )

  for iface_key in sorted(iface_keys):
    try:
      remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
      if remainder:
        raise KeyError("invalid keys: %s" % remainder)

      output += "%s:\n" % iface_key
      for key,required in key_order:
        if datadump[iface_key].has_key(key):
          output += "  %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
      output += "\n\n"
    except Exception as e:
      print "# Error while processing interface %s" % iface_key
      raise

  return output



def generate_wleiden_yaml(datadump, header=True):
  """ Generate (petty) version of wleiden.yaml"""
  output = generate_header(datadump, "#") if header else ''

  for key in datadump.keys():
    if key.startswith('autogen_'):
      del datadump[key]
    # Interface autogen cleanups
    elif type(datadump[key]) == dict:
      for key2 in datadump[key].keys():
        if key2.startswith('autogen_'):
          del datadump[key][key2]
      
  output += format_wleiden_yaml(datadump)
  return output

def generate_nanostation_config(datadump, iface, ns_type):
  #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
  datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
  
  datadump.update(datadump['iface_%s' % iface])

  return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump

def generate_yaml(datadump):
  return generate_config(datadump['nodename'], "wleiden.yaml", datadump)



def generate_config(node, config, datadump=None):
  """ Print configuration file 'config' of 'node' """
  output = ""
  try:
    # Load config file
    if datadump == None:
      datadump = get_yaml(node)

    if config == 'wleiden.yaml':
      output += generate_wleiden_yaml(datadump)
    elif config == 'authorized_keys':
      f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
      output += f.read()
      node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
      # Fetch local keys if existing
      if os.path.exists(node_keys):
         output += open(node_keys, 'r').read()
      f.close()
    elif config == 'dnsmasq.conf':
      output += generate_dnsmasq_conf(datadump)
    elif config == 'dhcpd.conf':
      output += generate_dhcpd_conf(datadump)
    elif config == 'rc.conf.local':
      output += generate_rc_conf_local(datadump)
    elif config == 'resolv.conf':
      output += generate_resolv_conf(datadump)
    elif config == 'ntp.conf':
      output += generate_ntp_conf(datadump)
    elif config == 'motd':
      output += generate_motd(datadump)
    elif config == 'pf.hybrid.conf.local':
      output += generate_pf_hybrid_conf_local(datadump)
    elif config.startswith('vr'):
      interface, ns_type = config.strip('.yaml').split('-')
      output += generate_nanostation_config(datadump, interface, ns_type)
    else:
      assert False, "Config not found!"
  except IOError, e:
    output += "[ERROR] Config file not found"
  return output



def process_cgi_request(environ=os.environ):
  """ When calling from CGI """
  response_headers = []
  content_type = 'text/plain'

  # Update repository if requested
  form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
  if form and form.has_key("action") and "update" in form["action"]:
    output = "[INFO] Updating subverion, please wait...\n"
    output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
    output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
    output += "[INFO] All done, redirecting in 5 seconds"
    response_headers += [
      ('Refresh', '5; url=.'),
    ]
    reload_cache()
  else:
    base_uri = environ['PATH_INFO']
    uri = base_uri.strip('/').split('/')

    output = "Template Holder"
    if base_uri.endswith('/create/network.kml'):
        content_type='application/vnd.google-earth.kml+xml'
        output = make_network_kml.make_graph()
    elif base_uri.endswith('/api/get/nodeplanner.json'):
        content_type='application/json'
        output = make_network_kml.make_nodeplanner_json()
    elif not uri[0]:
      if is_text_request(environ):
        output = '\n'.join(get_hostlist())
      else:
        content_type = 'text/html'
        output = generate_title(get_hostlist())
    elif len(uri) == 1:
      if is_text_request(environ):
        output = generate_node(uri[0])
      else:
        content_type = 'text/html'
        output = generate_node_overview(uri[0])
    elif len(uri) == 2:
      output = generate_config(uri[0], uri[1])
    else:
      assert False, "Invalid option"

  # Return response
  response_headers += [
    ('Content-type', content_type),
    ('Content-Length', str(len(output))),
  ]
  return(response_headers, str(output))


def get_realname(datadump):
  # Proxy naming convention is special, as the proxy name is also included in
  # the nodename, when it comes to the numbered proxies.
  if datadump['nodetype'] == 'Proxy':
    realname = datadump['nodetype'] + datadump['nodename'].replace('proxy','')
  else:
    # By default the full name is listed and also a shortname CNAME for easy use.
    realname = datadump['nodetype'] + datadump['nodename']
  return(realname)



def make_dns(output_dir = 'dns', external = False):
  items = dict()

  # hostname is key, IP is value
  wleiden_zone = defaultdict(list)
  wleiden_cname = dict()

  pool = dict()
  for node in get_hostlist():
    datadump = get_yaml(node)

    # Proxy naming convention is special
    fqdn = datadump['autogen_realname']
    if datadump['nodetype'] in ['CNode', 'Hybrid']:
      wleiden_cname[datadump['nodename']] = fqdn
 
    if datadump.has_key('rdr_host'):
      remote_target = datadump['rdr_host']
    elif datadump.has_key('remote_access') and datadump['remote_access']:
      remote_target = datadump['remote_access'].split(':')[0]
    else:
      remote_target = None

    if remote_target:
      try:
        parseaddr(remote_target)
        wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
      except (IndexError, ValueError):
        wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'


    wleiden_zone[fqdn].append((datadump['masterip'], True))

    # Hacking to get proper DHCP IPs and hostnames
    for iface_key in get_interface_keys(datadump):
      iface_name = iface_key.replace('_','-')
      (ip, cidr) = datadump[iface_key]['ip'].split('/')
      try:
        (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
        datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
        dhcp_part = ".".join(ip.split('.')[0:3])
        if ip != datadump['masterip']:
          wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, False))
        for i in range(int(dhcp_start), int(dhcp_stop) + 1):
          wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
      except (AttributeError, ValueError, KeyError):
        # First push it into a pool, to indentify the counter-part later on
        addr = parseaddr(ip)
        cidr = int(cidr)
        addr = addr & ~((1 << (32 - cidr)) - 1)
        if pool.has_key(addr):
          pool[addr] += [(iface_name, fqdn, ip)]
        else:
          pool[addr] = [(iface_name, fqdn, ip)]
        continue



  #  WL uses an /29 to configure an interface. IP's are ordered like this: 
  #     MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4) 
    
  sn = lambda x: re.sub(r'(?i)^cnode','',x)

  # Automatic naming convention of interlinks namely 2 + remote.lower()
  for (key,value) in pool.iteritems():
    # Make sure they are sorted from low-ip to high-ip
    value = sorted(value, key=lambda x: parseaddr(x[2]))

    if len(value) == 1:
      (iface_name, fqdn, ip) = value[0]
      wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))

      # Device DNS names
      if 'cnode' in fqdn.lower():
        wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
        wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))

    elif len(value) == 2:
      (a_iface_name, a_fqdn, a_ip) = value[0]
      (b_iface_name, b_fqdn, b_ip) = value[1]
      wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
      wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))

      # Device DNS names
      if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
        wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
        wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
        wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
        wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
        wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
        wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)

    else:
      pool_members = [k[1] for k in value]
      for item in value:
        (iface_name, fqdn, ip) = item
        wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))

  # Include static DNS entries
  # XXX: Should they override the autogenerated results?
  # XXX: Convert input to yaml more useable.
  # Format:
  ##; this is a comment
  ## roomburgh=CNodeRoomburgh1
  ## apkerk1.CNodeVosko=172.17.176.8 ;this as well
  dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))

  # Hack to allow special entries, for development
  wleiden_raw = {}

  for line in dns_list:
    reverse = False
    k, items = line.items()[0]
    if type(items) == dict:
      if items.has_key('reverse'):
        reverse = items['reverse']
        items = items['a']
      else:
        items = items['cname']
    items = [items] if type(items) != list else items
    for item in items:
      if item.startswith('IN '):
        wleiden_raw[k] = item
      elif valid_addr(item):
        wleiden_zone[k].append((item, reverse))
      else:
        wleiden_cname[k] = item

  # Hack to get dynamic pool listing
  def chunks(l, n):
      return [l[i:i+n] for i in range(0, len(l), n)]

  ntp_servers = [x[0] for x in get_nameservers()]
  for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
    for ntp_server in chunk:
      wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))

  details = dict()
  # 24 updates a day allowed
  details['serial'] = time.strftime('%Y%m%d%H')

  if external:
    dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
  else:
    dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]

  details['master'] = dns_masters[0]
  details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])

  dns_header = '''
$TTL 3h
%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
        ; Serial, Refresh, Retry, Expire, Neg. cache TTL

%(ns_servers)s
  \n'''


  if not os.path.isdir(output_dir):
    os.makedirs(output_dir)
  details['zone'] = 'wleiden.net'
  f = open(os.path.join(output_dir,"db." + details['zone']), "w")
  f.write(dns_header % details)

  for host,items in wleiden_zone.iteritems():
    for ip,reverse in items:
      if ip not in ['0.0.0.0']:
        f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
  for source,dest in wleiden_cname.iteritems():
    dest = dest if dest.endswith('.') else dest + ".wleiden.net."
    f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
  for source, dest in wleiden_raw.iteritems():
    f.write("%s.wleiden.net. %s\n" % (source, dest))
  f.close()

  # Create whole bunch of specific sub arpa zones. To keep it compliant
  for s in range(16,32):
    details['zone'] = '%i.172.in-addr.arpa' % s
    f = open(os.path.join(output_dir,"db." + details['zone']), "w")
    f.write(dns_header % details)

    #XXX: Not effient, fix to proper data structure and do checks at other
    # stages
    for host,items in wleiden_zone.iteritems():
      for ip,reverse in items:
        if not reverse:
          continue
        if valid_addr(ip):
          if valid_addr(ip):
            if int(ip.split('.')[1]) == s:
              rev_ip = '.'.join(reversed(ip.split('.')))
              f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
    f.close()


def usage():
  print """Usage: %(prog)s <argument>
Argument:
\tstandalone [port]            =  Run configurator webserver [8000]
\tdns [outputdir]              =  Generate BIND compliant zone files in dns [./dns]
\tnagios-export [--heavy-load] =  Generate basic nagios configuration file.
\tfull-export                  =  Generate yaml export script for heatmap.
\tstatic [outputdir]           =  Generate all config files and store on disk
\t                                with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
\ttest <node> [<file>]         =  Receive output for certain node [all files].
\ttest-cgi <node> <file>       =  Receive output of CGI script [all files].
\tlist <status> <items>        =  List systems which have certain status

Arguments:
\t<node>    = NodeName (example: HybridRick)
\t<file>    = %(files)s
\t<status>  = all|up|down|planned
\t<items>   = systems|nodes|proxies

NOTE FOR DEVELOPERS; you can test your changes like this:
  BEFORE any changes in this code:
  $ ./gformat.py static /tmp/pre
  AFTER the changes:
  $ ./gformat.py static /tmp/post
  VIEW differences and VERIFY all are OK:
  $ diff -urI 'Generated' -r /tmp/pre /tmp/post
""" % { 'prog' : sys.argv[0], 'files'  : '|'.join(files) }
  exit(0)


def is_text_request(environ=os.environ):
  """ Find out whether we are calling from the CLI or any text based CLI utility """
  try:
    return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
  except KeyError:
    return True

def switchFormat(setting):
  if setting:
    return "YES"
  else:
    return "NO"

def rlinput(prompt, prefill=''):
   import readline
   readline.set_startup_hook(lambda: readline.insert_text(prefill))
   try:
      return raw_input(prompt)
   finally:
      readline.set_startup_hook()

def fix_conflict(left, right, default='i'):
  while True:
    print "## %-30s | %-30s" % (left, right)
    c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
    if not c:
      c = default

    if c in ['l','1']:
      return left
    elif c in ['r','2']:
      return right
    elif c in ['e', '3']:
      return rlinput("Edit: ", "%30s | %30s" % (left, right))
    elif c in ['i', '4']:
      return None
    else:
      print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c



def print_cgi_response(response_headers, output):
  """Could we not use some kind of wsgi wrapper to make this output?"""
  for header in response_headers:
     print "%s: %s" % header
  print
  print output


def fill_cache():
 ''' Poor man re-loading of few cache items (the slow ones) '''
 for host in get_hostlist():
   get_yaml(host)


def reload_cache():
  clear_cache()
  fill_cache()
  

def main():
  """Hard working sub"""
  # Allow easy hacking using the CLI
  if not os.environ.has_key('PATH_INFO'):
    if len(sys.argv) < 2:
      usage()

    if sys.argv[1] == "standalone":
      import SocketServer
      import CGIHTTPServer
      # Hop to the right working directory.
      os.chdir(os.path.dirname(__file__))
      try:
        PORT = int(sys.argv[2])
      except (IndexError,ValueError):
        PORT = 8000

      class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
        """ Serve this CGI from the root of the webserver """
        def is_cgi(self):
          if "favicon" in self.path:
            return False

          self.cgi_info = (os.path.basename(__file__), self.path)
          self.path = ''
          return True
      handler = MyCGIHTTPRequestHandler
      SocketServer.TCPServer.allow_reuse_address = True
      httpd = SocketServer.TCPServer(("", PORT), handler)
      httpd.server_name = 'localhost'
      httpd.server_port = PORT

      logger.info("serving at port %s", PORT)
      try:
        httpd.serve_forever()
      except KeyboardInterrupt:
        httpd.shutdown()
        logger.info("All done goodbye")
    elif sys.argv[1] == "test":
      # Basic argument validation
      try:
        node = sys.argv[2]
        datadump = get_yaml(node)
      except IndexError:
        print "Invalid argument"
        exit(1)
      except IOError as e:
        print e
        exit(1)
        

      # Get files to generate
      gen_files = sys.argv[3:] if len(sys.argv) > 3 else files

      # Actual config generation
      for config in gen_files:
         logger.info("## Generating %s %s", node, config)
         print generate_config(node, config, datadump)
    elif sys.argv[1] == "test-cgi":
      os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
      os.environ['SCRIPT_NAME'] = __file__
      response_headers, output = process_cgi_request()
      print_cgi_response(response_headers, output)
    elif sys.argv[1] == "static":
      items = dict()
      items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
      for node in get_hostlist():
        items['node'] = node
        items['wdir'] = "%(output_dir)s/%(node)s" % items
        if not os.path.isdir(items['wdir']):
          os.makedirs(items['wdir'])
        datadump = get_yaml(node)
        for config in files:
          items['config'] = config
          logger.info("## Generating %(node)s %(config)s" % items)
          f = open("%(wdir)s/%(config)s" % items, "w")
          f.write(generate_config(node, config, datadump))
          f.close()
    elif sys.argv[1] == "wind-export":
      items = dict()
      for node in get_hostlist():
        datadump = get_yaml(node)
        sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
        VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
        sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
        VALUES (
          (SELECT id FROM users WHERE username = 'rvdzwet'),
          (SELECT id FROM nodes WHERE name = '%(nodename)s'),
          'Y');""" % datadump
        #for config in files:
        #  items['config'] = config
        #  print "## Generating %(node)s %(config)s" % items
        #  f = open("%(wdir)s/%(config)s" % items, "w")
        #  f.write(generate_config(node, config, datadump))
        #  f.close()
      for node in get_hostlist():
        datadump = get_yaml(node)
        for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
          ifacedump = datadump[iface_key]
          if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
            ifacedump['nodename'] = datadump['nodename']
            if not ifacedump.has_key('channel') or not ifacedump['channel']:
              ifacedump['channel'] = 0
            sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
              VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
              '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
    elif sys.argv[1] == "nagios-export":
      try:
        heavy_load = (sys.argv[2] == "--heavy-load")
      except IndexError:
        heavy_load = False

      hostgroup_details = {
         'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
         'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
         'walphen' : 'Stichting Wireless Alphen',
         'westeinder' : 'WestEinder Plassen',
      }

      params = {
        'check_interval' : 5 if heavy_load else 120,
        'retry_interval' : 1  if heavy_load else 10,
        'max_check_attempts' : 10 if heavy_load else 6,
      }  

      print '''\
define host {
        name                  wleiden-node           ; Default Node Template
        use                   generic-host           ; Use the standard template as initial starting point
        check_period          24x7                   ; By default, FreeBSD hosts are checked round the clock
        check_interval        %(check_interval)s     ; Actively check the host every 5 minutes
        retry_interval        %(retry_interval)s     ; Schedule host check retries at 1 minute intervals
        max_check_attempts    %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
        check_command         check-host-alive       ; Default command to check FreeBSD hosts
        register              0                      ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
}

define service {
        name                  wleiden-service        ; Default Service Template
        use                   generic-service        ; Use the standard template as initial starting point
        check_period          24x7                   ; By default, FreeBSD hosts are checked round the clock
        check_interval        %(check_interval)s     ; Actively check the host every 5 minutes
        retry_interval        %(retry_interval)s     ; Schedule host check retries at 1 minute intervals
        max_check_attempts    %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
        register              0                      ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
}

# Please make sure to install:
# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
#
define command{
        command_name          check_netsnmp_disk
        command_line          $USER1$/check_netsnmp -H $HOSTADDRESS$ -o disk
}

define command{
        command_name          check_netsnmp_load
        command_line          $USER1$/check_netsnmp -H $HOSTADDRESS$ -o load
}

define command{
        command_name          check_netsnmp_proc
        command_line          $USER1$/check_netsnmp -H $HOSTADDRESS$ -o proc
}

define command{
        command_name          check_disk_snmp
        command_line          /usr/local/libexec/nagios/check_disk_snmp.pl -H $HOSTADDRESS$ -d $ARG1$
}

define command{
       command_name           check_by_ssh
       command_line           $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
}

define command{
        command_name          check_dns_wl
        command_line          $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
}


# TDB: dhcp leases
# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec

# TDB: internet status
# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file

# TDB: Advanced local passive checks
# /usr/local/libexec/nagios/check_by_ssh
''' % params

      print '''\
# Service Group, not displayed by default        
define hostgroup {
       hostgroup_name         srv_hybrid
       alias                  All Hybrid Nodes
       register 0
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   SSH
        check_command         check_ssh
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   HTTP
        check_command         check_http
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   DNS
        check_command         check_dns_wl!"www.wirelessleiden.nl"
}


define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   VAR space
        check_command         check_disk_snmp!/var
}
'''

      if heavy_load:
        print '''\
define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   SNMP
        check_command         check_snmp
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   NTP
        check_command         check_ntp_peer
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   LOAD
        check_command         check_netsnmp_load
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   PROC
        check_command         check_netsnmp_proc
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   DISK
        check_command         check_netsnmp_disk
}
'''
      for node in get_hostlist():
        datadump = get_yaml(node)
        if not datadump['status'] == 'up':
          continue
        if not hostgroup_details.has_key(datadump['monitoring_group']):
           hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
        print '''\
define host {
        use                   wleiden-node
        host_name             %(autogen_fqdn)s
        address               %(masterip)s
        hostgroups            srv_hybrid,%(monitoring_group)s
}
''' % datadump

      for name,alias in hostgroup_details.iteritems(): 
        print '''\
define hostgroup {
       hostgroup_name         %s
       alias                  %s
} ''' % (name, alias)      


    elif sys.argv[1] == "full-export":
      hosts = {}
      for node in get_hostlist():
        datadump = get_yaml(node)
        hosts[datadump['nodename']] = datadump
      print yaml.dump(hosts)

    elif sys.argv[1] == "dns":
      make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
    elif sys.argv[1] == "cleanup":
      # First generate all datadumps
      datadumps = dict()
      ssid_to_node = dict()
      for host in get_hostlist():
        logger.info("# Processing: %s", host)
        # Set some boring default values
        datadump = { 'board' : 'UNKNOWN' }
        datadump.update(get_yaml(host))
        datadumps[datadump['autogen_realname']] = datadump

      (poel, errors) = make_relations(datadumps)
      print "\n".join(["# WARNING: %s" % x for x in errors])

      for host,datadump in datadumps.iteritems():
        try:
          # Convert all yes and no to boolean values
          def fix_boolean(dump):
            for key in dump.keys():
              if type(dump[key]) == dict:
                dump[key] = fix_boolean(dump[key])
              elif str(dump[key]).lower() in ["yes", "true"]:
                dump[key] = True
              elif str(dump[key]).lower() in ["no", "false"]:
                # Compass richting no (Noord Oost) is valid input
                if key != "compass": dump[key] = False
            return dump
          datadump = fix_boolean(datadump)

          if datadump['rdnap_x'] and datadump['rdnap_y']:
            datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
          elif datadump['latitude'] and datadump['longitude']:
            datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])

          if datadump['nodename'].startswith('Proxy'):
            datadump['nodename'] = datadump['nodename'].lower()

          for iface_key in datadump['autogen_iface_keys']:
            try:
              # All our normal wireless cards are normal APs now
              if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
                datadump[iface_key]['mode'] = 'ap'
              # Wireless Leiden SSID have an consistent lowercase/uppercase
              if datadump[iface_key].has_key('ssid'):
                ssid = datadump[iface_key]['ssid']
                prefix = 'ap-WirelessLeiden-'
                if ssid.lower().startswith(prefix.lower()):
                  datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:] 
              if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
                datadump[iface_key]['mode'] = 'autogen-FIXME'
              if not datadump[iface_key].has_key('comment'):
                datadump[iface_key]['comment'] = 'autogen-FIXME'

              if datadump[iface_key].has_key('ns_mac'):
                datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()

              if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
                datadump[iface_key] = datadump[iface_key]['desc']

              # We are not using 802.11b anymore. OFDM is preferred over DSSS
              # due to better collision avoidance.
              if datadump[iface_key]['type'] == '11b':
                datadump[iface_key]['type'] = '11g'

              # Setting 802.11g channels to de-facto standards, to avoid
              # un-detected sharing with other overlapping channels
              # 
              # Technically we could also use channel 13 in NL, but this is not
              # recommended as foreign devices might not be able to select this
              # channel. Secondly using 1,5,9,13 instead is going to clash with
              # the de-facto usage of 1,6,11.
              # 
              # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
              channels_at_2400Mhz = (1,6,11)
              if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
                datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
                if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
                   datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)

              # Mandatory interface keys
              if not datadump[iface_key].has_key('status'):
                datadump[iface_key]['status'] = 'planned'

              x = datadump[iface_key]['comment']
              datadump[iface_key]['comment'] = x[0].upper() + x[1:]

              # Fixing bridge_type if none is found
              if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
                if not 'bridge_type' in datadump[iface_key]:
                   datadump[iface_key]['bridge_type'] = 'NanoStation M5'

              # Making sure description works
              if datadump[iface_key].has_key('desc'):
                if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
                  del datadump[iface_key]['desc']
                else:
                  print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
                  response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
                  if response:
                    datadump[iface_key]['comment'] = response
                    del datadump[iface_key]['desc']

              # Check DHCP configuration
              dhcp_type(datadump[iface_key])

              # Set the compass value based on the angle between the poels
              if datadump[iface_key].has_key('ns_ip'):
                my_pool = poel[network(datadump[iface_key]['ip'])]
                remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
                if remote_hosts:
                  compass_target = remote_hosts[0]
                  datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)

              # Monitoring Group default
              if not 'monitoring_group' in datadump:
                datadump['monitoring_group'] = 'wleiden'

            except Exception as e:
              print "# Error while processing interface %s" % iface_key
              raise
          store_yaml(datadump)
        except Exception as e:
          print "# Error while processing %s" % host
          raise
    elif sys.argv[1] == "list":
      use_fqdn = False
      if len(sys.argv) < 4 or not sys.argv[2] in ["up", "down", "planned", "all"]:
        usage()
      if sys.argv[3] == "nodes":
        systems = get_nodelist()
      elif sys.argv[3] == "proxies":
        systems = get_proxylist()
      elif sys.argv[3] == "systems":
        systems = get_hostlist()
      else:
        usage()
      if len(sys.argv) > 4:
        if sys.argv[4] == "fqdn":
          use_fqdn = True
        else:
          usage()

      for system in systems:
       datadump = get_yaml(system)

       output = datadump['autogen_fqdn'] if use_fqdn else system
       if sys.argv[2] == "all":
         print output
       elif datadump['status'] == sys.argv[2]:
         print output
    elif sys.argv[1] == "create":
      if sys.argv[2] == "network.kml":
        print make_network_kml.make_graph()
      elif sys.argv[2] == "host-ips.txt":
        for system in get_hostlist():
          datadump = get_yaml(system)
          ips = [datadump['masterip']]
          for ifkey in datadump['autogen_iface_keys']:
            ips.append(datadump[ifkey]['ip'].split('/')[0])
          print system, ' '.join(ips)
      elif sys.argv[2] == "host-pos.txt":
        for system in get_hostlist():
          datadump = get_yaml(system)
          print system, datadump['rdnap_x'], datadump['rdnap_y']
      elif sys.argv[2] == 'ssh_config':
        print  '''
Host *.wleiden.net
  User root

Host 172.16.*.*
  User root
'''
        for system in get_hostlist():
          datadump = get_yaml(system)
          print '''\
Host %s
  User root

Host %s
  User root

Host %s
  User root

Host %s
  User root
''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
      else:
        usage()
    else:
      usage()
  else:
    # Do not enable debugging for config requests as it highly clutters the output
    if not is_text_request():
      cgitb.enable()
    response_headers, output = process_cgi_request()
    print_cgi_response(response_headers, output)

def application(environ, start_response):
  status = '200 OK'
  response_headers, output = process_cgi_request(environ)
  start_response(status, response_headers)

  # Debugging only
  # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
  # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
  return [output]

if __name__ == "__main__":
  main()