source: genesis/tools/gformat.py@ 14171

#!/usr/local/bin/python
#
# vim:ts=2:et:sw=2:ai
# Wireless Leiden configuration generator, based on yaml files'
#
# XXX: This should be rewritten to make use of the ipaddr.py library.
#
# Sample apache configuration (mind the AcceptPathInfo!)
# Alias /config /usr/local/www/config
# <Directory /usr/local/www/config>
#    AddHandler cgi-script .py
#    Require all granted
# 
#    RewriteEngine on
#    RewriteCond %{REQUEST_FILENAME} !-f
#    RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
#    Options +FollowSymlinks +ExecCGI
# </Directory>
#
# Package dependencies list:
# FreeBSD:
#   pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
# Fedora:
#   yum install python-yaml pyproj proj-epsg python-jinja2
#
# Rick van der Zwet <info@rickvanderzwet.nl>
#

# Hack to make the script directory is also threated as a module search path.
import sys
import os
sys.path.append(os.path.dirname(__file__))

SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
SVNVERSION = filter(os.path.isfile, ('/usr/local/bin/svnversion', '/usr/bin/svnversion'))[0]

import argparse
import cgi
import cgitb
import copy
import glob
import make_network_kml
import math
import pyproj
import random
import re
import socket
import string
import subprocess
import textwrap
import time
import traceback
import urlparse

from pprint import pprint
from collections import defaultdict, OrderedDict
from sys import stderr
try:
  import yaml
except ImportError, e:
  print e
  print "[ERROR] Please install the python-yaml or devel/py-yaml package"
  exit(1)

try:
    from yaml import CLoader as Loader
    from yaml import CDumper as Dumper
except ImportError:
    from yaml import Loader, Dumper

from jinja2 import Environment, Template
def yesorno(value):
  return "YES" if bool(value) else "NO"
env = Environment()
env.filters['yesorno'] = yesorno
def render_template(datadump, template):
  result = env.from_string(template).render(datadump)
  # Make it look pretty to the naked eye, as jinja templates are not so
  # friendly when it comes to whitespace formatting
  ## Remove extra whitespace at end of line lstrip() style.
  result = re.sub(r'\n[\ ]+','\n', result)
  ## Include only a single newline between an definition and a comment
  result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
  ## Remove extra newlines after single comment
  result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
  return result

import logging
logging.basicConfig(format='# %(levelname)s: %(message)s' )
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)


if os.environ.has_key('CONFIGROOT'):
  NODE_DIR = os.environ['CONFIGROOT']
else:
  NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
__version__ = '$Id: gformat.py 14171 2018-05-22 19:53:23Z www $'

CACHE_DIR = os.path.abspath(os.path.dirname(__file__))

files = [
    'authorized_keys',
    'dnsmasq.conf',
    'dhcpd.conf',
    'rc.conf.local',
    'resolv.conf',
    'motd',
    'ntp.conf',
    'pf.hybrid.conf.local',
    'unbound.wleiden.conf',
    'wleiden.yaml',
    ]

# Global variables uses
OK = 10
DOWN = 20
UNKNOWN = 90


ileiden_proxies = OrderedDict()
normal_proxies = []
datadump_cache = {}
interface_list_cache = {}
rc_conf_local_cache = {}
nameservers_cache = []
relations_cache = None

NO_DHCP = 0
DHCP_CLIENT = 10
DHCP_SERVER = 20
def dhcp_type(item):
  if not item.has_key('dhcp'):
    return NO_DHCP
  elif not item['dhcp']:
    return NO_DHCP
  elif item['dhcp'].lower() == 'client':
    return DHCP_CLIENT
  else:
    # Validation Checks
    begin,end = map(int,item['dhcp'].split('-'))
    if begin >= end:
      raise ValueError("DHCP Start >= DHCP End")
    return DHCP_SERVER

def etrs2rd(lat, lon):
  p1 = pyproj.Proj(proj='latlon',datum='WGS84')
  p2 = pyproj.Proj(init='EPSG:28992')
  RDx, RDy = pyproj.transform(p1,p2,lon, lat)
  return (RDx, RDy)

def rd2etrs(RDx, RDy):
  p1 = pyproj.Proj(init='EPSG:28992')
  p2 = pyproj.Proj(proj='latlon',datum='WGS84')
  lon, lat = pyproj.transform(p1,p2, RDx, RDy)
  return (lat, lon)

def get_yaml(item,add_version_info=True):
  try:
    """ Get configuration yaml for 'item'"""
    if datadump_cache.has_key(item):
      return datadump_cache[item].copy()

    gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
    global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
    d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)

    # Default values
    datadump = { 
      'autogen_revision' : 'NOTFOUND',
      'autogen_gfile' : gfile,
      'service_proxy_ileiden' : False, 
    }
    f = open(gfile, 'r')
    datadump.update(yaml.load(f,Loader=Loader))
    datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
    if datadump['nodetype'] == 'Hybrid':
      # Some values are defined implicitly
      if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
        datadump['service_incoming_rdr'] = True
      # Use some boring defaults
      defaults = { 
        'service_proxy_normal' : False, 
        'service_accesspoint' : True, 
        'service_incoming_rdr' : False, 
        'service_concentrator' : False,
        'monitoring_group' : 'wleiden',
      }
      for (key,value) in defaults.iteritems():
        if not datadump.has_key(key):
          datadump[key] = value
    f.close()

    # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
    if add_version_info:
      p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
      lines = p.communicate()[0].split('\n')
      if p.returncode == 0:
        for line in lines:
          if line:
            (key, value) = line.split(': ')
            datadump["autogen_" + key.lower().replace(' ','_')] = value

    # Preformat certain needed variables for formatting and push those into special object
    datadump['autogen_iface_keys'] = get_interface_keys(datadump)

    wlan_count=0
    try:
      for key in get_interface_keys(datadump, True):
        datadump[key]['autogen_ifbase'] = key.split('_')[1]
        datadump[key]['autogen_vlan'] = False
        datadump[key]['autogen_vlan_alias'] = False

        datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
        datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
        datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key

        if datadump[key].has_key('parent'):
            if datadump[key].has_key('ip'):
                raise ValueError("Interface bridge member cannot have IP assigned")
            if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
                raise ValueError("Interface bridge member cannot have DHCP set")

        if datadump[key].has_key('ip'):
          datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]

        if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
          datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
          datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
          datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
          wlan_count += 1
        else:
          datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
          if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
            datadump[key]['autogen_if_dhcp'] = '.'.join(key.split('_')[1:3])
            datadump[key]['autogen_vlan'] = key.split('_')[2]
            datadump[key]['autogen_vlan_alias'] = '_alias' in key
            datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
          else:
            datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
            datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])

    except Exception as exc:
      exc.args = ("# Error while processing interface %s" % key,) + exc.args
      raise

    dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
      if dhcp_type(datadump[key]) == DHCP_SERVER]

    datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
    datadump['autogen_item'] = item

    datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
    datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
    datadump_cache[item] = datadump.copy()
  except Exception as exc:
    exc.args = ("# Error while processing %s" % item,) + exc.args
    raise
  return datadump


def store_yaml(datadump, header=False):
  """ Store configuration yaml for 'item'"""
  item = datadump['autogen_item'] 
  gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')

  output = generate_wleiden_yaml(datadump, header)

  f = open(gfile, 'w')
  f.write(output)
  f.close()


def network(ip):
  addr, mask = ip.split('/')
  # Not parsing of these folks please
  addr = parseaddr(addr)
  mask = int(mask)
  network = addr & ~((1 << (32 - mask)) - 1)
  return network
  


def make_relations():
  """ Process _ALL_ yaml files to get connection relations """
  global relations_cache

  if relations_cache:
    return relations_cache

  errors = []
  poel = defaultdict(list)

  for host in get_hostlist():
    datadump = get_yaml(host)
    try:
      for iface_key in get_interface_keys(datadump):
        # Bridge members has no IP assigned
        if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
          continue
        net_addr = network(datadump[iface_key]['ip'])
        poel[net_addr] += [(host,datadump[iface_key].copy())]
    except (KeyError, ValueError) as e:
      errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' +  str(e)))
      continue

  relations_cache = (poel, errors)
  return relations_cache



def valid_addr(addr):
  """ Show which address is valid in which are not """
  return str(addr).startswith('172.')

def get_hostlist():
  """ Combined hosts and proxy list"""
  return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])

def angle_between_points(lat1,lat2,long1,long2):
  """
  Return Angle in radians between two GPS coordinates
  See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
  """
  dy = lat2 - lat1
  dx = math.cos(lat1)*(long2 - long1)
  angle = math.atan2(dy,dx)
  return angle



def angle_to_cd(angle):
  """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """

  # For easy conversion get positive degree
  degrees = math.degrees(angle)
  abs_degrees = 360 + degrees if degrees < 0 else degrees

  # Numbers can be confusing calculate from the 4 main directions
  p = 22.5
  if abs_degrees < p:
    cd = "n"
  elif abs_degrees < (90 - p):
    cd = "no"
  elif abs_degrees < (90 + p):
    cd = "o"
  elif abs_degrees < (180 - p):
    cd = "zo"
  elif abs_degrees < (180 + p):
    cd = "z"
  elif abs_degrees < (270 - p):
    cd = "zw"
  elif abs_degrees < (270 + p):
    cd = "w"
  elif abs_degrees < (360 - p):
    cd = "nw"
  else:
    cd = "n"
  return cd



def cd_between_hosts(hostA, hostB, datadumps):
  # Using RDNAP coordinates
  dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
  dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
  return angle_to_cd(math.atan2(dx,dy))

  # GPS coordinates seems to fail somehow
  #latA = float(datadumps[hostA]['latitude'])
  #latB = float(datadumps[hostB]['latitude'])
  #lonA = float(datadumps[hostA]['longitude'])
  #lonB = float(datadumps[hostB]['longitude'])
  #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))


def generate_title(nodelist):
  """ Main overview page """
  items = { \
        'root'      :   ".",
        'version'   :   subprocess.Popen([SVNVERSION, "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
        }
  def fl(spaces, line):
    return (' ' * spaces) + line + '\n'

  output = """
<html>
  <head>
    <title>Wireless leiden Configurator - GFormat</title>
    <style type="text/css">
     th        {background-color: #999999}
     tr:nth-child(odd)    {background-color: #cccccc}
     tr:nth-child(even)   {background-color: #ffffff}
     th, td    {padding: 0.1em 1em}
    </style>
  </head>
  <body>
    <center>
    <form type="GET" action="%(root)s">
      <input type="hidden" name="action" value="update">
      <input type="submit" value="Update Configuration Database (SVN)">
    </form>
    <table>
      <caption><h3>Wireless Leiden Configurator  - Revision %(version)s</h3></caption>
  """ % items

  for node in nodelist:
    items['node'] = node
    output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
    for config in files:
      items['config'] = config
      output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
    output += fl(5, "</tr>")
  output += """
    </table>
    <hr />
    <em>%s</em>
    </center>
  </body>
</html>
  """ % __version__

  return output



def generate_node(node):
  """ Print overview of all files available for node """
  return "\n".join(files)

def generate_node_overview(host, datadump=False):
  """ Print overview of all files available for node """
  if not datadump:
    datadump = get_yaml(host)
  params = { 'host' : host }
  output = "<em><a href='..'>Back to overview</a></em><hr />"
  output += "<h2>Available files:</h2><ul>"
  for cf in files:
    params['cf'] = cf
    output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
  output += "</ul>"

  # Generate and connection listing
  output += "<h2>Connected To:</h2><ul>"
  (poel, errors) = make_relations()
  for network, hosts in poel.iteritems():
    if host in [x[0] for x in hosts]:
      if len(hosts) == 1:
        # Single not connected interface
        continue
      for remote,ifacedump in hosts:
        if remote == host: 
          # This side of the interface
          continue
        params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
        output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
  output += "</ul>"
  output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"

  output += "<hr /><em><a href='..'>Back to overview</a></em>"
  return output


def generate_header(datadump, ctag="#"):
  return """\
%(ctag)s
%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
%(ctag)s
""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }



def parseaddr(s):
  """ Process IPv4 CIDR notation addr to a (binary) number """
  f = s.split('.')
  return (long(f[0]) << 24L) + \
    (long(f[1]) << 16L) + \
    (long(f[2]) << 8L) + \
    long(f[3])



def showaddr(a):
  """ Display IPv4 addr in (dotted) CIDR notation """
  return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)


def is_member(ip, mask, canidate):
  """ Return True if canidate is part of ip/mask block"""
  ip_addr = parseaddr(ip)
  ip_canidate = parseaddr(canidate)
  mask = int(mask)
  ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
  ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
  return ip_addr == ip_canidate



def cidr2netmask(netmask):
  """ Given a 'netmask' return corresponding CIDR """
  return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))

def get_network(addr, mask):
  return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))


def generate_dhcpd_conf(datadump):
  """ Generate config file '/usr/local/etc/dhcpd.conf """
  # Redundency support, in cause local DNS server is not running/responding.
  datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
  output = generate_header(datadump)
  output += Template("""\
# option definitions common to all supported networks...
option domain-name "dhcp.{{ autogen_fqdn }}";

default-lease-time 600;
max-lease-time 7200;

# Use this to enble / disable dynamic dns updates globally.
#ddns-update-style none;

# If this DHCP server is the official DHCP server for the local
# network, the authoritative directive should be uncommented.
authoritative;

# Use this to send dhcp log messages to a different log file (you also
# have to hack syslog.conf to complete the redirection).
log-facility local7;

# UniFi Discovery Support
option space ubnt;
option ubnt.unifi-address code 1 = ip-address;
#
class "ubnt" {
        match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
        option vendor-class-identifier "ubnt";
        vendor-option-space ubnt;
}

#
# Interface definitions
#
\n\n""").render(datadump)


  # TODO: Use textwrap.fill instead
  def indent(text, count):
    return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))

  # Process DHCP blocks
  dhcp_out = defaultdict(list)
  for iface_key in get_interface_keys(datadump):
    ifname = datadump[iface_key]['autogen_ifbase']
    groupif = datadump[iface_key]['autogen_if_dhcp']
    if not datadump[iface_key].has_key('comment'):
      datadump[iface_key]['comment'] = None

    if not datadump[iface_key].has_key('ip'):
      continue

    dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])

    (addr, mask) = datadump[iface_key]['ip'].split('/')
    datadump[iface_key]['autogen_addr'] = addr
    datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
    datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)

    if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
      dhcp_out[groupif].append(textwrap.dedent("""\
        subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
          ### not autoritive
        }
      """ % datadump[iface_key]))
      continue

    (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
    dhcp_part = ".".join(addr.split('.')[0:3])
    datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
    datadump[iface_key]['autogen_dhcp_stop'] =  dhcp_part + "." + dhcp_stop
    datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])

    # Assume the first 10 IPs could be used for static entries
    if 'no_portal' in datadump:
      fixed = 5
      for mac in datadump['no_portal']:
        dhcp_out[groupif].append(textwrap.dedent("""\
            host fixed-%(ifname)s-%(fixed)s {
              hardware ethernet %(mac)s;
              fixed-address %(prefix)s.%(fixed)s;
            }
         """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
      fixed += 1

    if 'dhcp_fixed' in datadump[iface_key]:
      for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
        dhcp_out[groupif].append(textwrap.dedent("""\
            host fixed-%(host)s {
              hardware ethernet %(mac)s;
              fixed-address %(addr)s;
            }
         """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
        

    dhcp_out[groupif].append(textwrap.dedent("""\
      subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
        range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
        option routers %(autogen_addr)s;
        option domain-name-servers %(autogen_dns_servers)s;
        option ubnt.unifi-address 172.17.107.10;
      }
      """ % datadump[iface_key]))

  # Output the blocks in groups
  for ifname,value in sorted(dhcp_out.iteritems()):
      output += ("shared-network \"%s\" {\n" % ifname) + indent(''.join(value), 2).rstrip()  + '\n}\n\n'
  return output



def generate_dnsmasq_conf(datadump):
  """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
  output = generate_header(datadump)
  output += Template("""\
# DHCP server options
dhcp-authoritative
dhcp-fqdn
domain=dhcp.{{ autogen_fqdn }}
domain-needed
expand-hosts
log-async=100

# Low memory footprint
cache-size=10000

\n""").render(datadump)

  for iface_key in get_interface_keys(datadump):
    if not datadump[iface_key].has_key('comment'):
      datadump[iface_key]['comment'] = None
    output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]

    if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
      output += "# not autoritive\n\n"
      continue

    (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
    (ip, cidr) = datadump[iface_key]['ip'].split('/')
    datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)

    dhcp_part = ".".join(ip.split('.')[0:3])
    datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
    datadump[iface_key]['autogen_dhcp_stop'] =  dhcp_part + "." + dhcp_stop
    output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]

  return output


class AutoVivification(dict):
    """Implementation of perl's autovivification feature."""
    def __getitem__(self, item):
        try:
            return dict.__getitem__(self, item)
        except KeyError:
            value = self[item] = type(self)()
            return value

def make_interface_list(datadump):
  if interface_list_cache.has_key(datadump['autogen_item']):
    return (interface_list_cache[datadump['autogen_item']])
  # lo0 configuration:
  # - 172.32.255.1/32 is the proxy.wleiden.net deflector
  # - masterip is special as it needs to be assigned to at
  #   least one interface, so if not used assign to lo0
  addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
  vlan_list = defaultdict(list)
  bridge_list = defaultdict(list)
  flags_if = AutoVivification()
  dhclient_if = {'lo0' : False}

  # XXX: Find some way of send this output nicely
  output = ''

  masterip_used = False
  for iface_key in get_interface_keys(datadump):
    if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
      masterip_used = True
      break
  if not masterip_used:
    addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))

  if 'serviceid' in datadump:
    addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))

  for iface_key in get_interface_keys(datadump):
    ifacedump = datadump[iface_key]
    if ifacedump['autogen_bridge_alias']:
      ifname = ifacedump['autogen_ifbase']
    else:
      ifname = ifacedump['autogen_ifname']

    # If defined as vlan interface
    if ifacedump['autogen_vlan']:
      vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])

    # If defined as bridge interface
    if ifacedump['autogen_bridge_member']:
      bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])


    # Flag dhclient is possible
    if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
      dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT

    # Ethernet address
    if ifacedump.has_key('ether'):
        flags_if[ifname]['ether'] = ifacedump['ether']

    # Add interface IP to list
    if ifacedump.has_key('ip'):
      item = (ifacedump['ip'], ifacedump['comment'])
      if addrs_list.has_key(ifname):
        addrs_list[ifname].append(item)
      else:
        addrs_list[ifname] = [item]

    # Alias only needs IP assignment for now, this might change if we
    # are going to use virtual accesspoints
    if "alias" in iface_key:
      continue

    # XXX: Might want to deduct type directly from interface name
    if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
      # Default to station (client) mode
      ifacedump['autogen_wlanmode'] = "sta"
      if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
        ifacedump['autogen_wlanmode'] = "ap"

      if not ifacedump.has_key('channel'):
        if ifacedump['type'] == '11a':
          ifacedump['channel'] = 36
        else:
          ifacedump['channel'] = 1

      # Allow special hacks at the back like wds and stuff
      if not ifacedump.has_key('extra'):
        ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
      else:
        ifacedump['autogen_extra'] = ifacedump['extra']
        
      ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])

      output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
      output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
      output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
        "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
      output += "\n"

    elif ifacedump['type'] in ['ethernet', 'eth']:
      # No special config needed besides IP
      pass
    elif ifacedump['type'] in ['vlan']:
      # VLAN member has no special configuration
      pass
    else:
      assert False, "Unknown type " + ifacedump['type']

  store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
  interface_list_cache[datadump['autogen_item']] = store
  return(store)



def create_proxies_list():
  if not ileiden_proxies or not normal_proxies:
    # Placeholder for to-be-installed proxies, this will avoid updating the all
    # nodes to include this new machine, yet due to an unbound issue, this list
    # has to be kept small.

    for i in range(1,20):
      ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}

    for host in get_hostlist():
      hostdump = get_yaml(host)
      if hostdump['status'] == 'up':
        if hostdump['service_proxy_ileiden']:
          ileiden_proxies[hostdump['serviceid']] = hostdump
        if hostdump['service_proxy_normal']:
          normal_proxies.append(hostdump)



def generate_rc_conf_local(datadump):
  """ Generate configuration file '/etc/rc.conf.local' """
  item = datadump['autogen_item']
  if rc_conf_local_cache.has_key(item):
    return rc_conf_local_cache[item]

  if not datadump.has_key('ileiden'):
    datadump['autogen_ileiden_enable'] = False
  else:
    datadump['autogen_ileiden_enable'] = datadump['ileiden']

  datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])

  create_proxies_list()
  datadump['autogen_ileiden_proxies'] = ileiden_proxies
  datadump['autogen_normal_proxies'] = normal_proxies
  datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
  datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
  datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
  datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]

  output = generate_header(datadump, "#");
  output += render_template(datadump, """\
hostname='{{ autogen_fqdn }}'
location='{{ location }}'
nodetype="{{ nodetype }}"

#
# Configured listings
#
captive_portal_whitelist=""
{% if nodetype == "Proxy" %}
#
# Proxy Configuration
#
{% if gateway and service_proxy_ileiden -%}
defaultrouter="{{ gateway }}"
{% else -%}
#defaultrouter="NOTSET"
{% endif -%}
internalif="{{ internalif }}"
ileiden_enable="{{ autogen_ileiden_enable }}"
gateway_enable="{{ autogen_ileiden_enable }}"
pf_enable="yes"
pf_rules="/etc/pf.conf"
{% if autogen_ileiden_enable -%}
pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
lvrouted_enable="{{ autogen_ileiden_enable }}"
lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
{% else -%}
pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
{% endif -%}
{% if internalroute -%}
static_routes="wleiden"
route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
{% endif -%}

{% elif nodetype == "Hybrid" %}
  #
  # Hybrid Configuration
  #
  list_ileiden_proxies="
  {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
    {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
  {% endfor -%}
  "
  list_normal_proxies="
  {% for item in autogen_normal_proxies -%}
    {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
  {% endfor -%}
  "

  {% if autogen_dhcp_interfaces -%}
  captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
  {% else %}
  captive_portal_interfaces="dummy"
  {% endif %}
  externalif="{{ externalif|default('vr0', true) }}"
  masterip="{{ masterip }}"

  {% if gateway and service_proxy_ileiden %}
  defaultrouter="{{ gateway }}"
  {% else %}
  #defaultrouter="NOTSET"
  {% endif %}
  
  #
  # Defined services 
  #
  service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
  service_proxy_normal="{{ service_proxy_normal|yesorno }}"
  service_accesspoint="{{ service_accesspoint|yesorno }}"
  service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
  service_concentrator="{{ service_concentrator|yesorno }}"

  {% if service_proxy_ileiden %}
    pf_rules="/etc/pf.hybrid.conf"
    {% if service_concentrator %}
      pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
    {% else %}
      pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
    {% endif %}
    pf_flags="$pf_flags -D publicnat=80,443"
    lvrouted_flags="$lvrouted_flags -g"
  {% elif service_proxy_normal or service_incoming_rdr %}
    pf_rules="/etc/pf.hybrid.conf"
    pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
    pf_flags="$pf_flags -D publicnat=0"
    lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
    named_setfib="1"
    tinyproxy_setfib="1"
    dnsmasq_setfib="1"
    sshd_setfib="1"
  {% else %}
    named_auto_forward_only="YES"
    pf_rules="/etc/pf.node.conf"
    pf_flags=""
    lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
  {% endif %}
  {% if service_concentrator %}
    # Do mind installing certificates is NOT done automatically for security reasons
    openvpn_enable="YES"
    openvpn_configfile="/usr/local/etc/openvpn/client.conf"
  {% endif %}

  {% if service_proxy_normal %}
    tinyproxy_enable="yes"
  {% else %}
    pen_wrapper_enable="yes"
  {% endif %}

  {% if service_accesspoint %}
    pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
  {% endif %}

  {% if board == "ALIX2" or board == "net4801" %}
    #
    # ''Fat'' configuration, board has 256MB RAM
    #
    dnsmasq_enable="NO"
    named_enable="YES"
    unbound_enable="YES"
    {% if autogen_dhcp_interfaces -%}
    dhcpd_enable="YES"
    dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
    {% endif -%}
  {% elif board == "apu1d" %}
    #
    # ''Fat'' configuration, board has 1024MB RAM
    #
    dnsmasq_enable="NO"
    unbound_enable="YES"
    {% if autogen_dhcp_interfaces -%}
    dhcpd_enable="YES"
    dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
    {% endif -%}
  {% endif -%}
{% endif %}

#
# Script variables
#
attached_devices="{{ autogen_attached_devices|join(' ') }}"
neighbours="{{ autogen_neighbours|join(' ') }}"


#
# Interface definitions
#\n
""")

  (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
  for iface, vlans in sorted(vlan_list.items()):
    output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))

  # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
  for iface in sorted(vlan_list.keys()):
    if not iface in addrs_list.keys():
      output += "ifconfig_%s='up'\n" % iface

  output += "\n"

  # Bridge configuration:
  if bridge_list.keys():
    output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
    
  for iface in bridge_list.keys():
    output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))

  # Bridge member interfaces not containing a configuration should be marked active explcitly.
  for _,members in bridge_list.items():
    for iface in members:
      if not iface in addrs_list.keys():
        output += "ifconfig_%s='up'\n" % iface.replace('.','_')
  
  output += "\n"

  # Details like SSID
  if extra_ouput:
    output += extra_ouput.strip() + "\n"

  # Print IP address which needs to be assigned over here
  output += "\n"
  for iface,addrs in sorted(addrs_list.iteritems()):
    for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
      output += "# %s || %s || %s\n" % (iface, addr, comment)

    # Write DHCLIENT entry
    if iface in dhclient_if and dhclient_if[iface]:
      output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
      continue
    
    # Make sure the external address is always first as this is needed in the
    # firewall setup
    addrs = sorted(
        [x for x in addrs if not '0.0.0.0' in x[0]],
        key=lambda x: x[0].split('.')[0], 
        cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
      )

    idx_offset = 0
    # Set MAC is required
    if flags_if[iface].has_key('ether'):
      output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
      output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
      idx_offset += 1
    elif iface in bridge_list:  
      output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
      idx_offset += 1
    else:   
      output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
        
    for idx, addr in enumerate(addrs[1:]):
      output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0]) 

    output += "\n"

  rc_conf_local_cache[datadump['autogen_item']] = output
  return output



def get_all_configs():
  """ Get dict with key 'host' with all configs present """
  configs = dict()
  for host in get_hostlist():
    datadump = get_yaml(host)
    configs[host] = datadump
  return configs


def get_interface_keys(config, extra=False):
  """ Quick hack to get all interface keys, later stage convert this to a iterator """
  elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
  if extra == False:
    return filter(lambda x: not "extra" in x, elems)
  else:
    return elems


def get_used_ips(configs):
    """ Return array of all IPs used in config files"""
    ip_list = []
    for config in configs:
      ip_list.append(config['masterip'])
      if 'serviceid' in config:
        ip_list.append(config['serviceid'])
      for iface_key in get_interface_keys(config, True):
        l = config[iface_key]['ip']
        addr, mask = l.split('/')
        # Special case do not process
        if valid_addr(addr):
          ip_list.append(addr)
        else:
          logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
    return sorted(ip_list)



def get_nameservers(max_servers=None):
  if nameservers_cache:
    return nameservers_cache[0:max_servers]

  for host in get_hostlist():
    hostdump = get_yaml(host)
    if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
      nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))

  return nameservers_cache[0:max_servers]


def get_neighbours(datadump):
  (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)

  (poel, errors) = make_relations()
  table = []
  for iface,addrs in sorted(addrs_list.iteritems()):
    if iface in ['lo0']:
      continue

    for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
      if not addr.startswith('172.'):
        # Avoid listing internet connections as pool
        continue
      for neighbour in poel[network(addr)]:
        if neighbour[0] != datadump['autogen_item']:
          table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
  return table


def get_attached_devices(datadump, url=False):
  table = []
  for iface_key in get_interface_keys(datadump, True):
    ifacedump = datadump[iface_key]

    if not ifacedump.has_key('ns_ip'):
      continue

    x_ip = ifacedump['ns_ip'].split('/')[0]

    if 'mode' in ifacedump:
      x_mode = ifacedump['mode']
    else:
      x_mode = 'unknown'

    if 'bridge_type' in ifacedump:
      device_type = ifacedump['bridge_type']
    else:
      device_type = 'Unknown'

    table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
  return table


def generate_resolv_conf(datadump):
  """ Generate configuration file '/etc/resolv.conf' """
  # XXX: This should properly going to be an datastructure soon
  datadump['autogen_header'] = generate_header(datadump, "#")
  datadump['autogen_edge_nameservers'] = ''


  for masterip,realname in get_nameservers():
    datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)

  return Template("""\
{{ autogen_header }}
search wleiden.net

# Try local (cache) first
nameserver 127.0.0.1

{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
nameserver 8.8.8.8   # Google Public NameServer
nameserver 64.6.64.6 # Verisign Public NameServer
{% else -%}
# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
{{ autogen_edge_nameservers }}
{% endif -%}
""").render(datadump)



def generate_ntp_conf(datadump):
  """ Generate configuration file '/etc/ntp.conf' """
  # XXX: This should properly going to be an datastructure soon

  datadump['autogen_header'] = generate_header(datadump, "#")
  datadump['autogen_ntp_servers'] = ''
  for host in get_hostlist():
    hostdump = get_yaml(host)
    if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
      datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump

  return Template("""\
{{ autogen_header }}

{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
# Machine hooked to internet.
server 0.nl.pool.ntp.org iburst maxpoll 9
server 1.nl.pool.ntp.org iburst maxpoll 9
server 2.nl.pool.ntp.org iburst maxpoll 9
server 3.nl.pool.ntp.org iburst maxpoll 9
{% else -%}
# Local Wireless Leiden NTP Servers.
server 0.pool.ntp.wleiden.net iburst maxpoll 9
server 1.pool.ntp.wleiden.net iburst maxpoll 9
server 2.pool.ntp.wleiden.net iburst maxpoll 9
server 3.pool.ntp.wleiden.net iburst maxpoll 9

# All the configured NTP servers
{{ autogen_ntp_servers }}
{% endif %}

# If a server loses sync with all upstream servers, NTP clients
# no longer follow that server. The local clock can be configured
# to provide a time source when this happens, but it should usually
# be configured on just one server on a network. For more details see
# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
# The use of Orphan Mode may be preferable.
#
server 127.127.1.0
fudge 127.127.1.0 stratum 10
""").render(datadump)


def generate_pf_hybrid_conf_local(datadump):
  """ Generate configuration file '/etc/pf.hybrid.conf.local' """
  datadump['autogen_header'] = generate_header(datadump, "#")
  if datadump['service_incoming_rdr']:
    datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
  return Template("""\
{{ autogen_header }}

# Redirect some internal facing services outside (7)
# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
{% endfor -%}
# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
{% endfor -%}
""").render(datadump)

def generate_unbound_wleiden_conf(datadump):
  """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
  datadump['autogen_header'] = generate_header(datadump, "#")

  autogen_ips = []
  (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
  for iface,addrs in sorted(addrs_list.iteritems()):
    for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
      if addr.startswith('172'):
        autogen_ips.append((addr.split('/')[0], comment))
  datadump['autogen_ips'] = autogen_ips

  create_proxies_list()
  datadump['autogen_ileiden_proxies'] = ileiden_proxies
  return Template("""\
{{ autogen_header }}

server:
    ## Static definitions fail to start on systems with broken ue(4) interfaces
{%- for ip,comment in autogen_ips %}
    # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
{%- endfor %}
    ## Enabling wildcard matching as work-around
    interface: 0.0.0.0
    interface: ::0

forward-zone:
    name: '.'
{%- if service_proxy_ileiden %}
    forward-addr: 8.8.8.8        # Google DNS A
    forward-addr: 8.8.4.4        # Google DNS B
    forward-addr: 208.67.222.222 # OpenDNS DNS A
    forward-addr: 208.67.220.220 # OpenDNS DNS B
{% else -%}
{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
  {%- if loop.index <= 5 %}
    forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
  {%- endif %}
{%- endfor %}
{%- endif %}
""").render(datadump)

def generate_motd(datadump):
  """ Generate configuration file '/etc/motd' """
  output = Template("""\
FreeBSD run ``service motd onestart'' to make me look normal

  WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
  Loc: {{ location }}

Services:
{% if board == "ALIX2" or board == "net4801" -%}
{{" -"}} Core Node ({{ board }})
{% else -%}
{{" -"}} Hulp Node ({{ board }})
{% endif -%}
{% if service_proxy_normal -%}
{{" -"}} Normal Proxy
{% endif -%}
{% if service_proxy_ileiden -%} 
{{" -"}} iLeiden Proxy
{% endif -%}
{% if service_incoming_rdr -%} 
{{" -"}} Incoming port redirects
{% endif %}
Interlinks:\n
""").render(datadump)


  def make_table(table):
      if not table:
        return " - none\n"
      else:
        lines = ""
        col_width = [max(len(x) for x in col) for col in zip(*table)]
        for row in table:
          # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
          lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
        return lines

  (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
  table = []
  for iface,addrs in sorted(addrs_list.iteritems()):
    if iface in ['lo0']:
      continue
    for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
      table.append((iface, addr, comment))

  output += make_table(table)
  output += '\n'
  output += """\
Attached devices:
"""
  output += make_table(get_attached_devices(datadump, url=True))
  output += '\n'
  output += """\
Available neighbours:
"""
  output += make_table(get_neighbours(datadump))

  return output


def format_yaml_value(value):
  """ Get yaml value in right syntax for outputting """
  if isinstance(value,str):
    output = '"%s"' % value
  else:
    output = value
  return output



def format_wleiden_yaml(datadump):
  """ Special formatting to ensure it is editable"""
  output = "# Genesis config yaml style\n"
  output += "# vim:ts=2:et:sw=2:ai\n"
  output += "#\n"
  iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
  for key in sorted(set(datadump.keys()) - set(iface_keys)):
    if key == 'rdr_rules':
      output += '%-10s:\n' % 'rdr_rules'
      for rdr_rule in datadump[key]:
        output += '- %s\n' % rdr_rule
    else:
      output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))

  output += "\n\n"

  # Format (key, required)
  key_order = (
    ('comment', True),
    ('parent', False),
    ('ip', False),
    ('ipv6', False),
    ('ether', False),
    ('desc', True),
    ('sdesc', True),
    ('mode', True),
    ('type', True),
    ('extra_type', False), 
    ('channel', False),
    ('ssid', False),
    ('wlan_mac', False),
    ('dhcp', True),
    ('dhcp_fixed', False),
    ('compass', False),
    ('distance', False),
    ('ns_ip', False),
    ('repeater_ip', False),
    ('bullet2_ip', False),
    ('ns_mac', False),
    ('bullet2_mac', False),
    ('ns_type', False),
    ('bridge_type', False),
    ('encrypted', False),
    ('status', True),
  )

  for iface_key in sorted(iface_keys):
    try:
      remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
      if remainder:
        raise KeyError("invalid keys: %s" % remainder)

      output += "%s:\n" % iface_key
      for key,required in key_order:
        if datadump[iface_key].has_key(key):
          output += "  %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
      output += "\n\n"
    except Exception as exc:
      exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
      raise

  return output



def generate_wleiden_yaml(datadump, header=True):
  """ Generate (petty) version of wleiden.yaml"""
  output = generate_header(datadump, "#") if header else ''

  for key in datadump.keys():
    if key.startswith('autogen_'):
      del datadump[key]
    # Interface autogen cleanups
    elif type(datadump[key]) == dict:
      for key2 in datadump[key].keys():
        if key2.startswith('autogen_'):
          del datadump[key][key2]
      
  output += format_wleiden_yaml(datadump)
  return output

def generate_nanostation_config(datadump, iface, ns_type):
  #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
  datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
  
  datadump.update(datadump['iface_%s' % iface])

  return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump

def generate_yaml(datadump):
  return generate_config(datadump['nodename'], "wleiden.yaml", datadump)



def generate_config(node, config, datadump=None):
  """ Print configuration file 'config' of 'node' """
  output = ""
  try:
    # Load config file
    if datadump == None:
      datadump = get_yaml(node)

    if config == 'wleiden.yaml':
      output += generate_wleiden_yaml(datadump)
    elif config == 'authorized_keys':
      f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
      output += f.read()
      node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
      # Fetch local keys if existing
      if os.path.exists(node_keys):
         output += open(node_keys, 'r').read()
      f.close()
    elif config == 'dnsmasq.conf':
      output += generate_dnsmasq_conf(datadump)
    elif config == 'dhcpd.conf':
      output += generate_dhcpd_conf(datadump)
    elif config == 'rc.conf.local':
      output += generate_rc_conf_local(datadump)
    elif config == 'resolv.conf':
      output += generate_resolv_conf(datadump)
    elif config == 'ntp.conf':
      output += generate_ntp_conf(datadump)
    elif config == 'motd':
      output += generate_motd(datadump)
    elif config == 'pf.hybrid.conf.local':
      output += generate_pf_hybrid_conf_local(datadump)
    elif config == 'unbound.wleiden.conf':
      output += generate_unbound_wleiden_conf(datadump)
    elif config.startswith('vr'):
      interface, ns_type = config.strip('.yaml').split('-')
      output += generate_nanostation_config(datadump, interface, ns_type)
    else:
      assert False, "Config not found!"
  except IOError, e:
    output += "[ERROR] Config file not found"
  return output



def generate_static(output_dir, logging=True):
  items = {'output_dir' : output_dir}
  for node in get_hostlist():
    items['node'] = node
    items['wdir'] = "%(output_dir)s/%(node)s" % items
    if not os.path.isdir(items['wdir']):
      os.makedirs(items['wdir'])
    datadump = get_yaml(node)
    f = open("%(wdir)s/index.html" % items, "w")
    f.write(generate_node_overview(items['node'], datadump))
    f.close()
    for config in files:
      items['config'] = config
      if logging: logger.info("## Generating %(node)s %(config)s" % items)
      f = open("%(wdir)s/%(config)s" % items, "w")
      f.write(generate_config(node, config, datadump))
      f.close()



def process_cgi_request(environ=os.environ):
  """ When calling from CGI """
  response_headers = []
  content_type = 'text/plain'

  # Update repository if requested
  form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
  if form and form.has_key("action") and "update" in form["action"]:
    refresh_rate = 5
    output = "[INFO] Updating subverion, please wait...\n"
    old_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
    output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
    output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
    new_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
    if old_version != new_version:
        try:
            generate_static(CACHE_DIR, False)
        except:
            output += traceback.format_exc()
            refresh_rate = 120
            pass
    output += "[INFO] All done, redirecting in %s seconds" % refresh_rate
    response_headers += [
      ('Refresh', '%s; url=.' % refresh_rate),
    ]
  else:
    # Bootstap cache directory if none exists
    if not glob.glob(os.path.join(CACHE_DIR,'*','index.html')):
        generate_static(CACHE_DIR, False)

    # URL must be of format <prefix>/config/<path>
    base_uri = environ['REQUEST_URI']
    uri = filter(None, base_uri.split('/config/')[1].strip('/').split('/'))

    output = "Template Holder"
    if base_uri.endswith('/create/network.kml'):
        content_type='application/vnd.google-earth.kml+xml'
        output = make_network_kml.make_graph()
    elif base_uri.endswith('/api/get/nodeplanner.json'):
        content_type='application/json'
        output = make_network_kml.make_nodeplanner_json()
    elif not uri:
      if is_text_request(environ):
        output = '\n'.join(get_hostlist())
      else:
        content_type = 'text/html'
        output = generate_title(get_hostlist())
    elif len(uri) == 1:
      if is_text_request(environ):
        output = generate_node(uri[0])
      else:
        content_type = 'text/html'
        output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
    elif len(uri) == 2:
      output = generate_config(uri[0], uri[1])
    else:
      assert False, "Invalid option"

  # Return response
  response_headers += [
    ('Content-type', content_type),
    ('Content-Length', str(len(output))),
  ]
  return(response_headers, str(output))


def make_dns(output_dir = 'dns', external = False):
  items = dict()

  # hostname is key, IP is value
  wleiden_zone = defaultdict(list)
  wleiden_cname = dict()

  pool = dict()
  for node in get_hostlist():
    datadump = get_yaml(node)

    fqdn = datadump['nodename']
 
    if datadump.has_key('rdr_host'):
      remote_target = datadump['rdr_host']
    elif datadump.has_key('remote_access') and datadump['remote_access']:
      remote_target = datadump['remote_access'].split(':')[0]
    else:
      remote_target = None

    if remote_target:
      try:
        parseaddr(remote_target)
        wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
      except (IndexError, ValueError):
        wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'


    wleiden_zone[fqdn].append((datadump['masterip'], True))

    # Hacking to get proper DHCP IPs and hostnames
    for iface_key in get_interface_keys(datadump):
      iface_name = iface_key.replace('_','-')
      if 'ip' in datadump[iface_key]:
        (ip, cidr) = datadump[iface_key]['ip'].split('/')
      try:
        (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
        datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
        dhcp_part = ".".join(ip.split('.')[0:3])
        if ip != datadump['masterip']:
          wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
        for i in range(int(dhcp_start), int(dhcp_stop) + 1):
          wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
      except (AttributeError, ValueError, KeyError):
        # First push it into a pool, to indentify the counter-part later on
        addr = parseaddr(ip)
        cidr = int(cidr)
        addr = addr & ~((1 << (32 - cidr)) - 1)
        if pool.has_key(addr):
          pool[addr] += [(iface_name, fqdn, ip)]
        else:
          pool[addr] = [(iface_name, fqdn, ip)]
        continue



  #  WL uses an /29 to configure an interface. IP's are ordered like this: 
  #     MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4) 
    
  sn = lambda x: re.sub(r'(?i)^cnode','',x)

  # Automatic naming convention of interlinks namely 2 + remote.lower()
  for (key,value) in pool.iteritems():
    # Make sure they are sorted from low-ip to high-ip
    value = sorted(value, key=lambda x: parseaddr(x[2]))

    if len(value) == 1:
      (iface_name, fqdn, ip) = value[0]
      wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))

      # Device DNS names
      if 'cnode' in fqdn.lower():
        wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
        wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))

    elif len(value) == 2:
      (a_iface_name, a_fqdn, a_ip) = value[0]
      (b_iface_name, b_fqdn, b_ip) = value[1]
      wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
      wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))

      # Device DNS names
      if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
        wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
        wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
        wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
        wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
        wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
        wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)

    else:
      pool_members = [k[1] for k in value]
      for item in value:
        (iface_name, fqdn, ip) = item
        wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))

  # Include static DNS entries
  # XXX: Should they override the autogenerated results?
  # XXX: Convert input to yaml more useable.
  # Format:
  ##; this is a comment
  ## roomburgh=Roomburgh1
  ## apkerk1.Vosko=172.17.176.8 ;this as well
  dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))

  # Hack to allow special entries, for development
  wleiden_raw = {}

  for line in dns_list:
    reverse = False
    k, items = line.items()[0]
    if type(items) == dict:
      if items.has_key('reverse'):
        reverse = items['reverse']
        items = items['a']
      else:
        items = items['cname']
    items = [items] if type(items) != list else items
    for item in items:
      if item.startswith('IN '):
        wleiden_raw[k] = item
      elif valid_addr(item):
        wleiden_zone[k].append((item, reverse))
      else:
        wleiden_cname[k] = item

  # Hack to get dynamic pool listing
  def chunks(l, n):
      return [l[i:i+n] for i in range(0, len(l), n)]

  ntp_servers = [x[0] for x in get_nameservers()]
  for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
    for ntp_server in chunk:
      wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))

  details = dict()
  # 24 updates a day allowed
  details['serial'] = time.strftime('%Y%m%d%H')

  if external:
    dns_masters = ['ns1.vanderzwet.net', 'ns1.transip.net', 'ns2.transip.eu']
  else:
    dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]

  details['master'] = dns_masters[0]
  details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])

  dns_header = '''
$TTL 3h
%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
        ; Serial, Refresh, Retry, Expire, Neg. cache TTL

%(ns_servers)s
  \n'''


  if not os.path.isdir(output_dir):
    os.makedirs(output_dir)
  details['zone'] = 'wleiden.net'
  f = open(os.path.join(output_dir,"db." + details['zone']), "w")
  f.write(dns_header % details)

  for host,items in wleiden_zone.iteritems():
    for ip,reverse in items:
      if ip not in ['0.0.0.0']:
        f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
  for source,dest in wleiden_cname.iteritems():
    dest = dest if dest.endswith('.') else dest + ".wleiden.net."
    f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
  for source, dest in wleiden_raw.iteritems():
    f.write("%s.wleiden.net. %s\n" % (source, dest))
  f.close()

  # Create whole bunch of specific sub arpa zones. To keep it compliant
  for s in range(16,32):
    details['zone'] = '%i.172.in-addr.arpa' % s
    f = open(os.path.join(output_dir,"db." + details['zone']), "w")
    f.write(dns_header % details)

    #XXX: Not effient, fix to proper data structure and do checks at other
    # stages
    for host,items in wleiden_zone.iteritems():
      for ip,reverse in items:
        if not reverse:
          continue
        if valid_addr(ip):
          if valid_addr(ip):
            if int(ip.split('.')[1]) == s:
              rev_ip = '.'.join(reversed(ip.split('.')))
              f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
    f.close()


def usage():
  print """Usage: %(prog)s <argument>
Argument:
\tcleanup                      =  Cleanup all YAML files to specified format
\tstandalone [port]            =  Run configurator webserver [8000]
\tdns [outputdir]              =  Generate BIND compliant zone files in dns [./dns]
\tnagios-export [--heavy-load] =  Generate basic nagios configuration file.
\tfull-export                  =  Generate yaml export script for heatmap.
\tstatic [outputdir]           =  Generate all config files and store on disk
\t                                with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
\ttest <node> [<file>]         =  Receive output for certain node [all files].
\ttest-cgi <node> <file>       =  Receive output of CGI script [all files].
\tlist <status> <items>        =  List systems which have certain status
\tcreate network.kml           =  Create Network KML file for use in Google Earth

Arguments:
\t<node>    = NodeName (example: HybridRick)
\t<file>    = %(files)s
\t<status>  = all|up|down|planned
\t<items>   = systems|nodes|proxies

NOTE FOR DEVELOPERS; you can test your changes like this:
  BEFORE any changes in this code:
  $ ./gformat.py static /tmp/pre
  AFTER the changes:
  $ ./gformat.py static /tmp/post
  VIEW differences and VERIFY all are OK:
  $ diff -urI 'Generated' -r /tmp/pre /tmp/post
""" % { 'prog' : sys.argv[0], 'files'  : '|'.join(files) }
  exit(0)


def is_text_request(environ=os.environ):
  """ Find out whether we are calling from the CLI or any text based CLI utility """
  if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
    return True

  if 'HTTP_USER_AGENT' in environ:
    return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
  else:
    return False


def switchFormat(setting):
  if setting:
    return "YES"
  else:
    return "NO"

def rlinput(prompt, prefill=''):
   import readline
   readline.set_startup_hook(lambda: readline.insert_text(prefill))
   try:
      return raw_input(prompt)
   finally:
      readline.set_startup_hook()

def fix_conflict(left, right, default='i'):
  while True:
    print "## %-30s | %-30s" % (left, right)
    c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
    if not c:
      c = default

    if c in ['l','1']:
      return left
    elif c in ['r','2']:
      return right
    elif c in ['e', '3']:
      return rlinput("Edit: ", "%30s | %30s" % (left, right))
    elif c in ['i', '4']:
      return None
    else:
      print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c



def print_cgi_response(response_headers, output):
  for header in response_headers:
     print "%s: %s" % header
  print
  print output



def main():
  """Hard working sub"""
  # Allow easy hacking using the CLI
  if not os.environ.has_key('REQUEST_URI'):
    if len(sys.argv) < 2:
      usage()

    if sys.argv[1] == "standalone":
      import SocketServer
      import CGIHTTPServer
      # Hop to the right working directory.
      os.chdir(os.path.dirname(__file__))
      try:
        PORT = int(sys.argv[2])
      except (IndexError,ValueError):
        PORT = 8000

      class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
        """ Serve this CGI from the root of the webserver """
        def is_cgi(self):
          if "favicon" in self.path:
            return False

          self.cgi_info = (os.path.basename(__file__), self.path)
          self.path = ''
          return True
      handler = MyCGIHTTPRequestHandler
      SocketServer.TCPServer.allow_reuse_address = True
      httpd = SocketServer.TCPServer(("", PORT), handler)
      httpd.server_name = 'localhost'
      httpd.server_port = PORT

      logger.info("serving at port %s", PORT)
      try:
        httpd.serve_forever()
      except KeyboardInterrupt:
        httpd.shutdown()
        logger.info("All done goodbye")
    elif sys.argv[1] == "test":
      # Basic argument validation
      try:
        node = sys.argv[2]
      except IndexError:
        print "Invalid argument"
        exit(1)
      except IOError as e:
        print e
        exit(1)

      datadump = get_yaml(node)
        

      # Get files to generate
      gen_files = sys.argv[3:] if len(sys.argv) > 3 else files

      # Actual config generation
      for config in gen_files:
         logger.info("## Generating %s %s", node, config)
         print generate_config(node, config, datadump)
    elif sys.argv[1] == "test-cgi":
      os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
      os.environ['SCRIPT_NAME'] = __file__
      response_headers, output = process_cgi_request()
      print_cgi_response(response_headers, output)
    elif sys.argv[1] == "static":
      generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
    elif sys.argv[1] == "wind-export":
      items = dict()
      for node in get_hostlist():
        datadump = get_yaml(node)
        sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
        VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
        sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
        VALUES (
          (SELECT id FROM users WHERE username = 'rvdzwet'),
          (SELECT id FROM nodes WHERE name = '%(nodename)s'),
          'Y');""" % datadump
        #for config in files:
        #  items['config'] = config
        #  print "## Generating %(node)s %(config)s" % items
        #  f = open("%(wdir)s/%(config)s" % items, "w")
        #  f.write(generate_config(node, config, datadump))
        #  f.close()
      for node in get_hostlist():
        datadump = get_yaml(node)
        for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
          ifacedump = datadump[iface_key]
          if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
            ifacedump['nodename'] = datadump['nodename']
            if not ifacedump.has_key('channel') or not ifacedump['channel']:
              ifacedump['channel'] = 0
            sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
              VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
              '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
    elif sys.argv[1] == "smokeping-export":
        for host in get_hostlist():
            datadump = get_yaml(host)
            if datadump.get('service_proxy_normal', False) or datadump.get('service_proxy_ileiden', False):
                print textwrap.dedent("""\
                    ++ wleiden-gw-%(nodename)s
                    menu = %(nodename)s.gw
                    title = Wireless Leiden gateway %(nodename)s.gw.wleiden.net.
                    host = %(nodename)s.gw.wleiden.net.
                    """ % datadump)
    elif sys.argv[1] == "nagios-export":
      try:
        heavy_load = (sys.argv[2] == "--heavy-load")
      except IndexError:
        heavy_load = False

      hostgroup_details = {
         'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
         'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
         'walphen' : 'Stichting Wireless Alphen',
         'westeinder' : 'Westeinder Plassen',
      }

      # Convert IP to Host
      ip2host = {'root' : 'root'}
      for host in get_hostlist():
        datadump = get_yaml(host)
        ip2host[datadump['masterip']] = datadump['autogen_fqdn']
        for iface in get_interface_keys(datadump):
          if datadump[iface].has_key('autogen_gateway'):
            ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']

      # Find dependency tree based on output of lvrouted.mytree of nearest node
      parents = defaultdict(list)
      stack = ['root']
      prev_depth = 0
      if os.path.isfile('lvrouted.mytree'):
        for line in open('lvrouted.mytree').readlines():
          depth = line.count('\t')
          ip = line.strip().split()[0]
        
          if prev_depth < depth:
            try:
              parents[ip2host[ip]].append(ip2host[stack[-1]])
            except KeyError as e:
              print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
            stack.append(ip)
          elif prev_depth > depth:
            stack = stack[:(depth - prev_depth)]
          elif prev_depth == depth:
            try:
              parents[ip2host[ip]].append(ip2host[stack[-1]])
            except KeyError as e:
              print >> stderr, "# Unable to find %s in configuration files" % e.args[0]


        prev_depth = depth
      # Observe that some nodes has themself as parent or multiple parents
      # for now take only the first parent, other behaviour is yet to be explained



      params = {
        'check_interval' : 5 if heavy_load else 120,
        'retry_interval' : 1  if heavy_load else 10,
        'max_check_attempts' : 10 if heavy_load else 6,
        'notification_interval': 120 if heavy_load else 240,
      }  

      print '''\
define host {
        name                  wleiden-node           ; Default Node Template
        use                   generic-host           ; Use the standard template as initial starting point
        check_period          24x7                   ; By default, FreeBSD hosts are checked round the clock
        check_interval        %(check_interval)s     ; Actively check the host every 5 minutes
        retry_interval        %(retry_interval)s     ; Schedule host check retries at 1 minute intervals
        notification_interval %(notification_interval)s
        max_check_attempts    %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
        check_command         check-host-alive       ; Default command to check FreeBSD hosts
        register              0                      ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
}

define service {
        name                  wleiden-service        ; Default Service Template
        use                   generic-service        ; Use the standard template as initial starting point
        check_period          24x7                   ; By default, FreeBSD hosts are checked round the clock
        check_interval        %(check_interval)s     ; Actively check the host every 5 minutes
        retry_interval        %(retry_interval)s     ; Schedule host check retries at 1 minute intervals
        notification_interval %(notification_interval)s
        max_check_attempts    %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
        register              0                      ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
}

# Please make sure to install:
#   make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
# 
# Recompile net-mgmt/nagios-plugins to support check_snmp
#   make -C /usr/ports/net-mgmt/nagios-plugins
#
# Install net/bind-tools to allow v2/check_dns_wl to work:
#   pkg install bind-tools
#
define command{
        command_name          check_snmp_disk
        command_line          $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
}

define command{
        command_name          check_netsnmp_load
        command_line          $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
}

define command{
        command_name          check_netsnmp_proc
        command_line          $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
}

define command{
       command_name           check_by_ssh
       command_line           $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
}

define command{
        command_name          check_dns_wl
        command_line          $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
}

define command{
        command_name            check_snmp_uptime
        command_line            $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
}


# TDB: dhcp leases
# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec

# TDB: internet status
# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file

# TDB: Advanced local passive checks
# /usr/local/libexec/nagios/check_by_ssh
''' % params

      print '''\
# Service Group, not displayed by default        
define hostgroup {
       hostgroup_name         srv_hybrid
       alias                  All Hybrid Nodes
       register 0
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   SSH
        check_command         check_ssh
}

define service {
        use                   wleiden-service,service-pnp
        hostgroup_name        srv_hybrid
        service_description   HTTP
        check_command         check_http
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   DNS
        check_command         check_dns_wl!"www.wirelessleiden.nl"
}
'''

      if heavy_load:
        print '''\
define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   UPTIME
        check_command         check_snmp_uptime
}

#define service {
#        use                   wleiden-service
#        hostgroup_name        srv_hybrid
#        service_description   NTP
#        check_command         check_ntp_peer
#}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   LOAD
        check_command         check_netsnmp_load
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   PROC
        check_command         check_netsnmp_proc
}

define service {
        use                   wleiden-service
        hostgroup_name        srv_hybrid
        service_description   DISK
        check_command         check_snmp_disk
}
'''
      for node in get_hostlist():
        datadump = get_yaml(node)
        if not datadump['status'] == 'up':
          continue
        if not hostgroup_details.has_key(datadump['monitoring_group']):
           hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
        print '''\
define host {
        use                   wleiden-node,host-pnp
        contact_groups        admins  
        host_name             %(autogen_fqdn)s
        address               %(masterip)s
        hostgroups            srv_hybrid,%(monitoring_group)s\
''' % datadump
        if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
          print '''\
        parents               %(parents)s\
''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
        print '''\
}
'''


      for name,alias in hostgroup_details.iteritems(): 
        print '''\
define hostgroup {
       hostgroup_name         %s
       alias                  %s
} ''' % (name, alias)      


    elif sys.argv[1] == "full-export":
      hosts = {}
      for node in get_hostlist():
        datadump = get_yaml(node)
        hosts[datadump['nodename']] = datadump
      print yaml.dump(hosts)

    elif sys.argv[1] == "dns":
      make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
    elif sys.argv[1] == "cleanup":
      # First generate all datadumps
      datadumps = dict()
      ssid_to_node = dict()
      for host in get_hostlist():
        logger.info("# Processing: %s", host)
        # Set some boring default values
        datadump = { 'board' : 'UNKNOWN' }
        datadump.update(get_yaml(host))
        datadumps[datadump['nodename']] = datadump

      (poel, errors) = make_relations()
      print "\n".join(["# WARNING: %s" % x for x in errors])

      for host,datadump in datadumps.iteritems():
        try:
          # Convert all yes and no to boolean values
          def fix_boolean(dump):
            for key in dump.keys():
              if type(dump[key]) == dict:
                dump[key] = fix_boolean(dump[key])
              elif str(dump[key]).lower() in ["yes", "true"]:
                dump[key] = True
              elif str(dump[key]).lower() in ["no", "false"]:
                # Compass richting no (Noord Oost) is valid input
                if key != "compass": dump[key] = False
            return dump
          datadump = fix_boolean(datadump)

          if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
            if not 'latitude' in datadump and not 'longitude' in datadump:
              datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
          elif 'latitude' in  datadump and 'longitude' in datadump:
            if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
              datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
          # TODO: Compare outcome of both coordinate systems and validate against each-other

          if datadump['nodename'].startswith('Proxy'):
            datadump['nodename'] = datadump['nodename'].lower()

          for iface_key in get_interface_keys(datadump):
            try:
              # All our normal wireless cards are normal APs now
              if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
                datadump[iface_key]['mode'] = 'ap'
              # Wireless Leiden SSID have an consistent lowercase/uppercase
              if datadump[iface_key].has_key('ssid'):
                ssid = datadump[iface_key]['ssid']
                prefix = 'ap-WirelessLeiden-'
                if ssid.lower().startswith(prefix.lower()):
                  datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:] 
              if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
                datadump[iface_key]['mode'] = 'autogen-FIXME'
              if not datadump[iface_key].has_key('comment'):
                datadump[iface_key]['comment'] = 'autogen-FIXME'

              if datadump[iface_key].has_key('ns_mac'):
                datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()

              if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
                datadump[iface_key] = datadump[iface_key]['desc']

              # We are not using 802.11b anymore. OFDM is preferred over DSSS
              # due to better collision avoidance.
              if datadump[iface_key]['type'] == '11b':
                datadump[iface_key]['type'] = '11g'

              # Setting 802.11g channels to de-facto standards, to avoid
              # un-detected sharing with other overlapping channels
              # 
              # Technically we could also use channel 13 in NL, but this is not
              # recommended as foreign devices might not be able to select this
              # channel. Secondly using 1,5,9,13 instead is going to clash with
              # the de-facto usage of 1,6,11.
              # 
              # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
              channels_at_2400Mhz = (1,6,11)
              if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
                datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
                if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
                   datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)

              # Mandatory interface keys
              if not datadump[iface_key].has_key('status'):
                datadump[iface_key]['status'] = 'planned'

              x = datadump[iface_key]['comment']
              datadump[iface_key]['comment'] = x[0].upper() + x[1:]

              # Fixing bridge_type if none is found
              if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
                if not 'bridge_type' in datadump[iface_key]:
                   datadump[iface_key]['bridge_type'] = 'NanoStation M5'

              # Making sure description works
              if datadump[iface_key].has_key('desc'):
                if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
                  del datadump[iface_key]['desc']
                else:
                  print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
                  response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
                  if response:
                    datadump[iface_key]['comment'] = response
                    del datadump[iface_key]['desc']

              # Check DHCP configuration
              dhcp_type(datadump[iface_key])

              # Set the compass value based on the angle between the poels
              if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
                my_pool = poel[network(datadump[iface_key]['ip'])]
                remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
                if remote_hosts:
                  compass_target = remote_hosts[0]
                  datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
              # TODO: Compass wanted and actual direction might differ

              # Monitoring Group default
              if not 'monitoring_group' in datadump:
                datadump['monitoring_group'] = 'wleiden'

            except Exception as exc:
              exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
              raise
          store_yaml(datadump)
        except Exception as exc:
          exc.args = ("# Error while processing %s" % host,) + exc.args
          raise
    elif sys.argv[1] == "list":
      use_fqdn = False
      if len(sys.argv) < 4:
        usage()
      if not sys.argv[2] in ["up", "down", "planned", "all"]:
        usage()
      if not sys.argv[3] in ["nodes","proxies","systems"]:
        usage()

      if len(sys.argv) > 4:
        if sys.argv[4] == "fqdn":
          use_fqdn = True
        else:
          usage()

      for system in get_hostlist():
       datadump = get_yaml(system)
       if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
         continue

       output = datadump['autogen_fqdn'] if use_fqdn else system
       if sys.argv[2] == "all":
         print output
       elif datadump['status'] == sys.argv[2]:
         print output
    elif sys.argv[1] == "create":
      if sys.argv[2] == "network.kml":
        print make_network_kml.make_graph()
      elif sys.argv[2] == "host-ips.txt":
        for system in get_hostlist():
          datadump = get_yaml(system)
          ips = [datadump['masterip']]
          for ifkey in get_interface_keys(datadump):
            ips.append(datadump[ifkey]['ip'].split('/')[0])
          print system, ' '.join(ips)
      elif sys.argv[2] == "host-pos.txt":
        for system in get_hostlist():
          datadump = get_yaml(system)
          print system, datadump['rdnap_x'], datadump['rdnap_y']
      elif sys.argv[2] == 'ssh_config':
        print  '''
Host *.wleiden.net
  User root

Host 172.16.*.*
  User root
'''
        for system in get_hostlist():
          datadump = get_yaml(system)
          print '''\
Host %s
  User root

Host %s
  User root

Host %s
  User root

Host %s
  User root
''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
      else:
        usage()
    else:
      usage()
  else:
    # Do not enable debugging for config requests as it highly clutters the output
    if not is_text_request():
      cgitb.enable()

    try:
      response_headers, output = process_cgi_request()
    except:
      print ''
      print ''
      raise

    print_cgi_response(response_headers, output)

if __name__ == "__main__":
  main()