source: genesis/tools/gformat.py@ 13906

Last change on this file since 13906 was 13906, checked in by rick, 8 years ago

Support for UniFi discovery
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 78.6 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# FreeBSD:
28# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
29# Fedora:
30# yum install python-yaml pyproj proj-epsg python-jinja2
31#
32# Rick van der Zwet <info@rickvanderzwet.nl>
33#
34
35# Hack to make the script directory is also threated as a module search path.
36import sys
37import os
38sys.path.append(os.path.dirname(__file__))
39
40SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
41
42import argparse
43import cgi
44import cgitb
45import copy
46import glob
47import make_network_kml
48import math
49import pyproj
50import random
51import re
52import socket
53import string
54import subprocess
55import textwrap
56import time
57import urlparse
58
59from pprint import pprint
60from collections import defaultdict, OrderedDict
61from sys import stderr
62try:
63 import yaml
64except ImportError, e:
65 print e
66 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
67 exit(1)
68
69try:
70 from yaml import CLoader as Loader
71 from yaml import CDumper as Dumper
72except ImportError:
73 from yaml import Loader, Dumper
74
75from jinja2 import Environment, Template
76def yesorno(value):
77 return "YES" if bool(value) else "NO"
78env = Environment()
79env.filters['yesorno'] = yesorno
80def render_template(datadump, template):
81 result = env.from_string(template).render(datadump)
82 # Make it look pretty to the naked eye, as jinja templates are not so
83 # friendly when it comes to whitespace formatting
84 ## Remove extra whitespace at end of line lstrip() style.
85 result = re.sub(r'\n[\ ]+','\n', result)
86 ## Include only a single newline between an definition and a comment
87 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
88 ## Remove extra newlines after single comment
89 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
90 return result
91
92import logging
93logging.basicConfig(format='# %(levelname)s: %(message)s' )
94logger = logging.getLogger()
95logger.setLevel(logging.DEBUG)
96
97
98if os.environ.has_key('CONFIGROOT'):
99 NODE_DIR = os.environ['CONFIGROOT']
100else:
101 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
102__version__ = '$Id: gformat.py 13906 2017-06-04 19:09:02Z rick $'
103
104files = [
105 'authorized_keys',
106 'dnsmasq.conf',
107 'dhcpd.conf',
108 'rc.conf.local',
109 'resolv.conf',
110 'motd',
111 'ntp.conf',
112 'pf.hybrid.conf.local',
113 'unbound.wleiden.conf',
114 'wleiden.yaml',
115 ]
116
117# Global variables uses
118OK = 10
119DOWN = 20
120UNKNOWN = 90
121
122
123ileiden_proxies = OrderedDict()
124normal_proxies = []
125datadump_cache = {}
126interface_list_cache = {}
127rc_conf_local_cache = {}
128nameservers_cache = []
129relations_cache = None
130def clear_cache():
131 ''' Poor mans cache implementation '''
132 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
133 datadump_cache = {}
134 interface_list_cache = {}
135 rc_conf_local_cache = {}
136 ileiden_proxies = OrderedDict()
137 normal_proxies = []
138 nameservers_cache = []
139 relations_cache = None
140
141NO_DHCP = 0
142DHCP_CLIENT = 10
143DHCP_SERVER = 20
144def dhcp_type(item):
145 if not item.has_key('dhcp'):
146 return NO_DHCP
147 elif not item['dhcp']:
148 return NO_DHCP
149 elif item['dhcp'].lower() == 'client':
150 return DHCP_CLIENT
151 else:
152 # Validation Checks
153 begin,end = map(int,item['dhcp'].split('-'))
154 if begin >= end:
155 raise ValueError("DHCP Start >= DHCP End")
156 return DHCP_SERVER
157
158def etrs2rd(lat, lon):
159 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
160 p2 = pyproj.Proj(init='EPSG:28992')
161 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
162 return (RDx, RDy)
163
164def rd2etrs(RDx, RDy):
165 p1 = pyproj.Proj(init='EPSG:28992')
166 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
167 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
168 return (lat, lon)
169
170def get_yaml(item,add_version_info=True):
171 try:
172 """ Get configuration yaml for 'item'"""
173 if datadump_cache.has_key(item):
174 return datadump_cache[item].copy()
175
176 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
177 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
178 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
179
180 # Default values
181 datadump = {
182 'autogen_revision' : 'NOTFOUND',
183 'autogen_gfile' : gfile,
184 'service_proxy_ileiden' : False,
185 }
186 f = open(gfile, 'r')
187 datadump.update(yaml.load(f,Loader=Loader))
188 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
189 if datadump['nodetype'] == 'Hybrid':
190 # Some values are defined implicitly
191 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
192 datadump['service_incoming_rdr'] = True
193 # Use some boring defaults
194 defaults = {
195 'service_proxy_normal' : False,
196 'service_accesspoint' : True,
197 'service_incoming_rdr' : False,
198 'service_concentrator' : False,
199 'monitoring_group' : 'wleiden',
200 }
201 for (key,value) in defaults.iteritems():
202 if not datadump.has_key(key):
203 datadump[key] = value
204 f.close()
205
206 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
207 if add_version_info:
208 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
209 lines = p.communicate()[0].split('\n')
210 if p.returncode == 0:
211 for line in lines:
212 if line:
213 (key, value) = line.split(': ')
214 datadump["autogen_" + key.lower().replace(' ','_')] = value
215
216 # Preformat certain needed variables for formatting and push those into special object
217 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
218
219 wlan_count=0
220 try:
221 for key in get_interface_keys(datadump, True):
222 datadump[key]['autogen_ifbase'] = key.split('_')[1]
223 datadump[key]['autogen_vlan'] = False
224
225 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
226 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
227
228 if datadump[key].has_key('parent'):
229 if datadump[key].has_key('ip'):
230 raise ValueError("Interface bridge member cannot have IP assigned")
231 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
232 raise ValueError("Interface bridge member cannot have DHCP set")
233
234 if datadump[key].has_key('ip'):
235 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
236
237 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
238 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
239 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
240 wlan_count += 1
241 else:
242 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
243 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
244 datadump[key]['autogen_vlan'] = key.split('_')[2]
245 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
246 else:
247 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
248
249 except Exception:
250 print "# Error while processing interface %s" % key
251 raise
252
253 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
254 if dhcp_type(datadump[key]) == DHCP_SERVER]
255
256 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
257 datadump['autogen_item'] = item
258
259 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
260 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
261 datadump_cache[item] = datadump.copy()
262 except Exception:
263 print "# Error while processing %s" % item
264 raise
265 return datadump
266
267
268def store_yaml(datadump, header=False):
269 """ Store configuration yaml for 'item'"""
270 item = datadump['autogen_item']
271 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
272
273 output = generate_wleiden_yaml(datadump, header)
274
275 f = open(gfile, 'w')
276 f.write(output)
277 f.close()
278
279
280def network(ip):
281 addr, mask = ip.split('/')
282 # Not parsing of these folks please
283 addr = parseaddr(addr)
284 mask = int(mask)
285 network = addr & ~((1 << (32 - mask)) - 1)
286 return network
287
288
289
290def make_relations():
291 """ Process _ALL_ yaml files to get connection relations """
292 global relations_cache
293
294 if relations_cache:
295 return relations_cache
296
297 errors = []
298 poel = defaultdict(list)
299
300 for host in get_hostlist():
301 datadump = get_yaml(host)
302 try:
303 for iface_key in get_interface_keys(datadump):
304 # Bridge members has no IP assigned
305 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
306 continue
307 net_addr = network(datadump[iface_key]['ip'])
308 poel[net_addr] += [(host,datadump[iface_key].copy())]
309 except (KeyError, ValueError) as e:
310 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
311 continue
312
313 relations_cache = (poel, errors)
314 return relations_cache
315
316
317
318def valid_addr(addr):
319 """ Show which address is valid in which are not """
320 return str(addr).startswith('172.')
321
322def get_hostlist():
323 """ Combined hosts and proxy list"""
324 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
325
326def angle_between_points(lat1,lat2,long1,long2):
327 """
328 Return Angle in radians between two GPS coordinates
329 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
330 """
331 dy = lat2 - lat1
332 dx = math.cos(lat1)*(long2 - long1)
333 angle = math.atan2(dy,dx)
334 return angle
335
336
337
338def angle_to_cd(angle):
339 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
340
341 # For easy conversion get positive degree
342 degrees = math.degrees(angle)
343 abs_degrees = 360 + degrees if degrees < 0 else degrees
344
345 # Numbers can be confusing calculate from the 4 main directions
346 p = 22.5
347 if abs_degrees < p:
348 cd = "n"
349 elif abs_degrees < (90 - p):
350 cd = "no"
351 elif abs_degrees < (90 + p):
352 cd = "o"
353 elif abs_degrees < (180 - p):
354 cd = "zo"
355 elif abs_degrees < (180 + p):
356 cd = "z"
357 elif abs_degrees < (270 - p):
358 cd = "zw"
359 elif abs_degrees < (270 + p):
360 cd = "w"
361 elif abs_degrees < (360 - p):
362 cd = "nw"
363 else:
364 cd = "n"
365 return cd
366
367
368
369def cd_between_hosts(hostA, hostB, datadumps):
370 # Using RDNAP coordinates
371 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
372 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
373 return angle_to_cd(math.atan2(dx,dy))
374
375 # GPS coordinates seems to fail somehow
376 #latA = float(datadumps[hostA]['latitude'])
377 #latB = float(datadumps[hostB]['latitude'])
378 #lonA = float(datadumps[hostA]['longitude'])
379 #lonB = float(datadumps[hostB]['longitude'])
380 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
381
382
383def generate_title(nodelist):
384 """ Main overview page """
385 items = {'root' : "." }
386 def fl(spaces, line):
387 return (' ' * spaces) + line + '\n'
388
389 output = """
390<html>
391 <head>
392 <title>Wireless leiden Configurator - GFormat</title>
393 <style type="text/css">
394 th {background-color: #999999}
395 tr:nth-child(odd) {background-color: #cccccc}
396 tr:nth-child(even) {background-color: #ffffff}
397 th, td {padding: 0.1em 1em}
398 </style>
399 </head>
400 <body>
401 <center>
402 <form type="GET" action="%(root)s">
403 <input type="hidden" name="action" value="update">
404 <input type="submit" value="Update Configuration Database (SVN)">
405 </form>
406 <table>
407 <caption><h3>Wireless Leiden Configurator</h3></caption>
408 """ % items
409
410 for node in nodelist:
411 items['node'] = node
412 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
413 for config in files:
414 items['config'] = config
415 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
416 output += fl(5, "</tr>")
417 output += """
418 </table>
419 <hr />
420 <em>%s</em>
421 </center>
422 </body>
423</html>
424 """ % __version__
425
426 return output
427
428
429
430def generate_node(node):
431 """ Print overview of all files available for node """
432 return "\n".join(files)
433
434def generate_node_overview(host):
435 """ Print overview of all files available for node """
436 datadump = get_yaml(host)
437 params = { 'host' : host }
438 output = "<em><a href='..'>Back to overview</a></em><hr />"
439 output += "<h2>Available files:</h2><ul>"
440 for cf in files:
441 params['cf'] = cf
442 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
443 output += "</ul>"
444
445 # Generate and connection listing
446 output += "<h2>Connected To:</h2><ul>"
447 (poel, errors) = make_relations()
448 for network, hosts in poel.iteritems():
449 if host in [x[0] for x in hosts]:
450 if len(hosts) == 1:
451 # Single not connected interface
452 continue
453 for remote,ifacedump in hosts:
454 if remote == host:
455 # This side of the interface
456 continue
457 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
458 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
459 output += "</ul>"
460 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
461
462 output += "<hr /><em><a href='..'>Back to overview</a></em>"
463 return output
464
465
466def generate_header(datadump, ctag="#"):
467 return """\
468%(ctag)s
469%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
470%(ctag)s
471""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
472
473
474
475def parseaddr(s):
476 """ Process IPv4 CIDR notation addr to a (binary) number """
477 f = s.split('.')
478 return (long(f[0]) << 24L) + \
479 (long(f[1]) << 16L) + \
480 (long(f[2]) << 8L) + \
481 long(f[3])
482
483
484
485def showaddr(a):
486 """ Display IPv4 addr in (dotted) CIDR notation """
487 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
488
489
490def is_member(ip, mask, canidate):
491 """ Return True if canidate is part of ip/mask block"""
492 ip_addr = parseaddr(ip)
493 ip_canidate = parseaddr(canidate)
494 mask = int(mask)
495 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
496 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
497 return ip_addr == ip_canidate
498
499
500
501def cidr2netmask(netmask):
502 """ Given a 'netmask' return corresponding CIDR """
503 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
504
505def get_network(addr, mask):
506 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
507
508
509def generate_dhcpd_conf(datadump):
510 """ Generate config file '/usr/local/etc/dhcpd.conf """
511 # Redundency support, in cause local DNS server is not running/responding.
512 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
513 output = generate_header(datadump)
514 output += Template("""\
515# option definitions common to all supported networks...
516option domain-name "dhcp.{{ autogen_fqdn }}";
517
518default-lease-time 600;
519max-lease-time 7200;
520
521# Use this to enble / disable dynamic dns updates globally.
522#ddns-update-style none;
523
524# If this DHCP server is the official DHCP server for the local
525# network, the authoritative directive should be uncommented.
526authoritative;
527
528# Use this to send dhcp log messages to a different log file (you also
529# have to hack syslog.conf to complete the redirection).
530log-facility local7;
531
532# UniFi Discovery Support
533option space ubnt;
534option ubnt.unifi-address code 1 = ip-address;
535#
536class "ubnt" {
537 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
538 option vendor-class-identifier "ubnt";
539 vendor-option-space ubnt;
540}
541
542#
543# Interface definitions
544#
545\n\n""").render(datadump)
546
547
548 # TODO: Use textwrap.fill instead
549 def indent(text, count):
550 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
551
552 # Process DHCP blocks
553 dhcp_out = defaultdict(list)
554 for iface_key in get_interface_keys(datadump):
555 ifname = datadump[iface_key]['autogen_ifbase']
556 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
557 if not datadump[iface_key].has_key('comment'):
558 datadump[iface_key]['comment'] = None
559
560 if not datadump[iface_key].has_key('ip'):
561 continue
562
563 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
564
565 (addr, mask) = datadump[iface_key]['ip'].split('/')
566 datadump[iface_key]['autogen_addr'] = addr
567 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
568 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
569
570 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
571 dhcp_out[groupif].append(textwrap.dedent("""\
572 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
573 ### not autoritive
574 }
575 """ % datadump[iface_key]))
576 continue
577
578 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
579 dhcp_part = ".".join(addr.split('.')[0:3])
580 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
581 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
582 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
583
584 # Assume the first 10 IPs could be used for static entries
585 if 'no_portal' in datadump:
586 fixed = 5
587 for mac in datadump['no_portal']:
588 dhcp_out[groupif].append(textwrap.dedent("""\
589 host fixed-%(ifname)s-%(fixed)s {
590 hardware ethernet %(mac)s;
591 fixed-address %(prefix)s.%(fixed)s;
592 }
593 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
594 fixed += 1
595
596 if 'dhcp_fixed' in datadump[iface_key]:
597 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
598 dhcp_out[groupif].append(textwrap.dedent("""\
599 host fixed-%(host)s {
600 hardware ethernet %(mac)s;
601 fixed-address %(addr)s;
602 }
603 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
604
605
606 dhcp_out[groupif].append(textwrap.dedent("""\
607 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
608 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
609 option routers %(autogen_addr)s;
610 option domain-name-servers %(autogen_dns_servers)s;
611 option ubnt.unifi-address 172.17.107.10;
612 }
613 """ % datadump[iface_key]))
614
615 # Output the blocks in groups
616 for ifname,value in sorted(dhcp_out.iteritems()):
617 if len(value) > 2:
618 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
619 else:
620 output += ''.join(value) + "\n\n"
621 return output
622
623
624
625def generate_dnsmasq_conf(datadump):
626 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
627 output = generate_header(datadump)
628 output += Template("""\
629# DHCP server options
630dhcp-authoritative
631dhcp-fqdn
632domain=dhcp.{{ autogen_fqdn }}
633domain-needed
634expand-hosts
635log-async=100
636
637# Low memory footprint
638cache-size=10000
639
640\n""").render(datadump)
641
642 for iface_key in get_interface_keys(datadump):
643 if not datadump[iface_key].has_key('comment'):
644 datadump[iface_key]['comment'] = None
645 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
646
647 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
648 output += "# not autoritive\n\n"
649 continue
650
651 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
652 (ip, cidr) = datadump[iface_key]['ip'].split('/')
653 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
654
655 dhcp_part = ".".join(ip.split('.')[0:3])
656 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
657 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
658 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
659
660 return output
661
662
663class AutoVivification(dict):
664 """Implementation of perl's autovivification feature."""
665 def __getitem__(self, item):
666 try:
667 return dict.__getitem__(self, item)
668 except KeyError:
669 value = self[item] = type(self)()
670 return value
671
672def make_interface_list(datadump):
673 if interface_list_cache.has_key(datadump['autogen_item']):
674 return (interface_list_cache[datadump['autogen_item']])
675 # lo0 configuration:
676 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
677 # - masterip is special as it needs to be assigned to at
678 # least one interface, so if not used assign to lo0
679 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
680 vlan_list = defaultdict(list)
681 bridge_list = defaultdict(list)
682 flags_if = AutoVivification()
683 dhclient_if = {'lo0' : False}
684
685 # XXX: Find some way of send this output nicely
686 output = ''
687
688 masterip_used = False
689 for iface_key in get_interface_keys(datadump):
690 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
691 masterip_used = True
692 break
693 if not masterip_used:
694 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
695
696 if 'serviceid' in datadump:
697 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
698
699 for iface_key in get_interface_keys(datadump):
700 ifacedump = datadump[iface_key]
701 ifname = ifacedump['autogen_ifname']
702
703 # If defined as vlan interface
704 if ifacedump['autogen_vlan']:
705 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
706
707 # If defined as bridge interface
708 if ifacedump['autogen_bridge_member']:
709 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
710
711 # Flag dhclient is possible
712 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
713 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
714
715 # Ethernet address
716 if ifacedump.has_key('ether'):
717 flags_if[ifname]['ether'] = ifacedump['ether']
718
719 # Add interface IP to list
720 if ifacedump.has_key('ip'):
721 item = (ifacedump['ip'], ifacedump['comment'])
722 if addrs_list.has_key(ifname):
723 addrs_list[ifname].append(item)
724 else:
725 addrs_list[ifname] = [item]
726
727 # Alias only needs IP assignment for now, this might change if we
728 # are going to use virtual accesspoints
729 if "alias" in iface_key:
730 continue
731
732 # XXX: Might want to deduct type directly from interface name
733 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
734 # Default to station (client) mode
735 ifacedump['autogen_wlanmode'] = "sta"
736 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
737 ifacedump['autogen_wlanmode'] = "ap"
738
739 if not ifacedump.has_key('channel'):
740 if ifacedump['type'] == '11a':
741 ifacedump['channel'] = 36
742 else:
743 ifacedump['channel'] = 1
744
745 # Allow special hacks at the back like wds and stuff
746 if not ifacedump.has_key('extra'):
747 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
748 else:
749 ifacedump['autogen_extra'] = ifacedump['extra']
750
751 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
752
753 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
754 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
755 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
756 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
757 output += "\n"
758
759 elif ifacedump['type'] in ['ethernet', 'eth']:
760 # No special config needed besides IP
761 pass
762 elif ifacedump['type'] in ['vlan']:
763 # VLAN member has no special configuration
764 pass
765 else:
766 assert False, "Unknown type " + ifacedump['type']
767
768 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
769 interface_list_cache[datadump['autogen_item']] = store
770 return(store)
771
772
773
774def create_proxies_list():
775 if not ileiden_proxies or not normal_proxies:
776 # Placeholder for to-be-installed proxies, this will avoid updating the all
777 # nodes to include this new machine, yet due to an unbound issue, this list
778 # has to be kept small.
779
780 for i in range(1,20):
781 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
782
783 for host in get_hostlist():
784 hostdump = get_yaml(host)
785 if hostdump['status'] == 'up':
786 if hostdump['service_proxy_ileiden']:
787 ileiden_proxies[hostdump['serviceid']] = hostdump
788 if hostdump['service_proxy_normal']:
789 normal_proxies.append(hostdump)
790
791
792
793def generate_rc_conf_local(datadump):
794 """ Generate configuration file '/etc/rc.conf.local' """
795 item = datadump['autogen_item']
796 if rc_conf_local_cache.has_key(item):
797 return rc_conf_local_cache[item]
798
799 if not datadump.has_key('ileiden'):
800 datadump['autogen_ileiden_enable'] = False
801 else:
802 datadump['autogen_ileiden_enable'] = datadump['ileiden']
803
804 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
805
806 create_proxies_list()
807 datadump['autogen_ileiden_proxies'] = ileiden_proxies
808 datadump['autogen_normal_proxies'] = normal_proxies
809 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
810 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
811 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
812 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
813
814 output = generate_header(datadump, "#");
815 output += render_template(datadump, """\
816hostname='{{ autogen_fqdn }}'
817location='{{ location }}'
818nodetype="{{ nodetype }}"
819
820#
821# Configured listings
822#
823captive_portal_whitelist=""
824{% if nodetype == "Proxy" %}
825#
826# Proxy Configuration
827#
828{% if gateway and service_proxy_ileiden -%}
829defaultrouter="{{ gateway }}"
830{% else -%}
831#defaultrouter="NOTSET"
832{% endif -%}
833internalif="{{ internalif }}"
834ileiden_enable="{{ autogen_ileiden_enable }}"
835gateway_enable="{{ autogen_ileiden_enable }}"
836pf_enable="yes"
837pf_rules="/etc/pf.conf"
838{% if autogen_ileiden_enable -%}
839pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
840lvrouted_enable="{{ autogen_ileiden_enable }}"
841lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
842{% else -%}
843pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
844{% endif -%}
845{% if internalroute -%}
846static_routes="wleiden"
847route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
848{% endif -%}
849
850{% elif nodetype == "Hybrid" %}
851 #
852 # Hybrid Configuration
853 #
854 list_ileiden_proxies="
855 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
856 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
857 {% endfor -%}
858 "
859 list_normal_proxies="
860 {% for item in autogen_normal_proxies -%}
861 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
862 {% endfor -%}
863 "
864
865 {% if autogen_dhcp_interfaces -%}
866 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
867 {% else %}
868 captive_portal_interfaces="dummy"
869 {% endif %}
870 externalif="{{ externalif|default('vr0', true) }}"
871 masterip="{{ masterip }}"
872
873 {% if gateway and service_proxy_ileiden %}
874 defaultrouter="{{ gateway }}"
875 {% else %}
876 #defaultrouter="NOTSET"
877 {% endif %}
878
879 #
880 # Defined services
881 #
882 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
883 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
884 service_accesspoint="{{ service_accesspoint|yesorno }}"
885 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
886 service_concentrator="{{ service_concentrator|yesorno }}"
887
888 {% if service_proxy_ileiden %}
889 pf_rules="/etc/pf.hybrid.conf"
890 {% if service_concentrator %}
891 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
892 {% else %}
893 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
894 {% endif %}
895 pf_flags="$pf_flags -D publicnat=80,443"
896 lvrouted_flags="$lvrouted_flags -g"
897 {% elif service_proxy_normal or service_incoming_rdr %}
898 pf_rules="/etc/pf.hybrid.conf"
899 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
900 pf_flags="$pf_flags -D publicnat=0"
901 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
902 named_setfib="1"
903 tinyproxy_setfib="1"
904 dnsmasq_setfib="1"
905 sshd_setfib="1"
906 {% else %}
907 named_auto_forward_only="YES"
908 pf_rules="/etc/pf.node.conf"
909 pf_flags=""
910 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
911 {% endif %}
912 {% if service_concentrator %}
913 # Do mind installing certificates is NOT done automatically for security reasons
914 openvpn_enable="YES"
915 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
916 {% endif %}
917
918 {% if service_proxy_normal %}
919 tinyproxy_enable="yes"
920 {% else %}
921 pen_wrapper_enable="yes"
922 {% endif %}
923
924 {% if service_accesspoint %}
925 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
926 {% endif %}
927
928 {% if board == "ALIX2" %}
929 #
930 # ''Fat'' configuration, board has 256MB RAM
931 #
932 dnsmasq_enable="NO"
933 named_enable="YES"
934 unbound_enable="YES"
935 {% if autogen_dhcp_interfaces -%}
936 dhcpd_enable="YES"
937 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
938 {% endif -%}
939 {% elif board == "apu1d" %}
940 #
941 # ''Fat'' configuration, board has 1024MB RAM
942 #
943 dnsmasq_enable="NO"
944 local_unbound_enable="YES"
945 {% if autogen_dhcp_interfaces -%}
946 dhcpd_enable="YES"
947 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
948 {% endif -%}
949 {% endif -%}
950{% endif %}
951
952#
953# Script variables
954#
955attached_devices="{{ autogen_attached_devices|join(' ') }}"
956neighbours="{{ autogen_neighbours|join(' ') }}"
957
958
959#
960# Interface definitions
961#\n
962""")
963
964 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
965 for iface, vlans in sorted(vlan_list.items()):
966 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
967
968 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
969 for iface in sorted(vlan_list.keys()):
970 if not iface in addrs_list.keys():
971 output += "ifconfig_%s='up'\n" % iface
972
973 output += "\n"
974
975 # Bridge configuration:
976 if bridge_list.keys():
977 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
978
979 for iface in bridge_list.keys():
980 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
981
982 # Bridge member interfaces not containing a configuration should be marked active explcitly.
983 for _,members in bridge_list.items():
984 for iface in members:
985 if not iface in addrs_list.keys():
986 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
987
988 output += "\n"
989
990 # Details like SSID
991 if extra_ouput:
992 output += extra_ouput.strip() + "\n"
993
994 # Print IP address which needs to be assigned over here
995 output += "\n"
996 for iface,addrs in sorted(addrs_list.iteritems()):
997 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
998 output += "# %s || %s || %s\n" % (iface, addr, comment)
999
1000 # Write DHCLIENT entry
1001 if iface in dhclient_if and dhclient_if[iface]:
1002 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1003 continue
1004
1005 # Make sure the external address is always first as this is needed in the
1006 # firewall setup
1007 addrs = sorted(
1008 [x for x in addrs if not '0.0.0.0' in x[0]],
1009 key=lambda x: x[0].split('.')[0],
1010 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1011 )
1012
1013 idx_offset = 0
1014 # Set MAC is required
1015 if flags_if[iface].has_key('ether'):
1016 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1017 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1018 idx_offset += 1
1019 else:
1020 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1021
1022 for idx, addr in enumerate(addrs[1:]):
1023 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1024
1025 output += "\n"
1026
1027 rc_conf_local_cache[datadump['autogen_item']] = output
1028 return output
1029
1030
1031
1032def get_all_configs():
1033 """ Get dict with key 'host' with all configs present """
1034 configs = dict()
1035 for host in get_hostlist():
1036 datadump = get_yaml(host)
1037 configs[host] = datadump
1038 return configs
1039
1040
1041def get_interface_keys(config, extra=False):
1042 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1043 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1044 if extra == False:
1045 return filter(lambda x: not "extra" in x, elems)
1046 else:
1047 return elems
1048
1049
1050def get_used_ips(configs):
1051 """ Return array of all IPs used in config files"""
1052 ip_list = []
1053 for config in configs:
1054 ip_list.append(config['masterip'])
1055 if 'serviceid' in config:
1056 ip_list.append(config['serviceid'])
1057 for iface_key in get_interface_keys(config, True):
1058 l = config[iface_key]['ip']
1059 addr, mask = l.split('/')
1060 # Special case do not process
1061 if valid_addr(addr):
1062 ip_list.append(addr)
1063 else:
1064 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1065 return sorted(ip_list)
1066
1067
1068
1069def get_nameservers(max_servers=None):
1070 if nameservers_cache:
1071 return nameservers_cache[0:max_servers]
1072
1073 for host in get_hostlist():
1074 hostdump = get_yaml(host)
1075 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1076 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
1077
1078 return nameservers_cache[0:max_servers]
1079
1080
1081def get_neighbours(datadump):
1082 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1083
1084 (poel, errors) = make_relations()
1085 table = []
1086 for iface,addrs in sorted(addrs_list.iteritems()):
1087 if iface in ['lo0']:
1088 continue
1089
1090 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1091 if not addr.startswith('172.'):
1092 # Avoid listing internet connections as pool
1093 continue
1094 for neighbour in poel[network(addr)]:
1095 if neighbour[0] != datadump['autogen_item']:
1096 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1097 return table
1098
1099
1100def get_attached_devices(datadump, url=False):
1101 table = []
1102 for iface_key in get_interface_keys(datadump, True):
1103 ifacedump = datadump[iface_key]
1104
1105 if not ifacedump.has_key('ns_ip'):
1106 continue
1107
1108 x_ip = ifacedump['ns_ip'].split('/')[0]
1109
1110 if 'mode' in ifacedump:
1111 x_mode = ifacedump['mode']
1112 else:
1113 x_mode = 'unknown'
1114
1115 if 'bridge_type' in ifacedump:
1116 device_type = ifacedump['bridge_type']
1117 else:
1118 device_type = 'Unknown'
1119
1120 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1121 return table
1122
1123
1124def generate_resolv_conf(datadump):
1125 """ Generate configuration file '/etc/resolv.conf' """
1126 # XXX: This should properly going to be an datastructure soon
1127 datadump['autogen_header'] = generate_header(datadump, "#")
1128 datadump['autogen_edge_nameservers'] = ''
1129
1130
1131 for masterip,realname in get_nameservers():
1132 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1133
1134 return Template("""\
1135{{ autogen_header }}
1136search wleiden.net
1137
1138# Try local (cache) first
1139nameserver 127.0.0.1
1140
1141{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1142nameserver 8.8.8.8 # Google Public NameServer
1143nameserver 64.6.64.6 # Verisign Public NameServer
1144{% else -%}
1145# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1146{{ autogen_edge_nameservers }}
1147{% endif -%}
1148""").render(datadump)
1149
1150
1151
1152def generate_ntp_conf(datadump):
1153 """ Generate configuration file '/etc/ntp.conf' """
1154 # XXX: This should properly going to be an datastructure soon
1155
1156 datadump['autogen_header'] = generate_header(datadump, "#")
1157 datadump['autogen_ntp_servers'] = ''
1158 for host in get_hostlist():
1159 hostdump = get_yaml(host)
1160 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1161 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1162
1163 return Template("""\
1164{{ autogen_header }}
1165
1166{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1167# Machine hooked to internet.
1168server 0.nl.pool.ntp.org iburst maxpoll 9
1169server 1.nl.pool.ntp.org iburst maxpoll 9
1170server 2.nl.pool.ntp.org iburst maxpoll 9
1171server 3.nl.pool.ntp.org iburst maxpoll 9
1172{% else -%}
1173# Local Wireless Leiden NTP Servers.
1174server 0.pool.ntp.wleiden.net iburst maxpoll 9
1175server 1.pool.ntp.wleiden.net iburst maxpoll 9
1176server 2.pool.ntp.wleiden.net iburst maxpoll 9
1177server 3.pool.ntp.wleiden.net iburst maxpoll 9
1178
1179# All the configured NTP servers
1180{{ autogen_ntp_servers }}
1181{% endif %}
1182
1183# If a server loses sync with all upstream servers, NTP clients
1184# no longer follow that server. The local clock can be configured
1185# to provide a time source when this happens, but it should usually
1186# be configured on just one server on a network. For more details see
1187# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1188# The use of Orphan Mode may be preferable.
1189#
1190server 127.127.1.0
1191fudge 127.127.1.0 stratum 10
1192""").render(datadump)
1193
1194
1195def generate_pf_hybrid_conf_local(datadump):
1196 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1197 datadump['autogen_header'] = generate_header(datadump, "#")
1198 if datadump['service_incoming_rdr']:
1199 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1200 return Template("""\
1201{{ autogen_header }}
1202
1203# Redirect some internal facing services outside (7)
1204# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1205{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1206rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1207{% endfor -%}
1208# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1209{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1210rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1211{% endfor -%}
1212""").render(datadump)
1213
1214def generate_unbound_wleiden_conf(datadump):
1215 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1216 datadump['autogen_header'] = generate_header(datadump, "#")
1217
1218 autogen_ips = []
1219 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1220 for iface,addrs in sorted(addrs_list.iteritems()):
1221 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1222 if addr.startswith('172'):
1223 autogen_ips.append((addr.split('/')[0], comment))
1224 datadump['autogen_ips'] = autogen_ips
1225
1226 create_proxies_list()
1227 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1228 return Template("""\
1229{{ autogen_header }}
1230
1231server:
1232{%- for ip,comment in autogen_ips %}
1233 interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1234{%- endfor %}
1235
1236forward-zone:
1237 name: '.'
1238{%- if service_proxy_ileiden %}
1239 forward-addr: 8.8.8.8 # Google DNS A
1240 forward-addr: 8.8.4.4 # Google DNS B
1241 forward-addr: 208.67.222.222 # OpenDNS DNS A
1242 forward-addr: 208.67.220.220 # OpenDNS DNS B
1243{% else -%}
1244{% for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1245 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1246{%- endfor %}
1247{% endif -%}
1248""").render(datadump)
1249
1250def generate_motd(datadump):
1251 """ Generate configuration file '/etc/motd' """
1252 output = Template("""\
1253FreeBSD run ``service motd onestart'' to make me look normal
1254
1255 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1256 Loc: {{ location }}
1257
1258Services:
1259{% if board == "ALIX2" -%}
1260{{" -"}} Core Node ({{ board }})
1261{% else -%}
1262{{" -"}} Hulp Node ({{ board }})
1263{% endif -%}
1264{% if service_proxy_normal -%}
1265{{" -"}} Normal Proxy
1266{% endif -%}
1267{% if service_proxy_ileiden -%}
1268{{" -"}} iLeiden Proxy
1269{% endif -%}
1270{% if service_incoming_rdr -%}
1271{{" -"}} Incoming port redirects
1272{% endif %}
1273Interlinks:\n
1274""").render(datadump)
1275
1276
1277 def make_table(table):
1278 if not table:
1279 return " - none\n"
1280 else:
1281 lines = ""
1282 col_width = [max(len(x) for x in col) for col in zip(*table)]
1283 for row in table:
1284 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1285 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1286 return lines
1287
1288 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1289 table = []
1290 for iface,addrs in sorted(addrs_list.iteritems()):
1291 if iface in ['lo0']:
1292 continue
1293 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1294 table.append((iface, addr, comment))
1295
1296 output += make_table(table)
1297 output += '\n'
1298 output += """\
1299Attached devices:
1300"""
1301 output += make_table(get_attached_devices(datadump, url=True))
1302 output += '\n'
1303 output += """\
1304Available neighbours:
1305"""
1306 output += make_table(get_neighbours(datadump))
1307
1308 return output
1309
1310
1311def format_yaml_value(value):
1312 """ Get yaml value in right syntax for outputting """
1313 if isinstance(value,str):
1314 output = '"%s"' % value
1315 else:
1316 output = value
1317 return output
1318
1319
1320
1321def format_wleiden_yaml(datadump):
1322 """ Special formatting to ensure it is editable"""
1323 output = "# Genesis config yaml style\n"
1324 output += "# vim:ts=2:et:sw=2:ai\n"
1325 output += "#\n"
1326 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1327 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1328 if key == 'rdr_rules':
1329 output += '%-10s:\n' % 'rdr_rules'
1330 for rdr_rule in datadump[key]:
1331 output += '- %s\n' % rdr_rule
1332 else:
1333 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1334
1335 output += "\n\n"
1336
1337 # Format (key, required)
1338 key_order = (
1339 ('comment', True),
1340 ('parent', False),
1341 ('ip', False),
1342 ('ether', False),
1343 ('desc', True),
1344 ('sdesc', True),
1345 ('mode', True),
1346 ('type', True),
1347 ('extra_type', False),
1348 ('channel', False),
1349 ('ssid', False),
1350 ('wlan_mac', False),
1351 ('dhcp', True),
1352 ('dhcp_fixed', False),
1353 ('compass', False),
1354 ('distance', False),
1355 ('ns_ip', False),
1356 ('repeater_ip', False),
1357 ('bullet2_ip', False),
1358 ('ns_mac', False),
1359 ('bullet2_mac', False),
1360 ('ns_type', False),
1361 ('bridge_type', False),
1362 ('status', True),
1363 )
1364
1365 for iface_key in sorted(iface_keys):
1366 try:
1367 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1368 if remainder:
1369 raise KeyError("invalid keys: %s" % remainder)
1370
1371 output += "%s:\n" % iface_key
1372 for key,required in key_order:
1373 if datadump[iface_key].has_key(key):
1374 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1375 output += "\n\n"
1376 except Exception:
1377 print "# Error while processing interface %s" % iface_key
1378 raise
1379
1380 return output
1381
1382
1383
1384def generate_wleiden_yaml(datadump, header=True):
1385 """ Generate (petty) version of wleiden.yaml"""
1386 output = generate_header(datadump, "#") if header else ''
1387
1388 for key in datadump.keys():
1389 if key.startswith('autogen_'):
1390 del datadump[key]
1391 # Interface autogen cleanups
1392 elif type(datadump[key]) == dict:
1393 for key2 in datadump[key].keys():
1394 if key2.startswith('autogen_'):
1395 del datadump[key][key2]
1396
1397 output += format_wleiden_yaml(datadump)
1398 return output
1399
1400def generate_nanostation_config(datadump, iface, ns_type):
1401 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1402 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1403
1404 datadump.update(datadump['iface_%s' % iface])
1405
1406 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1407
1408def generate_yaml(datadump):
1409 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1410
1411
1412
1413def generate_config(node, config, datadump=None):
1414 """ Print configuration file 'config' of 'node' """
1415 output = ""
1416 try:
1417 # Load config file
1418 if datadump == None:
1419 datadump = get_yaml(node)
1420
1421 if config == 'wleiden.yaml':
1422 output += generate_wleiden_yaml(datadump)
1423 elif config == 'authorized_keys':
1424 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1425 output += f.read()
1426 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1427 # Fetch local keys if existing
1428 if os.path.exists(node_keys):
1429 output += open(node_keys, 'r').read()
1430 f.close()
1431 elif config == 'dnsmasq.conf':
1432 output += generate_dnsmasq_conf(datadump)
1433 elif config == 'dhcpd.conf':
1434 output += generate_dhcpd_conf(datadump)
1435 elif config == 'rc.conf.local':
1436 output += generate_rc_conf_local(datadump)
1437 elif config == 'resolv.conf':
1438 output += generate_resolv_conf(datadump)
1439 elif config == 'ntp.conf':
1440 output += generate_ntp_conf(datadump)
1441 elif config == 'motd':
1442 output += generate_motd(datadump)
1443 elif config == 'pf.hybrid.conf.local':
1444 output += generate_pf_hybrid_conf_local(datadump)
1445 elif config == 'unbound.wleiden.conf':
1446 output += generate_unbound_wleiden_conf(datadump)
1447 elif config.startswith('vr'):
1448 interface, ns_type = config.strip('.yaml').split('-')
1449 output += generate_nanostation_config(datadump, interface, ns_type)
1450 else:
1451 assert False, "Config not found!"
1452 except IOError, e:
1453 output += "[ERROR] Config file not found"
1454 return output
1455
1456
1457
1458def process_cgi_request(environ=os.environ):
1459 """ When calling from CGI """
1460 response_headers = []
1461 content_type = 'text/plain'
1462
1463 # Update repository if requested
1464 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1465 if form and form.has_key("action") and "update" in form["action"]:
1466 output = "[INFO] Updating subverion, please wait...\n"
1467 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1468 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1469 output += "[INFO] All done, redirecting in 5 seconds"
1470 response_headers += [
1471 ('Refresh', '5; url=.'),
1472 ]
1473 reload_cache()
1474 else:
1475 base_uri = environ['PATH_INFO']
1476 uri = base_uri.strip('/').split('/')
1477
1478 output = "Template Holder"
1479 if base_uri.endswith('/create/network.kml'):
1480 content_type='application/vnd.google-earth.kml+xml'
1481 output = make_network_kml.make_graph()
1482 elif base_uri.endswith('/api/get/nodeplanner.json'):
1483 content_type='application/json'
1484 output = make_network_kml.make_nodeplanner_json()
1485 elif not uri[0]:
1486 if is_text_request(environ):
1487 output = '\n'.join(get_hostlist())
1488 else:
1489 content_type = 'text/html'
1490 output = generate_title(get_hostlist())
1491 elif len(uri) == 1:
1492 if is_text_request(environ):
1493 output = generate_node(uri[0])
1494 else:
1495 content_type = 'text/html'
1496 output = generate_node_overview(uri[0])
1497 elif len(uri) == 2:
1498 output = generate_config(uri[0], uri[1])
1499 else:
1500 assert False, "Invalid option"
1501
1502 # Return response
1503 response_headers += [
1504 ('Content-type', content_type),
1505 ('Content-Length', str(len(output))),
1506 ]
1507 return(response_headers, str(output))
1508
1509
1510def make_dns(output_dir = 'dns', external = False):
1511 items = dict()
1512
1513 # hostname is key, IP is value
1514 wleiden_zone = defaultdict(list)
1515 wleiden_cname = dict()
1516
1517 pool = dict()
1518 for node in get_hostlist():
1519 datadump = get_yaml(node)
1520
1521 fqdn = datadump['nodename']
1522
1523 if datadump.has_key('rdr_host'):
1524 remote_target = datadump['rdr_host']
1525 elif datadump.has_key('remote_access') and datadump['remote_access']:
1526 remote_target = datadump['remote_access'].split(':')[0]
1527 else:
1528 remote_target = None
1529
1530 if remote_target:
1531 try:
1532 parseaddr(remote_target)
1533 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1534 except (IndexError, ValueError):
1535 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1536
1537
1538 wleiden_zone[fqdn].append((datadump['masterip'], True))
1539
1540 # Hacking to get proper DHCP IPs and hostnames
1541 for iface_key in get_interface_keys(datadump):
1542 iface_name = iface_key.replace('_','-')
1543 if 'ip' in datadump[iface_key]:
1544 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1545 try:
1546 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1547 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1548 dhcp_part = ".".join(ip.split('.')[0:3])
1549 if ip != datadump['masterip']:
1550 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1551 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1552 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1553 except (AttributeError, ValueError, KeyError):
1554 # First push it into a pool, to indentify the counter-part later on
1555 addr = parseaddr(ip)
1556 cidr = int(cidr)
1557 addr = addr & ~((1 << (32 - cidr)) - 1)
1558 if pool.has_key(addr):
1559 pool[addr] += [(iface_name, fqdn, ip)]
1560 else:
1561 pool[addr] = [(iface_name, fqdn, ip)]
1562 continue
1563
1564
1565
1566 # WL uses an /29 to configure an interface. IP's are ordered like this:
1567 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1568
1569 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1570
1571 # Automatic naming convention of interlinks namely 2 + remote.lower()
1572 for (key,value) in pool.iteritems():
1573 # Make sure they are sorted from low-ip to high-ip
1574 value = sorted(value, key=lambda x: parseaddr(x[2]))
1575
1576 if len(value) == 1:
1577 (iface_name, fqdn, ip) = value[0]
1578 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1579
1580 # Device DNS names
1581 if 'cnode' in fqdn.lower():
1582 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1583 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1584
1585 elif len(value) == 2:
1586 (a_iface_name, a_fqdn, a_ip) = value[0]
1587 (b_iface_name, b_fqdn, b_ip) = value[1]
1588 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1589 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1590
1591 # Device DNS names
1592 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1593 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1594 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1595 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1596 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1597 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1598 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1599
1600 else:
1601 pool_members = [k[1] for k in value]
1602 for item in value:
1603 (iface_name, fqdn, ip) = item
1604 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1605
1606 # Include static DNS entries
1607 # XXX: Should they override the autogenerated results?
1608 # XXX: Convert input to yaml more useable.
1609 # Format:
1610 ##; this is a comment
1611 ## roomburgh=Roomburgh1
1612 ## apkerk1.Vosko=172.17.176.8 ;this as well
1613 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1614
1615 # Hack to allow special entries, for development
1616 wleiden_raw = {}
1617
1618 for line in dns_list:
1619 reverse = False
1620 k, items = line.items()[0]
1621 if type(items) == dict:
1622 if items.has_key('reverse'):
1623 reverse = items['reverse']
1624 items = items['a']
1625 else:
1626 items = items['cname']
1627 items = [items] if type(items) != list else items
1628 for item in items:
1629 if item.startswith('IN '):
1630 wleiden_raw[k] = item
1631 elif valid_addr(item):
1632 wleiden_zone[k].append((item, reverse))
1633 else:
1634 wleiden_cname[k] = item
1635
1636 # Hack to get dynamic pool listing
1637 def chunks(l, n):
1638 return [l[i:i+n] for i in range(0, len(l), n)]
1639
1640 ntp_servers = [x[0] for x in get_nameservers()]
1641 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1642 for ntp_server in chunk:
1643 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1644
1645 details = dict()
1646 # 24 updates a day allowed
1647 details['serial'] = time.strftime('%Y%m%d%H')
1648
1649 if external:
1650 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1651 else:
1652 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1653
1654 details['master'] = dns_masters[0]
1655 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1656
1657 dns_header = '''
1658$TTL 3h
1659%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1660 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1661
1662%(ns_servers)s
1663 \n'''
1664
1665
1666 if not os.path.isdir(output_dir):
1667 os.makedirs(output_dir)
1668 details['zone'] = 'wleiden.net'
1669 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1670 f.write(dns_header % details)
1671
1672 for host,items in wleiden_zone.iteritems():
1673 for ip,reverse in items:
1674 if ip not in ['0.0.0.0']:
1675 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1676 for source,dest in wleiden_cname.iteritems():
1677 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1678 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1679 for source, dest in wleiden_raw.iteritems():
1680 f.write("%s.wleiden.net. %s\n" % (source, dest))
1681 f.close()
1682
1683 # Create whole bunch of specific sub arpa zones. To keep it compliant
1684 for s in range(16,32):
1685 details['zone'] = '%i.172.in-addr.arpa' % s
1686 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1687 f.write(dns_header % details)
1688
1689 #XXX: Not effient, fix to proper data structure and do checks at other
1690 # stages
1691 for host,items in wleiden_zone.iteritems():
1692 for ip,reverse in items:
1693 if not reverse:
1694 continue
1695 if valid_addr(ip):
1696 if valid_addr(ip):
1697 if int(ip.split('.')[1]) == s:
1698 rev_ip = '.'.join(reversed(ip.split('.')))
1699 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1700 f.close()
1701
1702
1703def usage():
1704 print """Usage: %(prog)s <argument>
1705Argument:
1706\tcleanup = Cleanup all YAML files to specified format
1707\tstandalone [port] = Run configurator webserver [8000]
1708\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1709\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1710\tfull-export = Generate yaml export script for heatmap.
1711\tstatic [outputdir] = Generate all config files and store on disk
1712\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1713\ttest <node> [<file>] = Receive output for certain node [all files].
1714\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1715\tlist <status> <items> = List systems which have certain status
1716\tcreate network.kml = Create Network KML file for use in Google Earth
1717
1718Arguments:
1719\t<node> = NodeName (example: HybridRick)
1720\t<file> = %(files)s
1721\t<status> = all|up|down|planned
1722\t<items> = systems|nodes|proxies
1723
1724NOTE FOR DEVELOPERS; you can test your changes like this:
1725 BEFORE any changes in this code:
1726 $ ./gformat.py static /tmp/pre
1727 AFTER the changes:
1728 $ ./gformat.py static /tmp/post
1729 VIEW differences and VERIFY all are OK:
1730 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1731""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1732 exit(0)
1733
1734
1735def is_text_request(environ=os.environ):
1736 """ Find out whether we are calling from the CLI or any text based CLI utility """
1737 try:
1738 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1739 except KeyError:
1740 return True
1741
1742def switchFormat(setting):
1743 if setting:
1744 return "YES"
1745 else:
1746 return "NO"
1747
1748def rlinput(prompt, prefill=''):
1749 import readline
1750 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1751 try:
1752 return raw_input(prompt)
1753 finally:
1754 readline.set_startup_hook()
1755
1756def fix_conflict(left, right, default='i'):
1757 while True:
1758 print "## %-30s | %-30s" % (left, right)
1759 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1760 if not c:
1761 c = default
1762
1763 if c in ['l','1']:
1764 return left
1765 elif c in ['r','2']:
1766 return right
1767 elif c in ['e', '3']:
1768 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1769 elif c in ['i', '4']:
1770 return None
1771 else:
1772 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1773
1774
1775
1776def print_cgi_response(response_headers, output):
1777 """Could we not use some kind of wsgi wrapper to make this output?"""
1778 for header in response_headers:
1779 print "%s: %s" % header
1780 print
1781 print output
1782
1783
1784def fill_cache():
1785 ''' Poor man re-loading of few cache items (the slow ones) '''
1786 for host in get_hostlist():
1787 get_yaml(host)
1788
1789
1790def reload_cache():
1791 clear_cache()
1792 fill_cache()
1793
1794
1795def main():
1796 """Hard working sub"""
1797 # Allow easy hacking using the CLI
1798 if not os.environ.has_key('PATH_INFO'):
1799 if len(sys.argv) < 2:
1800 usage()
1801
1802 if sys.argv[1] == "standalone":
1803 import SocketServer
1804 import CGIHTTPServer
1805 # Hop to the right working directory.
1806 os.chdir(os.path.dirname(__file__))
1807 try:
1808 PORT = int(sys.argv[2])
1809 except (IndexError,ValueError):
1810 PORT = 8000
1811
1812 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1813 """ Serve this CGI from the root of the webserver """
1814 def is_cgi(self):
1815 if "favicon" in self.path:
1816 return False
1817
1818 self.cgi_info = (os.path.basename(__file__), self.path)
1819 self.path = ''
1820 return True
1821 handler = MyCGIHTTPRequestHandler
1822 SocketServer.TCPServer.allow_reuse_address = True
1823 httpd = SocketServer.TCPServer(("", PORT), handler)
1824 httpd.server_name = 'localhost'
1825 httpd.server_port = PORT
1826
1827 logger.info("serving at port %s", PORT)
1828 try:
1829 httpd.serve_forever()
1830 except KeyboardInterrupt:
1831 httpd.shutdown()
1832 logger.info("All done goodbye")
1833 elif sys.argv[1] == "test":
1834 # Basic argument validation
1835 try:
1836 node = sys.argv[2]
1837 except IndexError:
1838 print "Invalid argument"
1839 exit(1)
1840 except IOError as e:
1841 print e
1842 exit(1)
1843
1844 datadump = get_yaml(node)
1845
1846
1847 # Get files to generate
1848 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1849
1850 # Actual config generation
1851 for config in gen_files:
1852 logger.info("## Generating %s %s", node, config)
1853 print generate_config(node, config, datadump)
1854 elif sys.argv[1] == "test-cgi":
1855 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1856 os.environ['SCRIPT_NAME'] = __file__
1857 response_headers, output = process_cgi_request()
1858 print_cgi_response(response_headers, output)
1859 elif sys.argv[1] == "static":
1860 items = dict()
1861 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1862 for node in get_hostlist():
1863 items['node'] = node
1864 items['wdir'] = "%(output_dir)s/%(node)s" % items
1865 if not os.path.isdir(items['wdir']):
1866 os.makedirs(items['wdir'])
1867 datadump = get_yaml(node)
1868 for config in files:
1869 items['config'] = config
1870 logger.info("## Generating %(node)s %(config)s" % items)
1871 f = open("%(wdir)s/%(config)s" % items, "w")
1872 f.write(generate_config(node, config, datadump))
1873 f.close()
1874 elif sys.argv[1] == "wind-export":
1875 items = dict()
1876 for node in get_hostlist():
1877 datadump = get_yaml(node)
1878 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1879 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1880 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1881 VALUES (
1882 (SELECT id FROM users WHERE username = 'rvdzwet'),
1883 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1884 'Y');""" % datadump
1885 #for config in files:
1886 # items['config'] = config
1887 # print "## Generating %(node)s %(config)s" % items
1888 # f = open("%(wdir)s/%(config)s" % items, "w")
1889 # f.write(generate_config(node, config, datadump))
1890 # f.close()
1891 for node in get_hostlist():
1892 datadump = get_yaml(node)
1893 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1894 ifacedump = datadump[iface_key]
1895 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1896 ifacedump['nodename'] = datadump['nodename']
1897 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1898 ifacedump['channel'] = 0
1899 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1900 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1901 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1902 elif sys.argv[1] == "nagios-export":
1903 try:
1904 heavy_load = (sys.argv[2] == "--heavy-load")
1905 except IndexError:
1906 heavy_load = False
1907
1908 hostgroup_details = {
1909 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1910 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1911 'walphen' : 'Stichting Wireless Alphen',
1912 'westeinder' : 'Westeinder Plassen',
1913 }
1914
1915 # Convert IP to Host
1916 ip2host = {'root' : 'root'}
1917 for host in get_hostlist():
1918 datadump = get_yaml(host)
1919 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1920 for iface in get_interface_keys(datadump):
1921 if datadump[iface].has_key('autogen_gateway'):
1922 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1923
1924 # Find dependency tree based on output of lvrouted.mytree of nearest node
1925 parents = defaultdict(list)
1926 stack = ['root']
1927 prev_depth = 0
1928 for line in open('lvrouted.mytree').readlines():
1929 depth = line.count('\t')
1930 ip = line.strip().split()[0]
1931
1932 if prev_depth < depth:
1933 try:
1934 parents[ip2host[ip]].append(ip2host[stack[-1]])
1935 except KeyError as e:
1936 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1937 stack.append(ip)
1938 elif prev_depth > depth:
1939 stack = stack[:(depth - prev_depth)]
1940 elif prev_depth == depth:
1941 try:
1942 parents[ip2host[ip]].append(ip2host[stack[-1]])
1943 except KeyError as e:
1944 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1945
1946
1947 prev_depth = depth
1948 # Observe that some nodes has themself as parent or multiple parents
1949 # for now take only the first parent, other behaviour is yet to be explained
1950
1951
1952
1953 params = {
1954 'check_interval' : 5 if heavy_load else 120,
1955 'retry_interval' : 1 if heavy_load else 10,
1956 'max_check_attempts' : 10 if heavy_load else 6,
1957 'notification_interval': 120 if heavy_load else 240,
1958 }
1959
1960 print '''\
1961define host {
1962 name wleiden-node ; Default Node Template
1963 use generic-host ; Use the standard template as initial starting point
1964 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1965 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1966 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1967 notification_interval %(notification_interval)s
1968 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1969 check_command check-host-alive ; Default command to check FreeBSD hosts
1970 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1971}
1972
1973define service {
1974 name wleiden-service ; Default Service Template
1975 use generic-service ; Use the standard template as initial starting point
1976 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1977 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1978 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1979 notification_interval %(notification_interval)s
1980 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1981 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1982}
1983
1984# Please make sure to install:
1985# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1986#
1987# Recompile net-mgmt/nagios-plugins to support check_snmp
1988# make -C /usr/ports/net-mgmt/nagios-plugins
1989#
1990# Install net/bind-tools to allow v2/check_dns_wl to work:
1991# pkg install bind-tools
1992#
1993define command{
1994 command_name check_snmp_disk
1995 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1996}
1997
1998define command{
1999 command_name check_netsnmp_load
2000 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2001}
2002
2003define command{
2004 command_name check_netsnmp_proc
2005 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2006}
2007
2008define command{
2009 command_name check_by_ssh
2010 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2011}
2012
2013define command{
2014 command_name check_dns_wl
2015 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2016}
2017
2018define command{
2019 command_name check_snmp_uptime
2020 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2021}
2022
2023
2024# TDB: dhcp leases
2025# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2026
2027# TDB: internet status
2028# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2029
2030# TDB: Advanced local passive checks
2031# /usr/local/libexec/nagios/check_by_ssh
2032''' % params
2033
2034 print '''\
2035# Service Group, not displayed by default
2036define hostgroup {
2037 hostgroup_name srv_hybrid
2038 alias All Hybrid Nodes
2039 register 0
2040}
2041
2042define service {
2043 use wleiden-service
2044 hostgroup_name srv_hybrid
2045 service_description SSH
2046 check_command check_ssh
2047}
2048
2049define service {
2050 use wleiden-service,service-pnp
2051 hostgroup_name srv_hybrid
2052 service_description HTTP
2053 check_command check_http
2054}
2055
2056define service {
2057 use wleiden-service
2058 hostgroup_name srv_hybrid
2059 service_description DNS
2060 check_command check_dns_wl!"www.wirelessleiden.nl"
2061}
2062'''
2063
2064 if heavy_load:
2065 print '''\
2066define service {
2067 use wleiden-service
2068 hostgroup_name srv_hybrid
2069 service_description UPTIME
2070 check_command check_snmp_uptime
2071}
2072
2073#define service {
2074# use wleiden-service
2075# hostgroup_name srv_hybrid
2076# service_description NTP
2077# check_command check_ntp_peer
2078#}
2079
2080define service {
2081 use wleiden-service
2082 hostgroup_name srv_hybrid
2083 service_description LOAD
2084 check_command check_netsnmp_load
2085}
2086
2087define service {
2088 use wleiden-service
2089 hostgroup_name srv_hybrid
2090 service_description PROC
2091 check_command check_netsnmp_proc
2092}
2093
2094define service {
2095 use wleiden-service
2096 hostgroup_name srv_hybrid
2097 service_description DISK
2098 check_command check_snmp_disk
2099}
2100'''
2101 for node in get_hostlist():
2102 datadump = get_yaml(node)
2103 if not datadump['status'] == 'up':
2104 continue
2105 if not hostgroup_details.has_key(datadump['monitoring_group']):
2106 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2107 print '''\
2108define host {
2109 use wleiden-node,host-pnp
2110 contact_groups admins
2111 host_name %(autogen_fqdn)s
2112 address %(masterip)s
2113 hostgroups srv_hybrid,%(monitoring_group)s\
2114''' % datadump
2115 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2116 print '''\
2117 parents %(parents)s\
2118''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2119 print '''\
2120}
2121'''
2122
2123
2124 for name,alias in hostgroup_details.iteritems():
2125 print '''\
2126define hostgroup {
2127 hostgroup_name %s
2128 alias %s
2129} ''' % (name, alias)
2130
2131
2132 elif sys.argv[1] == "full-export":
2133 hosts = {}
2134 for node in get_hostlist():
2135 datadump = get_yaml(node)
2136 hosts[datadump['nodename']] = datadump
2137 print yaml.dump(hosts)
2138
2139 elif sys.argv[1] == "dns":
2140 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2141 elif sys.argv[1] == "cleanup":
2142 # First generate all datadumps
2143 datadumps = dict()
2144 ssid_to_node = dict()
2145 for host in get_hostlist():
2146 logger.info("# Processing: %s", host)
2147 # Set some boring default values
2148 datadump = { 'board' : 'UNKNOWN' }
2149 datadump.update(get_yaml(host))
2150 datadumps[datadump['nodename']] = datadump
2151
2152 (poel, errors) = make_relations()
2153 print "\n".join(["# WARNING: %s" % x for x in errors])
2154
2155 for host,datadump in datadumps.iteritems():
2156 try:
2157 # Convert all yes and no to boolean values
2158 def fix_boolean(dump):
2159 for key in dump.keys():
2160 if type(dump[key]) == dict:
2161 dump[key] = fix_boolean(dump[key])
2162 elif str(dump[key]).lower() in ["yes", "true"]:
2163 dump[key] = True
2164 elif str(dump[key]).lower() in ["no", "false"]:
2165 # Compass richting no (Noord Oost) is valid input
2166 if key != "compass": dump[key] = False
2167 return dump
2168 datadump = fix_boolean(datadump)
2169
2170 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2171 if not 'latitude' in datadump and not 'longitude' in datadump:
2172 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2173 elif 'latitude' in datadump and 'longitude' in datadump:
2174 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2175 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2176 # TODO: Compare outcome of both coordinate systems and validate against each-other
2177
2178 if datadump['nodename'].startswith('Proxy'):
2179 datadump['nodename'] = datadump['nodename'].lower()
2180
2181 for iface_key in get_interface_keys(datadump):
2182 try:
2183 # All our normal wireless cards are normal APs now
2184 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2185 datadump[iface_key]['mode'] = 'ap'
2186 # Wireless Leiden SSID have an consistent lowercase/uppercase
2187 if datadump[iface_key].has_key('ssid'):
2188 ssid = datadump[iface_key]['ssid']
2189 prefix = 'ap-WirelessLeiden-'
2190 if ssid.lower().startswith(prefix.lower()):
2191 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2192 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2193 datadump[iface_key]['mode'] = 'autogen-FIXME'
2194 if not datadump[iface_key].has_key('comment'):
2195 datadump[iface_key]['comment'] = 'autogen-FIXME'
2196
2197 if datadump[iface_key].has_key('ns_mac'):
2198 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2199
2200 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2201 datadump[iface_key] = datadump[iface_key]['desc']
2202
2203 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2204 # due to better collision avoidance.
2205 if datadump[iface_key]['type'] == '11b':
2206 datadump[iface_key]['type'] = '11g'
2207
2208 # Setting 802.11g channels to de-facto standards, to avoid
2209 # un-detected sharing with other overlapping channels
2210 #
2211 # Technically we could also use channel 13 in NL, but this is not
2212 # recommended as foreign devices might not be able to select this
2213 # channel. Secondly using 1,5,9,13 instead is going to clash with
2214 # the de-facto usage of 1,6,11.
2215 #
2216 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2217 channels_at_2400Mhz = (1,6,11)
2218 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2219 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2220 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2221 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2222
2223 # Mandatory interface keys
2224 if not datadump[iface_key].has_key('status'):
2225 datadump[iface_key]['status'] = 'planned'
2226
2227 x = datadump[iface_key]['comment']
2228 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2229
2230 # Fixing bridge_type if none is found
2231 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2232 if not 'bridge_type' in datadump[iface_key]:
2233 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2234
2235 # Making sure description works
2236 if datadump[iface_key].has_key('desc'):
2237 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2238 del datadump[iface_key]['desc']
2239 else:
2240 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2241 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2242 if response:
2243 datadump[iface_key]['comment'] = response
2244 del datadump[iface_key]['desc']
2245
2246 # Check DHCP configuration
2247 dhcp_type(datadump[iface_key])
2248
2249 # Set the compass value based on the angle between the poels
2250 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2251 my_pool = poel[network(datadump[iface_key]['ip'])]
2252 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2253 if remote_hosts:
2254 compass_target = remote_hosts[0]
2255 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2256 # TODO: Compass wanted and actual direction might differ
2257
2258 # Monitoring Group default
2259 if not 'monitoring_group' in datadump:
2260 datadump['monitoring_group'] = 'wleiden'
2261
2262 except Exception:
2263 print "# Error while processing interface %s" % iface_key
2264 raise
2265 store_yaml(datadump)
2266 except Exception:
2267 print "# Error while processing %s" % host
2268 raise
2269 elif sys.argv[1] == "list":
2270 use_fqdn = False
2271 if len(sys.argv) < 4:
2272 usage()
2273 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2274 usage()
2275 if not sys.argv[3] in ["nodes","proxies","systems"]:
2276 usage()
2277
2278 if len(sys.argv) > 4:
2279 if sys.argv[4] == "fqdn":
2280 use_fqdn = True
2281 else:
2282 usage()
2283
2284 for system in get_hostlist():
2285 datadump = get_yaml(system)
2286 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2287 continue
2288
2289 output = datadump['autogen_fqdn'] if use_fqdn else system
2290 if sys.argv[2] == "all":
2291 print output
2292 elif datadump['status'] == sys.argv[2]:
2293 print output
2294 elif sys.argv[1] == "create":
2295 if sys.argv[2] == "network.kml":
2296 print make_network_kml.make_graph()
2297 elif sys.argv[2] == "host-ips.txt":
2298 for system in get_hostlist():
2299 datadump = get_yaml(system)
2300 ips = [datadump['masterip']]
2301 for ifkey in get_interface_keys(datadump):
2302 ips.append(datadump[ifkey]['ip'].split('/')[0])
2303 print system, ' '.join(ips)
2304 elif sys.argv[2] == "host-pos.txt":
2305 for system in get_hostlist():
2306 datadump = get_yaml(system)
2307 print system, datadump['rdnap_x'], datadump['rdnap_y']
2308 elif sys.argv[2] == 'ssh_config':
2309 print '''
2310Host *.wleiden.net
2311 User root
2312
2313Host 172.16.*.*
2314 User root
2315'''
2316 for system in get_hostlist():
2317 datadump = get_yaml(system)
2318 print '''\
2319Host %s
2320 User root
2321
2322Host %s
2323 User root
2324
2325Host %s
2326 User root
2327
2328Host %s
2329 User root
2330''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2331 else:
2332 usage()
2333 else:
2334 usage()
2335 else:
2336 # Do not enable debugging for config requests as it highly clutters the output
2337 if not is_text_request():
2338 cgitb.enable()
2339 response_headers, output = process_cgi_request()
2340 print_cgi_response(response_headers, output)
2341
2342def application(environ, start_response):
2343 status = '200 OK'
2344 response_headers, output = process_cgi_request(environ)
2345 start_response(status, response_headers)
2346
2347 # Debugging only
2348 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2349 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2350 return [output]
2351
2352if __name__ == "__main__":
2353 main()
Note: See TracBrowser for help on using the repository browser.