source: genesis/tools/gformat.py@ 13929

Last change on this file since 13929 was 13929, checked in by rick, 8 years ago

Fix invalid interfaces list with DHCP on alias

When DHCP is configured on a bridge alias it should use the 'real' interface
to listen for DHCP requests instead of the bridgeX.aliasY generated ones.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 79.0 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# FreeBSD:
28# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
29# Fedora:
30# yum install python-yaml pyproj proj-epsg python-jinja2
31#
32# Rick van der Zwet <info@rickvanderzwet.nl>
33#
34
35# Hack to make the script directory is also threated as a module search path.
36import sys
37import os
38sys.path.append(os.path.dirname(__file__))
39
40SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
41
42import argparse
43import cgi
44import cgitb
45import copy
46import glob
47import make_network_kml
48import math
49import pyproj
50import random
51import re
52import socket
53import string
54import subprocess
55import textwrap
56import time
57import urlparse
58
59from pprint import pprint
60from collections import defaultdict, OrderedDict
61from sys import stderr
62try:
63 import yaml
64except ImportError, e:
65 print e
66 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
67 exit(1)
68
69try:
70 from yaml import CLoader as Loader
71 from yaml import CDumper as Dumper
72except ImportError:
73 from yaml import Loader, Dumper
74
75from jinja2 import Environment, Template
76def yesorno(value):
77 return "YES" if bool(value) else "NO"
78env = Environment()
79env.filters['yesorno'] = yesorno
80def render_template(datadump, template):
81 result = env.from_string(template).render(datadump)
82 # Make it look pretty to the naked eye, as jinja templates are not so
83 # friendly when it comes to whitespace formatting
84 ## Remove extra whitespace at end of line lstrip() style.
85 result = re.sub(r'\n[\ ]+','\n', result)
86 ## Include only a single newline between an definition and a comment
87 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
88 ## Remove extra newlines after single comment
89 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
90 return result
91
92import logging
93logging.basicConfig(format='# %(levelname)s: %(message)s' )
94logger = logging.getLogger()
95logger.setLevel(logging.DEBUG)
96
97
98if os.environ.has_key('CONFIGROOT'):
99 NODE_DIR = os.environ['CONFIGROOT']
100else:
101 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
102__version__ = '$Id: gformat.py 13929 2017-06-21 21:32:38Z rick $'
103
104files = [
105 'authorized_keys',
106 'dnsmasq.conf',
107 'dhcpd.conf',
108 'rc.conf.local',
109 'resolv.conf',
110 'motd',
111 'ntp.conf',
112 'pf.hybrid.conf.local',
113 'unbound.wleiden.conf',
114 'wleiden.yaml',
115 ]
116
117# Global variables uses
118OK = 10
119DOWN = 20
120UNKNOWN = 90
121
122
123ileiden_proxies = OrderedDict()
124normal_proxies = []
125datadump_cache = {}
126interface_list_cache = {}
127rc_conf_local_cache = {}
128nameservers_cache = []
129relations_cache = None
130def clear_cache():
131 ''' Poor mans cache implementation '''
132 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
133 datadump_cache = {}
134 interface_list_cache = {}
135 rc_conf_local_cache = {}
136 ileiden_proxies = OrderedDict()
137 normal_proxies = []
138 nameservers_cache = []
139 relations_cache = None
140
141NO_DHCP = 0
142DHCP_CLIENT = 10
143DHCP_SERVER = 20
144def dhcp_type(item):
145 if not item.has_key('dhcp'):
146 return NO_DHCP
147 elif not item['dhcp']:
148 return NO_DHCP
149 elif item['dhcp'].lower() == 'client':
150 return DHCP_CLIENT
151 else:
152 # Validation Checks
153 begin,end = map(int,item['dhcp'].split('-'))
154 if begin >= end:
155 raise ValueError("DHCP Start >= DHCP End")
156 return DHCP_SERVER
157
158def etrs2rd(lat, lon):
159 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
160 p2 = pyproj.Proj(init='EPSG:28992')
161 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
162 return (RDx, RDy)
163
164def rd2etrs(RDx, RDy):
165 p1 = pyproj.Proj(init='EPSG:28992')
166 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
167 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
168 return (lat, lon)
169
170def get_yaml(item,add_version_info=True):
171 try:
172 """ Get configuration yaml for 'item'"""
173 if datadump_cache.has_key(item):
174 return datadump_cache[item].copy()
175
176 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
177 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
178 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
179
180 # Default values
181 datadump = {
182 'autogen_revision' : 'NOTFOUND',
183 'autogen_gfile' : gfile,
184 'service_proxy_ileiden' : False,
185 }
186 f = open(gfile, 'r')
187 datadump.update(yaml.load(f,Loader=Loader))
188 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
189 if datadump['nodetype'] == 'Hybrid':
190 # Some values are defined implicitly
191 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
192 datadump['service_incoming_rdr'] = True
193 # Use some boring defaults
194 defaults = {
195 'service_proxy_normal' : False,
196 'service_accesspoint' : True,
197 'service_incoming_rdr' : False,
198 'service_concentrator' : False,
199 'monitoring_group' : 'wleiden',
200 }
201 for (key,value) in defaults.iteritems():
202 if not datadump.has_key(key):
203 datadump[key] = value
204 f.close()
205
206 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
207 if add_version_info:
208 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
209 lines = p.communicate()[0].split('\n')
210 if p.returncode == 0:
211 for line in lines:
212 if line:
213 (key, value) = line.split(': ')
214 datadump["autogen_" + key.lower().replace(' ','_')] = value
215
216 # Preformat certain needed variables for formatting and push those into special object
217 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
218
219 wlan_count=0
220 try:
221 for key in get_interface_keys(datadump, True):
222 datadump[key]['autogen_ifbase'] = key.split('_')[1]
223 datadump[key]['autogen_vlan'] = False
224
225 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
226 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
227
228 if datadump[key].has_key('parent'):
229 if datadump[key].has_key('ip'):
230 raise ValueError("Interface bridge member cannot have IP assigned")
231 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
232 raise ValueError("Interface bridge member cannot have DHCP set")
233
234 if datadump[key].has_key('ip'):
235 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
236
237 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
238 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
239 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
240 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
241 wlan_count += 1
242 else:
243 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
244 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
245 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
246 datadump[key]['autogen_vlan'] = key.split('_')[2]
247 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
248 else:
249 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
250
251 except Exception:
252 print "# Error while processing interface %s" % key
253 raise
254
255 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
256 if dhcp_type(datadump[key]) == DHCP_SERVER]
257
258 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
259 datadump['autogen_item'] = item
260
261 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
262 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
263 datadump_cache[item] = datadump.copy()
264 except Exception:
265 print "# Error while processing %s" % item
266 raise
267 return datadump
268
269
270def store_yaml(datadump, header=False):
271 """ Store configuration yaml for 'item'"""
272 item = datadump['autogen_item']
273 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
274
275 output = generate_wleiden_yaml(datadump, header)
276
277 f = open(gfile, 'w')
278 f.write(output)
279 f.close()
280
281
282def network(ip):
283 addr, mask = ip.split('/')
284 # Not parsing of these folks please
285 addr = parseaddr(addr)
286 mask = int(mask)
287 network = addr & ~((1 << (32 - mask)) - 1)
288 return network
289
290
291
292def make_relations():
293 """ Process _ALL_ yaml files to get connection relations """
294 global relations_cache
295
296 if relations_cache:
297 return relations_cache
298
299 errors = []
300 poel = defaultdict(list)
301
302 for host in get_hostlist():
303 datadump = get_yaml(host)
304 try:
305 for iface_key in get_interface_keys(datadump):
306 # Bridge members has no IP assigned
307 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
308 continue
309 net_addr = network(datadump[iface_key]['ip'])
310 poel[net_addr] += [(host,datadump[iface_key].copy())]
311 except (KeyError, ValueError) as e:
312 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
313 continue
314
315 relations_cache = (poel, errors)
316 return relations_cache
317
318
319
320def valid_addr(addr):
321 """ Show which address is valid in which are not """
322 return str(addr).startswith('172.')
323
324def get_hostlist():
325 """ Combined hosts and proxy list"""
326 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
327
328def angle_between_points(lat1,lat2,long1,long2):
329 """
330 Return Angle in radians between two GPS coordinates
331 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
332 """
333 dy = lat2 - lat1
334 dx = math.cos(lat1)*(long2 - long1)
335 angle = math.atan2(dy,dx)
336 return angle
337
338
339
340def angle_to_cd(angle):
341 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
342
343 # For easy conversion get positive degree
344 degrees = math.degrees(angle)
345 abs_degrees = 360 + degrees if degrees < 0 else degrees
346
347 # Numbers can be confusing calculate from the 4 main directions
348 p = 22.5
349 if abs_degrees < p:
350 cd = "n"
351 elif abs_degrees < (90 - p):
352 cd = "no"
353 elif abs_degrees < (90 + p):
354 cd = "o"
355 elif abs_degrees < (180 - p):
356 cd = "zo"
357 elif abs_degrees < (180 + p):
358 cd = "z"
359 elif abs_degrees < (270 - p):
360 cd = "zw"
361 elif abs_degrees < (270 + p):
362 cd = "w"
363 elif abs_degrees < (360 - p):
364 cd = "nw"
365 else:
366 cd = "n"
367 return cd
368
369
370
371def cd_between_hosts(hostA, hostB, datadumps):
372 # Using RDNAP coordinates
373 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
374 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
375 return angle_to_cd(math.atan2(dx,dy))
376
377 # GPS coordinates seems to fail somehow
378 #latA = float(datadumps[hostA]['latitude'])
379 #latB = float(datadumps[hostB]['latitude'])
380 #lonA = float(datadumps[hostA]['longitude'])
381 #lonB = float(datadumps[hostB]['longitude'])
382 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
383
384
385def generate_title(nodelist):
386 """ Main overview page """
387 items = {'root' : "." }
388 def fl(spaces, line):
389 return (' ' * spaces) + line + '\n'
390
391 output = """
392<html>
393 <head>
394 <title>Wireless leiden Configurator - GFormat</title>
395 <style type="text/css">
396 th {background-color: #999999}
397 tr:nth-child(odd) {background-color: #cccccc}
398 tr:nth-child(even) {background-color: #ffffff}
399 th, td {padding: 0.1em 1em}
400 </style>
401 </head>
402 <body>
403 <center>
404 <form type="GET" action="%(root)s">
405 <input type="hidden" name="action" value="update">
406 <input type="submit" value="Update Configuration Database (SVN)">
407 </form>
408 <table>
409 <caption><h3>Wireless Leiden Configurator</h3></caption>
410 """ % items
411
412 for node in nodelist:
413 items['node'] = node
414 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
415 for config in files:
416 items['config'] = config
417 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
418 output += fl(5, "</tr>")
419 output += """
420 </table>
421 <hr />
422 <em>%s</em>
423 </center>
424 </body>
425</html>
426 """ % __version__
427
428 return output
429
430
431
432def generate_node(node):
433 """ Print overview of all files available for node """
434 return "\n".join(files)
435
436def generate_node_overview(host):
437 """ Print overview of all files available for node """
438 datadump = get_yaml(host)
439 params = { 'host' : host }
440 output = "<em><a href='..'>Back to overview</a></em><hr />"
441 output += "<h2>Available files:</h2><ul>"
442 for cf in files:
443 params['cf'] = cf
444 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
445 output += "</ul>"
446
447 # Generate and connection listing
448 output += "<h2>Connected To:</h2><ul>"
449 (poel, errors) = make_relations()
450 for network, hosts in poel.iteritems():
451 if host in [x[0] for x in hosts]:
452 if len(hosts) == 1:
453 # Single not connected interface
454 continue
455 for remote,ifacedump in hosts:
456 if remote == host:
457 # This side of the interface
458 continue
459 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
460 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
461 output += "</ul>"
462 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
463
464 output += "<hr /><em><a href='..'>Back to overview</a></em>"
465 return output
466
467
468def generate_header(datadump, ctag="#"):
469 return """\
470%(ctag)s
471%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
472%(ctag)s
473""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
474
475
476
477def parseaddr(s):
478 """ Process IPv4 CIDR notation addr to a (binary) number """
479 f = s.split('.')
480 return (long(f[0]) << 24L) + \
481 (long(f[1]) << 16L) + \
482 (long(f[2]) << 8L) + \
483 long(f[3])
484
485
486
487def showaddr(a):
488 """ Display IPv4 addr in (dotted) CIDR notation """
489 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
490
491
492def is_member(ip, mask, canidate):
493 """ Return True if canidate is part of ip/mask block"""
494 ip_addr = parseaddr(ip)
495 ip_canidate = parseaddr(canidate)
496 mask = int(mask)
497 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
498 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
499 return ip_addr == ip_canidate
500
501
502
503def cidr2netmask(netmask):
504 """ Given a 'netmask' return corresponding CIDR """
505 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
506
507def get_network(addr, mask):
508 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
509
510
511def generate_dhcpd_conf(datadump):
512 """ Generate config file '/usr/local/etc/dhcpd.conf """
513 # Redundency support, in cause local DNS server is not running/responding.
514 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
515 output = generate_header(datadump)
516 output += Template("""\
517# option definitions common to all supported networks...
518option domain-name "dhcp.{{ autogen_fqdn }}";
519
520default-lease-time 600;
521max-lease-time 7200;
522
523# Use this to enble / disable dynamic dns updates globally.
524#ddns-update-style none;
525
526# If this DHCP server is the official DHCP server for the local
527# network, the authoritative directive should be uncommented.
528authoritative;
529
530# Use this to send dhcp log messages to a different log file (you also
531# have to hack syslog.conf to complete the redirection).
532log-facility local7;
533
534# UniFi Discovery Support
535option space ubnt;
536option ubnt.unifi-address code 1 = ip-address;
537#
538class "ubnt" {
539 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
540 option vendor-class-identifier "ubnt";
541 vendor-option-space ubnt;
542}
543
544#
545# Interface definitions
546#
547\n\n""").render(datadump)
548
549
550 # TODO: Use textwrap.fill instead
551 def indent(text, count):
552 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
553
554 # Process DHCP blocks
555 dhcp_out = defaultdict(list)
556 for iface_key in get_interface_keys(datadump):
557 ifname = datadump[iface_key]['autogen_ifbase']
558 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
559 if not datadump[iface_key].has_key('comment'):
560 datadump[iface_key]['comment'] = None
561
562 if not datadump[iface_key].has_key('ip'):
563 continue
564
565 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
566
567 (addr, mask) = datadump[iface_key]['ip'].split('/')
568 datadump[iface_key]['autogen_addr'] = addr
569 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
570 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
571
572 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
573 dhcp_out[groupif].append(textwrap.dedent("""\
574 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
575 ### not autoritive
576 }
577 """ % datadump[iface_key]))
578 continue
579
580 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
581 dhcp_part = ".".join(addr.split('.')[0:3])
582 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
583 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
584 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
585
586 # Assume the first 10 IPs could be used for static entries
587 if 'no_portal' in datadump:
588 fixed = 5
589 for mac in datadump['no_portal']:
590 dhcp_out[groupif].append(textwrap.dedent("""\
591 host fixed-%(ifname)s-%(fixed)s {
592 hardware ethernet %(mac)s;
593 fixed-address %(prefix)s.%(fixed)s;
594 }
595 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
596 fixed += 1
597
598 if 'dhcp_fixed' in datadump[iface_key]:
599 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
600 dhcp_out[groupif].append(textwrap.dedent("""\
601 host fixed-%(host)s {
602 hardware ethernet %(mac)s;
603 fixed-address %(addr)s;
604 }
605 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
606
607
608 dhcp_out[groupif].append(textwrap.dedent("""\
609 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
610 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
611 option routers %(autogen_addr)s;
612 option domain-name-servers %(autogen_dns_servers)s;
613 option ubnt.unifi-address 172.17.107.10;
614 }
615 """ % datadump[iface_key]))
616
617 # Output the blocks in groups
618 for ifname,value in sorted(dhcp_out.iteritems()):
619 if len(value) > 2:
620 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
621 else:
622 output += ''.join(value) + "\n\n"
623 return output
624
625
626
627def generate_dnsmasq_conf(datadump):
628 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
629 output = generate_header(datadump)
630 output += Template("""\
631# DHCP server options
632dhcp-authoritative
633dhcp-fqdn
634domain=dhcp.{{ autogen_fqdn }}
635domain-needed
636expand-hosts
637log-async=100
638
639# Low memory footprint
640cache-size=10000
641
642\n""").render(datadump)
643
644 for iface_key in get_interface_keys(datadump):
645 if not datadump[iface_key].has_key('comment'):
646 datadump[iface_key]['comment'] = None
647 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
648
649 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
650 output += "# not autoritive\n\n"
651 continue
652
653 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
654 (ip, cidr) = datadump[iface_key]['ip'].split('/')
655 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
656
657 dhcp_part = ".".join(ip.split('.')[0:3])
658 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
659 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
660 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
661
662 return output
663
664
665class AutoVivification(dict):
666 """Implementation of perl's autovivification feature."""
667 def __getitem__(self, item):
668 try:
669 return dict.__getitem__(self, item)
670 except KeyError:
671 value = self[item] = type(self)()
672 return value
673
674def make_interface_list(datadump):
675 if interface_list_cache.has_key(datadump['autogen_item']):
676 return (interface_list_cache[datadump['autogen_item']])
677 # lo0 configuration:
678 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
679 # - masterip is special as it needs to be assigned to at
680 # least one interface, so if not used assign to lo0
681 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
682 vlan_list = defaultdict(list)
683 bridge_list = defaultdict(list)
684 flags_if = AutoVivification()
685 dhclient_if = {'lo0' : False}
686
687 # XXX: Find some way of send this output nicely
688 output = ''
689
690 masterip_used = False
691 for iface_key in get_interface_keys(datadump):
692 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
693 masterip_used = True
694 break
695 if not masterip_used:
696 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
697
698 if 'serviceid' in datadump:
699 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
700
701 for iface_key in get_interface_keys(datadump):
702 ifacedump = datadump[iface_key]
703 ifname = ifacedump['autogen_ifname']
704
705 # If defined as vlan interface
706 if ifacedump['autogen_vlan']:
707 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
708
709 # If defined as bridge interface
710 if ifacedump['autogen_bridge_member']:
711 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
712
713 # Flag dhclient is possible
714 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
715 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
716
717 # Ethernet address
718 if ifacedump.has_key('ether'):
719 flags_if[ifname]['ether'] = ifacedump['ether']
720
721 # Add interface IP to list
722 if ifacedump.has_key('ip'):
723 item = (ifacedump['ip'], ifacedump['comment'])
724 if addrs_list.has_key(ifname):
725 addrs_list[ifname].append(item)
726 else:
727 addrs_list[ifname] = [item]
728
729 # Alias only needs IP assignment for now, this might change if we
730 # are going to use virtual accesspoints
731 if "alias" in iface_key:
732 continue
733
734 # XXX: Might want to deduct type directly from interface name
735 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
736 # Default to station (client) mode
737 ifacedump['autogen_wlanmode'] = "sta"
738 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
739 ifacedump['autogen_wlanmode'] = "ap"
740
741 if not ifacedump.has_key('channel'):
742 if ifacedump['type'] == '11a':
743 ifacedump['channel'] = 36
744 else:
745 ifacedump['channel'] = 1
746
747 # Allow special hacks at the back like wds and stuff
748 if not ifacedump.has_key('extra'):
749 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
750 else:
751 ifacedump['autogen_extra'] = ifacedump['extra']
752
753 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
754
755 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
756 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
757 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
758 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
759 output += "\n"
760
761 elif ifacedump['type'] in ['ethernet', 'eth']:
762 # No special config needed besides IP
763 pass
764 elif ifacedump['type'] in ['vlan']:
765 # VLAN member has no special configuration
766 pass
767 else:
768 assert False, "Unknown type " + ifacedump['type']
769
770 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
771 interface_list_cache[datadump['autogen_item']] = store
772 return(store)
773
774
775
776def create_proxies_list():
777 if not ileiden_proxies or not normal_proxies:
778 # Placeholder for to-be-installed proxies, this will avoid updating the all
779 # nodes to include this new machine, yet due to an unbound issue, this list
780 # has to be kept small.
781
782 for i in range(1,20):
783 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
784
785 for host in get_hostlist():
786 hostdump = get_yaml(host)
787 if hostdump['status'] == 'up':
788 if hostdump['service_proxy_ileiden']:
789 ileiden_proxies[hostdump['serviceid']] = hostdump
790 if hostdump['service_proxy_normal']:
791 normal_proxies.append(hostdump)
792
793
794
795def generate_rc_conf_local(datadump):
796 """ Generate configuration file '/etc/rc.conf.local' """
797 item = datadump['autogen_item']
798 if rc_conf_local_cache.has_key(item):
799 return rc_conf_local_cache[item]
800
801 if not datadump.has_key('ileiden'):
802 datadump['autogen_ileiden_enable'] = False
803 else:
804 datadump['autogen_ileiden_enable'] = datadump['ileiden']
805
806 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
807
808 create_proxies_list()
809 datadump['autogen_ileiden_proxies'] = ileiden_proxies
810 datadump['autogen_normal_proxies'] = normal_proxies
811 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
812 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
813 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
814 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
815
816 output = generate_header(datadump, "#");
817 output += render_template(datadump, """\
818hostname='{{ autogen_fqdn }}'
819location='{{ location }}'
820nodetype="{{ nodetype }}"
821
822#
823# Configured listings
824#
825captive_portal_whitelist=""
826{% if nodetype == "Proxy" %}
827#
828# Proxy Configuration
829#
830{% if gateway and service_proxy_ileiden -%}
831defaultrouter="{{ gateway }}"
832{% else -%}
833#defaultrouter="NOTSET"
834{% endif -%}
835internalif="{{ internalif }}"
836ileiden_enable="{{ autogen_ileiden_enable }}"
837gateway_enable="{{ autogen_ileiden_enable }}"
838pf_enable="yes"
839pf_rules="/etc/pf.conf"
840{% if autogen_ileiden_enable -%}
841pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
842lvrouted_enable="{{ autogen_ileiden_enable }}"
843lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
844{% else -%}
845pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
846{% endif -%}
847{% if internalroute -%}
848static_routes="wleiden"
849route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
850{% endif -%}
851
852{% elif nodetype == "Hybrid" %}
853 #
854 # Hybrid Configuration
855 #
856 list_ileiden_proxies="
857 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
858 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
859 {% endfor -%}
860 "
861 list_normal_proxies="
862 {% for item in autogen_normal_proxies -%}
863 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
864 {% endfor -%}
865 "
866
867 {% if autogen_dhcp_interfaces -%}
868 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
869 {% else %}
870 captive_portal_interfaces="dummy"
871 {% endif %}
872 externalif="{{ externalif|default('vr0', true) }}"
873 masterip="{{ masterip }}"
874
875 {% if gateway and service_proxy_ileiden %}
876 defaultrouter="{{ gateway }}"
877 {% else %}
878 #defaultrouter="NOTSET"
879 {% endif %}
880
881 #
882 # Defined services
883 #
884 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
885 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
886 service_accesspoint="{{ service_accesspoint|yesorno }}"
887 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
888 service_concentrator="{{ service_concentrator|yesorno }}"
889
890 {% if service_proxy_ileiden %}
891 pf_rules="/etc/pf.hybrid.conf"
892 {% if service_concentrator %}
893 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
894 {% else %}
895 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
896 {% endif %}
897 pf_flags="$pf_flags -D publicnat=80,443"
898 lvrouted_flags="$lvrouted_flags -g"
899 {% elif service_proxy_normal or service_incoming_rdr %}
900 pf_rules="/etc/pf.hybrid.conf"
901 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
902 pf_flags="$pf_flags -D publicnat=0"
903 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
904 named_setfib="1"
905 tinyproxy_setfib="1"
906 dnsmasq_setfib="1"
907 sshd_setfib="1"
908 {% else %}
909 named_auto_forward_only="YES"
910 pf_rules="/etc/pf.node.conf"
911 pf_flags=""
912 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
913 {% endif %}
914 {% if service_concentrator %}
915 # Do mind installing certificates is NOT done automatically for security reasons
916 openvpn_enable="YES"
917 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
918 {% endif %}
919
920 {% if service_proxy_normal %}
921 tinyproxy_enable="yes"
922 {% else %}
923 pen_wrapper_enable="yes"
924 {% endif %}
925
926 {% if service_accesspoint %}
927 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
928 {% endif %}
929
930 {% if board == "ALIX2" or board == "net4801" %}
931 #
932 # ''Fat'' configuration, board has 256MB RAM
933 #
934 dnsmasq_enable="NO"
935 named_enable="YES"
936 unbound_enable="YES"
937 {% if autogen_dhcp_interfaces -%}
938 dhcpd_enable="YES"
939 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
940 {% endif -%}
941 {% elif board == "apu1d" %}
942 #
943 # ''Fat'' configuration, board has 1024MB RAM
944 #
945 dnsmasq_enable="NO"
946 unbound_enable="YES"
947 {% if autogen_dhcp_interfaces -%}
948 dhcpd_enable="YES"
949 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
950 {% endif -%}
951 {% endif -%}
952{% endif %}
953
954#
955# Script variables
956#
957attached_devices="{{ autogen_attached_devices|join(' ') }}"
958neighbours="{{ autogen_neighbours|join(' ') }}"
959
960
961#
962# Interface definitions
963#\n
964""")
965
966 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
967 for iface, vlans in sorted(vlan_list.items()):
968 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
969
970 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
971 for iface in sorted(vlan_list.keys()):
972 if not iface in addrs_list.keys():
973 output += "ifconfig_%s='up'\n" % iface
974
975 output += "\n"
976
977 # Bridge configuration:
978 if bridge_list.keys():
979 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
980
981 for iface in bridge_list.keys():
982 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
983
984 # Bridge member interfaces not containing a configuration should be marked active explcitly.
985 for _,members in bridge_list.items():
986 for iface in members:
987 if not iface in addrs_list.keys():
988 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
989
990 output += "\n"
991
992 # Details like SSID
993 if extra_ouput:
994 output += extra_ouput.strip() + "\n"
995
996 # Print IP address which needs to be assigned over here
997 output += "\n"
998 for iface,addrs in sorted(addrs_list.iteritems()):
999 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1000 output += "# %s || %s || %s\n" % (iface, addr, comment)
1001
1002 # Write DHCLIENT entry
1003 if iface in dhclient_if and dhclient_if[iface]:
1004 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1005 continue
1006
1007 # Make sure the external address is always first as this is needed in the
1008 # firewall setup
1009 addrs = sorted(
1010 [x for x in addrs if not '0.0.0.0' in x[0]],
1011 key=lambda x: x[0].split('.')[0],
1012 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1013 )
1014
1015 idx_offset = 0
1016 # Set MAC is required
1017 if flags_if[iface].has_key('ether'):
1018 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1019 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1020 idx_offset += 1
1021 else:
1022 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1023
1024 for idx, addr in enumerate(addrs[1:]):
1025 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1026
1027 output += "\n"
1028
1029 rc_conf_local_cache[datadump['autogen_item']] = output
1030 return output
1031
1032
1033
1034def get_all_configs():
1035 """ Get dict with key 'host' with all configs present """
1036 configs = dict()
1037 for host in get_hostlist():
1038 datadump = get_yaml(host)
1039 configs[host] = datadump
1040 return configs
1041
1042
1043def get_interface_keys(config, extra=False):
1044 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1045 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1046 if extra == False:
1047 return filter(lambda x: not "extra" in x, elems)
1048 else:
1049 return elems
1050
1051
1052def get_used_ips(configs):
1053 """ Return array of all IPs used in config files"""
1054 ip_list = []
1055 for config in configs:
1056 ip_list.append(config['masterip'])
1057 if 'serviceid' in config:
1058 ip_list.append(config['serviceid'])
1059 for iface_key in get_interface_keys(config, True):
1060 l = config[iface_key]['ip']
1061 addr, mask = l.split('/')
1062 # Special case do not process
1063 if valid_addr(addr):
1064 ip_list.append(addr)
1065 else:
1066 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1067 return sorted(ip_list)
1068
1069
1070
1071def get_nameservers(max_servers=None):
1072 if nameservers_cache:
1073 return nameservers_cache[0:max_servers]
1074
1075 for host in get_hostlist():
1076 hostdump = get_yaml(host)
1077 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1078 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1079
1080 return nameservers_cache[0:max_servers]
1081
1082
1083def get_neighbours(datadump):
1084 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1085
1086 (poel, errors) = make_relations()
1087 table = []
1088 for iface,addrs in sorted(addrs_list.iteritems()):
1089 if iface in ['lo0']:
1090 continue
1091
1092 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1093 if not addr.startswith('172.'):
1094 # Avoid listing internet connections as pool
1095 continue
1096 for neighbour in poel[network(addr)]:
1097 if neighbour[0] != datadump['autogen_item']:
1098 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1099 return table
1100
1101
1102def get_attached_devices(datadump, url=False):
1103 table = []
1104 for iface_key in get_interface_keys(datadump, True):
1105 ifacedump = datadump[iface_key]
1106
1107 if not ifacedump.has_key('ns_ip'):
1108 continue
1109
1110 x_ip = ifacedump['ns_ip'].split('/')[0]
1111
1112 if 'mode' in ifacedump:
1113 x_mode = ifacedump['mode']
1114 else:
1115 x_mode = 'unknown'
1116
1117 if 'bridge_type' in ifacedump:
1118 device_type = ifacedump['bridge_type']
1119 else:
1120 device_type = 'Unknown'
1121
1122 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1123 return table
1124
1125
1126def generate_resolv_conf(datadump):
1127 """ Generate configuration file '/etc/resolv.conf' """
1128 # XXX: This should properly going to be an datastructure soon
1129 datadump['autogen_header'] = generate_header(datadump, "#")
1130 datadump['autogen_edge_nameservers'] = ''
1131
1132
1133 for masterip,realname in get_nameservers():
1134 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1135
1136 return Template("""\
1137{{ autogen_header }}
1138search wleiden.net
1139
1140# Try local (cache) first
1141nameserver 127.0.0.1
1142
1143{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1144nameserver 8.8.8.8 # Google Public NameServer
1145nameserver 64.6.64.6 # Verisign Public NameServer
1146{% else -%}
1147# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1148{{ autogen_edge_nameservers }}
1149{% endif -%}
1150""").render(datadump)
1151
1152
1153
1154def generate_ntp_conf(datadump):
1155 """ Generate configuration file '/etc/ntp.conf' """
1156 # XXX: This should properly going to be an datastructure soon
1157
1158 datadump['autogen_header'] = generate_header(datadump, "#")
1159 datadump['autogen_ntp_servers'] = ''
1160 for host in get_hostlist():
1161 hostdump = get_yaml(host)
1162 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1163 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1164
1165 return Template("""\
1166{{ autogen_header }}
1167
1168{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1169# Machine hooked to internet.
1170server 0.nl.pool.ntp.org iburst maxpoll 9
1171server 1.nl.pool.ntp.org iburst maxpoll 9
1172server 2.nl.pool.ntp.org iburst maxpoll 9
1173server 3.nl.pool.ntp.org iburst maxpoll 9
1174{% else -%}
1175# Local Wireless Leiden NTP Servers.
1176server 0.pool.ntp.wleiden.net iburst maxpoll 9
1177server 1.pool.ntp.wleiden.net iburst maxpoll 9
1178server 2.pool.ntp.wleiden.net iburst maxpoll 9
1179server 3.pool.ntp.wleiden.net iburst maxpoll 9
1180
1181# All the configured NTP servers
1182{{ autogen_ntp_servers }}
1183{% endif %}
1184
1185# If a server loses sync with all upstream servers, NTP clients
1186# no longer follow that server. The local clock can be configured
1187# to provide a time source when this happens, but it should usually
1188# be configured on just one server on a network. For more details see
1189# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1190# The use of Orphan Mode may be preferable.
1191#
1192server 127.127.1.0
1193fudge 127.127.1.0 stratum 10
1194""").render(datadump)
1195
1196
1197def generate_pf_hybrid_conf_local(datadump):
1198 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1199 datadump['autogen_header'] = generate_header(datadump, "#")
1200 if datadump['service_incoming_rdr']:
1201 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1202 return Template("""\
1203{{ autogen_header }}
1204
1205# Redirect some internal facing services outside (7)
1206# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1207{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1208rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1209{% endfor -%}
1210# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1211{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1212rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1213{% endfor -%}
1214""").render(datadump)
1215
1216def generate_unbound_wleiden_conf(datadump):
1217 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1218 datadump['autogen_header'] = generate_header(datadump, "#")
1219
1220 autogen_ips = []
1221 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1222 for iface,addrs in sorted(addrs_list.iteritems()):
1223 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1224 if addr.startswith('172'):
1225 autogen_ips.append((addr.split('/')[0], comment))
1226 datadump['autogen_ips'] = autogen_ips
1227
1228 create_proxies_list()
1229 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1230 return Template("""\
1231{{ autogen_header }}
1232
1233server:
1234 ## Static definitions fail to start on systems with broken ue(4) interfaces
1235{%- for ip,comment in autogen_ips %}
1236 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1237{%- endfor %}
1238 ## Enabling wildcard matching as work-around
1239 interface: 0.0.0.0
1240 interface: ::0
1241
1242forward-zone:
1243 name: '.'
1244{%- if service_proxy_ileiden %}
1245 forward-addr: 8.8.8.8 # Google DNS A
1246 forward-addr: 8.8.4.4 # Google DNS B
1247 forward-addr: 208.67.222.222 # OpenDNS DNS A
1248 forward-addr: 208.67.220.220 # OpenDNS DNS B
1249{% else -%}
1250{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1251 {%- if loop.index <= 5 %}
1252 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1253 {%- endif %}
1254{%- endfor %}
1255{%- endif %}
1256""").render(datadump)
1257
1258def generate_motd(datadump):
1259 """ Generate configuration file '/etc/motd' """
1260 output = Template("""\
1261FreeBSD run ``service motd onestart'' to make me look normal
1262
1263 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1264 Loc: {{ location }}
1265
1266Services:
1267{% if board == "ALIX2" or board == "net4801" -%}
1268{{" -"}} Core Node ({{ board }})
1269{% else -%}
1270{{" -"}} Hulp Node ({{ board }})
1271{% endif -%}
1272{% if service_proxy_normal -%}
1273{{" -"}} Normal Proxy
1274{% endif -%}
1275{% if service_proxy_ileiden -%}
1276{{" -"}} iLeiden Proxy
1277{% endif -%}
1278{% if service_incoming_rdr -%}
1279{{" -"}} Incoming port redirects
1280{% endif %}
1281Interlinks:\n
1282""").render(datadump)
1283
1284
1285 def make_table(table):
1286 if not table:
1287 return " - none\n"
1288 else:
1289 lines = ""
1290 col_width = [max(len(x) for x in col) for col in zip(*table)]
1291 for row in table:
1292 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1293 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1294 return lines
1295
1296 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1297 table = []
1298 for iface,addrs in sorted(addrs_list.iteritems()):
1299 if iface in ['lo0']:
1300 continue
1301 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1302 table.append((iface, addr, comment))
1303
1304 output += make_table(table)
1305 output += '\n'
1306 output += """\
1307Attached devices:
1308"""
1309 output += make_table(get_attached_devices(datadump, url=True))
1310 output += '\n'
1311 output += """\
1312Available neighbours:
1313"""
1314 output += make_table(get_neighbours(datadump))
1315
1316 return output
1317
1318
1319def format_yaml_value(value):
1320 """ Get yaml value in right syntax for outputting """
1321 if isinstance(value,str):
1322 output = '"%s"' % value
1323 else:
1324 output = value
1325 return output
1326
1327
1328
1329def format_wleiden_yaml(datadump):
1330 """ Special formatting to ensure it is editable"""
1331 output = "# Genesis config yaml style\n"
1332 output += "# vim:ts=2:et:sw=2:ai\n"
1333 output += "#\n"
1334 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1335 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1336 if key == 'rdr_rules':
1337 output += '%-10s:\n' % 'rdr_rules'
1338 for rdr_rule in datadump[key]:
1339 output += '- %s\n' % rdr_rule
1340 else:
1341 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1342
1343 output += "\n\n"
1344
1345 # Format (key, required)
1346 key_order = (
1347 ('comment', True),
1348 ('parent', False),
1349 ('ip', False),
1350 ('ether', False),
1351 ('desc', True),
1352 ('sdesc', True),
1353 ('mode', True),
1354 ('type', True),
1355 ('extra_type', False),
1356 ('channel', False),
1357 ('ssid', False),
1358 ('wlan_mac', False),
1359 ('dhcp', True),
1360 ('dhcp_fixed', False),
1361 ('compass', False),
1362 ('distance', False),
1363 ('ns_ip', False),
1364 ('repeater_ip', False),
1365 ('bullet2_ip', False),
1366 ('ns_mac', False),
1367 ('bullet2_mac', False),
1368 ('ns_type', False),
1369 ('bridge_type', False),
1370 ('status', True),
1371 )
1372
1373 for iface_key in sorted(iface_keys):
1374 try:
1375 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1376 if remainder:
1377 raise KeyError("invalid keys: %s" % remainder)
1378
1379 output += "%s:\n" % iface_key
1380 for key,required in key_order:
1381 if datadump[iface_key].has_key(key):
1382 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1383 output += "\n\n"
1384 except Exception:
1385 print "# Error while processing interface %s" % iface_key
1386 raise
1387
1388 return output
1389
1390
1391
1392def generate_wleiden_yaml(datadump, header=True):
1393 """ Generate (petty) version of wleiden.yaml"""
1394 output = generate_header(datadump, "#") if header else ''
1395
1396 for key in datadump.keys():
1397 if key.startswith('autogen_'):
1398 del datadump[key]
1399 # Interface autogen cleanups
1400 elif type(datadump[key]) == dict:
1401 for key2 in datadump[key].keys():
1402 if key2.startswith('autogen_'):
1403 del datadump[key][key2]
1404
1405 output += format_wleiden_yaml(datadump)
1406 return output
1407
1408def generate_nanostation_config(datadump, iface, ns_type):
1409 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1410 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1411
1412 datadump.update(datadump['iface_%s' % iface])
1413
1414 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1415
1416def generate_yaml(datadump):
1417 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1418
1419
1420
1421def generate_config(node, config, datadump=None):
1422 """ Print configuration file 'config' of 'node' """
1423 output = ""
1424 try:
1425 # Load config file
1426 if datadump == None:
1427 datadump = get_yaml(node)
1428
1429 if config == 'wleiden.yaml':
1430 output += generate_wleiden_yaml(datadump)
1431 elif config == 'authorized_keys':
1432 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1433 output += f.read()
1434 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1435 # Fetch local keys if existing
1436 if os.path.exists(node_keys):
1437 output += open(node_keys, 'r').read()
1438 f.close()
1439 elif config == 'dnsmasq.conf':
1440 output += generate_dnsmasq_conf(datadump)
1441 elif config == 'dhcpd.conf':
1442 output += generate_dhcpd_conf(datadump)
1443 elif config == 'rc.conf.local':
1444 output += generate_rc_conf_local(datadump)
1445 elif config == 'resolv.conf':
1446 output += generate_resolv_conf(datadump)
1447 elif config == 'ntp.conf':
1448 output += generate_ntp_conf(datadump)
1449 elif config == 'motd':
1450 output += generate_motd(datadump)
1451 elif config == 'pf.hybrid.conf.local':
1452 output += generate_pf_hybrid_conf_local(datadump)
1453 elif config == 'unbound.wleiden.conf':
1454 output += generate_unbound_wleiden_conf(datadump)
1455 elif config.startswith('vr'):
1456 interface, ns_type = config.strip('.yaml').split('-')
1457 output += generate_nanostation_config(datadump, interface, ns_type)
1458 else:
1459 assert False, "Config not found!"
1460 except IOError, e:
1461 output += "[ERROR] Config file not found"
1462 return output
1463
1464
1465
1466def process_cgi_request(environ=os.environ):
1467 """ When calling from CGI """
1468 response_headers = []
1469 content_type = 'text/plain'
1470
1471 # Update repository if requested
1472 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1473 if form and form.has_key("action") and "update" in form["action"]:
1474 output = "[INFO] Updating subverion, please wait...\n"
1475 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1476 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1477 output += "[INFO] All done, redirecting in 5 seconds"
1478 response_headers += [
1479 ('Refresh', '5; url=.'),
1480 ]
1481 reload_cache()
1482 else:
1483 base_uri = environ['PATH_INFO']
1484 uri = base_uri.strip('/').split('/')
1485
1486 output = "Template Holder"
1487 if base_uri.endswith('/create/network.kml'):
1488 content_type='application/vnd.google-earth.kml+xml'
1489 output = make_network_kml.make_graph()
1490 elif base_uri.endswith('/api/get/nodeplanner.json'):
1491 content_type='application/json'
1492 output = make_network_kml.make_nodeplanner_json()
1493 elif not uri[0]:
1494 if is_text_request(environ):
1495 output = '\n'.join(get_hostlist())
1496 else:
1497 content_type = 'text/html'
1498 output = generate_title(get_hostlist())
1499 elif len(uri) == 1:
1500 if is_text_request(environ):
1501 output = generate_node(uri[0])
1502 else:
1503 content_type = 'text/html'
1504 output = generate_node_overview(uri[0])
1505 elif len(uri) == 2:
1506 output = generate_config(uri[0], uri[1])
1507 else:
1508 assert False, "Invalid option"
1509
1510 # Return response
1511 response_headers += [
1512 ('Content-type', content_type),
1513 ('Content-Length', str(len(output))),
1514 ]
1515 return(response_headers, str(output))
1516
1517
1518def make_dns(output_dir = 'dns', external = False):
1519 items = dict()
1520
1521 # hostname is key, IP is value
1522 wleiden_zone = defaultdict(list)
1523 wleiden_cname = dict()
1524
1525 pool = dict()
1526 for node in get_hostlist():
1527 datadump = get_yaml(node)
1528
1529 fqdn = datadump['nodename']
1530
1531 if datadump.has_key('rdr_host'):
1532 remote_target = datadump['rdr_host']
1533 elif datadump.has_key('remote_access') and datadump['remote_access']:
1534 remote_target = datadump['remote_access'].split(':')[0]
1535 else:
1536 remote_target = None
1537
1538 if remote_target:
1539 try:
1540 parseaddr(remote_target)
1541 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1542 except (IndexError, ValueError):
1543 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1544
1545
1546 wleiden_zone[fqdn].append((datadump['masterip'], True))
1547
1548 # Hacking to get proper DHCP IPs and hostnames
1549 for iface_key in get_interface_keys(datadump):
1550 iface_name = iface_key.replace('_','-')
1551 if 'ip' in datadump[iface_key]:
1552 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1553 try:
1554 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1555 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1556 dhcp_part = ".".join(ip.split('.')[0:3])
1557 if ip != datadump['masterip']:
1558 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1559 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1560 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1561 except (AttributeError, ValueError, KeyError):
1562 # First push it into a pool, to indentify the counter-part later on
1563 addr = parseaddr(ip)
1564 cidr = int(cidr)
1565 addr = addr & ~((1 << (32 - cidr)) - 1)
1566 if pool.has_key(addr):
1567 pool[addr] += [(iface_name, fqdn, ip)]
1568 else:
1569 pool[addr] = [(iface_name, fqdn, ip)]
1570 continue
1571
1572
1573
1574 # WL uses an /29 to configure an interface. IP's are ordered like this:
1575 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1576
1577 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1578
1579 # Automatic naming convention of interlinks namely 2 + remote.lower()
1580 for (key,value) in pool.iteritems():
1581 # Make sure they are sorted from low-ip to high-ip
1582 value = sorted(value, key=lambda x: parseaddr(x[2]))
1583
1584 if len(value) == 1:
1585 (iface_name, fqdn, ip) = value[0]
1586 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1587
1588 # Device DNS names
1589 if 'cnode' in fqdn.lower():
1590 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1591 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1592
1593 elif len(value) == 2:
1594 (a_iface_name, a_fqdn, a_ip) = value[0]
1595 (b_iface_name, b_fqdn, b_ip) = value[1]
1596 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1597 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1598
1599 # Device DNS names
1600 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1601 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1602 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1603 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1604 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1605 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1606 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1607
1608 else:
1609 pool_members = [k[1] for k in value]
1610 for item in value:
1611 (iface_name, fqdn, ip) = item
1612 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1613
1614 # Include static DNS entries
1615 # XXX: Should they override the autogenerated results?
1616 # XXX: Convert input to yaml more useable.
1617 # Format:
1618 ##; this is a comment
1619 ## roomburgh=Roomburgh1
1620 ## apkerk1.Vosko=172.17.176.8 ;this as well
1621 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1622
1623 # Hack to allow special entries, for development
1624 wleiden_raw = {}
1625
1626 for line in dns_list:
1627 reverse = False
1628 k, items = line.items()[0]
1629 if type(items) == dict:
1630 if items.has_key('reverse'):
1631 reverse = items['reverse']
1632 items = items['a']
1633 else:
1634 items = items['cname']
1635 items = [items] if type(items) != list else items
1636 for item in items:
1637 if item.startswith('IN '):
1638 wleiden_raw[k] = item
1639 elif valid_addr(item):
1640 wleiden_zone[k].append((item, reverse))
1641 else:
1642 wleiden_cname[k] = item
1643
1644 # Hack to get dynamic pool listing
1645 def chunks(l, n):
1646 return [l[i:i+n] for i in range(0, len(l), n)]
1647
1648 ntp_servers = [x[0] for x in get_nameservers()]
1649 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1650 for ntp_server in chunk:
1651 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1652
1653 details = dict()
1654 # 24 updates a day allowed
1655 details['serial'] = time.strftime('%Y%m%d%H')
1656
1657 if external:
1658 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1659 else:
1660 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1661
1662 details['master'] = dns_masters[0]
1663 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1664
1665 dns_header = '''
1666$TTL 3h
1667%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1668 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1669
1670%(ns_servers)s
1671 \n'''
1672
1673
1674 if not os.path.isdir(output_dir):
1675 os.makedirs(output_dir)
1676 details['zone'] = 'wleiden.net'
1677 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1678 f.write(dns_header % details)
1679
1680 for host,items in wleiden_zone.iteritems():
1681 for ip,reverse in items:
1682 if ip not in ['0.0.0.0']:
1683 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1684 for source,dest in wleiden_cname.iteritems():
1685 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1686 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1687 for source, dest in wleiden_raw.iteritems():
1688 f.write("%s.wleiden.net. %s\n" % (source, dest))
1689 f.close()
1690
1691 # Create whole bunch of specific sub arpa zones. To keep it compliant
1692 for s in range(16,32):
1693 details['zone'] = '%i.172.in-addr.arpa' % s
1694 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1695 f.write(dns_header % details)
1696
1697 #XXX: Not effient, fix to proper data structure and do checks at other
1698 # stages
1699 for host,items in wleiden_zone.iteritems():
1700 for ip,reverse in items:
1701 if not reverse:
1702 continue
1703 if valid_addr(ip):
1704 if valid_addr(ip):
1705 if int(ip.split('.')[1]) == s:
1706 rev_ip = '.'.join(reversed(ip.split('.')))
1707 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1708 f.close()
1709
1710
1711def usage():
1712 print """Usage: %(prog)s <argument>
1713Argument:
1714\tcleanup = Cleanup all YAML files to specified format
1715\tstandalone [port] = Run configurator webserver [8000]
1716\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1717\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1718\tfull-export = Generate yaml export script for heatmap.
1719\tstatic [outputdir] = Generate all config files and store on disk
1720\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1721\ttest <node> [<file>] = Receive output for certain node [all files].
1722\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1723\tlist <status> <items> = List systems which have certain status
1724\tcreate network.kml = Create Network KML file for use in Google Earth
1725
1726Arguments:
1727\t<node> = NodeName (example: HybridRick)
1728\t<file> = %(files)s
1729\t<status> = all|up|down|planned
1730\t<items> = systems|nodes|proxies
1731
1732NOTE FOR DEVELOPERS; you can test your changes like this:
1733 BEFORE any changes in this code:
1734 $ ./gformat.py static /tmp/pre
1735 AFTER the changes:
1736 $ ./gformat.py static /tmp/post
1737 VIEW differences and VERIFY all are OK:
1738 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1739""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1740 exit(0)
1741
1742
1743def is_text_request(environ=os.environ):
1744 """ Find out whether we are calling from the CLI or any text based CLI utility """
1745 try:
1746 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1747 except KeyError:
1748 return True
1749
1750def switchFormat(setting):
1751 if setting:
1752 return "YES"
1753 else:
1754 return "NO"
1755
1756def rlinput(prompt, prefill=''):
1757 import readline
1758 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1759 try:
1760 return raw_input(prompt)
1761 finally:
1762 readline.set_startup_hook()
1763
1764def fix_conflict(left, right, default='i'):
1765 while True:
1766 print "## %-30s | %-30s" % (left, right)
1767 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1768 if not c:
1769 c = default
1770
1771 if c in ['l','1']:
1772 return left
1773 elif c in ['r','2']:
1774 return right
1775 elif c in ['e', '3']:
1776 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1777 elif c in ['i', '4']:
1778 return None
1779 else:
1780 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1781
1782
1783
1784def print_cgi_response(response_headers, output):
1785 """Could we not use some kind of wsgi wrapper to make this output?"""
1786 for header in response_headers:
1787 print "%s: %s" % header
1788 print
1789 print output
1790
1791
1792def fill_cache():
1793 ''' Poor man re-loading of few cache items (the slow ones) '''
1794 for host in get_hostlist():
1795 get_yaml(host)
1796
1797
1798def reload_cache():
1799 clear_cache()
1800 fill_cache()
1801
1802
1803def main():
1804 """Hard working sub"""
1805 # Allow easy hacking using the CLI
1806 if not os.environ.has_key('PATH_INFO'):
1807 if len(sys.argv) < 2:
1808 usage()
1809
1810 if sys.argv[1] == "standalone":
1811 import SocketServer
1812 import CGIHTTPServer
1813 # Hop to the right working directory.
1814 os.chdir(os.path.dirname(__file__))
1815 try:
1816 PORT = int(sys.argv[2])
1817 except (IndexError,ValueError):
1818 PORT = 8000
1819
1820 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1821 """ Serve this CGI from the root of the webserver """
1822 def is_cgi(self):
1823 if "favicon" in self.path:
1824 return False
1825
1826 self.cgi_info = (os.path.basename(__file__), self.path)
1827 self.path = ''
1828 return True
1829 handler = MyCGIHTTPRequestHandler
1830 SocketServer.TCPServer.allow_reuse_address = True
1831 httpd = SocketServer.TCPServer(("", PORT), handler)
1832 httpd.server_name = 'localhost'
1833 httpd.server_port = PORT
1834
1835 logger.info("serving at port %s", PORT)
1836 try:
1837 httpd.serve_forever()
1838 except KeyboardInterrupt:
1839 httpd.shutdown()
1840 logger.info("All done goodbye")
1841 elif sys.argv[1] == "test":
1842 # Basic argument validation
1843 try:
1844 node = sys.argv[2]
1845 except IndexError:
1846 print "Invalid argument"
1847 exit(1)
1848 except IOError as e:
1849 print e
1850 exit(1)
1851
1852 datadump = get_yaml(node)
1853
1854
1855 # Get files to generate
1856 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1857
1858 # Actual config generation
1859 for config in gen_files:
1860 logger.info("## Generating %s %s", node, config)
1861 print generate_config(node, config, datadump)
1862 elif sys.argv[1] == "test-cgi":
1863 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1864 os.environ['SCRIPT_NAME'] = __file__
1865 response_headers, output = process_cgi_request()
1866 print_cgi_response(response_headers, output)
1867 elif sys.argv[1] == "static":
1868 items = dict()
1869 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1870 for node in get_hostlist():
1871 items['node'] = node
1872 items['wdir'] = "%(output_dir)s/%(node)s" % items
1873 if not os.path.isdir(items['wdir']):
1874 os.makedirs(items['wdir'])
1875 datadump = get_yaml(node)
1876 for config in files:
1877 items['config'] = config
1878 logger.info("## Generating %(node)s %(config)s" % items)
1879 f = open("%(wdir)s/%(config)s" % items, "w")
1880 f.write(generate_config(node, config, datadump))
1881 f.close()
1882 elif sys.argv[1] == "wind-export":
1883 items = dict()
1884 for node in get_hostlist():
1885 datadump = get_yaml(node)
1886 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1887 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1888 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1889 VALUES (
1890 (SELECT id FROM users WHERE username = 'rvdzwet'),
1891 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1892 'Y');""" % datadump
1893 #for config in files:
1894 # items['config'] = config
1895 # print "## Generating %(node)s %(config)s" % items
1896 # f = open("%(wdir)s/%(config)s" % items, "w")
1897 # f.write(generate_config(node, config, datadump))
1898 # f.close()
1899 for node in get_hostlist():
1900 datadump = get_yaml(node)
1901 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1902 ifacedump = datadump[iface_key]
1903 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1904 ifacedump['nodename'] = datadump['nodename']
1905 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1906 ifacedump['channel'] = 0
1907 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1908 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1909 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1910 elif sys.argv[1] == "nagios-export":
1911 try:
1912 heavy_load = (sys.argv[2] == "--heavy-load")
1913 except IndexError:
1914 heavy_load = False
1915
1916 hostgroup_details = {
1917 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1918 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1919 'walphen' : 'Stichting Wireless Alphen',
1920 'westeinder' : 'Westeinder Plassen',
1921 }
1922
1923 # Convert IP to Host
1924 ip2host = {'root' : 'root'}
1925 for host in get_hostlist():
1926 datadump = get_yaml(host)
1927 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1928 for iface in get_interface_keys(datadump):
1929 if datadump[iface].has_key('autogen_gateway'):
1930 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1931
1932 # Find dependency tree based on output of lvrouted.mytree of nearest node
1933 parents = defaultdict(list)
1934 stack = ['root']
1935 prev_depth = 0
1936 for line in open('lvrouted.mytree').readlines():
1937 depth = line.count('\t')
1938 ip = line.strip().split()[0]
1939
1940 if prev_depth < depth:
1941 try:
1942 parents[ip2host[ip]].append(ip2host[stack[-1]])
1943 except KeyError as e:
1944 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1945 stack.append(ip)
1946 elif prev_depth > depth:
1947 stack = stack[:(depth - prev_depth)]
1948 elif prev_depth == depth:
1949 try:
1950 parents[ip2host[ip]].append(ip2host[stack[-1]])
1951 except KeyError as e:
1952 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1953
1954
1955 prev_depth = depth
1956 # Observe that some nodes has themself as parent or multiple parents
1957 # for now take only the first parent, other behaviour is yet to be explained
1958
1959
1960
1961 params = {
1962 'check_interval' : 5 if heavy_load else 120,
1963 'retry_interval' : 1 if heavy_load else 10,
1964 'max_check_attempts' : 10 if heavy_load else 6,
1965 'notification_interval': 120 if heavy_load else 240,
1966 }
1967
1968 print '''\
1969define host {
1970 name wleiden-node ; Default Node Template
1971 use generic-host ; Use the standard template as initial starting point
1972 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1973 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1974 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1975 notification_interval %(notification_interval)s
1976 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1977 check_command check-host-alive ; Default command to check FreeBSD hosts
1978 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1979}
1980
1981define service {
1982 name wleiden-service ; Default Service Template
1983 use generic-service ; Use the standard template as initial starting point
1984 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1985 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1986 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1987 notification_interval %(notification_interval)s
1988 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1989 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1990}
1991
1992# Please make sure to install:
1993# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1994#
1995# Recompile net-mgmt/nagios-plugins to support check_snmp
1996# make -C /usr/ports/net-mgmt/nagios-plugins
1997#
1998# Install net/bind-tools to allow v2/check_dns_wl to work:
1999# pkg install bind-tools
2000#
2001define command{
2002 command_name check_snmp_disk
2003 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2004}
2005
2006define command{
2007 command_name check_netsnmp_load
2008 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2009}
2010
2011define command{
2012 command_name check_netsnmp_proc
2013 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2014}
2015
2016define command{
2017 command_name check_by_ssh
2018 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2019}
2020
2021define command{
2022 command_name check_dns_wl
2023 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2024}
2025
2026define command{
2027 command_name check_snmp_uptime
2028 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2029}
2030
2031
2032# TDB: dhcp leases
2033# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2034
2035# TDB: internet status
2036# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2037
2038# TDB: Advanced local passive checks
2039# /usr/local/libexec/nagios/check_by_ssh
2040''' % params
2041
2042 print '''\
2043# Service Group, not displayed by default
2044define hostgroup {
2045 hostgroup_name srv_hybrid
2046 alias All Hybrid Nodes
2047 register 0
2048}
2049
2050define service {
2051 use wleiden-service
2052 hostgroup_name srv_hybrid
2053 service_description SSH
2054 check_command check_ssh
2055}
2056
2057define service {
2058 use wleiden-service,service-pnp
2059 hostgroup_name srv_hybrid
2060 service_description HTTP
2061 check_command check_http
2062}
2063
2064define service {
2065 use wleiden-service
2066 hostgroup_name srv_hybrid
2067 service_description DNS
2068 check_command check_dns_wl!"www.wirelessleiden.nl"
2069}
2070'''
2071
2072 if heavy_load:
2073 print '''\
2074define service {
2075 use wleiden-service
2076 hostgroup_name srv_hybrid
2077 service_description UPTIME
2078 check_command check_snmp_uptime
2079}
2080
2081#define service {
2082# use wleiden-service
2083# hostgroup_name srv_hybrid
2084# service_description NTP
2085# check_command check_ntp_peer
2086#}
2087
2088define service {
2089 use wleiden-service
2090 hostgroup_name srv_hybrid
2091 service_description LOAD
2092 check_command check_netsnmp_load
2093}
2094
2095define service {
2096 use wleiden-service
2097 hostgroup_name srv_hybrid
2098 service_description PROC
2099 check_command check_netsnmp_proc
2100}
2101
2102define service {
2103 use wleiden-service
2104 hostgroup_name srv_hybrid
2105 service_description DISK
2106 check_command check_snmp_disk
2107}
2108'''
2109 for node in get_hostlist():
2110 datadump = get_yaml(node)
2111 if not datadump['status'] == 'up':
2112 continue
2113 if not hostgroup_details.has_key(datadump['monitoring_group']):
2114 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2115 print '''\
2116define host {
2117 use wleiden-node,host-pnp
2118 contact_groups admins
2119 host_name %(autogen_fqdn)s
2120 address %(masterip)s
2121 hostgroups srv_hybrid,%(monitoring_group)s\
2122''' % datadump
2123 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2124 print '''\
2125 parents %(parents)s\
2126''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2127 print '''\
2128}
2129'''
2130
2131
2132 for name,alias in hostgroup_details.iteritems():
2133 print '''\
2134define hostgroup {
2135 hostgroup_name %s
2136 alias %s
2137} ''' % (name, alias)
2138
2139
2140 elif sys.argv[1] == "full-export":
2141 hosts = {}
2142 for node in get_hostlist():
2143 datadump = get_yaml(node)
2144 hosts[datadump['nodename']] = datadump
2145 print yaml.dump(hosts)
2146
2147 elif sys.argv[1] == "dns":
2148 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2149 elif sys.argv[1] == "cleanup":
2150 # First generate all datadumps
2151 datadumps = dict()
2152 ssid_to_node = dict()
2153 for host in get_hostlist():
2154 logger.info("# Processing: %s", host)
2155 # Set some boring default values
2156 datadump = { 'board' : 'UNKNOWN' }
2157 datadump.update(get_yaml(host))
2158 datadumps[datadump['nodename']] = datadump
2159
2160 (poel, errors) = make_relations()
2161 print "\n".join(["# WARNING: %s" % x for x in errors])
2162
2163 for host,datadump in datadumps.iteritems():
2164 try:
2165 # Convert all yes and no to boolean values
2166 def fix_boolean(dump):
2167 for key in dump.keys():
2168 if type(dump[key]) == dict:
2169 dump[key] = fix_boolean(dump[key])
2170 elif str(dump[key]).lower() in ["yes", "true"]:
2171 dump[key] = True
2172 elif str(dump[key]).lower() in ["no", "false"]:
2173 # Compass richting no (Noord Oost) is valid input
2174 if key != "compass": dump[key] = False
2175 return dump
2176 datadump = fix_boolean(datadump)
2177
2178 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2179 if not 'latitude' in datadump and not 'longitude' in datadump:
2180 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2181 elif 'latitude' in datadump and 'longitude' in datadump:
2182 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2183 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2184 # TODO: Compare outcome of both coordinate systems and validate against each-other
2185
2186 if datadump['nodename'].startswith('Proxy'):
2187 datadump['nodename'] = datadump['nodename'].lower()
2188
2189 for iface_key in get_interface_keys(datadump):
2190 try:
2191 # All our normal wireless cards are normal APs now
2192 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2193 datadump[iface_key]['mode'] = 'ap'
2194 # Wireless Leiden SSID have an consistent lowercase/uppercase
2195 if datadump[iface_key].has_key('ssid'):
2196 ssid = datadump[iface_key]['ssid']
2197 prefix = 'ap-WirelessLeiden-'
2198 if ssid.lower().startswith(prefix.lower()):
2199 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2200 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2201 datadump[iface_key]['mode'] = 'autogen-FIXME'
2202 if not datadump[iface_key].has_key('comment'):
2203 datadump[iface_key]['comment'] = 'autogen-FIXME'
2204
2205 if datadump[iface_key].has_key('ns_mac'):
2206 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2207
2208 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2209 datadump[iface_key] = datadump[iface_key]['desc']
2210
2211 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2212 # due to better collision avoidance.
2213 if datadump[iface_key]['type'] == '11b':
2214 datadump[iface_key]['type'] = '11g'
2215
2216 # Setting 802.11g channels to de-facto standards, to avoid
2217 # un-detected sharing with other overlapping channels
2218 #
2219 # Technically we could also use channel 13 in NL, but this is not
2220 # recommended as foreign devices might not be able to select this
2221 # channel. Secondly using 1,5,9,13 instead is going to clash with
2222 # the de-facto usage of 1,6,11.
2223 #
2224 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2225 channels_at_2400Mhz = (1,6,11)
2226 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2227 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2228 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2229 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2230
2231 # Mandatory interface keys
2232 if not datadump[iface_key].has_key('status'):
2233 datadump[iface_key]['status'] = 'planned'
2234
2235 x = datadump[iface_key]['comment']
2236 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2237
2238 # Fixing bridge_type if none is found
2239 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2240 if not 'bridge_type' in datadump[iface_key]:
2241 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2242
2243 # Making sure description works
2244 if datadump[iface_key].has_key('desc'):
2245 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2246 del datadump[iface_key]['desc']
2247 else:
2248 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2249 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2250 if response:
2251 datadump[iface_key]['comment'] = response
2252 del datadump[iface_key]['desc']
2253
2254 # Check DHCP configuration
2255 dhcp_type(datadump[iface_key])
2256
2257 # Set the compass value based on the angle between the poels
2258 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2259 my_pool = poel[network(datadump[iface_key]['ip'])]
2260 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2261 if remote_hosts:
2262 compass_target = remote_hosts[0]
2263 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2264 # TODO: Compass wanted and actual direction might differ
2265
2266 # Monitoring Group default
2267 if not 'monitoring_group' in datadump:
2268 datadump['monitoring_group'] = 'wleiden'
2269
2270 except Exception:
2271 print "# Error while processing interface %s" % iface_key
2272 raise
2273 store_yaml(datadump)
2274 except Exception:
2275 print "# Error while processing %s" % host
2276 raise
2277 elif sys.argv[1] == "list":
2278 use_fqdn = False
2279 if len(sys.argv) < 4:
2280 usage()
2281 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2282 usage()
2283 if not sys.argv[3] in ["nodes","proxies","systems"]:
2284 usage()
2285
2286 if len(sys.argv) > 4:
2287 if sys.argv[4] == "fqdn":
2288 use_fqdn = True
2289 else:
2290 usage()
2291
2292 for system in get_hostlist():
2293 datadump = get_yaml(system)
2294 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2295 continue
2296
2297 output = datadump['autogen_fqdn'] if use_fqdn else system
2298 if sys.argv[2] == "all":
2299 print output
2300 elif datadump['status'] == sys.argv[2]:
2301 print output
2302 elif sys.argv[1] == "create":
2303 if sys.argv[2] == "network.kml":
2304 print make_network_kml.make_graph()
2305 elif sys.argv[2] == "host-ips.txt":
2306 for system in get_hostlist():
2307 datadump = get_yaml(system)
2308 ips = [datadump['masterip']]
2309 for ifkey in get_interface_keys(datadump):
2310 ips.append(datadump[ifkey]['ip'].split('/')[0])
2311 print system, ' '.join(ips)
2312 elif sys.argv[2] == "host-pos.txt":
2313 for system in get_hostlist():
2314 datadump = get_yaml(system)
2315 print system, datadump['rdnap_x'], datadump['rdnap_y']
2316 elif sys.argv[2] == 'ssh_config':
2317 print '''
2318Host *.wleiden.net
2319 User root
2320
2321Host 172.16.*.*
2322 User root
2323'''
2324 for system in get_hostlist():
2325 datadump = get_yaml(system)
2326 print '''\
2327Host %s
2328 User root
2329
2330Host %s
2331 User root
2332
2333Host %s
2334 User root
2335
2336Host %s
2337 User root
2338''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2339 else:
2340 usage()
2341 else:
2342 usage()
2343 else:
2344 # Do not enable debugging for config requests as it highly clutters the output
2345 if not is_text_request():
2346 cgitb.enable()
2347 response_headers, output = process_cgi_request()
2348 print_cgi_response(response_headers, output)
2349
2350def application(environ, start_response):
2351 status = '200 OK'
2352 response_headers, output = process_cgi_request(environ)
2353 start_response(status, response_headers)
2354
2355 # Debugging only
2356 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2357 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2358 return [output]
2359
2360if __name__ == "__main__":
2361 main()
Note: See TracBrowser for help on using the repository browser.