source: genesis/tools/gformat.py@ 13867

Last change on this file since 13867 was 13863, checked in by rick, 8 years ago

Add ability to provide comments for tracking.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 77.8 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# FreeBSD:
28# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
29# Fedora:
30# yum install python-yaml pyproj proj-epsg python-jinja2
31#
32# Rick van der Zwet <info@rickvanderzwet.nl>
33#
34
35# Hack to make the script directory is also threated as a module search path.
36import sys
37import os
38sys.path.append(os.path.dirname(__file__))
39
40SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
41
42import argparse
43import cgi
44import cgitb
45import copy
46import glob
47import make_network_kml
48import math
49import pyproj
50import random
51import re
52import socket
53import string
54import subprocess
55import textwrap
56import time
57import urlparse
58
59from pprint import pprint
60from collections import defaultdict, OrderedDict
61from sys import stderr
62try:
63 import yaml
64except ImportError, e:
65 print e
66 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
67 exit(1)
68
69try:
70 from yaml import CLoader as Loader
71 from yaml import CDumper as Dumper
72except ImportError:
73 from yaml import Loader, Dumper
74
75from jinja2 import Environment, Template
76def yesorno(value):
77 return "YES" if bool(value) else "NO"
78env = Environment()
79env.filters['yesorno'] = yesorno
80def render_template(datadump, template):
81 result = env.from_string(template).render(datadump)
82 # Make it look pretty to the naked eye, as jinja templates are not so
83 # friendly when it comes to whitespace formatting
84 ## Remove extra whitespace at end of line lstrip() style.
85 result = re.sub(r'\n[\ ]+','\n', result)
86 ## Include only a single newline between an definition and a comment
87 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
88 ## Remove extra newlines after single comment
89 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
90 return result
91
92import logging
93logging.basicConfig(format='# %(levelname)s: %(message)s' )
94logger = logging.getLogger()
95logger.setLevel(logging.DEBUG)
96
97
98if os.environ.has_key('CONFIGROOT'):
99 NODE_DIR = os.environ['CONFIGROOT']
100else:
101 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
102__version__ = '$Id: gformat.py 13863 2017-05-09 06:43:17Z rick $'
103
104files = [
105 'authorized_keys',
106 'dnsmasq.conf',
107 'dhcpd.conf',
108 'rc.conf.local',
109 'resolv.conf',
110 'motd',
111 'ntp.conf',
112 'pf.hybrid.conf.local',
113 'unbound.wleiden.conf',
114 'wleiden.yaml',
115 ]
116
117# Global variables uses
118OK = 10
119DOWN = 20
120UNKNOWN = 90
121
122
123ileiden_proxies = OrderedDict()
124normal_proxies = []
125datadump_cache = {}
126interface_list_cache = {}
127rc_conf_local_cache = {}
128nameservers_cache = []
129relations_cache = None
130def clear_cache():
131 ''' Poor mans cache implementation '''
132 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
133 datadump_cache = {}
134 interface_list_cache = {}
135 rc_conf_local_cache = {}
136 ileiden_proxies = OrderedDict()
137 normal_proxies = []
138 nameservers_cache = []
139 relations_cache = None
140
141NO_DHCP = 0
142DHCP_CLIENT = 10
143DHCP_SERVER = 20
144def dhcp_type(item):
145 if not item.has_key('dhcp'):
146 return NO_DHCP
147 elif not item['dhcp']:
148 return NO_DHCP
149 elif item['dhcp'].lower() == 'client':
150 return DHCP_CLIENT
151 else:
152 # Validation Checks
153 begin,end = map(int,item['dhcp'].split('-'))
154 if begin >= end:
155 raise ValueError("DHCP Start >= DHCP End")
156 return DHCP_SERVER
157
158def etrs2rd(lat, lon):
159 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
160 p2 = pyproj.Proj(init='EPSG:28992')
161 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
162 return (RDx, RDy)
163
164def rd2etrs(RDx, RDy):
165 p1 = pyproj.Proj(init='EPSG:28992')
166 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
167 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
168 return (lat, lon)
169
170def get_yaml(item,add_version_info=True):
171 try:
172 """ Get configuration yaml for 'item'"""
173 if datadump_cache.has_key(item):
174 return datadump_cache[item].copy()
175
176 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
177 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
178 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
179
180 # Default values
181 datadump = {
182 'autogen_revision' : 'NOTFOUND',
183 'autogen_gfile' : gfile,
184 'service_proxy_ileiden' : False,
185 }
186 f = open(gfile, 'r')
187 datadump.update(yaml.load(f,Loader=Loader))
188 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
189 if datadump['nodetype'] == 'Hybrid':
190 # Some values are defined implicitly
191 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
192 datadump['service_incoming_rdr'] = True
193 # Use some boring defaults
194 defaults = {
195 'service_proxy_normal' : False,
196 'service_accesspoint' : True,
197 'service_incoming_rdr' : False,
198 'service_concentrator' : False,
199 'monitoring_group' : 'wleiden',
200 }
201 for (key,value) in defaults.iteritems():
202 if not datadump.has_key(key):
203 datadump[key] = value
204 f.close()
205
206 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
207 if add_version_info:
208 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
209 lines = p.communicate()[0].split('\n')
210 if p.returncode == 0:
211 for line in lines:
212 if line:
213 (key, value) = line.split(': ')
214 datadump["autogen_" + key.lower().replace(' ','_')] = value
215
216 # Preformat certain needed variables for formatting and push those into special object
217 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
218
219 wlan_count=0
220 try:
221 for key in get_interface_keys(datadump, True):
222 datadump[key]['autogen_ifbase'] = key.split('_')[1]
223 datadump[key]['autogen_vlan'] = False
224
225 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
226 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
227
228 if datadump[key].has_key('parent'):
229 if datadump[key].has_key('ip'):
230 raise ValueError("Interface bridge member cannot have IP assigned")
231 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
232 raise ValueError("Interface bridge member cannot have DHCP set")
233
234 if datadump[key].has_key('ip'):
235 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
236
237 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
238 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
239 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
240 wlan_count += 1
241 else:
242 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
243 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
244 datadump[key]['autogen_vlan'] = key.split('_')[2]
245 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
246 else:
247 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
248
249 except Exception:
250 print "# Error while processing interface %s" % key
251 raise
252
253 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
254 if dhcp_type(datadump[key]) == DHCP_SERVER]
255
256 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
257 datadump['autogen_item'] = item
258
259 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
260 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
261 datadump_cache[item] = datadump.copy()
262 except Exception:
263 print "# Error while processing %s" % item
264 raise
265 return datadump
266
267
268def store_yaml(datadump, header=False):
269 """ Store configuration yaml for 'item'"""
270 item = datadump['autogen_item']
271 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
272
273 output = generate_wleiden_yaml(datadump, header)
274
275 f = open(gfile, 'w')
276 f.write(output)
277 f.close()
278
279
280def network(ip):
281 addr, mask = ip.split('/')
282 # Not parsing of these folks please
283 addr = parseaddr(addr)
284 mask = int(mask)
285 network = addr & ~((1 << (32 - mask)) - 1)
286 return network
287
288
289
290def make_relations():
291 """ Process _ALL_ yaml files to get connection relations """
292 global relations_cache
293
294 if relations_cache:
295 return relations_cache
296
297 errors = []
298 poel = defaultdict(list)
299
300 for host in get_hostlist():
301 datadump = get_yaml(host)
302 try:
303 for iface_key in get_interface_keys(datadump):
304 # Bridge members has no IP assigned
305 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
306 continue
307 net_addr = network(datadump[iface_key]['ip'])
308 poel[net_addr] += [(host,datadump[iface_key].copy())]
309 except (KeyError, ValueError) as e:
310 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
311 continue
312
313 relations_cache = (poel, errors)
314 return relations_cache
315
316
317
318def valid_addr(addr):
319 """ Show which address is valid in which are not """
320 return str(addr).startswith('172.')
321
322def get_hostlist():
323 """ Combined hosts and proxy list"""
324 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
325
326def angle_between_points(lat1,lat2,long1,long2):
327 """
328 Return Angle in radians between two GPS coordinates
329 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
330 """
331 dy = lat2 - lat1
332 dx = math.cos(lat1)*(long2 - long1)
333 angle = math.atan2(dy,dx)
334 return angle
335
336
337
338def angle_to_cd(angle):
339 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
340
341 # For easy conversion get positive degree
342 degrees = math.degrees(angle)
343 abs_degrees = 360 + degrees if degrees < 0 else degrees
344
345 # Numbers can be confusing calculate from the 4 main directions
346 p = 22.5
347 if abs_degrees < p:
348 cd = "n"
349 elif abs_degrees < (90 - p):
350 cd = "no"
351 elif abs_degrees < (90 + p):
352 cd = "o"
353 elif abs_degrees < (180 - p):
354 cd = "zo"
355 elif abs_degrees < (180 + p):
356 cd = "z"
357 elif abs_degrees < (270 - p):
358 cd = "zw"
359 elif abs_degrees < (270 + p):
360 cd = "w"
361 elif abs_degrees < (360 - p):
362 cd = "nw"
363 else:
364 cd = "n"
365 return cd
366
367
368
369def cd_between_hosts(hostA, hostB, datadumps):
370 # Using RDNAP coordinates
371 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
372 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
373 return angle_to_cd(math.atan2(dx,dy))
374
375 # GPS coordinates seems to fail somehow
376 #latA = float(datadumps[hostA]['latitude'])
377 #latB = float(datadumps[hostB]['latitude'])
378 #lonA = float(datadumps[hostA]['longitude'])
379 #lonB = float(datadumps[hostB]['longitude'])
380 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
381
382
383def generate_title(nodelist):
384 """ Main overview page """
385 items = {'root' : "." }
386 def fl(spaces, line):
387 return (' ' * spaces) + line + '\n'
388
389 output = """
390<html>
391 <head>
392 <title>Wireless leiden Configurator - GFormat</title>
393 <style type="text/css">
394 th {background-color: #999999}
395 tr:nth-child(odd) {background-color: #cccccc}
396 tr:nth-child(even) {background-color: #ffffff}
397 th, td {padding: 0.1em 1em}
398 </style>
399 </head>
400 <body>
401 <center>
402 <form type="GET" action="%(root)s">
403 <input type="hidden" name="action" value="update">
404 <input type="submit" value="Update Configuration Database (SVN)">
405 </form>
406 <table>
407 <caption><h3>Wireless Leiden Configurator</h3></caption>
408 """ % items
409
410 for node in nodelist:
411 items['node'] = node
412 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
413 for config in files:
414 items['config'] = config
415 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
416 output += fl(5, "</tr>")
417 output += """
418 </table>
419 <hr />
420 <em>%s</em>
421 </center>
422 </body>
423</html>
424 """ % __version__
425
426 return output
427
428
429
430def generate_node(node):
431 """ Print overview of all files available for node """
432 return "\n".join(files)
433
434def generate_node_overview(host):
435 """ Print overview of all files available for node """
436 datadump = get_yaml(host)
437 params = { 'host' : host }
438 output = "<em><a href='..'>Back to overview</a></em><hr />"
439 output += "<h2>Available files:</h2><ul>"
440 for cf in files:
441 params['cf'] = cf
442 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
443 output += "</ul>"
444
445 # Generate and connection listing
446 output += "<h2>Connected To:</h2><ul>"
447 (poel, errors) = make_relations()
448 for network, hosts in poel.iteritems():
449 if host in [x[0] for x in hosts]:
450 if len(hosts) == 1:
451 # Single not connected interface
452 continue
453 for remote,ifacedump in hosts:
454 if remote == host:
455 # This side of the interface
456 continue
457 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
458 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
459 output += "</ul>"
460 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
461
462 output += "<hr /><em><a href='..'>Back to overview</a></em>"
463 return output
464
465
466def generate_header(datadump, ctag="#"):
467 return """\
468%(ctag)s
469%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
470%(ctag)s
471""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
472
473
474
475def parseaddr(s):
476 """ Process IPv4 CIDR notation addr to a (binary) number """
477 f = s.split('.')
478 return (long(f[0]) << 24L) + \
479 (long(f[1]) << 16L) + \
480 (long(f[2]) << 8L) + \
481 long(f[3])
482
483
484
485def showaddr(a):
486 """ Display IPv4 addr in (dotted) CIDR notation """
487 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
488
489
490def is_member(ip, mask, canidate):
491 """ Return True if canidate is part of ip/mask block"""
492 ip_addr = parseaddr(ip)
493 ip_canidate = parseaddr(canidate)
494 mask = int(mask)
495 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
496 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
497 return ip_addr == ip_canidate
498
499
500
501def cidr2netmask(netmask):
502 """ Given a 'netmask' return corresponding CIDR """
503 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
504
505def get_network(addr, mask):
506 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
507
508
509def generate_dhcpd_conf(datadump):
510 """ Generate config file '/usr/local/etc/dhcpd.conf """
511 # Redundency support, in cause local DNS server is not running/responding.
512 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
513 output = generate_header(datadump)
514 output += Template("""\
515# option definitions common to all supported networks...
516option domain-name "dhcp.{{ autogen_fqdn }}";
517
518default-lease-time 600;
519max-lease-time 7200;
520
521# Use this to enble / disable dynamic dns updates globally.
522#ddns-update-style none;
523
524# If this DHCP server is the official DHCP server for the local
525# network, the authoritative directive should be uncommented.
526authoritative;
527
528# Use this to send dhcp log messages to a different log file (you also
529# have to hack syslog.conf to complete the redirection).
530log-facility local7;
531
532#
533# Interface definitions
534#
535\n\n""").render(datadump)
536
537
538 # TODO: Use textwrap.fill instead
539 def indent(text, count):
540 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
541
542 # Process DHCP blocks
543 dhcp_out = defaultdict(list)
544 for iface_key in get_interface_keys(datadump):
545 ifname = datadump[iface_key]['autogen_ifbase']
546 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
547 if not datadump[iface_key].has_key('comment'):
548 datadump[iface_key]['comment'] = None
549
550 if not datadump[iface_key].has_key('ip'):
551 continue
552
553 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
554
555 (addr, mask) = datadump[iface_key]['ip'].split('/')
556 datadump[iface_key]['autogen_addr'] = addr
557 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
558 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
559
560 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
561 dhcp_out[groupif].append(textwrap.dedent("""\
562 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
563 ### not autoritive
564 }
565 """ % datadump[iface_key]))
566 continue
567
568 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
569 dhcp_part = ".".join(addr.split('.')[0:3])
570 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
571 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
572 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
573
574 # Assume the first 10 IPs could be used for static entries
575 if 'no_portal' in datadump:
576 fixed = 5
577 for mac in datadump['no_portal']:
578 dhcp_out[groupif].append(textwrap.dedent("""\
579 host fixed-%(ifname)s-%(fixed)s {
580 hardware ethernet %(mac)s;
581 fixed-address %(prefix)s.%(fixed)s;
582 }
583 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
584 fixed += 1
585
586 dhcp_out[groupif].append(textwrap.dedent("""\
587 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
588 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
589 option routers %(autogen_addr)s;
590 option domain-name-servers %(autogen_dns_servers)s;
591
592 }
593 """ % datadump[iface_key]))
594
595 # Output the blocks in groups
596 for ifname,value in sorted(dhcp_out.iteritems()):
597 if len(value) > 2:
598 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
599 else:
600 output += ''.join(value) + "\n\n"
601 return output
602
603
604
605def generate_dnsmasq_conf(datadump):
606 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
607 output = generate_header(datadump)
608 output += Template("""\
609# DHCP server options
610dhcp-authoritative
611dhcp-fqdn
612domain=dhcp.{{ autogen_fqdn }}
613domain-needed
614expand-hosts
615log-async=100
616
617# Low memory footprint
618cache-size=10000
619
620\n""").render(datadump)
621
622 for iface_key in get_interface_keys(datadump):
623 if not datadump[iface_key].has_key('comment'):
624 datadump[iface_key]['comment'] = None
625 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
626
627 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
628 output += "# not autoritive\n\n"
629 continue
630
631 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
632 (ip, cidr) = datadump[iface_key]['ip'].split('/')
633 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
634
635 dhcp_part = ".".join(ip.split('.')[0:3])
636 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
637 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
638 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
639
640 return output
641
642
643class AutoVivification(dict):
644 """Implementation of perl's autovivification feature."""
645 def __getitem__(self, item):
646 try:
647 return dict.__getitem__(self, item)
648 except KeyError:
649 value = self[item] = type(self)()
650 return value
651
652def make_interface_list(datadump):
653 if interface_list_cache.has_key(datadump['autogen_item']):
654 return (interface_list_cache[datadump['autogen_item']])
655 # lo0 configuration:
656 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
657 # - masterip is special as it needs to be assigned to at
658 # least one interface, so if not used assign to lo0
659 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
660 vlan_list = defaultdict(list)
661 bridge_list = defaultdict(list)
662 flags_if = AutoVivification()
663 dhclient_if = {'lo0' : False}
664
665 # XXX: Find some way of send this output nicely
666 output = ''
667
668 masterip_used = False
669 for iface_key in get_interface_keys(datadump):
670 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
671 masterip_used = True
672 break
673 if not masterip_used:
674 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
675
676 if 'serviceid' in datadump:
677 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
678
679 for iface_key in get_interface_keys(datadump):
680 ifacedump = datadump[iface_key]
681 ifname = ifacedump['autogen_ifname']
682
683 # If defined as vlan interface
684 if ifacedump['autogen_vlan']:
685 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
686
687 # If defined as bridge interface
688 if ifacedump['autogen_bridge_member']:
689 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
690
691 # Flag dhclient is possible
692 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
693 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
694
695 # Ethernet address
696 if ifacedump.has_key('ether'):
697 flags_if[ifname]['ether'] = ifacedump['ether']
698
699 # Add interface IP to list
700 if ifacedump.has_key('ip'):
701 item = (ifacedump['ip'], ifacedump['comment'])
702 if addrs_list.has_key(ifname):
703 addrs_list[ifname].append(item)
704 else:
705 addrs_list[ifname] = [item]
706
707 # Alias only needs IP assignment for now, this might change if we
708 # are going to use virtual accesspoints
709 if "alias" in iface_key:
710 continue
711
712 # XXX: Might want to deduct type directly from interface name
713 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
714 # Default to station (client) mode
715 ifacedump['autogen_wlanmode'] = "sta"
716 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
717 ifacedump['autogen_wlanmode'] = "ap"
718
719 if not ifacedump.has_key('channel'):
720 if ifacedump['type'] == '11a':
721 ifacedump['channel'] = 36
722 else:
723 ifacedump['channel'] = 1
724
725 # Allow special hacks at the back like wds and stuff
726 if not ifacedump.has_key('extra'):
727 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
728 else:
729 ifacedump['autogen_extra'] = ifacedump['extra']
730
731 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
732
733 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
734 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
735 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
736 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
737 output += "\n"
738
739 elif ifacedump['type'] in ['ethernet', 'eth']:
740 # No special config needed besides IP
741 pass
742 elif ifacedump['type'] in ['vlan']:
743 # VLAN member has no special configuration
744 pass
745 else:
746 assert False, "Unknown type " + ifacedump['type']
747
748 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
749 interface_list_cache[datadump['autogen_item']] = store
750 return(store)
751
752
753
754def create_proxies_list():
755 if not ileiden_proxies or not normal_proxies:
756 # Placeholder for to-be-installed proxies, this will avoid updating the all
757 # nodes to include this new machine, yet due to an unbound issue, this list
758 # has to be kept small.
759
760 for i in range(1,20):
761 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
762
763 for host in get_hostlist():
764 hostdump = get_yaml(host)
765 if hostdump['status'] == 'up':
766 if hostdump['service_proxy_ileiden']:
767 ileiden_proxies[hostdump['serviceid']] = hostdump
768 if hostdump['service_proxy_normal']:
769 normal_proxies.append(hostdump)
770
771
772
773def generate_rc_conf_local(datadump):
774 """ Generate configuration file '/etc/rc.conf.local' """
775 item = datadump['autogen_item']
776 if rc_conf_local_cache.has_key(item):
777 return rc_conf_local_cache[item]
778
779 if not datadump.has_key('ileiden'):
780 datadump['autogen_ileiden_enable'] = False
781 else:
782 datadump['autogen_ileiden_enable'] = datadump['ileiden']
783
784 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
785
786 create_proxies_list()
787 datadump['autogen_ileiden_proxies'] = ileiden_proxies
788 datadump['autogen_normal_proxies'] = normal_proxies
789 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
790 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
791 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
792 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
793
794 output = generate_header(datadump, "#");
795 output += render_template(datadump, """\
796hostname='{{ autogen_fqdn }}'
797location='{{ location }}'
798nodetype="{{ nodetype }}"
799
800#
801# Configured listings
802#
803captive_portal_whitelist=""
804{% if nodetype == "Proxy" %}
805#
806# Proxy Configuration
807#
808{% if gateway and service_proxy_ileiden -%}
809defaultrouter="{{ gateway }}"
810{% else -%}
811#defaultrouter="NOTSET"
812{% endif -%}
813internalif="{{ internalif }}"
814ileiden_enable="{{ autogen_ileiden_enable }}"
815gateway_enable="{{ autogen_ileiden_enable }}"
816pf_enable="yes"
817pf_rules="/etc/pf.conf"
818{% if autogen_ileiden_enable -%}
819pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
820lvrouted_enable="{{ autogen_ileiden_enable }}"
821lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
822{% else -%}
823pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
824{% endif -%}
825{% if internalroute -%}
826static_routes="wleiden"
827route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
828{% endif -%}
829
830{% elif nodetype == "Hybrid" %}
831 #
832 # Hybrid Configuration
833 #
834 list_ileiden_proxies="
835 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
836 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
837 {% endfor -%}
838 "
839 list_normal_proxies="
840 {% for item in autogen_normal_proxies -%}
841 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
842 {% endfor -%}
843 "
844
845 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
846 externalif="{{ externalif|default('vr0', true) }}"
847 masterip="{{ masterip }}"
848
849 {% if gateway and service_proxy_ileiden %}
850 defaultrouter="{{ gateway }}"
851 {% else %}
852 #defaultrouter="NOTSET"
853 {% endif %}
854
855 #
856 # Defined services
857 #
858 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
859 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
860 service_accesspoint="{{ service_accesspoint|yesorno }}"
861 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
862 service_concentrator="{{ service_concentrator|yesorno }}"
863
864 {% if service_proxy_ileiden %}
865 pf_rules="/etc/pf.hybrid.conf"
866 {% if service_concentrator %}
867 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
868 {% else %}
869 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
870 {% endif %}
871 pf_flags="$pf_flags -D publicnat=80,443"
872 lvrouted_flags="$lvrouted_flags -g"
873 {% elif service_proxy_normal or service_incoming_rdr %}
874 pf_rules="/etc/pf.hybrid.conf"
875 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
876 pf_flags="$pf_flags -D publicnat=0"
877 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
878 named_setfib="1"
879 tinyproxy_setfib="1"
880 dnsmasq_setfib="1"
881 sshd_setfib="1"
882 {% else %}
883 named_auto_forward_only="YES"
884 pf_rules="/etc/pf.node.conf"
885 pf_flags=""
886 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
887 {% endif %}
888 {% if service_concentrator %}
889 # Do mind installing certificates is NOT done automatically for security reasons
890 openvpn_enable="YES"
891 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
892 {% endif %}
893
894 {% if service_proxy_normal %}
895 tinyproxy_enable="yes"
896 {% else %}
897 pen_wrapper_enable="yes"
898 {% endif %}
899
900 {% if service_accesspoint %}
901 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
902 {% endif %}
903
904 {% if board == "ALIX2" %}
905 #
906 # ''Fat'' configuration, board has 256MB RAM
907 #
908 dnsmasq_enable="NO"
909 named_enable="YES"
910 unbound_enable="YES"
911 {% if autogen_dhcp_interfaces -%}
912 dhcpd_enable="YES"
913 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
914 {% endif -%}
915 {% elif board == "apu1d" %}
916 #
917 # ''Fat'' configuration, board has 1024MB RAM
918 #
919 dnsmasq_enable="NO"
920 local_unbound_enable="YES"
921 {% if autogen_dhcp_interfaces -%}
922 dhcpd_enable="YES"
923 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
924 {% endif -%}
925 {% endif -%}
926{% endif %}
927
928#
929# Script variables
930#
931attached_devices="{{ autogen_attached_devices|join(' ') }}"
932neighbours="{{ autogen_neighbours|join(' ') }}"
933
934
935#
936# Interface definitions
937#\n
938""")
939
940 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
941 for iface, vlans in sorted(vlan_list.items()):
942 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
943
944 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
945 for iface in sorted(vlan_list.keys()):
946 if not iface in addrs_list.keys():
947 output += "ifconfig_%s='up'\n" % iface
948
949 output += "\n"
950
951 # Bridge configuration:
952 if bridge_list.keys():
953 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
954
955 for iface in bridge_list.keys():
956 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
957
958 # Bridge member interfaces not containing a configuration should be marked active explcitly.
959 for _,members in bridge_list.items():
960 for iface in members:
961 if not iface in addrs_list.keys():
962 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
963
964 output += "\n"
965
966 # Details like SSID
967 if extra_ouput:
968 output += extra_ouput.strip() + "\n"
969
970 # Print IP address which needs to be assigned over here
971 output += "\n"
972 for iface,addrs in sorted(addrs_list.iteritems()):
973 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
974 output += "# %s || %s || %s\n" % (iface, addr, comment)
975
976 # Write DHCLIENT entry
977 if iface in dhclient_if and dhclient_if[iface]:
978 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
979 continue
980
981 # Make sure the external address is always first as this is needed in the
982 # firewall setup
983 addrs = sorted(
984 [x for x in addrs if not '0.0.0.0' in x[0]],
985 key=lambda x: x[0].split('.')[0],
986 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
987 )
988
989 idx_offset = 0
990 # Set MAC is required
991 if flags_if[iface].has_key('ether'):
992 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
993 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
994 idx_offset += 1
995 else:
996 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
997
998 for idx, addr in enumerate(addrs[1:]):
999 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1000
1001 output += "\n"
1002
1003 rc_conf_local_cache[datadump['autogen_item']] = output
1004 return output
1005
1006
1007
1008def get_all_configs():
1009 """ Get dict with key 'host' with all configs present """
1010 configs = dict()
1011 for host in get_hostlist():
1012 datadump = get_yaml(host)
1013 configs[host] = datadump
1014 return configs
1015
1016
1017def get_interface_keys(config, extra=False):
1018 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1019 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1020 if extra == False:
1021 return filter(lambda x: not "extra" in x, elems)
1022 else:
1023 return elems
1024
1025
1026def get_used_ips(configs):
1027 """ Return array of all IPs used in config files"""
1028 ip_list = []
1029 for config in configs:
1030 ip_list.append(config['masterip'])
1031 if 'serviceid' in config:
1032 ip_list.append(config['serviceid'])
1033 for iface_key in get_interface_keys(config, True):
1034 l = config[iface_key]['ip']
1035 addr, mask = l.split('/')
1036 # Special case do not process
1037 if valid_addr(addr):
1038 ip_list.append(addr)
1039 else:
1040 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1041 return sorted(ip_list)
1042
1043
1044
1045def get_nameservers(max_servers=None):
1046 if nameservers_cache:
1047 return nameservers_cache[0:max_servers]
1048
1049 for host in get_hostlist():
1050 hostdump = get_yaml(host)
1051 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1052 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
1053
1054 return nameservers_cache[0:max_servers]
1055
1056
1057def get_neighbours(datadump):
1058 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1059
1060 (poel, errors) = make_relations()
1061 table = []
1062 for iface,addrs in sorted(addrs_list.iteritems()):
1063 if iface in ['lo0']:
1064 continue
1065
1066 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1067 if not addr.startswith('172.'):
1068 # Avoid listing internet connections as pool
1069 continue
1070 for neighbour in poel[network(addr)]:
1071 if neighbour[0] != datadump['autogen_item']:
1072 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1073 return table
1074
1075
1076def get_attached_devices(datadump, url=False):
1077 table = []
1078 for iface_key in get_interface_keys(datadump, True):
1079 ifacedump = datadump[iface_key]
1080
1081 if not ifacedump.has_key('ns_ip'):
1082 continue
1083
1084 x_ip = ifacedump['ns_ip'].split('/')[0]
1085
1086 if 'mode' in ifacedump:
1087 x_mode = ifacedump['mode']
1088 else:
1089 x_mode = 'unknown'
1090
1091 if 'bridge_type' in ifacedump:
1092 device_type = ifacedump['bridge_type']
1093 else:
1094 device_type = 'Unknown'
1095
1096 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1097 return table
1098
1099
1100def generate_resolv_conf(datadump):
1101 """ Generate configuration file '/etc/resolv.conf' """
1102 # XXX: This should properly going to be an datastructure soon
1103 datadump['autogen_header'] = generate_header(datadump, "#")
1104 datadump['autogen_edge_nameservers'] = ''
1105
1106
1107 for masterip,realname in get_nameservers():
1108 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1109
1110 return Template("""\
1111{{ autogen_header }}
1112search wleiden.net
1113
1114# Try local (cache) first
1115nameserver 127.0.0.1
1116
1117{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1118nameserver 8.8.8.8 # Google Public NameServer
1119nameserver 64.6.64.6 # Verisign Public NameServer
1120{% else -%}
1121# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1122{{ autogen_edge_nameservers }}
1123{% endif -%}
1124""").render(datadump)
1125
1126
1127
1128def generate_ntp_conf(datadump):
1129 """ Generate configuration file '/etc/ntp.conf' """
1130 # XXX: This should properly going to be an datastructure soon
1131
1132 datadump['autogen_header'] = generate_header(datadump, "#")
1133 datadump['autogen_ntp_servers'] = ''
1134 for host in get_hostlist():
1135 hostdump = get_yaml(host)
1136 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1137 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1138
1139 return Template("""\
1140{{ autogen_header }}
1141
1142{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1143# Machine hooked to internet.
1144server 0.nl.pool.ntp.org iburst maxpoll 9
1145server 1.nl.pool.ntp.org iburst maxpoll 9
1146server 2.nl.pool.ntp.org iburst maxpoll 9
1147server 3.nl.pool.ntp.org iburst maxpoll 9
1148{% else -%}
1149# Local Wireless Leiden NTP Servers.
1150server 0.pool.ntp.wleiden.net iburst maxpoll 9
1151server 1.pool.ntp.wleiden.net iburst maxpoll 9
1152server 2.pool.ntp.wleiden.net iburst maxpoll 9
1153server 3.pool.ntp.wleiden.net iburst maxpoll 9
1154
1155# All the configured NTP servers
1156{{ autogen_ntp_servers }}
1157{% endif %}
1158
1159# If a server loses sync with all upstream servers, NTP clients
1160# no longer follow that server. The local clock can be configured
1161# to provide a time source when this happens, but it should usually
1162# be configured on just one server on a network. For more details see
1163# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1164# The use of Orphan Mode may be preferable.
1165#
1166server 127.127.1.0
1167fudge 127.127.1.0 stratum 10
1168""").render(datadump)
1169
1170
1171def generate_pf_hybrid_conf_local(datadump):
1172 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1173 datadump['autogen_header'] = generate_header(datadump, "#")
1174 if datadump['service_incoming_rdr']:
1175 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1176 return Template("""\
1177{{ autogen_header }}
1178
1179# Redirect some internal facing services outside (7)
1180# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1181{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1182rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "-%14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1183{% endfor -%}
1184# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1185{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1186rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }} # {{ comment }}
1187{% endfor -%}
1188""").render(datadump)
1189
1190def generate_unbound_wleiden_conf(datadump):
1191 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1192 datadump['autogen_header'] = generate_header(datadump, "#")
1193
1194 autogen_ips = []
1195 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1196 for iface,addrs in sorted(addrs_list.iteritems()):
1197 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1198 if addr.startswith('172'):
1199 autogen_ips.append((addr.split('/')[0], comment))
1200 datadump['autogen_ips'] = autogen_ips
1201
1202 create_proxies_list()
1203 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1204 return Template("""\
1205{{ autogen_header }}
1206
1207server:
1208{%- for ip,comment in autogen_ips %}
1209 interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1210{%- endfor %}
1211
1212forward-zone:
1213 name: '.'
1214{%- if service_proxy_ileiden %}
1215 forward-addr: 8.8.8.8 # Google DNS A
1216 forward-addr: 8.8.4.4 # Google DNS B
1217 forward-addr: 208.67.222.222 # OpenDNS DNS A
1218 forward-addr: 208.67.220.220 # OpenDNS DNS B
1219{% else -%}
1220{% for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1221 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1222{%- endfor %}
1223{% endif -%}
1224""").render(datadump)
1225
1226def generate_motd(datadump):
1227 """ Generate configuration file '/etc/motd' """
1228 output = Template("""\
1229FreeBSD run ``service motd onestart'' to make me look normal
1230
1231 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1232 Loc: {{ location }}
1233
1234Services:
1235{% if board == "ALIX2" -%}
1236{{" -"}} Core Node ({{ board }})
1237{% else -%}
1238{{" -"}} Hulp Node ({{ board }})
1239{% endif -%}
1240{% if service_proxy_normal -%}
1241{{" -"}} Normal Proxy
1242{% endif -%}
1243{% if service_proxy_ileiden -%}
1244{{" -"}} iLeiden Proxy
1245{% endif -%}
1246{% if service_incoming_rdr -%}
1247{{" -"}} Incoming port redirects
1248{% endif %}
1249Interlinks:\n
1250""").render(datadump)
1251
1252
1253 def make_table(table):
1254 if not table:
1255 return " - none\n"
1256 else:
1257 lines = ""
1258 col_width = [max(len(x) for x in col) for col in zip(*table)]
1259 for row in table:
1260 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1261 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1262 return lines
1263
1264 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1265 table = []
1266 for iface,addrs in sorted(addrs_list.iteritems()):
1267 if iface in ['lo0']:
1268 continue
1269 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1270 table.append((iface, addr, comment))
1271
1272 output += make_table(table)
1273 output += '\n'
1274 output += """\
1275Attached devices:
1276"""
1277 output += make_table(get_attached_devices(datadump, url=True))
1278 output += '\n'
1279 output += """\
1280Available neighbours:
1281"""
1282 output += make_table(get_neighbours(datadump))
1283
1284 return output
1285
1286
1287def format_yaml_value(value):
1288 """ Get yaml value in right syntax for outputting """
1289 if isinstance(value,str):
1290 output = '"%s"' % value
1291 else:
1292 output = value
1293 return output
1294
1295
1296
1297def format_wleiden_yaml(datadump):
1298 """ Special formatting to ensure it is editable"""
1299 output = "# Genesis config yaml style\n"
1300 output += "# vim:ts=2:et:sw=2:ai\n"
1301 output += "#\n"
1302 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1303 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1304 if key == 'rdr_rules':
1305 output += '%-10s:\n' % 'rdr_rules'
1306 for rdr_rule in datadump[key]:
1307 output += '- %s\n' % rdr_rule
1308 else:
1309 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1310
1311 output += "\n\n"
1312
1313 # Format (key, required)
1314 key_order = (
1315 ('comment', True),
1316 ('parent', False),
1317 ('ip', False),
1318 ('ether', False),
1319 ('desc', True),
1320 ('sdesc', True),
1321 ('mode', True),
1322 ('type', True),
1323 ('extra_type', False),
1324 ('channel', False),
1325 ('ssid', False),
1326 ('wlan_mac', False),
1327 ('dhcp', True),
1328 ('compass', False),
1329 ('distance', False),
1330 ('ns_ip', False),
1331 ('repeater_ip', False),
1332 ('bullet2_ip', False),
1333 ('ns_mac', False),
1334 ('bullet2_mac', False),
1335 ('ns_type', False),
1336 ('bridge_type', False),
1337 ('status', True),
1338 )
1339
1340 for iface_key in sorted(iface_keys):
1341 try:
1342 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1343 if remainder:
1344 raise KeyError("invalid keys: %s" % remainder)
1345
1346 output += "%s:\n" % iface_key
1347 for key,required in key_order:
1348 if datadump[iface_key].has_key(key):
1349 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1350 output += "\n\n"
1351 except Exception:
1352 print "# Error while processing interface %s" % iface_key
1353 raise
1354
1355 return output
1356
1357
1358
1359def generate_wleiden_yaml(datadump, header=True):
1360 """ Generate (petty) version of wleiden.yaml"""
1361 output = generate_header(datadump, "#") if header else ''
1362
1363 for key in datadump.keys():
1364 if key.startswith('autogen_'):
1365 del datadump[key]
1366 # Interface autogen cleanups
1367 elif type(datadump[key]) == dict:
1368 for key2 in datadump[key].keys():
1369 if key2.startswith('autogen_'):
1370 del datadump[key][key2]
1371
1372 output += format_wleiden_yaml(datadump)
1373 return output
1374
1375def generate_nanostation_config(datadump, iface, ns_type):
1376 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1377 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1378
1379 datadump.update(datadump['iface_%s' % iface])
1380
1381 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1382
1383def generate_yaml(datadump):
1384 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1385
1386
1387
1388def generate_config(node, config, datadump=None):
1389 """ Print configuration file 'config' of 'node' """
1390 output = ""
1391 try:
1392 # Load config file
1393 if datadump == None:
1394 datadump = get_yaml(node)
1395
1396 if config == 'wleiden.yaml':
1397 output += generate_wleiden_yaml(datadump)
1398 elif config == 'authorized_keys':
1399 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1400 output += f.read()
1401 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1402 # Fetch local keys if existing
1403 if os.path.exists(node_keys):
1404 output += open(node_keys, 'r').read()
1405 f.close()
1406 elif config == 'dnsmasq.conf':
1407 output += generate_dnsmasq_conf(datadump)
1408 elif config == 'dhcpd.conf':
1409 output += generate_dhcpd_conf(datadump)
1410 elif config == 'rc.conf.local':
1411 output += generate_rc_conf_local(datadump)
1412 elif config == 'resolv.conf':
1413 output += generate_resolv_conf(datadump)
1414 elif config == 'ntp.conf':
1415 output += generate_ntp_conf(datadump)
1416 elif config == 'motd':
1417 output += generate_motd(datadump)
1418 elif config == 'pf.hybrid.conf.local':
1419 output += generate_pf_hybrid_conf_local(datadump)
1420 elif config == 'unbound.wleiden.conf':
1421 output += generate_unbound_wleiden_conf(datadump)
1422 elif config.startswith('vr'):
1423 interface, ns_type = config.strip('.yaml').split('-')
1424 output += generate_nanostation_config(datadump, interface, ns_type)
1425 else:
1426 assert False, "Config not found!"
1427 except IOError, e:
1428 output += "[ERROR] Config file not found"
1429 return output
1430
1431
1432
1433def process_cgi_request(environ=os.environ):
1434 """ When calling from CGI """
1435 response_headers = []
1436 content_type = 'text/plain'
1437
1438 # Update repository if requested
1439 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1440 if form and form.has_key("action") and "update" in form["action"]:
1441 output = "[INFO] Updating subverion, please wait...\n"
1442 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1443 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1444 output += "[INFO] All done, redirecting in 5 seconds"
1445 response_headers += [
1446 ('Refresh', '5; url=.'),
1447 ]
1448 reload_cache()
1449 else:
1450 base_uri = environ['PATH_INFO']
1451 uri = base_uri.strip('/').split('/')
1452
1453 output = "Template Holder"
1454 if base_uri.endswith('/create/network.kml'):
1455 content_type='application/vnd.google-earth.kml+xml'
1456 output = make_network_kml.make_graph()
1457 elif base_uri.endswith('/api/get/nodeplanner.json'):
1458 content_type='application/json'
1459 output = make_network_kml.make_nodeplanner_json()
1460 elif not uri[0]:
1461 if is_text_request(environ):
1462 output = '\n'.join(get_hostlist())
1463 else:
1464 content_type = 'text/html'
1465 output = generate_title(get_hostlist())
1466 elif len(uri) == 1:
1467 if is_text_request(environ):
1468 output = generate_node(uri[0])
1469 else:
1470 content_type = 'text/html'
1471 output = generate_node_overview(uri[0])
1472 elif len(uri) == 2:
1473 output = generate_config(uri[0], uri[1])
1474 else:
1475 assert False, "Invalid option"
1476
1477 # Return response
1478 response_headers += [
1479 ('Content-type', content_type),
1480 ('Content-Length', str(len(output))),
1481 ]
1482 return(response_headers, str(output))
1483
1484
1485def make_dns(output_dir = 'dns', external = False):
1486 items = dict()
1487
1488 # hostname is key, IP is value
1489 wleiden_zone = defaultdict(list)
1490 wleiden_cname = dict()
1491
1492 pool = dict()
1493 for node in get_hostlist():
1494 datadump = get_yaml(node)
1495
1496 fqdn = datadump['nodename']
1497
1498 if datadump.has_key('rdr_host'):
1499 remote_target = datadump['rdr_host']
1500 elif datadump.has_key('remote_access') and datadump['remote_access']:
1501 remote_target = datadump['remote_access'].split(':')[0]
1502 else:
1503 remote_target = None
1504
1505 if remote_target:
1506 try:
1507 parseaddr(remote_target)
1508 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1509 except (IndexError, ValueError):
1510 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1511
1512
1513 wleiden_zone[fqdn].append((datadump['masterip'], True))
1514
1515 # Hacking to get proper DHCP IPs and hostnames
1516 for iface_key in get_interface_keys(datadump):
1517 iface_name = iface_key.replace('_','-')
1518 if 'ip' in datadump[iface_key]:
1519 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1520 try:
1521 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1522 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1523 dhcp_part = ".".join(ip.split('.')[0:3])
1524 if ip != datadump['masterip']:
1525 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1526 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1527 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1528 except (AttributeError, ValueError, KeyError):
1529 # First push it into a pool, to indentify the counter-part later on
1530 addr = parseaddr(ip)
1531 cidr = int(cidr)
1532 addr = addr & ~((1 << (32 - cidr)) - 1)
1533 if pool.has_key(addr):
1534 pool[addr] += [(iface_name, fqdn, ip)]
1535 else:
1536 pool[addr] = [(iface_name, fqdn, ip)]
1537 continue
1538
1539
1540
1541 # WL uses an /29 to configure an interface. IP's are ordered like this:
1542 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1543
1544 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1545
1546 # Automatic naming convention of interlinks namely 2 + remote.lower()
1547 for (key,value) in pool.iteritems():
1548 # Make sure they are sorted from low-ip to high-ip
1549 value = sorted(value, key=lambda x: parseaddr(x[2]))
1550
1551 if len(value) == 1:
1552 (iface_name, fqdn, ip) = value[0]
1553 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1554
1555 # Device DNS names
1556 if 'cnode' in fqdn.lower():
1557 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1558 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1559
1560 elif len(value) == 2:
1561 (a_iface_name, a_fqdn, a_ip) = value[0]
1562 (b_iface_name, b_fqdn, b_ip) = value[1]
1563 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1564 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1565
1566 # Device DNS names
1567 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1568 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1569 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1570 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1571 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1572 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1573 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1574
1575 else:
1576 pool_members = [k[1] for k in value]
1577 for item in value:
1578 (iface_name, fqdn, ip) = item
1579 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1580
1581 # Include static DNS entries
1582 # XXX: Should they override the autogenerated results?
1583 # XXX: Convert input to yaml more useable.
1584 # Format:
1585 ##; this is a comment
1586 ## roomburgh=Roomburgh1
1587 ## apkerk1.Vosko=172.17.176.8 ;this as well
1588 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1589
1590 # Hack to allow special entries, for development
1591 wleiden_raw = {}
1592
1593 for line in dns_list:
1594 reverse = False
1595 k, items = line.items()[0]
1596 if type(items) == dict:
1597 if items.has_key('reverse'):
1598 reverse = items['reverse']
1599 items = items['a']
1600 else:
1601 items = items['cname']
1602 items = [items] if type(items) != list else items
1603 for item in items:
1604 if item.startswith('IN '):
1605 wleiden_raw[k] = item
1606 elif valid_addr(item):
1607 wleiden_zone[k].append((item, reverse))
1608 else:
1609 wleiden_cname[k] = item
1610
1611 # Hack to get dynamic pool listing
1612 def chunks(l, n):
1613 return [l[i:i+n] for i in range(0, len(l), n)]
1614
1615 ntp_servers = [x[0] for x in get_nameservers()]
1616 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1617 for ntp_server in chunk:
1618 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1619
1620 details = dict()
1621 # 24 updates a day allowed
1622 details['serial'] = time.strftime('%Y%m%d%H')
1623
1624 if external:
1625 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1626 else:
1627 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1628
1629 details['master'] = dns_masters[0]
1630 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1631
1632 dns_header = '''
1633$TTL 3h
1634%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1635 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1636
1637%(ns_servers)s
1638 \n'''
1639
1640
1641 if not os.path.isdir(output_dir):
1642 os.makedirs(output_dir)
1643 details['zone'] = 'wleiden.net'
1644 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1645 f.write(dns_header % details)
1646
1647 for host,items in wleiden_zone.iteritems():
1648 for ip,reverse in items:
1649 if ip not in ['0.0.0.0']:
1650 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1651 for source,dest in wleiden_cname.iteritems():
1652 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1653 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1654 for source, dest in wleiden_raw.iteritems():
1655 f.write("%s.wleiden.net. %s\n" % (source, dest))
1656 f.close()
1657
1658 # Create whole bunch of specific sub arpa zones. To keep it compliant
1659 for s in range(16,32):
1660 details['zone'] = '%i.172.in-addr.arpa' % s
1661 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1662 f.write(dns_header % details)
1663
1664 #XXX: Not effient, fix to proper data structure and do checks at other
1665 # stages
1666 for host,items in wleiden_zone.iteritems():
1667 for ip,reverse in items:
1668 if not reverse:
1669 continue
1670 if valid_addr(ip):
1671 if valid_addr(ip):
1672 if int(ip.split('.')[1]) == s:
1673 rev_ip = '.'.join(reversed(ip.split('.')))
1674 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1675 f.close()
1676
1677
1678def usage():
1679 print """Usage: %(prog)s <argument>
1680Argument:
1681\tcleanup = Cleanup all YAML files to specified format
1682\tstandalone [port] = Run configurator webserver [8000]
1683\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1684\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1685\tfull-export = Generate yaml export script for heatmap.
1686\tstatic [outputdir] = Generate all config files and store on disk
1687\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1688\ttest <node> [<file>] = Receive output for certain node [all files].
1689\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1690\tlist <status> <items> = List systems which have certain status
1691\tcreate network.kml = Create Network KML file for use in Google Earth
1692
1693Arguments:
1694\t<node> = NodeName (example: HybridRick)
1695\t<file> = %(files)s
1696\t<status> = all|up|down|planned
1697\t<items> = systems|nodes|proxies
1698
1699NOTE FOR DEVELOPERS; you can test your changes like this:
1700 BEFORE any changes in this code:
1701 $ ./gformat.py static /tmp/pre
1702 AFTER the changes:
1703 $ ./gformat.py static /tmp/post
1704 VIEW differences and VERIFY all are OK:
1705 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1706""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1707 exit(0)
1708
1709
1710def is_text_request(environ=os.environ):
1711 """ Find out whether we are calling from the CLI or any text based CLI utility """
1712 try:
1713 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1714 except KeyError:
1715 return True
1716
1717def switchFormat(setting):
1718 if setting:
1719 return "YES"
1720 else:
1721 return "NO"
1722
1723def rlinput(prompt, prefill=''):
1724 import readline
1725 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1726 try:
1727 return raw_input(prompt)
1728 finally:
1729 readline.set_startup_hook()
1730
1731def fix_conflict(left, right, default='i'):
1732 while True:
1733 print "## %-30s | %-30s" % (left, right)
1734 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1735 if not c:
1736 c = default
1737
1738 if c in ['l','1']:
1739 return left
1740 elif c in ['r','2']:
1741 return right
1742 elif c in ['e', '3']:
1743 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1744 elif c in ['i', '4']:
1745 return None
1746 else:
1747 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1748
1749
1750
1751def print_cgi_response(response_headers, output):
1752 """Could we not use some kind of wsgi wrapper to make this output?"""
1753 for header in response_headers:
1754 print "%s: %s" % header
1755 print
1756 print output
1757
1758
1759def fill_cache():
1760 ''' Poor man re-loading of few cache items (the slow ones) '''
1761 for host in get_hostlist():
1762 get_yaml(host)
1763
1764
1765def reload_cache():
1766 clear_cache()
1767 fill_cache()
1768
1769
1770def main():
1771 """Hard working sub"""
1772 # Allow easy hacking using the CLI
1773 if not os.environ.has_key('PATH_INFO'):
1774 if len(sys.argv) < 2:
1775 usage()
1776
1777 if sys.argv[1] == "standalone":
1778 import SocketServer
1779 import CGIHTTPServer
1780 # Hop to the right working directory.
1781 os.chdir(os.path.dirname(__file__))
1782 try:
1783 PORT = int(sys.argv[2])
1784 except (IndexError,ValueError):
1785 PORT = 8000
1786
1787 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1788 """ Serve this CGI from the root of the webserver """
1789 def is_cgi(self):
1790 if "favicon" in self.path:
1791 return False
1792
1793 self.cgi_info = (os.path.basename(__file__), self.path)
1794 self.path = ''
1795 return True
1796 handler = MyCGIHTTPRequestHandler
1797 SocketServer.TCPServer.allow_reuse_address = True
1798 httpd = SocketServer.TCPServer(("", PORT), handler)
1799 httpd.server_name = 'localhost'
1800 httpd.server_port = PORT
1801
1802 logger.info("serving at port %s", PORT)
1803 try:
1804 httpd.serve_forever()
1805 except KeyboardInterrupt:
1806 httpd.shutdown()
1807 logger.info("All done goodbye")
1808 elif sys.argv[1] == "test":
1809 # Basic argument validation
1810 try:
1811 node = sys.argv[2]
1812 except IndexError:
1813 print "Invalid argument"
1814 exit(1)
1815 except IOError as e:
1816 print e
1817 exit(1)
1818
1819 datadump = get_yaml(node)
1820
1821
1822 # Get files to generate
1823 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1824
1825 # Actual config generation
1826 for config in gen_files:
1827 logger.info("## Generating %s %s", node, config)
1828 print generate_config(node, config, datadump)
1829 elif sys.argv[1] == "test-cgi":
1830 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1831 os.environ['SCRIPT_NAME'] = __file__
1832 response_headers, output = process_cgi_request()
1833 print_cgi_response(response_headers, output)
1834 elif sys.argv[1] == "static":
1835 items = dict()
1836 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1837 for node in get_hostlist():
1838 items['node'] = node
1839 items['wdir'] = "%(output_dir)s/%(node)s" % items
1840 if not os.path.isdir(items['wdir']):
1841 os.makedirs(items['wdir'])
1842 datadump = get_yaml(node)
1843 for config in files:
1844 items['config'] = config
1845 logger.info("## Generating %(node)s %(config)s" % items)
1846 f = open("%(wdir)s/%(config)s" % items, "w")
1847 f.write(generate_config(node, config, datadump))
1848 f.close()
1849 elif sys.argv[1] == "wind-export":
1850 items = dict()
1851 for node in get_hostlist():
1852 datadump = get_yaml(node)
1853 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1854 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1855 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1856 VALUES (
1857 (SELECT id FROM users WHERE username = 'rvdzwet'),
1858 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1859 'Y');""" % datadump
1860 #for config in files:
1861 # items['config'] = config
1862 # print "## Generating %(node)s %(config)s" % items
1863 # f = open("%(wdir)s/%(config)s" % items, "w")
1864 # f.write(generate_config(node, config, datadump))
1865 # f.close()
1866 for node in get_hostlist():
1867 datadump = get_yaml(node)
1868 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1869 ifacedump = datadump[iface_key]
1870 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1871 ifacedump['nodename'] = datadump['nodename']
1872 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1873 ifacedump['channel'] = 0
1874 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1875 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1876 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1877 elif sys.argv[1] == "nagios-export":
1878 try:
1879 heavy_load = (sys.argv[2] == "--heavy-load")
1880 except IndexError:
1881 heavy_load = False
1882
1883 hostgroup_details = {
1884 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1885 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1886 'walphen' : 'Stichting Wireless Alphen',
1887 'westeinder' : 'Westeinder Plassen',
1888 }
1889
1890 # Convert IP to Host
1891 ip2host = {'root' : 'root'}
1892 for host in get_hostlist():
1893 datadump = get_yaml(host)
1894 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1895 for iface in get_interface_keys(datadump):
1896 if datadump[iface].has_key('autogen_gateway'):
1897 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1898
1899 # Find dependency tree based on output of lvrouted.mytree of nearest node
1900 parents = defaultdict(list)
1901 stack = ['root']
1902 prev_depth = 0
1903 for line in open('lvrouted.mytree').readlines():
1904 depth = line.count('\t')
1905 ip = line.strip().split()[0]
1906
1907 if prev_depth < depth:
1908 try:
1909 parents[ip2host[ip]].append(ip2host[stack[-1]])
1910 except KeyError as e:
1911 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1912 stack.append(ip)
1913 elif prev_depth > depth:
1914 stack = stack[:(depth - prev_depth)]
1915 elif prev_depth == depth:
1916 try:
1917 parents[ip2host[ip]].append(ip2host[stack[-1]])
1918 except KeyError as e:
1919 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1920
1921
1922 prev_depth = depth
1923 # Observe that some nodes has themself as parent or multiple parents
1924 # for now take only the first parent, other behaviour is yet to be explained
1925
1926
1927
1928 params = {
1929 'check_interval' : 5 if heavy_load else 120,
1930 'retry_interval' : 1 if heavy_load else 10,
1931 'max_check_attempts' : 10 if heavy_load else 6,
1932 'notification_interval': 120 if heavy_load else 240,
1933 }
1934
1935 print '''\
1936define host {
1937 name wleiden-node ; Default Node Template
1938 use generic-host ; Use the standard template as initial starting point
1939 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1940 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1941 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1942 notification_interval %(notification_interval)s
1943 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1944 check_command check-host-alive ; Default command to check FreeBSD hosts
1945 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1946}
1947
1948define service {
1949 name wleiden-service ; Default Service Template
1950 use generic-service ; Use the standard template as initial starting point
1951 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1952 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1953 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1954 notification_interval %(notification_interval)s
1955 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1956 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1957}
1958
1959# Please make sure to install:
1960# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1961#
1962# Recompile net-mgmt/nagios-plugins to support check_snmp
1963# make -C /usr/ports/net-mgmt/nagios-plugins
1964#
1965# Install net/bind-tools to allow v2/check_dns_wl to work:
1966# pkg install bind-tools
1967#
1968define command{
1969 command_name check_snmp_disk
1970 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1971}
1972
1973define command{
1974 command_name check_netsnmp_load
1975 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1976}
1977
1978define command{
1979 command_name check_netsnmp_proc
1980 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1981}
1982
1983define command{
1984 command_name check_by_ssh
1985 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1986}
1987
1988define command{
1989 command_name check_dns_wl
1990 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1991}
1992
1993define command{
1994 command_name check_snmp_uptime
1995 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1996}
1997
1998
1999# TDB: dhcp leases
2000# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2001
2002# TDB: internet status
2003# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2004
2005# TDB: Advanced local passive checks
2006# /usr/local/libexec/nagios/check_by_ssh
2007''' % params
2008
2009 print '''\
2010# Service Group, not displayed by default
2011define hostgroup {
2012 hostgroup_name srv_hybrid
2013 alias All Hybrid Nodes
2014 register 0
2015}
2016
2017define service {
2018 use wleiden-service
2019 hostgroup_name srv_hybrid
2020 service_description SSH
2021 check_command check_ssh
2022}
2023
2024define service {
2025 use wleiden-service,service-pnp
2026 hostgroup_name srv_hybrid
2027 service_description HTTP
2028 check_command check_http
2029}
2030
2031define service {
2032 use wleiden-service
2033 hostgroup_name srv_hybrid
2034 service_description DNS
2035 check_command check_dns_wl!"www.wirelessleiden.nl"
2036}
2037'''
2038
2039 if heavy_load:
2040 print '''\
2041define service {
2042 use wleiden-service
2043 hostgroup_name srv_hybrid
2044 service_description UPTIME
2045 check_command check_snmp_uptime
2046}
2047
2048#define service {
2049# use wleiden-service
2050# hostgroup_name srv_hybrid
2051# service_description NTP
2052# check_command check_ntp_peer
2053#}
2054
2055define service {
2056 use wleiden-service
2057 hostgroup_name srv_hybrid
2058 service_description LOAD
2059 check_command check_netsnmp_load
2060}
2061
2062define service {
2063 use wleiden-service
2064 hostgroup_name srv_hybrid
2065 service_description PROC
2066 check_command check_netsnmp_proc
2067}
2068
2069define service {
2070 use wleiden-service
2071 hostgroup_name srv_hybrid
2072 service_description DISK
2073 check_command check_snmp_disk
2074}
2075'''
2076 for node in get_hostlist():
2077 datadump = get_yaml(node)
2078 if not datadump['status'] == 'up':
2079 continue
2080 if not hostgroup_details.has_key(datadump['monitoring_group']):
2081 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2082 print '''\
2083define host {
2084 use wleiden-node,host-pnp
2085 contact_groups admins
2086 host_name %(autogen_fqdn)s
2087 address %(masterip)s
2088 hostgroups srv_hybrid,%(monitoring_group)s\
2089''' % datadump
2090 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2091 print '''\
2092 parents %(parents)s\
2093''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2094 print '''\
2095}
2096'''
2097
2098
2099 for name,alias in hostgroup_details.iteritems():
2100 print '''\
2101define hostgroup {
2102 hostgroup_name %s
2103 alias %s
2104} ''' % (name, alias)
2105
2106
2107 elif sys.argv[1] == "full-export":
2108 hosts = {}
2109 for node in get_hostlist():
2110 datadump = get_yaml(node)
2111 hosts[datadump['nodename']] = datadump
2112 print yaml.dump(hosts)
2113
2114 elif sys.argv[1] == "dns":
2115 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2116 elif sys.argv[1] == "cleanup":
2117 # First generate all datadumps
2118 datadumps = dict()
2119 ssid_to_node = dict()
2120 for host in get_hostlist():
2121 logger.info("# Processing: %s", host)
2122 # Set some boring default values
2123 datadump = { 'board' : 'UNKNOWN' }
2124 datadump.update(get_yaml(host))
2125 datadumps[datadump['nodename']] = datadump
2126
2127 (poel, errors) = make_relations()
2128 print "\n".join(["# WARNING: %s" % x for x in errors])
2129
2130 for host,datadump in datadumps.iteritems():
2131 try:
2132 # Convert all yes and no to boolean values
2133 def fix_boolean(dump):
2134 for key in dump.keys():
2135 if type(dump[key]) == dict:
2136 dump[key] = fix_boolean(dump[key])
2137 elif str(dump[key]).lower() in ["yes", "true"]:
2138 dump[key] = True
2139 elif str(dump[key]).lower() in ["no", "false"]:
2140 # Compass richting no (Noord Oost) is valid input
2141 if key != "compass": dump[key] = False
2142 return dump
2143 datadump = fix_boolean(datadump)
2144
2145 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2146 if not 'latitude' in datadump and not 'longitude' in datadump:
2147 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2148 elif 'latitude' in datadump and 'longitude' in datadump:
2149 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2150 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2151 # TODO: Compare outcome of both coordinate systems and validate against each-other
2152
2153 if datadump['nodename'].startswith('Proxy'):
2154 datadump['nodename'] = datadump['nodename'].lower()
2155
2156 for iface_key in get_interface_keys(datadump):
2157 try:
2158 # All our normal wireless cards are normal APs now
2159 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2160 datadump[iface_key]['mode'] = 'ap'
2161 # Wireless Leiden SSID have an consistent lowercase/uppercase
2162 if datadump[iface_key].has_key('ssid'):
2163 ssid = datadump[iface_key]['ssid']
2164 prefix = 'ap-WirelessLeiden-'
2165 if ssid.lower().startswith(prefix.lower()):
2166 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2167 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2168 datadump[iface_key]['mode'] = 'autogen-FIXME'
2169 if not datadump[iface_key].has_key('comment'):
2170 datadump[iface_key]['comment'] = 'autogen-FIXME'
2171
2172 if datadump[iface_key].has_key('ns_mac'):
2173 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2174
2175 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2176 datadump[iface_key] = datadump[iface_key]['desc']
2177
2178 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2179 # due to better collision avoidance.
2180 if datadump[iface_key]['type'] == '11b':
2181 datadump[iface_key]['type'] = '11g'
2182
2183 # Setting 802.11g channels to de-facto standards, to avoid
2184 # un-detected sharing with other overlapping channels
2185 #
2186 # Technically we could also use channel 13 in NL, but this is not
2187 # recommended as foreign devices might not be able to select this
2188 # channel. Secondly using 1,5,9,13 instead is going to clash with
2189 # the de-facto usage of 1,6,11.
2190 #
2191 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2192 channels_at_2400Mhz = (1,6,11)
2193 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2194 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2195 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2196 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2197
2198 # Mandatory interface keys
2199 if not datadump[iface_key].has_key('status'):
2200 datadump[iface_key]['status'] = 'planned'
2201
2202 x = datadump[iface_key]['comment']
2203 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2204
2205 # Fixing bridge_type if none is found
2206 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2207 if not 'bridge_type' in datadump[iface_key]:
2208 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2209
2210 # Making sure description works
2211 if datadump[iface_key].has_key('desc'):
2212 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2213 del datadump[iface_key]['desc']
2214 else:
2215 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2216 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2217 if response:
2218 datadump[iface_key]['comment'] = response
2219 del datadump[iface_key]['desc']
2220
2221 # Check DHCP configuration
2222 dhcp_type(datadump[iface_key])
2223
2224 # Set the compass value based on the angle between the poels
2225 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2226 my_pool = poel[network(datadump[iface_key]['ip'])]
2227 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2228 if remote_hosts:
2229 compass_target = remote_hosts[0]
2230 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2231 # TODO: Compass wanted and actual direction might differ
2232
2233 # Monitoring Group default
2234 if not 'monitoring_group' in datadump:
2235 datadump['monitoring_group'] = 'wleiden'
2236
2237 except Exception:
2238 print "# Error while processing interface %s" % iface_key
2239 raise
2240 store_yaml(datadump)
2241 except Exception:
2242 print "# Error while processing %s" % host
2243 raise
2244 elif sys.argv[1] == "list":
2245 use_fqdn = False
2246 if len(sys.argv) < 4:
2247 usage()
2248 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2249 usage()
2250 if not sys.argv[3] in ["nodes","proxies","systems"]:
2251 usage()
2252
2253 if len(sys.argv) > 4:
2254 if sys.argv[4] == "fqdn":
2255 use_fqdn = True
2256 else:
2257 usage()
2258
2259 for system in get_hostlist():
2260 datadump = get_yaml(system)
2261 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2262 continue
2263
2264 output = datadump['autogen_fqdn'] if use_fqdn else system
2265 if sys.argv[2] == "all":
2266 print output
2267 elif datadump['status'] == sys.argv[2]:
2268 print output
2269 elif sys.argv[1] == "create":
2270 if sys.argv[2] == "network.kml":
2271 print make_network_kml.make_graph()
2272 elif sys.argv[2] == "host-ips.txt":
2273 for system in get_hostlist():
2274 datadump = get_yaml(system)
2275 ips = [datadump['masterip']]
2276 for ifkey in get_interface_keys(datadump):
2277 ips.append(datadump[ifkey]['ip'].split('/')[0])
2278 print system, ' '.join(ips)
2279 elif sys.argv[2] == "host-pos.txt":
2280 for system in get_hostlist():
2281 datadump = get_yaml(system)
2282 print system, datadump['rdnap_x'], datadump['rdnap_y']
2283 elif sys.argv[2] == 'ssh_config':
2284 print '''
2285Host *.wleiden.net
2286 User root
2287
2288Host 172.16.*.*
2289 User root
2290'''
2291 for system in get_hostlist():
2292 datadump = get_yaml(system)
2293 print '''\
2294Host %s
2295 User root
2296
2297Host %s
2298 User root
2299
2300Host %s
2301 User root
2302
2303Host %s
2304 User root
2305''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2306 else:
2307 usage()
2308 else:
2309 usage()
2310 else:
2311 # Do not enable debugging for config requests as it highly clutters the output
2312 if not is_text_request():
2313 cgitb.enable()
2314 response_headers, output = process_cgi_request()
2315 print_cgi_response(response_headers, output)
2316
2317def application(environ, start_response):
2318 status = '200 OK'
2319 response_headers, output = process_cgi_request(environ)
2320 start_response(status, response_headers)
2321
2322 # Debugging only
2323 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2324 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2325 return [output]
2326
2327if __name__ == "__main__":
2328 main()
Note: See TracBrowser for help on using the repository browser.