source: genesis/tools/gformat.py@ 13896

Last change on this file since 13896 was 13896, checked in by rick, 8 years ago

Setting a dummy captive portal to avoid syntax errors in pf.conf files

Syntax errors will cause the pf not to load.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 78.3 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# FreeBSD:
28# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
29# Fedora:
30# yum install python-yaml pyproj proj-epsg python-jinja2
31#
32# Rick van der Zwet <info@rickvanderzwet.nl>
33#
34
35# Hack to make the script directory is also threated as a module search path.
36import sys
37import os
38sys.path.append(os.path.dirname(__file__))
39
40SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
41
42import argparse
43import cgi
44import cgitb
45import copy
46import glob
47import make_network_kml
48import math
49import pyproj
50import random
51import re
52import socket
53import string
54import subprocess
55import textwrap
56import time
57import urlparse
58
59from pprint import pprint
60from collections import defaultdict, OrderedDict
61from sys import stderr
62try:
63 import yaml
64except ImportError, e:
65 print e
66 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
67 exit(1)
68
69try:
70 from yaml import CLoader as Loader
71 from yaml import CDumper as Dumper
72except ImportError:
73 from yaml import Loader, Dumper
74
75from jinja2 import Environment, Template
76def yesorno(value):
77 return "YES" if bool(value) else "NO"
78env = Environment()
79env.filters['yesorno'] = yesorno
80def render_template(datadump, template):
81 result = env.from_string(template).render(datadump)
82 # Make it look pretty to the naked eye, as jinja templates are not so
83 # friendly when it comes to whitespace formatting
84 ## Remove extra whitespace at end of line lstrip() style.
85 result = re.sub(r'\n[\ ]+','\n', result)
86 ## Include only a single newline between an definition and a comment
87 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
88 ## Remove extra newlines after single comment
89 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
90 return result
91
92import logging
93logging.basicConfig(format='# %(levelname)s: %(message)s' )
94logger = logging.getLogger()
95logger.setLevel(logging.DEBUG)
96
97
98if os.environ.has_key('CONFIGROOT'):
99 NODE_DIR = os.environ['CONFIGROOT']
100else:
101 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
102__version__ = '$Id: gformat.py 13896 2017-06-02 13:22:55Z rick $'
103
104files = [
105 'authorized_keys',
106 'dnsmasq.conf',
107 'dhcpd.conf',
108 'rc.conf.local',
109 'resolv.conf',
110 'motd',
111 'ntp.conf',
112 'pf.hybrid.conf.local',
113 'unbound.wleiden.conf',
114 'wleiden.yaml',
115 ]
116
117# Global variables uses
118OK = 10
119DOWN = 20
120UNKNOWN = 90
121
122
123ileiden_proxies = OrderedDict()
124normal_proxies = []
125datadump_cache = {}
126interface_list_cache = {}
127rc_conf_local_cache = {}
128nameservers_cache = []
129relations_cache = None
130def clear_cache():
131 ''' Poor mans cache implementation '''
132 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
133 datadump_cache = {}
134 interface_list_cache = {}
135 rc_conf_local_cache = {}
136 ileiden_proxies = OrderedDict()
137 normal_proxies = []
138 nameservers_cache = []
139 relations_cache = None
140
141NO_DHCP = 0
142DHCP_CLIENT = 10
143DHCP_SERVER = 20
144def dhcp_type(item):
145 if not item.has_key('dhcp'):
146 return NO_DHCP
147 elif not item['dhcp']:
148 return NO_DHCP
149 elif item['dhcp'].lower() == 'client':
150 return DHCP_CLIENT
151 else:
152 # Validation Checks
153 begin,end = map(int,item['dhcp'].split('-'))
154 if begin >= end:
155 raise ValueError("DHCP Start >= DHCP End")
156 return DHCP_SERVER
157
158def etrs2rd(lat, lon):
159 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
160 p2 = pyproj.Proj(init='EPSG:28992')
161 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
162 return (RDx, RDy)
163
164def rd2etrs(RDx, RDy):
165 p1 = pyproj.Proj(init='EPSG:28992')
166 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
167 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
168 return (lat, lon)
169
170def get_yaml(item,add_version_info=True):
171 try:
172 """ Get configuration yaml for 'item'"""
173 if datadump_cache.has_key(item):
174 return datadump_cache[item].copy()
175
176 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
177 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
178 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
179
180 # Default values
181 datadump = {
182 'autogen_revision' : 'NOTFOUND',
183 'autogen_gfile' : gfile,
184 'service_proxy_ileiden' : False,
185 }
186 f = open(gfile, 'r')
187 datadump.update(yaml.load(f,Loader=Loader))
188 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
189 if datadump['nodetype'] == 'Hybrid':
190 # Some values are defined implicitly
191 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
192 datadump['service_incoming_rdr'] = True
193 # Use some boring defaults
194 defaults = {
195 'service_proxy_normal' : False,
196 'service_accesspoint' : True,
197 'service_incoming_rdr' : False,
198 'service_concentrator' : False,
199 'monitoring_group' : 'wleiden',
200 }
201 for (key,value) in defaults.iteritems():
202 if not datadump.has_key(key):
203 datadump[key] = value
204 f.close()
205
206 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
207 if add_version_info:
208 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
209 lines = p.communicate()[0].split('\n')
210 if p.returncode == 0:
211 for line in lines:
212 if line:
213 (key, value) = line.split(': ')
214 datadump["autogen_" + key.lower().replace(' ','_')] = value
215
216 # Preformat certain needed variables for formatting and push those into special object
217 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
218
219 wlan_count=0
220 try:
221 for key in get_interface_keys(datadump, True):
222 datadump[key]['autogen_ifbase'] = key.split('_')[1]
223 datadump[key]['autogen_vlan'] = False
224
225 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
226 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
227
228 if datadump[key].has_key('parent'):
229 if datadump[key].has_key('ip'):
230 raise ValueError("Interface bridge member cannot have IP assigned")
231 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
232 raise ValueError("Interface bridge member cannot have DHCP set")
233
234 if datadump[key].has_key('ip'):
235 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
236
237 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
238 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
239 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
240 wlan_count += 1
241 else:
242 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
243 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
244 datadump[key]['autogen_vlan'] = key.split('_')[2]
245 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
246 else:
247 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
248
249 except Exception:
250 print "# Error while processing interface %s" % key
251 raise
252
253 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
254 if dhcp_type(datadump[key]) == DHCP_SERVER]
255
256 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
257 datadump['autogen_item'] = item
258
259 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
260 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
261 datadump_cache[item] = datadump.copy()
262 except Exception:
263 print "# Error while processing %s" % item
264 raise
265 return datadump
266
267
268def store_yaml(datadump, header=False):
269 """ Store configuration yaml for 'item'"""
270 item = datadump['autogen_item']
271 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
272
273 output = generate_wleiden_yaml(datadump, header)
274
275 f = open(gfile, 'w')
276 f.write(output)
277 f.close()
278
279
280def network(ip):
281 addr, mask = ip.split('/')
282 # Not parsing of these folks please
283 addr = parseaddr(addr)
284 mask = int(mask)
285 network = addr & ~((1 << (32 - mask)) - 1)
286 return network
287
288
289
290def make_relations():
291 """ Process _ALL_ yaml files to get connection relations """
292 global relations_cache
293
294 if relations_cache:
295 return relations_cache
296
297 errors = []
298 poel = defaultdict(list)
299
300 for host in get_hostlist():
301 datadump = get_yaml(host)
302 try:
303 for iface_key in get_interface_keys(datadump):
304 # Bridge members has no IP assigned
305 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
306 continue
307 net_addr = network(datadump[iface_key]['ip'])
308 poel[net_addr] += [(host,datadump[iface_key].copy())]
309 except (KeyError, ValueError) as e:
310 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
311 continue
312
313 relations_cache = (poel, errors)
314 return relations_cache
315
316
317
318def valid_addr(addr):
319 """ Show which address is valid in which are not """
320 return str(addr).startswith('172.')
321
322def get_hostlist():
323 """ Combined hosts and proxy list"""
324 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
325
326def angle_between_points(lat1,lat2,long1,long2):
327 """
328 Return Angle in radians between two GPS coordinates
329 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
330 """
331 dy = lat2 - lat1
332 dx = math.cos(lat1)*(long2 - long1)
333 angle = math.atan2(dy,dx)
334 return angle
335
336
337
338def angle_to_cd(angle):
339 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
340
341 # For easy conversion get positive degree
342 degrees = math.degrees(angle)
343 abs_degrees = 360 + degrees if degrees < 0 else degrees
344
345 # Numbers can be confusing calculate from the 4 main directions
346 p = 22.5
347 if abs_degrees < p:
348 cd = "n"
349 elif abs_degrees < (90 - p):
350 cd = "no"
351 elif abs_degrees < (90 + p):
352 cd = "o"
353 elif abs_degrees < (180 - p):
354 cd = "zo"
355 elif abs_degrees < (180 + p):
356 cd = "z"
357 elif abs_degrees < (270 - p):
358 cd = "zw"
359 elif abs_degrees < (270 + p):
360 cd = "w"
361 elif abs_degrees < (360 - p):
362 cd = "nw"
363 else:
364 cd = "n"
365 return cd
366
367
368
369def cd_between_hosts(hostA, hostB, datadumps):
370 # Using RDNAP coordinates
371 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
372 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
373 return angle_to_cd(math.atan2(dx,dy))
374
375 # GPS coordinates seems to fail somehow
376 #latA = float(datadumps[hostA]['latitude'])
377 #latB = float(datadumps[hostB]['latitude'])
378 #lonA = float(datadumps[hostA]['longitude'])
379 #lonB = float(datadumps[hostB]['longitude'])
380 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
381
382
383def generate_title(nodelist):
384 """ Main overview page """
385 items = {'root' : "." }
386 def fl(spaces, line):
387 return (' ' * spaces) + line + '\n'
388
389 output = """
390<html>
391 <head>
392 <title>Wireless leiden Configurator - GFormat</title>
393 <style type="text/css">
394 th {background-color: #999999}
395 tr:nth-child(odd) {background-color: #cccccc}
396 tr:nth-child(even) {background-color: #ffffff}
397 th, td {padding: 0.1em 1em}
398 </style>
399 </head>
400 <body>
401 <center>
402 <form type="GET" action="%(root)s">
403 <input type="hidden" name="action" value="update">
404 <input type="submit" value="Update Configuration Database (SVN)">
405 </form>
406 <table>
407 <caption><h3>Wireless Leiden Configurator</h3></caption>
408 """ % items
409
410 for node in nodelist:
411 items['node'] = node
412 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
413 for config in files:
414 items['config'] = config
415 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
416 output += fl(5, "</tr>")
417 output += """
418 </table>
419 <hr />
420 <em>%s</em>
421 </center>
422 </body>
423</html>
424 """ % __version__
425
426 return output
427
428
429
430def generate_node(node):
431 """ Print overview of all files available for node """
432 return "\n".join(files)
433
434def generate_node_overview(host):
435 """ Print overview of all files available for node """
436 datadump = get_yaml(host)
437 params = { 'host' : host }
438 output = "<em><a href='..'>Back to overview</a></em><hr />"
439 output += "<h2>Available files:</h2><ul>"
440 for cf in files:
441 params['cf'] = cf
442 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
443 output += "</ul>"
444
445 # Generate and connection listing
446 output += "<h2>Connected To:</h2><ul>"
447 (poel, errors) = make_relations()
448 for network, hosts in poel.iteritems():
449 if host in [x[0] for x in hosts]:
450 if len(hosts) == 1:
451 # Single not connected interface
452 continue
453 for remote,ifacedump in hosts:
454 if remote == host:
455 # This side of the interface
456 continue
457 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
458 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
459 output += "</ul>"
460 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
461
462 output += "<hr /><em><a href='..'>Back to overview</a></em>"
463 return output
464
465
466def generate_header(datadump, ctag="#"):
467 return """\
468%(ctag)s
469%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
470%(ctag)s
471""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
472
473
474
475def parseaddr(s):
476 """ Process IPv4 CIDR notation addr to a (binary) number """
477 f = s.split('.')
478 return (long(f[0]) << 24L) + \
479 (long(f[1]) << 16L) + \
480 (long(f[2]) << 8L) + \
481 long(f[3])
482
483
484
485def showaddr(a):
486 """ Display IPv4 addr in (dotted) CIDR notation """
487 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
488
489
490def is_member(ip, mask, canidate):
491 """ Return True if canidate is part of ip/mask block"""
492 ip_addr = parseaddr(ip)
493 ip_canidate = parseaddr(canidate)
494 mask = int(mask)
495 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
496 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
497 return ip_addr == ip_canidate
498
499
500
501def cidr2netmask(netmask):
502 """ Given a 'netmask' return corresponding CIDR """
503 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
504
505def get_network(addr, mask):
506 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
507
508
509def generate_dhcpd_conf(datadump):
510 """ Generate config file '/usr/local/etc/dhcpd.conf """
511 # Redundency support, in cause local DNS server is not running/responding.
512 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
513 output = generate_header(datadump)
514 output += Template("""\
515# option definitions common to all supported networks...
516option domain-name "dhcp.{{ autogen_fqdn }}";
517
518default-lease-time 600;
519max-lease-time 7200;
520
521# Use this to enble / disable dynamic dns updates globally.
522#ddns-update-style none;
523
524# If this DHCP server is the official DHCP server for the local
525# network, the authoritative directive should be uncommented.
526authoritative;
527
528# Use this to send dhcp log messages to a different log file (you also
529# have to hack syslog.conf to complete the redirection).
530log-facility local7;
531
532#
533# Interface definitions
534#
535\n\n""").render(datadump)
536
537
538 # TODO: Use textwrap.fill instead
539 def indent(text, count):
540 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
541
542 # Process DHCP blocks
543 dhcp_out = defaultdict(list)
544 for iface_key in get_interface_keys(datadump):
545 ifname = datadump[iface_key]['autogen_ifbase']
546 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
547 if not datadump[iface_key].has_key('comment'):
548 datadump[iface_key]['comment'] = None
549
550 if not datadump[iface_key].has_key('ip'):
551 continue
552
553 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
554
555 (addr, mask) = datadump[iface_key]['ip'].split('/')
556 datadump[iface_key]['autogen_addr'] = addr
557 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
558 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
559
560 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
561 dhcp_out[groupif].append(textwrap.dedent("""\
562 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
563 ### not autoritive
564 }
565 """ % datadump[iface_key]))
566 continue
567
568 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
569 dhcp_part = ".".join(addr.split('.')[0:3])
570 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
571 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
572 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
573
574 # Assume the first 10 IPs could be used for static entries
575 if 'no_portal' in datadump:
576 fixed = 5
577 for mac in datadump['no_portal']:
578 dhcp_out[groupif].append(textwrap.dedent("""\
579 host fixed-%(ifname)s-%(fixed)s {
580 hardware ethernet %(mac)s;
581 fixed-address %(prefix)s.%(fixed)s;
582 }
583 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
584 fixed += 1
585
586 if 'dhcp_fixed' in datadump[iface_key]:
587 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
588 dhcp_out[groupif].append(textwrap.dedent("""\
589 host fixed-%(host)s {
590 hardware ethernet %(mac)s;
591 fixed-address %(addr)s;
592 }
593 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
594
595
596 dhcp_out[groupif].append(textwrap.dedent("""\
597 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
598 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
599 option routers %(autogen_addr)s;
600 option domain-name-servers %(autogen_dns_servers)s;
601
602 }
603 """ % datadump[iface_key]))
604
605 # Output the blocks in groups
606 for ifname,value in sorted(dhcp_out.iteritems()):
607 if len(value) > 2:
608 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
609 else:
610 output += ''.join(value) + "\n\n"
611 return output
612
613
614
615def generate_dnsmasq_conf(datadump):
616 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
617 output = generate_header(datadump)
618 output += Template("""\
619# DHCP server options
620dhcp-authoritative
621dhcp-fqdn
622domain=dhcp.{{ autogen_fqdn }}
623domain-needed
624expand-hosts
625log-async=100
626
627# Low memory footprint
628cache-size=10000
629
630\n""").render(datadump)
631
632 for iface_key in get_interface_keys(datadump):
633 if not datadump[iface_key].has_key('comment'):
634 datadump[iface_key]['comment'] = None
635 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
636
637 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
638 output += "# not autoritive\n\n"
639 continue
640
641 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
642 (ip, cidr) = datadump[iface_key]['ip'].split('/')
643 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
644
645 dhcp_part = ".".join(ip.split('.')[0:3])
646 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
647 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
648 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
649
650 return output
651
652
653class AutoVivification(dict):
654 """Implementation of perl's autovivification feature."""
655 def __getitem__(self, item):
656 try:
657 return dict.__getitem__(self, item)
658 except KeyError:
659 value = self[item] = type(self)()
660 return value
661
662def make_interface_list(datadump):
663 if interface_list_cache.has_key(datadump['autogen_item']):
664 return (interface_list_cache[datadump['autogen_item']])
665 # lo0 configuration:
666 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
667 # - masterip is special as it needs to be assigned to at
668 # least one interface, so if not used assign to lo0
669 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
670 vlan_list = defaultdict(list)
671 bridge_list = defaultdict(list)
672 flags_if = AutoVivification()
673 dhclient_if = {'lo0' : False}
674
675 # XXX: Find some way of send this output nicely
676 output = ''
677
678 masterip_used = False
679 for iface_key in get_interface_keys(datadump):
680 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
681 masterip_used = True
682 break
683 if not masterip_used:
684 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
685
686 if 'serviceid' in datadump:
687 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
688
689 for iface_key in get_interface_keys(datadump):
690 ifacedump = datadump[iface_key]
691 ifname = ifacedump['autogen_ifname']
692
693 # If defined as vlan interface
694 if ifacedump['autogen_vlan']:
695 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
696
697 # If defined as bridge interface
698 if ifacedump['autogen_bridge_member']:
699 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
700
701 # Flag dhclient is possible
702 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
703 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
704
705 # Ethernet address
706 if ifacedump.has_key('ether'):
707 flags_if[ifname]['ether'] = ifacedump['ether']
708
709 # Add interface IP to list
710 if ifacedump.has_key('ip'):
711 item = (ifacedump['ip'], ifacedump['comment'])
712 if addrs_list.has_key(ifname):
713 addrs_list[ifname].append(item)
714 else:
715 addrs_list[ifname] = [item]
716
717 # Alias only needs IP assignment for now, this might change if we
718 # are going to use virtual accesspoints
719 if "alias" in iface_key:
720 continue
721
722 # XXX: Might want to deduct type directly from interface name
723 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
724 # Default to station (client) mode
725 ifacedump['autogen_wlanmode'] = "sta"
726 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
727 ifacedump['autogen_wlanmode'] = "ap"
728
729 if not ifacedump.has_key('channel'):
730 if ifacedump['type'] == '11a':
731 ifacedump['channel'] = 36
732 else:
733 ifacedump['channel'] = 1
734
735 # Allow special hacks at the back like wds and stuff
736 if not ifacedump.has_key('extra'):
737 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
738 else:
739 ifacedump['autogen_extra'] = ifacedump['extra']
740
741 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
742
743 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
744 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
745 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
746 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
747 output += "\n"
748
749 elif ifacedump['type'] in ['ethernet', 'eth']:
750 # No special config needed besides IP
751 pass
752 elif ifacedump['type'] in ['vlan']:
753 # VLAN member has no special configuration
754 pass
755 else:
756 assert False, "Unknown type " + ifacedump['type']
757
758 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
759 interface_list_cache[datadump['autogen_item']] = store
760 return(store)
761
762
763
764def create_proxies_list():
765 if not ileiden_proxies or not normal_proxies:
766 # Placeholder for to-be-installed proxies, this will avoid updating the all
767 # nodes to include this new machine, yet due to an unbound issue, this list
768 # has to be kept small.
769
770 for i in range(1,20):
771 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
772
773 for host in get_hostlist():
774 hostdump = get_yaml(host)
775 if hostdump['status'] == 'up':
776 if hostdump['service_proxy_ileiden']:
777 ileiden_proxies[hostdump['serviceid']] = hostdump
778 if hostdump['service_proxy_normal']:
779 normal_proxies.append(hostdump)
780
781
782
783def generate_rc_conf_local(datadump):
784 """ Generate configuration file '/etc/rc.conf.local' """
785 item = datadump['autogen_item']
786 if rc_conf_local_cache.has_key(item):
787 return rc_conf_local_cache[item]
788
789 if not datadump.has_key('ileiden'):
790 datadump['autogen_ileiden_enable'] = False
791 else:
792 datadump['autogen_ileiden_enable'] = datadump['ileiden']
793
794 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
795
796 create_proxies_list()
797 datadump['autogen_ileiden_proxies'] = ileiden_proxies
798 datadump['autogen_normal_proxies'] = normal_proxies
799 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
800 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
801 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
802 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
803
804 output = generate_header(datadump, "#");
805 output += render_template(datadump, """\
806hostname='{{ autogen_fqdn }}'
807location='{{ location }}'
808nodetype="{{ nodetype }}"
809
810#
811# Configured listings
812#
813captive_portal_whitelist=""
814{% if nodetype == "Proxy" %}
815#
816# Proxy Configuration
817#
818{% if gateway and service_proxy_ileiden -%}
819defaultrouter="{{ gateway }}"
820{% else -%}
821#defaultrouter="NOTSET"
822{% endif -%}
823internalif="{{ internalif }}"
824ileiden_enable="{{ autogen_ileiden_enable }}"
825gateway_enable="{{ autogen_ileiden_enable }}"
826pf_enable="yes"
827pf_rules="/etc/pf.conf"
828{% if autogen_ileiden_enable -%}
829pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
830lvrouted_enable="{{ autogen_ileiden_enable }}"
831lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
832{% else -%}
833pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
834{% endif -%}
835{% if internalroute -%}
836static_routes="wleiden"
837route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
838{% endif -%}
839
840{% elif nodetype == "Hybrid" %}
841 #
842 # Hybrid Configuration
843 #
844 list_ileiden_proxies="
845 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
846 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
847 {% endfor -%}
848 "
849 list_normal_proxies="
850 {% for item in autogen_normal_proxies -%}
851 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
852 {% endfor -%}
853 "
854
855 {% if autogen_dhcp_interfaces -%}
856 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
857 {% else %}
858 captive_portal_interfaces="dummy"
859 {% endif %}
860 externalif="{{ externalif|default('vr0', true) }}"
861 masterip="{{ masterip }}"
862
863 {% if gateway and service_proxy_ileiden %}
864 defaultrouter="{{ gateway }}"
865 {% else %}
866 #defaultrouter="NOTSET"
867 {% endif %}
868
869 #
870 # Defined services
871 #
872 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
873 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
874 service_accesspoint="{{ service_accesspoint|yesorno }}"
875 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
876 service_concentrator="{{ service_concentrator|yesorno }}"
877
878 {% if service_proxy_ileiden %}
879 pf_rules="/etc/pf.hybrid.conf"
880 {% if service_concentrator %}
881 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
882 {% else %}
883 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
884 {% endif %}
885 pf_flags="$pf_flags -D publicnat=80,443"
886 lvrouted_flags="$lvrouted_flags -g"
887 {% elif service_proxy_normal or service_incoming_rdr %}
888 pf_rules="/etc/pf.hybrid.conf"
889 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
890 pf_flags="$pf_flags -D publicnat=0"
891 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
892 named_setfib="1"
893 tinyproxy_setfib="1"
894 dnsmasq_setfib="1"
895 sshd_setfib="1"
896 {% else %}
897 named_auto_forward_only="YES"
898 pf_rules="/etc/pf.node.conf"
899 pf_flags=""
900 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
901 {% endif %}
902 {% if service_concentrator %}
903 # Do mind installing certificates is NOT done automatically for security reasons
904 openvpn_enable="YES"
905 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
906 {% endif %}
907
908 {% if service_proxy_normal %}
909 tinyproxy_enable="yes"
910 {% else %}
911 pen_wrapper_enable="yes"
912 {% endif %}
913
914 {% if service_accesspoint %}
915 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
916 {% endif %}
917
918 {% if board == "ALIX2" %}
919 #
920 # ''Fat'' configuration, board has 256MB RAM
921 #
922 dnsmasq_enable="NO"
923 named_enable="YES"
924 unbound_enable="YES"
925 {% if autogen_dhcp_interfaces -%}
926 dhcpd_enable="YES"
927 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
928 {% endif -%}
929 {% elif board == "apu1d" %}
930 #
931 # ''Fat'' configuration, board has 1024MB RAM
932 #
933 dnsmasq_enable="NO"
934 local_unbound_enable="YES"
935 {% if autogen_dhcp_interfaces -%}
936 dhcpd_enable="YES"
937 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
938 {% endif -%}
939 {% endif -%}
940{% endif %}
941
942#
943# Script variables
944#
945attached_devices="{{ autogen_attached_devices|join(' ') }}"
946neighbours="{{ autogen_neighbours|join(' ') }}"
947
948
949#
950# Interface definitions
951#\n
952""")
953
954 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
955 for iface, vlans in sorted(vlan_list.items()):
956 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
957
958 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
959 for iface in sorted(vlan_list.keys()):
960 if not iface in addrs_list.keys():
961 output += "ifconfig_%s='up'\n" % iface
962
963 output += "\n"
964
965 # Bridge configuration:
966 if bridge_list.keys():
967 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
968
969 for iface in bridge_list.keys():
970 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
971
972 # Bridge member interfaces not containing a configuration should be marked active explcitly.
973 for _,members in bridge_list.items():
974 for iface in members:
975 if not iface in addrs_list.keys():
976 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
977
978 output += "\n"
979
980 # Details like SSID
981 if extra_ouput:
982 output += extra_ouput.strip() + "\n"
983
984 # Print IP address which needs to be assigned over here
985 output += "\n"
986 for iface,addrs in sorted(addrs_list.iteritems()):
987 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
988 output += "# %s || %s || %s\n" % (iface, addr, comment)
989
990 # Write DHCLIENT entry
991 if iface in dhclient_if and dhclient_if[iface]:
992 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
993 continue
994
995 # Make sure the external address is always first as this is needed in the
996 # firewall setup
997 addrs = sorted(
998 [x for x in addrs if not '0.0.0.0' in x[0]],
999 key=lambda x: x[0].split('.')[0],
1000 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1001 )
1002
1003 idx_offset = 0
1004 # Set MAC is required
1005 if flags_if[iface].has_key('ether'):
1006 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1007 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1008 idx_offset += 1
1009 else:
1010 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1011
1012 for idx, addr in enumerate(addrs[1:]):
1013 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1014
1015 output += "\n"
1016
1017 rc_conf_local_cache[datadump['autogen_item']] = output
1018 return output
1019
1020
1021
1022def get_all_configs():
1023 """ Get dict with key 'host' with all configs present """
1024 configs = dict()
1025 for host in get_hostlist():
1026 datadump = get_yaml(host)
1027 configs[host] = datadump
1028 return configs
1029
1030
1031def get_interface_keys(config, extra=False):
1032 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1033 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1034 if extra == False:
1035 return filter(lambda x: not "extra" in x, elems)
1036 else:
1037 return elems
1038
1039
1040def get_used_ips(configs):
1041 """ Return array of all IPs used in config files"""
1042 ip_list = []
1043 for config in configs:
1044 ip_list.append(config['masterip'])
1045 if 'serviceid' in config:
1046 ip_list.append(config['serviceid'])
1047 for iface_key in get_interface_keys(config, True):
1048 l = config[iface_key]['ip']
1049 addr, mask = l.split('/')
1050 # Special case do not process
1051 if valid_addr(addr):
1052 ip_list.append(addr)
1053 else:
1054 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1055 return sorted(ip_list)
1056
1057
1058
1059def get_nameservers(max_servers=None):
1060 if nameservers_cache:
1061 return nameservers_cache[0:max_servers]
1062
1063 for host in get_hostlist():
1064 hostdump = get_yaml(host)
1065 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1066 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
1067
1068 return nameservers_cache[0:max_servers]
1069
1070
1071def get_neighbours(datadump):
1072 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1073
1074 (poel, errors) = make_relations()
1075 table = []
1076 for iface,addrs in sorted(addrs_list.iteritems()):
1077 if iface in ['lo0']:
1078 continue
1079
1080 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1081 if not addr.startswith('172.'):
1082 # Avoid listing internet connections as pool
1083 continue
1084 for neighbour in poel[network(addr)]:
1085 if neighbour[0] != datadump['autogen_item']:
1086 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1087 return table
1088
1089
1090def get_attached_devices(datadump, url=False):
1091 table = []
1092 for iface_key in get_interface_keys(datadump, True):
1093 ifacedump = datadump[iface_key]
1094
1095 if not ifacedump.has_key('ns_ip'):
1096 continue
1097
1098 x_ip = ifacedump['ns_ip'].split('/')[0]
1099
1100 if 'mode' in ifacedump:
1101 x_mode = ifacedump['mode']
1102 else:
1103 x_mode = 'unknown'
1104
1105 if 'bridge_type' in ifacedump:
1106 device_type = ifacedump['bridge_type']
1107 else:
1108 device_type = 'Unknown'
1109
1110 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1111 return table
1112
1113
1114def generate_resolv_conf(datadump):
1115 """ Generate configuration file '/etc/resolv.conf' """
1116 # XXX: This should properly going to be an datastructure soon
1117 datadump['autogen_header'] = generate_header(datadump, "#")
1118 datadump['autogen_edge_nameservers'] = ''
1119
1120
1121 for masterip,realname in get_nameservers():
1122 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1123
1124 return Template("""\
1125{{ autogen_header }}
1126search wleiden.net
1127
1128# Try local (cache) first
1129nameserver 127.0.0.1
1130
1131{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1132nameserver 8.8.8.8 # Google Public NameServer
1133nameserver 64.6.64.6 # Verisign Public NameServer
1134{% else -%}
1135# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1136{{ autogen_edge_nameservers }}
1137{% endif -%}
1138""").render(datadump)
1139
1140
1141
1142def generate_ntp_conf(datadump):
1143 """ Generate configuration file '/etc/ntp.conf' """
1144 # XXX: This should properly going to be an datastructure soon
1145
1146 datadump['autogen_header'] = generate_header(datadump, "#")
1147 datadump['autogen_ntp_servers'] = ''
1148 for host in get_hostlist():
1149 hostdump = get_yaml(host)
1150 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1151 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1152
1153 return Template("""\
1154{{ autogen_header }}
1155
1156{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1157# Machine hooked to internet.
1158server 0.nl.pool.ntp.org iburst maxpoll 9
1159server 1.nl.pool.ntp.org iburst maxpoll 9
1160server 2.nl.pool.ntp.org iburst maxpoll 9
1161server 3.nl.pool.ntp.org iburst maxpoll 9
1162{% else -%}
1163# Local Wireless Leiden NTP Servers.
1164server 0.pool.ntp.wleiden.net iburst maxpoll 9
1165server 1.pool.ntp.wleiden.net iburst maxpoll 9
1166server 2.pool.ntp.wleiden.net iburst maxpoll 9
1167server 3.pool.ntp.wleiden.net iburst maxpoll 9
1168
1169# All the configured NTP servers
1170{{ autogen_ntp_servers }}
1171{% endif %}
1172
1173# If a server loses sync with all upstream servers, NTP clients
1174# no longer follow that server. The local clock can be configured
1175# to provide a time source when this happens, but it should usually
1176# be configured on just one server on a network. For more details see
1177# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1178# The use of Orphan Mode may be preferable.
1179#
1180server 127.127.1.0
1181fudge 127.127.1.0 stratum 10
1182""").render(datadump)
1183
1184
1185def generate_pf_hybrid_conf_local(datadump):
1186 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1187 datadump['autogen_header'] = generate_header(datadump, "#")
1188 if datadump['service_incoming_rdr']:
1189 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1190 return Template("""\
1191{{ autogen_header }}
1192
1193# Redirect some internal facing services outside (7)
1194# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1195{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1196rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1197{% endfor -%}
1198# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1199{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1200rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1201{% endfor -%}
1202""").render(datadump)
1203
1204def generate_unbound_wleiden_conf(datadump):
1205 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1206 datadump['autogen_header'] = generate_header(datadump, "#")
1207
1208 autogen_ips = []
1209 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1210 for iface,addrs in sorted(addrs_list.iteritems()):
1211 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1212 if addr.startswith('172'):
1213 autogen_ips.append((addr.split('/')[0], comment))
1214 datadump['autogen_ips'] = autogen_ips
1215
1216 create_proxies_list()
1217 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1218 return Template("""\
1219{{ autogen_header }}
1220
1221server:
1222{%- for ip,comment in autogen_ips %}
1223 interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1224{%- endfor %}
1225
1226forward-zone:
1227 name: '.'
1228{%- if service_proxy_ileiden %}
1229 forward-addr: 8.8.8.8 # Google DNS A
1230 forward-addr: 8.8.4.4 # Google DNS B
1231 forward-addr: 208.67.222.222 # OpenDNS DNS A
1232 forward-addr: 208.67.220.220 # OpenDNS DNS B
1233{% else -%}
1234{% for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1235 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1236{%- endfor %}
1237{% endif -%}
1238""").render(datadump)
1239
1240def generate_motd(datadump):
1241 """ Generate configuration file '/etc/motd' """
1242 output = Template("""\
1243FreeBSD run ``service motd onestart'' to make me look normal
1244
1245 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1246 Loc: {{ location }}
1247
1248Services:
1249{% if board == "ALIX2" -%}
1250{{" -"}} Core Node ({{ board }})
1251{% else -%}
1252{{" -"}} Hulp Node ({{ board }})
1253{% endif -%}
1254{% if service_proxy_normal -%}
1255{{" -"}} Normal Proxy
1256{% endif -%}
1257{% if service_proxy_ileiden -%}
1258{{" -"}} iLeiden Proxy
1259{% endif -%}
1260{% if service_incoming_rdr -%}
1261{{" -"}} Incoming port redirects
1262{% endif %}
1263Interlinks:\n
1264""").render(datadump)
1265
1266
1267 def make_table(table):
1268 if not table:
1269 return " - none\n"
1270 else:
1271 lines = ""
1272 col_width = [max(len(x) for x in col) for col in zip(*table)]
1273 for row in table:
1274 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1275 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1276 return lines
1277
1278 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1279 table = []
1280 for iface,addrs in sorted(addrs_list.iteritems()):
1281 if iface in ['lo0']:
1282 continue
1283 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1284 table.append((iface, addr, comment))
1285
1286 output += make_table(table)
1287 output += '\n'
1288 output += """\
1289Attached devices:
1290"""
1291 output += make_table(get_attached_devices(datadump, url=True))
1292 output += '\n'
1293 output += """\
1294Available neighbours:
1295"""
1296 output += make_table(get_neighbours(datadump))
1297
1298 return output
1299
1300
1301def format_yaml_value(value):
1302 """ Get yaml value in right syntax for outputting """
1303 if isinstance(value,str):
1304 output = '"%s"' % value
1305 else:
1306 output = value
1307 return output
1308
1309
1310
1311def format_wleiden_yaml(datadump):
1312 """ Special formatting to ensure it is editable"""
1313 output = "# Genesis config yaml style\n"
1314 output += "# vim:ts=2:et:sw=2:ai\n"
1315 output += "#\n"
1316 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1317 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1318 if key == 'rdr_rules':
1319 output += '%-10s:\n' % 'rdr_rules'
1320 for rdr_rule in datadump[key]:
1321 output += '- %s\n' % rdr_rule
1322 else:
1323 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1324
1325 output += "\n\n"
1326
1327 # Format (key, required)
1328 key_order = (
1329 ('comment', True),
1330 ('parent', False),
1331 ('ip', False),
1332 ('ether', False),
1333 ('desc', True),
1334 ('sdesc', True),
1335 ('mode', True),
1336 ('type', True),
1337 ('extra_type', False),
1338 ('channel', False),
1339 ('ssid', False),
1340 ('wlan_mac', False),
1341 ('dhcp', True),
1342 ('dhcp_fixed', False),
1343 ('compass', False),
1344 ('distance', False),
1345 ('ns_ip', False),
1346 ('repeater_ip', False),
1347 ('bullet2_ip', False),
1348 ('ns_mac', False),
1349 ('bullet2_mac', False),
1350 ('ns_type', False),
1351 ('bridge_type', False),
1352 ('status', True),
1353 )
1354
1355 for iface_key in sorted(iface_keys):
1356 try:
1357 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1358 if remainder:
1359 raise KeyError("invalid keys: %s" % remainder)
1360
1361 output += "%s:\n" % iface_key
1362 for key,required in key_order:
1363 if datadump[iface_key].has_key(key):
1364 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1365 output += "\n\n"
1366 except Exception:
1367 print "# Error while processing interface %s" % iface_key
1368 raise
1369
1370 return output
1371
1372
1373
1374def generate_wleiden_yaml(datadump, header=True):
1375 """ Generate (petty) version of wleiden.yaml"""
1376 output = generate_header(datadump, "#") if header else ''
1377
1378 for key in datadump.keys():
1379 if key.startswith('autogen_'):
1380 del datadump[key]
1381 # Interface autogen cleanups
1382 elif type(datadump[key]) == dict:
1383 for key2 in datadump[key].keys():
1384 if key2.startswith('autogen_'):
1385 del datadump[key][key2]
1386
1387 output += format_wleiden_yaml(datadump)
1388 return output
1389
1390def generate_nanostation_config(datadump, iface, ns_type):
1391 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1392 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1393
1394 datadump.update(datadump['iface_%s' % iface])
1395
1396 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1397
1398def generate_yaml(datadump):
1399 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1400
1401
1402
1403def generate_config(node, config, datadump=None):
1404 """ Print configuration file 'config' of 'node' """
1405 output = ""
1406 try:
1407 # Load config file
1408 if datadump == None:
1409 datadump = get_yaml(node)
1410
1411 if config == 'wleiden.yaml':
1412 output += generate_wleiden_yaml(datadump)
1413 elif config == 'authorized_keys':
1414 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1415 output += f.read()
1416 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1417 # Fetch local keys if existing
1418 if os.path.exists(node_keys):
1419 output += open(node_keys, 'r').read()
1420 f.close()
1421 elif config == 'dnsmasq.conf':
1422 output += generate_dnsmasq_conf(datadump)
1423 elif config == 'dhcpd.conf':
1424 output += generate_dhcpd_conf(datadump)
1425 elif config == 'rc.conf.local':
1426 output += generate_rc_conf_local(datadump)
1427 elif config == 'resolv.conf':
1428 output += generate_resolv_conf(datadump)
1429 elif config == 'ntp.conf':
1430 output += generate_ntp_conf(datadump)
1431 elif config == 'motd':
1432 output += generate_motd(datadump)
1433 elif config == 'pf.hybrid.conf.local':
1434 output += generate_pf_hybrid_conf_local(datadump)
1435 elif config == 'unbound.wleiden.conf':
1436 output += generate_unbound_wleiden_conf(datadump)
1437 elif config.startswith('vr'):
1438 interface, ns_type = config.strip('.yaml').split('-')
1439 output += generate_nanostation_config(datadump, interface, ns_type)
1440 else:
1441 assert False, "Config not found!"
1442 except IOError, e:
1443 output += "[ERROR] Config file not found"
1444 return output
1445
1446
1447
1448def process_cgi_request(environ=os.environ):
1449 """ When calling from CGI """
1450 response_headers = []
1451 content_type = 'text/plain'
1452
1453 # Update repository if requested
1454 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1455 if form and form.has_key("action") and "update" in form["action"]:
1456 output = "[INFO] Updating subverion, please wait...\n"
1457 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1458 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1459 output += "[INFO] All done, redirecting in 5 seconds"
1460 response_headers += [
1461 ('Refresh', '5; url=.'),
1462 ]
1463 reload_cache()
1464 else:
1465 base_uri = environ['PATH_INFO']
1466 uri = base_uri.strip('/').split('/')
1467
1468 output = "Template Holder"
1469 if base_uri.endswith('/create/network.kml'):
1470 content_type='application/vnd.google-earth.kml+xml'
1471 output = make_network_kml.make_graph()
1472 elif base_uri.endswith('/api/get/nodeplanner.json'):
1473 content_type='application/json'
1474 output = make_network_kml.make_nodeplanner_json()
1475 elif not uri[0]:
1476 if is_text_request(environ):
1477 output = '\n'.join(get_hostlist())
1478 else:
1479 content_type = 'text/html'
1480 output = generate_title(get_hostlist())
1481 elif len(uri) == 1:
1482 if is_text_request(environ):
1483 output = generate_node(uri[0])
1484 else:
1485 content_type = 'text/html'
1486 output = generate_node_overview(uri[0])
1487 elif len(uri) == 2:
1488 output = generate_config(uri[0], uri[1])
1489 else:
1490 assert False, "Invalid option"
1491
1492 # Return response
1493 response_headers += [
1494 ('Content-type', content_type),
1495 ('Content-Length', str(len(output))),
1496 ]
1497 return(response_headers, str(output))
1498
1499
1500def make_dns(output_dir = 'dns', external = False):
1501 items = dict()
1502
1503 # hostname is key, IP is value
1504 wleiden_zone = defaultdict(list)
1505 wleiden_cname = dict()
1506
1507 pool = dict()
1508 for node in get_hostlist():
1509 datadump = get_yaml(node)
1510
1511 fqdn = datadump['nodename']
1512
1513 if datadump.has_key('rdr_host'):
1514 remote_target = datadump['rdr_host']
1515 elif datadump.has_key('remote_access') and datadump['remote_access']:
1516 remote_target = datadump['remote_access'].split(':')[0]
1517 else:
1518 remote_target = None
1519
1520 if remote_target:
1521 try:
1522 parseaddr(remote_target)
1523 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1524 except (IndexError, ValueError):
1525 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1526
1527
1528 wleiden_zone[fqdn].append((datadump['masterip'], True))
1529
1530 # Hacking to get proper DHCP IPs and hostnames
1531 for iface_key in get_interface_keys(datadump):
1532 iface_name = iface_key.replace('_','-')
1533 if 'ip' in datadump[iface_key]:
1534 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1535 try:
1536 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1537 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1538 dhcp_part = ".".join(ip.split('.')[0:3])
1539 if ip != datadump['masterip']:
1540 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1541 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1542 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1543 except (AttributeError, ValueError, KeyError):
1544 # First push it into a pool, to indentify the counter-part later on
1545 addr = parseaddr(ip)
1546 cidr = int(cidr)
1547 addr = addr & ~((1 << (32 - cidr)) - 1)
1548 if pool.has_key(addr):
1549 pool[addr] += [(iface_name, fqdn, ip)]
1550 else:
1551 pool[addr] = [(iface_name, fqdn, ip)]
1552 continue
1553
1554
1555
1556 # WL uses an /29 to configure an interface. IP's are ordered like this:
1557 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1558
1559 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1560
1561 # Automatic naming convention of interlinks namely 2 + remote.lower()
1562 for (key,value) in pool.iteritems():
1563 # Make sure they are sorted from low-ip to high-ip
1564 value = sorted(value, key=lambda x: parseaddr(x[2]))
1565
1566 if len(value) == 1:
1567 (iface_name, fqdn, ip) = value[0]
1568 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1569
1570 # Device DNS names
1571 if 'cnode' in fqdn.lower():
1572 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1573 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1574
1575 elif len(value) == 2:
1576 (a_iface_name, a_fqdn, a_ip) = value[0]
1577 (b_iface_name, b_fqdn, b_ip) = value[1]
1578 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1579 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1580
1581 # Device DNS names
1582 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1583 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1584 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1585 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1586 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1587 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1588 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1589
1590 else:
1591 pool_members = [k[1] for k in value]
1592 for item in value:
1593 (iface_name, fqdn, ip) = item
1594 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1595
1596 # Include static DNS entries
1597 # XXX: Should they override the autogenerated results?
1598 # XXX: Convert input to yaml more useable.
1599 # Format:
1600 ##; this is a comment
1601 ## roomburgh=Roomburgh1
1602 ## apkerk1.Vosko=172.17.176.8 ;this as well
1603 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1604
1605 # Hack to allow special entries, for development
1606 wleiden_raw = {}
1607
1608 for line in dns_list:
1609 reverse = False
1610 k, items = line.items()[0]
1611 if type(items) == dict:
1612 if items.has_key('reverse'):
1613 reverse = items['reverse']
1614 items = items['a']
1615 else:
1616 items = items['cname']
1617 items = [items] if type(items) != list else items
1618 for item in items:
1619 if item.startswith('IN '):
1620 wleiden_raw[k] = item
1621 elif valid_addr(item):
1622 wleiden_zone[k].append((item, reverse))
1623 else:
1624 wleiden_cname[k] = item
1625
1626 # Hack to get dynamic pool listing
1627 def chunks(l, n):
1628 return [l[i:i+n] for i in range(0, len(l), n)]
1629
1630 ntp_servers = [x[0] for x in get_nameservers()]
1631 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1632 for ntp_server in chunk:
1633 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1634
1635 details = dict()
1636 # 24 updates a day allowed
1637 details['serial'] = time.strftime('%Y%m%d%H')
1638
1639 if external:
1640 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1641 else:
1642 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1643
1644 details['master'] = dns_masters[0]
1645 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1646
1647 dns_header = '''
1648$TTL 3h
1649%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1650 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1651
1652%(ns_servers)s
1653 \n'''
1654
1655
1656 if not os.path.isdir(output_dir):
1657 os.makedirs(output_dir)
1658 details['zone'] = 'wleiden.net'
1659 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1660 f.write(dns_header % details)
1661
1662 for host,items in wleiden_zone.iteritems():
1663 for ip,reverse in items:
1664 if ip not in ['0.0.0.0']:
1665 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1666 for source,dest in wleiden_cname.iteritems():
1667 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1668 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1669 for source, dest in wleiden_raw.iteritems():
1670 f.write("%s.wleiden.net. %s\n" % (source, dest))
1671 f.close()
1672
1673 # Create whole bunch of specific sub arpa zones. To keep it compliant
1674 for s in range(16,32):
1675 details['zone'] = '%i.172.in-addr.arpa' % s
1676 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1677 f.write(dns_header % details)
1678
1679 #XXX: Not effient, fix to proper data structure and do checks at other
1680 # stages
1681 for host,items in wleiden_zone.iteritems():
1682 for ip,reverse in items:
1683 if not reverse:
1684 continue
1685 if valid_addr(ip):
1686 if valid_addr(ip):
1687 if int(ip.split('.')[1]) == s:
1688 rev_ip = '.'.join(reversed(ip.split('.')))
1689 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1690 f.close()
1691
1692
1693def usage():
1694 print """Usage: %(prog)s <argument>
1695Argument:
1696\tcleanup = Cleanup all YAML files to specified format
1697\tstandalone [port] = Run configurator webserver [8000]
1698\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1699\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1700\tfull-export = Generate yaml export script for heatmap.
1701\tstatic [outputdir] = Generate all config files and store on disk
1702\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1703\ttest <node> [<file>] = Receive output for certain node [all files].
1704\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1705\tlist <status> <items> = List systems which have certain status
1706\tcreate network.kml = Create Network KML file for use in Google Earth
1707
1708Arguments:
1709\t<node> = NodeName (example: HybridRick)
1710\t<file> = %(files)s
1711\t<status> = all|up|down|planned
1712\t<items> = systems|nodes|proxies
1713
1714NOTE FOR DEVELOPERS; you can test your changes like this:
1715 BEFORE any changes in this code:
1716 $ ./gformat.py static /tmp/pre
1717 AFTER the changes:
1718 $ ./gformat.py static /tmp/post
1719 VIEW differences and VERIFY all are OK:
1720 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1721""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1722 exit(0)
1723
1724
1725def is_text_request(environ=os.environ):
1726 """ Find out whether we are calling from the CLI or any text based CLI utility """
1727 try:
1728 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1729 except KeyError:
1730 return True
1731
1732def switchFormat(setting):
1733 if setting:
1734 return "YES"
1735 else:
1736 return "NO"
1737
1738def rlinput(prompt, prefill=''):
1739 import readline
1740 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1741 try:
1742 return raw_input(prompt)
1743 finally:
1744 readline.set_startup_hook()
1745
1746def fix_conflict(left, right, default='i'):
1747 while True:
1748 print "## %-30s | %-30s" % (left, right)
1749 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1750 if not c:
1751 c = default
1752
1753 if c in ['l','1']:
1754 return left
1755 elif c in ['r','2']:
1756 return right
1757 elif c in ['e', '3']:
1758 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1759 elif c in ['i', '4']:
1760 return None
1761 else:
1762 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1763
1764
1765
1766def print_cgi_response(response_headers, output):
1767 """Could we not use some kind of wsgi wrapper to make this output?"""
1768 for header in response_headers:
1769 print "%s: %s" % header
1770 print
1771 print output
1772
1773
1774def fill_cache():
1775 ''' Poor man re-loading of few cache items (the slow ones) '''
1776 for host in get_hostlist():
1777 get_yaml(host)
1778
1779
1780def reload_cache():
1781 clear_cache()
1782 fill_cache()
1783
1784
1785def main():
1786 """Hard working sub"""
1787 # Allow easy hacking using the CLI
1788 if not os.environ.has_key('PATH_INFO'):
1789 if len(sys.argv) < 2:
1790 usage()
1791
1792 if sys.argv[1] == "standalone":
1793 import SocketServer
1794 import CGIHTTPServer
1795 # Hop to the right working directory.
1796 os.chdir(os.path.dirname(__file__))
1797 try:
1798 PORT = int(sys.argv[2])
1799 except (IndexError,ValueError):
1800 PORT = 8000
1801
1802 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1803 """ Serve this CGI from the root of the webserver """
1804 def is_cgi(self):
1805 if "favicon" in self.path:
1806 return False
1807
1808 self.cgi_info = (os.path.basename(__file__), self.path)
1809 self.path = ''
1810 return True
1811 handler = MyCGIHTTPRequestHandler
1812 SocketServer.TCPServer.allow_reuse_address = True
1813 httpd = SocketServer.TCPServer(("", PORT), handler)
1814 httpd.server_name = 'localhost'
1815 httpd.server_port = PORT
1816
1817 logger.info("serving at port %s", PORT)
1818 try:
1819 httpd.serve_forever()
1820 except KeyboardInterrupt:
1821 httpd.shutdown()
1822 logger.info("All done goodbye")
1823 elif sys.argv[1] == "test":
1824 # Basic argument validation
1825 try:
1826 node = sys.argv[2]
1827 except IndexError:
1828 print "Invalid argument"
1829 exit(1)
1830 except IOError as e:
1831 print e
1832 exit(1)
1833
1834 datadump = get_yaml(node)
1835
1836
1837 # Get files to generate
1838 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1839
1840 # Actual config generation
1841 for config in gen_files:
1842 logger.info("## Generating %s %s", node, config)
1843 print generate_config(node, config, datadump)
1844 elif sys.argv[1] == "test-cgi":
1845 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1846 os.environ['SCRIPT_NAME'] = __file__
1847 response_headers, output = process_cgi_request()
1848 print_cgi_response(response_headers, output)
1849 elif sys.argv[1] == "static":
1850 items = dict()
1851 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1852 for node in get_hostlist():
1853 items['node'] = node
1854 items['wdir'] = "%(output_dir)s/%(node)s" % items
1855 if not os.path.isdir(items['wdir']):
1856 os.makedirs(items['wdir'])
1857 datadump = get_yaml(node)
1858 for config in files:
1859 items['config'] = config
1860 logger.info("## Generating %(node)s %(config)s" % items)
1861 f = open("%(wdir)s/%(config)s" % items, "w")
1862 f.write(generate_config(node, config, datadump))
1863 f.close()
1864 elif sys.argv[1] == "wind-export":
1865 items = dict()
1866 for node in get_hostlist():
1867 datadump = get_yaml(node)
1868 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1869 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1870 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1871 VALUES (
1872 (SELECT id FROM users WHERE username = 'rvdzwet'),
1873 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1874 'Y');""" % datadump
1875 #for config in files:
1876 # items['config'] = config
1877 # print "## Generating %(node)s %(config)s" % items
1878 # f = open("%(wdir)s/%(config)s" % items, "w")
1879 # f.write(generate_config(node, config, datadump))
1880 # f.close()
1881 for node in get_hostlist():
1882 datadump = get_yaml(node)
1883 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1884 ifacedump = datadump[iface_key]
1885 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1886 ifacedump['nodename'] = datadump['nodename']
1887 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1888 ifacedump['channel'] = 0
1889 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1890 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1891 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1892 elif sys.argv[1] == "nagios-export":
1893 try:
1894 heavy_load = (sys.argv[2] == "--heavy-load")
1895 except IndexError:
1896 heavy_load = False
1897
1898 hostgroup_details = {
1899 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1900 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1901 'walphen' : 'Stichting Wireless Alphen',
1902 'westeinder' : 'Westeinder Plassen',
1903 }
1904
1905 # Convert IP to Host
1906 ip2host = {'root' : 'root'}
1907 for host in get_hostlist():
1908 datadump = get_yaml(host)
1909 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1910 for iface in get_interface_keys(datadump):
1911 if datadump[iface].has_key('autogen_gateway'):
1912 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1913
1914 # Find dependency tree based on output of lvrouted.mytree of nearest node
1915 parents = defaultdict(list)
1916 stack = ['root']
1917 prev_depth = 0
1918 for line in open('lvrouted.mytree').readlines():
1919 depth = line.count('\t')
1920 ip = line.strip().split()[0]
1921
1922 if prev_depth < depth:
1923 try:
1924 parents[ip2host[ip]].append(ip2host[stack[-1]])
1925 except KeyError as e:
1926 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1927 stack.append(ip)
1928 elif prev_depth > depth:
1929 stack = stack[:(depth - prev_depth)]
1930 elif prev_depth == depth:
1931 try:
1932 parents[ip2host[ip]].append(ip2host[stack[-1]])
1933 except KeyError as e:
1934 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1935
1936
1937 prev_depth = depth
1938 # Observe that some nodes has themself as parent or multiple parents
1939 # for now take only the first parent, other behaviour is yet to be explained
1940
1941
1942
1943 params = {
1944 'check_interval' : 5 if heavy_load else 120,
1945 'retry_interval' : 1 if heavy_load else 10,
1946 'max_check_attempts' : 10 if heavy_load else 6,
1947 'notification_interval': 120 if heavy_load else 240,
1948 }
1949
1950 print '''\
1951define host {
1952 name wleiden-node ; Default Node Template
1953 use generic-host ; Use the standard template as initial starting point
1954 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1955 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1956 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1957 notification_interval %(notification_interval)s
1958 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1959 check_command check-host-alive ; Default command to check FreeBSD hosts
1960 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1961}
1962
1963define service {
1964 name wleiden-service ; Default Service Template
1965 use generic-service ; Use the standard template as initial starting point
1966 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1967 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1968 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1969 notification_interval %(notification_interval)s
1970 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1971 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1972}
1973
1974# Please make sure to install:
1975# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1976#
1977# Recompile net-mgmt/nagios-plugins to support check_snmp
1978# make -C /usr/ports/net-mgmt/nagios-plugins
1979#
1980# Install net/bind-tools to allow v2/check_dns_wl to work:
1981# pkg install bind-tools
1982#
1983define command{
1984 command_name check_snmp_disk
1985 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1986}
1987
1988define command{
1989 command_name check_netsnmp_load
1990 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1991}
1992
1993define command{
1994 command_name check_netsnmp_proc
1995 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1996}
1997
1998define command{
1999 command_name check_by_ssh
2000 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2001}
2002
2003define command{
2004 command_name check_dns_wl
2005 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2006}
2007
2008define command{
2009 command_name check_snmp_uptime
2010 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2011}
2012
2013
2014# TDB: dhcp leases
2015# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2016
2017# TDB: internet status
2018# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2019
2020# TDB: Advanced local passive checks
2021# /usr/local/libexec/nagios/check_by_ssh
2022''' % params
2023
2024 print '''\
2025# Service Group, not displayed by default
2026define hostgroup {
2027 hostgroup_name srv_hybrid
2028 alias All Hybrid Nodes
2029 register 0
2030}
2031
2032define service {
2033 use wleiden-service
2034 hostgroup_name srv_hybrid
2035 service_description SSH
2036 check_command check_ssh
2037}
2038
2039define service {
2040 use wleiden-service,service-pnp
2041 hostgroup_name srv_hybrid
2042 service_description HTTP
2043 check_command check_http
2044}
2045
2046define service {
2047 use wleiden-service
2048 hostgroup_name srv_hybrid
2049 service_description DNS
2050 check_command check_dns_wl!"www.wirelessleiden.nl"
2051}
2052'''
2053
2054 if heavy_load:
2055 print '''\
2056define service {
2057 use wleiden-service
2058 hostgroup_name srv_hybrid
2059 service_description UPTIME
2060 check_command check_snmp_uptime
2061}
2062
2063#define service {
2064# use wleiden-service
2065# hostgroup_name srv_hybrid
2066# service_description NTP
2067# check_command check_ntp_peer
2068#}
2069
2070define service {
2071 use wleiden-service
2072 hostgroup_name srv_hybrid
2073 service_description LOAD
2074 check_command check_netsnmp_load
2075}
2076
2077define service {
2078 use wleiden-service
2079 hostgroup_name srv_hybrid
2080 service_description PROC
2081 check_command check_netsnmp_proc
2082}
2083
2084define service {
2085 use wleiden-service
2086 hostgroup_name srv_hybrid
2087 service_description DISK
2088 check_command check_snmp_disk
2089}
2090'''
2091 for node in get_hostlist():
2092 datadump = get_yaml(node)
2093 if not datadump['status'] == 'up':
2094 continue
2095 if not hostgroup_details.has_key(datadump['monitoring_group']):
2096 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2097 print '''\
2098define host {
2099 use wleiden-node,host-pnp
2100 contact_groups admins
2101 host_name %(autogen_fqdn)s
2102 address %(masterip)s
2103 hostgroups srv_hybrid,%(monitoring_group)s\
2104''' % datadump
2105 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2106 print '''\
2107 parents %(parents)s\
2108''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2109 print '''\
2110}
2111'''
2112
2113
2114 for name,alias in hostgroup_details.iteritems():
2115 print '''\
2116define hostgroup {
2117 hostgroup_name %s
2118 alias %s
2119} ''' % (name, alias)
2120
2121
2122 elif sys.argv[1] == "full-export":
2123 hosts = {}
2124 for node in get_hostlist():
2125 datadump = get_yaml(node)
2126 hosts[datadump['nodename']] = datadump
2127 print yaml.dump(hosts)
2128
2129 elif sys.argv[1] == "dns":
2130 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2131 elif sys.argv[1] == "cleanup":
2132 # First generate all datadumps
2133 datadumps = dict()
2134 ssid_to_node = dict()
2135 for host in get_hostlist():
2136 logger.info("# Processing: %s", host)
2137 # Set some boring default values
2138 datadump = { 'board' : 'UNKNOWN' }
2139 datadump.update(get_yaml(host))
2140 datadumps[datadump['nodename']] = datadump
2141
2142 (poel, errors) = make_relations()
2143 print "\n".join(["# WARNING: %s" % x for x in errors])
2144
2145 for host,datadump in datadumps.iteritems():
2146 try:
2147 # Convert all yes and no to boolean values
2148 def fix_boolean(dump):
2149 for key in dump.keys():
2150 if type(dump[key]) == dict:
2151 dump[key] = fix_boolean(dump[key])
2152 elif str(dump[key]).lower() in ["yes", "true"]:
2153 dump[key] = True
2154 elif str(dump[key]).lower() in ["no", "false"]:
2155 # Compass richting no (Noord Oost) is valid input
2156 if key != "compass": dump[key] = False
2157 return dump
2158 datadump = fix_boolean(datadump)
2159
2160 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2161 if not 'latitude' in datadump and not 'longitude' in datadump:
2162 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2163 elif 'latitude' in datadump and 'longitude' in datadump:
2164 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2165 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2166 # TODO: Compare outcome of both coordinate systems and validate against each-other
2167
2168 if datadump['nodename'].startswith('Proxy'):
2169 datadump['nodename'] = datadump['nodename'].lower()
2170
2171 for iface_key in get_interface_keys(datadump):
2172 try:
2173 # All our normal wireless cards are normal APs now
2174 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2175 datadump[iface_key]['mode'] = 'ap'
2176 # Wireless Leiden SSID have an consistent lowercase/uppercase
2177 if datadump[iface_key].has_key('ssid'):
2178 ssid = datadump[iface_key]['ssid']
2179 prefix = 'ap-WirelessLeiden-'
2180 if ssid.lower().startswith(prefix.lower()):
2181 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2182 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2183 datadump[iface_key]['mode'] = 'autogen-FIXME'
2184 if not datadump[iface_key].has_key('comment'):
2185 datadump[iface_key]['comment'] = 'autogen-FIXME'
2186
2187 if datadump[iface_key].has_key('ns_mac'):
2188 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2189
2190 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2191 datadump[iface_key] = datadump[iface_key]['desc']
2192
2193 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2194 # due to better collision avoidance.
2195 if datadump[iface_key]['type'] == '11b':
2196 datadump[iface_key]['type'] = '11g'
2197
2198 # Setting 802.11g channels to de-facto standards, to avoid
2199 # un-detected sharing with other overlapping channels
2200 #
2201 # Technically we could also use channel 13 in NL, but this is not
2202 # recommended as foreign devices might not be able to select this
2203 # channel. Secondly using 1,5,9,13 instead is going to clash with
2204 # the de-facto usage of 1,6,11.
2205 #
2206 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2207 channels_at_2400Mhz = (1,6,11)
2208 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2209 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2210 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2211 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2212
2213 # Mandatory interface keys
2214 if not datadump[iface_key].has_key('status'):
2215 datadump[iface_key]['status'] = 'planned'
2216
2217 x = datadump[iface_key]['comment']
2218 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2219
2220 # Fixing bridge_type if none is found
2221 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2222 if not 'bridge_type' in datadump[iface_key]:
2223 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2224
2225 # Making sure description works
2226 if datadump[iface_key].has_key('desc'):
2227 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2228 del datadump[iface_key]['desc']
2229 else:
2230 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2231 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2232 if response:
2233 datadump[iface_key]['comment'] = response
2234 del datadump[iface_key]['desc']
2235
2236 # Check DHCP configuration
2237 dhcp_type(datadump[iface_key])
2238
2239 # Set the compass value based on the angle between the poels
2240 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2241 my_pool = poel[network(datadump[iface_key]['ip'])]
2242 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2243 if remote_hosts:
2244 compass_target = remote_hosts[0]
2245 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2246 # TODO: Compass wanted and actual direction might differ
2247
2248 # Monitoring Group default
2249 if not 'monitoring_group' in datadump:
2250 datadump['monitoring_group'] = 'wleiden'
2251
2252 except Exception:
2253 print "# Error while processing interface %s" % iface_key
2254 raise
2255 store_yaml(datadump)
2256 except Exception:
2257 print "# Error while processing %s" % host
2258 raise
2259 elif sys.argv[1] == "list":
2260 use_fqdn = False
2261 if len(sys.argv) < 4:
2262 usage()
2263 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2264 usage()
2265 if not sys.argv[3] in ["nodes","proxies","systems"]:
2266 usage()
2267
2268 if len(sys.argv) > 4:
2269 if sys.argv[4] == "fqdn":
2270 use_fqdn = True
2271 else:
2272 usage()
2273
2274 for system in get_hostlist():
2275 datadump = get_yaml(system)
2276 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2277 continue
2278
2279 output = datadump['autogen_fqdn'] if use_fqdn else system
2280 if sys.argv[2] == "all":
2281 print output
2282 elif datadump['status'] == sys.argv[2]:
2283 print output
2284 elif sys.argv[1] == "create":
2285 if sys.argv[2] == "network.kml":
2286 print make_network_kml.make_graph()
2287 elif sys.argv[2] == "host-ips.txt":
2288 for system in get_hostlist():
2289 datadump = get_yaml(system)
2290 ips = [datadump['masterip']]
2291 for ifkey in get_interface_keys(datadump):
2292 ips.append(datadump[ifkey]['ip'].split('/')[0])
2293 print system, ' '.join(ips)
2294 elif sys.argv[2] == "host-pos.txt":
2295 for system in get_hostlist():
2296 datadump = get_yaml(system)
2297 print system, datadump['rdnap_x'], datadump['rdnap_y']
2298 elif sys.argv[2] == 'ssh_config':
2299 print '''
2300Host *.wleiden.net
2301 User root
2302
2303Host 172.16.*.*
2304 User root
2305'''
2306 for system in get_hostlist():
2307 datadump = get_yaml(system)
2308 print '''\
2309Host %s
2310 User root
2311
2312Host %s
2313 User root
2314
2315Host %s
2316 User root
2317
2318Host %s
2319 User root
2320''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2321 else:
2322 usage()
2323 else:
2324 usage()
2325 else:
2326 # Do not enable debugging for config requests as it highly clutters the output
2327 if not is_text_request():
2328 cgitb.enable()
2329 response_headers, output = process_cgi_request()
2330 print_cgi_response(response_headers, output)
2331
2332def application(environ, start_response):
2333 status = '200 OK'
2334 response_headers, output = process_cgi_request(environ)
2335 start_response(status, response_headers)
2336
2337 # Debugging only
2338 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2339 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2340 return [output]
2341
2342if __name__ == "__main__":
2343 main()
Note: See TracBrowser for help on using the repository browser.