source: genesis/tools/gformat.py@ 13916

Last change on this file since 13916 was 13916, checked in by rick, 8 years ago

Avoid query of system with broken upstream DNS

Since serviceid is removed when node is not connected to the internet.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 78.8 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# FreeBSD:
28# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
29# Fedora:
30# yum install python-yaml pyproj proj-epsg python-jinja2
31#
32# Rick van der Zwet <info@rickvanderzwet.nl>
33#
34
35# Hack to make the script directory is also threated as a module search path.
36import sys
37import os
38sys.path.append(os.path.dirname(__file__))
39
40SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
41
42import argparse
43import cgi
44import cgitb
45import copy
46import glob
47import make_network_kml
48import math
49import pyproj
50import random
51import re
52import socket
53import string
54import subprocess
55import textwrap
56import time
57import urlparse
58
59from pprint import pprint
60from collections import defaultdict, OrderedDict
61from sys import stderr
62try:
63 import yaml
64except ImportError, e:
65 print e
66 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
67 exit(1)
68
69try:
70 from yaml import CLoader as Loader
71 from yaml import CDumper as Dumper
72except ImportError:
73 from yaml import Loader, Dumper
74
75from jinja2 import Environment, Template
76def yesorno(value):
77 return "YES" if bool(value) else "NO"
78env = Environment()
79env.filters['yesorno'] = yesorno
80def render_template(datadump, template):
81 result = env.from_string(template).render(datadump)
82 # Make it look pretty to the naked eye, as jinja templates are not so
83 # friendly when it comes to whitespace formatting
84 ## Remove extra whitespace at end of line lstrip() style.
85 result = re.sub(r'\n[\ ]+','\n', result)
86 ## Include only a single newline between an definition and a comment
87 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
88 ## Remove extra newlines after single comment
89 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
90 return result
91
92import logging
93logging.basicConfig(format='# %(levelname)s: %(message)s' )
94logger = logging.getLogger()
95logger.setLevel(logging.DEBUG)
96
97
98if os.environ.has_key('CONFIGROOT'):
99 NODE_DIR = os.environ['CONFIGROOT']
100else:
101 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
102__version__ = '$Id: gformat.py 13916 2017-06-13 20:52:37Z rick $'
103
104files = [
105 'authorized_keys',
106 'dnsmasq.conf',
107 'dhcpd.conf',
108 'rc.conf.local',
109 'resolv.conf',
110 'motd',
111 'ntp.conf',
112 'pf.hybrid.conf.local',
113 'unbound.wleiden.conf',
114 'wleiden.yaml',
115 ]
116
117# Global variables uses
118OK = 10
119DOWN = 20
120UNKNOWN = 90
121
122
123ileiden_proxies = OrderedDict()
124normal_proxies = []
125datadump_cache = {}
126interface_list_cache = {}
127rc_conf_local_cache = {}
128nameservers_cache = []
129relations_cache = None
130def clear_cache():
131 ''' Poor mans cache implementation '''
132 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
133 datadump_cache = {}
134 interface_list_cache = {}
135 rc_conf_local_cache = {}
136 ileiden_proxies = OrderedDict()
137 normal_proxies = []
138 nameservers_cache = []
139 relations_cache = None
140
141NO_DHCP = 0
142DHCP_CLIENT = 10
143DHCP_SERVER = 20
144def dhcp_type(item):
145 if not item.has_key('dhcp'):
146 return NO_DHCP
147 elif not item['dhcp']:
148 return NO_DHCP
149 elif item['dhcp'].lower() == 'client':
150 return DHCP_CLIENT
151 else:
152 # Validation Checks
153 begin,end = map(int,item['dhcp'].split('-'))
154 if begin >= end:
155 raise ValueError("DHCP Start >= DHCP End")
156 return DHCP_SERVER
157
158def etrs2rd(lat, lon):
159 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
160 p2 = pyproj.Proj(init='EPSG:28992')
161 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
162 return (RDx, RDy)
163
164def rd2etrs(RDx, RDy):
165 p1 = pyproj.Proj(init='EPSG:28992')
166 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
167 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
168 return (lat, lon)
169
170def get_yaml(item,add_version_info=True):
171 try:
172 """ Get configuration yaml for 'item'"""
173 if datadump_cache.has_key(item):
174 return datadump_cache[item].copy()
175
176 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
177 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
178 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
179
180 # Default values
181 datadump = {
182 'autogen_revision' : 'NOTFOUND',
183 'autogen_gfile' : gfile,
184 'service_proxy_ileiden' : False,
185 }
186 f = open(gfile, 'r')
187 datadump.update(yaml.load(f,Loader=Loader))
188 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
189 if datadump['nodetype'] == 'Hybrid':
190 # Some values are defined implicitly
191 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
192 datadump['service_incoming_rdr'] = True
193 # Use some boring defaults
194 defaults = {
195 'service_proxy_normal' : False,
196 'service_accesspoint' : True,
197 'service_incoming_rdr' : False,
198 'service_concentrator' : False,
199 'monitoring_group' : 'wleiden',
200 }
201 for (key,value) in defaults.iteritems():
202 if not datadump.has_key(key):
203 datadump[key] = value
204 f.close()
205
206 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
207 if add_version_info:
208 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
209 lines = p.communicate()[0].split('\n')
210 if p.returncode == 0:
211 for line in lines:
212 if line:
213 (key, value) = line.split(': ')
214 datadump["autogen_" + key.lower().replace(' ','_')] = value
215
216 # Preformat certain needed variables for formatting and push those into special object
217 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
218
219 wlan_count=0
220 try:
221 for key in get_interface_keys(datadump, True):
222 datadump[key]['autogen_ifbase'] = key.split('_')[1]
223 datadump[key]['autogen_vlan'] = False
224
225 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
226 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
227
228 if datadump[key].has_key('parent'):
229 if datadump[key].has_key('ip'):
230 raise ValueError("Interface bridge member cannot have IP assigned")
231 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
232 raise ValueError("Interface bridge member cannot have DHCP set")
233
234 if datadump[key].has_key('ip'):
235 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
236
237 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
238 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
239 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
240 wlan_count += 1
241 else:
242 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
243 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
244 datadump[key]['autogen_vlan'] = key.split('_')[2]
245 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
246 else:
247 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
248
249 except Exception:
250 print "# Error while processing interface %s" % key
251 raise
252
253 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
254 if dhcp_type(datadump[key]) == DHCP_SERVER]
255
256 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
257 datadump['autogen_item'] = item
258
259 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
260 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
261 datadump_cache[item] = datadump.copy()
262 except Exception:
263 print "# Error while processing %s" % item
264 raise
265 return datadump
266
267
268def store_yaml(datadump, header=False):
269 """ Store configuration yaml for 'item'"""
270 item = datadump['autogen_item']
271 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
272
273 output = generate_wleiden_yaml(datadump, header)
274
275 f = open(gfile, 'w')
276 f.write(output)
277 f.close()
278
279
280def network(ip):
281 addr, mask = ip.split('/')
282 # Not parsing of these folks please
283 addr = parseaddr(addr)
284 mask = int(mask)
285 network = addr & ~((1 << (32 - mask)) - 1)
286 return network
287
288
289
290def make_relations():
291 """ Process _ALL_ yaml files to get connection relations """
292 global relations_cache
293
294 if relations_cache:
295 return relations_cache
296
297 errors = []
298 poel = defaultdict(list)
299
300 for host in get_hostlist():
301 datadump = get_yaml(host)
302 try:
303 for iface_key in get_interface_keys(datadump):
304 # Bridge members has no IP assigned
305 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
306 continue
307 net_addr = network(datadump[iface_key]['ip'])
308 poel[net_addr] += [(host,datadump[iface_key].copy())]
309 except (KeyError, ValueError) as e:
310 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
311 continue
312
313 relations_cache = (poel, errors)
314 return relations_cache
315
316
317
318def valid_addr(addr):
319 """ Show which address is valid in which are not """
320 return str(addr).startswith('172.')
321
322def get_hostlist():
323 """ Combined hosts and proxy list"""
324 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
325
326def angle_between_points(lat1,lat2,long1,long2):
327 """
328 Return Angle in radians between two GPS coordinates
329 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
330 """
331 dy = lat2 - lat1
332 dx = math.cos(lat1)*(long2 - long1)
333 angle = math.atan2(dy,dx)
334 return angle
335
336
337
338def angle_to_cd(angle):
339 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
340
341 # For easy conversion get positive degree
342 degrees = math.degrees(angle)
343 abs_degrees = 360 + degrees if degrees < 0 else degrees
344
345 # Numbers can be confusing calculate from the 4 main directions
346 p = 22.5
347 if abs_degrees < p:
348 cd = "n"
349 elif abs_degrees < (90 - p):
350 cd = "no"
351 elif abs_degrees < (90 + p):
352 cd = "o"
353 elif abs_degrees < (180 - p):
354 cd = "zo"
355 elif abs_degrees < (180 + p):
356 cd = "z"
357 elif abs_degrees < (270 - p):
358 cd = "zw"
359 elif abs_degrees < (270 + p):
360 cd = "w"
361 elif abs_degrees < (360 - p):
362 cd = "nw"
363 else:
364 cd = "n"
365 return cd
366
367
368
369def cd_between_hosts(hostA, hostB, datadumps):
370 # Using RDNAP coordinates
371 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
372 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
373 return angle_to_cd(math.atan2(dx,dy))
374
375 # GPS coordinates seems to fail somehow
376 #latA = float(datadumps[hostA]['latitude'])
377 #latB = float(datadumps[hostB]['latitude'])
378 #lonA = float(datadumps[hostA]['longitude'])
379 #lonB = float(datadumps[hostB]['longitude'])
380 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
381
382
383def generate_title(nodelist):
384 """ Main overview page """
385 items = {'root' : "." }
386 def fl(spaces, line):
387 return (' ' * spaces) + line + '\n'
388
389 output = """
390<html>
391 <head>
392 <title>Wireless leiden Configurator - GFormat</title>
393 <style type="text/css">
394 th {background-color: #999999}
395 tr:nth-child(odd) {background-color: #cccccc}
396 tr:nth-child(even) {background-color: #ffffff}
397 th, td {padding: 0.1em 1em}
398 </style>
399 </head>
400 <body>
401 <center>
402 <form type="GET" action="%(root)s">
403 <input type="hidden" name="action" value="update">
404 <input type="submit" value="Update Configuration Database (SVN)">
405 </form>
406 <table>
407 <caption><h3>Wireless Leiden Configurator</h3></caption>
408 """ % items
409
410 for node in nodelist:
411 items['node'] = node
412 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
413 for config in files:
414 items['config'] = config
415 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
416 output += fl(5, "</tr>")
417 output += """
418 </table>
419 <hr />
420 <em>%s</em>
421 </center>
422 </body>
423</html>
424 """ % __version__
425
426 return output
427
428
429
430def generate_node(node):
431 """ Print overview of all files available for node """
432 return "\n".join(files)
433
434def generate_node_overview(host):
435 """ Print overview of all files available for node """
436 datadump = get_yaml(host)
437 params = { 'host' : host }
438 output = "<em><a href='..'>Back to overview</a></em><hr />"
439 output += "<h2>Available files:</h2><ul>"
440 for cf in files:
441 params['cf'] = cf
442 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
443 output += "</ul>"
444
445 # Generate and connection listing
446 output += "<h2>Connected To:</h2><ul>"
447 (poel, errors) = make_relations()
448 for network, hosts in poel.iteritems():
449 if host in [x[0] for x in hosts]:
450 if len(hosts) == 1:
451 # Single not connected interface
452 continue
453 for remote,ifacedump in hosts:
454 if remote == host:
455 # This side of the interface
456 continue
457 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
458 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
459 output += "</ul>"
460 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
461
462 output += "<hr /><em><a href='..'>Back to overview</a></em>"
463 return output
464
465
466def generate_header(datadump, ctag="#"):
467 return """\
468%(ctag)s
469%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
470%(ctag)s
471""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
472
473
474
475def parseaddr(s):
476 """ Process IPv4 CIDR notation addr to a (binary) number """
477 f = s.split('.')
478 return (long(f[0]) << 24L) + \
479 (long(f[1]) << 16L) + \
480 (long(f[2]) << 8L) + \
481 long(f[3])
482
483
484
485def showaddr(a):
486 """ Display IPv4 addr in (dotted) CIDR notation """
487 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
488
489
490def is_member(ip, mask, canidate):
491 """ Return True if canidate is part of ip/mask block"""
492 ip_addr = parseaddr(ip)
493 ip_canidate = parseaddr(canidate)
494 mask = int(mask)
495 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
496 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
497 return ip_addr == ip_canidate
498
499
500
501def cidr2netmask(netmask):
502 """ Given a 'netmask' return corresponding CIDR """
503 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
504
505def get_network(addr, mask):
506 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
507
508
509def generate_dhcpd_conf(datadump):
510 """ Generate config file '/usr/local/etc/dhcpd.conf """
511 # Redundency support, in cause local DNS server is not running/responding.
512 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
513 output = generate_header(datadump)
514 output += Template("""\
515# option definitions common to all supported networks...
516option domain-name "dhcp.{{ autogen_fqdn }}";
517
518default-lease-time 600;
519max-lease-time 7200;
520
521# Use this to enble / disable dynamic dns updates globally.
522#ddns-update-style none;
523
524# If this DHCP server is the official DHCP server for the local
525# network, the authoritative directive should be uncommented.
526authoritative;
527
528# Use this to send dhcp log messages to a different log file (you also
529# have to hack syslog.conf to complete the redirection).
530log-facility local7;
531
532# UniFi Discovery Support
533option space ubnt;
534option ubnt.unifi-address code 1 = ip-address;
535#
536class "ubnt" {
537 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
538 option vendor-class-identifier "ubnt";
539 vendor-option-space ubnt;
540}
541
542#
543# Interface definitions
544#
545\n\n""").render(datadump)
546
547
548 # TODO: Use textwrap.fill instead
549 def indent(text, count):
550 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
551
552 # Process DHCP blocks
553 dhcp_out = defaultdict(list)
554 for iface_key in get_interface_keys(datadump):
555 ifname = datadump[iface_key]['autogen_ifbase']
556 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
557 if not datadump[iface_key].has_key('comment'):
558 datadump[iface_key]['comment'] = None
559
560 if not datadump[iface_key].has_key('ip'):
561 continue
562
563 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
564
565 (addr, mask) = datadump[iface_key]['ip'].split('/')
566 datadump[iface_key]['autogen_addr'] = addr
567 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
568 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
569
570 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
571 dhcp_out[groupif].append(textwrap.dedent("""\
572 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
573 ### not autoritive
574 }
575 """ % datadump[iface_key]))
576 continue
577
578 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
579 dhcp_part = ".".join(addr.split('.')[0:3])
580 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
581 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
582 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
583
584 # Assume the first 10 IPs could be used for static entries
585 if 'no_portal' in datadump:
586 fixed = 5
587 for mac in datadump['no_portal']:
588 dhcp_out[groupif].append(textwrap.dedent("""\
589 host fixed-%(ifname)s-%(fixed)s {
590 hardware ethernet %(mac)s;
591 fixed-address %(prefix)s.%(fixed)s;
592 }
593 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
594 fixed += 1
595
596 if 'dhcp_fixed' in datadump[iface_key]:
597 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
598 dhcp_out[groupif].append(textwrap.dedent("""\
599 host fixed-%(host)s {
600 hardware ethernet %(mac)s;
601 fixed-address %(addr)s;
602 }
603 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
604
605
606 dhcp_out[groupif].append(textwrap.dedent("""\
607 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
608 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
609 option routers %(autogen_addr)s;
610 option domain-name-servers %(autogen_dns_servers)s;
611 option ubnt.unifi-address 172.17.107.10;
612 }
613 """ % datadump[iface_key]))
614
615 # Output the blocks in groups
616 for ifname,value in sorted(dhcp_out.iteritems()):
617 if len(value) > 2:
618 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
619 else:
620 output += ''.join(value) + "\n\n"
621 return output
622
623
624
625def generate_dnsmasq_conf(datadump):
626 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
627 output = generate_header(datadump)
628 output += Template("""\
629# DHCP server options
630dhcp-authoritative
631dhcp-fqdn
632domain=dhcp.{{ autogen_fqdn }}
633domain-needed
634expand-hosts
635log-async=100
636
637# Low memory footprint
638cache-size=10000
639
640\n""").render(datadump)
641
642 for iface_key in get_interface_keys(datadump):
643 if not datadump[iface_key].has_key('comment'):
644 datadump[iface_key]['comment'] = None
645 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
646
647 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
648 output += "# not autoritive\n\n"
649 continue
650
651 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
652 (ip, cidr) = datadump[iface_key]['ip'].split('/')
653 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
654
655 dhcp_part = ".".join(ip.split('.')[0:3])
656 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
657 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
658 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
659
660 return output
661
662
663class AutoVivification(dict):
664 """Implementation of perl's autovivification feature."""
665 def __getitem__(self, item):
666 try:
667 return dict.__getitem__(self, item)
668 except KeyError:
669 value = self[item] = type(self)()
670 return value
671
672def make_interface_list(datadump):
673 if interface_list_cache.has_key(datadump['autogen_item']):
674 return (interface_list_cache[datadump['autogen_item']])
675 # lo0 configuration:
676 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
677 # - masterip is special as it needs to be assigned to at
678 # least one interface, so if not used assign to lo0
679 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
680 vlan_list = defaultdict(list)
681 bridge_list = defaultdict(list)
682 flags_if = AutoVivification()
683 dhclient_if = {'lo0' : False}
684
685 # XXX: Find some way of send this output nicely
686 output = ''
687
688 masterip_used = False
689 for iface_key in get_interface_keys(datadump):
690 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
691 masterip_used = True
692 break
693 if not masterip_used:
694 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
695
696 if 'serviceid' in datadump:
697 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
698
699 for iface_key in get_interface_keys(datadump):
700 ifacedump = datadump[iface_key]
701 ifname = ifacedump['autogen_ifname']
702
703 # If defined as vlan interface
704 if ifacedump['autogen_vlan']:
705 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
706
707 # If defined as bridge interface
708 if ifacedump['autogen_bridge_member']:
709 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
710
711 # Flag dhclient is possible
712 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
713 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
714
715 # Ethernet address
716 if ifacedump.has_key('ether'):
717 flags_if[ifname]['ether'] = ifacedump['ether']
718
719 # Add interface IP to list
720 if ifacedump.has_key('ip'):
721 item = (ifacedump['ip'], ifacedump['comment'])
722 if addrs_list.has_key(ifname):
723 addrs_list[ifname].append(item)
724 else:
725 addrs_list[ifname] = [item]
726
727 # Alias only needs IP assignment for now, this might change if we
728 # are going to use virtual accesspoints
729 if "alias" in iface_key:
730 continue
731
732 # XXX: Might want to deduct type directly from interface name
733 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
734 # Default to station (client) mode
735 ifacedump['autogen_wlanmode'] = "sta"
736 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
737 ifacedump['autogen_wlanmode'] = "ap"
738
739 if not ifacedump.has_key('channel'):
740 if ifacedump['type'] == '11a':
741 ifacedump['channel'] = 36
742 else:
743 ifacedump['channel'] = 1
744
745 # Allow special hacks at the back like wds and stuff
746 if not ifacedump.has_key('extra'):
747 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
748 else:
749 ifacedump['autogen_extra'] = ifacedump['extra']
750
751 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
752
753 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
754 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
755 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
756 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
757 output += "\n"
758
759 elif ifacedump['type'] in ['ethernet', 'eth']:
760 # No special config needed besides IP
761 pass
762 elif ifacedump['type'] in ['vlan']:
763 # VLAN member has no special configuration
764 pass
765 else:
766 assert False, "Unknown type " + ifacedump['type']
767
768 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
769 interface_list_cache[datadump['autogen_item']] = store
770 return(store)
771
772
773
774def create_proxies_list():
775 if not ileiden_proxies or not normal_proxies:
776 # Placeholder for to-be-installed proxies, this will avoid updating the all
777 # nodes to include this new machine, yet due to an unbound issue, this list
778 # has to be kept small.
779
780 for i in range(1,20):
781 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
782
783 for host in get_hostlist():
784 hostdump = get_yaml(host)
785 if hostdump['status'] == 'up':
786 if hostdump['service_proxy_ileiden']:
787 ileiden_proxies[hostdump['serviceid']] = hostdump
788 if hostdump['service_proxy_normal']:
789 normal_proxies.append(hostdump)
790
791
792
793def generate_rc_conf_local(datadump):
794 """ Generate configuration file '/etc/rc.conf.local' """
795 item = datadump['autogen_item']
796 if rc_conf_local_cache.has_key(item):
797 return rc_conf_local_cache[item]
798
799 if not datadump.has_key('ileiden'):
800 datadump['autogen_ileiden_enable'] = False
801 else:
802 datadump['autogen_ileiden_enable'] = datadump['ileiden']
803
804 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
805
806 create_proxies_list()
807 datadump['autogen_ileiden_proxies'] = ileiden_proxies
808 datadump['autogen_normal_proxies'] = normal_proxies
809 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
810 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
811 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
812 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
813
814 output = generate_header(datadump, "#");
815 output += render_template(datadump, """\
816hostname='{{ autogen_fqdn }}'
817location='{{ location }}'
818nodetype="{{ nodetype }}"
819
820#
821# Configured listings
822#
823captive_portal_whitelist=""
824{% if nodetype == "Proxy" %}
825#
826# Proxy Configuration
827#
828{% if gateway and service_proxy_ileiden -%}
829defaultrouter="{{ gateway }}"
830{% else -%}
831#defaultrouter="NOTSET"
832{% endif -%}
833internalif="{{ internalif }}"
834ileiden_enable="{{ autogen_ileiden_enable }}"
835gateway_enable="{{ autogen_ileiden_enable }}"
836pf_enable="yes"
837pf_rules="/etc/pf.conf"
838{% if autogen_ileiden_enable -%}
839pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
840lvrouted_enable="{{ autogen_ileiden_enable }}"
841lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
842{% else -%}
843pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
844{% endif -%}
845{% if internalroute -%}
846static_routes="wleiden"
847route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
848{% endif -%}
849
850{% elif nodetype == "Hybrid" %}
851 #
852 # Hybrid Configuration
853 #
854 list_ileiden_proxies="
855 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
856 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
857 {% endfor -%}
858 "
859 list_normal_proxies="
860 {% for item in autogen_normal_proxies -%}
861 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
862 {% endfor -%}
863 "
864
865 {% if autogen_dhcp_interfaces -%}
866 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
867 {% else %}
868 captive_portal_interfaces="dummy"
869 {% endif %}
870 externalif="{{ externalif|default('vr0', true) }}"
871 masterip="{{ masterip }}"
872
873 {% if gateway and service_proxy_ileiden %}
874 defaultrouter="{{ gateway }}"
875 {% else %}
876 #defaultrouter="NOTSET"
877 {% endif %}
878
879 #
880 # Defined services
881 #
882 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
883 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
884 service_accesspoint="{{ service_accesspoint|yesorno }}"
885 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
886 service_concentrator="{{ service_concentrator|yesorno }}"
887
888 {% if service_proxy_ileiden %}
889 pf_rules="/etc/pf.hybrid.conf"
890 {% if service_concentrator %}
891 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
892 {% else %}
893 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
894 {% endif %}
895 pf_flags="$pf_flags -D publicnat=80,443"
896 lvrouted_flags="$lvrouted_flags -g"
897 {% elif service_proxy_normal or service_incoming_rdr %}
898 pf_rules="/etc/pf.hybrid.conf"
899 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
900 pf_flags="$pf_flags -D publicnat=0"
901 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
902 named_setfib="1"
903 tinyproxy_setfib="1"
904 dnsmasq_setfib="1"
905 sshd_setfib="1"
906 {% else %}
907 named_auto_forward_only="YES"
908 pf_rules="/etc/pf.node.conf"
909 pf_flags=""
910 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
911 {% endif %}
912 {% if service_concentrator %}
913 # Do mind installing certificates is NOT done automatically for security reasons
914 openvpn_enable="YES"
915 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
916 {% endif %}
917
918 {% if service_proxy_normal %}
919 tinyproxy_enable="yes"
920 {% else %}
921 pen_wrapper_enable="yes"
922 {% endif %}
923
924 {% if service_accesspoint %}
925 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
926 {% endif %}
927
928 {% if board == "ALIX2" %}
929 #
930 # ''Fat'' configuration, board has 256MB RAM
931 #
932 dnsmasq_enable="NO"
933 named_enable="YES"
934 unbound_enable="YES"
935 {% if autogen_dhcp_interfaces -%}
936 dhcpd_enable="YES"
937 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
938 {% endif -%}
939 {% elif board == "apu1d" %}
940 #
941 # ''Fat'' configuration, board has 1024MB RAM
942 #
943 dnsmasq_enable="NO"
944 unbound_enable="YES"
945 {% if autogen_dhcp_interfaces -%}
946 dhcpd_enable="YES"
947 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
948 {% endif -%}
949 {% endif -%}
950{% endif %}
951
952#
953# Script variables
954#
955attached_devices="{{ autogen_attached_devices|join(' ') }}"
956neighbours="{{ autogen_neighbours|join(' ') }}"
957
958
959#
960# Interface definitions
961#\n
962""")
963
964 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
965 for iface, vlans in sorted(vlan_list.items()):
966 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
967
968 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
969 for iface in sorted(vlan_list.keys()):
970 if not iface in addrs_list.keys():
971 output += "ifconfig_%s='up'\n" % iface
972
973 output += "\n"
974
975 # Bridge configuration:
976 if bridge_list.keys():
977 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
978
979 for iface in bridge_list.keys():
980 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
981
982 # Bridge member interfaces not containing a configuration should be marked active explcitly.
983 for _,members in bridge_list.items():
984 for iface in members:
985 if not iface in addrs_list.keys():
986 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
987
988 output += "\n"
989
990 # Details like SSID
991 if extra_ouput:
992 output += extra_ouput.strip() + "\n"
993
994 # Print IP address which needs to be assigned over here
995 output += "\n"
996 for iface,addrs in sorted(addrs_list.iteritems()):
997 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
998 output += "# %s || %s || %s\n" % (iface, addr, comment)
999
1000 # Write DHCLIENT entry
1001 if iface in dhclient_if and dhclient_if[iface]:
1002 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1003 continue
1004
1005 # Make sure the external address is always first as this is needed in the
1006 # firewall setup
1007 addrs = sorted(
1008 [x for x in addrs if not '0.0.0.0' in x[0]],
1009 key=lambda x: x[0].split('.')[0],
1010 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1011 )
1012
1013 idx_offset = 0
1014 # Set MAC is required
1015 if flags_if[iface].has_key('ether'):
1016 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1017 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1018 idx_offset += 1
1019 else:
1020 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1021
1022 for idx, addr in enumerate(addrs[1:]):
1023 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1024
1025 output += "\n"
1026
1027 rc_conf_local_cache[datadump['autogen_item']] = output
1028 return output
1029
1030
1031
1032def get_all_configs():
1033 """ Get dict with key 'host' with all configs present """
1034 configs = dict()
1035 for host in get_hostlist():
1036 datadump = get_yaml(host)
1037 configs[host] = datadump
1038 return configs
1039
1040
1041def get_interface_keys(config, extra=False):
1042 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1043 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1044 if extra == False:
1045 return filter(lambda x: not "extra" in x, elems)
1046 else:
1047 return elems
1048
1049
1050def get_used_ips(configs):
1051 """ Return array of all IPs used in config files"""
1052 ip_list = []
1053 for config in configs:
1054 ip_list.append(config['masterip'])
1055 if 'serviceid' in config:
1056 ip_list.append(config['serviceid'])
1057 for iface_key in get_interface_keys(config, True):
1058 l = config[iface_key]['ip']
1059 addr, mask = l.split('/')
1060 # Special case do not process
1061 if valid_addr(addr):
1062 ip_list.append(addr)
1063 else:
1064 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1065 return sorted(ip_list)
1066
1067
1068
1069def get_nameservers(max_servers=None):
1070 if nameservers_cache:
1071 return nameservers_cache[0:max_servers]
1072
1073 for host in get_hostlist():
1074 hostdump = get_yaml(host)
1075 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1076 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1077
1078 return nameservers_cache[0:max_servers]
1079
1080
1081def get_neighbours(datadump):
1082 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1083
1084 (poel, errors) = make_relations()
1085 table = []
1086 for iface,addrs in sorted(addrs_list.iteritems()):
1087 if iface in ['lo0']:
1088 continue
1089
1090 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1091 if not addr.startswith('172.'):
1092 # Avoid listing internet connections as pool
1093 continue
1094 for neighbour in poel[network(addr)]:
1095 if neighbour[0] != datadump['autogen_item']:
1096 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1097 return table
1098
1099
1100def get_attached_devices(datadump, url=False):
1101 table = []
1102 for iface_key in get_interface_keys(datadump, True):
1103 ifacedump = datadump[iface_key]
1104
1105 if not ifacedump.has_key('ns_ip'):
1106 continue
1107
1108 x_ip = ifacedump['ns_ip'].split('/')[0]
1109
1110 if 'mode' in ifacedump:
1111 x_mode = ifacedump['mode']
1112 else:
1113 x_mode = 'unknown'
1114
1115 if 'bridge_type' in ifacedump:
1116 device_type = ifacedump['bridge_type']
1117 else:
1118 device_type = 'Unknown'
1119
1120 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1121 return table
1122
1123
1124def generate_resolv_conf(datadump):
1125 """ Generate configuration file '/etc/resolv.conf' """
1126 # XXX: This should properly going to be an datastructure soon
1127 datadump['autogen_header'] = generate_header(datadump, "#")
1128 datadump['autogen_edge_nameservers'] = ''
1129
1130
1131 for masterip,realname in get_nameservers():
1132 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1133
1134 return Template("""\
1135{{ autogen_header }}
1136search wleiden.net
1137
1138# Try local (cache) first
1139nameserver 127.0.0.1
1140
1141{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1142nameserver 8.8.8.8 # Google Public NameServer
1143nameserver 64.6.64.6 # Verisign Public NameServer
1144{% else -%}
1145# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1146{{ autogen_edge_nameservers }}
1147{% endif -%}
1148""").render(datadump)
1149
1150
1151
1152def generate_ntp_conf(datadump):
1153 """ Generate configuration file '/etc/ntp.conf' """
1154 # XXX: This should properly going to be an datastructure soon
1155
1156 datadump['autogen_header'] = generate_header(datadump, "#")
1157 datadump['autogen_ntp_servers'] = ''
1158 for host in get_hostlist():
1159 hostdump = get_yaml(host)
1160 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1161 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1162
1163 return Template("""\
1164{{ autogen_header }}
1165
1166{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1167# Machine hooked to internet.
1168server 0.nl.pool.ntp.org iburst maxpoll 9
1169server 1.nl.pool.ntp.org iburst maxpoll 9
1170server 2.nl.pool.ntp.org iburst maxpoll 9
1171server 3.nl.pool.ntp.org iburst maxpoll 9
1172{% else -%}
1173# Local Wireless Leiden NTP Servers.
1174server 0.pool.ntp.wleiden.net iburst maxpoll 9
1175server 1.pool.ntp.wleiden.net iburst maxpoll 9
1176server 2.pool.ntp.wleiden.net iburst maxpoll 9
1177server 3.pool.ntp.wleiden.net iburst maxpoll 9
1178
1179# All the configured NTP servers
1180{{ autogen_ntp_servers }}
1181{% endif %}
1182
1183# If a server loses sync with all upstream servers, NTP clients
1184# no longer follow that server. The local clock can be configured
1185# to provide a time source when this happens, but it should usually
1186# be configured on just one server on a network. For more details see
1187# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1188# The use of Orphan Mode may be preferable.
1189#
1190server 127.127.1.0
1191fudge 127.127.1.0 stratum 10
1192""").render(datadump)
1193
1194
1195def generate_pf_hybrid_conf_local(datadump):
1196 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1197 datadump['autogen_header'] = generate_header(datadump, "#")
1198 if datadump['service_incoming_rdr']:
1199 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1200 return Template("""\
1201{{ autogen_header }}
1202
1203# Redirect some internal facing services outside (7)
1204# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1205{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1206rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1207{% endfor -%}
1208# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1209{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1210rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1211{% endfor -%}
1212""").render(datadump)
1213
1214def generate_unbound_wleiden_conf(datadump):
1215 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1216 datadump['autogen_header'] = generate_header(datadump, "#")
1217
1218 autogen_ips = []
1219 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1220 for iface,addrs in sorted(addrs_list.iteritems()):
1221 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1222 if addr.startswith('172'):
1223 autogen_ips.append((addr.split('/')[0], comment))
1224 datadump['autogen_ips'] = autogen_ips
1225
1226 create_proxies_list()
1227 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1228 return Template("""\
1229{{ autogen_header }}
1230
1231server:
1232 ## Static definitions fail to start on systems with broken ue(4) interfaces
1233{%- for ip,comment in autogen_ips %}
1234 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1235{%- endfor %}
1236 ## Enabling wildcard matching as work-around
1237 interface: 0.0.0.0
1238 interface: ::0
1239
1240forward-zone:
1241 name: '.'
1242{%- if service_proxy_ileiden %}
1243 forward-addr: 8.8.8.8 # Google DNS A
1244 forward-addr: 8.8.4.4 # Google DNS B
1245 forward-addr: 208.67.222.222 # OpenDNS DNS A
1246 forward-addr: 208.67.220.220 # OpenDNS DNS B
1247{% else -%}
1248{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1249 {%- if loop.index <= 5 %}
1250 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1251 {%- endif %}
1252{%- endfor %}
1253{%- endif %}
1254""").render(datadump)
1255
1256def generate_motd(datadump):
1257 """ Generate configuration file '/etc/motd' """
1258 output = Template("""\
1259FreeBSD run ``service motd onestart'' to make me look normal
1260
1261 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1262 Loc: {{ location }}
1263
1264Services:
1265{% if board == "ALIX2" -%}
1266{{" -"}} Core Node ({{ board }})
1267{% else -%}
1268{{" -"}} Hulp Node ({{ board }})
1269{% endif -%}
1270{% if service_proxy_normal -%}
1271{{" -"}} Normal Proxy
1272{% endif -%}
1273{% if service_proxy_ileiden -%}
1274{{" -"}} iLeiden Proxy
1275{% endif -%}
1276{% if service_incoming_rdr -%}
1277{{" -"}} Incoming port redirects
1278{% endif %}
1279Interlinks:\n
1280""").render(datadump)
1281
1282
1283 def make_table(table):
1284 if not table:
1285 return " - none\n"
1286 else:
1287 lines = ""
1288 col_width = [max(len(x) for x in col) for col in zip(*table)]
1289 for row in table:
1290 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1291 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1292 return lines
1293
1294 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1295 table = []
1296 for iface,addrs in sorted(addrs_list.iteritems()):
1297 if iface in ['lo0']:
1298 continue
1299 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1300 table.append((iface, addr, comment))
1301
1302 output += make_table(table)
1303 output += '\n'
1304 output += """\
1305Attached devices:
1306"""
1307 output += make_table(get_attached_devices(datadump, url=True))
1308 output += '\n'
1309 output += """\
1310Available neighbours:
1311"""
1312 output += make_table(get_neighbours(datadump))
1313
1314 return output
1315
1316
1317def format_yaml_value(value):
1318 """ Get yaml value in right syntax for outputting """
1319 if isinstance(value,str):
1320 output = '"%s"' % value
1321 else:
1322 output = value
1323 return output
1324
1325
1326
1327def format_wleiden_yaml(datadump):
1328 """ Special formatting to ensure it is editable"""
1329 output = "# Genesis config yaml style\n"
1330 output += "# vim:ts=2:et:sw=2:ai\n"
1331 output += "#\n"
1332 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1333 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1334 if key == 'rdr_rules':
1335 output += '%-10s:\n' % 'rdr_rules'
1336 for rdr_rule in datadump[key]:
1337 output += '- %s\n' % rdr_rule
1338 else:
1339 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1340
1341 output += "\n\n"
1342
1343 # Format (key, required)
1344 key_order = (
1345 ('comment', True),
1346 ('parent', False),
1347 ('ip', False),
1348 ('ether', False),
1349 ('desc', True),
1350 ('sdesc', True),
1351 ('mode', True),
1352 ('type', True),
1353 ('extra_type', False),
1354 ('channel', False),
1355 ('ssid', False),
1356 ('wlan_mac', False),
1357 ('dhcp', True),
1358 ('dhcp_fixed', False),
1359 ('compass', False),
1360 ('distance', False),
1361 ('ns_ip', False),
1362 ('repeater_ip', False),
1363 ('bullet2_ip', False),
1364 ('ns_mac', False),
1365 ('bullet2_mac', False),
1366 ('ns_type', False),
1367 ('bridge_type', False),
1368 ('status', True),
1369 )
1370
1371 for iface_key in sorted(iface_keys):
1372 try:
1373 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1374 if remainder:
1375 raise KeyError("invalid keys: %s" % remainder)
1376
1377 output += "%s:\n" % iface_key
1378 for key,required in key_order:
1379 if datadump[iface_key].has_key(key):
1380 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1381 output += "\n\n"
1382 except Exception:
1383 print "# Error while processing interface %s" % iface_key
1384 raise
1385
1386 return output
1387
1388
1389
1390def generate_wleiden_yaml(datadump, header=True):
1391 """ Generate (petty) version of wleiden.yaml"""
1392 output = generate_header(datadump, "#") if header else ''
1393
1394 for key in datadump.keys():
1395 if key.startswith('autogen_'):
1396 del datadump[key]
1397 # Interface autogen cleanups
1398 elif type(datadump[key]) == dict:
1399 for key2 in datadump[key].keys():
1400 if key2.startswith('autogen_'):
1401 del datadump[key][key2]
1402
1403 output += format_wleiden_yaml(datadump)
1404 return output
1405
1406def generate_nanostation_config(datadump, iface, ns_type):
1407 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1408 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1409
1410 datadump.update(datadump['iface_%s' % iface])
1411
1412 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1413
1414def generate_yaml(datadump):
1415 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1416
1417
1418
1419def generate_config(node, config, datadump=None):
1420 """ Print configuration file 'config' of 'node' """
1421 output = ""
1422 try:
1423 # Load config file
1424 if datadump == None:
1425 datadump = get_yaml(node)
1426
1427 if config == 'wleiden.yaml':
1428 output += generate_wleiden_yaml(datadump)
1429 elif config == 'authorized_keys':
1430 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1431 output += f.read()
1432 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1433 # Fetch local keys if existing
1434 if os.path.exists(node_keys):
1435 output += open(node_keys, 'r').read()
1436 f.close()
1437 elif config == 'dnsmasq.conf':
1438 output += generate_dnsmasq_conf(datadump)
1439 elif config == 'dhcpd.conf':
1440 output += generate_dhcpd_conf(datadump)
1441 elif config == 'rc.conf.local':
1442 output += generate_rc_conf_local(datadump)
1443 elif config == 'resolv.conf':
1444 output += generate_resolv_conf(datadump)
1445 elif config == 'ntp.conf':
1446 output += generate_ntp_conf(datadump)
1447 elif config == 'motd':
1448 output += generate_motd(datadump)
1449 elif config == 'pf.hybrid.conf.local':
1450 output += generate_pf_hybrid_conf_local(datadump)
1451 elif config == 'unbound.wleiden.conf':
1452 output += generate_unbound_wleiden_conf(datadump)
1453 elif config.startswith('vr'):
1454 interface, ns_type = config.strip('.yaml').split('-')
1455 output += generate_nanostation_config(datadump, interface, ns_type)
1456 else:
1457 assert False, "Config not found!"
1458 except IOError, e:
1459 output += "[ERROR] Config file not found"
1460 return output
1461
1462
1463
1464def process_cgi_request(environ=os.environ):
1465 """ When calling from CGI """
1466 response_headers = []
1467 content_type = 'text/plain'
1468
1469 # Update repository if requested
1470 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1471 if form and form.has_key("action") and "update" in form["action"]:
1472 output = "[INFO] Updating subverion, please wait...\n"
1473 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1474 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1475 output += "[INFO] All done, redirecting in 5 seconds"
1476 response_headers += [
1477 ('Refresh', '5; url=.'),
1478 ]
1479 reload_cache()
1480 else:
1481 base_uri = environ['PATH_INFO']
1482 uri = base_uri.strip('/').split('/')
1483
1484 output = "Template Holder"
1485 if base_uri.endswith('/create/network.kml'):
1486 content_type='application/vnd.google-earth.kml+xml'
1487 output = make_network_kml.make_graph()
1488 elif base_uri.endswith('/api/get/nodeplanner.json'):
1489 content_type='application/json'
1490 output = make_network_kml.make_nodeplanner_json()
1491 elif not uri[0]:
1492 if is_text_request(environ):
1493 output = '\n'.join(get_hostlist())
1494 else:
1495 content_type = 'text/html'
1496 output = generate_title(get_hostlist())
1497 elif len(uri) == 1:
1498 if is_text_request(environ):
1499 output = generate_node(uri[0])
1500 else:
1501 content_type = 'text/html'
1502 output = generate_node_overview(uri[0])
1503 elif len(uri) == 2:
1504 output = generate_config(uri[0], uri[1])
1505 else:
1506 assert False, "Invalid option"
1507
1508 # Return response
1509 response_headers += [
1510 ('Content-type', content_type),
1511 ('Content-Length', str(len(output))),
1512 ]
1513 return(response_headers, str(output))
1514
1515
1516def make_dns(output_dir = 'dns', external = False):
1517 items = dict()
1518
1519 # hostname is key, IP is value
1520 wleiden_zone = defaultdict(list)
1521 wleiden_cname = dict()
1522
1523 pool = dict()
1524 for node in get_hostlist():
1525 datadump = get_yaml(node)
1526
1527 fqdn = datadump['nodename']
1528
1529 if datadump.has_key('rdr_host'):
1530 remote_target = datadump['rdr_host']
1531 elif datadump.has_key('remote_access') and datadump['remote_access']:
1532 remote_target = datadump['remote_access'].split(':')[0]
1533 else:
1534 remote_target = None
1535
1536 if remote_target:
1537 try:
1538 parseaddr(remote_target)
1539 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1540 except (IndexError, ValueError):
1541 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1542
1543
1544 wleiden_zone[fqdn].append((datadump['masterip'], True))
1545
1546 # Hacking to get proper DHCP IPs and hostnames
1547 for iface_key in get_interface_keys(datadump):
1548 iface_name = iface_key.replace('_','-')
1549 if 'ip' in datadump[iface_key]:
1550 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1551 try:
1552 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1553 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1554 dhcp_part = ".".join(ip.split('.')[0:3])
1555 if ip != datadump['masterip']:
1556 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1557 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1558 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1559 except (AttributeError, ValueError, KeyError):
1560 # First push it into a pool, to indentify the counter-part later on
1561 addr = parseaddr(ip)
1562 cidr = int(cidr)
1563 addr = addr & ~((1 << (32 - cidr)) - 1)
1564 if pool.has_key(addr):
1565 pool[addr] += [(iface_name, fqdn, ip)]
1566 else:
1567 pool[addr] = [(iface_name, fqdn, ip)]
1568 continue
1569
1570
1571
1572 # WL uses an /29 to configure an interface. IP's are ordered like this:
1573 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1574
1575 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1576
1577 # Automatic naming convention of interlinks namely 2 + remote.lower()
1578 for (key,value) in pool.iteritems():
1579 # Make sure they are sorted from low-ip to high-ip
1580 value = sorted(value, key=lambda x: parseaddr(x[2]))
1581
1582 if len(value) == 1:
1583 (iface_name, fqdn, ip) = value[0]
1584 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1585
1586 # Device DNS names
1587 if 'cnode' in fqdn.lower():
1588 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1589 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1590
1591 elif len(value) == 2:
1592 (a_iface_name, a_fqdn, a_ip) = value[0]
1593 (b_iface_name, b_fqdn, b_ip) = value[1]
1594 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1595 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1596
1597 # Device DNS names
1598 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1599 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1600 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1601 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1602 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1603 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1604 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1605
1606 else:
1607 pool_members = [k[1] for k in value]
1608 for item in value:
1609 (iface_name, fqdn, ip) = item
1610 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1611
1612 # Include static DNS entries
1613 # XXX: Should they override the autogenerated results?
1614 # XXX: Convert input to yaml more useable.
1615 # Format:
1616 ##; this is a comment
1617 ## roomburgh=Roomburgh1
1618 ## apkerk1.Vosko=172.17.176.8 ;this as well
1619 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1620
1621 # Hack to allow special entries, for development
1622 wleiden_raw = {}
1623
1624 for line in dns_list:
1625 reverse = False
1626 k, items = line.items()[0]
1627 if type(items) == dict:
1628 if items.has_key('reverse'):
1629 reverse = items['reverse']
1630 items = items['a']
1631 else:
1632 items = items['cname']
1633 items = [items] if type(items) != list else items
1634 for item in items:
1635 if item.startswith('IN '):
1636 wleiden_raw[k] = item
1637 elif valid_addr(item):
1638 wleiden_zone[k].append((item, reverse))
1639 else:
1640 wleiden_cname[k] = item
1641
1642 # Hack to get dynamic pool listing
1643 def chunks(l, n):
1644 return [l[i:i+n] for i in range(0, len(l), n)]
1645
1646 ntp_servers = [x[0] for x in get_nameservers()]
1647 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1648 for ntp_server in chunk:
1649 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1650
1651 details = dict()
1652 # 24 updates a day allowed
1653 details['serial'] = time.strftime('%Y%m%d%H')
1654
1655 if external:
1656 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1657 else:
1658 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1659
1660 details['master'] = dns_masters[0]
1661 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1662
1663 dns_header = '''
1664$TTL 3h
1665%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1666 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1667
1668%(ns_servers)s
1669 \n'''
1670
1671
1672 if not os.path.isdir(output_dir):
1673 os.makedirs(output_dir)
1674 details['zone'] = 'wleiden.net'
1675 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1676 f.write(dns_header % details)
1677
1678 for host,items in wleiden_zone.iteritems():
1679 for ip,reverse in items:
1680 if ip not in ['0.0.0.0']:
1681 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1682 for source,dest in wleiden_cname.iteritems():
1683 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1684 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1685 for source, dest in wleiden_raw.iteritems():
1686 f.write("%s.wleiden.net. %s\n" % (source, dest))
1687 f.close()
1688
1689 # Create whole bunch of specific sub arpa zones. To keep it compliant
1690 for s in range(16,32):
1691 details['zone'] = '%i.172.in-addr.arpa' % s
1692 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1693 f.write(dns_header % details)
1694
1695 #XXX: Not effient, fix to proper data structure and do checks at other
1696 # stages
1697 for host,items in wleiden_zone.iteritems():
1698 for ip,reverse in items:
1699 if not reverse:
1700 continue
1701 if valid_addr(ip):
1702 if valid_addr(ip):
1703 if int(ip.split('.')[1]) == s:
1704 rev_ip = '.'.join(reversed(ip.split('.')))
1705 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1706 f.close()
1707
1708
1709def usage():
1710 print """Usage: %(prog)s <argument>
1711Argument:
1712\tcleanup = Cleanup all YAML files to specified format
1713\tstandalone [port] = Run configurator webserver [8000]
1714\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1715\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1716\tfull-export = Generate yaml export script for heatmap.
1717\tstatic [outputdir] = Generate all config files and store on disk
1718\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1719\ttest <node> [<file>] = Receive output for certain node [all files].
1720\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1721\tlist <status> <items> = List systems which have certain status
1722\tcreate network.kml = Create Network KML file for use in Google Earth
1723
1724Arguments:
1725\t<node> = NodeName (example: HybridRick)
1726\t<file> = %(files)s
1727\t<status> = all|up|down|planned
1728\t<items> = systems|nodes|proxies
1729
1730NOTE FOR DEVELOPERS; you can test your changes like this:
1731 BEFORE any changes in this code:
1732 $ ./gformat.py static /tmp/pre
1733 AFTER the changes:
1734 $ ./gformat.py static /tmp/post
1735 VIEW differences and VERIFY all are OK:
1736 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1737""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1738 exit(0)
1739
1740
1741def is_text_request(environ=os.environ):
1742 """ Find out whether we are calling from the CLI or any text based CLI utility """
1743 try:
1744 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1745 except KeyError:
1746 return True
1747
1748def switchFormat(setting):
1749 if setting:
1750 return "YES"
1751 else:
1752 return "NO"
1753
1754def rlinput(prompt, prefill=''):
1755 import readline
1756 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1757 try:
1758 return raw_input(prompt)
1759 finally:
1760 readline.set_startup_hook()
1761
1762def fix_conflict(left, right, default='i'):
1763 while True:
1764 print "## %-30s | %-30s" % (left, right)
1765 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1766 if not c:
1767 c = default
1768
1769 if c in ['l','1']:
1770 return left
1771 elif c in ['r','2']:
1772 return right
1773 elif c in ['e', '3']:
1774 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1775 elif c in ['i', '4']:
1776 return None
1777 else:
1778 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1779
1780
1781
1782def print_cgi_response(response_headers, output):
1783 """Could we not use some kind of wsgi wrapper to make this output?"""
1784 for header in response_headers:
1785 print "%s: %s" % header
1786 print
1787 print output
1788
1789
1790def fill_cache():
1791 ''' Poor man re-loading of few cache items (the slow ones) '''
1792 for host in get_hostlist():
1793 get_yaml(host)
1794
1795
1796def reload_cache():
1797 clear_cache()
1798 fill_cache()
1799
1800
1801def main():
1802 """Hard working sub"""
1803 # Allow easy hacking using the CLI
1804 if not os.environ.has_key('PATH_INFO'):
1805 if len(sys.argv) < 2:
1806 usage()
1807
1808 if sys.argv[1] == "standalone":
1809 import SocketServer
1810 import CGIHTTPServer
1811 # Hop to the right working directory.
1812 os.chdir(os.path.dirname(__file__))
1813 try:
1814 PORT = int(sys.argv[2])
1815 except (IndexError,ValueError):
1816 PORT = 8000
1817
1818 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1819 """ Serve this CGI from the root of the webserver """
1820 def is_cgi(self):
1821 if "favicon" in self.path:
1822 return False
1823
1824 self.cgi_info = (os.path.basename(__file__), self.path)
1825 self.path = ''
1826 return True
1827 handler = MyCGIHTTPRequestHandler
1828 SocketServer.TCPServer.allow_reuse_address = True
1829 httpd = SocketServer.TCPServer(("", PORT), handler)
1830 httpd.server_name = 'localhost'
1831 httpd.server_port = PORT
1832
1833 logger.info("serving at port %s", PORT)
1834 try:
1835 httpd.serve_forever()
1836 except KeyboardInterrupt:
1837 httpd.shutdown()
1838 logger.info("All done goodbye")
1839 elif sys.argv[1] == "test":
1840 # Basic argument validation
1841 try:
1842 node = sys.argv[2]
1843 except IndexError:
1844 print "Invalid argument"
1845 exit(1)
1846 except IOError as e:
1847 print e
1848 exit(1)
1849
1850 datadump = get_yaml(node)
1851
1852
1853 # Get files to generate
1854 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1855
1856 # Actual config generation
1857 for config in gen_files:
1858 logger.info("## Generating %s %s", node, config)
1859 print generate_config(node, config, datadump)
1860 elif sys.argv[1] == "test-cgi":
1861 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1862 os.environ['SCRIPT_NAME'] = __file__
1863 response_headers, output = process_cgi_request()
1864 print_cgi_response(response_headers, output)
1865 elif sys.argv[1] == "static":
1866 items = dict()
1867 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1868 for node in get_hostlist():
1869 items['node'] = node
1870 items['wdir'] = "%(output_dir)s/%(node)s" % items
1871 if not os.path.isdir(items['wdir']):
1872 os.makedirs(items['wdir'])
1873 datadump = get_yaml(node)
1874 for config in files:
1875 items['config'] = config
1876 logger.info("## Generating %(node)s %(config)s" % items)
1877 f = open("%(wdir)s/%(config)s" % items, "w")
1878 f.write(generate_config(node, config, datadump))
1879 f.close()
1880 elif sys.argv[1] == "wind-export":
1881 items = dict()
1882 for node in get_hostlist():
1883 datadump = get_yaml(node)
1884 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1885 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1886 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1887 VALUES (
1888 (SELECT id FROM users WHERE username = 'rvdzwet'),
1889 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1890 'Y');""" % datadump
1891 #for config in files:
1892 # items['config'] = config
1893 # print "## Generating %(node)s %(config)s" % items
1894 # f = open("%(wdir)s/%(config)s" % items, "w")
1895 # f.write(generate_config(node, config, datadump))
1896 # f.close()
1897 for node in get_hostlist():
1898 datadump = get_yaml(node)
1899 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1900 ifacedump = datadump[iface_key]
1901 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1902 ifacedump['nodename'] = datadump['nodename']
1903 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1904 ifacedump['channel'] = 0
1905 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1906 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1907 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1908 elif sys.argv[1] == "nagios-export":
1909 try:
1910 heavy_load = (sys.argv[2] == "--heavy-load")
1911 except IndexError:
1912 heavy_load = False
1913
1914 hostgroup_details = {
1915 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1916 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1917 'walphen' : 'Stichting Wireless Alphen',
1918 'westeinder' : 'Westeinder Plassen',
1919 }
1920
1921 # Convert IP to Host
1922 ip2host = {'root' : 'root'}
1923 for host in get_hostlist():
1924 datadump = get_yaml(host)
1925 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1926 for iface in get_interface_keys(datadump):
1927 if datadump[iface].has_key('autogen_gateway'):
1928 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1929
1930 # Find dependency tree based on output of lvrouted.mytree of nearest node
1931 parents = defaultdict(list)
1932 stack = ['root']
1933 prev_depth = 0
1934 for line in open('lvrouted.mytree').readlines():
1935 depth = line.count('\t')
1936 ip = line.strip().split()[0]
1937
1938 if prev_depth < depth:
1939 try:
1940 parents[ip2host[ip]].append(ip2host[stack[-1]])
1941 except KeyError as e:
1942 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1943 stack.append(ip)
1944 elif prev_depth > depth:
1945 stack = stack[:(depth - prev_depth)]
1946 elif prev_depth == depth:
1947 try:
1948 parents[ip2host[ip]].append(ip2host[stack[-1]])
1949 except KeyError as e:
1950 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1951
1952
1953 prev_depth = depth
1954 # Observe that some nodes has themself as parent or multiple parents
1955 # for now take only the first parent, other behaviour is yet to be explained
1956
1957
1958
1959 params = {
1960 'check_interval' : 5 if heavy_load else 120,
1961 'retry_interval' : 1 if heavy_load else 10,
1962 'max_check_attempts' : 10 if heavy_load else 6,
1963 'notification_interval': 120 if heavy_load else 240,
1964 }
1965
1966 print '''\
1967define host {
1968 name wleiden-node ; Default Node Template
1969 use generic-host ; Use the standard template as initial starting point
1970 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1971 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1972 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1973 notification_interval %(notification_interval)s
1974 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1975 check_command check-host-alive ; Default command to check FreeBSD hosts
1976 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1977}
1978
1979define service {
1980 name wleiden-service ; Default Service Template
1981 use generic-service ; Use the standard template as initial starting point
1982 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1983 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1984 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1985 notification_interval %(notification_interval)s
1986 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1987 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1988}
1989
1990# Please make sure to install:
1991# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1992#
1993# Recompile net-mgmt/nagios-plugins to support check_snmp
1994# make -C /usr/ports/net-mgmt/nagios-plugins
1995#
1996# Install net/bind-tools to allow v2/check_dns_wl to work:
1997# pkg install bind-tools
1998#
1999define command{
2000 command_name check_snmp_disk
2001 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2002}
2003
2004define command{
2005 command_name check_netsnmp_load
2006 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2007}
2008
2009define command{
2010 command_name check_netsnmp_proc
2011 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2012}
2013
2014define command{
2015 command_name check_by_ssh
2016 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2017}
2018
2019define command{
2020 command_name check_dns_wl
2021 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2022}
2023
2024define command{
2025 command_name check_snmp_uptime
2026 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2027}
2028
2029
2030# TDB: dhcp leases
2031# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2032
2033# TDB: internet status
2034# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2035
2036# TDB: Advanced local passive checks
2037# /usr/local/libexec/nagios/check_by_ssh
2038''' % params
2039
2040 print '''\
2041# Service Group, not displayed by default
2042define hostgroup {
2043 hostgroup_name srv_hybrid
2044 alias All Hybrid Nodes
2045 register 0
2046}
2047
2048define service {
2049 use wleiden-service
2050 hostgroup_name srv_hybrid
2051 service_description SSH
2052 check_command check_ssh
2053}
2054
2055define service {
2056 use wleiden-service,service-pnp
2057 hostgroup_name srv_hybrid
2058 service_description HTTP
2059 check_command check_http
2060}
2061
2062define service {
2063 use wleiden-service
2064 hostgroup_name srv_hybrid
2065 service_description DNS
2066 check_command check_dns_wl!"www.wirelessleiden.nl"
2067}
2068'''
2069
2070 if heavy_load:
2071 print '''\
2072define service {
2073 use wleiden-service
2074 hostgroup_name srv_hybrid
2075 service_description UPTIME
2076 check_command check_snmp_uptime
2077}
2078
2079#define service {
2080# use wleiden-service
2081# hostgroup_name srv_hybrid
2082# service_description NTP
2083# check_command check_ntp_peer
2084#}
2085
2086define service {
2087 use wleiden-service
2088 hostgroup_name srv_hybrid
2089 service_description LOAD
2090 check_command check_netsnmp_load
2091}
2092
2093define service {
2094 use wleiden-service
2095 hostgroup_name srv_hybrid
2096 service_description PROC
2097 check_command check_netsnmp_proc
2098}
2099
2100define service {
2101 use wleiden-service
2102 hostgroup_name srv_hybrid
2103 service_description DISK
2104 check_command check_snmp_disk
2105}
2106'''
2107 for node in get_hostlist():
2108 datadump = get_yaml(node)
2109 if not datadump['status'] == 'up':
2110 continue
2111 if not hostgroup_details.has_key(datadump['monitoring_group']):
2112 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2113 print '''\
2114define host {
2115 use wleiden-node,host-pnp
2116 contact_groups admins
2117 host_name %(autogen_fqdn)s
2118 address %(masterip)s
2119 hostgroups srv_hybrid,%(monitoring_group)s\
2120''' % datadump
2121 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2122 print '''\
2123 parents %(parents)s\
2124''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2125 print '''\
2126}
2127'''
2128
2129
2130 for name,alias in hostgroup_details.iteritems():
2131 print '''\
2132define hostgroup {
2133 hostgroup_name %s
2134 alias %s
2135} ''' % (name, alias)
2136
2137
2138 elif sys.argv[1] == "full-export":
2139 hosts = {}
2140 for node in get_hostlist():
2141 datadump = get_yaml(node)
2142 hosts[datadump['nodename']] = datadump
2143 print yaml.dump(hosts)
2144
2145 elif sys.argv[1] == "dns":
2146 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2147 elif sys.argv[1] == "cleanup":
2148 # First generate all datadumps
2149 datadumps = dict()
2150 ssid_to_node = dict()
2151 for host in get_hostlist():
2152 logger.info("# Processing: %s", host)
2153 # Set some boring default values
2154 datadump = { 'board' : 'UNKNOWN' }
2155 datadump.update(get_yaml(host))
2156 datadumps[datadump['nodename']] = datadump
2157
2158 (poel, errors) = make_relations()
2159 print "\n".join(["# WARNING: %s" % x for x in errors])
2160
2161 for host,datadump in datadumps.iteritems():
2162 try:
2163 # Convert all yes and no to boolean values
2164 def fix_boolean(dump):
2165 for key in dump.keys():
2166 if type(dump[key]) == dict:
2167 dump[key] = fix_boolean(dump[key])
2168 elif str(dump[key]).lower() in ["yes", "true"]:
2169 dump[key] = True
2170 elif str(dump[key]).lower() in ["no", "false"]:
2171 # Compass richting no (Noord Oost) is valid input
2172 if key != "compass": dump[key] = False
2173 return dump
2174 datadump = fix_boolean(datadump)
2175
2176 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2177 if not 'latitude' in datadump and not 'longitude' in datadump:
2178 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2179 elif 'latitude' in datadump and 'longitude' in datadump:
2180 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2181 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2182 # TODO: Compare outcome of both coordinate systems and validate against each-other
2183
2184 if datadump['nodename'].startswith('Proxy'):
2185 datadump['nodename'] = datadump['nodename'].lower()
2186
2187 for iface_key in get_interface_keys(datadump):
2188 try:
2189 # All our normal wireless cards are normal APs now
2190 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2191 datadump[iface_key]['mode'] = 'ap'
2192 # Wireless Leiden SSID have an consistent lowercase/uppercase
2193 if datadump[iface_key].has_key('ssid'):
2194 ssid = datadump[iface_key]['ssid']
2195 prefix = 'ap-WirelessLeiden-'
2196 if ssid.lower().startswith(prefix.lower()):
2197 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2198 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2199 datadump[iface_key]['mode'] = 'autogen-FIXME'
2200 if not datadump[iface_key].has_key('comment'):
2201 datadump[iface_key]['comment'] = 'autogen-FIXME'
2202
2203 if datadump[iface_key].has_key('ns_mac'):
2204 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2205
2206 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2207 datadump[iface_key] = datadump[iface_key]['desc']
2208
2209 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2210 # due to better collision avoidance.
2211 if datadump[iface_key]['type'] == '11b':
2212 datadump[iface_key]['type'] = '11g'
2213
2214 # Setting 802.11g channels to de-facto standards, to avoid
2215 # un-detected sharing with other overlapping channels
2216 #
2217 # Technically we could also use channel 13 in NL, but this is not
2218 # recommended as foreign devices might not be able to select this
2219 # channel. Secondly using 1,5,9,13 instead is going to clash with
2220 # the de-facto usage of 1,6,11.
2221 #
2222 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2223 channels_at_2400Mhz = (1,6,11)
2224 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2225 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2226 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2227 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2228
2229 # Mandatory interface keys
2230 if not datadump[iface_key].has_key('status'):
2231 datadump[iface_key]['status'] = 'planned'
2232
2233 x = datadump[iface_key]['comment']
2234 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2235
2236 # Fixing bridge_type if none is found
2237 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2238 if not 'bridge_type' in datadump[iface_key]:
2239 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2240
2241 # Making sure description works
2242 if datadump[iface_key].has_key('desc'):
2243 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2244 del datadump[iface_key]['desc']
2245 else:
2246 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2247 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2248 if response:
2249 datadump[iface_key]['comment'] = response
2250 del datadump[iface_key]['desc']
2251
2252 # Check DHCP configuration
2253 dhcp_type(datadump[iface_key])
2254
2255 # Set the compass value based on the angle between the poels
2256 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2257 my_pool = poel[network(datadump[iface_key]['ip'])]
2258 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2259 if remote_hosts:
2260 compass_target = remote_hosts[0]
2261 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2262 # TODO: Compass wanted and actual direction might differ
2263
2264 # Monitoring Group default
2265 if not 'monitoring_group' in datadump:
2266 datadump['monitoring_group'] = 'wleiden'
2267
2268 except Exception:
2269 print "# Error while processing interface %s" % iface_key
2270 raise
2271 store_yaml(datadump)
2272 except Exception:
2273 print "# Error while processing %s" % host
2274 raise
2275 elif sys.argv[1] == "list":
2276 use_fqdn = False
2277 if len(sys.argv) < 4:
2278 usage()
2279 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2280 usage()
2281 if not sys.argv[3] in ["nodes","proxies","systems"]:
2282 usage()
2283
2284 if len(sys.argv) > 4:
2285 if sys.argv[4] == "fqdn":
2286 use_fqdn = True
2287 else:
2288 usage()
2289
2290 for system in get_hostlist():
2291 datadump = get_yaml(system)
2292 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2293 continue
2294
2295 output = datadump['autogen_fqdn'] if use_fqdn else system
2296 if sys.argv[2] == "all":
2297 print output
2298 elif datadump['status'] == sys.argv[2]:
2299 print output
2300 elif sys.argv[1] == "create":
2301 if sys.argv[2] == "network.kml":
2302 print make_network_kml.make_graph()
2303 elif sys.argv[2] == "host-ips.txt":
2304 for system in get_hostlist():
2305 datadump = get_yaml(system)
2306 ips = [datadump['masterip']]
2307 for ifkey in get_interface_keys(datadump):
2308 ips.append(datadump[ifkey]['ip'].split('/')[0])
2309 print system, ' '.join(ips)
2310 elif sys.argv[2] == "host-pos.txt":
2311 for system in get_hostlist():
2312 datadump = get_yaml(system)
2313 print system, datadump['rdnap_x'], datadump['rdnap_y']
2314 elif sys.argv[2] == 'ssh_config':
2315 print '''
2316Host *.wleiden.net
2317 User root
2318
2319Host 172.16.*.*
2320 User root
2321'''
2322 for system in get_hostlist():
2323 datadump = get_yaml(system)
2324 print '''\
2325Host %s
2326 User root
2327
2328Host %s
2329 User root
2330
2331Host %s
2332 User root
2333
2334Host %s
2335 User root
2336''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2337 else:
2338 usage()
2339 else:
2340 usage()
2341 else:
2342 # Do not enable debugging for config requests as it highly clutters the output
2343 if not is_text_request():
2344 cgitb.enable()
2345 response_headers, output = process_cgi_request()
2346 print_cgi_response(response_headers, output)
2347
2348def application(environ, start_response):
2349 status = '200 OK'
2350 response_headers, output = process_cgi_request(environ)
2351 start_response(status, response_headers)
2352
2353 # Debugging only
2354 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2355 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2356 return [output]
2357
2358if __name__ == "__main__":
2359 main()
Note: See TracBrowser for help on using the repository browser.