source: genesis/tools/gformat.py@ 13706

Last change on this file since 13706 was 13705, checked in by rick, 8 years ago

No dhcp allowed when interface is bridge member.

Added pre-checking validation, since it is an likely canidate for confusing and
committing the duplicates will cause the config files generation to fail.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 75.9 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict, OrderedDict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13705 2017-01-15 22:25:04Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'unbound.wleiden.conf',
111 'wleiden.yaml',
112 ]
113
114# Global variables uses
115OK = 10
116DOWN = 20
117UNKNOWN = 90
118
119
120ileiden_proxies = OrderedDict()
121normal_proxies = []
122datadump_cache = {}
123interface_list_cache = {}
124rc_conf_local_cache = {}
125nameservers_cache = []
126relations_cache = None
127def clear_cache():
128 ''' Poor mans cache implementation '''
129 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
130 datadump_cache = {}
131 interface_list_cache = {}
132 rc_conf_local_cache = {}
133 ileiden_proxies = OrderedDict()
134 normal_proxies = []
135 nameservers_cache = []
136 relations_cache = None
137
138NO_DHCP = 0
139DHCP_CLIENT = 10
140DHCP_SERVER = 20
141def dhcp_type(item):
142 if not item.has_key('dhcp'):
143 return NO_DHCP
144 elif not item['dhcp']:
145 return NO_DHCP
146 elif item['dhcp'].lower() == 'client':
147 return DHCP_CLIENT
148 else:
149 # Validation Checks
150 begin,end = map(int,item['dhcp'].split('-'))
151 if begin >= end:
152 raise ValueError("DHCP Start >= DHCP End")
153 return DHCP_SERVER
154
155def etrs2rd(lat, lon):
156 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
157 p2 = pyproj.Proj(init='EPSG:28992')
158 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
159 return (RDx, RDy)
160
161def rd2etrs(RDx, RDy):
162 p1 = pyproj.Proj(init='EPSG:28992')
163 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
164 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
165 return (lat, lon)
166
167def get_yaml(item,add_version_info=True):
168 try:
169 """ Get configuration yaml for 'item'"""
170 if datadump_cache.has_key(item):
171 return datadump_cache[item].copy()
172
173 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
174
175 # Default values
176 datadump = {
177 'autogen_revision' : 'NOTFOUND',
178 'autogen_gfile' : gfile,
179 'service_proxy_ileiden' : False,
180 }
181 f = open(gfile, 'r')
182 datadump.update(yaml.load(f,Loader=Loader))
183 if datadump['nodetype'] == 'Hybrid':
184 # Some values are defined implicitly
185 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
186 datadump['service_incoming_rdr'] = True
187 # Use some boring defaults
188 defaults = {
189 'service_proxy_normal' : False,
190 'service_accesspoint' : True,
191 'service_incoming_rdr' : False,
192 'service_concentrator' : False,
193 'monitoring_group' : 'wleiden',
194 }
195 for (key,value) in defaults.iteritems():
196 if not datadump.has_key(key):
197 datadump[key] = value
198 f.close()
199
200 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
201 if add_version_info:
202 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
203 lines = p.communicate()[0].split('\n')
204 if p.returncode == 0:
205 for line in lines:
206 if line:
207 (key, value) = line.split(': ')
208 datadump["autogen_" + key.lower().replace(' ','_')] = value
209
210 # Preformat certain needed variables for formatting and push those into special object
211 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
212
213 wlan_count=0
214 try:
215 for key in get_interface_keys(datadump, True):
216 datadump[key]['autogen_ifbase'] = key.split('_')[1]
217 datadump[key]['autogen_vlan'] = False
218
219 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
220 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
221
222 if datadump[key].has_key('parent'):
223 if datadump[key].has_key('ip'):
224 raise ValueError("Interface bridge member cannot have IP assigned")
225 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
226 raise ValueError("Interface bridge member cannot have DHCP set")
227
228 if datadump[key].has_key('ip'):
229 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
230
231 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
232 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
233 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
234 wlan_count += 1
235 else:
236 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
237 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
238 datadump[key]['autogen_vlan'] = key.split('_')[2]
239 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
240 else:
241 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
242
243 except Exception:
244 print "# Error while processing interface %s" % key
245 raise
246
247 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
248 if dhcp_type(datadump[key]) == DHCP_SERVER]
249
250 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
251 datadump['autogen_item'] = item
252
253 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
254 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
255 datadump_cache[item] = datadump.copy()
256 except Exception:
257 print "# Error while processing %s" % item
258 raise
259 return datadump
260
261
262def store_yaml(datadump, header=False):
263 """ Store configuration yaml for 'item'"""
264 item = datadump['autogen_item']
265 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
266
267 output = generate_wleiden_yaml(datadump, header)
268
269 f = open(gfile, 'w')
270 f.write(output)
271 f.close()
272
273
274def network(ip):
275 addr, mask = ip.split('/')
276 # Not parsing of these folks please
277 addr = parseaddr(addr)
278 mask = int(mask)
279 network = addr & ~((1 << (32 - mask)) - 1)
280 return network
281
282
283
284def make_relations():
285 """ Process _ALL_ yaml files to get connection relations """
286 global relations_cache
287
288 if relations_cache:
289 return relations_cache
290
291 errors = []
292 poel = defaultdict(list)
293
294 for host in get_hostlist():
295 datadump = get_yaml(host)
296 try:
297 for iface_key in get_interface_keys(datadump):
298 net_addr = network(datadump[iface_key]['ip'])
299 poel[net_addr] += [(host,datadump[iface_key].copy())]
300 except (KeyError, ValueError), e:
301 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
302 continue
303
304 relations_cache = (poel, errors)
305 return relations_cache
306
307
308
309def valid_addr(addr):
310 """ Show which address is valid in which are not """
311 return str(addr).startswith('172.')
312
313def get_hostlist():
314 """ Combined hosts and proxy list"""
315 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
316
317def angle_between_points(lat1,lat2,long1,long2):
318 """
319 Return Angle in radians between two GPS coordinates
320 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
321 """
322 dy = lat2 - lat1
323 dx = math.cos(lat1)*(long2 - long1)
324 angle = math.atan2(dy,dx)
325 return angle
326
327
328
329def angle_to_cd(angle):
330 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
331
332 # For easy conversion get positive degree
333 degrees = math.degrees(angle)
334 abs_degrees = 360 + degrees if degrees < 0 else degrees
335
336 # Numbers can be confusing calculate from the 4 main directions
337 p = 22.5
338 if abs_degrees < p:
339 cd = "n"
340 elif abs_degrees < (90 - p):
341 cd = "no"
342 elif abs_degrees < (90 + p):
343 cd = "o"
344 elif abs_degrees < (180 - p):
345 cd = "zo"
346 elif abs_degrees < (180 + p):
347 cd = "z"
348 elif abs_degrees < (270 - p):
349 cd = "zw"
350 elif abs_degrees < (270 + p):
351 cd = "w"
352 elif abs_degrees < (360 - p):
353 cd = "nw"
354 else:
355 cd = "n"
356 return cd
357
358
359
360def cd_between_hosts(hostA, hostB, datadumps):
361 # Using RDNAP coordinates
362 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
363 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
364 return angle_to_cd(math.atan2(dx,dy))
365
366 # GPS coordinates seems to fail somehow
367 #latA = float(datadumps[hostA]['latitude'])
368 #latB = float(datadumps[hostB]['latitude'])
369 #lonA = float(datadumps[hostA]['longitude'])
370 #lonB = float(datadumps[hostB]['longitude'])
371 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
372
373
374def generate_title(nodelist):
375 """ Main overview page """
376 items = {'root' : "." }
377 def fl(spaces, line):
378 return (' ' * spaces) + line + '\n'
379
380 output = """
381<html>
382 <head>
383 <title>Wireless leiden Configurator - GFormat</title>
384 <style type="text/css">
385 th {background-color: #999999}
386 tr:nth-child(odd) {background-color: #cccccc}
387 tr:nth-child(even) {background-color: #ffffff}
388 th, td {padding: 0.1em 1em}
389 </style>
390 </head>
391 <body>
392 <center>
393 <form type="GET" action="%(root)s">
394 <input type="hidden" name="action" value="update">
395 <input type="submit" value="Update Configuration Database (SVN)">
396 </form>
397 <table>
398 <caption><h3>Wireless Leiden Configurator</h3></caption>
399 """ % items
400
401 for node in nodelist:
402 items['node'] = node
403 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
404 for config in files:
405 items['config'] = config
406 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
407 output += fl(5, "</tr>")
408 output += """
409 </table>
410 <hr />
411 <em>%s</em>
412 </center>
413 </body>
414</html>
415 """ % __version__
416
417 return output
418
419
420
421def generate_node(node):
422 """ Print overview of all files available for node """
423 return "\n".join(files)
424
425def generate_node_overview(host):
426 """ Print overview of all files available for node """
427 datadump = get_yaml(host)
428 params = { 'host' : host }
429 output = "<em><a href='..'>Back to overview</a></em><hr />"
430 output += "<h2>Available files:</h2><ul>"
431 for cf in files:
432 params['cf'] = cf
433 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
434 output += "</ul>"
435
436 # Generate and connection listing
437 output += "<h2>Connected To:</h2><ul>"
438 (poel, errors) = make_relations()
439 for network, hosts in poel.iteritems():
440 if host in [x[0] for x in hosts]:
441 if len(hosts) == 1:
442 # Single not connected interface
443 continue
444 for remote,ifacedump in hosts:
445 if remote == host:
446 # This side of the interface
447 continue
448 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
449 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
450 output += "</ul>"
451 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
452
453 output += "<hr /><em><a href='..'>Back to overview</a></em>"
454 return output
455
456
457def generate_header(datadump, ctag="#"):
458 return """\
459%(ctag)s
460%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
461%(ctag)s
462""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
463
464
465
466def parseaddr(s):
467 """ Process IPv4 CIDR notation addr to a (binary) number """
468 f = s.split('.')
469 return (long(f[0]) << 24L) + \
470 (long(f[1]) << 16L) + \
471 (long(f[2]) << 8L) + \
472 long(f[3])
473
474
475
476def showaddr(a):
477 """ Display IPv4 addr in (dotted) CIDR notation """
478 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
479
480
481def is_member(ip, mask, canidate):
482 """ Return True if canidate is part of ip/mask block"""
483 ip_addr = parseaddr(ip)
484 ip_canidate = parseaddr(canidate)
485 mask = int(mask)
486 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
487 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
488 return ip_addr == ip_canidate
489
490
491
492def cidr2netmask(netmask):
493 """ Given a 'netmask' return corresponding CIDR """
494 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
495
496def get_network(addr, mask):
497 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
498
499
500def generate_dhcpd_conf(datadump):
501 """ Generate config file '/usr/local/etc/dhcpd.conf """
502 output = generate_header(datadump)
503 output += Template("""\
504# option definitions common to all supported networks...
505option domain-name "dhcp.{{ autogen_fqdn }}";
506
507default-lease-time 600;
508max-lease-time 7200;
509
510# Use this to enble / disable dynamic dns updates globally.
511#ddns-update-style none;
512
513# If this DHCP server is the official DHCP server for the local
514# network, the authoritative directive should be uncommented.
515authoritative;
516
517# Use this to send dhcp log messages to a different log file (you also
518# have to hack syslog.conf to complete the redirection).
519log-facility local7;
520
521#
522# Interface definitions
523#
524\n\n""").render(datadump)
525
526
527 # TODO: Use textwrap.fill instead
528 def indent(text, count):
529 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
530
531 dhcp_out = defaultdict(list)
532 for iface_key in get_interface_keys(datadump):
533 ifname = datadump[iface_key]['autogen_ifbase']
534 if not datadump[iface_key].has_key('comment'):
535 datadump[iface_key]['comment'] = None
536
537 if not datadump[iface_key].has_key('ip'):
538 continue
539
540 dhcp_out[iface_key].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
541
542 (addr, mask) = datadump[iface_key]['ip'].split('/')
543 datadump[iface_key]['autogen_addr'] = addr
544 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
545 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
546
547 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
548 dhcp_out[iface_key].append(textwrap.dedent("""\
549 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
550 ### not autoritive
551 }
552 """ % datadump[iface_key]))
553 continue
554
555 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
556 dhcp_part = ".".join(addr.split('.')[0:3])
557 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
558 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
559
560 # Assume the first 10 IPs could be used for static entries
561 if 'no_portal' in datadump:
562 fixed = 5
563 for mac in datadump['no_portal']:
564 dhcp_out[iface_key].append(textwrap.dedent("""\
565 host fixed-%(ifname)s-%(fixed)s {
566 hardware ethernet %(mac)s;
567 fixed-address %(prefix)s.%(fixed)s;
568 }
569 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
570 fixed += 1
571
572 dhcp_out[iface_key].append(textwrap.dedent("""\
573 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
574 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
575 option routers %(autogen_addr)s;
576 option domain-name-servers %(autogen_addr)s;
577 }
578 """ % datadump[iface_key]))
579
580 for ifname,value in dhcp_out.iteritems():
581 if len(value) > 2:
582 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
583 else:
584 output += ''.join(value) + "\n\n"
585 return output
586
587
588
589def generate_dnsmasq_conf(datadump):
590 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
591 output = generate_header(datadump)
592 output += Template("""\
593# DHCP server options
594dhcp-authoritative
595dhcp-fqdn
596domain=dhcp.{{ autogen_fqdn }}
597domain-needed
598expand-hosts
599log-async=100
600
601# Low memory footprint
602cache-size=10000
603
604\n""").render(datadump)
605
606 for iface_key in get_interface_keys(datadump):
607 if not datadump[iface_key].has_key('comment'):
608 datadump[iface_key]['comment'] = None
609 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
610
611 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
612 output += "# not autoritive\n\n"
613 continue
614
615 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
616 (ip, cidr) = datadump[iface_key]['ip'].split('/')
617 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
618
619 dhcp_part = ".".join(ip.split('.')[0:3])
620 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
621 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
622 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
623
624 return output
625
626
627class AutoVivification(dict):
628 """Implementation of perl's autovivification feature."""
629 def __getitem__(self, item):
630 try:
631 return dict.__getitem__(self, item)
632 except KeyError:
633 value = self[item] = type(self)()
634 return value
635
636def make_interface_list(datadump):
637 if interface_list_cache.has_key(datadump['autogen_item']):
638 return (interface_list_cache[datadump['autogen_item']])
639 # lo0 configuration:
640 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
641 # - masterip is special as it needs to be assigned to at
642 # least one interface, so if not used assign to lo0
643 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
644 vlan_list = defaultdict(list)
645 bridge_list = defaultdict(list)
646 flags_if = AutoVivification()
647 dhclient_if = {'lo0' : False}
648
649 # XXX: Find some way of send this output nicely
650 output = ''
651
652 masterip_used = False
653 for iface_key in get_interface_keys(datadump):
654 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
655 masterip_used = True
656 break
657 if not masterip_used:
658 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
659
660 if 'serviceid' in datadump:
661 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
662
663 for iface_key in get_interface_keys(datadump):
664 ifacedump = datadump[iface_key]
665 ifname = ifacedump['autogen_ifname']
666
667 # If defined as vlan interface
668 if ifacedump['autogen_vlan']:
669 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
670
671 # If defined as bridge interface
672 if ifacedump['autogen_bridge_member']:
673 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
674
675 # Flag dhclient is possible
676 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
677 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
678
679 # Ethernet address
680 if ifacedump.has_key('ether'):
681 flags_if[ifname]['ether'] = ifacedump['ether']
682
683 # Add interface IP to list
684 if ifacedump.has_key('ip'):
685 item = (ifacedump['ip'], ifacedump['comment'])
686 if addrs_list.has_key(ifname):
687 addrs_list[ifname].append(item)
688 else:
689 addrs_list[ifname] = [item]
690
691 # Alias only needs IP assignment for now, this might change if we
692 # are going to use virtual accesspoints
693 if "alias" in iface_key:
694 continue
695
696 # XXX: Might want to deduct type directly from interface name
697 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
698 # Default to station (client) mode
699 ifacedump['autogen_wlanmode'] = "sta"
700 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
701 ifacedump['autogen_wlanmode'] = "ap"
702
703 if not ifacedump.has_key('channel'):
704 if ifacedump['type'] == '11a':
705 ifacedump['channel'] = 36
706 else:
707 ifacedump['channel'] = 1
708
709 # Allow special hacks at the back like wds and stuff
710 if not ifacedump.has_key('extra'):
711 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
712 else:
713 ifacedump['autogen_extra'] = ifacedump['extra']
714
715 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
716
717 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
718 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
719 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
720 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
721 output += "\n"
722
723 elif ifacedump['type'] in ['ethernet', 'eth']:
724 # No special config needed besides IP
725 pass
726 elif ifacedump['type'] in ['vlan']:
727 # VLAN member has no special configuration
728 pass
729 else:
730 assert False, "Unknown type " + ifacedump['type']
731
732 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
733 interface_list_cache[datadump['autogen_item']] = store
734 return(store)
735
736
737
738def create_proxies_list():
739 if not ileiden_proxies or not normal_proxies:
740 for i in range(1,50):
741 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
742
743 for host in get_hostlist():
744 hostdump = get_yaml(host)
745 if hostdump['status'] == 'up':
746 if hostdump['service_proxy_ileiden']:
747 ileiden_proxies[hostdump['serviceid']] = hostdump
748 if hostdump['service_proxy_normal']:
749 normal_proxies.append(hostdump)
750
751
752
753def generate_rc_conf_local(datadump):
754 """ Generate configuration file '/etc/rc.conf.local' """
755 item = datadump['autogen_item']
756 if rc_conf_local_cache.has_key(item):
757 return rc_conf_local_cache[item]
758
759 if not datadump.has_key('ileiden'):
760 datadump['autogen_ileiden_enable'] = False
761 else:
762 datadump['autogen_ileiden_enable'] = datadump['ileiden']
763
764 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
765
766 create_proxies_list()
767 datadump['autogen_ileiden_proxies'] = ileiden_proxies
768 datadump['autogen_normal_proxies'] = normal_proxies
769 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
770 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
771 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
772 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
773
774 output = generate_header(datadump, "#");
775 output += render_template(datadump, """\
776hostname='{{ autogen_fqdn }}'
777location='{{ location }}'
778nodetype="{{ nodetype }}"
779
780#
781# Configured listings
782#
783captive_portal_whitelist=""
784{% if nodetype == "Proxy" %}
785#
786# Proxy Configuration
787#
788{% if gateway and service_proxy_ileiden -%}
789defaultrouter="{{ gateway }}"
790{% else -%}
791#defaultrouter="NOTSET"
792{% endif -%}
793internalif="{{ internalif }}"
794ileiden_enable="{{ autogen_ileiden_enable }}"
795gateway_enable="{{ autogen_ileiden_enable }}"
796pf_enable="yes"
797pf_rules="/etc/pf.conf"
798{% if autogen_ileiden_enable -%}
799pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
800lvrouted_enable="{{ autogen_ileiden_enable }}"
801lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
802{% else -%}
803pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
804{% endif -%}
805{% if internalroute -%}
806static_routes="wleiden"
807route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
808{% endif -%}
809
810{% elif nodetype == "Hybrid" %}
811 #
812 # Hybrid Configuration
813 #
814 list_ileiden_proxies="
815 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
816 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
817 {% endfor -%}
818 "
819 list_normal_proxies="
820 {% for item in autogen_normal_proxies -%}
821 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
822 {% endfor -%}
823 "
824
825 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
826 externalif="{{ externalif|default('vr0', true) }}"
827 masterip="{{ masterip }}"
828
829 {% if gateway and service_proxy_ileiden %}
830 defaultrouter="{{ gateway }}"
831 {% else %}
832 #defaultrouter="NOTSET"
833 {% endif %}
834
835 #
836 # Defined services
837 #
838 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
839 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
840 service_accesspoint="{{ service_accesspoint|yesorno }}"
841 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
842 service_concentrator="{{ service_concentrator|yesorno }}"
843
844 {% if service_proxy_ileiden %}
845 pf_rules="/etc/pf.hybrid.conf"
846 {% if service_concentrator %}
847 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
848 {% else %}
849 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
850 {% endif %}
851 pf_flags="$pf_flags -D publicnat=80,443"
852 lvrouted_flags="$lvrouted_flags -g"
853 {% elif service_proxy_normal or service_incoming_rdr %}
854 pf_rules="/etc/pf.hybrid.conf"
855 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
856 pf_flags="$pf_flags -D publicnat=0"
857 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
858 named_setfib="1"
859 tinyproxy_setfib="1"
860 dnsmasq_setfib="1"
861 sshd_setfib="1"
862 {% else %}
863 named_auto_forward_only="YES"
864 pf_rules="/etc/pf.node.conf"
865 pf_flags=""
866 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
867 {% endif %}
868 {% if service_concentrator %}
869 # Do mind installing certificates is NOT done automatically for security reasons
870 openvpn_enable="YES"
871 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
872 {% endif %}
873
874 {% if service_proxy_normal %}
875 tinyproxy_enable="yes"
876 {% else %}
877 pen_wrapper_enable="yes"
878 {% endif %}
879
880 {% if service_accesspoint %}
881 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
882 {% endif %}
883
884 {% if board == "ALIX2" %}
885 #
886 # ''Fat'' configuration, board has 256MB RAM
887 #
888 dnsmasq_enable="NO"
889 named_enable="YES"
890 unbound_enable="YES"
891 {% if autogen_dhcp_interfaces -%}
892 dhcpd_enable="YES"
893 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
894 {% endif -%}
895 {% elif board == "apu1d" %}
896 #
897 # ''Fat'' configuration, board has 1024MB RAM
898 #
899 dnsmasq_enable="NO"
900 local_unbound_enable="YES"
901 {% if autogen_dhcp_interfaces -%}
902 dhcpd_enable="YES"
903 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
904 {% endif -%}
905 {% endif -%}
906{% endif %}
907
908#
909# Script variables
910#
911attached_devices="{{ autogen_attached_devices|join(' ') }}"
912neighbours="{{ autogen_neighbours|join(' ') }}"
913
914
915#
916# Interface definitions
917#\n
918""")
919
920 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
921 for iface, vlans in sorted(vlan_list.items()):
922 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
923
924 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
925 for iface in sorted(vlan_list.keys()):
926 if not iface in addrs_list.keys():
927 output += "ifconfig_%s='up'\n" % iface
928
929 output += "\n"
930
931 # Bridge configuration:
932 if bridge_list.keys():
933 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
934
935 for iface in bridge_list.keys():
936 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
937
938 # Bridge member interfaces not containing a configuration should be marked active explcitly.
939 for _,members in bridge_list.items():
940 for iface in members:
941 if not iface in addrs_list.keys():
942 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
943
944 output += "\n"
945
946 # Details like SSID
947 if extra_ouput:
948 output += extra_ouput.strip() + "\n"
949
950 # Print IP address which needs to be assigned over here
951 output += "\n"
952 for iface,addrs in sorted(addrs_list.iteritems()):
953 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
954 output += "# %s || %s || %s\n" % (iface, addr, comment)
955
956 # Write DHCLIENT entry
957 if iface in dhclient_if and dhclient_if[iface]:
958 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
959 continue
960
961 # Make sure the external address is always first as this is needed in the
962 # firewall setup
963 addrs = sorted(
964 [x for x in addrs if not '0.0.0.0' in x[0]],
965 key=lambda x: x[0].split('.')[0],
966 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
967 )
968
969 idx_offset = 0
970 # Set MAC is required
971 if flags_if[iface].has_key('ether'):
972 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
973 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
974 idx_offset += 1
975 else:
976 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
977
978 for idx, addr in enumerate(addrs[1:]):
979 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
980
981 output += "\n"
982
983 rc_conf_local_cache[datadump['autogen_item']] = output
984 return output
985
986
987
988def get_all_configs():
989 """ Get dict with key 'host' with all configs present """
990 configs = dict()
991 for host in get_hostlist():
992 datadump = get_yaml(host)
993 configs[host] = datadump
994 return configs
995
996
997def get_interface_keys(config, extra=False):
998 """ Quick hack to get all interface keys, later stage convert this to a iterator """
999 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1000 if extra == False:
1001 return filter(lambda x: not "extra" in x, elems)
1002 else:
1003 return elems
1004
1005
1006def get_used_ips(configs):
1007 """ Return array of all IPs used in config files"""
1008 ip_list = []
1009 for config in configs:
1010 ip_list.append(config['masterip'])
1011 if 'serviceid' in config:
1012 ip_list.append(config['serviceid'])
1013 for iface_key in get_interface_keys(config, True):
1014 l = config[iface_key]['ip']
1015 addr, mask = l.split('/')
1016 # Special case do not process
1017 if valid_addr(addr):
1018 ip_list.append(addr)
1019 else:
1020 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1021 return sorted(ip_list)
1022
1023
1024
1025def get_nameservers(max_servers=None):
1026 if nameservers_cache:
1027 return nameservers_cache[0:max_servers]
1028
1029 for host in get_hostlist():
1030 hostdump = get_yaml(host)
1031 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1032 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
1033
1034 return nameservers_cache[0:max_servers]
1035
1036
1037def get_neighbours(datadump):
1038 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1039
1040 (poel, errors) = make_relations()
1041 table = []
1042 for iface,addrs in sorted(addrs_list.iteritems()):
1043 if iface in ['lo0']:
1044 continue
1045
1046 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1047 if not addr.startswith('172.'):
1048 # Avoid listing internet connections as pool
1049 continue
1050 for neighbour in poel[network(addr)]:
1051 if neighbour[0] != datadump['autogen_item']:
1052 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1053 return table
1054
1055
1056def get_attached_devices(datadump, url=False):
1057 table = []
1058 for iface_key in get_interface_keys(datadump, True):
1059 ifacedump = datadump[iface_key]
1060
1061 if not ifacedump.has_key('ns_ip'):
1062 continue
1063
1064 x_ip = ifacedump['ns_ip'].split('/')[0]
1065
1066 if 'mode' in ifacedump:
1067 x_mode = ifacedump['mode']
1068 else:
1069 x_mode = 'unknown'
1070
1071 if 'bridge_type' in ifacedump:
1072 device_type = ifacedump['bridge_type']
1073 else:
1074 device_type = 'Unknown'
1075
1076 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1077 return table
1078
1079
1080def generate_resolv_conf(datadump):
1081 """ Generate configuration file '/etc/resolv.conf' """
1082 # XXX: This should properly going to be an datastructure soon
1083 datadump['autogen_header'] = generate_header(datadump, "#")
1084 datadump['autogen_edge_nameservers'] = ''
1085
1086
1087 for masterip,realname in get_nameservers():
1088 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1089
1090 return Template("""\
1091{{ autogen_header }}
1092search wleiden.net
1093
1094# Try local (cache) first
1095nameserver 127.0.0.1
1096
1097{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1098nameserver 8.8.8.8 # Google Public NameServer
1099nameserver 4.2.2.1 # Level3 Public NameServer
1100{% else -%}
1101# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1102{{ autogen_edge_nameservers }}
1103{% endif -%}
1104""").render(datadump)
1105
1106
1107
1108def generate_ntp_conf(datadump):
1109 """ Generate configuration file '/etc/ntp.conf' """
1110 # XXX: This should properly going to be an datastructure soon
1111
1112 datadump['autogen_header'] = generate_header(datadump, "#")
1113 datadump['autogen_ntp_servers'] = ''
1114 for host in get_hostlist():
1115 hostdump = get_yaml(host)
1116 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1117 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1118
1119 return Template("""\
1120{{ autogen_header }}
1121
1122{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1123# Machine hooked to internet.
1124server 0.nl.pool.ntp.org iburst maxpoll 9
1125server 1.nl.pool.ntp.org iburst maxpoll 9
1126server 2.nl.pool.ntp.org iburst maxpoll 9
1127server 3.nl.pool.ntp.org iburst maxpoll 9
1128{% else -%}
1129# Local Wireless Leiden NTP Servers.
1130server 0.pool.ntp.wleiden.net iburst maxpoll 9
1131server 1.pool.ntp.wleiden.net iburst maxpoll 9
1132server 2.pool.ntp.wleiden.net iburst maxpoll 9
1133server 3.pool.ntp.wleiden.net iburst maxpoll 9
1134
1135# All the configured NTP servers
1136{{ autogen_ntp_servers }}
1137{% endif %}
1138
1139# If a server loses sync with all upstream servers, NTP clients
1140# no longer follow that server. The local clock can be configured
1141# to provide a time source when this happens, but it should usually
1142# be configured on just one server on a network. For more details see
1143# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1144# The use of Orphan Mode may be preferable.
1145#
1146server 127.127.1.0
1147fudge 127.127.1.0 stratum 10
1148""").render(datadump)
1149
1150
1151def generate_pf_hybrid_conf_local(datadump):
1152 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1153 datadump['autogen_header'] = generate_header(datadump, "#")
1154 return Template("""\
1155{{ autogen_header }}
1156
1157# Redirect some internal facing services outside (7)
1158# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1159{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1160rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1161{% endfor -%}
1162""").render(datadump)
1163
1164def generate_unbound_wleiden_conf(datadump):
1165 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1166 datadump['autogen_header'] = generate_header(datadump, "#")
1167
1168 autogen_ips = []
1169 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1170 for iface,addrs in sorted(addrs_list.iteritems()):
1171 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1172 if addr.startswith('172'):
1173 autogen_ips.append((addr.split('/')[0], comment))
1174 datadump['autogen_ips'] = autogen_ips
1175
1176 create_proxies_list()
1177 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1178 return Template("""\
1179{{ autogen_header }}
1180
1181server:
1182{%- for ip,comment in autogen_ips %}
1183 interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1184{%- endfor %}
1185
1186forward-zone:
1187 name: '.'
1188{%- if service_proxy_ileiden %}
1189 forward-addr: 8.8.8.8 # Google DNS A
1190 forward-addr: 8.8.4.4 # Google DNS B
1191 forward-addr: 208.67.222.222 # OpenDNS DNS A
1192 forward-addr: 208.67.220.220 # OpenDNS DNS B
1193{% else -%}
1194{% for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1195 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1196{%- endfor %}
1197{% endif -%}
1198""").render(datadump)
1199
1200def generate_motd(datadump):
1201 """ Generate configuration file '/etc/motd' """
1202 output = Template("""\
1203FreeBSD run ``service motd onestart'' to make me look normal
1204
1205 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1206 Loc: {{ location }}
1207
1208Services:
1209{% if board == "ALIX2" -%}
1210{{" -"}} Core Node ({{ board }})
1211{% else -%}
1212{{" -"}} Hulp Node ({{ board }})
1213{% endif -%}
1214{% if service_proxy_normal -%}
1215{{" -"}} Normal Proxy
1216{% endif -%}
1217{% if service_proxy_ileiden -%}
1218{{" -"}} iLeiden Proxy
1219{% endif -%}
1220{% if service_incoming_rdr -%}
1221{{" -"}} Incoming port redirects
1222{% endif %}
1223Interlinks:\n
1224""").render(datadump)
1225
1226
1227 def make_table(table):
1228 if not table:
1229 return " - none\n"
1230 else:
1231 lines = ""
1232 col_width = [max(len(x) for x in col) for col in zip(*table)]
1233 for row in table:
1234 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1235 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1236 return lines
1237
1238 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1239 table = []
1240 for iface,addrs in sorted(addrs_list.iteritems()):
1241 if iface in ['lo0']:
1242 continue
1243 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1244 table.append((iface, addr, comment))
1245
1246 output += make_table(table)
1247 output += '\n'
1248 output += """\
1249Attached devices:
1250"""
1251 output += make_table(get_attached_devices(datadump, url=True))
1252 output += '\n'
1253 output += """\
1254Available neighbours:
1255"""
1256 output += make_table(get_neighbours(datadump))
1257
1258 return output
1259
1260
1261def format_yaml_value(value):
1262 """ Get yaml value in right syntax for outputting """
1263 if isinstance(value,str):
1264 output = '"%s"' % value
1265 else:
1266 output = value
1267 return output
1268
1269
1270
1271def format_wleiden_yaml(datadump):
1272 """ Special formatting to ensure it is editable"""
1273 output = "# Genesis config yaml style\n"
1274 output += "# vim:ts=2:et:sw=2:ai\n"
1275 output += "#\n"
1276 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1277 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1278 if key == 'rdr_rules':
1279 output += '%-10s:\n' % 'rdr_rules'
1280 for rdr_rule in datadump[key]:
1281 output += '- %s\n' % rdr_rule
1282 else:
1283 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1284
1285 output += "\n\n"
1286
1287 # Format (key, required)
1288 key_order = (
1289 ('comment', True),
1290 ('parent', False),
1291 ('ip', False),
1292 ('ether', False),
1293 ('desc', True),
1294 ('sdesc', True),
1295 ('mode', True),
1296 ('type', True),
1297 ('extra_type', False),
1298 ('channel', False),
1299 ('ssid', False),
1300 ('wlan_mac', False),
1301 ('dhcp', True),
1302 ('compass', False),
1303 ('distance', False),
1304 ('ns_ip', False),
1305 ('repeater_ip', False),
1306 ('bullet2_ip', False),
1307 ('ns_mac', False),
1308 ('bullet2_mac', False),
1309 ('ns_type', False),
1310 ('bridge_type', False),
1311 ('status', True),
1312 )
1313
1314 for iface_key in sorted(iface_keys):
1315 try:
1316 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1317 if remainder:
1318 raise KeyError("invalid keys: %s" % remainder)
1319
1320 output += "%s:\n" % iface_key
1321 for key,required in key_order:
1322 if datadump[iface_key].has_key(key):
1323 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1324 output += "\n\n"
1325 except Exception:
1326 print "# Error while processing interface %s" % iface_key
1327 raise
1328
1329 return output
1330
1331
1332
1333def generate_wleiden_yaml(datadump, header=True):
1334 """ Generate (petty) version of wleiden.yaml"""
1335 output = generate_header(datadump, "#") if header else ''
1336
1337 for key in datadump.keys():
1338 if key.startswith('autogen_'):
1339 del datadump[key]
1340 # Interface autogen cleanups
1341 elif type(datadump[key]) == dict:
1342 for key2 in datadump[key].keys():
1343 if key2.startswith('autogen_'):
1344 del datadump[key][key2]
1345
1346 output += format_wleiden_yaml(datadump)
1347 return output
1348
1349def generate_nanostation_config(datadump, iface, ns_type):
1350 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1351 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1352
1353 datadump.update(datadump['iface_%s' % iface])
1354
1355 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1356
1357def generate_yaml(datadump):
1358 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1359
1360
1361
1362def generate_config(node, config, datadump=None):
1363 """ Print configuration file 'config' of 'node' """
1364 output = ""
1365 try:
1366 # Load config file
1367 if datadump == None:
1368 datadump = get_yaml(node)
1369
1370 if config == 'wleiden.yaml':
1371 output += generate_wleiden_yaml(datadump)
1372 elif config == 'authorized_keys':
1373 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1374 output += f.read()
1375 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1376 # Fetch local keys if existing
1377 if os.path.exists(node_keys):
1378 output += open(node_keys, 'r').read()
1379 f.close()
1380 elif config == 'dnsmasq.conf':
1381 output += generate_dnsmasq_conf(datadump)
1382 elif config == 'dhcpd.conf':
1383 output += generate_dhcpd_conf(datadump)
1384 elif config == 'rc.conf.local':
1385 output += generate_rc_conf_local(datadump)
1386 elif config == 'resolv.conf':
1387 output += generate_resolv_conf(datadump)
1388 elif config == 'ntp.conf':
1389 output += generate_ntp_conf(datadump)
1390 elif config == 'motd':
1391 output += generate_motd(datadump)
1392 elif config == 'pf.hybrid.conf.local':
1393 output += generate_pf_hybrid_conf_local(datadump)
1394 elif config == 'unbound.wleiden.conf':
1395 output += generate_unbound_wleiden_conf(datadump)
1396 elif config.startswith('vr'):
1397 interface, ns_type = config.strip('.yaml').split('-')
1398 output += generate_nanostation_config(datadump, interface, ns_type)
1399 else:
1400 assert False, "Config not found!"
1401 except IOError, e:
1402 output += "[ERROR] Config file not found"
1403 return output
1404
1405
1406
1407def process_cgi_request(environ=os.environ):
1408 """ When calling from CGI """
1409 response_headers = []
1410 content_type = 'text/plain'
1411
1412 # Update repository if requested
1413 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1414 if form and form.has_key("action") and "update" in form["action"]:
1415 output = "[INFO] Updating subverion, please wait...\n"
1416 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1417 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1418 output += "[INFO] All done, redirecting in 5 seconds"
1419 response_headers += [
1420 ('Refresh', '5; url=.'),
1421 ]
1422 reload_cache()
1423 else:
1424 base_uri = environ['PATH_INFO']
1425 uri = base_uri.strip('/').split('/')
1426
1427 output = "Template Holder"
1428 if base_uri.endswith('/create/network.kml'):
1429 content_type='application/vnd.google-earth.kml+xml'
1430 output = make_network_kml.make_graph()
1431 elif base_uri.endswith('/api/get/nodeplanner.json'):
1432 content_type='application/json'
1433 output = make_network_kml.make_nodeplanner_json()
1434 elif not uri[0]:
1435 if is_text_request(environ):
1436 output = '\n'.join(get_hostlist())
1437 else:
1438 content_type = 'text/html'
1439 output = generate_title(get_hostlist())
1440 elif len(uri) == 1:
1441 if is_text_request(environ):
1442 output = generate_node(uri[0])
1443 else:
1444 content_type = 'text/html'
1445 output = generate_node_overview(uri[0])
1446 elif len(uri) == 2:
1447 output = generate_config(uri[0], uri[1])
1448 else:
1449 assert False, "Invalid option"
1450
1451 # Return response
1452 response_headers += [
1453 ('Content-type', content_type),
1454 ('Content-Length', str(len(output))),
1455 ]
1456 return(response_headers, str(output))
1457
1458
1459def make_dns(output_dir = 'dns', external = False):
1460 items = dict()
1461
1462 # hostname is key, IP is value
1463 wleiden_zone = defaultdict(list)
1464 wleiden_cname = dict()
1465
1466 pool = dict()
1467 for node in get_hostlist():
1468 datadump = get_yaml(node)
1469
1470 fqdn = datadump['nodename']
1471
1472 if datadump.has_key('rdr_host'):
1473 remote_target = datadump['rdr_host']
1474 elif datadump.has_key('remote_access') and datadump['remote_access']:
1475 remote_target = datadump['remote_access'].split(':')[0]
1476 else:
1477 remote_target = None
1478
1479 if remote_target:
1480 try:
1481 parseaddr(remote_target)
1482 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1483 except (IndexError, ValueError):
1484 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1485
1486
1487 wleiden_zone[fqdn].append((datadump['masterip'], True))
1488
1489 # Hacking to get proper DHCP IPs and hostnames
1490 for iface_key in get_interface_keys(datadump):
1491 iface_name = iface_key.replace('_','-')
1492 if 'ip' in datadump[iface_key]:
1493 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1494 try:
1495 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1496 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1497 dhcp_part = ".".join(ip.split('.')[0:3])
1498 if ip != datadump['masterip']:
1499 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1500 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1501 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1502 except (AttributeError, ValueError, KeyError):
1503 # First push it into a pool, to indentify the counter-part later on
1504 addr = parseaddr(ip)
1505 cidr = int(cidr)
1506 addr = addr & ~((1 << (32 - cidr)) - 1)
1507 if pool.has_key(addr):
1508 pool[addr] += [(iface_name, fqdn, ip)]
1509 else:
1510 pool[addr] = [(iface_name, fqdn, ip)]
1511 continue
1512
1513
1514
1515 # WL uses an /29 to configure an interface. IP's are ordered like this:
1516 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1517
1518 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1519
1520 # Automatic naming convention of interlinks namely 2 + remote.lower()
1521 for (key,value) in pool.iteritems():
1522 # Make sure they are sorted from low-ip to high-ip
1523 value = sorted(value, key=lambda x: parseaddr(x[2]))
1524
1525 if len(value) == 1:
1526 (iface_name, fqdn, ip) = value[0]
1527 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1528
1529 # Device DNS names
1530 if 'cnode' in fqdn.lower():
1531 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1532 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1533
1534 elif len(value) == 2:
1535 (a_iface_name, a_fqdn, a_ip) = value[0]
1536 (b_iface_name, b_fqdn, b_ip) = value[1]
1537 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1538 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1539
1540 # Device DNS names
1541 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1542 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1543 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1544 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1545 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1546 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1547 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1548
1549 else:
1550 pool_members = [k[1] for k in value]
1551 for item in value:
1552 (iface_name, fqdn, ip) = item
1553 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1554
1555 # Include static DNS entries
1556 # XXX: Should they override the autogenerated results?
1557 # XXX: Convert input to yaml more useable.
1558 # Format:
1559 ##; this is a comment
1560 ## roomburgh=Roomburgh1
1561 ## apkerk1.Vosko=172.17.176.8 ;this as well
1562 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1563
1564 # Hack to allow special entries, for development
1565 wleiden_raw = {}
1566
1567 for line in dns_list:
1568 reverse = False
1569 k, items = line.items()[0]
1570 if type(items) == dict:
1571 if items.has_key('reverse'):
1572 reverse = items['reverse']
1573 items = items['a']
1574 else:
1575 items = items['cname']
1576 items = [items] if type(items) != list else items
1577 for item in items:
1578 if item.startswith('IN '):
1579 wleiden_raw[k] = item
1580 elif valid_addr(item):
1581 wleiden_zone[k].append((item, reverse))
1582 else:
1583 wleiden_cname[k] = item
1584
1585 # Hack to get dynamic pool listing
1586 def chunks(l, n):
1587 return [l[i:i+n] for i in range(0, len(l), n)]
1588
1589 ntp_servers = [x[0] for x in get_nameservers()]
1590 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1591 for ntp_server in chunk:
1592 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1593
1594 details = dict()
1595 # 24 updates a day allowed
1596 details['serial'] = time.strftime('%Y%m%d%H')
1597
1598 if external:
1599 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1600 else:
1601 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1602
1603 details['master'] = dns_masters[0]
1604 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1605
1606 dns_header = '''
1607$TTL 3h
1608%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1609 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1610
1611%(ns_servers)s
1612 \n'''
1613
1614
1615 if not os.path.isdir(output_dir):
1616 os.makedirs(output_dir)
1617 details['zone'] = 'wleiden.net'
1618 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1619 f.write(dns_header % details)
1620
1621 for host,items in wleiden_zone.iteritems():
1622 for ip,reverse in items:
1623 if ip not in ['0.0.0.0']:
1624 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1625 for source,dest in wleiden_cname.iteritems():
1626 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1627 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1628 for source, dest in wleiden_raw.iteritems():
1629 f.write("%s.wleiden.net. %s\n" % (source, dest))
1630 f.close()
1631
1632 # Create whole bunch of specific sub arpa zones. To keep it compliant
1633 for s in range(16,32):
1634 details['zone'] = '%i.172.in-addr.arpa' % s
1635 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1636 f.write(dns_header % details)
1637
1638 #XXX: Not effient, fix to proper data structure and do checks at other
1639 # stages
1640 for host,items in wleiden_zone.iteritems():
1641 for ip,reverse in items:
1642 if not reverse:
1643 continue
1644 if valid_addr(ip):
1645 if valid_addr(ip):
1646 if int(ip.split('.')[1]) == s:
1647 rev_ip = '.'.join(reversed(ip.split('.')))
1648 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1649 f.close()
1650
1651
1652def usage():
1653 print """Usage: %(prog)s <argument>
1654Argument:
1655\tcleanup = Cleanup all YAML files to specified format
1656\tstandalone [port] = Run configurator webserver [8000]
1657\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1658\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1659\tfull-export = Generate yaml export script for heatmap.
1660\tstatic [outputdir] = Generate all config files and store on disk
1661\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1662\ttest <node> [<file>] = Receive output for certain node [all files].
1663\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1664\tlist <status> <items> = List systems which have certain status
1665\tcreate network.kml = Create Network KML file for use in Google Earth
1666
1667Arguments:
1668\t<node> = NodeName (example: HybridRick)
1669\t<file> = %(files)s
1670\t<status> = all|up|down|planned
1671\t<items> = systems|nodes|proxies
1672
1673NOTE FOR DEVELOPERS; you can test your changes like this:
1674 BEFORE any changes in this code:
1675 $ ./gformat.py static /tmp/pre
1676 AFTER the changes:
1677 $ ./gformat.py static /tmp/post
1678 VIEW differences and VERIFY all are OK:
1679 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1680""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1681 exit(0)
1682
1683
1684def is_text_request(environ=os.environ):
1685 """ Find out whether we are calling from the CLI or any text based CLI utility """
1686 try:
1687 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1688 except KeyError:
1689 return True
1690
1691def switchFormat(setting):
1692 if setting:
1693 return "YES"
1694 else:
1695 return "NO"
1696
1697def rlinput(prompt, prefill=''):
1698 import readline
1699 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1700 try:
1701 return raw_input(prompt)
1702 finally:
1703 readline.set_startup_hook()
1704
1705def fix_conflict(left, right, default='i'):
1706 while True:
1707 print "## %-30s | %-30s" % (left, right)
1708 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1709 if not c:
1710 c = default
1711
1712 if c in ['l','1']:
1713 return left
1714 elif c in ['r','2']:
1715 return right
1716 elif c in ['e', '3']:
1717 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1718 elif c in ['i', '4']:
1719 return None
1720 else:
1721 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1722
1723
1724
1725def print_cgi_response(response_headers, output):
1726 """Could we not use some kind of wsgi wrapper to make this output?"""
1727 for header in response_headers:
1728 print "%s: %s" % header
1729 print
1730 print output
1731
1732
1733def fill_cache():
1734 ''' Poor man re-loading of few cache items (the slow ones) '''
1735 for host in get_hostlist():
1736 get_yaml(host)
1737
1738
1739def reload_cache():
1740 clear_cache()
1741 fill_cache()
1742
1743
1744def main():
1745 """Hard working sub"""
1746 # Allow easy hacking using the CLI
1747 if not os.environ.has_key('PATH_INFO'):
1748 if len(sys.argv) < 2:
1749 usage()
1750
1751 if sys.argv[1] == "standalone":
1752 import SocketServer
1753 import CGIHTTPServer
1754 # Hop to the right working directory.
1755 os.chdir(os.path.dirname(__file__))
1756 try:
1757 PORT = int(sys.argv[2])
1758 except (IndexError,ValueError):
1759 PORT = 8000
1760
1761 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1762 """ Serve this CGI from the root of the webserver """
1763 def is_cgi(self):
1764 if "favicon" in self.path:
1765 return False
1766
1767 self.cgi_info = (os.path.basename(__file__), self.path)
1768 self.path = ''
1769 return True
1770 handler = MyCGIHTTPRequestHandler
1771 SocketServer.TCPServer.allow_reuse_address = True
1772 httpd = SocketServer.TCPServer(("", PORT), handler)
1773 httpd.server_name = 'localhost'
1774 httpd.server_port = PORT
1775
1776 logger.info("serving at port %s", PORT)
1777 try:
1778 httpd.serve_forever()
1779 except KeyboardInterrupt:
1780 httpd.shutdown()
1781 logger.info("All done goodbye")
1782 elif sys.argv[1] == "test":
1783 # Basic argument validation
1784 try:
1785 node = sys.argv[2]
1786 except IndexError:
1787 print "Invalid argument"
1788 exit(1)
1789 except IOError as e:
1790 print e
1791 exit(1)
1792
1793 datadump = get_yaml(node)
1794
1795
1796 # Get files to generate
1797 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1798
1799 # Actual config generation
1800 for config in gen_files:
1801 logger.info("## Generating %s %s", node, config)
1802 print generate_config(node, config, datadump)
1803 elif sys.argv[1] == "test-cgi":
1804 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1805 os.environ['SCRIPT_NAME'] = __file__
1806 response_headers, output = process_cgi_request()
1807 print_cgi_response(response_headers, output)
1808 elif sys.argv[1] == "static":
1809 items = dict()
1810 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1811 for node in get_hostlist():
1812 items['node'] = node
1813 items['wdir'] = "%(output_dir)s/%(node)s" % items
1814 if not os.path.isdir(items['wdir']):
1815 os.makedirs(items['wdir'])
1816 datadump = get_yaml(node)
1817 for config in files:
1818 items['config'] = config
1819 logger.info("## Generating %(node)s %(config)s" % items)
1820 f = open("%(wdir)s/%(config)s" % items, "w")
1821 f.write(generate_config(node, config, datadump))
1822 f.close()
1823 elif sys.argv[1] == "wind-export":
1824 items = dict()
1825 for node in get_hostlist():
1826 datadump = get_yaml(node)
1827 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1828 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1829 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1830 VALUES (
1831 (SELECT id FROM users WHERE username = 'rvdzwet'),
1832 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1833 'Y');""" % datadump
1834 #for config in files:
1835 # items['config'] = config
1836 # print "## Generating %(node)s %(config)s" % items
1837 # f = open("%(wdir)s/%(config)s" % items, "w")
1838 # f.write(generate_config(node, config, datadump))
1839 # f.close()
1840 for node in get_hostlist():
1841 datadump = get_yaml(node)
1842 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1843 ifacedump = datadump[iface_key]
1844 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1845 ifacedump['nodename'] = datadump['nodename']
1846 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1847 ifacedump['channel'] = 0
1848 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1849 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1850 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1851 elif sys.argv[1] == "nagios-export":
1852 try:
1853 heavy_load = (sys.argv[2] == "--heavy-load")
1854 except IndexError:
1855 heavy_load = False
1856
1857 hostgroup_details = {
1858 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1859 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1860 'walphen' : 'Stichting Wireless Alphen',
1861 'westeinder' : 'Westeinder Plassen',
1862 }
1863
1864 # Convert IP to Host
1865 ip2host = {'root' : 'root'}
1866 for host in get_hostlist():
1867 datadump = get_yaml(host)
1868 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1869 for iface in get_interface_keys(datadump):
1870 if datadump[iface].has_key('autogen_gateway'):
1871 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1872
1873 # Find dependency tree based on output of lvrouted.mytree of nearest node
1874 parents = defaultdict(list)
1875 stack = ['root']
1876 prev_depth = 0
1877 for line in open('lvrouted.mytree').readlines():
1878 depth = line.count('\t')
1879 ip = line.strip().split()[0]
1880
1881 if prev_depth < depth:
1882 try:
1883 parents[ip2host[ip]].append(ip2host[stack[-1]])
1884 except KeyError as e:
1885 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1886 stack.append(ip)
1887 elif prev_depth > depth:
1888 stack = stack[:(depth - prev_depth)]
1889 elif prev_depth == depth:
1890 try:
1891 parents[ip2host[ip]].append(ip2host[stack[-1]])
1892 except KeyError as e:
1893 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1894
1895
1896 prev_depth = depth
1897 # Observe that some nodes has themself as parent or multiple parents
1898 # for now take only the first parent, other behaviour is yet to be explained
1899
1900
1901
1902 params = {
1903 'check_interval' : 5 if heavy_load else 120,
1904 'retry_interval' : 1 if heavy_load else 10,
1905 'max_check_attempts' : 10 if heavy_load else 6,
1906 'notification_interval': 120 if heavy_load else 240,
1907 }
1908
1909 print '''\
1910define host {
1911 name wleiden-node ; Default Node Template
1912 use generic-host ; Use the standard template as initial starting point
1913 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1914 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1915 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1916 notification_interval %(notification_interval)s
1917 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1918 check_command check-host-alive ; Default command to check FreeBSD hosts
1919 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1920}
1921
1922define service {
1923 name wleiden-service ; Default Service Template
1924 use generic-service ; Use the standard template as initial starting point
1925 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1926 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1927 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1928 notification_interval %(notification_interval)s
1929 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1930 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1931}
1932
1933# Please make sure to install:
1934# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1935#
1936# Recompile net-mgmt/nagios-plugins to support check_snmp
1937# make -C /usr/ports/net-mgmt/nagios-plugins
1938#
1939# Install net/bind-tools to allow v2/check_dns_wl to work:
1940# pkg install bind-tools
1941#
1942define command{
1943 command_name check_snmp_disk
1944 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1945}
1946
1947define command{
1948 command_name check_netsnmp_load
1949 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1950}
1951
1952define command{
1953 command_name check_netsnmp_proc
1954 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1955}
1956
1957define command{
1958 command_name check_by_ssh
1959 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1960}
1961
1962define command{
1963 command_name check_dns_wl
1964 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1965}
1966
1967define command{
1968 command_name check_snmp_uptime
1969 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1970}
1971
1972
1973# TDB: dhcp leases
1974# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1975
1976# TDB: internet status
1977# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1978
1979# TDB: Advanced local passive checks
1980# /usr/local/libexec/nagios/check_by_ssh
1981''' % params
1982
1983 print '''\
1984# Service Group, not displayed by default
1985define hostgroup {
1986 hostgroup_name srv_hybrid
1987 alias All Hybrid Nodes
1988 register 0
1989}
1990
1991define service {
1992 use wleiden-service
1993 hostgroup_name srv_hybrid
1994 service_description SSH
1995 check_command check_ssh
1996}
1997
1998define service {
1999 use wleiden-service,service-pnp
2000 hostgroup_name srv_hybrid
2001 service_description HTTP
2002 check_command check_http
2003}
2004
2005define service {
2006 use wleiden-service
2007 hostgroup_name srv_hybrid
2008 service_description DNS
2009 check_command check_dns_wl!"www.wirelessleiden.nl"
2010}
2011'''
2012
2013 if heavy_load:
2014 print '''\
2015define service {
2016 use wleiden-service
2017 hostgroup_name srv_hybrid
2018 service_description UPTIME
2019 check_command check_snmp_uptime
2020}
2021
2022#define service {
2023# use wleiden-service
2024# hostgroup_name srv_hybrid
2025# service_description NTP
2026# check_command check_ntp_peer
2027#}
2028
2029define service {
2030 use wleiden-service
2031 hostgroup_name srv_hybrid
2032 service_description LOAD
2033 check_command check_netsnmp_load
2034}
2035
2036define service {
2037 use wleiden-service
2038 hostgroup_name srv_hybrid
2039 service_description PROC
2040 check_command check_netsnmp_proc
2041}
2042
2043define service {
2044 use wleiden-service
2045 hostgroup_name srv_hybrid
2046 service_description DISK
2047 check_command check_snmp_disk
2048}
2049'''
2050 for node in get_hostlist():
2051 datadump = get_yaml(node)
2052 if not datadump['status'] == 'up':
2053 continue
2054 if not hostgroup_details.has_key(datadump['monitoring_group']):
2055 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2056 print '''\
2057define host {
2058 use wleiden-node,host-pnp
2059 contact_groups admins
2060 host_name %(autogen_fqdn)s
2061 address %(masterip)s
2062 hostgroups srv_hybrid,%(monitoring_group)s\
2063''' % datadump
2064 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2065 print '''\
2066 parents %(parents)s\
2067''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2068 print '''\
2069}
2070'''
2071
2072
2073 for name,alias in hostgroup_details.iteritems():
2074 print '''\
2075define hostgroup {
2076 hostgroup_name %s
2077 alias %s
2078} ''' % (name, alias)
2079
2080
2081 elif sys.argv[1] == "full-export":
2082 hosts = {}
2083 for node in get_hostlist():
2084 datadump = get_yaml(node)
2085 hosts[datadump['nodename']] = datadump
2086 print yaml.dump(hosts)
2087
2088 elif sys.argv[1] == "dns":
2089 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2090 elif sys.argv[1] == "cleanup":
2091 # First generate all datadumps
2092 datadumps = dict()
2093 ssid_to_node = dict()
2094 for host in get_hostlist():
2095 logger.info("# Processing: %s", host)
2096 # Set some boring default values
2097 datadump = { 'board' : 'UNKNOWN' }
2098 datadump.update(get_yaml(host))
2099 datadumps[datadump['nodename']] = datadump
2100
2101 (poel, errors) = make_relations()
2102 print "\n".join(["# WARNING: %s" % x for x in errors])
2103
2104 for host,datadump in datadumps.iteritems():
2105 try:
2106 # Convert all yes and no to boolean values
2107 def fix_boolean(dump):
2108 for key in dump.keys():
2109 if type(dump[key]) == dict:
2110 dump[key] = fix_boolean(dump[key])
2111 elif str(dump[key]).lower() in ["yes", "true"]:
2112 dump[key] = True
2113 elif str(dump[key]).lower() in ["no", "false"]:
2114 # Compass richting no (Noord Oost) is valid input
2115 if key != "compass": dump[key] = False
2116 return dump
2117 datadump = fix_boolean(datadump)
2118
2119 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2120 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2121 elif 'latitude' in datadump and 'longitude' in datadump:
2122 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2123
2124 if datadump['nodename'].startswith('Proxy'):
2125 datadump['nodename'] = datadump['nodename'].lower()
2126
2127 for iface_key in get_interface_keys(datadump):
2128 try:
2129 # All our normal wireless cards are normal APs now
2130 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2131 datadump[iface_key]['mode'] = 'ap'
2132 # Wireless Leiden SSID have an consistent lowercase/uppercase
2133 if datadump[iface_key].has_key('ssid'):
2134 ssid = datadump[iface_key]['ssid']
2135 prefix = 'ap-WirelessLeiden-'
2136 if ssid.lower().startswith(prefix.lower()):
2137 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2138 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2139 datadump[iface_key]['mode'] = 'autogen-FIXME'
2140 if not datadump[iface_key].has_key('comment'):
2141 datadump[iface_key]['comment'] = 'autogen-FIXME'
2142
2143 if datadump[iface_key].has_key('ns_mac'):
2144 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2145
2146 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2147 datadump[iface_key] = datadump[iface_key]['desc']
2148
2149 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2150 # due to better collision avoidance.
2151 if datadump[iface_key]['type'] == '11b':
2152 datadump[iface_key]['type'] = '11g'
2153
2154 # Setting 802.11g channels to de-facto standards, to avoid
2155 # un-detected sharing with other overlapping channels
2156 #
2157 # Technically we could also use channel 13 in NL, but this is not
2158 # recommended as foreign devices might not be able to select this
2159 # channel. Secondly using 1,5,9,13 instead is going to clash with
2160 # the de-facto usage of 1,6,11.
2161 #
2162 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2163 channels_at_2400Mhz = (1,6,11)
2164 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2165 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2166 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2167 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2168
2169 # Mandatory interface keys
2170 if not datadump[iface_key].has_key('status'):
2171 datadump[iface_key]['status'] = 'planned'
2172
2173 x = datadump[iface_key]['comment']
2174 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2175
2176 # Fixing bridge_type if none is found
2177 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2178 if not 'bridge_type' in datadump[iface_key]:
2179 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2180
2181 # Making sure description works
2182 if datadump[iface_key].has_key('desc'):
2183 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2184 del datadump[iface_key]['desc']
2185 else:
2186 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2187 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2188 if response:
2189 datadump[iface_key]['comment'] = response
2190 del datadump[iface_key]['desc']
2191
2192 # Check DHCP configuration
2193 dhcp_type(datadump[iface_key])
2194
2195 # Set the compass value based on the angle between the poels
2196 if datadump[iface_key].has_key('ns_ip'):
2197 my_pool = poel[network(datadump[iface_key]['ip'])]
2198 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2199 if remote_hosts:
2200 compass_target = remote_hosts[0]
2201 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2202
2203 # Monitoring Group default
2204 if not 'monitoring_group' in datadump:
2205 datadump['monitoring_group'] = 'wleiden'
2206
2207 except Exception:
2208 print "# Error while processing interface %s" % iface_key
2209 raise
2210 store_yaml(datadump)
2211 except Exception:
2212 print "# Error while processing %s" % host
2213 raise
2214 elif sys.argv[1] == "list":
2215 use_fqdn = False
2216 if len(sys.argv) < 4:
2217 usage()
2218 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2219 usage()
2220 if not sys.argv[3] in ["nodes","proxies","systems"]:
2221 usage()
2222
2223 if len(sys.argv) > 4:
2224 if sys.argv[4] == "fqdn":
2225 use_fqdn = True
2226 else:
2227 usage()
2228
2229 for system in get_hostlist():
2230 datadump = get_yaml(system)
2231 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2232 continue
2233
2234 output = datadump['autogen_fqdn'] if use_fqdn else system
2235 if sys.argv[2] == "all":
2236 print output
2237 elif datadump['status'] == sys.argv[2]:
2238 print output
2239 elif sys.argv[1] == "create":
2240 if sys.argv[2] == "network.kml":
2241 print make_network_kml.make_graph()
2242 elif sys.argv[2] == "host-ips.txt":
2243 for system in get_hostlist():
2244 datadump = get_yaml(system)
2245 ips = [datadump['masterip']]
2246 for ifkey in get_interface_keys(datadump):
2247 ips.append(datadump[ifkey]['ip'].split('/')[0])
2248 print system, ' '.join(ips)
2249 elif sys.argv[2] == "host-pos.txt":
2250 for system in get_hostlist():
2251 datadump = get_yaml(system)
2252 print system, datadump['rdnap_x'], datadump['rdnap_y']
2253 elif sys.argv[2] == 'ssh_config':
2254 print '''
2255Host *.wleiden.net
2256 User root
2257
2258Host 172.16.*.*
2259 User root
2260'''
2261 for system in get_hostlist():
2262 datadump = get_yaml(system)
2263 print '''\
2264Host %s
2265 User root
2266
2267Host %s
2268 User root
2269
2270Host %s
2271 User root
2272
2273Host %s
2274 User root
2275''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2276 else:
2277 usage()
2278 else:
2279 usage()
2280 else:
2281 # Do not enable debugging for config requests as it highly clutters the output
2282 if not is_text_request():
2283 cgitb.enable()
2284 response_headers, output = process_cgi_request()
2285 print_cgi_response(response_headers, output)
2286
2287def application(environ, start_response):
2288 status = '200 OK'
2289 response_headers, output = process_cgi_request(environ)
2290 start_response(status, response_headers)
2291
2292 # Debugging only
2293 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2294 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2295 return [output]
2296
2297if __name__ == "__main__":
2298 main()
Note: See TracBrowser for help on using the repository browser.