source: genesis/tools/gformat.py@ 13615

Last change on this file since 13615 was 13606, checked in by rick, 8 years ago

Update network.kml and provide pointer on how-to create it.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 73.7 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13606 2016-08-17 08:57:31Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = []
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126def clear_cache():
127 ''' Poor mans cache implementation '''
128 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
129 datadump_cache = {}
130 interface_list_cache = {}
131 rc_conf_local_cache = {}
132 ileiden_proxies = []
133 normal_proxies = []
134 nameservers_cache = []
135 relations_cache = None
136
137NO_DHCP = 0
138DHCP_CLIENT = 10
139DHCP_SERVER = 20
140def dhcp_type(item):
141 if not item.has_key('dhcp'):
142 return NO_DHCP
143 elif not item['dhcp']:
144 return NO_DHCP
145 elif item['dhcp'].lower() == 'client':
146 return DHCP_CLIENT
147 else:
148 # Validation Checks
149 begin,end = map(int,item['dhcp'].split('-'))
150 if begin >= end:
151 raise ValueError("DHCP Start >= DHCP End")
152 return DHCP_SERVER
153
154def etrs2rd(lat, lon):
155 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
156 p2 = pyproj.Proj(init='EPSG:28992')
157 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
158 return (RDx, RDy)
159
160def rd2etrs(RDx, RDy):
161 p1 = pyproj.Proj(init='EPSG:28992')
162 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
163 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
164 return (lat, lon)
165
166def get_yaml(item,add_version_info=True):
167 try:
168 """ Get configuration yaml for 'item'"""
169 if datadump_cache.has_key(item):
170 return datadump_cache[item].copy()
171
172 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
173
174 # Default values
175 datadump = {
176 'autogen_revision' : 'NOTFOUND',
177 'autogen_gfile' : gfile,
178 'service_proxy_ileiden' : False,
179 }
180 f = open(gfile, 'r')
181 datadump.update(yaml.load(f,Loader=Loader))
182 if datadump['nodetype'] == 'Hybrid':
183 # Some values are defined implicitly
184 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
185 datadump['service_incoming_rdr'] = True
186 # Use some boring defaults
187 defaults = {
188 'service_proxy_normal' : False,
189 'service_accesspoint' : True,
190 'service_incoming_rdr' : False,
191 'service_concentrator' : False,
192 'monitoring_group' : 'wleiden',
193 }
194 for (key,value) in defaults.iteritems():
195 if not datadump.has_key(key):
196 datadump[key] = value
197 f.close()
198
199 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
200 if add_version_info:
201 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
202 lines = p.communicate()[0].split('\n')
203 if p.returncode == 0:
204 for line in lines:
205 if line:
206 (key, value) = line.split(': ')
207 datadump["autogen_" + key.lower().replace(' ','_')] = value
208
209 # Preformat certain needed variables for formatting and push those into special object
210 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
211
212 wlan_count=0
213 try:
214 for key in get_interface_keys(datadump, True):
215 datadump[key]['autogen_ifbase'] = key.split('_')[1]
216 datadump[key]['autogen_vlan'] = False
217
218 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
219 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
220 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
221 wlan_count += 1
222 else:
223 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
224 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
225 datadump[key]['autogen_vlan'] = key.split('_')[2]
226
227 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
228 if datadump[key]['autogen_bridge'] and not 'alias' in key:
229 datadump[key]['autogen_bridge_interfaces'] = datadump[key]['members'].split()
230 except Exception:
231 print "# Error while processing interface %s" % key
232 raise
233
234 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
235 if dhcp_type(datadump[key]) == DHCP_SERVER]
236
237 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
238 datadump['autogen_item'] = item
239
240 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
241 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
242 datadump_cache[item] = datadump.copy()
243 except Exception:
244 print "# Error while processing %s" % item
245 raise
246 return datadump
247
248
249def store_yaml(datadump, header=False):
250 """ Store configuration yaml for 'item'"""
251 item = datadump['autogen_item']
252 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
253
254 output = generate_wleiden_yaml(datadump, header)
255
256 f = open(gfile, 'w')
257 f.write(output)
258 f.close()
259
260
261def network(ip):
262 addr, mask = ip.split('/')
263 # Not parsing of these folks please
264 addr = parseaddr(addr)
265 mask = int(mask)
266 network = addr & ~((1 << (32 - mask)) - 1)
267 return network
268
269
270
271def make_relations():
272 """ Process _ALL_ yaml files to get connection relations """
273 global relations_cache
274
275 if relations_cache:
276 return relations_cache
277
278 errors = []
279 poel = defaultdict(list)
280
281 for host in get_hostlist():
282 datadump = get_yaml(host)
283 try:
284 for iface_key in get_interface_keys(datadump):
285 net_addr = network(datadump[iface_key]['ip'])
286 poel[net_addr] += [(host,datadump[iface_key].copy())]
287 except (KeyError, ValueError), e:
288 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
289 continue
290
291 relations_cache = (poel, errors)
292 return relations_cache
293
294
295
296def valid_addr(addr):
297 """ Show which address is valid in which are not """
298 return str(addr).startswith('172.')
299
300def get_hostlist():
301 """ Combined hosts and proxy list"""
302 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
303
304def angle_between_points(lat1,lat2,long1,long2):
305 """
306 Return Angle in radians between two GPS coordinates
307 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
308 """
309 dy = lat2 - lat1
310 dx = math.cos(lat1)*(long2 - long1)
311 angle = math.atan2(dy,dx)
312 return angle
313
314
315
316def angle_to_cd(angle):
317 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
318
319 # For easy conversion get positive degree
320 degrees = math.degrees(angle)
321 abs_degrees = 360 + degrees if degrees < 0 else degrees
322
323 # Numbers can be confusing calculate from the 4 main directions
324 p = 22.5
325 if abs_degrees < p:
326 cd = "n"
327 elif abs_degrees < (90 - p):
328 cd = "no"
329 elif abs_degrees < (90 + p):
330 cd = "o"
331 elif abs_degrees < (180 - p):
332 cd = "zo"
333 elif abs_degrees < (180 + p):
334 cd = "z"
335 elif abs_degrees < (270 - p):
336 cd = "zw"
337 elif abs_degrees < (270 + p):
338 cd = "w"
339 elif abs_degrees < (360 - p):
340 cd = "nw"
341 else:
342 cd = "n"
343 return cd
344
345
346
347def cd_between_hosts(hostA, hostB, datadumps):
348 # Using RDNAP coordinates
349 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
350 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
351 return angle_to_cd(math.atan2(dx,dy))
352
353 # GPS coordinates seems to fail somehow
354 #latA = float(datadumps[hostA]['latitude'])
355 #latB = float(datadumps[hostB]['latitude'])
356 #lonA = float(datadumps[hostA]['longitude'])
357 #lonB = float(datadumps[hostB]['longitude'])
358 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
359
360
361def generate_title(nodelist):
362 """ Main overview page """
363 items = {'root' : "." }
364 def fl(spaces, line):
365 return (' ' * spaces) + line + '\n'
366
367 output = """
368<html>
369 <head>
370 <title>Wireless leiden Configurator - GFormat</title>
371 <style type="text/css">
372 th {background-color: #999999}
373 tr:nth-child(odd) {background-color: #cccccc}
374 tr:nth-child(even) {background-color: #ffffff}
375 th, td {padding: 0.1em 1em}
376 </style>
377 </head>
378 <body>
379 <center>
380 <form type="GET" action="%(root)s">
381 <input type="hidden" name="action" value="update">
382 <input type="submit" value="Update Configuration Database (SVN)">
383 </form>
384 <table>
385 <caption><h3>Wireless Leiden Configurator</h3></caption>
386 """ % items
387
388 for node in nodelist:
389 items['node'] = node
390 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
391 for config in files:
392 items['config'] = config
393 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
394 output += fl(5, "</tr>")
395 output += """
396 </table>
397 <hr />
398 <em>%s</em>
399 </center>
400 </body>
401</html>
402 """ % __version__
403
404 return output
405
406
407
408def generate_node(node):
409 """ Print overview of all files available for node """
410 return "\n".join(files)
411
412def generate_node_overview(host):
413 """ Print overview of all files available for node """
414 datadump = get_yaml(host)
415 params = { 'host' : host }
416 output = "<em><a href='..'>Back to overview</a></em><hr />"
417 output += "<h2>Available files:</h2><ul>"
418 for cf in files:
419 params['cf'] = cf
420 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
421 output += "</ul>"
422
423 # Generate and connection listing
424 output += "<h2>Connected To:</h2><ul>"
425 (poel, errors) = make_relations()
426 for network, hosts in poel.iteritems():
427 if host in [x[0] for x in hosts]:
428 if len(hosts) == 1:
429 # Single not connected interface
430 continue
431 for remote,ifacedump in hosts:
432 if remote == host:
433 # This side of the interface
434 continue
435 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
436 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
437 output += "</ul>"
438 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
439
440 output += "<hr /><em><a href='..'>Back to overview</a></em>"
441 return output
442
443
444def generate_header(datadump, ctag="#"):
445 return """\
446%(ctag)s
447%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
448%(ctag)s
449""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
450
451
452
453def parseaddr(s):
454 """ Process IPv4 CIDR notation addr to a (binary) number """
455 f = s.split('.')
456 return (long(f[0]) << 24L) + \
457 (long(f[1]) << 16L) + \
458 (long(f[2]) << 8L) + \
459 long(f[3])
460
461
462
463def showaddr(a):
464 """ Display IPv4 addr in (dotted) CIDR notation """
465 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
466
467
468def is_member(ip, mask, canidate):
469 """ Return True if canidate is part of ip/mask block"""
470 ip_addr = parseaddr(ip)
471 ip_canidate = parseaddr(canidate)
472 mask = int(mask)
473 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
474 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
475 return ip_addr == ip_canidate
476
477
478
479def cidr2netmask(netmask):
480 """ Given a 'netmask' return corresponding CIDR """
481 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
482
483def get_network(addr, mask):
484 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
485
486
487def generate_dhcpd_conf(datadump):
488 """ Generate config file '/usr/local/etc/dhcpd.conf """
489 output = generate_header(datadump)
490 output += Template("""\
491# option definitions common to all supported networks...
492option domain-name "dhcp.{{ autogen_fqdn }}";
493
494default-lease-time 600;
495max-lease-time 7200;
496
497# Use this to enble / disable dynamic dns updates globally.
498#ddns-update-style none;
499
500# If this DHCP server is the official DHCP server for the local
501# network, the authoritative directive should be uncommented.
502authoritative;
503
504# Use this to send dhcp log messages to a different log file (you also
505# have to hack syslog.conf to complete the redirection).
506log-facility local7;
507
508#
509# Interface definitions
510#
511\n\n""").render(datadump)
512
513
514 # TODO: Use textwrap.fill instead
515 def indent(text, count):
516 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
517
518 dhcp_out = defaultdict(list)
519 for iface_key in get_interface_keys(datadump):
520 ifname = datadump[iface_key]['autogen_ifbase']
521 if not datadump[iface_key].has_key('comment'):
522 datadump[iface_key]['comment'] = None
523
524 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
525 dhcp_out[ifname].append("## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key])
526 else:
527 dhcp_out['bridge0'].append(" ## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key])
528
529 (addr, mask) = datadump[iface_key]['ip'].split('/')
530 datadump[iface_key]['autogen_addr'] = addr
531 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
532 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
533
534 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
535 dhcp_out[ifname].append(textwrap.dedent("""\
536 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
537 ### not autoritive
538 }
539 """ % datadump[iface_key]))
540 continue
541
542 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
543 dhcp_part = ".".join(addr.split('.')[0:3])
544 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
545 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
546
547 # Assume the first 10 IPs could be used for static entries
548 if 'no_portal' in datadump:
549 fixed = 5
550 for mac in datadump['no_portal']:
551 dhcp_out['bridge0'].append("""\
552 host fixed-%(ifname)s-%(fixed)s {
553 hardware ethernet %(mac)s;
554 fixed-address %(prefix)s.%(fixed)s;
555 }
556""" % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed })
557 fixed += 1
558
559 dhcp_out['bridge0'].append("""\
560 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
561 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
562 option routers %(autogen_addr)s;
563 option domain-name-servers %(autogen_addr)s;
564 }
565
566""" % datadump[iface_key])
567
568 for ifname,value in dhcp_out.iteritems():
569 if ifname == 'bridge0':
570 output += ("shared-network %s {\n" % ifname) + ''.join(value) + '}\n\n'
571 elif len(value) > 2:
572 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
573 else:
574 output += ''.join(value) + "\n\n"
575 return output
576
577
578
579def generate_dnsmasq_conf(datadump):
580 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
581 output = generate_header(datadump)
582 output += Template("""\
583# DHCP server options
584dhcp-authoritative
585dhcp-fqdn
586domain=dhcp.{{ autogen_fqdn }}
587domain-needed
588expand-hosts
589log-async=100
590
591# Low memory footprint
592cache-size=10000
593
594\n""").render(datadump)
595
596 for iface_key in get_interface_keys(datadump):
597 if not datadump[iface_key].has_key('comment'):
598 datadump[iface_key]['comment'] = None
599 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
600
601 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
602 output += "# not autoritive\n\n"
603 continue
604
605 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
606 (ip, cidr) = datadump[iface_key]['ip'].split('/')
607 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
608
609 dhcp_part = ".".join(ip.split('.')[0:3])
610 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
611 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
612 output += "dhcp-range=%(autogen_ifname)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
613
614 return output
615
616
617class AutoVivification(dict):
618 """Implementation of perl's autovivification feature."""
619 def __getitem__(self, item):
620 try:
621 return dict.__getitem__(self, item)
622 except KeyError:
623 value = self[item] = type(self)()
624 return value
625
626def make_interface_list(datadump):
627 if interface_list_cache.has_key(datadump['autogen_item']):
628 return (interface_list_cache[datadump['autogen_item']])
629 # lo0 configuration:
630 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
631 # - masterip is special as it needs to be assigned to at
632 # least one interface, so if not used assign to lo0
633 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
634 vlan_list = defaultdict(list)
635 flags_if = AutoVivification()
636 dhclient_if = {'lo0' : False}
637
638 # XXX: Find some way of send this output nicely
639 output = ''
640
641 masterip_used = False
642 for iface_key in get_interface_keys(datadump):
643 if datadump[iface_key]['ip'].startswith(datadump['masterip']):
644 masterip_used = True
645 break
646 if not masterip_used:
647 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
648
649 for iface_key in get_interface_keys(datadump):
650 ifacedump = datadump[iface_key]
651 ifname = ifacedump['autogen_ifname']
652
653 # If defined as vlan interface
654 if ifacedump['autogen_vlan']:
655 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
656
657 # Flag dhclient is possible
658 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
659 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
660
661 # Ethernet address
662 if ifacedump.has_key('ether'):
663 flags_if[ifname]['ether'] = ifacedump['ether']
664
665 # DHCP interfaces are to be added to bridge0
666 if ifname.replace('_','.') in datadump['autogen_dhcp_interfaces']:
667 ifname_base = 'bridge0'
668 else:
669 ifname_base = ifname
670
671 # Add interface IP to list
672 item = (ifacedump['ip'], ifacedump['comment'])
673 if addrs_list.has_key(ifname_base):
674 addrs_list[ifname_base].append(item)
675 else:
676 addrs_list[ifname_base] = [item]
677
678 # Alias only needs IP assignment for now, this might change if we
679 # are going to use virtual accesspoints
680 if "alias" in iface_key:
681 continue
682
683 # XXX: Might want to deduct type directly from interface name
684 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
685 # Default to station (client) mode
686 ifacedump['autogen_wlanmode'] = "sta"
687 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
688 ifacedump['autogen_wlanmode'] = "ap"
689
690 if not ifacedump.has_key('channel'):
691 if ifacedump['type'] == '11a':
692 ifacedump['channel'] = 36
693 else:
694 ifacedump['channel'] = 1
695
696 # Allow special hacks at the back like wds and stuff
697 if not ifacedump.has_key('extra'):
698 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
699 else:
700 ifacedump['autogen_extra'] = ifacedump['extra']
701
702 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
703
704 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
705 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
706 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
707 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
708 output += "\n"
709
710 elif ifacedump['type'] in ['ethernet', 'eth']:
711 # No special config needed besides IP
712 if ifacedump['autogen_bridge']:
713 output += "cloned_interfaces='%(autogen_ifname)s'\n" % ifacedump
714 output += "ifconfig_%s='addm %s up'\n" % (ifacedump['autogen_ifname'], ' addm '.join(ifacedump['autogen_bridge_interfaces']))
715 for member in ifacedump['autogen_bridge_interfaces']:
716 output += "ifconfig_%s='up'\n" % member
717 output += "\n"
718 else:
719 assert False, "Unknown type " + ifacedump['type']
720
721 store = (addrs_list, vlan_list, dhclient_if, flags_if, output)
722 interface_list_cache[datadump['autogen_item']] = store
723 return(store)
724
725
726
727def generate_rc_conf_local(datadump):
728 """ Generate configuration file '/etc/rc.conf.local' """
729 item = datadump['autogen_item']
730 if rc_conf_local_cache.has_key(item):
731 return rc_conf_local_cache[item]
732
733 if not datadump.has_key('ileiden'):
734 datadump['autogen_ileiden_enable'] = False
735 else:
736 datadump['autogen_ileiden_enable'] = datadump['ileiden']
737
738 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
739
740 if not ileiden_proxies or not normal_proxies:
741 for host in get_hostlist():
742 hostdump = get_yaml(host)
743 if hostdump['status'] == 'up':
744 if hostdump['service_proxy_ileiden']:
745 ileiden_proxies.append(hostdump)
746 if hostdump['service_proxy_normal']:
747 normal_proxies.append(hostdump)
748
749 datadump['autogen_ileiden_proxies'] = ileiden_proxies
750 datadump['autogen_normal_proxies'] = normal_proxies
751 datadump['autogen_ileiden_proxies_ips'] = ','.join([x['masterip'] for x in ileiden_proxies])
752 datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
753 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
754 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
755 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
756 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
757
758 output = generate_header(datadump, "#");
759 output += render_template(datadump, """\
760hostname='{{ autogen_fqdn }}'
761location='{{ location }}'
762nodetype="{{ nodetype }}"
763
764#
765# Configured listings
766#
767captive_portal_whitelist=""
768{% if nodetype == "Proxy" %}
769#
770# Proxy Configuration
771#
772{% if gateway and service_proxy_ileiden -%}
773defaultrouter="{{ gateway }}"
774{% else -%}
775#defaultrouter="NOTSET"
776{% endif -%}
777internalif="{{ internalif }}"
778ileiden_enable="{{ autogen_ileiden_enable }}"
779gateway_enable="{{ autogen_ileiden_enable }}"
780pf_enable="yes"
781pf_rules="/etc/pf.conf"
782{% if autogen_ileiden_enable -%}
783pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
784lvrouted_enable="{{ autogen_ileiden_enable }}"
785lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
786{% else -%}
787pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
788{% endif -%}
789{% if internalroute -%}
790static_routes="wleiden"
791route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
792{% endif -%}
793
794{% elif nodetype == "Hybrid" %}
795 #
796 # Hybrid Configuration
797 #
798 list_ileiden_proxies="
799 {% for item in autogen_ileiden_proxies -%}
800 {{ "%-16s"|format(item.masterip) }} # {{ item.nodename }}
801 {% endfor -%}
802 "
803 list_normal_proxies="
804 {% for item in autogen_normal_proxies -%}
805 {{ "%-16s"|format(item.masterip) }} # {{ item.nodename }}
806 {% endfor -%}
807 "
808
809 captive_portal_interfaces="bridge0"
810 externalif="{{ externalif|default('vr0', true) }}"
811 masterip="{{ masterip }}"
812
813 {% if gateway and service_proxy_ileiden %}
814 defaultrouter="{{ gateway }}"
815 {% else %}
816 #defaultrouter="NOTSET"
817 {% endif %}
818
819 #
820 # Defined services
821 #
822 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
823 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
824 service_accesspoint="{{ service_accesspoint|yesorno }}"
825 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
826 service_concentrator="{{ service_concentrator|yesorno }}"
827
828 {% if service_proxy_ileiden %}
829 pf_rules="/etc/pf.hybrid.conf"
830 {% if service_concentrator %}
831 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
832 {% else %}
833 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
834 {% endif %}
835 pf_flags="$pf_flags -D publicnat=80,443"
836 lvrouted_flags="$lvrouted_flags -g"
837 {% elif service_proxy_normal or service_incoming_rdr %}
838 pf_rules="/etc/pf.hybrid.conf"
839 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
840 pf_flags="$pf_flags -D publicnat=0"
841 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
842 named_setfib="1"
843 tinyproxy_setfib="1"
844 dnsmasq_setfib="1"
845 sshd_setfib="1"
846 {% else %}
847 named_auto_forward_only="YES"
848 pf_rules="/etc/pf.node.conf"
849 pf_flags=""
850 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
851 {% endif %}
852 {% if service_concentrator %}
853 # Do mind installing certificates is NOT done automatically for security reasons
854 openvpn_enable="YES"
855 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
856 {% endif %}
857
858 {% if service_proxy_normal %}
859 tinyproxy_enable="yes"
860 {% else %}
861 pen_wrapper_enable="yes"
862 {% endif %}
863
864 {% if service_accesspoint %}
865 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
866 {% endif %}
867
868 {% if board == "ALIX2" %}
869 #
870 # ''Fat'' configuration, board has 256MB RAM
871 #
872 dnsmasq_enable="NO"
873 named_enable="YES"
874 {% if autogen_dhcp_interfaces -%}
875 dhcpd_enable="YES"
876 dhcpd_flags="$dhcpd_flags bridge0"
877 {% endif -%}
878 {% elif board == "apu1d" %}
879 #
880 # ''Fat'' configuration, board has 1024MB RAM
881 #
882 dnsmasq_enable="NO"
883 local_unbound_enable="YES"
884 {% if autogen_dhcp_interfaces -%}
885 dhcpd_enable="YES"
886 dhcpd_flags="$dhcpd_flags bridge0"
887 {% endif -%}
888 {% endif -%}
889{% endif %}
890
891#
892# Script variables
893#
894attached_devices="{{ autogen_attached_devices|join(' ') }}"
895neighbours="{{ autogen_neighbours|join(' ') }}"
896
897
898#
899# Interface definitions
900#\n
901""")
902
903 (addrs_list, vlan_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
904 for iface, vlans in vlan_list.items():
905 output += 'vlans_%s="%s"\n' % (iface, ' '.join(vlans))
906
907 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
908 for iface in vlan_list.keys():
909 if not iface in addrs_list.keys():
910 output += "ifconfig_%s='up'\n" % iface
911
912 output += "\n"
913
914 # Details like SSID
915 if extra_ouput:
916 output += extra_ouput.strip() + "\n"
917
918 # Print IP address which needs to be assigned over here
919 output += "\n"
920 for iface,addrs in sorted(addrs_list.iteritems()):
921 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
922 output += "# %s || %s || %s\n" % (iface, addr, comment)
923
924 # Write DHCLIENT entry
925 if iface in dhclient_if and dhclient_if[iface]:
926 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
927 continue
928
929 # Make sure the external address is always first as this is needed in the
930 # firewall setup
931 addrs = sorted(
932 [x for x in addrs if not '0.0.0.0' in x[0]],
933 key=lambda x: x[0].split('.')[0],
934 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
935 )
936
937 idx_offset = 0
938 if iface == 'bridge0':
939 output += "cloned_interfaces='bridge0'\n"
940 output += "autobridge_interfaces='bridge0'\n"
941 output += "autobridge_bridge0='%s'\n" % ' '.join(datadump['autogen_dhcp_interfaces'])
942 output += "ifconfig_bridge0='%s up'\n" % addrs[0][0]
943 else:
944 if flags_if[iface].has_key('ether'):
945 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
946 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
947 idx_offset += 1
948 else:
949 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
950
951 for idx, addr in enumerate(addrs[1:]):
952 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
953 if iface == 'bridge0':
954 for dhcp_iface in datadump['autogen_dhcp_interfaces']:
955 output += "ifconfig_%s='up'\n" % dhcp_iface.replace('.','_')
956 output += "\n"
957
958 rc_conf_local_cache[datadump['autogen_item']] = output
959 return output
960
961
962
963def get_all_configs():
964 """ Get dict with key 'host' with all configs present """
965 configs = dict()
966 for host in get_hostlist():
967 datadump = get_yaml(host)
968 configs[host] = datadump
969 return configs
970
971
972def get_interface_keys(config, extra=False):
973 """ Quick hack to get all interface keys, later stage convert this to a iterator """
974 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
975 if extra == False:
976 return filter(lambda x: not "extra" in x, elems)
977 else:
978 return elems
979
980
981def get_used_ips(configs):
982 """ Return array of all IPs used in config files"""
983 ip_list = []
984 for config in configs:
985 ip_list.append(config['masterip'])
986 for iface_key in get_interface_keys(config, True):
987 l = config[iface_key]['ip']
988 addr, mask = l.split('/')
989 # Special case do not process
990 if valid_addr(addr):
991 ip_list.append(addr)
992 else:
993 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
994 return sorted(ip_list)
995
996
997
998def get_nameservers(max_servers=None):
999 if nameservers_cache:
1000 return nameservers_cache[0:max_servers]
1001
1002 for host in get_hostlist():
1003 hostdump = get_yaml(host)
1004 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1005 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
1006
1007 return nameservers_cache[0:max_servers]
1008
1009
1010def get_neighbours(datadump):
1011 (addrs_list, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1012
1013 (poel, errors) = make_relations()
1014 table = []
1015 for iface,addrs in sorted(addrs_list.iteritems()):
1016 if iface in ['lo0']:
1017 continue
1018
1019 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1020 if not addr.startswith('172.'):
1021 # Avoid listing internet connections as pool
1022 continue
1023 for neighbour in poel[network(addr)]:
1024 if neighbour[0] != datadump['autogen_item']:
1025 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_ifname'] + ")", neighbour[1]['comment']))
1026 return table
1027
1028
1029def get_attached_devices(datadump, url=False):
1030 table = []
1031 for iface_key in get_interface_keys(datadump, True):
1032 # Quick to avoid listing ath(4) interface as attached device
1033 if 'ath0' in iface_key:
1034 continue
1035 ifacedump = datadump[iface_key]
1036 if ifacedump.has_key('ns_ip'):
1037 x_ip = ifacedump['ns_ip'].split('/')[0]
1038 else:
1039 x_ip = ifacedump['ip'].split('/')[0]
1040
1041 if 'mode' in ifacedump:
1042 x_mode = ifacedump['mode']
1043 else:
1044 x_mode = 'unknown'
1045
1046 if 'bridge_type' in ifacedump:
1047 device_type = ifacedump['bridge_type']
1048 else:
1049 device_type = 'Unknown'
1050
1051 table.append((ifacedump['autogen_ifname'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1052 return table
1053
1054
1055def generate_resolv_conf(datadump):
1056 """ Generate configuration file '/etc/resolv.conf' """
1057 # XXX: This should properly going to be an datastructure soon
1058 datadump['autogen_header'] = generate_header(datadump, "#")
1059 datadump['autogen_edge_nameservers'] = ''
1060
1061
1062 for masterip,realname in get_nameservers():
1063 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1064
1065 return Template("""\
1066{{ autogen_header }}
1067search wleiden.net
1068
1069# Try local (cache) first
1070nameserver 127.0.0.1
1071
1072{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1073nameserver 8.8.8.8 # Google Public NameServer
1074nameserver 4.2.2.1 # Level3 Public NameServer
1075{% else -%}
1076# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1077{{ autogen_edge_nameservers }}
1078{% endif -%}
1079""").render(datadump)
1080
1081
1082
1083def generate_ntp_conf(datadump):
1084 """ Generate configuration file '/etc/ntp.conf' """
1085 # XXX: This should properly going to be an datastructure soon
1086
1087 datadump['autogen_header'] = generate_header(datadump, "#")
1088 datadump['autogen_ntp_servers'] = ''
1089 for host in get_hostlist():
1090 hostdump = get_yaml(host)
1091 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1092 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1093
1094 return Template("""\
1095{{ autogen_header }}
1096
1097{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1098# Machine hooked to internet.
1099server 0.nl.pool.ntp.org iburst maxpoll 9
1100server 1.nl.pool.ntp.org iburst maxpoll 9
1101server 2.nl.pool.ntp.org iburst maxpoll 9
1102server 3.nl.pool.ntp.org iburst maxpoll 9
1103{% else -%}
1104# Local Wireless Leiden NTP Servers.
1105server 0.pool.ntp.wleiden.net iburst maxpoll 9
1106server 1.pool.ntp.wleiden.net iburst maxpoll 9
1107server 2.pool.ntp.wleiden.net iburst maxpoll 9
1108server 3.pool.ntp.wleiden.net iburst maxpoll 9
1109
1110# All the configured NTP servers
1111{{ autogen_ntp_servers }}
1112{% endif %}
1113
1114# If a server loses sync with all upstream servers, NTP clients
1115# no longer follow that server. The local clock can be configured
1116# to provide a time source when this happens, but it should usually
1117# be configured on just one server on a network. For more details see
1118# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1119# The use of Orphan Mode may be preferable.
1120#
1121server 127.127.1.0
1122fudge 127.127.1.0 stratum 10
1123""").render(datadump)
1124
1125
1126def generate_pf_hybrid_conf_local(datadump):
1127 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1128 datadump['autogen_header'] = generate_header(datadump, "#")
1129 return Template("""\
1130{{ autogen_header }}
1131
1132# Redirect some internal facing services outside (7)
1133# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1134{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1135rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1136{% endfor -%}
1137""").render(datadump)
1138
1139def generate_motd(datadump):
1140 """ Generate configuration file '/etc/motd' """
1141 output = Template("""\
1142FreeBSD run ``service motd onestart'' to make me look normal
1143
1144 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1145 Loc: {{ location }}
1146
1147Services:
1148{% if board == "ALIX2" -%}
1149{{" -"}} Core Node ({{ board }})
1150{% else -%}
1151{{" -"}} Hulp Node ({{ board }})
1152{% endif -%}
1153{% if service_proxy_normal -%}
1154{{" -"}} Normal Proxy
1155{% endif -%}
1156{% if service_proxy_ileiden -%}
1157{{" -"}} iLeiden Proxy
1158{% endif -%}
1159{% if service_incoming_rdr -%}
1160{{" -"}} Incoming port redirects
1161{% endif %}
1162Interlinks:\n
1163""").render(datadump)
1164
1165
1166 def make_table(table):
1167 if not table:
1168 return " - none\n"
1169 else:
1170 lines = ""
1171 col_width = [max(len(x) for x in col) for col in zip(*table)]
1172 for row in table:
1173 lines += " - " + " || ".join("{:{}}".format(x, col_width[i]) for i, x in enumerate(row)) + "\n"
1174 return lines
1175
1176 (addrs_list, vlan_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1177 table = []
1178 for iface,addrs in sorted(addrs_list.iteritems()):
1179 if iface in ['lo0']:
1180 continue
1181 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1182 table.append((iface, addr, comment))
1183
1184 output += make_table(table)
1185 output += '\n'
1186 output += """\
1187Attached devices:
1188"""
1189 output += make_table(get_attached_devices(datadump, url=True))
1190 output += '\n'
1191 output += """\
1192Available neighbours:
1193"""
1194 output += make_table(get_neighbours(datadump))
1195
1196 return output
1197
1198
1199def format_yaml_value(value):
1200 """ Get yaml value in right syntax for outputting """
1201 if isinstance(value,str):
1202 output = '"%s"' % value
1203 else:
1204 output = value
1205 return output
1206
1207
1208
1209def format_wleiden_yaml(datadump):
1210 """ Special formatting to ensure it is editable"""
1211 output = "# Genesis config yaml style\n"
1212 output += "# vim:ts=2:et:sw=2:ai\n"
1213 output += "#\n"
1214 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1215 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1216 if key == 'rdr_rules':
1217 output += '%-10s:\n' % 'rdr_rules'
1218 for rdr_rule in datadump[key]:
1219 output += '- %s\n' % rdr_rule
1220 else:
1221 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1222
1223 output += "\n\n"
1224
1225 # Format (key, required)
1226 key_order = (
1227 ('comment', True),
1228 ('ip', True),
1229 ('ether', False),
1230 ('desc', True),
1231 ('sdesc', True),
1232 ('mode', True),
1233 ('type', True),
1234 ('extra_type', False),
1235 ('channel', False),
1236 ('ssid', False),
1237 ('wlan_mac', False),
1238 ('dhcp', True),
1239 ('compass', False),
1240 ('distance', False),
1241 ('ns_ip', False),
1242 ('repeater_ip', False),
1243 ('bullet2_ip', False),
1244 ('ns_mac', False),
1245 ('bullet2_mac', False),
1246 ('ns_type', False),
1247 ('bridge_type', False),
1248 ('members', True),
1249 ('status', True),
1250 )
1251
1252 for iface_key in sorted(iface_keys):
1253 try:
1254 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1255 if remainder:
1256 raise KeyError("invalid keys: %s" % remainder)
1257
1258 output += "%s:\n" % iface_key
1259 for key,required in key_order:
1260 if datadump[iface_key].has_key(key):
1261 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1262 output += "\n\n"
1263 except Exception:
1264 print "# Error while processing interface %s" % iface_key
1265 raise
1266
1267 return output
1268
1269
1270
1271def generate_wleiden_yaml(datadump, header=True):
1272 """ Generate (petty) version of wleiden.yaml"""
1273 output = generate_header(datadump, "#") if header else ''
1274
1275 for key in datadump.keys():
1276 if key.startswith('autogen_'):
1277 del datadump[key]
1278 # Interface autogen cleanups
1279 elif type(datadump[key]) == dict:
1280 for key2 in datadump[key].keys():
1281 if key2.startswith('autogen_'):
1282 del datadump[key][key2]
1283
1284 output += format_wleiden_yaml(datadump)
1285 return output
1286
1287def generate_nanostation_config(datadump, iface, ns_type):
1288 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1289 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1290
1291 datadump.update(datadump['iface_%s' % iface])
1292
1293 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1294
1295def generate_yaml(datadump):
1296 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1297
1298
1299
1300def generate_config(node, config, datadump=None):
1301 """ Print configuration file 'config' of 'node' """
1302 output = ""
1303 try:
1304 # Load config file
1305 if datadump == None:
1306 datadump = get_yaml(node)
1307
1308 if config == 'wleiden.yaml':
1309 output += generate_wleiden_yaml(datadump)
1310 elif config == 'authorized_keys':
1311 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1312 output += f.read()
1313 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1314 # Fetch local keys if existing
1315 if os.path.exists(node_keys):
1316 output += open(node_keys, 'r').read()
1317 f.close()
1318 elif config == 'dnsmasq.conf':
1319 output += generate_dnsmasq_conf(datadump)
1320 elif config == 'dhcpd.conf':
1321 output += generate_dhcpd_conf(datadump)
1322 elif config == 'rc.conf.local':
1323 output += generate_rc_conf_local(datadump)
1324 elif config == 'resolv.conf':
1325 output += generate_resolv_conf(datadump)
1326 elif config == 'ntp.conf':
1327 output += generate_ntp_conf(datadump)
1328 elif config == 'motd':
1329 output += generate_motd(datadump)
1330 elif config == 'pf.hybrid.conf.local':
1331 output += generate_pf_hybrid_conf_local(datadump)
1332 elif config.startswith('vr'):
1333 interface, ns_type = config.strip('.yaml').split('-')
1334 output += generate_nanostation_config(datadump, interface, ns_type)
1335 else:
1336 assert False, "Config not found!"
1337 except IOError, e:
1338 output += "[ERROR] Config file not found"
1339 return output
1340
1341
1342
1343def process_cgi_request(environ=os.environ):
1344 """ When calling from CGI """
1345 response_headers = []
1346 content_type = 'text/plain'
1347
1348 # Update repository if requested
1349 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1350 if form and form.has_key("action") and "update" in form["action"]:
1351 output = "[INFO] Updating subverion, please wait...\n"
1352 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1353 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1354 output += "[INFO] All done, redirecting in 5 seconds"
1355 response_headers += [
1356 ('Refresh', '5; url=.'),
1357 ]
1358 reload_cache()
1359 else:
1360 base_uri = environ['PATH_INFO']
1361 uri = base_uri.strip('/').split('/')
1362
1363 output = "Template Holder"
1364 if base_uri.endswith('/create/network.kml'):
1365 content_type='application/vnd.google-earth.kml+xml'
1366 output = make_network_kml.make_graph()
1367 elif base_uri.endswith('/api/get/nodeplanner.json'):
1368 content_type='application/json'
1369 output = make_network_kml.make_nodeplanner_json()
1370 elif not uri[0]:
1371 if is_text_request(environ):
1372 output = '\n'.join(get_hostlist())
1373 else:
1374 content_type = 'text/html'
1375 output = generate_title(get_hostlist())
1376 elif len(uri) == 1:
1377 if is_text_request(environ):
1378 output = generate_node(uri[0])
1379 else:
1380 content_type = 'text/html'
1381 output = generate_node_overview(uri[0])
1382 elif len(uri) == 2:
1383 output = generate_config(uri[0], uri[1])
1384 else:
1385 assert False, "Invalid option"
1386
1387 # Return response
1388 response_headers += [
1389 ('Content-type', content_type),
1390 ('Content-Length', str(len(output))),
1391 ]
1392 return(response_headers, str(output))
1393
1394
1395def make_dns(output_dir = 'dns', external = False):
1396 items = dict()
1397
1398 # hostname is key, IP is value
1399 wleiden_zone = defaultdict(list)
1400 wleiden_cname = dict()
1401
1402 pool = dict()
1403 for node in get_hostlist():
1404 datadump = get_yaml(node)
1405
1406 fqdn = datadump['nodename']
1407
1408 if datadump.has_key('rdr_host'):
1409 remote_target = datadump['rdr_host']
1410 elif datadump.has_key('remote_access') and datadump['remote_access']:
1411 remote_target = datadump['remote_access'].split(':')[0]
1412 else:
1413 remote_target = None
1414
1415 if remote_target:
1416 try:
1417 parseaddr(remote_target)
1418 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1419 except (IndexError, ValueError):
1420 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1421
1422
1423 wleiden_zone[fqdn].append((datadump['masterip'], True))
1424
1425 # Hacking to get proper DHCP IPs and hostnames
1426 for iface_key in get_interface_keys(datadump):
1427 iface_name = iface_key.replace('_','-')
1428 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1429 try:
1430 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1431 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1432 dhcp_part = ".".join(ip.split('.')[0:3])
1433 if ip != datadump['masterip']:
1434 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, False))
1435 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1436 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1437 except (AttributeError, ValueError, KeyError):
1438 # First push it into a pool, to indentify the counter-part later on
1439 addr = parseaddr(ip)
1440 cidr = int(cidr)
1441 addr = addr & ~((1 << (32 - cidr)) - 1)
1442 if pool.has_key(addr):
1443 pool[addr] += [(iface_name, fqdn, ip)]
1444 else:
1445 pool[addr] = [(iface_name, fqdn, ip)]
1446 continue
1447
1448
1449
1450 # WL uses an /29 to configure an interface. IP's are ordered like this:
1451 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1452
1453 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1454
1455 # Automatic naming convention of interlinks namely 2 + remote.lower()
1456 for (key,value) in pool.iteritems():
1457 # Make sure they are sorted from low-ip to high-ip
1458 value = sorted(value, key=lambda x: parseaddr(x[2]))
1459
1460 if len(value) == 1:
1461 (iface_name, fqdn, ip) = value[0]
1462 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1463
1464 # Device DNS names
1465 if 'cnode' in fqdn.lower():
1466 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1467 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1468
1469 elif len(value) == 2:
1470 (a_iface_name, a_fqdn, a_ip) = value[0]
1471 (b_iface_name, b_fqdn, b_ip) = value[1]
1472 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1473 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1474
1475 # Device DNS names
1476 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1477 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1478 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1479 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1480 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1481 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1482 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1483
1484 else:
1485 pool_members = [k[1] for k in value]
1486 for item in value:
1487 (iface_name, fqdn, ip) = item
1488 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1489
1490 # Include static DNS entries
1491 # XXX: Should they override the autogenerated results?
1492 # XXX: Convert input to yaml more useable.
1493 # Format:
1494 ##; this is a comment
1495 ## roomburgh=Roomburgh1
1496 ## apkerk1.Vosko=172.17.176.8 ;this as well
1497 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1498
1499 # Hack to allow special entries, for development
1500 wleiden_raw = {}
1501
1502 for line in dns_list:
1503 reverse = False
1504 k, items = line.items()[0]
1505 if type(items) == dict:
1506 if items.has_key('reverse'):
1507 reverse = items['reverse']
1508 items = items['a']
1509 else:
1510 items = items['cname']
1511 items = [items] if type(items) != list else items
1512 for item in items:
1513 if item.startswith('IN '):
1514 wleiden_raw[k] = item
1515 elif valid_addr(item):
1516 wleiden_zone[k].append((item, reverse))
1517 else:
1518 wleiden_cname[k] = item
1519
1520 # Hack to get dynamic pool listing
1521 def chunks(l, n):
1522 return [l[i:i+n] for i in range(0, len(l), n)]
1523
1524 ntp_servers = [x[0] for x in get_nameservers()]
1525 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1526 for ntp_server in chunk:
1527 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1528
1529 details = dict()
1530 # 24 updates a day allowed
1531 details['serial'] = time.strftime('%Y%m%d%H')
1532
1533 if external:
1534 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1535 else:
1536 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1537
1538 details['master'] = dns_masters[0]
1539 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1540
1541 dns_header = '''
1542$TTL 3h
1543%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1544 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1545
1546%(ns_servers)s
1547 \n'''
1548
1549
1550 if not os.path.isdir(output_dir):
1551 os.makedirs(output_dir)
1552 details['zone'] = 'wleiden.net'
1553 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1554 f.write(dns_header % details)
1555
1556 for host,items in wleiden_zone.iteritems():
1557 for ip,reverse in items:
1558 if ip not in ['0.0.0.0']:
1559 f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
1560 for source,dest in wleiden_cname.iteritems():
1561 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1562 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1563 for source, dest in wleiden_raw.iteritems():
1564 f.write("%s.wleiden.net. %s\n" % (source, dest))
1565 f.close()
1566
1567 # Create whole bunch of specific sub arpa zones. To keep it compliant
1568 for s in range(16,32):
1569 details['zone'] = '%i.172.in-addr.arpa' % s
1570 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1571 f.write(dns_header % details)
1572
1573 #XXX: Not effient, fix to proper data structure and do checks at other
1574 # stages
1575 for host,items in wleiden_zone.iteritems():
1576 for ip,reverse in items:
1577 if not reverse:
1578 continue
1579 if valid_addr(ip):
1580 if valid_addr(ip):
1581 if int(ip.split('.')[1]) == s:
1582 rev_ip = '.'.join(reversed(ip.split('.')))
1583 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1584 f.close()
1585
1586
1587def usage():
1588 print """Usage: %(prog)s <argument>
1589Argument:
1590\tcleanup = Cleanup all YAML files to specified format
1591\tstandalone [port] = Run configurator webserver [8000]
1592\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1593\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1594\tfull-export = Generate yaml export script for heatmap.
1595\tstatic [outputdir] = Generate all config files and store on disk
1596\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1597\ttest <node> [<file>] = Receive output for certain node [all files].
1598\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1599\tlist <status> <items> = List systems which have certain status
1600\tcreate network.kml = Create Network KML file for use in Google Earth
1601
1602Arguments:
1603\t<node> = NodeName (example: HybridRick)
1604\t<file> = %(files)s
1605\t<status> = all|up|down|planned
1606\t<items> = systems|nodes|proxies
1607
1608NOTE FOR DEVELOPERS; you can test your changes like this:
1609 BEFORE any changes in this code:
1610 $ ./gformat.py static /tmp/pre
1611 AFTER the changes:
1612 $ ./gformat.py static /tmp/post
1613 VIEW differences and VERIFY all are OK:
1614 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1615""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1616 exit(0)
1617
1618
1619def is_text_request(environ=os.environ):
1620 """ Find out whether we are calling from the CLI or any text based CLI utility """
1621 try:
1622 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1623 except KeyError:
1624 return True
1625
1626def switchFormat(setting):
1627 if setting:
1628 return "YES"
1629 else:
1630 return "NO"
1631
1632def rlinput(prompt, prefill=''):
1633 import readline
1634 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1635 try:
1636 return raw_input(prompt)
1637 finally:
1638 readline.set_startup_hook()
1639
1640def fix_conflict(left, right, default='i'):
1641 while True:
1642 print "## %-30s | %-30s" % (left, right)
1643 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1644 if not c:
1645 c = default
1646
1647 if c in ['l','1']:
1648 return left
1649 elif c in ['r','2']:
1650 return right
1651 elif c in ['e', '3']:
1652 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1653 elif c in ['i', '4']:
1654 return None
1655 else:
1656 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1657
1658
1659
1660def print_cgi_response(response_headers, output):
1661 """Could we not use some kind of wsgi wrapper to make this output?"""
1662 for header in response_headers:
1663 print "%s: %s" % header
1664 print
1665 print output
1666
1667
1668def fill_cache():
1669 ''' Poor man re-loading of few cache items (the slow ones) '''
1670 for host in get_hostlist():
1671 get_yaml(host)
1672
1673
1674def reload_cache():
1675 clear_cache()
1676 fill_cache()
1677
1678
1679def main():
1680 """Hard working sub"""
1681 # Allow easy hacking using the CLI
1682 if not os.environ.has_key('PATH_INFO'):
1683 if len(sys.argv) < 2:
1684 usage()
1685
1686 if sys.argv[1] == "standalone":
1687 import SocketServer
1688 import CGIHTTPServer
1689 # Hop to the right working directory.
1690 os.chdir(os.path.dirname(__file__))
1691 try:
1692 PORT = int(sys.argv[2])
1693 except (IndexError,ValueError):
1694 PORT = 8000
1695
1696 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1697 """ Serve this CGI from the root of the webserver """
1698 def is_cgi(self):
1699 if "favicon" in self.path:
1700 return False
1701
1702 self.cgi_info = (os.path.basename(__file__), self.path)
1703 self.path = ''
1704 return True
1705 handler = MyCGIHTTPRequestHandler
1706 SocketServer.TCPServer.allow_reuse_address = True
1707 httpd = SocketServer.TCPServer(("", PORT), handler)
1708 httpd.server_name = 'localhost'
1709 httpd.server_port = PORT
1710
1711 logger.info("serving at port %s", PORT)
1712 try:
1713 httpd.serve_forever()
1714 except KeyboardInterrupt:
1715 httpd.shutdown()
1716 logger.info("All done goodbye")
1717 elif sys.argv[1] == "test":
1718 # Basic argument validation
1719 try:
1720 node = sys.argv[2]
1721 except IndexError:
1722 print "Invalid argument"
1723 exit(1)
1724 except IOError as e:
1725 print e
1726 exit(1)
1727
1728 datadump = get_yaml(node)
1729
1730
1731 # Get files to generate
1732 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1733
1734 # Actual config generation
1735 for config in gen_files:
1736 logger.info("## Generating %s %s", node, config)
1737 print generate_config(node, config, datadump)
1738 elif sys.argv[1] == "test-cgi":
1739 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1740 os.environ['SCRIPT_NAME'] = __file__
1741 response_headers, output = process_cgi_request()
1742 print_cgi_response(response_headers, output)
1743 elif sys.argv[1] == "static":
1744 items = dict()
1745 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1746 for node in get_hostlist():
1747 items['node'] = node
1748 items['wdir'] = "%(output_dir)s/%(node)s" % items
1749 if not os.path.isdir(items['wdir']):
1750 os.makedirs(items['wdir'])
1751 datadump = get_yaml(node)
1752 for config in files:
1753 items['config'] = config
1754 logger.info("## Generating %(node)s %(config)s" % items)
1755 f = open("%(wdir)s/%(config)s" % items, "w")
1756 f.write(generate_config(node, config, datadump))
1757 f.close()
1758 elif sys.argv[1] == "wind-export":
1759 items = dict()
1760 for node in get_hostlist():
1761 datadump = get_yaml(node)
1762 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1763 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1764 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1765 VALUES (
1766 (SELECT id FROM users WHERE username = 'rvdzwet'),
1767 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1768 'Y');""" % datadump
1769 #for config in files:
1770 # items['config'] = config
1771 # print "## Generating %(node)s %(config)s" % items
1772 # f = open("%(wdir)s/%(config)s" % items, "w")
1773 # f.write(generate_config(node, config, datadump))
1774 # f.close()
1775 for node in get_hostlist():
1776 datadump = get_yaml(node)
1777 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1778 ifacedump = datadump[iface_key]
1779 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1780 ifacedump['nodename'] = datadump['nodename']
1781 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1782 ifacedump['channel'] = 0
1783 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1784 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1785 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1786 elif sys.argv[1] == "nagios-export":
1787 try:
1788 heavy_load = (sys.argv[2] == "--heavy-load")
1789 except IndexError:
1790 heavy_load = False
1791
1792 hostgroup_details = {
1793 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1794 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1795 'walphen' : 'Stichting Wireless Alphen',
1796 'westeinder' : 'Westeinder Plassen',
1797 }
1798
1799 # Convert IP to Host
1800 ip2host = {'root' : 'root'}
1801 for host in get_hostlist():
1802 datadump = get_yaml(host)
1803 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1804 for iface in get_interface_keys(datadump):
1805 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1806
1807 # Find dependency tree based on output of lvrouted.mytree of nearest node
1808 parents = defaultdict(list)
1809 stack = ['root']
1810 prev_depth = 0
1811 for line in open('lvrouted.mytree').readlines():
1812 depth = line.count('\t')
1813 ip = line.strip().split()[0]
1814
1815 if prev_depth < depth:
1816 try:
1817 parents[ip2host[ip]].append(ip2host[stack[-1]])
1818 except KeyError as e:
1819 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1820 stack.append(ip)
1821 elif prev_depth > depth:
1822 stack = stack[:(depth - prev_depth)]
1823 elif prev_depth == depth:
1824 try:
1825 parents[ip2host[ip]].append(ip2host[stack[-1]])
1826 except KeyError as e:
1827 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1828
1829
1830 prev_depth = depth
1831 # Observe that some nodes has themself as parent or multiple parents
1832 # for now take only the first parent, other behaviour is yet to be explained
1833
1834
1835
1836 params = {
1837 'check_interval' : 5 if heavy_load else 120,
1838 'retry_interval' : 1 if heavy_load else 10,
1839 'max_check_attempts' : 10 if heavy_load else 6,
1840 'notification_interval': 120 if heavy_load else 240,
1841 }
1842
1843 print '''\
1844define host {
1845 name wleiden-node ; Default Node Template
1846 use generic-host ; Use the standard template as initial starting point
1847 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1848 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1849 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1850 notification_interval %(notification_interval)s
1851 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1852 check_command check-host-alive ; Default command to check FreeBSD hosts
1853 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1854}
1855
1856define service {
1857 name wleiden-service ; Default Service Template
1858 use generic-service ; Use the standard template as initial starting point
1859 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1860 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1861 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1862 notification_interval %(notification_interval)s
1863 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1864 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1865}
1866
1867# Please make sure to install:
1868# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1869#
1870# Recompile net-mgmt/nagios-plugins to support check_snmp
1871# make -C /usr/ports/net-mgmt/nagios-plugins
1872#
1873# Install net/bind-tools to allow v2/check_dns_wl to work:
1874# pkg install bind-tools
1875#
1876define command{
1877 command_name check_snmp_disk
1878 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1879}
1880
1881define command{
1882 command_name check_netsnmp_load
1883 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1884}
1885
1886define command{
1887 command_name check_netsnmp_proc
1888 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1889}
1890
1891define command{
1892 command_name check_by_ssh
1893 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1894}
1895
1896define command{
1897 command_name check_dns_wl
1898 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1899}
1900
1901define command{
1902 command_name check_snmp_uptime
1903 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1904}
1905
1906
1907# TDB: dhcp leases
1908# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1909
1910# TDB: internet status
1911# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1912
1913# TDB: Advanced local passive checks
1914# /usr/local/libexec/nagios/check_by_ssh
1915''' % params
1916
1917 print '''\
1918# Service Group, not displayed by default
1919define hostgroup {
1920 hostgroup_name srv_hybrid
1921 alias All Hybrid Nodes
1922 register 0
1923}
1924
1925define service {
1926 use wleiden-service
1927 hostgroup_name srv_hybrid
1928 service_description SSH
1929 check_command check_ssh
1930}
1931
1932define service {
1933 use wleiden-service,service-pnp
1934 hostgroup_name srv_hybrid
1935 service_description HTTP
1936 check_command check_http
1937}
1938
1939define service {
1940 use wleiden-service
1941 hostgroup_name srv_hybrid
1942 service_description DNS
1943 check_command check_dns_wl!"www.wirelessleiden.nl"
1944}
1945'''
1946
1947 if heavy_load:
1948 print '''\
1949define service {
1950 use wleiden-service
1951 hostgroup_name srv_hybrid
1952 service_description UPTIME
1953 check_command check_snmp_uptime
1954}
1955
1956#define service {
1957# use wleiden-service
1958# hostgroup_name srv_hybrid
1959# service_description NTP
1960# check_command check_ntp_peer
1961#}
1962
1963define service {
1964 use wleiden-service
1965 hostgroup_name srv_hybrid
1966 service_description LOAD
1967 check_command check_netsnmp_load
1968}
1969
1970define service {
1971 use wleiden-service
1972 hostgroup_name srv_hybrid
1973 service_description PROC
1974 check_command check_netsnmp_proc
1975}
1976
1977define service {
1978 use wleiden-service
1979 hostgroup_name srv_hybrid
1980 service_description DISK
1981 check_command check_snmp_disk
1982}
1983'''
1984 for node in get_hostlist():
1985 datadump = get_yaml(node)
1986 if not datadump['status'] == 'up':
1987 continue
1988 if not hostgroup_details.has_key(datadump['monitoring_group']):
1989 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
1990 print '''\
1991define host {
1992 use wleiden-node,host-pnp
1993 contact_groups admins
1994 host_name %(autogen_fqdn)s
1995 address %(masterip)s
1996 hostgroups srv_hybrid,%(monitoring_group)s\
1997''' % datadump
1998 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
1999 print '''\
2000 parents %(parents)s\
2001''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2002 print '''\
2003}
2004'''
2005
2006
2007 for name,alias in hostgroup_details.iteritems():
2008 print '''\
2009define hostgroup {
2010 hostgroup_name %s
2011 alias %s
2012} ''' % (name, alias)
2013
2014
2015 elif sys.argv[1] == "full-export":
2016 hosts = {}
2017 for node in get_hostlist():
2018 datadump = get_yaml(node)
2019 hosts[datadump['nodename']] = datadump
2020 print yaml.dump(hosts)
2021
2022 elif sys.argv[1] == "dns":
2023 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2024 elif sys.argv[1] == "cleanup":
2025 # First generate all datadumps
2026 datadumps = dict()
2027 ssid_to_node = dict()
2028 for host in get_hostlist():
2029 logger.info("# Processing: %s", host)
2030 # Set some boring default values
2031 datadump = { 'board' : 'UNKNOWN' }
2032 datadump.update(get_yaml(host))
2033 datadumps[datadump['nodename']] = datadump
2034
2035 (poel, errors) = make_relations()
2036 print "\n".join(["# WARNING: %s" % x for x in errors])
2037
2038 for host,datadump in datadumps.iteritems():
2039 try:
2040 # Convert all yes and no to boolean values
2041 def fix_boolean(dump):
2042 for key in dump.keys():
2043 if type(dump[key]) == dict:
2044 dump[key] = fix_boolean(dump[key])
2045 elif str(dump[key]).lower() in ["yes", "true"]:
2046 dump[key] = True
2047 elif str(dump[key]).lower() in ["no", "false"]:
2048 # Compass richting no (Noord Oost) is valid input
2049 if key != "compass": dump[key] = False
2050 return dump
2051 datadump = fix_boolean(datadump)
2052
2053 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2054 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2055 elif 'latitude' in datadump and 'longitude' in datadump:
2056 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2057
2058 if datadump['nodename'].startswith('Proxy'):
2059 datadump['nodename'] = datadump['nodename'].lower()
2060
2061 for iface_key in get_interface_keys(datadump):
2062 try:
2063 # All our normal wireless cards are normal APs now
2064 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2065 datadump[iface_key]['mode'] = 'ap'
2066 # Wireless Leiden SSID have an consistent lowercase/uppercase
2067 if datadump[iface_key].has_key('ssid'):
2068 ssid = datadump[iface_key]['ssid']
2069 prefix = 'ap-WirelessLeiden-'
2070 if ssid.lower().startswith(prefix.lower()):
2071 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2072 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2073 datadump[iface_key]['mode'] = 'autogen-FIXME'
2074 if not datadump[iface_key].has_key('comment'):
2075 datadump[iface_key]['comment'] = 'autogen-FIXME'
2076
2077 if datadump[iface_key].has_key('ns_mac'):
2078 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2079
2080 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2081 datadump[iface_key] = datadump[iface_key]['desc']
2082
2083 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2084 # due to better collision avoidance.
2085 if datadump[iface_key]['type'] == '11b':
2086 datadump[iface_key]['type'] = '11g'
2087
2088 # Setting 802.11g channels to de-facto standards, to avoid
2089 # un-detected sharing with other overlapping channels
2090 #
2091 # Technically we could also use channel 13 in NL, but this is not
2092 # recommended as foreign devices might not be able to select this
2093 # channel. Secondly using 1,5,9,13 instead is going to clash with
2094 # the de-facto usage of 1,6,11.
2095 #
2096 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2097 channels_at_2400Mhz = (1,6,11)
2098 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2099 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2100 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2101 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2102
2103 # Mandatory interface keys
2104 if not datadump[iface_key].has_key('status'):
2105 datadump[iface_key]['status'] = 'planned'
2106
2107 x = datadump[iface_key]['comment']
2108 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2109
2110 # Fixing bridge_type if none is found
2111 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2112 if not 'bridge_type' in datadump[iface_key]:
2113 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2114
2115 # Making sure description works
2116 if datadump[iface_key].has_key('desc'):
2117 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2118 del datadump[iface_key]['desc']
2119 else:
2120 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2121 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2122 if response:
2123 datadump[iface_key]['comment'] = response
2124 del datadump[iface_key]['desc']
2125
2126 # Check DHCP configuration
2127 dhcp_type(datadump[iface_key])
2128
2129 # Set the compass value based on the angle between the poels
2130 if datadump[iface_key].has_key('ns_ip'):
2131 my_pool = poel[network(datadump[iface_key]['ip'])]
2132 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2133 if remote_hosts:
2134 compass_target = remote_hosts[0]
2135 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2136
2137 # Monitoring Group default
2138 if not 'monitoring_group' in datadump:
2139 datadump['monitoring_group'] = 'wleiden'
2140
2141 except Exception:
2142 print "# Error while processing interface %s" % iface_key
2143 raise
2144 store_yaml(datadump)
2145 except Exception:
2146 print "# Error while processing %s" % host
2147 raise
2148 elif sys.argv[1] == "list":
2149 use_fqdn = False
2150 if len(sys.argv) < 4:
2151 usage()
2152 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2153 usage()
2154 if not sys.argv[3] in ["nodes","proxies","systems"]:
2155 usage()
2156
2157 if len(sys.argv) > 4:
2158 if sys.argv[4] == "fqdn":
2159 use_fqdn = True
2160 else:
2161 usage()
2162
2163 for system in get_hostlist():
2164 datadump = get_yaml(system)
2165 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2166 continue
2167
2168 output = datadump['autogen_fqdn'] if use_fqdn else system
2169 if sys.argv[2] == "all":
2170 print output
2171 elif datadump['status'] == sys.argv[2]:
2172 print output
2173 elif sys.argv[1] == "create":
2174 if sys.argv[2] == "network.kml":
2175 print make_network_kml.make_graph()
2176 elif sys.argv[2] == "host-ips.txt":
2177 for system in get_hostlist():
2178 datadump = get_yaml(system)
2179 ips = [datadump['masterip']]
2180 for ifkey in get_interface_keys(datadump):
2181 ips.append(datadump[ifkey]['ip'].split('/')[0])
2182 print system, ' '.join(ips)
2183 elif sys.argv[2] == "host-pos.txt":
2184 for system in get_hostlist():
2185 datadump = get_yaml(system)
2186 print system, datadump['rdnap_x'], datadump['rdnap_y']
2187 elif sys.argv[2] == 'ssh_config':
2188 print '''
2189Host *.wleiden.net
2190 User root
2191
2192Host 172.16.*.*
2193 User root
2194'''
2195 for system in get_hostlist():
2196 datadump = get_yaml(system)
2197 print '''\
2198Host %s
2199 User root
2200
2201Host %s
2202 User root
2203
2204Host %s
2205 User root
2206
2207Host %s
2208 User root
2209''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2210 else:
2211 usage()
2212 else:
2213 usage()
2214 else:
2215 # Do not enable debugging for config requests as it highly clutters the output
2216 if not is_text_request():
2217 cgitb.enable()
2218 response_headers, output = process_cgi_request()
2219 print_cgi_response(response_headers, output)
2220
2221def application(environ, start_response):
2222 status = '200 OK'
2223 response_headers, output = process_cgi_request(environ)
2224 start_response(status, response_headers)
2225
2226 # Debugging only
2227 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2228 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2229 return [output]
2230
2231if __name__ == "__main__":
2232 main()
Note: See TracBrowser for help on using the repository browser.