source: genesis/tools/gformat.py@ 13521

Last change on this file since 13521 was 13516, checked in by rick, 9 years ago

Supporting spaces within SSIDs to create names like "WiFi Zoeterwoude".
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 72.3 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13516 2015-12-03 07:42:47Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = []
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126def clear_cache():
127 ''' Poor mans cache implementation '''
128 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
129 datadump_cache = {}
130 interface_list_cache = {}
131 rc_conf_local_cache = {}
132 ileiden_proxies = []
133 normal_proxies = []
134 nameservers_cache = []
135 relations_cache = None
136
137NO_DHCP = 0
138DHCP_CLIENT = 10
139DHCP_SERVER = 20
140def dhcp_type(item):
141 if not item.has_key('dhcp'):
142 return NO_DHCP
143 elif not item['dhcp']:
144 return NO_DHCP
145 elif item['dhcp'].lower() == 'client':
146 return DHCP_CLIENT
147 else:
148 # Validation Checks
149 begin,end = map(int,item['dhcp'].split('-'))
150 if begin >= end:
151 raise ValueError("DHCP Start >= DHCP End")
152 return DHCP_SERVER
153
154def etrs2rd(lat, lon):
155 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
156 p2 = pyproj.Proj(init='EPSG:28992')
157 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
158 return (RDx, RDy)
159
160def rd2etrs(RDx, RDy):
161 p1 = pyproj.Proj(init='EPSG:28992')
162 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
163 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
164 return (lat, lon)
165
166def get_yaml(item,add_version_info=True):
167 try:
168 """ Get configuration yaml for 'item'"""
169 if datadump_cache.has_key(item):
170 return datadump_cache[item].copy()
171
172 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
173
174 # Default values
175 datadump = {
176 'autogen_revision' : 'NOTFOUND',
177 'autogen_gfile' : gfile,
178 'service_proxy_ileiden' : False,
179 }
180 f = open(gfile, 'r')
181 datadump.update(yaml.load(f,Loader=Loader))
182 if datadump['nodetype'] == 'Hybrid':
183 # Some values are defined implicitly
184 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
185 datadump['service_incoming_rdr'] = True
186 # Use some boring defaults
187 defaults = {
188 'service_proxy_normal' : False,
189 'service_accesspoint' : True,
190 'service_incoming_rdr' : False,
191 'service_concentrator' : False,
192 'monitoring_group' : 'wleiden',
193 }
194 for (key,value) in defaults.iteritems():
195 if not datadump.has_key(key):
196 datadump[key] = value
197 f.close()
198
199 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
200 if add_version_info:
201 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
202 lines = p.communicate()[0].split('\n')
203 if p.returncode == 0:
204 for line in lines:
205 if line:
206 (key, value) = line.split(': ')
207 datadump["autogen_" + key.lower().replace(' ','_')] = value
208
209 # Preformat certain needed variables for formatting and push those into special object
210 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
211
212 wlan_count=0
213 try:
214 for key in get_interface_keys(datadump, True):
215 datadump[key]['autogen_ifbase'] = key.split('_')[1]
216 datadump[key]['autogen_vlan'] = False
217
218 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
219 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
220 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
221 wlan_count += 1
222 else:
223 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
224 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
225 datadump[key]['autogen_vlan'] = key.split('_')[2]
226
227 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
228 if datadump[key]['autogen_bridge'] and not 'alias' in key:
229 datadump[key]['autogen_bridge_interfaces'] = datadump[key]['members'].split()
230 except Exception:
231 print "# Error while processing interface %s" % key
232 raise
233
234 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
235 if dhcp_type(datadump[key]) == DHCP_SERVER]
236
237 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
238 datadump['autogen_item'] = item
239
240 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
241 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
242 datadump_cache[item] = datadump.copy()
243 except Exception:
244 print "# Error while processing %s" % item
245 raise
246 return datadump
247
248
249def store_yaml(datadump, header=False):
250 """ Store configuration yaml for 'item'"""
251 item = datadump['autogen_item']
252 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
253
254 output = generate_wleiden_yaml(datadump, header)
255
256 f = open(gfile, 'w')
257 f.write(output)
258 f.close()
259
260
261def network(ip):
262 addr, mask = ip.split('/')
263 # Not parsing of these folks please
264 addr = parseaddr(addr)
265 mask = int(mask)
266 network = addr & ~((1 << (32 - mask)) - 1)
267 return network
268
269
270
271def make_relations():
272 """ Process _ALL_ yaml files to get connection relations """
273 global relations_cache
274
275 if relations_cache:
276 return relations_cache
277
278 errors = []
279 poel = defaultdict(list)
280
281 for host in get_hostlist():
282 datadump = get_yaml(host)
283 try:
284 for iface_key in get_interface_keys(datadump):
285 net_addr = network(datadump[iface_key]['ip'])
286 poel[net_addr] += [(host,datadump[iface_key].copy())]
287 except (KeyError, ValueError), e:
288 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
289 continue
290
291 relations_cache = (poel, errors)
292 return relations_cache
293
294
295
296def valid_addr(addr):
297 """ Show which address is valid in which are not """
298 return str(addr).startswith('172.')
299
300def get_hostlist():
301 """ Combined hosts and proxy list"""
302 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
303
304def angle_between_points(lat1,lat2,long1,long2):
305 """
306 Return Angle in radians between two GPS coordinates
307 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
308 """
309 dy = lat2 - lat1
310 dx = math.cos(lat1)*(long2 - long1)
311 angle = math.atan2(dy,dx)
312 return angle
313
314
315
316def angle_to_cd(angle):
317 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
318
319 # For easy conversion get positive degree
320 degrees = math.degrees(angle)
321 abs_degrees = 360 + degrees if degrees < 0 else degrees
322
323 # Numbers can be confusing calculate from the 4 main directions
324 p = 22.5
325 if abs_degrees < p:
326 cd = "n"
327 elif abs_degrees < (90 - p):
328 cd = "no"
329 elif abs_degrees < (90 + p):
330 cd = "o"
331 elif abs_degrees < (180 - p):
332 cd = "zo"
333 elif abs_degrees < (180 + p):
334 cd = "z"
335 elif abs_degrees < (270 - p):
336 cd = "zw"
337 elif abs_degrees < (270 + p):
338 cd = "w"
339 elif abs_degrees < (360 - p):
340 cd = "nw"
341 else:
342 cd = "n"
343 return cd
344
345
346
347def cd_between_hosts(hostA, hostB, datadumps):
348 # Using RDNAP coordinates
349 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
350 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
351 return angle_to_cd(math.atan2(dx,dy))
352
353 # GPS coordinates seems to fail somehow
354 #latA = float(datadumps[hostA]['latitude'])
355 #latB = float(datadumps[hostB]['latitude'])
356 #lonA = float(datadumps[hostA]['longitude'])
357 #lonB = float(datadumps[hostB]['longitude'])
358 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
359
360
361def generate_title(nodelist):
362 """ Main overview page """
363 items = {'root' : "." }
364 def fl(spaces, line):
365 return (' ' * spaces) + line + '\n'
366
367 output = """
368<html>
369 <head>
370 <title>Wireless leiden Configurator - GFormat</title>
371 <style type="text/css">
372 th {background-color: #999999}
373 tr:nth-child(odd) {background-color: #cccccc}
374 tr:nth-child(even) {background-color: #ffffff}
375 th, td {padding: 0.1em 1em}
376 </style>
377 </head>
378 <body>
379 <center>
380 <form type="GET" action="%(root)s">
381 <input type="hidden" name="action" value="update">
382 <input type="submit" value="Update Configuration Database (SVN)">
383 </form>
384 <table>
385 <caption><h3>Wireless Leiden Configurator</h3></caption>
386 """ % items
387
388 for node in nodelist:
389 items['node'] = node
390 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
391 for config in files:
392 items['config'] = config
393 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
394 output += fl(5, "</tr>")
395 output += """
396 </table>
397 <hr />
398 <em>%s</em>
399 </center>
400 </body>
401</html>
402 """ % __version__
403
404 return output
405
406
407
408def generate_node(node):
409 """ Print overview of all files available for node """
410 return "\n".join(files)
411
412def generate_node_overview(host):
413 """ Print overview of all files available for node """
414 datadump = get_yaml(host)
415 params = { 'host' : host }
416 output = "<em><a href='..'>Back to overview</a></em><hr />"
417 output += "<h2>Available files:</h2><ul>"
418 for cf in files:
419 params['cf'] = cf
420 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
421 output += "</ul>"
422
423 # Generate and connection listing
424 output += "<h2>Connected To:</h2><ul>"
425 (poel, errors) = make_relations()
426 for network, hosts in poel.iteritems():
427 if host in [x[0] for x in hosts]:
428 if len(hosts) == 1:
429 # Single not connected interface
430 continue
431 for remote,ifacedump in hosts:
432 if remote == host:
433 # This side of the interface
434 continue
435 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
436 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
437 output += "</ul>"
438 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
439
440 output += "<hr /><em><a href='..'>Back to overview</a></em>"
441 return output
442
443
444def generate_header(datadump, ctag="#"):
445 return """\
446%(ctag)s
447%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
448%(ctag)s
449""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
450
451
452
453def parseaddr(s):
454 """ Process IPv4 CIDR notation addr to a (binary) number """
455 f = s.split('.')
456 return (long(f[0]) << 24L) + \
457 (long(f[1]) << 16L) + \
458 (long(f[2]) << 8L) + \
459 long(f[3])
460
461
462
463def showaddr(a):
464 """ Display IPv4 addr in (dotted) CIDR notation """
465 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
466
467
468def is_member(ip, mask, canidate):
469 """ Return True if canidate is part of ip/mask block"""
470 ip_addr = parseaddr(ip)
471 ip_canidate = parseaddr(canidate)
472 mask = int(mask)
473 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
474 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
475 return ip_addr == ip_canidate
476
477
478
479def cidr2netmask(netmask):
480 """ Given a 'netmask' return corresponding CIDR """
481 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
482
483def get_network(addr, mask):
484 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
485
486
487def generate_dhcpd_conf(datadump):
488 """ Generate config file '/usr/local/etc/dhcpd.conf """
489 output = generate_header(datadump)
490 output += Template("""\
491# option definitions common to all supported networks...
492option domain-name "dhcp.{{ autogen_fqdn }}";
493
494default-lease-time 600;
495max-lease-time 7200;
496
497# Use this to enble / disable dynamic dns updates globally.
498#ddns-update-style none;
499
500# If this DHCP server is the official DHCP server for the local
501# network, the authoritative directive should be uncommented.
502authoritative;
503
504# Use this to send dhcp log messages to a different log file (you also
505# have to hack syslog.conf to complete the redirection).
506log-facility local7;
507
508#
509# Interface definitions
510#
511\n""").render(datadump)
512
513 dhcp_out = defaultdict(list)
514 for iface_key in get_interface_keys(datadump):
515 ifname = datadump[iface_key]['autogen_ifbase']
516 if not datadump[iface_key].has_key('comment'):
517 datadump[iface_key]['comment'] = None
518
519 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
520 dhcp_out[iface_key].append(" ## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key])
521 else:
522 dhcp_out[ifname].append(" ## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key])
523
524 (addr, mask) = datadump[iface_key]['ip'].split('/')
525 datadump[iface_key]['autogen_addr'] = addr
526 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
527 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
528 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
529 dhcp_out[iface_key].append(" subnet %(autogen_subnet)s netmask %(autogen_netmask)s {\n ### not autoritive\n }\n" % \
530 datadump[iface_key])
531 continue
532
533 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
534 dhcp_part = ".".join(addr.split('.')[0:3])
535 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
536 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
537
538 # Assume the first 10 IPs could be used for static entries
539 if 'no_portal' in datadump:
540 fixed = 5
541 for mac in datadump['no_portal']:
542 dhcp_out[ifname].append("""\
543 host fixed-%(ifname)s-%(fixed)s {
544 hardware ethernet %(mac)s;
545 fixed-address %(prefix)s.%(fixed)s;
546 }
547""" % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed })
548 fixed += 1
549
550 dhcp_out[ifname].append("""\
551 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
552 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
553 option routers %(autogen_addr)s;
554 option domain-name-servers %(autogen_addr)s;
555 }
556""" % datadump[iface_key])
557
558 for ifname,value in dhcp_out.iteritems():
559 output += ("shared-network %s {\n" % ifname) + ''.join(value) + '}\n\n'
560 return output
561
562
563
564def generate_dnsmasq_conf(datadump):
565 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
566 output = generate_header(datadump)
567 output += Template("""\
568# DHCP server options
569dhcp-authoritative
570dhcp-fqdn
571domain=dhcp.{{ autogen_fqdn }}
572domain-needed
573expand-hosts
574log-async=100
575
576# Low memory footprint
577cache-size=10000
578
579\n""").render(datadump)
580
581 for iface_key in get_interface_keys(datadump):
582 if not datadump[iface_key].has_key('comment'):
583 datadump[iface_key]['comment'] = None
584 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
585
586 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
587 output += "# not autoritive\n\n"
588 continue
589
590 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
591 (ip, cidr) = datadump[iface_key]['ip'].split('/')
592 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
593
594 dhcp_part = ".".join(ip.split('.')[0:3])
595 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
596 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
597 output += "dhcp-range=%(autogen_ifname)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
598
599 return output
600
601
602def make_interface_list(datadump):
603 if interface_list_cache.has_key(datadump['autogen_item']):
604 return (interface_list_cache[datadump['autogen_item']])
605 # lo0 configuration:
606 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
607 # - masterip is special as it needs to be assigned to at
608 # least one interface, so if not used assign to lo0
609 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
610 vlan_list = defaultdict(list)
611 dhclient_if = {'lo0' : False}
612
613 # XXX: Find some way of send this output nicely
614 output = ''
615
616 masterip_used = False
617 for iface_key in get_interface_keys(datadump):
618 if datadump[iface_key]['ip'].startswith(datadump['masterip']):
619 masterip_used = True
620 break
621 if not masterip_used:
622 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
623
624 for iface_key in get_interface_keys(datadump):
625 ifacedump = datadump[iface_key]
626 ifname = ifacedump['autogen_ifname']
627
628 # If defined as vlan interface
629 if ifacedump['autogen_vlan']:
630 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
631
632 # Flag dhclient is possible
633 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
634 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
635
636 # DHCP interfaces are to be added to bridge0
637 if ifname.replace('_','.') in datadump['autogen_dhcp_interfaces']:
638 ifname_base = 'bridge0'
639 else:
640 ifname_base = ifname
641
642 # Add interface IP to list
643 item = (ifacedump['ip'], ifacedump['comment'])
644 if addrs_list.has_key(ifname_base):
645 addrs_list[ifname_base].append(item)
646 else:
647 addrs_list[ifname_base] = [item]
648
649 # Alias only needs IP assignment for now, this might change if we
650 # are going to use virtual accesspoints
651 if "alias" in iface_key:
652 continue
653
654 # XXX: Might want to deduct type directly from interface name
655 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
656 # Default to station (client) mode
657 ifacedump['autogen_wlanmode'] = "sta"
658 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
659 ifacedump['autogen_wlanmode'] = "ap"
660
661 if not ifacedump.has_key('channel'):
662 if ifacedump['type'] == '11a':
663 ifacedump['channel'] = 36
664 else:
665 ifacedump['channel'] = 1
666
667 # Allow special hacks at the back like wds and stuff
668 if not ifacedump.has_key('extra'):
669 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
670 else:
671 ifacedump['autogen_extra'] = ifacedump['extra']
672
673 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
674 output += ("create_args_%(autogen_ifname)s='wlanmode %(autogen_wlanmode)s mode " +\
675 "%(type)s ssid \"%(ssid)s\" %(autogen_extra)s channel %(channel)s'\n") % ifacedump
676 output += "\n"
677
678 elif ifacedump['type'] in ['ethernet', 'eth']:
679 # No special config needed besides IP
680 if ifacedump['autogen_bridge']:
681 output += "cloned_interfaces='%(autogen_ifname)s'\n" % ifacedump
682 output += "ifconfig_%s='addm %s up'\n" % (ifacedump['autogen_ifname'], ' addm '.join(ifacedump['autogen_bridge_interfaces']))
683 for member in ifacedump['autogen_bridge_interfaces']:
684 output += "ifconfig_%s='up'\n" % member
685 output += "\n"
686 else:
687 assert False, "Unknown type " + ifacedump['type']
688
689 store = (addrs_list, vlan_list, dhclient_if, output)
690 interface_list_cache[datadump['autogen_item']] = store
691 return(store)
692
693
694
695def generate_rc_conf_local(datadump):
696 """ Generate configuration file '/etc/rc.conf.local' """
697 item = datadump['autogen_item']
698 if rc_conf_local_cache.has_key(item):
699 return rc_conf_local_cache[item]
700
701 if not datadump.has_key('ileiden'):
702 datadump['autogen_ileiden_enable'] = False
703 else:
704 datadump['autogen_ileiden_enable'] = datadump['ileiden']
705
706 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
707
708 if not ileiden_proxies or not normal_proxies:
709 for host in get_hostlist():
710 hostdump = get_yaml(host)
711 if hostdump['status'] == 'up':
712 if hostdump['service_proxy_ileiden']:
713 ileiden_proxies.append(hostdump)
714 if hostdump['service_proxy_normal']:
715 normal_proxies.append(hostdump)
716
717 datadump['autogen_ileiden_proxies'] = ileiden_proxies
718 datadump['autogen_normal_proxies'] = normal_proxies
719 datadump['autogen_ileiden_proxies_ips'] = ','.join([x['masterip'] for x in ileiden_proxies])
720 datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
721 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
722 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
723 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
724 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
725
726 output = generate_header(datadump, "#");
727 output += render_template(datadump, """\
728hostname='{{ autogen_fqdn }}'
729location='{{ location }}'
730nodetype="{{ nodetype }}"
731
732#
733# Configured listings
734#
735captive_portal_whitelist=""
736{% if nodetype == "Proxy" %}
737#
738# Proxy Configuration
739#
740{% if gateway and service_proxy_ileiden -%}
741defaultrouter="{{ gateway }}"
742{% else -%}
743#defaultrouter="NOTSET"
744{% endif -%}
745internalif="{{ internalif }}"
746ileiden_enable="{{ autogen_ileiden_enable }}"
747gateway_enable="{{ autogen_ileiden_enable }}"
748pf_enable="yes"
749pf_rules="/etc/pf.conf"
750{% if autogen_ileiden_enable -%}
751pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
752lvrouted_enable="{{ autogen_ileiden_enable }}"
753lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
754{% else -%}
755pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
756{% endif -%}
757{% if internalroute -%}
758static_routes="wleiden"
759route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
760{% endif -%}
761
762{% elif nodetype == "Hybrid" %}
763 #
764 # Hybrid Configuration
765 #
766 list_ileiden_proxies="
767 {% for item in autogen_ileiden_proxies -%}
768 {{ "%-16s"|format(item.masterip) }} # {{ item.nodename }}
769 {% endfor -%}
770 "
771 list_normal_proxies="
772 {% for item in autogen_normal_proxies -%}
773 {{ "%-16s"|format(item.masterip) }} # {{ item.nodename }}
774 {% endfor -%}
775 "
776
777 captive_portal_interfaces="bridge0"
778 externalif="{{ externalif|default('vr0', true) }}"
779 masterip="{{ masterip }}"
780
781 {% if gateway and service_proxy_ileiden %}
782 defaultrouter="{{ gateway }}"
783 {% else %}
784 #defaultrouter="NOTSET"
785 {% endif %}
786
787 #
788 # Defined services
789 #
790 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
791 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
792 service_accesspoint="{{ service_accesspoint|yesorno }}"
793 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
794 service_concentrator="{{ service_concentrator|yesorno }}"
795
796 {% if service_proxy_ileiden %}
797 pf_rules="/etc/pf.hybrid.conf"
798 {% if service_concentrator %}
799 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
800 {% else %}
801 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
802 {% endif %}
803 pf_flags="$pf_flags -D publicnat=80,443"
804 lvrouted_flags="$lvrouted_flags -g"
805 {% elif service_proxy_normal or service_incoming_rdr %}
806 pf_rules="/etc/pf.hybrid.conf"
807 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
808 pf_flags="$pf_flags -D publicnat=0"
809 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
810 named_setfib="1"
811 tinyproxy_setfib="1"
812 dnsmasq_setfib="1"
813 sshd_setfib="1"
814 {% else %}
815 named_auto_forward_only="YES"
816 pf_rules="/etc/pf.node.conf"
817 pf_flags=""
818 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
819 {% endif %}
820 {% if service_concentrator %}
821 # Do mind installing certificates is NOT done automatically for security reasons
822 openvpn_enable="YES"
823 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
824 {% endif %}
825
826 {% if service_proxy_normal %}
827 tinyproxy_enable="yes"
828 {% else %}
829 pen_wrapper_enable="yes"
830 {% endif %}
831
832 {% if service_accesspoint %}
833 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
834 {% endif %}
835
836 {% if board == "ALIX2" %}
837 #
838 # ''Fat'' configuration, board has 256MB RAM
839 #
840 dnsmasq_enable="NO"
841 named_enable="YES"
842 {% if autogen_dhcp_interfaces -%}
843 dhcpd_enable="YES"
844 dhcpd_flags="$dhcpd_flags bridge0"
845 {% endif -%}
846 {% elif board == "apu1d" %}
847 #
848 # ''Fat'' configuration, board has 1024MB RAM
849 #
850 dnsmasq_enable="NO"
851 local_unbound_enable="YES"
852 {% if autogen_dhcp_interfaces -%}
853 dhcpd_enable="YES"
854 dhcpd_flags="$dhcpd_flags bridge0"
855 {% endif -%}
856 {% endif -%}
857{% endif %}
858
859#
860# Script variables
861#
862attached_devices="{{ autogen_attached_devices|join(' ') }}"
863neighbours="{{ autogen_neighbours|join(' ') }}"
864
865
866#
867# Interface definitions
868#\n
869""")
870
871 (addrs_list, vlan_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
872 for iface, vlans in vlan_list.items():
873 output += 'vlans_%s="%s"\n' % (iface, ' '.join(vlans))
874
875 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
876 for iface in vlan_list.keys():
877 if not iface in addrs_list.keys():
878 output += "ifconfig_%s='up'\n" % iface
879
880 output += "\n"
881 output += "cloned_interfaces='bridge0'\n"
882
883 # Details like SSID
884 if extra_ouput:
885 output += extra_ouput.strip() + "\n"
886
887 # Print IP address which needs to be assigned over here
888 output += "\n"
889 for iface,addrs in sorted(addrs_list.iteritems()):
890 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
891 output += "# %s || %s || %s\n" % (iface, addr, comment)
892
893 # Write DHCLIENT entry
894 if iface in dhclient_if and dhclient_if[iface]:
895 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
896
897 # Make sure the external address is always first as this is needed in the
898 # firewall setup
899 addrs = sorted(
900 [x for x in addrs if not '0.0.0.0' in x[0]],
901 key=lambda x: x[0].split('.')[0],
902 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
903 )
904
905 if iface == 'bridge0':
906 output += "ifconfig_bridge0='addm %s %s up'\n" % (' addm '.join(datadump['autogen_dhcp_interfaces']), addrs[0][0])
907 else:
908 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
909 for idx, addr in enumerate(addrs[1:]):
910 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx, addr[0])
911 if iface == 'bridge0':
912 for dhcp_iface in datadump['autogen_dhcp_interfaces']:
913 output += "ifconfig_%s='up'\n" % dhcp_iface.replace('.','_')
914 output += "\n"
915
916 rc_conf_local_cache[datadump['autogen_item']] = output
917 return output
918
919
920
921def get_all_configs():
922 """ Get dict with key 'host' with all configs present """
923 configs = dict()
924 for host in get_hostlist():
925 datadump = get_yaml(host)
926 configs[host] = datadump
927 return configs
928
929
930def get_interface_keys(config, extra=False):
931 """ Quick hack to get all interface keys, later stage convert this to a iterator """
932 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
933 if extra == False:
934 return filter(lambda x: not "extra" in x, elems)
935 else:
936 return elems
937
938
939def get_used_ips(configs):
940 """ Return array of all IPs used in config files"""
941 ip_list = []
942 for config in configs:
943 ip_list.append(config['masterip'])
944 for iface_key in get_interface_keys(config, True):
945 l = config[iface_key]['ip']
946 addr, mask = l.split('/')
947 # Special case do not process
948 if valid_addr(addr):
949 ip_list.append(addr)
950 else:
951 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
952 return sorted(ip_list)
953
954
955
956def get_nameservers(max_servers=None):
957 if nameservers_cache:
958 return nameservers_cache[0:max_servers]
959
960 for host in get_hostlist():
961 hostdump = get_yaml(host)
962 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
963 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
964
965 return nameservers_cache[0:max_servers]
966
967
968def get_neighbours(datadump):
969 (addrs_list, _, dhclient_if, extra_ouput) = make_interface_list(datadump)
970
971 (poel, errors) = make_relations()
972 table = []
973 for iface,addrs in sorted(addrs_list.iteritems()):
974 if iface in ['lo0']:
975 continue
976
977 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
978 if not addr.startswith('172.'):
979 # Avoid listing internet connections as pool
980 continue
981 for neighbour in poel[network(addr)]:
982 if neighbour[0] != datadump['autogen_item']:
983 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_ifname'] + ")", neighbour[1]['comment']))
984 return table
985
986
987def get_attached_devices(datadump, url=False):
988 table = []
989 for iface_key in get_interface_keys(datadump, True):
990 # Quick to avoid listing ath(4) interface as attached device
991 if 'ath0' in iface_key:
992 continue
993 ifacedump = datadump[iface_key]
994 if ifacedump.has_key('ns_ip'):
995 x_ip = ifacedump['ns_ip'].split('/')[0]
996 else:
997 x_ip = ifacedump['ip'].split('/')[0]
998
999 if 'mode' in ifacedump:
1000 x_mode = ifacedump['mode']
1001 else:
1002 x_mode = 'unknown'
1003
1004 if 'bridge_type' in ifacedump:
1005 device_type = ifacedump['bridge_type']
1006 else:
1007 device_type = 'Unknown'
1008
1009 table.append((ifacedump['autogen_ifname'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1010 return table
1011
1012
1013def generate_resolv_conf(datadump):
1014 """ Generate configuration file '/etc/resolv.conf' """
1015 # XXX: This should properly going to be an datastructure soon
1016 datadump['autogen_header'] = generate_header(datadump, "#")
1017 datadump['autogen_edge_nameservers'] = ''
1018
1019
1020 for masterip,realname in get_nameservers():
1021 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1022
1023 return Template("""\
1024{{ autogen_header }}
1025search wleiden.net
1026
1027# Try local (cache) first
1028nameserver 127.0.0.1
1029
1030{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1031nameserver 8.8.8.8 # Google Public NameServer
1032nameserver 4.2.2.1 # Level3 Public NameServer
1033{% else -%}
1034# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1035{{ autogen_edge_nameservers }}
1036{% endif -%}
1037""").render(datadump)
1038
1039
1040
1041def generate_ntp_conf(datadump):
1042 """ Generate configuration file '/etc/ntp.conf' """
1043 # XXX: This should properly going to be an datastructure soon
1044
1045 datadump['autogen_header'] = generate_header(datadump, "#")
1046 datadump['autogen_ntp_servers'] = ''
1047 for host in get_hostlist():
1048 hostdump = get_yaml(host)
1049 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1050 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1051
1052 return Template("""\
1053{{ autogen_header }}
1054
1055{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1056# Machine hooked to internet.
1057server 0.nl.pool.ntp.org iburst maxpoll 9
1058server 1.nl.pool.ntp.org iburst maxpoll 9
1059server 2.nl.pool.ntp.org iburst maxpoll 9
1060server 3.nl.pool.ntp.org iburst maxpoll 9
1061{% else -%}
1062# Local Wireless Leiden NTP Servers.
1063server 0.pool.ntp.wleiden.net iburst maxpoll 9
1064server 1.pool.ntp.wleiden.net iburst maxpoll 9
1065server 2.pool.ntp.wleiden.net iburst maxpoll 9
1066server 3.pool.ntp.wleiden.net iburst maxpoll 9
1067
1068# All the configured NTP servers
1069{{ autogen_ntp_servers }}
1070{% endif %}
1071
1072# If a server loses sync with all upstream servers, NTP clients
1073# no longer follow that server. The local clock can be configured
1074# to provide a time source when this happens, but it should usually
1075# be configured on just one server on a network. For more details see
1076# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1077# The use of Orphan Mode may be preferable.
1078#
1079server 127.127.1.0
1080fudge 127.127.1.0 stratum 10
1081""").render(datadump)
1082
1083
1084def generate_pf_hybrid_conf_local(datadump):
1085 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1086 datadump['autogen_header'] = generate_header(datadump, "#")
1087 return Template("""\
1088{{ autogen_header }}
1089
1090# Redirect some internal facing services outside (7)
1091# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1092{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1093rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1094{% endfor -%}
1095""").render(datadump)
1096
1097def generate_motd(datadump):
1098 """ Generate configuration file '/etc/motd' """
1099 output = Template("""\
1100FreeBSD run ``service motd onestart'' to make me look normal
1101
1102 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1103 Loc: {{ location }}
1104
1105Services:
1106{% if board == "ALIX2" -%}
1107{{" -"}} Core Node ({{ board }})
1108{% else -%}
1109{{" -"}} Hulp Node ({{ board }})
1110{% endif -%}
1111{% if service_proxy_normal -%}
1112{{" -"}} Normal Proxy
1113{% endif -%}
1114{% if service_proxy_ileiden -%}
1115{{" -"}} iLeiden Proxy
1116{% endif -%}
1117{% if service_incoming_rdr -%}
1118{{" -"}} Incoming port redirects
1119{% endif %}
1120Interlinks:\n
1121""").render(datadump)
1122
1123
1124 def make_table(table):
1125 if not table:
1126 return " - none\n"
1127 else:
1128 lines = ""
1129 col_width = [max(len(x) for x in col) for col in zip(*table)]
1130 for row in table:
1131 lines += " - " + " || ".join("{:{}}".format(x, col_width[i]) for i, x in enumerate(row)) + "\n"
1132 return lines
1133
1134 (addrs_list, vlan_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
1135 table = []
1136 for iface,addrs in sorted(addrs_list.iteritems()):
1137 if iface in ['lo0']:
1138 continue
1139 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1140 table.append((iface, addr, comment))
1141
1142 output += make_table(table)
1143 output += '\n'
1144 output += """\
1145Attached devices:
1146"""
1147 output += make_table(get_attached_devices(datadump, url=True))
1148 output += '\n'
1149 output += """\
1150Available neighbours:
1151"""
1152 output += make_table(get_neighbours(datadump))
1153
1154 return output
1155
1156
1157def format_yaml_value(value):
1158 """ Get yaml value in right syntax for outputting """
1159 if isinstance(value,str):
1160 output = '"%s"' % value
1161 else:
1162 output = value
1163 return output
1164
1165
1166
1167def format_wleiden_yaml(datadump):
1168 """ Special formatting to ensure it is editable"""
1169 output = "# Genesis config yaml style\n"
1170 output += "# vim:ts=2:et:sw=2:ai\n"
1171 output += "#\n"
1172 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1173 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1174 if key == 'rdr_rules':
1175 output += '%-10s:\n' % 'rdr_rules'
1176 for rdr_rule in datadump[key]:
1177 output += '- %s\n' % rdr_rule
1178 else:
1179 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1180
1181 output += "\n\n"
1182
1183 # Format (key, required)
1184 key_order = (
1185 ('comment', True),
1186 ('ip', True),
1187 ('desc', True),
1188 ('sdesc', True),
1189 ('mode', True),
1190 ('type', True),
1191 ('extra_type', False),
1192 ('channel', False),
1193 ('ssid', False),
1194 ('wlan_mac', False),
1195 ('dhcp', True),
1196 ('compass', False),
1197 ('distance', False),
1198 ('ns_ip', False),
1199 ('repeater_ip', False),
1200 ('bullet2_ip', False),
1201 ('ns_mac', False),
1202 ('bullet2_mac', False),
1203 ('ns_type', False),
1204 ('bridge_type', False),
1205 ('members', True),
1206 ('status', True),
1207 )
1208
1209 for iface_key in sorted(iface_keys):
1210 try:
1211 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1212 if remainder:
1213 raise KeyError("invalid keys: %s" % remainder)
1214
1215 output += "%s:\n" % iface_key
1216 for key,required in key_order:
1217 if datadump[iface_key].has_key(key):
1218 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1219 output += "\n\n"
1220 except Exception:
1221 print "# Error while processing interface %s" % iface_key
1222 raise
1223
1224 return output
1225
1226
1227
1228def generate_wleiden_yaml(datadump, header=True):
1229 """ Generate (petty) version of wleiden.yaml"""
1230 output = generate_header(datadump, "#") if header else ''
1231
1232 for key in datadump.keys():
1233 if key.startswith('autogen_'):
1234 del datadump[key]
1235 # Interface autogen cleanups
1236 elif type(datadump[key]) == dict:
1237 for key2 in datadump[key].keys():
1238 if key2.startswith('autogen_'):
1239 del datadump[key][key2]
1240
1241 output += format_wleiden_yaml(datadump)
1242 return output
1243
1244def generate_nanostation_config(datadump, iface, ns_type):
1245 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1246 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1247
1248 datadump.update(datadump['iface_%s' % iface])
1249
1250 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1251
1252def generate_yaml(datadump):
1253 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1254
1255
1256
1257def generate_config(node, config, datadump=None):
1258 """ Print configuration file 'config' of 'node' """
1259 output = ""
1260 try:
1261 # Load config file
1262 if datadump == None:
1263 datadump = get_yaml(node)
1264
1265 if config == 'wleiden.yaml':
1266 output += generate_wleiden_yaml(datadump)
1267 elif config == 'authorized_keys':
1268 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1269 output += f.read()
1270 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1271 # Fetch local keys if existing
1272 if os.path.exists(node_keys):
1273 output += open(node_keys, 'r').read()
1274 f.close()
1275 elif config == 'dnsmasq.conf':
1276 output += generate_dnsmasq_conf(datadump)
1277 elif config == 'dhcpd.conf':
1278 output += generate_dhcpd_conf(datadump)
1279 elif config == 'rc.conf.local':
1280 output += generate_rc_conf_local(datadump)
1281 elif config == 'resolv.conf':
1282 output += generate_resolv_conf(datadump)
1283 elif config == 'ntp.conf':
1284 output += generate_ntp_conf(datadump)
1285 elif config == 'motd':
1286 output += generate_motd(datadump)
1287 elif config == 'pf.hybrid.conf.local':
1288 output += generate_pf_hybrid_conf_local(datadump)
1289 elif config.startswith('vr'):
1290 interface, ns_type = config.strip('.yaml').split('-')
1291 output += generate_nanostation_config(datadump, interface, ns_type)
1292 else:
1293 assert False, "Config not found!"
1294 except IOError, e:
1295 output += "[ERROR] Config file not found"
1296 return output
1297
1298
1299
1300def process_cgi_request(environ=os.environ):
1301 """ When calling from CGI """
1302 response_headers = []
1303 content_type = 'text/plain'
1304
1305 # Update repository if requested
1306 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1307 if form and form.has_key("action") and "update" in form["action"]:
1308 output = "[INFO] Updating subverion, please wait...\n"
1309 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1310 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1311 output += "[INFO] All done, redirecting in 5 seconds"
1312 response_headers += [
1313 ('Refresh', '5; url=.'),
1314 ]
1315 reload_cache()
1316 else:
1317 base_uri = environ['PATH_INFO']
1318 uri = base_uri.strip('/').split('/')
1319
1320 output = "Template Holder"
1321 if base_uri.endswith('/create/network.kml'):
1322 content_type='application/vnd.google-earth.kml+xml'
1323 output = make_network_kml.make_graph()
1324 elif base_uri.endswith('/api/get/nodeplanner.json'):
1325 content_type='application/json'
1326 output = make_network_kml.make_nodeplanner_json()
1327 elif not uri[0]:
1328 if is_text_request(environ):
1329 output = '\n'.join(get_hostlist())
1330 else:
1331 content_type = 'text/html'
1332 output = generate_title(get_hostlist())
1333 elif len(uri) == 1:
1334 if is_text_request(environ):
1335 output = generate_node(uri[0])
1336 else:
1337 content_type = 'text/html'
1338 output = generate_node_overview(uri[0])
1339 elif len(uri) == 2:
1340 output = generate_config(uri[0], uri[1])
1341 else:
1342 assert False, "Invalid option"
1343
1344 # Return response
1345 response_headers += [
1346 ('Content-type', content_type),
1347 ('Content-Length', str(len(output))),
1348 ]
1349 return(response_headers, str(output))
1350
1351
1352def make_dns(output_dir = 'dns', external = False):
1353 items = dict()
1354
1355 # hostname is key, IP is value
1356 wleiden_zone = defaultdict(list)
1357 wleiden_cname = dict()
1358
1359 pool = dict()
1360 for node in get_hostlist():
1361 datadump = get_yaml(node)
1362
1363 # Proxy naming convention is special
1364 fqdn = datadump['nodename']
1365 if datadump['nodetype'] in ['Hybrid']:
1366 wleiden_cname[datadump['nodename']] = fqdn
1367
1368 if datadump.has_key('rdr_host'):
1369 remote_target = datadump['rdr_host']
1370 elif datadump.has_key('remote_access') and datadump['remote_access']:
1371 remote_target = datadump['remote_access'].split(':')[0]
1372 else:
1373 remote_target = None
1374
1375 if remote_target:
1376 try:
1377 parseaddr(remote_target)
1378 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1379 except (IndexError, ValueError):
1380 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1381
1382
1383 wleiden_zone[fqdn].append((datadump['masterip'], True))
1384
1385 # Hacking to get proper DHCP IPs and hostnames
1386 for iface_key in get_interface_keys(datadump):
1387 iface_name = iface_key.replace('_','-')
1388 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1389 try:
1390 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1391 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1392 dhcp_part = ".".join(ip.split('.')[0:3])
1393 if ip != datadump['masterip']:
1394 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, False))
1395 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1396 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1397 except (AttributeError, ValueError, KeyError):
1398 # First push it into a pool, to indentify the counter-part later on
1399 addr = parseaddr(ip)
1400 cidr = int(cidr)
1401 addr = addr & ~((1 << (32 - cidr)) - 1)
1402 if pool.has_key(addr):
1403 pool[addr] += [(iface_name, fqdn, ip)]
1404 else:
1405 pool[addr] = [(iface_name, fqdn, ip)]
1406 continue
1407
1408
1409
1410 # WL uses an /29 to configure an interface. IP's are ordered like this:
1411 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1412
1413 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1414
1415 # Automatic naming convention of interlinks namely 2 + remote.lower()
1416 for (key,value) in pool.iteritems():
1417 # Make sure they are sorted from low-ip to high-ip
1418 value = sorted(value, key=lambda x: parseaddr(x[2]))
1419
1420 if len(value) == 1:
1421 (iface_name, fqdn, ip) = value[0]
1422 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1423
1424 # Device DNS names
1425 if 'cnode' in fqdn.lower():
1426 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1427 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1428
1429 elif len(value) == 2:
1430 (a_iface_name, a_fqdn, a_ip) = value[0]
1431 (b_iface_name, b_fqdn, b_ip) = value[1]
1432 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1433 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1434
1435 # Device DNS names
1436 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1437 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1438 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1439 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1440 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1441 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1442 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1443
1444 else:
1445 pool_members = [k[1] for k in value]
1446 for item in value:
1447 (iface_name, fqdn, ip) = item
1448 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1449
1450 # Include static DNS entries
1451 # XXX: Should they override the autogenerated results?
1452 # XXX: Convert input to yaml more useable.
1453 # Format:
1454 ##; this is a comment
1455 ## roomburgh=Roomburgh1
1456 ## apkerk1.Vosko=172.17.176.8 ;this as well
1457 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1458
1459 # Hack to allow special entries, for development
1460 wleiden_raw = {}
1461
1462 for line in dns_list:
1463 reverse = False
1464 k, items = line.items()[0]
1465 if type(items) == dict:
1466 if items.has_key('reverse'):
1467 reverse = items['reverse']
1468 items = items['a']
1469 else:
1470 items = items['cname']
1471 items = [items] if type(items) != list else items
1472 for item in items:
1473 if item.startswith('IN '):
1474 wleiden_raw[k] = item
1475 elif valid_addr(item):
1476 wleiden_zone[k].append((item, reverse))
1477 else:
1478 wleiden_cname[k] = item
1479
1480 # Hack to get dynamic pool listing
1481 def chunks(l, n):
1482 return [l[i:i+n] for i in range(0, len(l), n)]
1483
1484 ntp_servers = [x[0] for x in get_nameservers()]
1485 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1486 for ntp_server in chunk:
1487 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1488
1489 details = dict()
1490 # 24 updates a day allowed
1491 details['serial'] = time.strftime('%Y%m%d%H')
1492
1493 if external:
1494 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1495 else:
1496 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1497
1498 details['master'] = dns_masters[0]
1499 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1500
1501 dns_header = '''
1502$TTL 3h
1503%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1504 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1505
1506%(ns_servers)s
1507 \n'''
1508
1509
1510 if not os.path.isdir(output_dir):
1511 os.makedirs(output_dir)
1512 details['zone'] = 'wleiden.net'
1513 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1514 f.write(dns_header % details)
1515
1516 for host,items in wleiden_zone.iteritems():
1517 for ip,reverse in items:
1518 if ip not in ['0.0.0.0']:
1519 f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
1520 for source,dest in wleiden_cname.iteritems():
1521 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1522 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1523 for source, dest in wleiden_raw.iteritems():
1524 f.write("%s.wleiden.net. %s\n" % (source, dest))
1525 f.close()
1526
1527 # Create whole bunch of specific sub arpa zones. To keep it compliant
1528 for s in range(16,32):
1529 details['zone'] = '%i.172.in-addr.arpa' % s
1530 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1531 f.write(dns_header % details)
1532
1533 #XXX: Not effient, fix to proper data structure and do checks at other
1534 # stages
1535 for host,items in wleiden_zone.iteritems():
1536 for ip,reverse in items:
1537 if not reverse:
1538 continue
1539 if valid_addr(ip):
1540 if valid_addr(ip):
1541 if int(ip.split('.')[1]) == s:
1542 rev_ip = '.'.join(reversed(ip.split('.')))
1543 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1544 f.close()
1545
1546
1547def usage():
1548 print """Usage: %(prog)s <argument>
1549Argument:
1550\tcleanup = Cleanup all YAML files to specified format
1551\tstandalone [port] = Run configurator webserver [8000]
1552\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1553\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1554\tfull-export = Generate yaml export script for heatmap.
1555\tstatic [outputdir] = Generate all config files and store on disk
1556\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1557\ttest <node> [<file>] = Receive output for certain node [all files].
1558\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1559\tlist <status> <items> = List systems which have certain status
1560
1561Arguments:
1562\t<node> = NodeName (example: HybridRick)
1563\t<file> = %(files)s
1564\t<status> = all|up|down|planned
1565\t<items> = systems|nodes|proxies
1566
1567NOTE FOR DEVELOPERS; you can test your changes like this:
1568 BEFORE any changes in this code:
1569 $ ./gformat.py static /tmp/pre
1570 AFTER the changes:
1571 $ ./gformat.py static /tmp/post
1572 VIEW differences and VERIFY all are OK:
1573 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1574""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1575 exit(0)
1576
1577
1578def is_text_request(environ=os.environ):
1579 """ Find out whether we are calling from the CLI or any text based CLI utility """
1580 try:
1581 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1582 except KeyError:
1583 return True
1584
1585def switchFormat(setting):
1586 if setting:
1587 return "YES"
1588 else:
1589 return "NO"
1590
1591def rlinput(prompt, prefill=''):
1592 import readline
1593 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1594 try:
1595 return raw_input(prompt)
1596 finally:
1597 readline.set_startup_hook()
1598
1599def fix_conflict(left, right, default='i'):
1600 while True:
1601 print "## %-30s | %-30s" % (left, right)
1602 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1603 if not c:
1604 c = default
1605
1606 if c in ['l','1']:
1607 return left
1608 elif c in ['r','2']:
1609 return right
1610 elif c in ['e', '3']:
1611 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1612 elif c in ['i', '4']:
1613 return None
1614 else:
1615 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1616
1617
1618
1619def print_cgi_response(response_headers, output):
1620 """Could we not use some kind of wsgi wrapper to make this output?"""
1621 for header in response_headers:
1622 print "%s: %s" % header
1623 print
1624 print output
1625
1626
1627def fill_cache():
1628 ''' Poor man re-loading of few cache items (the slow ones) '''
1629 for host in get_hostlist():
1630 get_yaml(host)
1631
1632
1633def reload_cache():
1634 clear_cache()
1635 fill_cache()
1636
1637
1638def main():
1639 """Hard working sub"""
1640 # Allow easy hacking using the CLI
1641 if not os.environ.has_key('PATH_INFO'):
1642 if len(sys.argv) < 2:
1643 usage()
1644
1645 if sys.argv[1] == "standalone":
1646 import SocketServer
1647 import CGIHTTPServer
1648 # Hop to the right working directory.
1649 os.chdir(os.path.dirname(__file__))
1650 try:
1651 PORT = int(sys.argv[2])
1652 except (IndexError,ValueError):
1653 PORT = 8000
1654
1655 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1656 """ Serve this CGI from the root of the webserver """
1657 def is_cgi(self):
1658 if "favicon" in self.path:
1659 return False
1660
1661 self.cgi_info = (os.path.basename(__file__), self.path)
1662 self.path = ''
1663 return True
1664 handler = MyCGIHTTPRequestHandler
1665 SocketServer.TCPServer.allow_reuse_address = True
1666 httpd = SocketServer.TCPServer(("", PORT), handler)
1667 httpd.server_name = 'localhost'
1668 httpd.server_port = PORT
1669
1670 logger.info("serving at port %s", PORT)
1671 try:
1672 httpd.serve_forever()
1673 except KeyboardInterrupt:
1674 httpd.shutdown()
1675 logger.info("All done goodbye")
1676 elif sys.argv[1] == "test":
1677 # Basic argument validation
1678 try:
1679 node = sys.argv[2]
1680 except IndexError:
1681 print "Invalid argument"
1682 exit(1)
1683 except IOError as e:
1684 print e
1685 exit(1)
1686
1687 datadump = get_yaml(node)
1688
1689
1690 # Get files to generate
1691 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1692
1693 # Actual config generation
1694 for config in gen_files:
1695 logger.info("## Generating %s %s", node, config)
1696 print generate_config(node, config, datadump)
1697 elif sys.argv[1] == "test-cgi":
1698 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1699 os.environ['SCRIPT_NAME'] = __file__
1700 response_headers, output = process_cgi_request()
1701 print_cgi_response(response_headers, output)
1702 elif sys.argv[1] == "static":
1703 items = dict()
1704 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1705 for node in get_hostlist():
1706 items['node'] = node
1707 items['wdir'] = "%(output_dir)s/%(node)s" % items
1708 if not os.path.isdir(items['wdir']):
1709 os.makedirs(items['wdir'])
1710 datadump = get_yaml(node)
1711 for config in files:
1712 items['config'] = config
1713 logger.info("## Generating %(node)s %(config)s" % items)
1714 f = open("%(wdir)s/%(config)s" % items, "w")
1715 f.write(generate_config(node, config, datadump))
1716 f.close()
1717 elif sys.argv[1] == "wind-export":
1718 items = dict()
1719 for node in get_hostlist():
1720 datadump = get_yaml(node)
1721 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1722 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1723 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1724 VALUES (
1725 (SELECT id FROM users WHERE username = 'rvdzwet'),
1726 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1727 'Y');""" % datadump
1728 #for config in files:
1729 # items['config'] = config
1730 # print "## Generating %(node)s %(config)s" % items
1731 # f = open("%(wdir)s/%(config)s" % items, "w")
1732 # f.write(generate_config(node, config, datadump))
1733 # f.close()
1734 for node in get_hostlist():
1735 datadump = get_yaml(node)
1736 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1737 ifacedump = datadump[iface_key]
1738 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1739 ifacedump['nodename'] = datadump['nodename']
1740 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1741 ifacedump['channel'] = 0
1742 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1743 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1744 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1745 elif sys.argv[1] == "nagios-export":
1746 try:
1747 heavy_load = (sys.argv[2] == "--heavy-load")
1748 except IndexError:
1749 heavy_load = False
1750
1751 hostgroup_details = {
1752 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1753 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1754 'walphen' : 'Stichting Wireless Alphen',
1755 'westeinder' : 'Westeinder Plassen',
1756 }
1757
1758 # Convert IP to Host
1759 ip2host = {'root' : 'root'}
1760 for host in get_hostlist():
1761 datadump = get_yaml(host)
1762 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1763 for iface in get_interface_keys(datadump):
1764 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1765
1766 # Find dependency tree based on output of lvrouted.mytree of nearest node
1767 parents = defaultdict(list)
1768 stack = ['root']
1769 prev_depth = 0
1770 for line in open('lvrouted.mytree').readlines():
1771 depth = line.count('\t')
1772 ip = line.strip().split()[0]
1773
1774 if prev_depth < depth:
1775 try:
1776 parents[ip2host[ip]].append(ip2host[stack[-1]])
1777 except KeyError as e:
1778 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1779 stack.append(ip)
1780 elif prev_depth > depth:
1781 stack = stack[:(depth - prev_depth)]
1782 elif prev_depth == depth:
1783 try:
1784 parents[ip2host[ip]].append(ip2host[stack[-1]])
1785 except KeyError as e:
1786 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1787
1788
1789 prev_depth = depth
1790 # Observe that some nodes has themself as parent or multiple parents
1791 # for now take only the first parent, other behaviour is yet to be explained
1792
1793
1794
1795 params = {
1796 'check_interval' : 5 if heavy_load else 120,
1797 'retry_interval' : 1 if heavy_load else 10,
1798 'max_check_attempts' : 10 if heavy_load else 6,
1799 'notification_interval': 120 if heavy_load else 240,
1800 }
1801
1802 print '''\
1803define host {
1804 name wleiden-node ; Default Node Template
1805 use generic-host ; Use the standard template as initial starting point
1806 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1807 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1808 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1809 notification_interval %(notification_interval)s
1810 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1811 check_command check-host-alive ; Default command to check FreeBSD hosts
1812 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1813}
1814
1815define service {
1816 name wleiden-service ; Default Service Template
1817 use generic-service ; Use the standard template as initial starting point
1818 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1819 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1820 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1821 notification_interval %(notification_interval)s
1822 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1823 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1824}
1825
1826# Please make sure to install:
1827# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1828#
1829# Recompile net-mgmt/nagios-plugins to support check_snmp
1830# make -C /usr/ports/net-mgmt/nagios-plugins
1831#
1832# Install net/bind-tools to allow v2/check_dns_wl to work:
1833# pkg install bind-tools
1834#
1835define command{
1836 command_name check_snmp_disk
1837 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1838}
1839
1840define command{
1841 command_name check_netsnmp_load
1842 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1843}
1844
1845define command{
1846 command_name check_netsnmp_proc
1847 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1848}
1849
1850define command{
1851 command_name check_by_ssh
1852 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1853}
1854
1855define command{
1856 command_name check_dns_wl
1857 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1858}
1859
1860define command{
1861 command_name check_snmp_uptime
1862 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1863}
1864
1865
1866# TDB: dhcp leases
1867# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1868
1869# TDB: internet status
1870# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1871
1872# TDB: Advanced local passive checks
1873# /usr/local/libexec/nagios/check_by_ssh
1874''' % params
1875
1876 print '''\
1877# Service Group, not displayed by default
1878define hostgroup {
1879 hostgroup_name srv_hybrid
1880 alias All Hybrid Nodes
1881 register 0
1882}
1883
1884define service {
1885 use wleiden-service
1886 hostgroup_name srv_hybrid
1887 service_description SSH
1888 check_command check_ssh
1889}
1890
1891define service {
1892 use wleiden-service,service-pnp
1893 hostgroup_name srv_hybrid
1894 service_description HTTP
1895 check_command check_http
1896}
1897
1898define service {
1899 use wleiden-service
1900 hostgroup_name srv_hybrid
1901 service_description DNS
1902 check_command check_dns_wl!"www.wirelessleiden.nl"
1903}
1904'''
1905
1906 if heavy_load:
1907 print '''\
1908define service {
1909 use wleiden-service
1910 hostgroup_name srv_hybrid
1911 service_description UPTIME
1912 check_command check_snmp_uptime
1913}
1914
1915#define service {
1916# use wleiden-service
1917# hostgroup_name srv_hybrid
1918# service_description NTP
1919# check_command check_ntp_peer
1920#}
1921
1922define service {
1923 use wleiden-service
1924 hostgroup_name srv_hybrid
1925 service_description LOAD
1926 check_command check_netsnmp_load
1927}
1928
1929define service {
1930 use wleiden-service
1931 hostgroup_name srv_hybrid
1932 service_description PROC
1933 check_command check_netsnmp_proc
1934}
1935
1936define service {
1937 use wleiden-service
1938 hostgroup_name srv_hybrid
1939 service_description DISK
1940 check_command check_snmp_disk
1941}
1942'''
1943 for node in get_hostlist():
1944 datadump = get_yaml(node)
1945 if not datadump['status'] == 'up':
1946 continue
1947 if not hostgroup_details.has_key(datadump['monitoring_group']):
1948 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
1949 print '''\
1950define host {
1951 use wleiden-node,host-pnp
1952 contact_groups admins
1953 host_name %(autogen_fqdn)s
1954 address %(masterip)s
1955 hostgroups srv_hybrid,%(monitoring_group)s\
1956''' % datadump
1957 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
1958 print '''\
1959 parents %(parents)s\
1960''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
1961 print '''\
1962}
1963'''
1964
1965
1966 for name,alias in hostgroup_details.iteritems():
1967 print '''\
1968define hostgroup {
1969 hostgroup_name %s
1970 alias %s
1971} ''' % (name, alias)
1972
1973
1974 elif sys.argv[1] == "full-export":
1975 hosts = {}
1976 for node in get_hostlist():
1977 datadump = get_yaml(node)
1978 hosts[datadump['nodename']] = datadump
1979 print yaml.dump(hosts)
1980
1981 elif sys.argv[1] == "dns":
1982 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
1983 elif sys.argv[1] == "cleanup":
1984 # First generate all datadumps
1985 datadumps = dict()
1986 ssid_to_node = dict()
1987 for host in get_hostlist():
1988 logger.info("# Processing: %s", host)
1989 # Set some boring default values
1990 datadump = { 'board' : 'UNKNOWN' }
1991 datadump.update(get_yaml(host))
1992 datadumps[datadump['nodename']] = datadump
1993
1994 (poel, errors) = make_relations()
1995 print "\n".join(["# WARNING: %s" % x for x in errors])
1996
1997 for host,datadump in datadumps.iteritems():
1998 try:
1999 # Convert all yes and no to boolean values
2000 def fix_boolean(dump):
2001 for key in dump.keys():
2002 if type(dump[key]) == dict:
2003 dump[key] = fix_boolean(dump[key])
2004 elif str(dump[key]).lower() in ["yes", "true"]:
2005 dump[key] = True
2006 elif str(dump[key]).lower() in ["no", "false"]:
2007 # Compass richting no (Noord Oost) is valid input
2008 if key != "compass": dump[key] = False
2009 return dump
2010 datadump = fix_boolean(datadump)
2011
2012 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2013 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2014 elif 'latitude' in datadump and 'longitude' in datadump:
2015 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2016
2017 if datadump['nodename'].startswith('Proxy'):
2018 datadump['nodename'] = datadump['nodename'].lower()
2019
2020 for iface_key in get_interface_keys(datadump):
2021 try:
2022 # All our normal wireless cards are normal APs now
2023 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2024 datadump[iface_key]['mode'] = 'ap'
2025 # Wireless Leiden SSID have an consistent lowercase/uppercase
2026 if datadump[iface_key].has_key('ssid'):
2027 ssid = datadump[iface_key]['ssid']
2028 prefix = 'ap-WirelessLeiden-'
2029 if ssid.lower().startswith(prefix.lower()):
2030 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2031 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2032 datadump[iface_key]['mode'] = 'autogen-FIXME'
2033 if not datadump[iface_key].has_key('comment'):
2034 datadump[iface_key]['comment'] = 'autogen-FIXME'
2035
2036 if datadump[iface_key].has_key('ns_mac'):
2037 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2038
2039 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2040 datadump[iface_key] = datadump[iface_key]['desc']
2041
2042 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2043 # due to better collision avoidance.
2044 if datadump[iface_key]['type'] == '11b':
2045 datadump[iface_key]['type'] = '11g'
2046
2047 # Setting 802.11g channels to de-facto standards, to avoid
2048 # un-detected sharing with other overlapping channels
2049 #
2050 # Technically we could also use channel 13 in NL, but this is not
2051 # recommended as foreign devices might not be able to select this
2052 # channel. Secondly using 1,5,9,13 instead is going to clash with
2053 # the de-facto usage of 1,6,11.
2054 #
2055 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2056 channels_at_2400Mhz = (1,6,11)
2057 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2058 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2059 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2060 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2061
2062 # Mandatory interface keys
2063 if not datadump[iface_key].has_key('status'):
2064 datadump[iface_key]['status'] = 'planned'
2065
2066 x = datadump[iface_key]['comment']
2067 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2068
2069 # Fixing bridge_type if none is found
2070 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2071 if not 'bridge_type' in datadump[iface_key]:
2072 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2073
2074 # Making sure description works
2075 if datadump[iface_key].has_key('desc'):
2076 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2077 del datadump[iface_key]['desc']
2078 else:
2079 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2080 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2081 if response:
2082 datadump[iface_key]['comment'] = response
2083 del datadump[iface_key]['desc']
2084
2085 # Check DHCP configuration
2086 dhcp_type(datadump[iface_key])
2087
2088 # Set the compass value based on the angle between the poels
2089 if datadump[iface_key].has_key('ns_ip'):
2090 my_pool = poel[network(datadump[iface_key]['ip'])]
2091 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2092 if remote_hosts:
2093 compass_target = remote_hosts[0]
2094 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2095
2096 # Monitoring Group default
2097 if not 'monitoring_group' in datadump:
2098 datadump['monitoring_group'] = 'wleiden'
2099
2100 except Exception:
2101 print "# Error while processing interface %s" % iface_key
2102 raise
2103 store_yaml(datadump)
2104 except Exception:
2105 print "# Error while processing %s" % host
2106 raise
2107 elif sys.argv[1] == "list":
2108 use_fqdn = False
2109 if len(sys.argv) < 4:
2110 usage()
2111 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2112 usage()
2113 if not sys.argv[3] in ["nodes","proxies","systems"]:
2114 usage()
2115
2116 if len(sys.argv) > 4:
2117 if sys.argv[4] == "fqdn":
2118 use_fqdn = True
2119 else:
2120 usage()
2121
2122 for system in get_hostlist():
2123 datadump = get_yaml(system)
2124 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2125 continue
2126
2127 output = datadump['autogen_fqdn'] if use_fqdn else system
2128 if sys.argv[2] == "all":
2129 print output
2130 elif datadump['status'] == sys.argv[2]:
2131 print output
2132 elif sys.argv[1] == "create":
2133 if sys.argv[2] == "network.kml":
2134 print make_network_kml.make_graph()
2135 elif sys.argv[2] == "host-ips.txt":
2136 for system in get_hostlist():
2137 datadump = get_yaml(system)
2138 ips = [datadump['masterip']]
2139 for ifkey in get_interface_keys(datadump):
2140 ips.append(datadump[ifkey]['ip'].split('/')[0])
2141 print system, ' '.join(ips)
2142 elif sys.argv[2] == "host-pos.txt":
2143 for system in get_hostlist():
2144 datadump = get_yaml(system)
2145 print system, datadump['rdnap_x'], datadump['rdnap_y']
2146 elif sys.argv[2] == 'ssh_config':
2147 print '''
2148Host *.wleiden.net
2149 User root
2150
2151Host 172.16.*.*
2152 User root
2153'''
2154 for system in get_hostlist():
2155 datadump = get_yaml(system)
2156 print '''\
2157Host %s
2158 User root
2159
2160Host %s
2161 User root
2162
2163Host %s
2164 User root
2165
2166Host %s
2167 User root
2168''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2169 else:
2170 usage()
2171 else:
2172 usage()
2173 else:
2174 # Do not enable debugging for config requests as it highly clutters the output
2175 if not is_text_request():
2176 cgitb.enable()
2177 response_headers, output = process_cgi_request()
2178 print_cgi_response(response_headers, output)
2179
2180def application(environ, start_response):
2181 status = '200 OK'
2182 response_headers, output = process_cgi_request(environ)
2183 start_response(status, response_headers)
2184
2185 # Debugging only
2186 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2187 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2188 return [output]
2189
2190if __name__ == "__main__":
2191 main()
Note: See TracBrowser for help on using the repository browser.