source: genesis/tools/gformat.py@ 13418

Last change on this file since 13418 was 13418, checked in by rick, 9 years ago

Weed out CNode legacy code
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 71.2 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13418 2015-10-03 21:20:32Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = []
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126def clear_cache():
127 ''' Poor mans cache implementation '''
128 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
129 datadump_cache = {}
130 interface_list_cache = {}
131 rc_conf_local_cache = {}
132 ileiden_proxies = []
133 normal_proxies = []
134 nameservers_cache = []
135 relations_cache = None
136
137NO_DHCP = 0
138DHCP_CLIENT = 10
139DHCP_SERVER = 20
140def dhcp_type(item):
141 if not item.has_key('dhcp'):
142 return NO_DHCP
143 elif not item['dhcp']:
144 return NO_DHCP
145 elif item['dhcp'].lower() == 'client':
146 return DHCP_CLIENT
147 else:
148 # Validation Checks
149 begin,end = map(int,item['dhcp'].split('-'))
150 if begin >= end:
151 raise ValueError("DHCP Start >= DHCP End")
152 return DHCP_SERVER
153
154def etrs2rd(lat, lon):
155 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
156 p2 = pyproj.Proj(init='EPSG:28992')
157 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
158 return (RDx, RDy)
159
160def rd2etrs(RDx, RDy):
161 p1 = pyproj.Proj(init='EPSG:28992')
162 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
163 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
164 return (lat, lon)
165
166def get_yaml(item,add_version_info=True):
167 try:
168 """ Get configuration yaml for 'item'"""
169 if datadump_cache.has_key(item):
170 return datadump_cache[item].copy()
171
172 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
173
174 # Default values
175 datadump = {
176 'autogen_revision' : 'NOTFOUND',
177 'autogen_gfile' : gfile,
178 'service_proxy_ileiden' : False,
179 }
180 f = open(gfile, 'r')
181 datadump.update(yaml.load(f,Loader=Loader))
182 if datadump['nodetype'] == 'Hybrid':
183 # Some values are defined implicitly
184 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
185 datadump['service_incoming_rdr'] = True
186 # Use some boring defaults
187 defaults = {
188 'service_proxy_normal' : False,
189 'service_accesspoint' : True,
190 'service_incoming_rdr' : False,
191 'service_concentrator' : False,
192 'monitoring_group' : 'wleiden',
193 }
194 for (key,value) in defaults.iteritems():
195 if not datadump.has_key(key):
196 datadump[key] = value
197 f.close()
198
199 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
200 if add_version_info:
201 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
202 lines = p.communicate()[0].split('\n')
203 if p.returncode == 0:
204 for line in lines:
205 if line:
206 (key, value) = line.split(': ')
207 datadump["autogen_" + key.lower().replace(' ','_')] = value
208
209 # Preformat certain needed variables for formatting and push those into special object
210 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
211
212 wlan_count=0
213 try:
214 for key in get_interface_keys(datadump, True):
215 datadump[key]['autogen_ifbase'] = key.split('_')[1]
216 datadump[key]['autogen_vlan'] = False
217
218 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
219 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
220 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
221 wlan_count += 1
222 else:
223 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
224 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
225 datadump[key]['autogen_vlan'] = key.split('_')[2]
226
227 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
228 if datadump[key]['autogen_bridge'] and not 'alias' in key:
229 datadump[key]['autogen_bridge_interfaces'] = datadump[key]['members'].split()
230 except Exception:
231 print "# Error while processing interface %s" % key
232 raise
233
234 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
235 if dhcp_type(datadump[key]) == DHCP_SERVER]
236
237 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
238 datadump['autogen_item'] = item
239
240 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
241 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
242 datadump_cache[item] = datadump.copy()
243 except Exception:
244 print "# Error while processing %s" % item
245 raise
246 return datadump
247
248
249def store_yaml(datadump, header=False):
250 """ Store configuration yaml for 'item'"""
251 item = datadump['autogen_item']
252 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
253
254 output = generate_wleiden_yaml(datadump, header)
255
256 f = open(gfile, 'w')
257 f.write(output)
258 f.close()
259
260
261def network(ip):
262 addr, mask = ip.split('/')
263 # Not parsing of these folks please
264 addr = parseaddr(addr)
265 mask = int(mask)
266 network = addr & ~((1 << (32 - mask)) - 1)
267 return network
268
269
270
271def make_relations():
272 """ Process _ALL_ yaml files to get connection relations """
273 global relations_cache
274
275 if relations_cache:
276 return relations_cache
277
278 errors = []
279 poel = defaultdict(list)
280
281 for host in get_hostlist():
282 datadump = get_yaml(host)
283 try:
284 for iface_key in get_interface_keys(datadump):
285 net_addr = network(datadump[iface_key]['ip'])
286 poel[net_addr] += [(host,datadump[iface_key].copy())]
287 except (KeyError, ValueError), e:
288 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
289 continue
290
291 relations_cache = (poel, errors)
292 return relations_cache
293
294
295
296def valid_addr(addr):
297 """ Show which address is valid in which are not """
298 return str(addr).startswith('172.')
299
300def get_hostlist():
301 """ Combined hosts and proxy list"""
302 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
303
304def angle_between_points(lat1,lat2,long1,long2):
305 """
306 Return Angle in radians between two GPS coordinates
307 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
308 """
309 dy = lat2 - lat1
310 dx = math.cos(lat1)*(long2 - long1)
311 angle = math.atan2(dy,dx)
312 return angle
313
314
315
316def angle_to_cd(angle):
317 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
318
319 # For easy conversion get positive degree
320 degrees = math.degrees(angle)
321 abs_degrees = 360 + degrees if degrees < 0 else degrees
322
323 # Numbers can be confusing calculate from the 4 main directions
324 p = 22.5
325 if abs_degrees < p:
326 cd = "n"
327 elif abs_degrees < (90 - p):
328 cd = "no"
329 elif abs_degrees < (90 + p):
330 cd = "o"
331 elif abs_degrees < (180 - p):
332 cd = "zo"
333 elif abs_degrees < (180 + p):
334 cd = "z"
335 elif abs_degrees < (270 - p):
336 cd = "zw"
337 elif abs_degrees < (270 + p):
338 cd = "w"
339 elif abs_degrees < (360 - p):
340 cd = "nw"
341 else:
342 cd = "n"
343 return cd
344
345
346
347def cd_between_hosts(hostA, hostB, datadumps):
348 # Using RDNAP coordinates
349 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
350 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
351 return angle_to_cd(math.atan2(dx,dy))
352
353 # GPS coordinates seems to fail somehow
354 #latA = float(datadumps[hostA]['latitude'])
355 #latB = float(datadumps[hostB]['latitude'])
356 #lonA = float(datadumps[hostA]['longitude'])
357 #lonB = float(datadumps[hostB]['longitude'])
358 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
359
360
361def generate_title(nodelist):
362 """ Main overview page """
363 items = {'root' : "." }
364 def fl(spaces, line):
365 return (' ' * spaces) + line + '\n'
366
367 output = """
368<html>
369 <head>
370 <title>Wireless leiden Configurator - GFormat</title>
371 <style type="text/css">
372 th {background-color: #999999}
373 tr:nth-child(odd) {background-color: #cccccc}
374 tr:nth-child(even) {background-color: #ffffff}
375 th, td {padding: 0.1em 1em}
376 </style>
377 </head>
378 <body>
379 <center>
380 <form type="GET" action="%(root)s">
381 <input type="hidden" name="action" value="update">
382 <input type="submit" value="Update Configuration Database (SVN)">
383 </form>
384 <table>
385 <caption><h3>Wireless Leiden Configurator</h3></caption>
386 """ % items
387
388 for node in nodelist:
389 items['node'] = node
390 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
391 for config in files:
392 items['config'] = config
393 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
394 output += fl(5, "</tr>")
395 output += """
396 </table>
397 <hr />
398 <em>%s</em>
399 </center>
400 </body>
401</html>
402 """ % __version__
403
404 return output
405
406
407
408def generate_node(node):
409 """ Print overview of all files available for node """
410 return "\n".join(files)
411
412def generate_node_overview(host):
413 """ Print overview of all files available for node """
414 datadump = get_yaml(host)
415 params = { 'host' : host }
416 output = "<em><a href='..'>Back to overview</a></em><hr />"
417 output += "<h2>Available files:</h2><ul>"
418 for cf in files:
419 params['cf'] = cf
420 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
421 output += "</ul>"
422
423 # Generate and connection listing
424 output += "<h2>Connected To:</h2><ul>"
425 (poel, errors) = make_relations()
426 for network, hosts in poel.iteritems():
427 if host in [x[0] for x in hosts]:
428 if len(hosts) == 1:
429 # Single not connected interface
430 continue
431 for remote,ifacedump in hosts:
432 if remote == host:
433 # This side of the interface
434 continue
435 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
436 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
437 output += "</ul>"
438 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
439
440 output += "<hr /><em><a href='..'>Back to overview</a></em>"
441 return output
442
443
444def generate_header(datadump, ctag="#"):
445 return """\
446%(ctag)s
447%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
448%(ctag)s
449""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
450
451
452
453def parseaddr(s):
454 """ Process IPv4 CIDR notation addr to a (binary) number """
455 f = s.split('.')
456 return (long(f[0]) << 24L) + \
457 (long(f[1]) << 16L) + \
458 (long(f[2]) << 8L) + \
459 long(f[3])
460
461
462
463def showaddr(a):
464 """ Display IPv4 addr in (dotted) CIDR notation """
465 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
466
467
468def is_member(ip, mask, canidate):
469 """ Return True if canidate is part of ip/mask block"""
470 ip_addr = parseaddr(ip)
471 ip_canidate = parseaddr(canidate)
472 mask = int(mask)
473 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
474 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
475 return ip_addr == ip_canidate
476
477
478
479def cidr2netmask(netmask):
480 """ Given a 'netmask' return corresponding CIDR """
481 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
482
483def get_network(addr, mask):
484 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
485
486
487def generate_dhcpd_conf(datadump):
488 """ Generate config file '/usr/local/etc/dhcpd.conf """
489 output = generate_header(datadump)
490 output += Template("""\
491# option definitions common to all supported networks...
492option domain-name "dhcp.{{ autogen_fqdn }}";
493
494default-lease-time 600;
495max-lease-time 7200;
496
497# Use this to enble / disable dynamic dns updates globally.
498#ddns-update-style none;
499
500# If this DHCP server is the official DHCP server for the local
501# network, the authoritative directive should be uncommented.
502authoritative;
503
504# Use this to send dhcp log messages to a different log file (you also
505# have to hack syslog.conf to complete the redirection).
506log-facility local7;
507
508#
509# Interface definitions
510#
511\n""").render(datadump)
512
513 dhcp_out = defaultdict(list)
514 for iface_key in get_interface_keys(datadump):
515 ifname = datadump[iface_key]['autogen_ifname']
516 if not datadump[iface_key].has_key('comment'):
517 datadump[iface_key]['comment'] = None
518 dhcp_out[ifname].append(" ## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key])
519
520 (addr, mask) = datadump[iface_key]['ip'].split('/')
521 datadump[iface_key]['autogen_addr'] = addr
522 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
523 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
524 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
525 dhcp_out[ifname].append(" subnet %(autogen_subnet)s netmask %(autogen_netmask)s {\n ### not autoritive\n }\n" % \
526 datadump[iface_key])
527 continue
528
529 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
530 dhcp_part = ".".join(addr.split('.')[0:3])
531 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
532 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
533
534 # Assume the first 10 IPs could be used for static entries
535 if 'no_portal' in datadump:
536 fixed = 5
537 for mac in datadump['no_portal']:
538 dhcp_out[ifname].append("""\
539 host fixed-%(ifname)s-%(fixed)s {
540 hardware ethernet %(mac)s;
541 fixed-address %(prefix)s.%(fixed)s;
542 }
543""" % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed })
544 fixed += 1
545
546 dhcp_out[ifname].append("""\
547 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
548 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
549 option routers %(autogen_addr)s;
550 option domain-name-servers %(autogen_addr)s;
551 }
552""" % datadump[iface_key])
553
554 for ifname,value in dhcp_out.iteritems():
555 output += ("shared-network %s {\n" % ifname) + ''.join(value) + '}\n\n'
556 return output
557
558
559
560def generate_dnsmasq_conf(datadump):
561 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
562 output = generate_header(datadump)
563 output += Template("""\
564# DHCP server options
565dhcp-authoritative
566dhcp-fqdn
567domain=dhcp.{{ autogen_fqdn }}
568domain-needed
569expand-hosts
570log-async=100
571
572# Low memory footprint
573cache-size=10000
574
575\n""").render(datadump)
576
577 for iface_key in get_interface_keys(datadump):
578 if not datadump[iface_key].has_key('comment'):
579 datadump[iface_key]['comment'] = None
580 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
581
582 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
583 output += "# not autoritive\n\n"
584 continue
585
586 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
587 (ip, cidr) = datadump[iface_key]['ip'].split('/')
588 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
589
590 dhcp_part = ".".join(ip.split('.')[0:3])
591 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
592 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
593 output += "dhcp-range=%(autogen_ifname)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
594
595 return output
596
597
598def make_interface_list(datadump):
599 if interface_list_cache.has_key(datadump['autogen_item']):
600 return (interface_list_cache[datadump['autogen_item']])
601 # lo0 configuration:
602 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
603 # - masterip is special as it needs to be assigned to at
604 # least one interface, so if not used assign to lo0
605 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
606 vlan_list = defaultdict(list)
607 dhclient_if = {'lo0' : False}
608
609 # XXX: Find some way of send this output nicely
610 output = ''
611
612 masterip_used = False
613 for iface_key in get_interface_keys(datadump):
614 if datadump[iface_key]['ip'].startswith(datadump['masterip']):
615 masterip_used = True
616 break
617 if not masterip_used:
618 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
619
620 for iface_key in get_interface_keys(datadump):
621 ifacedump = datadump[iface_key]
622 ifname = ifacedump['autogen_ifname']
623
624 # If defined as vlan interface
625 if ifacedump['autogen_vlan']:
626 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
627
628 # Flag dhclient is possible
629 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
630 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
631
632 # Add interface IP to list
633 item = (ifacedump['ip'], ifacedump['comment'])
634 if addrs_list.has_key(ifname):
635 addrs_list[ifname].append(item)
636 else:
637 addrs_list[ifname] = [item]
638
639 # Alias only needs IP assignment for now, this might change if we
640 # are going to use virtual accesspoints
641 if "alias" in iface_key:
642 continue
643
644 # XXX: Might want to deduct type directly from interface name
645 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
646 # Default to station (client) mode
647 ifacedump['autogen_wlanmode'] = "sta"
648 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
649 ifacedump['autogen_wlanmode'] = "ap"
650
651 if not ifacedump.has_key('channel'):
652 if ifacedump['type'] == '11a':
653 ifacedump['channel'] = 36
654 else:
655 ifacedump['channel'] = 1
656
657 # Allow special hacks at the back like wds and stuff
658 if not ifacedump.has_key('extra'):
659 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
660 else:
661 ifacedump['autogen_extra'] = ifacedump['extra']
662
663 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
664 output += ("create_args_%(autogen_ifname)s='wlanmode %(autogen_wlanmode)s mode " +\
665 "%(type)s ssid %(ssid)s %(autogen_extra)s channel %(channel)s'\n") % ifacedump
666 output += "\n"
667
668 elif ifacedump['type'] in ['ethernet', 'eth']:
669 # No special config needed besides IP
670 if ifacedump['autogen_bridge']:
671 output += "cloned_interfaces='%(autogen_ifname)s'\n" % ifacedump
672 output += "ifconfig_%s='addm %s up'\n" % (ifacedump['autogen_ifname'], ' addm '.join(ifacedump['autogen_bridge_interfaces']))
673 for member in ifacedump['autogen_bridge_interfaces']:
674 output += "ifconfig_%s='up'\n" % member
675 output += "\n"
676 else:
677 assert False, "Unknown type " + ifacedump['type']
678
679 store = (addrs_list, vlan_list, dhclient_if, output)
680 interface_list_cache[datadump['autogen_item']] = store
681 return(store)
682
683
684
685def generate_rc_conf_local(datadump):
686 """ Generate configuration file '/etc/rc.conf.local' """
687 item = datadump['autogen_item']
688 if rc_conf_local_cache.has_key(item):
689 return rc_conf_local_cache[item]
690
691 if not datadump.has_key('ileiden'):
692 datadump['autogen_ileiden_enable'] = False
693 else:
694 datadump['autogen_ileiden_enable'] = datadump['ileiden']
695
696 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
697
698 if not ileiden_proxies or not normal_proxies:
699 for host in get_hostlist():
700 hostdump = get_yaml(host)
701 if hostdump['status'] == 'up':
702 if hostdump['service_proxy_ileiden']:
703 ileiden_proxies.append(hostdump)
704 if hostdump['service_proxy_normal']:
705 normal_proxies.append(hostdump)
706
707 datadump['autogen_ileiden_proxies'] = ileiden_proxies
708 datadump['autogen_normal_proxies'] = normal_proxies
709 datadump['autogen_ileiden_proxies_ips'] = ','.join([x['masterip'] for x in ileiden_proxies])
710 datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
711 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
712 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
713 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
714 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
715
716 output = generate_header(datadump, "#");
717 output += render_template(datadump, """\
718hostname='{{ autogen_fqdn }}'
719location='{{ location }}'
720nodetype="{{ nodetype }}"
721
722#
723# Configured listings
724#
725captive_portal_whitelist=""
726{% if nodetype == "Proxy" %}
727#
728# Proxy Configuration
729#
730{% if gateway and service_proxy_ileiden -%}
731defaultrouter="{{ gateway }}"
732{% else -%}
733#defaultrouter="NOTSET"
734{% endif -%}
735internalif="{{ internalif }}"
736ileiden_enable="{{ autogen_ileiden_enable }}"
737gateway_enable="{{ autogen_ileiden_enable }}"
738pf_enable="yes"
739pf_rules="/etc/pf.conf"
740{% if autogen_ileiden_enable -%}
741pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
742lvrouted_enable="{{ autogen_ileiden_enable }}"
743lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
744{% else -%}
745pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
746{% endif -%}
747{% if internalroute -%}
748static_routes="wleiden"
749route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
750{% endif -%}
751
752{% elif nodetype == "Hybrid" %}
753 #
754 # Hybrid Configuration
755 #
756 list_ileiden_proxies="
757 {% for item in autogen_ileiden_proxies -%}
758 {{ "%-16s"|format(item.masterip) }} # {{ item.nodename }}
759 {% endfor -%}
760 "
761 list_normal_proxies="
762 {% for item in autogen_normal_proxies -%}
763 {{ "%-16s"|format(item.masterip) }} # {{ item.nodename }}
764 {% endfor -%}
765 "
766
767 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',')|default('none', true) }}"
768 externalif="{{ externalif|default('vr0', true) }}"
769 masterip="{{ masterip }}"
770
771 {% if gateway and service_proxy_ileiden %}
772 defaultrouter="{{ gateway }}"
773 {% else %}
774 #defaultrouter="NOTSET"
775 {% endif %}
776
777 #
778 # Defined services
779 #
780 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
781 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
782 service_accesspoint="{{ service_accesspoint|yesorno }}"
783 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
784 service_concentrator="{{ service_concentrator|yesorno }}"
785
786 {% if service_proxy_ileiden %}
787 pf_rules="/etc/pf.hybrid.conf"
788 {% if service_concentrator %}
789 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
790 {% else %}
791 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
792 {% endif %}
793 pf_flags="$pf_flags -D publicnat=80,443"
794 lvrouted_flags="$lvrouted_flags -g"
795 {% elif service_proxy_normal or service_incoming_rdr %}
796 pf_rules="/etc/pf.hybrid.conf"
797 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
798 pf_flags="$pf_flags -D publicnat=0"
799 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
800 named_setfib="1"
801 tinyproxy_setfib="1"
802 dnsmasq_setfib="1"
803 sshd_setfib="1"
804 {% else %}
805 named_auto_forward_only="YES"
806 pf_rules="/etc/pf.node.conf"
807 pf_flags=""
808 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
809 {% endif %}
810 {% if service_concentrator %}
811 # Do mind installing certificates is NOT done automatically for security reasons
812 openvpn_enable="YES"
813 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
814 {% endif %}
815
816 {% if service_proxy_normal %}
817 tinyproxy_enable="yes"
818 {% else %}
819 pen_wrapper_enable="yes"
820 {% endif %}
821
822 {% if service_accesspoint %}
823 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
824 {% endif %}
825
826 {% if board == "ALIX2" %}
827 #
828 # ''Fat'' configuration, board has 256MB RAM
829 #
830 dnsmasq_enable="NO"
831 named_enable="YES"
832 {% if autogen_dhcp_interfaces -%}
833 dhcpd_enable="YES"
834 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
835 {% endif -%}
836 {% endif -%}
837{% endif %}
838
839#
840# Script variables
841#
842attached_devices="{{ autogen_attached_devices|join(' ') }}"
843neighbours="{{ autogen_neighbours|join(' ') }}"
844
845
846#
847# Interface definitions
848#\n
849""")
850
851 (addrs_list, vlan_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
852 for iface, vlans in vlan_list.items():
853 output += 'vlans_%s="%s"\n' % (iface, ' '.join(vlans))
854
855 # Details like SSID
856 if extra_ouput:
857 output += extra_ouput.strip() + "\n"
858
859 # Print IP address which needs to be assigned over here
860 output += "\n"
861 for iface,addrs in sorted(addrs_list.iteritems()):
862 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
863 output += "# %s || %s || %s\n" % (iface, addr, comment)
864
865 # Write DHCLIENT entry
866 if dhclient_if[iface]:
867 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
868
869 # Make sure the external address is always first as this is needed in the
870 # firewall setup
871 addrs = sorted(
872 [x for x in addrs if not '0.0.0.0' in x[0]],
873 key=lambda x: x[0].split('.')[0],
874 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
875 )
876 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
877 for idx, addr in enumerate(addrs[1:]):
878 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx, addr[0])
879 output += "\n"
880
881 rc_conf_local_cache[datadump['autogen_item']] = output
882 return output
883
884
885
886def get_all_configs():
887 """ Get dict with key 'host' with all configs present """
888 configs = dict()
889 for host in get_hostlist():
890 datadump = get_yaml(host)
891 configs[host] = datadump
892 return configs
893
894
895def get_interface_keys(config, extra=False):
896 """ Quick hack to get all interface keys, later stage convert this to a iterator """
897 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
898 if extra == False:
899 return filter(lambda x: not "extra" in x, elems)
900 else:
901 return elems
902
903
904def get_used_ips(configs):
905 """ Return array of all IPs used in config files"""
906 ip_list = []
907 for config in configs:
908 ip_list.append(config['masterip'])
909 for iface_key in get_interface_keys(config, True):
910 l = config[iface_key]['ip']
911 addr, mask = l.split('/')
912 # Special case do not process
913 if valid_addr(addr):
914 ip_list.append(addr)
915 else:
916 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
917 return sorted(ip_list)
918
919
920
921def get_nameservers(max_servers=None):
922 if nameservers_cache:
923 return nameservers_cache[0:max_servers]
924
925 for host in get_hostlist():
926 hostdump = get_yaml(host)
927 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
928 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
929
930 return nameservers_cache[0:max_servers]
931
932
933def get_neighbours(datadump):
934 (addrs_list, _, dhclient_if, extra_ouput) = make_interface_list(datadump)
935
936 (poel, errors) = make_relations()
937 table = []
938 for iface,addrs in sorted(addrs_list.iteritems()):
939 if iface in ['lo0']:
940 continue
941
942 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
943 if not addr.startswith('172.'):
944 # Avoid listing internet connections as pool
945 continue
946 for neighbour in poel[network(addr)]:
947 if neighbour[0] != datadump['autogen_item']:
948 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_ifname'] + ")", neighbour[1]['comment']))
949 return table
950
951
952def get_attached_devices(datadump, url=False):
953 table = []
954 for iface_key in get_interface_keys(datadump, True):
955 # Quick to avoid listing ath(4) interface as attached device
956 if 'ath0' in iface_key:
957 continue
958 ifacedump = datadump[iface_key]
959 if ifacedump.has_key('ns_ip'):
960 x_ip = ifacedump['ns_ip'].split('/')[0]
961 else:
962 x_ip = ifacedump['ip'].split('/')[0]
963
964 if 'mode' in ifacedump:
965 x_mode = ifacedump['mode']
966 else:
967 x_mode = 'unknown'
968
969 if 'bridge_type' in ifacedump:
970 device_type = ifacedump['bridge_type']
971 else:
972 device_type = 'Unknown'
973
974 table.append((ifacedump['autogen_ifname'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
975 return table
976
977
978def generate_resolv_conf(datadump):
979 """ Generate configuration file '/etc/resolv.conf' """
980 # XXX: This should properly going to be an datastructure soon
981 datadump['autogen_header'] = generate_header(datadump, "#")
982 datadump['autogen_edge_nameservers'] = ''
983
984
985 for masterip,realname in get_nameservers():
986 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
987
988 return Template("""\
989{{ autogen_header }}
990search wleiden.net
991
992# Try local (cache) first
993nameserver 127.0.0.1
994
995{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
996nameserver 8.8.8.8 # Google Public NameServer
997nameserver 4.2.2.1 # Level3 Public NameServer
998{% else -%}
999# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1000{{ autogen_edge_nameservers }}
1001{% endif -%}
1002""").render(datadump)
1003
1004
1005
1006def generate_ntp_conf(datadump):
1007 """ Generate configuration file '/etc/ntp.conf' """
1008 # XXX: This should properly going to be an datastructure soon
1009
1010 datadump['autogen_header'] = generate_header(datadump, "#")
1011 datadump['autogen_ntp_servers'] = ''
1012 for host in get_hostlist():
1013 hostdump = get_yaml(host)
1014 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1015 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1016
1017 return Template("""\
1018{{ autogen_header }}
1019
1020{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1021# Machine hooked to internet.
1022server 0.nl.pool.ntp.org iburst maxpoll 9
1023server 1.nl.pool.ntp.org iburst maxpoll 9
1024server 2.nl.pool.ntp.org iburst maxpoll 9
1025server 3.nl.pool.ntp.org iburst maxpoll 9
1026{% else -%}
1027# Local Wireless Leiden NTP Servers.
1028server 0.pool.ntp.wleiden.net iburst maxpoll 9
1029server 1.pool.ntp.wleiden.net iburst maxpoll 9
1030server 2.pool.ntp.wleiden.net iburst maxpoll 9
1031server 3.pool.ntp.wleiden.net iburst maxpoll 9
1032
1033# All the configured NTP servers
1034{{ autogen_ntp_servers }}
1035{% endif %}
1036
1037# If a server loses sync with all upstream servers, NTP clients
1038# no longer follow that server. The local clock can be configured
1039# to provide a time source when this happens, but it should usually
1040# be configured on just one server on a network. For more details see
1041# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1042# The use of Orphan Mode may be preferable.
1043#
1044server 127.127.1.0
1045fudge 127.127.1.0 stratum 10
1046""").render(datadump)
1047
1048
1049def generate_pf_hybrid_conf_local(datadump):
1050 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1051 datadump['autogen_header'] = generate_header(datadump, "#")
1052 return Template("""\
1053{{ autogen_header }}
1054
1055# Redirect some internal facing services outside (7)
1056# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1057{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1058rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1059{% endfor -%}
1060""").render(datadump)
1061
1062def generate_motd(datadump):
1063 """ Generate configuration file '/etc/motd' """
1064 output = Template("""\
1065FreeBSD run ``service motd onestart'' to make me look normal
1066
1067 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1068 Loc: {{ location }}
1069
1070Services:
1071{% if board == "ALIX2" -%}
1072{{" -"}} Core Node ({{ board }})
1073{% else -%}
1074{{" -"}} Hulp Node ({{ board }})
1075{% endif -%}
1076{% if service_proxy_normal -%}
1077{{" -"}} Normal Proxy
1078{% endif -%}
1079{% if service_proxy_ileiden -%}
1080{{" -"}} iLeiden Proxy
1081{% endif -%}
1082{% if service_incoming_rdr -%}
1083{{" -"}} Incoming port redirects
1084{% endif %}
1085Interlinks:\n
1086""").render(datadump)
1087
1088
1089 def make_table(table):
1090 if not table:
1091 return " - none\n"
1092 else:
1093 lines = ""
1094 col_width = [max(len(x) for x in col) for col in zip(*table)]
1095 for row in table:
1096 lines += " - " + " || ".join("{:{}}".format(x, col_width[i]) for i, x in enumerate(row)) + "\n"
1097 return lines
1098
1099 (addrs_list, vlan_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
1100 table = []
1101 for iface,addrs in sorted(addrs_list.iteritems()):
1102 if iface in ['lo0']:
1103 continue
1104 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1105 table.append((iface, addr, comment))
1106
1107 output += make_table(table)
1108 output += '\n'
1109 output += """\
1110Attached devices:
1111"""
1112 output += make_table(get_attached_devices(datadump, url=True))
1113 output += '\n'
1114 output += """\
1115Available neighbours:
1116"""
1117 output += make_table(get_neighbours(datadump))
1118
1119 return output
1120
1121
1122def format_yaml_value(value):
1123 """ Get yaml value in right syntax for outputting """
1124 if isinstance(value,str):
1125 output = '"%s"' % value
1126 else:
1127 output = value
1128 return output
1129
1130
1131
1132def format_wleiden_yaml(datadump):
1133 """ Special formatting to ensure it is editable"""
1134 output = "# Genesis config yaml style\n"
1135 output += "# vim:ts=2:et:sw=2:ai\n"
1136 output += "#\n"
1137 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1138 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1139 if key == 'rdr_rules':
1140 output += '%-10s:\n' % 'rdr_rules'
1141 for rdr_rule in datadump[key]:
1142 output += '- %s\n' % rdr_rule
1143 else:
1144 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1145
1146 output += "\n\n"
1147
1148 # Format (key, required)
1149 key_order = (
1150 ('comment', True),
1151 ('ip', True),
1152 ('desc', True),
1153 ('sdesc', True),
1154 ('mode', True),
1155 ('type', True),
1156 ('extra_type', False),
1157 ('channel', False),
1158 ('ssid', False),
1159 ('wlan_mac', False),
1160 ('dhcp', True),
1161 ('compass', False),
1162 ('distance', False),
1163 ('ns_ip', False),
1164 ('repeater_ip', False),
1165 ('bullet2_ip', False),
1166 ('ns_mac', False),
1167 ('bullet2_mac', False),
1168 ('ns_type', False),
1169 ('bridge_type', False),
1170 ('members', True),
1171 ('status', True),
1172 )
1173
1174 for iface_key in sorted(iface_keys):
1175 try:
1176 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1177 if remainder:
1178 raise KeyError("invalid keys: %s" % remainder)
1179
1180 output += "%s:\n" % iface_key
1181 for key,required in key_order:
1182 if datadump[iface_key].has_key(key):
1183 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1184 output += "\n\n"
1185 except Exception:
1186 print "# Error while processing interface %s" % iface_key
1187 raise
1188
1189 return output
1190
1191
1192
1193def generate_wleiden_yaml(datadump, header=True):
1194 """ Generate (petty) version of wleiden.yaml"""
1195 output = generate_header(datadump, "#") if header else ''
1196
1197 for key in datadump.keys():
1198 if key.startswith('autogen_'):
1199 del datadump[key]
1200 # Interface autogen cleanups
1201 elif type(datadump[key]) == dict:
1202 for key2 in datadump[key].keys():
1203 if key2.startswith('autogen_'):
1204 del datadump[key][key2]
1205
1206 output += format_wleiden_yaml(datadump)
1207 return output
1208
1209def generate_nanostation_config(datadump, iface, ns_type):
1210 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1211 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1212
1213 datadump.update(datadump['iface_%s' % iface])
1214
1215 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1216
1217def generate_yaml(datadump):
1218 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1219
1220
1221
1222def generate_config(node, config, datadump=None):
1223 """ Print configuration file 'config' of 'node' """
1224 output = ""
1225 try:
1226 # Load config file
1227 if datadump == None:
1228 datadump = get_yaml(node)
1229
1230 if config == 'wleiden.yaml':
1231 output += generate_wleiden_yaml(datadump)
1232 elif config == 'authorized_keys':
1233 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1234 output += f.read()
1235 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1236 # Fetch local keys if existing
1237 if os.path.exists(node_keys):
1238 output += open(node_keys, 'r').read()
1239 f.close()
1240 elif config == 'dnsmasq.conf':
1241 output += generate_dnsmasq_conf(datadump)
1242 elif config == 'dhcpd.conf':
1243 output += generate_dhcpd_conf(datadump)
1244 elif config == 'rc.conf.local':
1245 output += generate_rc_conf_local(datadump)
1246 elif config == 'resolv.conf':
1247 output += generate_resolv_conf(datadump)
1248 elif config == 'ntp.conf':
1249 output += generate_ntp_conf(datadump)
1250 elif config == 'motd':
1251 output += generate_motd(datadump)
1252 elif config == 'pf.hybrid.conf.local':
1253 output += generate_pf_hybrid_conf_local(datadump)
1254 elif config.startswith('vr'):
1255 interface, ns_type = config.strip('.yaml').split('-')
1256 output += generate_nanostation_config(datadump, interface, ns_type)
1257 else:
1258 assert False, "Config not found!"
1259 except IOError, e:
1260 output += "[ERROR] Config file not found"
1261 return output
1262
1263
1264
1265def process_cgi_request(environ=os.environ):
1266 """ When calling from CGI """
1267 response_headers = []
1268 content_type = 'text/plain'
1269
1270 # Update repository if requested
1271 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1272 if form and form.has_key("action") and "update" in form["action"]:
1273 output = "[INFO] Updating subverion, please wait...\n"
1274 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1275 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1276 output += "[INFO] All done, redirecting in 5 seconds"
1277 response_headers += [
1278 ('Refresh', '5; url=.'),
1279 ]
1280 reload_cache()
1281 else:
1282 base_uri = environ['PATH_INFO']
1283 uri = base_uri.strip('/').split('/')
1284
1285 output = "Template Holder"
1286 if base_uri.endswith('/create/network.kml'):
1287 content_type='application/vnd.google-earth.kml+xml'
1288 output = make_network_kml.make_graph()
1289 elif base_uri.endswith('/api/get/nodeplanner.json'):
1290 content_type='application/json'
1291 output = make_network_kml.make_nodeplanner_json()
1292 elif not uri[0]:
1293 if is_text_request(environ):
1294 output = '\n'.join(get_hostlist())
1295 else:
1296 content_type = 'text/html'
1297 output = generate_title(get_hostlist())
1298 elif len(uri) == 1:
1299 if is_text_request(environ):
1300 output = generate_node(uri[0])
1301 else:
1302 content_type = 'text/html'
1303 output = generate_node_overview(uri[0])
1304 elif len(uri) == 2:
1305 output = generate_config(uri[0], uri[1])
1306 else:
1307 assert False, "Invalid option"
1308
1309 # Return response
1310 response_headers += [
1311 ('Content-type', content_type),
1312 ('Content-Length', str(len(output))),
1313 ]
1314 return(response_headers, str(output))
1315
1316
1317def make_dns(output_dir = 'dns', external = False):
1318 items = dict()
1319
1320 # hostname is key, IP is value
1321 wleiden_zone = defaultdict(list)
1322 wleiden_cname = dict()
1323
1324 pool = dict()
1325 for node in get_hostlist():
1326 datadump = get_yaml(node)
1327
1328 # Proxy naming convention is special
1329 fqdn = datadump['nodename']
1330 if datadump['nodetype'] in ['Hybrid']:
1331 wleiden_cname[datadump['nodename']] = fqdn
1332
1333 if datadump.has_key('rdr_host'):
1334 remote_target = datadump['rdr_host']
1335 elif datadump.has_key('remote_access') and datadump['remote_access']:
1336 remote_target = datadump['remote_access'].split(':')[0]
1337 else:
1338 remote_target = None
1339
1340 if remote_target:
1341 try:
1342 parseaddr(remote_target)
1343 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1344 except (IndexError, ValueError):
1345 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1346
1347
1348 wleiden_zone[fqdn].append((datadump['masterip'], True))
1349
1350 # Hacking to get proper DHCP IPs and hostnames
1351 for iface_key in get_interface_keys(datadump):
1352 iface_name = iface_key.replace('_','-')
1353 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1354 try:
1355 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1356 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1357 dhcp_part = ".".join(ip.split('.')[0:3])
1358 if ip != datadump['masterip']:
1359 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, False))
1360 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1361 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1362 except (AttributeError, ValueError, KeyError):
1363 # First push it into a pool, to indentify the counter-part later on
1364 addr = parseaddr(ip)
1365 cidr = int(cidr)
1366 addr = addr & ~((1 << (32 - cidr)) - 1)
1367 if pool.has_key(addr):
1368 pool[addr] += [(iface_name, fqdn, ip)]
1369 else:
1370 pool[addr] = [(iface_name, fqdn, ip)]
1371 continue
1372
1373
1374
1375 # WL uses an /29 to configure an interface. IP's are ordered like this:
1376 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1377
1378 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1379
1380 # Automatic naming convention of interlinks namely 2 + remote.lower()
1381 for (key,value) in pool.iteritems():
1382 # Make sure they are sorted from low-ip to high-ip
1383 value = sorted(value, key=lambda x: parseaddr(x[2]))
1384
1385 if len(value) == 1:
1386 (iface_name, fqdn, ip) = value[0]
1387 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1388
1389 # Device DNS names
1390 if 'cnode' in fqdn.lower():
1391 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1392 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1393
1394 elif len(value) == 2:
1395 (a_iface_name, a_fqdn, a_ip) = value[0]
1396 (b_iface_name, b_fqdn, b_ip) = value[1]
1397 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1398 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1399
1400 # Device DNS names
1401 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1402 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1403 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1404 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1405 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1406 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1407 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1408
1409 else:
1410 pool_members = [k[1] for k in value]
1411 for item in value:
1412 (iface_name, fqdn, ip) = item
1413 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1414
1415 # Include static DNS entries
1416 # XXX: Should they override the autogenerated results?
1417 # XXX: Convert input to yaml more useable.
1418 # Format:
1419 ##; this is a comment
1420 ## roomburgh=Roomburgh1
1421 ## apkerk1.Vosko=172.17.176.8 ;this as well
1422 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1423
1424 # Hack to allow special entries, for development
1425 wleiden_raw = {}
1426
1427 for line in dns_list:
1428 reverse = False
1429 k, items = line.items()[0]
1430 if type(items) == dict:
1431 if items.has_key('reverse'):
1432 reverse = items['reverse']
1433 items = items['a']
1434 else:
1435 items = items['cname']
1436 items = [items] if type(items) != list else items
1437 for item in items:
1438 if item.startswith('IN '):
1439 wleiden_raw[k] = item
1440 elif valid_addr(item):
1441 wleiden_zone[k].append((item, reverse))
1442 else:
1443 wleiden_cname[k] = item
1444
1445 # Hack to get dynamic pool listing
1446 def chunks(l, n):
1447 return [l[i:i+n] for i in range(0, len(l), n)]
1448
1449 ntp_servers = [x[0] for x in get_nameservers()]
1450 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1451 for ntp_server in chunk:
1452 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1453
1454 details = dict()
1455 # 24 updates a day allowed
1456 details['serial'] = time.strftime('%Y%m%d%H')
1457
1458 if external:
1459 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1460 else:
1461 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1462
1463 details['master'] = dns_masters[0]
1464 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1465
1466 dns_header = '''
1467$TTL 3h
1468%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1469 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1470
1471%(ns_servers)s
1472 \n'''
1473
1474
1475 if not os.path.isdir(output_dir):
1476 os.makedirs(output_dir)
1477 details['zone'] = 'wleiden.net'
1478 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1479 f.write(dns_header % details)
1480
1481 for host,items in wleiden_zone.iteritems():
1482 for ip,reverse in items:
1483 if ip not in ['0.0.0.0']:
1484 f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
1485 for source,dest in wleiden_cname.iteritems():
1486 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1487 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1488 for source, dest in wleiden_raw.iteritems():
1489 f.write("%s.wleiden.net. %s\n" % (source, dest))
1490 f.close()
1491
1492 # Create whole bunch of specific sub arpa zones. To keep it compliant
1493 for s in range(16,32):
1494 details['zone'] = '%i.172.in-addr.arpa' % s
1495 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1496 f.write(dns_header % details)
1497
1498 #XXX: Not effient, fix to proper data structure and do checks at other
1499 # stages
1500 for host,items in wleiden_zone.iteritems():
1501 for ip,reverse in items:
1502 if not reverse:
1503 continue
1504 if valid_addr(ip):
1505 if valid_addr(ip):
1506 if int(ip.split('.')[1]) == s:
1507 rev_ip = '.'.join(reversed(ip.split('.')))
1508 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1509 f.close()
1510
1511
1512def usage():
1513 print """Usage: %(prog)s <argument>
1514Argument:
1515\tcleanup = Cleanup all YAML files to specified format
1516\tstandalone [port] = Run configurator webserver [8000]
1517\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1518\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1519\tfull-export = Generate yaml export script for heatmap.
1520\tstatic [outputdir] = Generate all config files and store on disk
1521\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1522\ttest <node> [<file>] = Receive output for certain node [all files].
1523\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1524\tlist <status> <items> = List systems which have certain status
1525
1526Arguments:
1527\t<node> = NodeName (example: HybridRick)
1528\t<file> = %(files)s
1529\t<status> = all|up|down|planned
1530\t<items> = systems|nodes|proxies
1531
1532NOTE FOR DEVELOPERS; you can test your changes like this:
1533 BEFORE any changes in this code:
1534 $ ./gformat.py static /tmp/pre
1535 AFTER the changes:
1536 $ ./gformat.py static /tmp/post
1537 VIEW differences and VERIFY all are OK:
1538 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1539""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1540 exit(0)
1541
1542
1543def is_text_request(environ=os.environ):
1544 """ Find out whether we are calling from the CLI or any text based CLI utility """
1545 try:
1546 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1547 except KeyError:
1548 return True
1549
1550def switchFormat(setting):
1551 if setting:
1552 return "YES"
1553 else:
1554 return "NO"
1555
1556def rlinput(prompt, prefill=''):
1557 import readline
1558 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1559 try:
1560 return raw_input(prompt)
1561 finally:
1562 readline.set_startup_hook()
1563
1564def fix_conflict(left, right, default='i'):
1565 while True:
1566 print "## %-30s | %-30s" % (left, right)
1567 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1568 if not c:
1569 c = default
1570
1571 if c in ['l','1']:
1572 return left
1573 elif c in ['r','2']:
1574 return right
1575 elif c in ['e', '3']:
1576 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1577 elif c in ['i', '4']:
1578 return None
1579 else:
1580 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1581
1582
1583
1584def print_cgi_response(response_headers, output):
1585 """Could we not use some kind of wsgi wrapper to make this output?"""
1586 for header in response_headers:
1587 print "%s: %s" % header
1588 print
1589 print output
1590
1591
1592def fill_cache():
1593 ''' Poor man re-loading of few cache items (the slow ones) '''
1594 for host in get_hostlist():
1595 get_yaml(host)
1596
1597
1598def reload_cache():
1599 clear_cache()
1600 fill_cache()
1601
1602
1603def main():
1604 """Hard working sub"""
1605 # Allow easy hacking using the CLI
1606 if not os.environ.has_key('PATH_INFO'):
1607 if len(sys.argv) < 2:
1608 usage()
1609
1610 if sys.argv[1] == "standalone":
1611 import SocketServer
1612 import CGIHTTPServer
1613 # Hop to the right working directory.
1614 os.chdir(os.path.dirname(__file__))
1615 try:
1616 PORT = int(sys.argv[2])
1617 except (IndexError,ValueError):
1618 PORT = 8000
1619
1620 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1621 """ Serve this CGI from the root of the webserver """
1622 def is_cgi(self):
1623 if "favicon" in self.path:
1624 return False
1625
1626 self.cgi_info = (os.path.basename(__file__), self.path)
1627 self.path = ''
1628 return True
1629 handler = MyCGIHTTPRequestHandler
1630 SocketServer.TCPServer.allow_reuse_address = True
1631 httpd = SocketServer.TCPServer(("", PORT), handler)
1632 httpd.server_name = 'localhost'
1633 httpd.server_port = PORT
1634
1635 logger.info("serving at port %s", PORT)
1636 try:
1637 httpd.serve_forever()
1638 except KeyboardInterrupt:
1639 httpd.shutdown()
1640 logger.info("All done goodbye")
1641 elif sys.argv[1] == "test":
1642 # Basic argument validation
1643 try:
1644 node = sys.argv[2]
1645 except IndexError:
1646 print "Invalid argument"
1647 exit(1)
1648 except IOError as e:
1649 print e
1650 exit(1)
1651
1652 datadump = get_yaml(node)
1653
1654
1655 # Get files to generate
1656 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1657
1658 # Actual config generation
1659 for config in gen_files:
1660 logger.info("## Generating %s %s", node, config)
1661 print generate_config(node, config, datadump)
1662 elif sys.argv[1] == "test-cgi":
1663 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1664 os.environ['SCRIPT_NAME'] = __file__
1665 response_headers, output = process_cgi_request()
1666 print_cgi_response(response_headers, output)
1667 elif sys.argv[1] == "static":
1668 items = dict()
1669 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1670 for node in get_hostlist():
1671 items['node'] = node
1672 items['wdir'] = "%(output_dir)s/%(node)s" % items
1673 if not os.path.isdir(items['wdir']):
1674 os.makedirs(items['wdir'])
1675 datadump = get_yaml(node)
1676 for config in files:
1677 items['config'] = config
1678 logger.info("## Generating %(node)s %(config)s" % items)
1679 f = open("%(wdir)s/%(config)s" % items, "w")
1680 f.write(generate_config(node, config, datadump))
1681 f.close()
1682 elif sys.argv[1] == "wind-export":
1683 items = dict()
1684 for node in get_hostlist():
1685 datadump = get_yaml(node)
1686 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1687 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1688 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1689 VALUES (
1690 (SELECT id FROM users WHERE username = 'rvdzwet'),
1691 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1692 'Y');""" % datadump
1693 #for config in files:
1694 # items['config'] = config
1695 # print "## Generating %(node)s %(config)s" % items
1696 # f = open("%(wdir)s/%(config)s" % items, "w")
1697 # f.write(generate_config(node, config, datadump))
1698 # f.close()
1699 for node in get_hostlist():
1700 datadump = get_yaml(node)
1701 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1702 ifacedump = datadump[iface_key]
1703 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1704 ifacedump['nodename'] = datadump['nodename']
1705 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1706 ifacedump['channel'] = 0
1707 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1708 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1709 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1710 elif sys.argv[1] == "nagios-export":
1711 try:
1712 heavy_load = (sys.argv[2] == "--heavy-load")
1713 except IndexError:
1714 heavy_load = False
1715
1716 hostgroup_details = {
1717 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1718 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1719 'walphen' : 'Stichting Wireless Alphen',
1720 'westeinder' : 'Westeinder Plassen',
1721 }
1722
1723 # Convert IP to Host
1724 ip2host = {'root' : 'root'}
1725 for host in get_hostlist():
1726 datadump = get_yaml(host)
1727 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1728 for iface in get_interface_keys(datadump):
1729 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1730
1731 # Find dependency tree based on output of lvrouted.mytree of nearest node
1732 parents = defaultdict(list)
1733 stack = ['root']
1734 prev_depth = 0
1735 for line in open('lvrouted.mytree').readlines():
1736 depth = line.count('\t')
1737 ip = line.strip().split()[0]
1738
1739 if prev_depth < depth:
1740 try:
1741 parents[ip2host[ip]].append(ip2host[stack[-1]])
1742 except KeyError as e:
1743 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1744 stack.append(ip)
1745 elif prev_depth > depth:
1746 stack = stack[:(depth - prev_depth)]
1747 elif prev_depth == depth:
1748 try:
1749 parents[ip2host[ip]].append(ip2host[stack[-1]])
1750 except KeyError as e:
1751 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1752
1753
1754 prev_depth = depth
1755 # Observe that some nodes has themself as parent or multiple parents
1756 # for now take only the first parent, other behaviour is yet to be explained
1757
1758
1759
1760 params = {
1761 'check_interval' : 5 if heavy_load else 120,
1762 'retry_interval' : 1 if heavy_load else 10,
1763 'max_check_attempts' : 10 if heavy_load else 6,
1764 'notification_interval': 120 if heavy_load else 240,
1765 }
1766
1767 print '''\
1768define host {
1769 name wleiden-node ; Default Node Template
1770 use generic-host ; Use the standard template as initial starting point
1771 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1772 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1773 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1774 notification_interval %(notification_interval)s
1775 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1776 check_command check-host-alive ; Default command to check FreeBSD hosts
1777 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1778}
1779
1780define service {
1781 name wleiden-service ; Default Service Template
1782 use generic-service ; Use the standard template as initial starting point
1783 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1784 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1785 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1786 notification_interval %(notification_interval)s
1787 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1788 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1789}
1790
1791# Please make sure to install:
1792# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1793#
1794# Recompile net-mgmt/nagios-plugins to support check_snmp
1795# make -C /usr/ports/net-mgmt/nagios-plugins
1796#
1797# Install net/bind-tools to allow v2/check_dns_wl to work:
1798# pkg install bind-tools
1799#
1800define command{
1801 command_name check_snmp_disk
1802 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1803}
1804
1805define command{
1806 command_name check_netsnmp_load
1807 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1808}
1809
1810define command{
1811 command_name check_netsnmp_proc
1812 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1813}
1814
1815define command{
1816 command_name check_by_ssh
1817 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1818}
1819
1820define command{
1821 command_name check_dns_wl
1822 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1823}
1824
1825define command{
1826 command_name check_snmp_uptime
1827 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1828}
1829
1830
1831# TDB: dhcp leases
1832# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1833
1834# TDB: internet status
1835# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1836
1837# TDB: Advanced local passive checks
1838# /usr/local/libexec/nagios/check_by_ssh
1839''' % params
1840
1841 print '''\
1842# Service Group, not displayed by default
1843define hostgroup {
1844 hostgroup_name srv_hybrid
1845 alias All Hybrid Nodes
1846 register 0
1847}
1848
1849define service {
1850 use wleiden-service
1851 hostgroup_name srv_hybrid
1852 service_description SSH
1853 check_command check_ssh
1854}
1855
1856define service {
1857 use wleiden-service,service-pnp
1858 hostgroup_name srv_hybrid
1859 service_description HTTP
1860 check_command check_http
1861}
1862
1863define service {
1864 use wleiden-service
1865 hostgroup_name srv_hybrid
1866 service_description DNS
1867 check_command check_dns_wl!"www.wirelessleiden.nl"
1868}
1869'''
1870
1871 if heavy_load:
1872 print '''\
1873define service {
1874 use wleiden-service
1875 hostgroup_name srv_hybrid
1876 service_description UPTIME
1877 check_command check_snmp_uptime
1878}
1879
1880#define service {
1881# use wleiden-service
1882# hostgroup_name srv_hybrid
1883# service_description NTP
1884# check_command check_ntp_peer
1885#}
1886
1887define service {
1888 use wleiden-service
1889 hostgroup_name srv_hybrid
1890 service_description LOAD
1891 check_command check_netsnmp_load
1892}
1893
1894define service {
1895 use wleiden-service
1896 hostgroup_name srv_hybrid
1897 service_description PROC
1898 check_command check_netsnmp_proc
1899}
1900
1901define service {
1902 use wleiden-service
1903 hostgroup_name srv_hybrid
1904 service_description DISK
1905 check_command check_snmp_disk
1906}
1907'''
1908 for node in get_hostlist():
1909 datadump = get_yaml(node)
1910 if not datadump['status'] == 'up':
1911 continue
1912 if not hostgroup_details.has_key(datadump['monitoring_group']):
1913 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
1914 print '''\
1915define host {
1916 use wleiden-node,host-pnp
1917 contact_groups admins
1918 host_name %(autogen_fqdn)s
1919 address %(masterip)s
1920 hostgroups srv_hybrid,%(monitoring_group)s\
1921''' % datadump
1922 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
1923 print '''\
1924 parents %(parents)s\
1925''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
1926 print '''\
1927}
1928'''
1929
1930
1931 for name,alias in hostgroup_details.iteritems():
1932 print '''\
1933define hostgroup {
1934 hostgroup_name %s
1935 alias %s
1936} ''' % (name, alias)
1937
1938
1939 elif sys.argv[1] == "full-export":
1940 hosts = {}
1941 for node in get_hostlist():
1942 datadump = get_yaml(node)
1943 hosts[datadump['nodename']] = datadump
1944 print yaml.dump(hosts)
1945
1946 elif sys.argv[1] == "dns":
1947 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
1948 elif sys.argv[1] == "cleanup":
1949 # First generate all datadumps
1950 datadumps = dict()
1951 ssid_to_node = dict()
1952 for host in get_hostlist():
1953 logger.info("# Processing: %s", host)
1954 # Set some boring default values
1955 datadump = { 'board' : 'UNKNOWN' }
1956 datadump.update(get_yaml(host))
1957 datadumps[datadump['nodename']] = datadump
1958
1959 (poel, errors) = make_relations()
1960 print "\n".join(["# WARNING: %s" % x for x in errors])
1961
1962 for host,datadump in datadumps.iteritems():
1963 try:
1964 # Convert all yes and no to boolean values
1965 def fix_boolean(dump):
1966 for key in dump.keys():
1967 if type(dump[key]) == dict:
1968 dump[key] = fix_boolean(dump[key])
1969 elif str(dump[key]).lower() in ["yes", "true"]:
1970 dump[key] = True
1971 elif str(dump[key]).lower() in ["no", "false"]:
1972 # Compass richting no (Noord Oost) is valid input
1973 if key != "compass": dump[key] = False
1974 return dump
1975 datadump = fix_boolean(datadump)
1976
1977 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
1978 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
1979 elif 'latitude' in datadump and 'longitude' in datadump:
1980 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
1981
1982 if datadump['nodename'].startswith('Proxy'):
1983 datadump['nodename'] = datadump['nodename'].lower()
1984
1985 for iface_key in get_interface_keys(datadump):
1986 try:
1987 # All our normal wireless cards are normal APs now
1988 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
1989 datadump[iface_key]['mode'] = 'ap'
1990 # Wireless Leiden SSID have an consistent lowercase/uppercase
1991 if datadump[iface_key].has_key('ssid'):
1992 ssid = datadump[iface_key]['ssid']
1993 prefix = 'ap-WirelessLeiden-'
1994 if ssid.lower().startswith(prefix.lower()):
1995 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
1996 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
1997 datadump[iface_key]['mode'] = 'autogen-FIXME'
1998 if not datadump[iface_key].has_key('comment'):
1999 datadump[iface_key]['comment'] = 'autogen-FIXME'
2000
2001 if datadump[iface_key].has_key('ns_mac'):
2002 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2003
2004 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2005 datadump[iface_key] = datadump[iface_key]['desc']
2006
2007 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2008 # due to better collision avoidance.
2009 if datadump[iface_key]['type'] == '11b':
2010 datadump[iface_key]['type'] = '11g'
2011
2012 # Setting 802.11g channels to de-facto standards, to avoid
2013 # un-detected sharing with other overlapping channels
2014 #
2015 # Technically we could also use channel 13 in NL, but this is not
2016 # recommended as foreign devices might not be able to select this
2017 # channel. Secondly using 1,5,9,13 instead is going to clash with
2018 # the de-facto usage of 1,6,11.
2019 #
2020 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2021 channels_at_2400Mhz = (1,6,11)
2022 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2023 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2024 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2025 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2026
2027 # Mandatory interface keys
2028 if not datadump[iface_key].has_key('status'):
2029 datadump[iface_key]['status'] = 'planned'
2030
2031 x = datadump[iface_key]['comment']
2032 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2033
2034 # Fixing bridge_type if none is found
2035 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2036 if not 'bridge_type' in datadump[iface_key]:
2037 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2038
2039 # Making sure description works
2040 if datadump[iface_key].has_key('desc'):
2041 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2042 del datadump[iface_key]['desc']
2043 else:
2044 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2045 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2046 if response:
2047 datadump[iface_key]['comment'] = response
2048 del datadump[iface_key]['desc']
2049
2050 # Check DHCP configuration
2051 dhcp_type(datadump[iface_key])
2052
2053 # Set the compass value based on the angle between the poels
2054 if datadump[iface_key].has_key('ns_ip'):
2055 my_pool = poel[network(datadump[iface_key]['ip'])]
2056 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2057 if remote_hosts:
2058 compass_target = remote_hosts[0]
2059 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2060
2061 # Monitoring Group default
2062 if not 'monitoring_group' in datadump:
2063 datadump['monitoring_group'] = 'wleiden'
2064
2065 except Exception:
2066 print "# Error while processing interface %s" % iface_key
2067 raise
2068 store_yaml(datadump)
2069 except Exception:
2070 print "# Error while processing %s" % host
2071 raise
2072 elif sys.argv[1] == "list":
2073 use_fqdn = False
2074 if len(sys.argv) < 4:
2075 usage()
2076 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2077 usage()
2078 if not sys.argv[3] in ["nodes","proxies","systems"]:
2079 usage()
2080
2081 if len(sys.argv) > 4:
2082 if sys.argv[4] == "fqdn":
2083 use_fqdn = True
2084 else:
2085 usage()
2086
2087 for system in get_hostlist():
2088 datadump = get_yaml(system)
2089 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2090 continue
2091
2092 output = datadump['autogen_fqdn'] if use_fqdn else system
2093 if sys.argv[2] == "all":
2094 print output
2095 elif datadump['status'] == sys.argv[2]:
2096 print output
2097 elif sys.argv[1] == "create":
2098 if sys.argv[2] == "network.kml":
2099 print make_network_kml.make_graph()
2100 elif sys.argv[2] == "host-ips.txt":
2101 for system in get_hostlist():
2102 datadump = get_yaml(system)
2103 ips = [datadump['masterip']]
2104 for ifkey in get_interface_keys(datadump):
2105 ips.append(datadump[ifkey]['ip'].split('/')[0])
2106 print system, ' '.join(ips)
2107 elif sys.argv[2] == "host-pos.txt":
2108 for system in get_hostlist():
2109 datadump = get_yaml(system)
2110 print system, datadump['rdnap_x'], datadump['rdnap_y']
2111 elif sys.argv[2] == 'ssh_config':
2112 print '''
2113Host *.wleiden.net
2114 User root
2115
2116Host 172.16.*.*
2117 User root
2118'''
2119 for system in get_hostlist():
2120 datadump = get_yaml(system)
2121 print '''\
2122Host %s
2123 User root
2124
2125Host %s
2126 User root
2127
2128Host %s
2129 User root
2130
2131Host %s
2132 User root
2133''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2134 else:
2135 usage()
2136 else:
2137 usage()
2138 else:
2139 # Do not enable debugging for config requests as it highly clutters the output
2140 if not is_text_request():
2141 cgitb.enable()
2142 response_headers, output = process_cgi_request()
2143 print_cgi_response(response_headers, output)
2144
2145def application(environ, start_response):
2146 status = '200 OK'
2147 response_headers, output = process_cgi_request(environ)
2148 start_response(status, response_headers)
2149
2150 # Debugging only
2151 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2152 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2153 return [output]
2154
2155if __name__ == "__main__":
2156 main()
Note: See TracBrowser for help on using the repository browser.