source: genesis/tools/gformat.py@ 13404

Last change on this file since 13404 was 13404, checked in by rick, 9 years ago

Removing references to old CNode and Hybrid prefixes
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 72.2 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13404 2015-09-20 19:09:40Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = []
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126def clear_cache():
127 ''' Poor mans cache implementation '''
128 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
129 datadump_cache = {}
130 interface_list_cache = {}
131 rc_conf_local_cache = {}
132 ileiden_proxies = []
133 normal_proxies = []
134 nameservers_cache = []
135 relations_cache = None
136
137NO_DHCP = 0
138DHCP_CLIENT = 10
139DHCP_SERVER = 20
140def dhcp_type(item):
141 if not item.has_key('dhcp'):
142 return NO_DHCP
143 elif not item['dhcp']:
144 return NO_DHCP
145 elif item['dhcp'].lower() == 'client':
146 return DHCP_CLIENT
147 else:
148 # Validation Checks
149 begin,end = map(int,item['dhcp'].split('-'))
150 if begin >= end:
151 raise ValueError("DHCP Start >= DHCP End")
152 return DHCP_SERVER
153
154def etrs2rd(lat, lon):
155 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
156 p2 = pyproj.Proj(init='EPSG:28992')
157 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
158 return (RDx, RDy)
159
160def rd2etrs(RDx, RDy):
161 p1 = pyproj.Proj(init='EPSG:28992')
162 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
163 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
164 return (lat, lon)
165
166def get_yaml(item,add_version_info=True):
167 try:
168 """ Get configuration yaml for 'item'"""
169 if datadump_cache.has_key(item):
170 return datadump_cache[item].copy()
171
172 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
173
174 # Default values
175 datadump = {
176 'autogen_revision' : 'NOTFOUND',
177 'autogen_gfile' : gfile,
178 'service_proxy_ileiden' : False,
179 }
180 f = open(gfile, 'r')
181 datadump.update(yaml.load(f,Loader=Loader))
182 if datadump['nodetype'] == 'Hybrid':
183 # Some values are defined implicitly
184 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
185 datadump['service_incoming_rdr'] = True
186 # Use some boring defaults
187 defaults = {
188 'service_proxy_normal' : False,
189 'service_accesspoint' : True,
190 'service_incoming_rdr' : False,
191 'service_concentrator' : False,
192 'monitoring_group' : 'wleiden',
193 }
194 for (key,value) in defaults.iteritems():
195 if not datadump.has_key(key):
196 datadump[key] = value
197 f.close()
198
199 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
200 if add_version_info:
201 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
202 lines = p.communicate()[0].split('\n')
203 if p.returncode == 0:
204 for line in lines:
205 if line:
206 (key, value) = line.split(': ')
207 datadump["autogen_" + key.lower().replace(' ','_')] = value
208
209 # Preformat certain needed variables for formatting and push those into special object
210 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
211
212 wlan_count=0
213 try:
214 for key in get_interface_keys(datadump, True):
215 datadump[key]['autogen_ifbase'] = key.split('_')[1]
216 datadump[key]['autogen_vlan'] = False
217
218 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
219 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
220 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
221 wlan_count += 1
222 else:
223 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
224 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
225 datadump[key]['autogen_vlan'] = key.split('_')[2]
226
227 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
228 if datadump[key]['autogen_bridge'] and not 'alias' in key:
229 datadump[key]['autogen_bridge_interfaces'] = datadump[key]['members'].split()
230 except Exception:
231 print "# Error while processing interface %s" % key
232 raise
233
234 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
235 if dhcp_type(datadump[key]) == DHCP_SERVER]
236
237 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
238 datadump['autogen_item'] = item
239
240 datadump['autogen_realname'] = get_realname(datadump)
241 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
242 datadump['autogen_fqdn'] = datadump['autogen_realname'] + '.' + datadump['autogen_domain']
243 datadump_cache[item] = datadump.copy()
244 except Exception:
245 print "# Error while processing %s" % item
246 raise
247 return datadump
248
249
250def store_yaml(datadump, header=False):
251 """ Store configuration yaml for 'item'"""
252 item = datadump['autogen_item']
253 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
254
255 output = generate_wleiden_yaml(datadump, header)
256
257 f = open(gfile, 'w')
258 f.write(output)
259 f.close()
260
261
262def network(ip):
263 addr, mask = ip.split('/')
264 # Not parsing of these folks please
265 addr = parseaddr(addr)
266 mask = int(mask)
267 network = addr & ~((1 << (32 - mask)) - 1)
268 return network
269
270
271
272def make_relations():
273 """ Process _ALL_ yaml files to get connection relations """
274 global relations_cache
275
276 if relations_cache:
277 return relations_cache
278
279 errors = []
280 poel = defaultdict(list)
281
282 for host in get_hostlist():
283 datadump = get_yaml(host)
284 try:
285 for iface_key in get_interface_keys(datadump):
286 net_addr = network(datadump[iface_key]['ip'])
287 poel[net_addr] += [(host,datadump[iface_key].copy())]
288 except (KeyError, ValueError), e:
289 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
290 continue
291
292 relations_cache = (poel, errors)
293 return relations_cache
294
295
296
297def valid_addr(addr):
298 """ Show which address is valid in which are not """
299 return str(addr).startswith('172.')
300
301def get_hostlist():
302 """ Combined hosts and proxy list"""
303 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
304
305def angle_between_points(lat1,lat2,long1,long2):
306 """
307 Return Angle in radians between two GPS coordinates
308 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
309 """
310 dy = lat2 - lat1
311 dx = math.cos(lat1)*(long2 - long1)
312 angle = math.atan2(dy,dx)
313 return angle
314
315
316
317def angle_to_cd(angle):
318 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
319
320 # For easy conversion get positive degree
321 degrees = math.degrees(angle)
322 abs_degrees = 360 + degrees if degrees < 0 else degrees
323
324 # Numbers can be confusing calculate from the 4 main directions
325 p = 22.5
326 if abs_degrees < p:
327 cd = "n"
328 elif abs_degrees < (90 - p):
329 cd = "no"
330 elif abs_degrees < (90 + p):
331 cd = "o"
332 elif abs_degrees < (180 - p):
333 cd = "zo"
334 elif abs_degrees < (180 + p):
335 cd = "z"
336 elif abs_degrees < (270 - p):
337 cd = "zw"
338 elif abs_degrees < (270 + p):
339 cd = "w"
340 elif abs_degrees < (360 - p):
341 cd = "nw"
342 else:
343 cd = "n"
344 return cd
345
346
347
348def cd_between_hosts(hostA, hostB, datadumps):
349 # Using RDNAP coordinates
350 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
351 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
352 return angle_to_cd(math.atan2(dx,dy))
353
354 # GPS coordinates seems to fail somehow
355 #latA = float(datadumps[hostA]['latitude'])
356 #latB = float(datadumps[hostB]['latitude'])
357 #lonA = float(datadumps[hostA]['longitude'])
358 #lonB = float(datadumps[hostB]['longitude'])
359 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
360
361
362def generate_title(nodelist):
363 """ Main overview page """
364 items = {'root' : "." }
365 def fl(spaces, line):
366 return (' ' * spaces) + line + '\n'
367
368 output = """
369<html>
370 <head>
371 <title>Wireless leiden Configurator - GFormat</title>
372 <style type="text/css">
373 th {background-color: #999999}
374 tr:nth-child(odd) {background-color: #cccccc}
375 tr:nth-child(even) {background-color: #ffffff}
376 th, td {padding: 0.1em 1em}
377 </style>
378 </head>
379 <body>
380 <center>
381 <form type="GET" action="%(root)s">
382 <input type="hidden" name="action" value="update">
383 <input type="submit" value="Update Configuration Database (SVN)">
384 </form>
385 <table>
386 <caption><h3>Wireless Leiden Configurator</h3></caption>
387 """ % items
388
389 for node in nodelist:
390 items['node'] = node
391 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
392 for config in files:
393 items['config'] = config
394 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
395 output += fl(5, "</tr>")
396 output += """
397 </table>
398 <hr />
399 <em>%s</em>
400 </center>
401 </body>
402</html>
403 """ % __version__
404
405 return output
406
407
408
409def generate_node(node):
410 """ Print overview of all files available for node """
411 return "\n".join(files)
412
413def generate_node_overview(host):
414 """ Print overview of all files available for node """
415 datadump = get_yaml(host)
416 params = { 'host' : host }
417 output = "<em><a href='..'>Back to overview</a></em><hr />"
418 output += "<h2>Available files:</h2><ul>"
419 for cf in files:
420 params['cf'] = cf
421 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
422 output += "</ul>"
423
424 # Generate and connection listing
425 output += "<h2>Connected To:</h2><ul>"
426 (poel, errors) = make_relations()
427 for network, hosts in poel.iteritems():
428 if host in [x[0] for x in hosts]:
429 if len(hosts) == 1:
430 # Single not connected interface
431 continue
432 for remote,ifacedump in hosts:
433 if remote == host:
434 # This side of the interface
435 continue
436 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
437 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
438 output += "</ul>"
439 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
440
441 output += "<hr /><em><a href='..'>Back to overview</a></em>"
442 return output
443
444
445def generate_header(datadump, ctag="#"):
446 return """\
447%(ctag)s
448%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
449%(ctag)s
450""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
451
452
453
454def parseaddr(s):
455 """ Process IPv4 CIDR notation addr to a (binary) number """
456 f = s.split('.')
457 return (long(f[0]) << 24L) + \
458 (long(f[1]) << 16L) + \
459 (long(f[2]) << 8L) + \
460 long(f[3])
461
462
463
464def showaddr(a):
465 """ Display IPv4 addr in (dotted) CIDR notation """
466 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
467
468
469def is_member(ip, mask, canidate):
470 """ Return True if canidate is part of ip/mask block"""
471 ip_addr = parseaddr(ip)
472 ip_canidate = parseaddr(canidate)
473 mask = int(mask)
474 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
475 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
476 return ip_addr == ip_canidate
477
478
479
480def cidr2netmask(netmask):
481 """ Given a 'netmask' return corresponding CIDR """
482 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
483
484def get_network(addr, mask):
485 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
486
487
488def generate_dhcpd_conf(datadump):
489 """ Generate config file '/usr/local/etc/dhcpd.conf """
490 output = generate_header(datadump)
491 output += Template("""\
492# option definitions common to all supported networks...
493option domain-name "dhcp.{{ autogen_fqdn }}";
494
495default-lease-time 600;
496max-lease-time 7200;
497
498# Use this to enble / disable dynamic dns updates globally.
499#ddns-update-style none;
500
501# If this DHCP server is the official DHCP server for the local
502# network, the authoritative directive should be uncommented.
503authoritative;
504
505# Use this to send dhcp log messages to a different log file (you also
506# have to hack syslog.conf to complete the redirection).
507log-facility local7;
508
509#
510# Interface definitions
511#
512\n""").render(datadump)
513
514 dhcp_out = defaultdict(list)
515 for iface_key in get_interface_keys(datadump):
516 ifname = datadump[iface_key]['autogen_ifname']
517 if not datadump[iface_key].has_key('comment'):
518 datadump[iface_key]['comment'] = None
519 dhcp_out[ifname].append(" ## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key])
520
521 (addr, mask) = datadump[iface_key]['ip'].split('/')
522 datadump[iface_key]['autogen_addr'] = addr
523 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
524 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
525 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
526 dhcp_out[ifname].append(" subnet %(autogen_subnet)s netmask %(autogen_netmask)s {\n ### not autoritive\n }\n" % \
527 datadump[iface_key])
528 continue
529
530 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
531 dhcp_part = ".".join(addr.split('.')[0:3])
532 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
533 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
534
535 # Assume the first 10 IPs could be used for static entries
536 if 'no_portal' in datadump:
537 fixed = 5
538 for mac in datadump['no_portal']:
539 dhcp_out[ifname].append("""\
540 host fixed-%(ifname)s-%(fixed)s {
541 hardware ethernet %(mac)s;
542 fixed-address %(prefix)s.%(fixed)s;
543 }
544""" % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed })
545 fixed += 1
546
547 dhcp_out[ifname].append("""\
548 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
549 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
550 option routers %(autogen_addr)s;
551 option domain-name-servers %(autogen_addr)s;
552 }
553""" % datadump[iface_key])
554
555 for ifname,value in dhcp_out.iteritems():
556 output += ("shared-network %s {\n" % ifname) + ''.join(value) + '}\n\n'
557 return output
558
559
560
561def generate_dnsmasq_conf(datadump):
562 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
563 output = generate_header(datadump)
564 output += Template("""\
565# DHCP server options
566dhcp-authoritative
567dhcp-fqdn
568domain=dhcp.{{ autogen_fqdn }}
569domain-needed
570expand-hosts
571log-async=100
572
573# Low memory footprint
574cache-size=10000
575
576\n""").render(datadump)
577
578 for iface_key in get_interface_keys(datadump):
579 if not datadump[iface_key].has_key('comment'):
580 datadump[iface_key]['comment'] = None
581 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
582
583 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
584 output += "# not autoritive\n\n"
585 continue
586
587 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
588 (ip, cidr) = datadump[iface_key]['ip'].split('/')
589 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
590
591 dhcp_part = ".".join(ip.split('.')[0:3])
592 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
593 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
594 output += "dhcp-range=%(autogen_ifname)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
595
596 return output
597
598
599def make_interface_list(datadump):
600 if interface_list_cache.has_key(datadump['autogen_item']):
601 return (interface_list_cache[datadump['autogen_item']])
602 # lo0 configuration:
603 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
604 # - masterip is special as it needs to be assigned to at
605 # least one interface, so if not used assign to lo0
606 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
607 vlan_list = defaultdict(list)
608 dhclient_if = {'lo0' : False}
609
610 # XXX: Find some way of send this output nicely
611 output = ''
612
613 masterip_used = False
614 for iface_key in get_interface_keys(datadump):
615 if datadump[iface_key]['ip'].startswith(datadump['masterip']):
616 masterip_used = True
617 break
618 if not masterip_used:
619 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
620
621 for iface_key in get_interface_keys(datadump):
622 ifacedump = datadump[iface_key]
623 ifname = ifacedump['autogen_ifname']
624
625 # If defined as vlan interface
626 if ifacedump['autogen_vlan']:
627 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
628
629 # Flag dhclient is possible
630 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
631 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
632
633 # Add interface IP to list
634 item = (ifacedump['ip'], ifacedump['comment'])
635 if addrs_list.has_key(ifname):
636 addrs_list[ifname].append(item)
637 else:
638 addrs_list[ifname] = [item]
639
640 # Alias only needs IP assignment for now, this might change if we
641 # are going to use virtual accesspoints
642 if "alias" in iface_key:
643 continue
644
645 # XXX: Might want to deduct type directly from interface name
646 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
647 # Default to station (client) mode
648 ifacedump['autogen_wlanmode'] = "sta"
649 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
650 ifacedump['autogen_wlanmode'] = "ap"
651
652 if not ifacedump.has_key('channel'):
653 if ifacedump['type'] == '11a':
654 ifacedump['channel'] = 36
655 else:
656 ifacedump['channel'] = 1
657
658 # Allow special hacks at the back like wds and stuff
659 if not ifacedump.has_key('extra'):
660 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
661 else:
662 ifacedump['autogen_extra'] = ifacedump['extra']
663
664 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
665 output += ("create_args_%(autogen_ifname)s='wlanmode %(autogen_wlanmode)s mode " +\
666 "%(type)s ssid %(ssid)s %(autogen_extra)s channel %(channel)s'\n") % ifacedump
667 output += "\n"
668
669 elif ifacedump['type'] in ['ethernet', 'eth']:
670 # No special config needed besides IP
671 if ifacedump['autogen_bridge']:
672 output += "cloned_interfaces='%(autogen_ifname)s'\n" % ifacedump
673 output += "ifconfig_%s='addm %s up'\n" % (ifacedump['autogen_ifname'], ' addm '.join(ifacedump['autogen_bridge_interfaces']))
674 for member in ifacedump['autogen_bridge_interfaces']:
675 output += "ifconfig_%s='up'\n" % member
676 output += "\n"
677 else:
678 assert False, "Unknown type " + ifacedump['type']
679
680 store = (addrs_list, vlan_list, dhclient_if, output)
681 interface_list_cache[datadump['autogen_item']] = store
682 return(store)
683
684
685
686def generate_rc_conf_local(datadump):
687 """ Generate configuration file '/etc/rc.conf.local' """
688 item = datadump['autogen_item']
689 if rc_conf_local_cache.has_key(item):
690 return rc_conf_local_cache[item]
691
692 if not datadump.has_key('ileiden'):
693 datadump['autogen_ileiden_enable'] = False
694 else:
695 datadump['autogen_ileiden_enable'] = datadump['ileiden']
696
697 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
698
699 if not ileiden_proxies or not normal_proxies:
700 for host in get_hostlist():
701 hostdump = get_yaml(host)
702 if hostdump['status'] == 'up':
703 if hostdump['service_proxy_ileiden']:
704 ileiden_proxies.append(hostdump)
705 if hostdump['service_proxy_normal']:
706 normal_proxies.append(hostdump)
707
708 datadump['autogen_ileiden_proxies'] = ileiden_proxies
709 datadump['autogen_normal_proxies'] = normal_proxies
710 datadump['autogen_ileiden_proxies_ips'] = ','.join([x['masterip'] for x in ileiden_proxies])
711 datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
712 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
713 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
714 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
715 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
716
717 output = generate_header(datadump, "#");
718 output += render_template(datadump, """\
719hostname='{{ autogen_fqdn }}'
720location='{{ location }}'
721nodetype="{{ nodetype }}"
722
723#
724# Configured listings
725#
726captive_portal_whitelist=""
727{% if nodetype == "Proxy" %}
728#
729# Proxy Configuration
730#
731{% if gateway and service_proxy_ileiden -%}
732defaultrouter="{{ gateway }}"
733{% else -%}
734#defaultrouter="NOTSET"
735{% endif -%}
736internalif="{{ internalif }}"
737ileiden_enable="{{ autogen_ileiden_enable }}"
738gateway_enable="{{ autogen_ileiden_enable }}"
739pf_enable="yes"
740pf_rules="/etc/pf.conf"
741{% if autogen_ileiden_enable -%}
742pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
743lvrouted_enable="{{ autogen_ileiden_enable }}"
744lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
745{% else -%}
746pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
747{% endif -%}
748{% if internalroute -%}
749static_routes="wleiden"
750route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
751{% endif -%}
752
753{% elif nodetype == "Hybrid" %}
754 #
755 # Hybrid Configuration
756 #
757 list_ileiden_proxies="
758 {% for item in autogen_ileiden_proxies -%}
759 {{ "%-16s"|format(item.masterip) }} # {{ item.autogen_realname }}
760 {% endfor -%}
761 "
762 list_normal_proxies="
763 {% for item in autogen_normal_proxies -%}
764 {{ "%-16s"|format(item.masterip) }} # {{ item.autogen_realname }}
765 {% endfor -%}
766 "
767
768 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',')|default('none', true) }}"
769 externalif="{{ externalif|default('vr0', true) }}"
770 masterip="{{ masterip }}"
771
772 {% if gateway and service_proxy_ileiden %}
773 defaultrouter="{{ gateway }}"
774 {% else %}
775 #defaultrouter="NOTSET"
776 {% endif %}
777
778 #
779 # Defined services
780 #
781 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
782 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
783 service_accesspoint="{{ service_accesspoint|yesorno }}"
784 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
785 service_concentrator="{{ service_concentrator|yesorno }}"
786
787 {% if service_proxy_ileiden %}
788 pf_rules="/etc/pf.hybrid.conf"
789 {% if service_concentrator %}
790 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
791 {% else %}
792 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
793 {% endif %}
794 pf_flags="$pf_flags -D publicnat=80,443"
795 lvrouted_flags="$lvrouted_flags -g"
796 {% elif service_proxy_normal or service_incoming_rdr %}
797 pf_rules="/etc/pf.hybrid.conf"
798 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
799 pf_flags="$pf_flags -D publicnat=0"
800 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
801 named_setfib="1"
802 tinyproxy_setfib="1"
803 dnsmasq_setfib="1"
804 sshd_setfib="1"
805 {% else %}
806 named_auto_forward_only="YES"
807 pf_rules="/etc/pf.node.conf"
808 pf_flags=""
809 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
810 {% endif %}
811 {% if service_concentrator %}
812 # Do mind installing certificates is NOT done automatically for security reasons
813 openvpn_enable="YES"
814 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
815 {% endif %}
816
817 {% if service_proxy_normal %}
818 tinyproxy_enable="yes"
819 {% else %}
820 pen_wrapper_enable="yes"
821 {% endif %}
822
823 {% if service_accesspoint %}
824 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
825 {% endif %}
826
827 {% if board == "ALIX2" %}
828 #
829 # ''Fat'' configuration, board has 256MB RAM
830 #
831 dnsmasq_enable="NO"
832 named_enable="YES"
833 {% if autogen_dhcp_interfaces -%}
834 dhcpd_enable="YES"
835 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
836 {% endif -%}
837 {% endif -%}
838{% elif nodetype == "CNode" %}
839#
840# NODE iLeiden Configuration
841#
842
843# iLeiden Proxies {{ autogen_ileiden_proxies_names }}
844list_ileiden_proxies="{{ autogen_ileiden_proxies_ips }}"
845# normal Proxies {{ autogen_normal_proxies_names }}
846list_normal_proxies="{{ autogen_normal_proxies_ips }}"
847
848captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
849
850lvrouted_flags="-u -s s00p3rs3kr3t -m 28 -z $list_ileiden_proxies"
851{% endif %}
852
853#
854# Script variables
855#
856attached_devices="{{ autogen_attached_devices|join(' ') }}"
857neighbours="{{ autogen_neighbours|join(' ') }}"
858
859
860#
861# Interface definitions
862#\n
863""")
864
865 (addrs_list, vlan_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
866 for iface, vlans in vlan_list.items():
867 output += 'vlans_%s="%s"\n' % (iface, ' '.join(vlans))
868
869 # Details like SSID
870 if extra_ouput:
871 output += extra_ouput.strip() + "\n"
872
873 # Print IP address which needs to be assigned over here
874 output += "\n"
875 for iface,addrs in sorted(addrs_list.iteritems()):
876 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
877 output += "# %s || %s || %s\n" % (iface, addr, comment)
878
879 # Write DHCLIENT entry
880 if dhclient_if[iface]:
881 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
882
883 # Make sure the external address is always first as this is needed in the
884 # firewall setup
885 addrs = sorted(
886 [x for x in addrs if not '0.0.0.0' in x[0]],
887 key=lambda x: x[0].split('.')[0],
888 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
889 )
890 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
891 for idx, addr in enumerate(addrs[1:]):
892 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx, addr[0])
893 output += "\n"
894
895 rc_conf_local_cache[datadump['autogen_item']] = output
896 return output
897
898
899
900def get_all_configs():
901 """ Get dict with key 'host' with all configs present """
902 configs = dict()
903 for host in get_hostlist():
904 datadump = get_yaml(host)
905 configs[host] = datadump
906 return configs
907
908
909def get_interface_keys(config, extra=False):
910 """ Quick hack to get all interface keys, later stage convert this to a iterator """
911 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
912 if extra == False:
913 return filter(lambda x: not "extra" in x, elems)
914 else:
915 return elems
916
917
918def get_used_ips(configs):
919 """ Return array of all IPs used in config files"""
920 ip_list = []
921 for config in configs:
922 ip_list.append(config['masterip'])
923 for iface_key in get_interface_keys(config, True):
924 l = config[iface_key]['ip']
925 addr, mask = l.split('/')
926 # Special case do not process
927 if valid_addr(addr):
928 ip_list.append(addr)
929 else:
930 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
931 return sorted(ip_list)
932
933
934
935def get_nameservers(max_servers=None):
936 if nameservers_cache:
937 return nameservers_cache[0:max_servers]
938
939 for host in get_hostlist():
940 hostdump = get_yaml(host)
941 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
942 nameservers_cache.append((hostdump['masterip'], hostdump['autogen_realname']))
943
944 return nameservers_cache[0:max_servers]
945
946
947def get_neighbours(datadump):
948 (addrs_list, _, dhclient_if, extra_ouput) = make_interface_list(datadump)
949
950 (poel, errors) = make_relations()
951 table = []
952 for iface,addrs in sorted(addrs_list.iteritems()):
953 if iface in ['lo0']:
954 continue
955
956 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
957 if not addr.startswith('172.'):
958 # Avoid listing internet connections as pool
959 continue
960 for neighbour in poel[network(addr)]:
961 if neighbour[0] != datadump['autogen_item']:
962 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_ifname'] + ")", neighbour[1]['comment']))
963 return table
964
965
966def get_attached_devices(datadump, url=False):
967 table = []
968 for iface_key in get_interface_keys(datadump, True):
969 # Quick to avoid listing ath(4) interface as attached device
970 if 'ath0' in iface_key:
971 continue
972 ifacedump = datadump[iface_key]
973 if ifacedump.has_key('ns_ip'):
974 x_ip = ifacedump['ns_ip'].split('/')[0]
975 else:
976 x_ip = ifacedump['ip'].split('/')[0]
977
978 if 'mode' in ifacedump:
979 x_mode = ifacedump['mode']
980 else:
981 x_mode = 'unknown'
982
983 if 'bridge_type' in ifacedump:
984 device_type = ifacedump['bridge_type']
985 else:
986 device_type = 'Unknown'
987
988 table.append((ifacedump['autogen_ifname'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
989 return table
990
991
992def generate_resolv_conf(datadump):
993 """ Generate configuration file '/etc/resolv.conf' """
994 # XXX: This should properly going to be an datastructure soon
995 datadump['autogen_header'] = generate_header(datadump, "#")
996 datadump['autogen_edge_nameservers'] = ''
997
998
999 for masterip,realname in get_nameservers():
1000 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1001
1002 return Template("""\
1003{{ autogen_header }}
1004search wleiden.net
1005
1006# Try local (cache) first
1007nameserver 127.0.0.1
1008
1009{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1010nameserver 8.8.8.8 # Google Public NameServer
1011nameserver 4.2.2.1 # Level3 Public NameServer
1012{% else -%}
1013# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1014{{ autogen_edge_nameservers }}
1015{% endif -%}
1016""").render(datadump)
1017
1018
1019
1020def generate_ntp_conf(datadump):
1021 """ Generate configuration file '/etc/ntp.conf' """
1022 # XXX: This should properly going to be an datastructure soon
1023
1024 datadump['autogen_header'] = generate_header(datadump, "#")
1025 datadump['autogen_ntp_servers'] = ''
1026 for host in get_hostlist():
1027 hostdump = get_yaml(host)
1028 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1029 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(autogen_realname)s\n" % hostdump
1030
1031 return Template("""\
1032{{ autogen_header }}
1033
1034{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1035# Machine hooked to internet.
1036server 0.nl.pool.ntp.org iburst maxpoll 9
1037server 1.nl.pool.ntp.org iburst maxpoll 9
1038server 2.nl.pool.ntp.org iburst maxpoll 9
1039server 3.nl.pool.ntp.org iburst maxpoll 9
1040{% else -%}
1041# Local Wireless Leiden NTP Servers.
1042server 0.pool.ntp.wleiden.net iburst maxpoll 9
1043server 1.pool.ntp.wleiden.net iburst maxpoll 9
1044server 2.pool.ntp.wleiden.net iburst maxpoll 9
1045server 3.pool.ntp.wleiden.net iburst maxpoll 9
1046
1047# All the configured NTP servers
1048{{ autogen_ntp_servers }}
1049{% endif %}
1050
1051# If a server loses sync with all upstream servers, NTP clients
1052# no longer follow that server. The local clock can be configured
1053# to provide a time source when this happens, but it should usually
1054# be configured on just one server on a network. For more details see
1055# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1056# The use of Orphan Mode may be preferable.
1057#
1058server 127.127.1.0
1059fudge 127.127.1.0 stratum 10
1060""").render(datadump)
1061
1062
1063def generate_pf_hybrid_conf_local(datadump):
1064 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1065 datadump['autogen_header'] = generate_header(datadump, "#")
1066 return Template("""\
1067{{ autogen_header }}
1068
1069# Redirect some internal facing services outside (7)
1070# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1071{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1072rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1073{% endfor -%}
1074""").render(datadump)
1075
1076def generate_motd(datadump):
1077 """ Generate configuration file '/etc/motd' """
1078 output = Template("""\
1079FreeBSD run ``service motd onestart'' to make me look normal
1080
1081 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1082 Loc: {{ location }}
1083
1084Services:
1085{% if board == "ALIX2" -%}
1086{{" -"}} Core Node ({{ board }})
1087{% else -%}
1088{{" -"}} Hulp Node ({{ board }})
1089{% endif -%}
1090{% if service_proxy_normal -%}
1091{{" -"}} Normal Proxy
1092{% endif -%}
1093{% if service_proxy_ileiden -%}
1094{{" -"}} iLeiden Proxy
1095{% endif -%}
1096{% if service_incoming_rdr -%}
1097{{" -"}} Incoming port redirects
1098{% endif %}
1099Interlinks:\n
1100""").render(datadump)
1101
1102
1103 def make_table(table):
1104 if not table:
1105 return " - none\n"
1106 else:
1107 lines = ""
1108 col_width = [max(len(x) for x in col) for col in zip(*table)]
1109 for row in table:
1110 lines += " - " + " || ".join("{:{}}".format(x, col_width[i]) for i, x in enumerate(row)) + "\n"
1111 return lines
1112
1113 (addrs_list, vlan_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
1114 table = []
1115 for iface,addrs in sorted(addrs_list.iteritems()):
1116 if iface in ['lo0']:
1117 continue
1118 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1119 table.append((iface, addr, comment))
1120
1121 output += make_table(table)
1122 output += '\n'
1123 output += """\
1124Attached devices:
1125"""
1126 output += make_table(get_attached_devices(datadump, url=True))
1127 output += '\n'
1128 output += """\
1129Available neighbours:
1130"""
1131 output += make_table(get_neighbours(datadump))
1132
1133 return output
1134
1135
1136def format_yaml_value(value):
1137 """ Get yaml value in right syntax for outputting """
1138 if isinstance(value,str):
1139 output = '"%s"' % value
1140 else:
1141 output = value
1142 return output
1143
1144
1145
1146def format_wleiden_yaml(datadump):
1147 """ Special formatting to ensure it is editable"""
1148 output = "# Genesis config yaml style\n"
1149 output += "# vim:ts=2:et:sw=2:ai\n"
1150 output += "#\n"
1151 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1152 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1153 if key == 'rdr_rules':
1154 output += '%-10s:\n' % 'rdr_rules'
1155 for rdr_rule in datadump[key]:
1156 output += '- %s\n' % rdr_rule
1157 else:
1158 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1159
1160 output += "\n\n"
1161
1162 # Format (key, required)
1163 key_order = (
1164 ('comment', True),
1165 ('ip', True),
1166 ('desc', True),
1167 ('sdesc', True),
1168 ('mode', True),
1169 ('type', True),
1170 ('extra_type', False),
1171 ('channel', False),
1172 ('ssid', False),
1173 ('wlan_mac', False),
1174 ('dhcp', True),
1175 ('compass', False),
1176 ('distance', False),
1177 ('ns_ip', False),
1178 ('repeater_ip', False),
1179 ('bullet2_ip', False),
1180 ('ns_mac', False),
1181 ('bullet2_mac', False),
1182 ('ns_type', False),
1183 ('bridge_type', False),
1184 ('members', True),
1185 ('status', True),
1186 )
1187
1188 for iface_key in sorted(iface_keys):
1189 try:
1190 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1191 if remainder:
1192 raise KeyError("invalid keys: %s" % remainder)
1193
1194 output += "%s:\n" % iface_key
1195 for key,required in key_order:
1196 if datadump[iface_key].has_key(key):
1197 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1198 output += "\n\n"
1199 except Exception:
1200 print "# Error while processing interface %s" % iface_key
1201 raise
1202
1203 return output
1204
1205
1206
1207def generate_wleiden_yaml(datadump, header=True):
1208 """ Generate (petty) version of wleiden.yaml"""
1209 output = generate_header(datadump, "#") if header else ''
1210
1211 for key in datadump.keys():
1212 if key.startswith('autogen_'):
1213 del datadump[key]
1214 # Interface autogen cleanups
1215 elif type(datadump[key]) == dict:
1216 for key2 in datadump[key].keys():
1217 if key2.startswith('autogen_'):
1218 del datadump[key][key2]
1219
1220 output += format_wleiden_yaml(datadump)
1221 return output
1222
1223def generate_nanostation_config(datadump, iface, ns_type):
1224 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1225 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1226
1227 datadump.update(datadump['iface_%s' % iface])
1228
1229 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1230
1231def generate_yaml(datadump):
1232 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1233
1234
1235
1236def generate_config(node, config, datadump=None):
1237 """ Print configuration file 'config' of 'node' """
1238 output = ""
1239 try:
1240 # Load config file
1241 if datadump == None:
1242 datadump = get_yaml(node)
1243
1244 if config == 'wleiden.yaml':
1245 output += generate_wleiden_yaml(datadump)
1246 elif config == 'authorized_keys':
1247 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1248 output += f.read()
1249 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1250 # Fetch local keys if existing
1251 if os.path.exists(node_keys):
1252 output += open(node_keys, 'r').read()
1253 f.close()
1254 elif config == 'dnsmasq.conf':
1255 output += generate_dnsmasq_conf(datadump)
1256 elif config == 'dhcpd.conf':
1257 output += generate_dhcpd_conf(datadump)
1258 elif config == 'rc.conf.local':
1259 output += generate_rc_conf_local(datadump)
1260 elif config == 'resolv.conf':
1261 output += generate_resolv_conf(datadump)
1262 elif config == 'ntp.conf':
1263 output += generate_ntp_conf(datadump)
1264 elif config == 'motd':
1265 output += generate_motd(datadump)
1266 elif config == 'pf.hybrid.conf.local':
1267 output += generate_pf_hybrid_conf_local(datadump)
1268 elif config.startswith('vr'):
1269 interface, ns_type = config.strip('.yaml').split('-')
1270 output += generate_nanostation_config(datadump, interface, ns_type)
1271 else:
1272 assert False, "Config not found!"
1273 except IOError, e:
1274 output += "[ERROR] Config file not found"
1275 return output
1276
1277
1278
1279def process_cgi_request(environ=os.environ):
1280 """ When calling from CGI """
1281 response_headers = []
1282 content_type = 'text/plain'
1283
1284 # Update repository if requested
1285 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1286 if form and form.has_key("action") and "update" in form["action"]:
1287 output = "[INFO] Updating subverion, please wait...\n"
1288 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1289 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1290 output += "[INFO] All done, redirecting in 5 seconds"
1291 response_headers += [
1292 ('Refresh', '5; url=.'),
1293 ]
1294 reload_cache()
1295 else:
1296 base_uri = environ['PATH_INFO']
1297 uri = base_uri.strip('/').split('/')
1298
1299 output = "Template Holder"
1300 if base_uri.endswith('/create/network.kml'):
1301 content_type='application/vnd.google-earth.kml+xml'
1302 output = make_network_kml.make_graph()
1303 elif base_uri.endswith('/api/get/nodeplanner.json'):
1304 content_type='application/json'
1305 output = make_network_kml.make_nodeplanner_json()
1306 elif not uri[0]:
1307 if is_text_request(environ):
1308 output = '\n'.join(get_hostlist())
1309 else:
1310 content_type = 'text/html'
1311 output = generate_title(get_hostlist())
1312 elif len(uri) == 1:
1313 if is_text_request(environ):
1314 output = generate_node(uri[0])
1315 else:
1316 content_type = 'text/html'
1317 output = generate_node_overview(uri[0])
1318 elif len(uri) == 2:
1319 output = generate_config(uri[0], uri[1])
1320 else:
1321 assert False, "Invalid option"
1322
1323 # Return response
1324 response_headers += [
1325 ('Content-type', content_type),
1326 ('Content-Length', str(len(output))),
1327 ]
1328 return(response_headers, str(output))
1329
1330
1331def get_realname(datadump):
1332 # Proxy naming convention is special, as the proxy name is also included in
1333 # the nodename, when it comes to the numbered proxies.
1334 if datadump['nodetype'] == 'Proxy':
1335 realname = datadump['nodetype'] + datadump['nodename'].replace('proxy','')
1336 else:
1337 # By default the full name is listed and also a shortname CNAME for easy use.
1338 realname = datadump['nodetype'] + datadump['nodename']
1339 return(realname)
1340
1341
1342
1343def make_dns(output_dir = 'dns', external = False):
1344 items = dict()
1345
1346 # hostname is key, IP is value
1347 wleiden_zone = defaultdict(list)
1348 wleiden_cname = dict()
1349
1350 pool = dict()
1351 for node in get_hostlist():
1352 datadump = get_yaml(node)
1353
1354 # Proxy naming convention is special
1355 fqdn = datadump['autogen_realname']
1356 if datadump['nodetype'] in ['CNode', 'Hybrid']:
1357 wleiden_cname[datadump['nodename']] = fqdn
1358
1359 if datadump.has_key('rdr_host'):
1360 remote_target = datadump['rdr_host']
1361 elif datadump.has_key('remote_access') and datadump['remote_access']:
1362 remote_target = datadump['remote_access'].split(':')[0]
1363 else:
1364 remote_target = None
1365
1366 if remote_target:
1367 try:
1368 parseaddr(remote_target)
1369 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1370 except (IndexError, ValueError):
1371 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1372
1373
1374 wleiden_zone[fqdn].append((datadump['masterip'], True))
1375
1376 # Hacking to get proper DHCP IPs and hostnames
1377 for iface_key in get_interface_keys(datadump):
1378 iface_name = iface_key.replace('_','-')
1379 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1380 try:
1381 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1382 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1383 dhcp_part = ".".join(ip.split('.')[0:3])
1384 if ip != datadump['masterip']:
1385 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, False))
1386 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1387 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1388 except (AttributeError, ValueError, KeyError):
1389 # First push it into a pool, to indentify the counter-part later on
1390 addr = parseaddr(ip)
1391 cidr = int(cidr)
1392 addr = addr & ~((1 << (32 - cidr)) - 1)
1393 if pool.has_key(addr):
1394 pool[addr] += [(iface_name, fqdn, ip)]
1395 else:
1396 pool[addr] = [(iface_name, fqdn, ip)]
1397 continue
1398
1399
1400
1401 # WL uses an /29 to configure an interface. IP's are ordered like this:
1402 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1403
1404 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1405
1406 # Automatic naming convention of interlinks namely 2 + remote.lower()
1407 for (key,value) in pool.iteritems():
1408 # Make sure they are sorted from low-ip to high-ip
1409 value = sorted(value, key=lambda x: parseaddr(x[2]))
1410
1411 if len(value) == 1:
1412 (iface_name, fqdn, ip) = value[0]
1413 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1414
1415 # Device DNS names
1416 if 'cnode' in fqdn.lower():
1417 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1418 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1419
1420 elif len(value) == 2:
1421 (a_iface_name, a_fqdn, a_ip) = value[0]
1422 (b_iface_name, b_fqdn, b_ip) = value[1]
1423 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1424 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1425
1426 # Device DNS names
1427 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1428 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1429 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1430 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1431 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1432 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1433 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1434
1435 else:
1436 pool_members = [k[1] for k in value]
1437 for item in value:
1438 (iface_name, fqdn, ip) = item
1439 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1440
1441 # Include static DNS entries
1442 # XXX: Should they override the autogenerated results?
1443 # XXX: Convert input to yaml more useable.
1444 # Format:
1445 ##; this is a comment
1446 ## roomburgh=CNodeRoomburgh1
1447 ## apkerk1.CNodeVosko=172.17.176.8 ;this as well
1448 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1449
1450 # Hack to allow special entries, for development
1451 wleiden_raw = {}
1452
1453 for line in dns_list:
1454 reverse = False
1455 k, items = line.items()[0]
1456 if type(items) == dict:
1457 if items.has_key('reverse'):
1458 reverse = items['reverse']
1459 items = items['a']
1460 else:
1461 items = items['cname']
1462 items = [items] if type(items) != list else items
1463 for item in items:
1464 if item.startswith('IN '):
1465 wleiden_raw[k] = item
1466 elif valid_addr(item):
1467 wleiden_zone[k].append((item, reverse))
1468 else:
1469 wleiden_cname[k] = item
1470
1471 # Hack to get dynamic pool listing
1472 def chunks(l, n):
1473 return [l[i:i+n] for i in range(0, len(l), n)]
1474
1475 ntp_servers = [x[0] for x in get_nameservers()]
1476 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1477 for ntp_server in chunk:
1478 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1479
1480 details = dict()
1481 # 24 updates a day allowed
1482 details['serial'] = time.strftime('%Y%m%d%H')
1483
1484 if external:
1485 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1486 else:
1487 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1488
1489 details['master'] = dns_masters[0]
1490 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1491
1492 dns_header = '''
1493$TTL 3h
1494%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1495 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1496
1497%(ns_servers)s
1498 \n'''
1499
1500
1501 if not os.path.isdir(output_dir):
1502 os.makedirs(output_dir)
1503 details['zone'] = 'wleiden.net'
1504 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1505 f.write(dns_header % details)
1506
1507 for host,items in wleiden_zone.iteritems():
1508 for ip,reverse in items:
1509 if ip not in ['0.0.0.0']:
1510 f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
1511 for source,dest in wleiden_cname.iteritems():
1512 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1513 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1514 for source, dest in wleiden_raw.iteritems():
1515 f.write("%s.wleiden.net. %s\n" % (source, dest))
1516 f.close()
1517
1518 # Create whole bunch of specific sub arpa zones. To keep it compliant
1519 for s in range(16,32):
1520 details['zone'] = '%i.172.in-addr.arpa' % s
1521 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1522 f.write(dns_header % details)
1523
1524 #XXX: Not effient, fix to proper data structure and do checks at other
1525 # stages
1526 for host,items in wleiden_zone.iteritems():
1527 for ip,reverse in items:
1528 if not reverse:
1529 continue
1530 if valid_addr(ip):
1531 if valid_addr(ip):
1532 if int(ip.split('.')[1]) == s:
1533 rev_ip = '.'.join(reversed(ip.split('.')))
1534 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1535 f.close()
1536
1537
1538def usage():
1539 print """Usage: %(prog)s <argument>
1540Argument:
1541\tcleanup = Cleanup all YAML files to specified format
1542\tstandalone [port] = Run configurator webserver [8000]
1543\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1544\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1545\tfull-export = Generate yaml export script for heatmap.
1546\tstatic [outputdir] = Generate all config files and store on disk
1547\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1548\ttest <node> [<file>] = Receive output for certain node [all files].
1549\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1550\tlist <status> <items> = List systems which have certain status
1551
1552Arguments:
1553\t<node> = NodeName (example: HybridRick)
1554\t<file> = %(files)s
1555\t<status> = all|up|down|planned
1556\t<items> = systems|nodes|proxies
1557
1558NOTE FOR DEVELOPERS; you can test your changes like this:
1559 BEFORE any changes in this code:
1560 $ ./gformat.py static /tmp/pre
1561 AFTER the changes:
1562 $ ./gformat.py static /tmp/post
1563 VIEW differences and VERIFY all are OK:
1564 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1565""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1566 exit(0)
1567
1568
1569def is_text_request(environ=os.environ):
1570 """ Find out whether we are calling from the CLI or any text based CLI utility """
1571 try:
1572 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1573 except KeyError:
1574 return True
1575
1576def switchFormat(setting):
1577 if setting:
1578 return "YES"
1579 else:
1580 return "NO"
1581
1582def rlinput(prompt, prefill=''):
1583 import readline
1584 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1585 try:
1586 return raw_input(prompt)
1587 finally:
1588 readline.set_startup_hook()
1589
1590def fix_conflict(left, right, default='i'):
1591 while True:
1592 print "## %-30s | %-30s" % (left, right)
1593 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1594 if not c:
1595 c = default
1596
1597 if c in ['l','1']:
1598 return left
1599 elif c in ['r','2']:
1600 return right
1601 elif c in ['e', '3']:
1602 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1603 elif c in ['i', '4']:
1604 return None
1605 else:
1606 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1607
1608
1609
1610def print_cgi_response(response_headers, output):
1611 """Could we not use some kind of wsgi wrapper to make this output?"""
1612 for header in response_headers:
1613 print "%s: %s" % header
1614 print
1615 print output
1616
1617
1618def fill_cache():
1619 ''' Poor man re-loading of few cache items (the slow ones) '''
1620 for host in get_hostlist():
1621 get_yaml(host)
1622
1623
1624def reload_cache():
1625 clear_cache()
1626 fill_cache()
1627
1628
1629def main():
1630 """Hard working sub"""
1631 # Allow easy hacking using the CLI
1632 if not os.environ.has_key('PATH_INFO'):
1633 if len(sys.argv) < 2:
1634 usage()
1635
1636 if sys.argv[1] == "standalone":
1637 import SocketServer
1638 import CGIHTTPServer
1639 # Hop to the right working directory.
1640 os.chdir(os.path.dirname(__file__))
1641 try:
1642 PORT = int(sys.argv[2])
1643 except (IndexError,ValueError):
1644 PORT = 8000
1645
1646 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1647 """ Serve this CGI from the root of the webserver """
1648 def is_cgi(self):
1649 if "favicon" in self.path:
1650 return False
1651
1652 self.cgi_info = (os.path.basename(__file__), self.path)
1653 self.path = ''
1654 return True
1655 handler = MyCGIHTTPRequestHandler
1656 SocketServer.TCPServer.allow_reuse_address = True
1657 httpd = SocketServer.TCPServer(("", PORT), handler)
1658 httpd.server_name = 'localhost'
1659 httpd.server_port = PORT
1660
1661 logger.info("serving at port %s", PORT)
1662 try:
1663 httpd.serve_forever()
1664 except KeyboardInterrupt:
1665 httpd.shutdown()
1666 logger.info("All done goodbye")
1667 elif sys.argv[1] == "test":
1668 # Basic argument validation
1669 try:
1670 node = sys.argv[2]
1671 except IndexError:
1672 print "Invalid argument"
1673 exit(1)
1674 except IOError as e:
1675 print e
1676 exit(1)
1677
1678 datadump = get_yaml(node)
1679
1680
1681 # Get files to generate
1682 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1683
1684 # Actual config generation
1685 for config in gen_files:
1686 logger.info("## Generating %s %s", node, config)
1687 print generate_config(node, config, datadump)
1688 elif sys.argv[1] == "test-cgi":
1689 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1690 os.environ['SCRIPT_NAME'] = __file__
1691 response_headers, output = process_cgi_request()
1692 print_cgi_response(response_headers, output)
1693 elif sys.argv[1] == "static":
1694 items = dict()
1695 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1696 for node in get_hostlist():
1697 items['node'] = node
1698 items['wdir'] = "%(output_dir)s/%(node)s" % items
1699 if not os.path.isdir(items['wdir']):
1700 os.makedirs(items['wdir'])
1701 datadump = get_yaml(node)
1702 for config in files:
1703 items['config'] = config
1704 logger.info("## Generating %(node)s %(config)s" % items)
1705 f = open("%(wdir)s/%(config)s" % items, "w")
1706 f.write(generate_config(node, config, datadump))
1707 f.close()
1708 elif sys.argv[1] == "wind-export":
1709 items = dict()
1710 for node in get_hostlist():
1711 datadump = get_yaml(node)
1712 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1713 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1714 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1715 VALUES (
1716 (SELECT id FROM users WHERE username = 'rvdzwet'),
1717 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1718 'Y');""" % datadump
1719 #for config in files:
1720 # items['config'] = config
1721 # print "## Generating %(node)s %(config)s" % items
1722 # f = open("%(wdir)s/%(config)s" % items, "w")
1723 # f.write(generate_config(node, config, datadump))
1724 # f.close()
1725 for node in get_hostlist():
1726 datadump = get_yaml(node)
1727 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1728 ifacedump = datadump[iface_key]
1729 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1730 ifacedump['nodename'] = datadump['nodename']
1731 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1732 ifacedump['channel'] = 0
1733 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1734 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1735 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1736 elif sys.argv[1] == "nagios-export":
1737 try:
1738 heavy_load = (sys.argv[2] == "--heavy-load")
1739 except IndexError:
1740 heavy_load = False
1741
1742 hostgroup_details = {
1743 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1744 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1745 'walphen' : 'Stichting Wireless Alphen',
1746 'westeinder' : 'Westeinder Plassen',
1747 }
1748
1749 # Convert IP to Host
1750 ip2host = {'root' : 'root'}
1751 for host in get_hostlist():
1752 datadump = get_yaml(host)
1753 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1754 for iface in get_interface_keys(datadump):
1755 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1756
1757 # Find dependency tree based on output of lvrouted.mytree of nearest node
1758 parents = defaultdict(list)
1759 stack = ['root']
1760 prev_depth = 0
1761 for line in open('lvrouted.mytree').readlines():
1762 depth = line.count('\t')
1763 ip = line.strip().split()[0]
1764
1765 if prev_depth < depth:
1766 try:
1767 parents[ip2host[ip]].append(ip2host[stack[-1]])
1768 except KeyError as e:
1769 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1770 stack.append(ip)
1771 elif prev_depth > depth:
1772 stack = stack[:(depth - prev_depth)]
1773 elif prev_depth == depth:
1774 try:
1775 parents[ip2host[ip]].append(ip2host[stack[-1]])
1776 except KeyError as e:
1777 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1778
1779
1780 prev_depth = depth
1781 # Observe that some nodes has themself as parent or multiple parents
1782 # for now take only the first parent, other behaviour is yet to be explained
1783
1784
1785
1786 params = {
1787 'check_interval' : 5 if heavy_load else 120,
1788 'retry_interval' : 1 if heavy_load else 10,
1789 'max_check_attempts' : 10 if heavy_load else 6,
1790 'notification_interval': 120 if heavy_load else 240,
1791 }
1792
1793 print '''\
1794define host {
1795 name wleiden-node ; Default Node Template
1796 use generic-host ; Use the standard template as initial starting point
1797 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1798 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1799 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1800 notification_interval %(notification_interval)s
1801 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1802 check_command check-host-alive ; Default command to check FreeBSD hosts
1803 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1804}
1805
1806define service {
1807 name wleiden-service ; Default Service Template
1808 use generic-service ; Use the standard template as initial starting point
1809 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1810 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1811 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1812 notification_interval %(notification_interval)s
1813 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1814 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1815}
1816
1817# Please make sure to install:
1818# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1819#
1820# Recompile net-mgmt/nagios-plugins to support check_snmp
1821# make -C /usr/ports/net-mgmt/nagios-plugins
1822#
1823# Install net/bind-tools to allow v2/check_dns_wl to work:
1824# pkg install bind-tools
1825#
1826define command{
1827 command_name check_snmp_disk
1828 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1829}
1830
1831define command{
1832 command_name check_netsnmp_load
1833 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1834}
1835
1836define command{
1837 command_name check_netsnmp_proc
1838 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1839}
1840
1841define command{
1842 command_name check_by_ssh
1843 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1844}
1845
1846define command{
1847 command_name check_dns_wl
1848 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1849}
1850
1851define command{
1852 command_name check_snmp_uptime
1853 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1854}
1855
1856
1857# TDB: dhcp leases
1858# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1859
1860# TDB: internet status
1861# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1862
1863# TDB: Advanced local passive checks
1864# /usr/local/libexec/nagios/check_by_ssh
1865''' % params
1866
1867 print '''\
1868# Service Group, not displayed by default
1869define hostgroup {
1870 hostgroup_name srv_hybrid
1871 alias All Hybrid Nodes
1872 register 0
1873}
1874
1875define service {
1876 use wleiden-service
1877 hostgroup_name srv_hybrid
1878 service_description SSH
1879 check_command check_ssh
1880}
1881
1882define service {
1883 use wleiden-service,service-pnp
1884 hostgroup_name srv_hybrid
1885 service_description HTTP
1886 check_command check_http
1887}
1888
1889define service {
1890 use wleiden-service
1891 hostgroup_name srv_hybrid
1892 service_description DNS
1893 check_command check_dns_wl!"www.wirelessleiden.nl"
1894}
1895'''
1896
1897 if heavy_load:
1898 print '''\
1899define service {
1900 use wleiden-service
1901 hostgroup_name srv_hybrid
1902 service_description UPTIME
1903 check_command check_snmp_uptime
1904}
1905
1906#define service {
1907# use wleiden-service
1908# hostgroup_name srv_hybrid
1909# service_description NTP
1910# check_command check_ntp_peer
1911#}
1912
1913define service {
1914 use wleiden-service
1915 hostgroup_name srv_hybrid
1916 service_description LOAD
1917 check_command check_netsnmp_load
1918}
1919
1920define service {
1921 use wleiden-service
1922 hostgroup_name srv_hybrid
1923 service_description PROC
1924 check_command check_netsnmp_proc
1925}
1926
1927define service {
1928 use wleiden-service
1929 hostgroup_name srv_hybrid
1930 service_description DISK
1931 check_command check_snmp_disk
1932}
1933'''
1934 for node in get_hostlist():
1935 datadump = get_yaml(node)
1936 if not datadump['status'] == 'up':
1937 continue
1938 if not hostgroup_details.has_key(datadump['monitoring_group']):
1939 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
1940 print '''\
1941define host {
1942 use wleiden-node,host-pnp
1943 contact_groups admins
1944 host_name %(autogen_fqdn)s
1945 address %(masterip)s
1946 hostgroups srv_hybrid,%(monitoring_group)s\
1947''' % datadump
1948 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
1949 print '''\
1950 parents %(parents)s\
1951''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
1952 print '''\
1953}
1954'''
1955
1956
1957 for name,alias in hostgroup_details.iteritems():
1958 print '''\
1959define hostgroup {
1960 hostgroup_name %s
1961 alias %s
1962} ''' % (name, alias)
1963
1964
1965 elif sys.argv[1] == "full-export":
1966 hosts = {}
1967 for node in get_hostlist():
1968 datadump = get_yaml(node)
1969 hosts[datadump['nodename']] = datadump
1970 print yaml.dump(hosts)
1971
1972 elif sys.argv[1] == "dns":
1973 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
1974 elif sys.argv[1] == "cleanup":
1975 # First generate all datadumps
1976 datadumps = dict()
1977 ssid_to_node = dict()
1978 for host in get_hostlist():
1979 logger.info("# Processing: %s", host)
1980 # Set some boring default values
1981 datadump = { 'board' : 'UNKNOWN' }
1982 datadump.update(get_yaml(host))
1983 datadumps[datadump['autogen_realname']] = datadump
1984
1985 (poel, errors) = make_relations()
1986 print "\n".join(["# WARNING: %s" % x for x in errors])
1987
1988 for host,datadump in datadumps.iteritems():
1989 try:
1990 # Convert all yes and no to boolean values
1991 def fix_boolean(dump):
1992 for key in dump.keys():
1993 if type(dump[key]) == dict:
1994 dump[key] = fix_boolean(dump[key])
1995 elif str(dump[key]).lower() in ["yes", "true"]:
1996 dump[key] = True
1997 elif str(dump[key]).lower() in ["no", "false"]:
1998 # Compass richting no (Noord Oost) is valid input
1999 if key != "compass": dump[key] = False
2000 return dump
2001 datadump = fix_boolean(datadump)
2002
2003 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2004 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2005 elif 'latitude' in datadump and 'longitude' in datadump:
2006 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2007
2008 if datadump['nodename'].startswith('Proxy'):
2009 datadump['nodename'] = datadump['nodename'].lower()
2010
2011 for iface_key in get_interface_keys(datadump):
2012 try:
2013 # All our normal wireless cards are normal APs now
2014 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2015 datadump[iface_key]['mode'] = 'ap'
2016 # Wireless Leiden SSID have an consistent lowercase/uppercase
2017 if datadump[iface_key].has_key('ssid'):
2018 ssid = datadump[iface_key]['ssid']
2019 prefix = 'ap-WirelessLeiden-'
2020 if ssid.lower().startswith(prefix.lower()):
2021 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2022 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2023 datadump[iface_key]['mode'] = 'autogen-FIXME'
2024 if not datadump[iface_key].has_key('comment'):
2025 datadump[iface_key]['comment'] = 'autogen-FIXME'
2026
2027 if datadump[iface_key].has_key('ns_mac'):
2028 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2029
2030 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2031 datadump[iface_key] = datadump[iface_key]['desc']
2032
2033 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2034 # due to better collision avoidance.
2035 if datadump[iface_key]['type'] == '11b':
2036 datadump[iface_key]['type'] = '11g'
2037
2038 # Setting 802.11g channels to de-facto standards, to avoid
2039 # un-detected sharing with other overlapping channels
2040 #
2041 # Technically we could also use channel 13 in NL, but this is not
2042 # recommended as foreign devices might not be able to select this
2043 # channel. Secondly using 1,5,9,13 instead is going to clash with
2044 # the de-facto usage of 1,6,11.
2045 #
2046 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2047 channels_at_2400Mhz = (1,6,11)
2048 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2049 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2050 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2051 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2052
2053 # Mandatory interface keys
2054 if not datadump[iface_key].has_key('status'):
2055 datadump[iface_key]['status'] = 'planned'
2056
2057 x = datadump[iface_key]['comment']
2058 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2059
2060 # Fixing bridge_type if none is found
2061 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2062 if not 'bridge_type' in datadump[iface_key]:
2063 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2064
2065 # Making sure description works
2066 if datadump[iface_key].has_key('desc'):
2067 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2068 del datadump[iface_key]['desc']
2069 else:
2070 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2071 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2072 if response:
2073 datadump[iface_key]['comment'] = response
2074 del datadump[iface_key]['desc']
2075
2076 # Check DHCP configuration
2077 dhcp_type(datadump[iface_key])
2078
2079 # Set the compass value based on the angle between the poels
2080 if datadump[iface_key].has_key('ns_ip'):
2081 my_pool = poel[network(datadump[iface_key]['ip'])]
2082 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2083 if remote_hosts:
2084 compass_target = remote_hosts[0]
2085 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2086
2087 # Monitoring Group default
2088 if not 'monitoring_group' in datadump:
2089 datadump['monitoring_group'] = 'wleiden'
2090
2091 except Exception:
2092 print "# Error while processing interface %s" % iface_key
2093 raise
2094 store_yaml(datadump)
2095 except Exception:
2096 print "# Error while processing %s" % host
2097 raise
2098 elif sys.argv[1] == "list":
2099 use_fqdn = False
2100 if len(sys.argv) < 4:
2101 usage()
2102 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2103 usage()
2104 if not sys.argv[3] in ["nodes","proxies","systems"]:
2105 usage()
2106
2107 if len(sys.argv) > 4:
2108 if sys.argv[4] == "fqdn":
2109 use_fqdn = True
2110 else:
2111 usage()
2112
2113 for system in get_hostlist():
2114 datadump = get_yaml(system)
2115 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2116 continue
2117
2118 output = datadump['autogen_fqdn'] if use_fqdn else system
2119 if sys.argv[2] == "all":
2120 print output
2121 elif datadump['status'] == sys.argv[2]:
2122 print output
2123 elif sys.argv[1] == "create":
2124 if sys.argv[2] == "network.kml":
2125 print make_network_kml.make_graph()
2126 elif sys.argv[2] == "host-ips.txt":
2127 for system in get_hostlist():
2128 datadump = get_yaml(system)
2129 ips = [datadump['masterip']]
2130 for ifkey in get_interface_keys(datadump):
2131 ips.append(datadump[ifkey]['ip'].split('/')[0])
2132 print system, ' '.join(ips)
2133 elif sys.argv[2] == "host-pos.txt":
2134 for system in get_hostlist():
2135 datadump = get_yaml(system)
2136 print system, datadump['rdnap_x'], datadump['rdnap_y']
2137 elif sys.argv[2] == 'ssh_config':
2138 print '''
2139Host *.wleiden.net
2140 User root
2141
2142Host 172.16.*.*
2143 User root
2144'''
2145 for system in get_hostlist():
2146 datadump = get_yaml(system)
2147 print '''\
2148Host %s
2149 User root
2150
2151Host %s
2152 User root
2153
2154Host %s
2155 User root
2156
2157Host %s
2158 User root
2159''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2160 else:
2161 usage()
2162 else:
2163 usage()
2164 else:
2165 # Do not enable debugging for config requests as it highly clutters the output
2166 if not is_text_request():
2167 cgitb.enable()
2168 response_headers, output = process_cgi_request()
2169 print_cgi_response(response_headers, output)
2170
2171def application(environ, start_response):
2172 status = '200 OK'
2173 response_headers, output = process_cgi_request(environ)
2174 start_response(status, response_headers)
2175
2176 # Debugging only
2177 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2178 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2179 return [output]
2180
2181if __name__ == "__main__":
2182 main()
Note: See TracBrowser for help on using the repository browser.