source: genesis/tools/gformat.py@ 13278

Last change on this file since 13278 was 13278, checked in by rick, 10 years ago

Enable PNP (graphing) on certain set of nodes
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 70.5 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13278 2015-06-13 19:37:14Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = []
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125def clear_cache():
126 ''' Poor mans cache implementation '''
127 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
128 datadump_cache = {}
129 interface_list_cache = {}
130 rc_conf_local_cache = {}
131 ileiden_proxies = []
132 normal_proxies = []
133 nameservers_cache = []
134
135NO_DHCP = 0
136DHCP_CLIENT = 10
137DHCP_SERVER = 20
138def dhcp_type(item):
139 if not item.has_key('dhcp'):
140 return NO_DHCP
141 elif not item['dhcp']:
142 return NO_DHCP
143 elif item['dhcp'].lower() == 'client':
144 return DHCP_CLIENT
145 else:
146 # Validation Checks
147 begin,end = map(int,item['dhcp'].split('-'))
148 if begin >= end:
149 raise ValueError("DHCP Start >= DHCP End")
150 return DHCP_SERVER
151
152def etrs2rd(lat, lon):
153 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
154 p2 = pyproj.Proj(init='EPSG:28992')
155 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
156 return (RDx, RDy)
157
158def rd2etrs(RDx, RDy):
159 p1 = pyproj.Proj(init='EPSG:28992')
160 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
161 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
162 return (lat, lon)
163
164def get_yaml(item,add_version_info=True):
165 try:
166 """ Get configuration yaml for 'item'"""
167 if datadump_cache.has_key(item):
168 return datadump_cache[item].copy()
169
170 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
171
172 # Default values
173 datadump = {
174 'autogen_revision' : 'NOTFOUND',
175 'autogen_gfile' : gfile,
176 }
177 f = open(gfile, 'r')
178 datadump.update(yaml.load(f,Loader=Loader))
179 if datadump['nodetype'] == 'Hybrid':
180 # Some values are defined implicitly
181 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
182 datadump['service_incoming_rdr'] = True
183 # Use some boring defaults
184 defaults = {
185 'service_proxy_normal' : False,
186 'service_proxy_ileiden' : False,
187 'service_accesspoint' : True,
188 'service_incoming_rdr' : False,
189 'service_concentrator' : False,
190 'monitoring_group' : 'wleiden',
191 }
192 for (key,value) in defaults.iteritems():
193 if not datadump.has_key(key):
194 datadump[key] = value
195 f.close()
196
197 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
198 if add_version_info:
199 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
200 lines = p.communicate()[0].split('\n')
201 if p.returncode == 0:
202 for line in lines:
203 if line:
204 (key, value) = line.split(': ')
205 datadump["autogen_" + key.lower().replace(' ','_')] = value
206
207 # Preformat certain needed variables for formatting and push those into special object
208 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
209
210 wlan_count=0
211 try:
212 for key in datadump['autogen_iface_keys']:
213 datadump[key]['autogen_ifbase'] = key.split('_')[1]
214 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
215 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
216 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
217 wlan_count += 1
218 else:
219 datadump[key]['autogen_ifname'] = datadump[key]['autogen_ifbase']
220 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
221 if datadump[key]['autogen_bridge'] and not 'alias' in key:
222 datadump[key]['autogen_bridge_interfaces'] = datadump[key]['members'].split()
223 except Exception as e:
224 print "# Error while processing interface %s" % key
225 raise
226
227 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
228 if dhcp_type(datadump[key]) == DHCP_SERVER]
229
230 datadump['autogen_dhcp_interfaces'] = dhcp_interfaces
231 datadump['autogen_item'] = item
232
233 datadump['autogen_realname'] = get_realname(datadump)
234 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
235 datadump['autogen_fqdn'] = datadump['autogen_realname'] + '.' + datadump['autogen_domain']
236 datadump_cache[item] = datadump.copy()
237 except Exception as e:
238 print "# Error while processing %s" % item
239 raise
240 return datadump
241
242
243def store_yaml(datadump, header=False):
244 """ Store configuration yaml for 'item'"""
245 item = datadump['autogen_item']
246 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
247
248 output = generate_wleiden_yaml(datadump, header)
249
250 f = open(gfile, 'w')
251 f.write(output)
252 f.close()
253
254
255def network(ip):
256 addr, mask = ip.split('/')
257 # Not parsing of these folks please
258 addr = parseaddr(addr)
259 mask = int(mask)
260 network = addr & ~((1 << (32 - mask)) - 1)
261 return network
262
263
264
265def make_relations(datadumps=dict()):
266 """ Process _ALL_ yaml files to get connection relations """
267 errors = []
268 poel = defaultdict(list)
269
270 if not datadumps:
271 for host in get_hostlist():
272 datadumps[host] = get_yaml(host)
273
274 for host, datadump in datadumps.iteritems():
275 try:
276 for iface_key in datadump['autogen_iface_keys']:
277 net_addr = network(datadump[iface_key]['ip'])
278 poel[net_addr] += [(host,datadump[iface_key])]
279 except (KeyError, ValueError), e:
280 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
281 continue
282 return (poel, errors)
283
284
285
286def valid_addr(addr):
287 """ Show which address is valid in which are not """
288 return str(addr).startswith('172.')
289
290def get_system_list(prefix):
291 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/%s*/wleiden.yaml" % (NODE_DIR, prefix))])
292
293get_hybridlist = lambda: get_system_list("Hybrid")
294get_nodelist = lambda: get_system_list("CNode")
295get_proxylist = lambda: get_system_list("Proxy")
296
297def get_hostlist():
298 """ Combined hosts and proxy list"""
299 return get_nodelist() + get_proxylist() + get_hybridlist()
300
301def angle_between_points(lat1,lat2,long1,long2):
302 """
303 Return Angle in radians between two GPS coordinates
304 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
305 """
306 dy = lat2 - lat1
307 dx = math.cos(lat1)*(long2 - long1)
308 angle = math.atan2(dy,dx)
309 return angle
310
311
312
313def angle_to_cd(angle):
314 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
315
316 # For easy conversion get positive degree
317 degrees = math.degrees(angle)
318 abs_degrees = 360 + degrees if degrees < 0 else degrees
319
320 # Numbers can be confusing calculate from the 4 main directions
321 p = 22.5
322 if abs_degrees < p:
323 cd = "n"
324 elif abs_degrees < (90 - p):
325 cd = "no"
326 elif abs_degrees < (90 + p):
327 cd = "o"
328 elif abs_degrees < (180 - p):
329 cd = "zo"
330 elif abs_degrees < (180 + p):
331 cd = "z"
332 elif abs_degrees < (270 - p):
333 cd = "zw"
334 elif abs_degrees < (270 + p):
335 cd = "w"
336 elif abs_degrees < (360 - p):
337 cd = "nw"
338 else:
339 cd = "n"
340 return cd
341
342
343
344def cd_between_hosts(hostA, hostB, datadumps):
345 # Using RDNAP coordinates
346 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
347 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
348 return angle_to_cd(math.atan2(dx,dy))
349
350 # GPS coordinates seems to fail somehow
351 #latA = float(datadumps[hostA]['latitude'])
352 #latB = float(datadumps[hostB]['latitude'])
353 #lonA = float(datadumps[hostA]['longitude'])
354 #lonB = float(datadumps[hostB]['longitude'])
355 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
356
357
358def generate_title(nodelist):
359 """ Main overview page """
360 items = {'root' : "." }
361 def fl(spaces, line):
362 return (' ' * spaces) + line + '\n'
363
364 output = """
365<html>
366 <head>
367 <title>Wireless leiden Configurator - GFormat</title>
368 <style type="text/css">
369 th {background-color: #999999}
370 tr:nth-child(odd) {background-color: #cccccc}
371 tr:nth-child(even) {background-color: #ffffff}
372 th, td {padding: 0.1em 1em}
373 </style>
374 </head>
375 <body>
376 <center>
377 <form type="GET" action="%(root)s">
378 <input type="hidden" name="action" value="update">
379 <input type="submit" value="Update Configuration Database (SVN)">
380 </form>
381 <table>
382 <caption><h3>Wireless Leiden Configurator</h3></caption>
383 """ % items
384
385 for node in nodelist:
386 items['node'] = node
387 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
388 for config in files:
389 items['config'] = config
390 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
391 output += fl(5, "</tr>")
392 output += """
393 </table>
394 <hr />
395 <em>%s</em>
396 </center>
397 </body>
398</html>
399 """ % __version__
400
401 return output
402
403
404
405def generate_node(node):
406 """ Print overview of all files available for node """
407 return "\n".join(files)
408
409def generate_node_overview(host):
410 """ Print overview of all files available for node """
411 datadump = get_yaml(host)
412 params = { 'host' : host }
413 output = "<em><a href='..'>Back to overview</a></em><hr />"
414 output += "<h2>Available files:</h2><ul>"
415 for cf in files:
416 params['cf'] = cf
417 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
418 output += "</ul>"
419
420 # Generate and connection listing
421 output += "<h2>Connected To:</h2><ul>"
422 (poel, errors) = make_relations()
423 for network, hosts in poel.iteritems():
424 if host in [x[0] for x in hosts]:
425 if len(hosts) == 1:
426 # Single not connected interface
427 continue
428 for remote,ifacedump in hosts:
429 if remote == host:
430 # This side of the interface
431 continue
432 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
433 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
434 output += "</ul>"
435 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
436
437 output += "<hr /><em><a href='..'>Back to overview</a></em>"
438 return output
439
440
441def generate_header(datadump, ctag="#"):
442 return """\
443%(ctag)s
444%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
445%(ctag)s Generated at %(date)s by %(host)s from r%(revision)s
446%(ctag)s
447""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
448
449
450
451def parseaddr(s):
452 """ Process IPv4 CIDR notation addr to a (binary) number """
453 f = s.split('.')
454 return (long(f[0]) << 24L) + \
455 (long(f[1]) << 16L) + \
456 (long(f[2]) << 8L) + \
457 long(f[3])
458
459
460
461def showaddr(a):
462 """ Display IPv4 addr in (dotted) CIDR notation """
463 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
464
465
466def is_member(ip, mask, canidate):
467 """ Return True if canidate is part of ip/mask block"""
468 ip_addr = parseaddr(ip)
469 ip_canidate = parseaddr(canidate)
470 mask = int(mask)
471 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
472 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
473 return ip_addr == ip_canidate
474
475
476
477def cidr2netmask(netmask):
478 """ Given a 'netmask' return corresponding CIDR """
479 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
480
481def get_network(addr, mask):
482 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
483
484
485def generate_dhcpd_conf(datadump):
486 """ Generate config file '/usr/local/etc/dhcpd.conf """
487 output = generate_header(datadump)
488 output += Template("""\
489# option definitions common to all supported networks...
490option domain-name "dhcp.{{ autogen_fqdn }}";
491
492default-lease-time 600;
493max-lease-time 7200;
494
495# Use this to enble / disable dynamic dns updates globally.
496#ddns-update-style none;
497
498# If this DHCP server is the official DHCP server for the local
499# network, the authoritative directive should be uncommented.
500authoritative;
501
502# Use this to send dhcp log messages to a different log file (you also
503# have to hack syslog.conf to complete the redirection).
504log-facility local7;
505
506#
507# Interface definitions
508#
509\n""").render(datadump)
510
511 dhcp_out = defaultdict(list)
512 for iface_key in datadump['autogen_iface_keys']:
513 ifname = datadump[iface_key]['autogen_ifname']
514 if not datadump[iface_key].has_key('comment'):
515 datadump[iface_key]['comment'] = None
516 dhcp_out[ifname].append(" ## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key])
517
518 (addr, mask) = datadump[iface_key]['ip'].split('/')
519 datadump[iface_key]['autogen_addr'] = addr
520 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
521 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
522 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
523 dhcp_out[ifname].append(" subnet %(autogen_subnet)s netmask %(autogen_netmask)s {\n ### not autoritive\n }\n" % \
524 datadump[iface_key])
525 continue
526
527 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
528 dhcp_part = ".".join(addr.split('.')[0:3])
529 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
530 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
531
532 # Assume the first 10 IPs could be used for static entries
533 if 'no_portal' in datadump:
534 fixed = 5
535 for mac in datadump['no_portal']:
536 dhcp_out[ifname].append("""\
537 host fixed-%(ifname)s-%(fixed)s {
538 hardware ethernet %(mac)s;
539 fixed-address %(prefix)s.%(fixed)s;
540 }
541""" % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed })
542 fixed += 1
543
544 dhcp_out[ifname].append("""\
545 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
546 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
547 option routers %(autogen_addr)s;
548 option domain-name-servers %(autogen_addr)s;
549 }
550""" % datadump[iface_key])
551
552 for ifname,value in dhcp_out.iteritems():
553 output += ("shared-network %s {\n" % ifname) + ''.join(value) + '}\n\n'
554 return output
555
556
557
558def generate_dnsmasq_conf(datadump):
559 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
560 output = generate_header(datadump)
561 output += Template("""\
562# DHCP server options
563dhcp-authoritative
564dhcp-fqdn
565domain=dhcp.{{ autogen_fqdn }}
566domain-needed
567expand-hosts
568log-async=100
569
570# Low memory footprint
571cache-size=10000
572
573\n""").render(datadump)
574
575 for iface_key in datadump['autogen_iface_keys']:
576 if not datadump[iface_key].has_key('comment'):
577 datadump[iface_key]['comment'] = None
578 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
579
580 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
581 output += "# not autoritive\n\n"
582 continue
583
584 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
585 (ip, cidr) = datadump[iface_key]['ip'].split('/')
586 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
587
588 dhcp_part = ".".join(ip.split('.')[0:3])
589 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
590 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
591 output += "dhcp-range=%(autogen_ifname)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
592
593 return output
594
595
596def make_interface_list(datadump):
597 if interface_list_cache.has_key(datadump['autogen_item']):
598 return (interface_list_cache[datadump['autogen_item']])
599 # lo0 configuration:
600 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
601 # - masterip is special as it needs to be assigned to at
602 # least one interface, so if not used assign to lo0
603 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
604 dhclient_if = {'lo0' : False}
605
606 # XXX: Find some way of send this output nicely
607 output = ''
608
609 masterip_used = False
610 for iface_key in datadump['autogen_iface_keys']:
611 if datadump[iface_key]['ip'].startswith(datadump['masterip']):
612 masterip_used = True
613 break
614 if not masterip_used:
615 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
616
617 for iface_key in datadump['autogen_iface_keys']:
618 ifacedump = datadump[iface_key]
619 ifname = ifacedump['autogen_ifname']
620
621 # Flag dhclient is possible
622 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
623 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
624
625 # Add interface IP to list
626 item = (ifacedump['ip'], ifacedump['comment'])
627 if addrs_list.has_key(ifname):
628 addrs_list[ifname].append(item)
629 else:
630 addrs_list[ifname] = [item]
631
632 # Alias only needs IP assignment for now, this might change if we
633 # are going to use virtual accesspoints
634 if "alias" in iface_key:
635 continue
636
637 # XXX: Might want to deduct type directly from interface name
638 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
639 # Default to station (client) mode
640 ifacedump['autogen_wlanmode'] = "sta"
641 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
642 ifacedump['autogen_wlanmode'] = "ap"
643
644 if not ifacedump.has_key('channel'):
645 if ifacedump['type'] == '11a':
646 ifacedump['channel'] = 36
647 else:
648 ifacedump['channel'] = 1
649
650 # Allow special hacks at the back like wds and stuff
651 if not ifacedump.has_key('extra'):
652 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
653 else:
654 ifacedump['autogen_extra'] = ifacedump['extra']
655
656 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
657 output += ("create_args_%(autogen_ifname)s='wlanmode %(autogen_wlanmode)s mode " +\
658 "%(type)s ssid %(ssid)s %(autogen_extra)s channel %(channel)s'\n") % ifacedump
659 output += "\n"
660
661 elif ifacedump['type'] in ['ethernet', 'eth']:
662 # No special config needed besides IP
663 if ifacedump['autogen_bridge']:
664 output += "cloned_interfaces='%(autogen_ifname)s'\n" % ifacedump
665 output += "ifconfig_%s='addm %s up'\n" % (ifacedump['autogen_ifname'], ' addm '.join(ifacedump['autogen_bridge_interfaces']))
666 for member in ifacedump['autogen_bridge_interfaces']:
667 output += "ifconfig_%s='up'\n" % member
668 output += "\n"
669 else:
670 assert False, "Unknown type " + ifacedump['type']
671
672 store = (addrs_list, dhclient_if, output)
673 interface_list_cache[datadump['autogen_item']] = store
674 return(store)
675
676
677
678def generate_rc_conf_local(datadump):
679 """ Generate configuration file '/etc/rc.conf.local' """
680 item = datadump['autogen_item']
681 if rc_conf_local_cache.has_key(item):
682 return rc_conf_local_cache[item]
683
684 if not datadump.has_key('ileiden'):
685 datadump['autogen_ileiden_enable'] = False
686 else:
687 datadump['autogen_ileiden_enable'] = datadump['ileiden']
688
689 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
690
691 if not ileiden_proxies or not normal_proxies:
692 for proxy in get_proxylist():
693 proxydump = get_yaml(proxy)
694 if proxydump['ileiden']:
695 ileiden_proxies.append(proxydump)
696 else:
697 normal_proxies.append(proxydump)
698 for host in get_hybridlist():
699 hostdump = get_yaml(host)
700 if hostdump['status'] == 'up':
701 if hostdump['service_proxy_ileiden']:
702 ileiden_proxies.append(hostdump)
703 if hostdump['service_proxy_normal']:
704 normal_proxies.append(hostdump)
705
706 datadump['autogen_ileiden_proxies'] = ileiden_proxies
707 datadump['autogen_normal_proxies'] = normal_proxies
708 datadump['autogen_ileiden_proxies_ips'] = ','.join([x['masterip'] for x in ileiden_proxies])
709 datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
710 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
711 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
712
713 output = generate_header(datadump, "#");
714 output += render_template(datadump, """\
715hostname='{{ autogen_fqdn }}'
716location='{{ location }}'
717nodetype="{{ nodetype }}"
718
719#
720# Configured listings
721#
722captive_portal_whitelist=""
723{% if nodetype == "Proxy" %}
724#
725# Proxy Configuration
726#
727{% if gateway -%}
728defaultrouter="{{ gateway }}"
729{% else -%}
730#defaultrouter="NOTSET"
731{% endif -%}
732internalif="{{ internalif }}"
733ileiden_enable="{{ autogen_ileiden_enable }}"
734gateway_enable="{{ autogen_ileiden_enable }}"
735pf_enable="yes"
736pf_rules="/etc/pf.conf"
737{% if autogen_ileiden_enable -%}
738pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
739lvrouted_enable="{{ autogen_ileiden_enable }}"
740lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
741{% else -%}
742pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
743{% endif -%}
744{% if internalroute -%}
745static_routes="wleiden"
746route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
747{% endif -%}
748
749{% elif nodetype == "Hybrid" %}
750 #
751 # Hybrid Configuration
752 #
753 list_ileiden_proxies="
754 {% for item in autogen_ileiden_proxies -%}
755 {{ "%-16s"|format(item.masterip) }} # {{ item.autogen_realname }}
756 {% endfor -%}
757 "
758 list_normal_proxies="
759 {% for item in autogen_normal_proxies -%}
760 {{ "%-16s"|format(item.masterip) }} # {{ item.autogen_realname }}
761 {% endfor -%}
762 "
763
764 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',')|default('none', true) }}"
765 externalif="{{ externalif|default('vr0', true) }}"
766 masterip="{{ masterip }}"
767
768 # Defined services
769 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
770 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
771 service_accesspoint="{{ service_accesspoint|yesorno }}"
772 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
773 service_concentrator="{{ service_concentrator|yesorno }}"
774 #
775
776 {% if service_proxy_ileiden %}
777 pf_rules="/etc/pf.hybrid.conf"
778 {% if service_concentrator %}
779 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
780 {% else %}
781 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
782 {% endif %}
783 pf_flags="$pf_flags -D publicnat=80,443"
784 lvrouted_flags="$lvrouted_flags -g"
785 {% elif service_proxy_normal or service_incoming_rdr %}
786 pf_rules="/etc/pf.hybrid.conf"
787 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
788 pf_flags="$pf_flags -D publicnat=0"
789 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
790 named_setfib="1"
791 tinyproxy_setfib="1"
792 dnsmasq_setfib="1"
793 sshd_setfib="1"
794 {% else %}
795 named_auto_forward_only="YES"
796 pf_rules="/etc/pf.node.conf"
797 pf_flags=""
798 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
799 {% endif %}
800 {% if service_concentrator %}
801 # Do mind installing certificates is NOT done automatically for security reasons
802 openvpn_enable="YES"
803 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
804 {% endif %}
805
806 {% if service_proxy_normal %}
807 tinyproxy_enable="yes"
808 {% else %}
809 pen_wrapper_enable="yes"
810 {% endif %}
811
812 {% if service_accesspoint %}
813 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
814 {% endif %}
815
816 {% if board == "ALIX2" %}
817 #
818 # ''Fat'' configuration, board has 256MB RAM
819 #
820 dnsmasq_enable="NO"
821 named_enable="YES"
822 {% if autogen_dhcp_interfaces -%}
823 dhcpd_enable="YES"
824 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
825 {% endif -%}
826 {% endif -%}
827
828 {% if gateway %}
829 defaultrouter="{{ gateway }}"
830 {% endif %}
831{% elif nodetype == "CNode" %}
832#
833# NODE iLeiden Configuration
834#
835
836# iLeiden Proxies {{ autogen_ileiden_proxies_names }}
837list_ileiden_proxies="{{ autogen_ileiden_proxies_ips }}"
838# normal Proxies {{ autogen_normal_proxies_names }}
839list_normal_proxies="{{ autogen_normal_proxies_ips }}"
840
841captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
842
843lvrouted_flags="-u -s s00p3rs3kr3t -m 28 -z $list_ileiden_proxies"
844{% endif %}
845
846#
847# Interface definitions
848#\n
849""")
850
851 (addrs_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
852 output += extra_ouput.strip() + "\n"
853
854 # Print IP address which needs to be assigned over here
855 output += "\n"
856 for iface,addrs in sorted(addrs_list.iteritems()):
857 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
858 output += "# %s || %s || %s\n" % (iface, addr, comment)
859
860 # Write DHCLIENT entry
861 if dhclient_if[iface]:
862 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
863
864 # Make sure the external address is always first as this is needed in the
865 # firewall setup
866 addrs = sorted(
867 [x for x in addrs if not '0.0.0.0' in x[0]],
868 key=lambda x: x[0].split('.')[0],
869 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
870 )
871 addr_str = " ".join([x[0] for x in addrs])
872 output += "ipv4_addrs_%s='%s'\n\n" % (iface, addr_str)
873
874 rc_conf_local_cache[datadump['autogen_item']] = output
875 return output
876
877
878
879
880def get_all_configs():
881 """ Get dict with key 'host' with all configs present """
882 configs = dict()
883 for host in get_hostlist():
884 datadump = get_yaml(host)
885 configs[host] = datadump
886 return configs
887
888
889def get_interface_keys(config):
890 """ Quick hack to get all interface keys, later stage convert this to a iterator """
891 return sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
892
893
894def get_used_ips(configs):
895 """ Return array of all IPs used in config files"""
896 ip_list = []
897 for config in configs:
898 ip_list.append(config['masterip'])
899 for iface_key in get_interface_keys(config):
900 l = config[iface_key]['ip']
901 addr, mask = l.split('/')
902 # Special case do not process
903 if valid_addr(addr):
904 ip_list.append(addr)
905 else:
906 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
907 return sorted(ip_list)
908
909
910
911def get_nameservers(max_servers=None):
912 if nameservers_cache:
913 return nameservers_cache[0:max_servers]
914
915 for host in get_hybridlist():
916 hostdump = get_yaml(host)
917 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
918 nameservers_cache.append((hostdump['masterip'], hostdump['autogen_realname']))
919 for host in get_proxylist():
920 hostdump = get_yaml(host)
921 if hostdump['status'] == 'up':
922 nameservers_cache.append((hostdump['masterip'], hostdump['autogen_realname']))
923
924 return nameservers_cache[0:max_servers]
925
926
927def generate_resolv_conf(datadump):
928 """ Generate configuration file '/etc/resolv.conf' """
929 # XXX: This should properly going to be an datastructure soon
930 datadump['autogen_header'] = generate_header(datadump, "#")
931 datadump['autogen_edge_nameservers'] = ''
932
933
934 for masterip,realname in get_nameservers():
935 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
936
937 return Template("""\
938{{ autogen_header }}
939search wleiden.net
940
941# Try local (cache) first
942nameserver 127.0.0.1
943
944{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
945nameserver 8.8.8.8 # Google Public NameServer
946nameserver 8.8.4.4 # Google Public NameServer
947{% else -%}
948# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
949{{ autogen_edge_nameservers }}
950{% endif -%}
951""").render(datadump)
952
953
954
955def generate_ntp_conf(datadump):
956 """ Generate configuration file '/etc/ntp.conf' """
957 # XXX: This should properly going to be an datastructure soon
958
959 datadump['autogen_header'] = generate_header(datadump, "#")
960 datadump['autogen_ntp_servers'] = ''
961 for host in get_proxylist():
962 hostdump = get_yaml(host)
963 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(autogen_realname)s\n" % hostdump
964 for host in get_hybridlist():
965 hostdump = get_yaml(host)
966 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
967 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(autogen_realname)s\n" % hostdump
968
969 return Template("""\
970{{ autogen_header }}
971
972{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
973# Machine hooked to internet.
974server 0.nl.pool.ntp.org iburst maxpoll 9
975server 1.nl.pool.ntp.org iburst maxpoll 9
976server 2.nl.pool.ntp.org iburst maxpoll 9
977server 3.nl.pool.ntp.org iburst maxpoll 9
978{% else -%}
979# Local Wireless Leiden NTP Servers.
980server 0.pool.ntp.wleiden.net iburst maxpoll 9
981server 1.pool.ntp.wleiden.net iburst maxpoll 9
982server 2.pool.ntp.wleiden.net iburst maxpoll 9
983server 3.pool.ntp.wleiden.net iburst maxpoll 9
984
985# All the configured NTP servers
986{{ autogen_ntp_servers }}
987{% endif %}
988
989# If a server loses sync with all upstream servers, NTP clients
990# no longer follow that server. The local clock can be configured
991# to provide a time source when this happens, but it should usually
992# be configured on just one server on a network. For more details see
993# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
994# The use of Orphan Mode may be preferable.
995#
996server 127.127.1.0
997fudge 127.127.1.0 stratum 10
998""").render(datadump)
999
1000
1001def generate_pf_hybrid_conf_local(datadump):
1002 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1003 datadump['autogen_header'] = generate_header(datadump, "#")
1004 return Template("""\
1005{{ autogen_header }}
1006
1007# Redirect some internal facing services outside (7)
1008# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1009{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1010rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1011{% endfor -%}
1012""").render(datadump)
1013
1014def generate_motd(datadump):
1015 """ Generate configuration file '/etc/motd' """
1016 output = Template("""\
1017FreeBSD run ``service motd onestart'' to make me look normal
1018
1019 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1020 Loc: {{ location }}
1021
1022Services:
1023{% if board == "ALIX2" -%}
1024{{" -"}} Core Node ({{ board }})
1025{% else -%}
1026{{" -"}} Hulp Node ({{ board }})
1027{% endif -%}
1028{% if service_proxy_normal -%}
1029{{" -"}} Normal Proxy
1030{% endif -%}
1031{% if service_proxy_ileiden -%}
1032{{" -"}} iLeiden Proxy
1033{% endif -%}
1034{% if service_incoming_rdr -%}
1035{{" -"}} Incoming port redirects
1036{% endif %}
1037Interlinks:\n
1038""").render(datadump)
1039
1040 (addrs_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
1041 # Just nasty hack to make the formatting looks nice
1042 iface_len = max(map(len,addrs_list.keys()))
1043 addr_len = max(map(len,[x[0] for x in [x[0] for x in addrs_list.values()]]))
1044 for iface,addrs in sorted(addrs_list.iteritems()):
1045 if iface in ['lo0']:
1046 continue
1047 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1048 output += " - %s || %s || %s\n" % (iface.ljust(iface_len), addr.ljust(addr_len), comment)
1049
1050 output += '\n'
1051 output += """\
1052Attached bridges:
1053"""
1054 has_item = False
1055 for iface_key in datadump['autogen_iface_keys']:
1056 ifacedump = datadump[iface_key]
1057 if ifacedump.has_key('ns_ip'):
1058 ifacedump['autogen_ns_ip'] = ifacedump['ns_ip'].split('/')[0]
1059 has_item = True
1060 output += " - %(autogen_ifname)s || %(mode)s || https://%(autogen_ns_ip)s\n" % ifacedump
1061 if not has_item:
1062 output += " - none\n"
1063
1064 return output
1065
1066
1067def format_yaml_value(value):
1068 """ Get yaml value in right syntax for outputting """
1069 if isinstance(value,str):
1070 output = '"%s"' % value
1071 else:
1072 output = value
1073 return output
1074
1075
1076
1077def format_wleiden_yaml(datadump):
1078 """ Special formatting to ensure it is editable"""
1079 output = "# Genesis config yaml style\n"
1080 output += "# vim:ts=2:et:sw=2:ai\n"
1081 output += "#\n"
1082 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1083 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1084 if key == 'rdr_rules':
1085 output += '%-10s:\n' % 'rdr_rules'
1086 for rdr_rule in datadump[key]:
1087 output += '- %s\n' % rdr_rule
1088 else:
1089 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1090
1091 output += "\n\n"
1092
1093 # Format (key, required)
1094 key_order = (
1095 ('comment', True),
1096 ('ip', True),
1097 ('desc', True),
1098 ('sdesc', True),
1099 ('mode', True),
1100 ('type', True),
1101 ('extra_type', False),
1102 ('channel', False),
1103 ('ssid', False),
1104 ('wlan_mac', False),
1105 ('dhcp', True),
1106 ('compass', False),
1107 ('distance', False),
1108 ('ns_ip', False),
1109 ('repeater_ip', False),
1110 ('bullet2_ip', False),
1111 ('ns_mac', False),
1112 ('bullet2_mac', False),
1113 ('ns_type', False),
1114 ('bridge_type', False),
1115 ('members', True),
1116 ('status', True),
1117 )
1118
1119 for iface_key in sorted(iface_keys):
1120 try:
1121 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1122 if remainder:
1123 raise KeyError("invalid keys: %s" % remainder)
1124
1125 output += "%s:\n" % iface_key
1126 for key,required in key_order:
1127 if datadump[iface_key].has_key(key):
1128 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1129 output += "\n\n"
1130 except Exception as e:
1131 print "# Error while processing interface %s" % iface_key
1132 raise
1133
1134 return output
1135
1136
1137
1138def generate_wleiden_yaml(datadump, header=True):
1139 """ Generate (petty) version of wleiden.yaml"""
1140 output = generate_header(datadump, "#") if header else ''
1141
1142 for key in datadump.keys():
1143 if key.startswith('autogen_'):
1144 del datadump[key]
1145 # Interface autogen cleanups
1146 elif type(datadump[key]) == dict:
1147 for key2 in datadump[key].keys():
1148 if key2.startswith('autogen_'):
1149 del datadump[key][key2]
1150
1151 output += format_wleiden_yaml(datadump)
1152 return output
1153
1154def generate_nanostation_config(datadump, iface, ns_type):
1155 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1156 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1157
1158 datadump.update(datadump['iface_%s' % iface])
1159
1160 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1161
1162def generate_yaml(datadump):
1163 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1164
1165
1166
1167def generate_config(node, config, datadump=None):
1168 """ Print configuration file 'config' of 'node' """
1169 output = ""
1170 try:
1171 # Load config file
1172 if datadump == None:
1173 datadump = get_yaml(node)
1174
1175 if config == 'wleiden.yaml':
1176 output += generate_wleiden_yaml(datadump)
1177 elif config == 'authorized_keys':
1178 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1179 output += f.read()
1180 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1181 # Fetch local keys if existing
1182 if os.path.exists(node_keys):
1183 output += open(node_keys, 'r').read()
1184 f.close()
1185 elif config == 'dnsmasq.conf':
1186 output += generate_dnsmasq_conf(datadump)
1187 elif config == 'dhcpd.conf':
1188 output += generate_dhcpd_conf(datadump)
1189 elif config == 'rc.conf.local':
1190 output += generate_rc_conf_local(datadump)
1191 elif config == 'resolv.conf':
1192 output += generate_resolv_conf(datadump)
1193 elif config == 'ntp.conf':
1194 output += generate_ntp_conf(datadump)
1195 elif config == 'motd':
1196 output += generate_motd(datadump)
1197 elif config == 'pf.hybrid.conf.local':
1198 output += generate_pf_hybrid_conf_local(datadump)
1199 elif config.startswith('vr'):
1200 interface, ns_type = config.strip('.yaml').split('-')
1201 output += generate_nanostation_config(datadump, interface, ns_type)
1202 else:
1203 assert False, "Config not found!"
1204 except IOError, e:
1205 output += "[ERROR] Config file not found"
1206 return output
1207
1208
1209
1210def process_cgi_request(environ=os.environ):
1211 """ When calling from CGI """
1212 response_headers = []
1213 content_type = 'text/plain'
1214
1215 # Update repository if requested
1216 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1217 if form and form.has_key("action") and "update" in form["action"]:
1218 output = "[INFO] Updating subverion, please wait...\n"
1219 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1220 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1221 output += "[INFO] All done, redirecting in 5 seconds"
1222 response_headers += [
1223 ('Refresh', '5; url=.'),
1224 ]
1225 reload_cache()
1226 else:
1227 base_uri = environ['PATH_INFO']
1228 uri = base_uri.strip('/').split('/')
1229
1230 output = "Template Holder"
1231 if base_uri.endswith('/create/network.kml'):
1232 content_type='application/vnd.google-earth.kml+xml'
1233 output = make_network_kml.make_graph()
1234 elif base_uri.endswith('/api/get/nodeplanner.json'):
1235 content_type='application/json'
1236 output = make_network_kml.make_nodeplanner_json()
1237 elif not uri[0]:
1238 if is_text_request(environ):
1239 output = '\n'.join(get_hostlist())
1240 else:
1241 content_type = 'text/html'
1242 output = generate_title(get_hostlist())
1243 elif len(uri) == 1:
1244 if is_text_request(environ):
1245 output = generate_node(uri[0])
1246 else:
1247 content_type = 'text/html'
1248 output = generate_node_overview(uri[0])
1249 elif len(uri) == 2:
1250 output = generate_config(uri[0], uri[1])
1251 else:
1252 assert False, "Invalid option"
1253
1254 # Return response
1255 response_headers += [
1256 ('Content-type', content_type),
1257 ('Content-Length', str(len(output))),
1258 ]
1259 return(response_headers, str(output))
1260
1261
1262def get_realname(datadump):
1263 # Proxy naming convention is special, as the proxy name is also included in
1264 # the nodename, when it comes to the numbered proxies.
1265 if datadump['nodetype'] == 'Proxy':
1266 realname = datadump['nodetype'] + datadump['nodename'].replace('proxy','')
1267 else:
1268 # By default the full name is listed and also a shortname CNAME for easy use.
1269 realname = datadump['nodetype'] + datadump['nodename']
1270 return(realname)
1271
1272
1273
1274def make_dns(output_dir = 'dns', external = False):
1275 items = dict()
1276
1277 # hostname is key, IP is value
1278 wleiden_zone = defaultdict(list)
1279 wleiden_cname = dict()
1280
1281 pool = dict()
1282 for node in get_hostlist():
1283 datadump = get_yaml(node)
1284
1285 # Proxy naming convention is special
1286 fqdn = datadump['autogen_realname']
1287 if datadump['nodetype'] in ['CNode', 'Hybrid']:
1288 wleiden_cname[datadump['nodename']] = fqdn
1289
1290 if datadump.has_key('rdr_host'):
1291 remote_target = datadump['rdr_host']
1292 elif datadump.has_key('remote_access') and datadump['remote_access']:
1293 remote_target = datadump['remote_access'].split(':')[0]
1294 else:
1295 remote_target = None
1296
1297 if remote_target:
1298 try:
1299 parseaddr(remote_target)
1300 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1301 except (IndexError, ValueError):
1302 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1303
1304
1305 wleiden_zone[fqdn].append((datadump['masterip'], True))
1306
1307 # Hacking to get proper DHCP IPs and hostnames
1308 for iface_key in get_interface_keys(datadump):
1309 iface_name = iface_key.replace('_','-')
1310 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1311 try:
1312 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1313 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1314 dhcp_part = ".".join(ip.split('.')[0:3])
1315 if ip != datadump['masterip']:
1316 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, False))
1317 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1318 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1319 except (AttributeError, ValueError, KeyError):
1320 # First push it into a pool, to indentify the counter-part later on
1321 addr = parseaddr(ip)
1322 cidr = int(cidr)
1323 addr = addr & ~((1 << (32 - cidr)) - 1)
1324 if pool.has_key(addr):
1325 pool[addr] += [(iface_name, fqdn, ip)]
1326 else:
1327 pool[addr] = [(iface_name, fqdn, ip)]
1328 continue
1329
1330
1331
1332 # WL uses an /29 to configure an interface. IP's are ordered like this:
1333 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1334
1335 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1336
1337 # Automatic naming convention of interlinks namely 2 + remote.lower()
1338 for (key,value) in pool.iteritems():
1339 # Make sure they are sorted from low-ip to high-ip
1340 value = sorted(value, key=lambda x: parseaddr(x[2]))
1341
1342 if len(value) == 1:
1343 (iface_name, fqdn, ip) = value[0]
1344 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1345
1346 # Device DNS names
1347 if 'cnode' in fqdn.lower():
1348 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1349 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1350
1351 elif len(value) == 2:
1352 (a_iface_name, a_fqdn, a_ip) = value[0]
1353 (b_iface_name, b_fqdn, b_ip) = value[1]
1354 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1355 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1356
1357 # Device DNS names
1358 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1359 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1360 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1361 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1362 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1363 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1364 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1365
1366 else:
1367 pool_members = [k[1] for k in value]
1368 for item in value:
1369 (iface_name, fqdn, ip) = item
1370 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1371
1372 # Include static DNS entries
1373 # XXX: Should they override the autogenerated results?
1374 # XXX: Convert input to yaml more useable.
1375 # Format:
1376 ##; this is a comment
1377 ## roomburgh=CNodeRoomburgh1
1378 ## apkerk1.CNodeVosko=172.17.176.8 ;this as well
1379 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1380
1381 # Hack to allow special entries, for development
1382 wleiden_raw = {}
1383
1384 for line in dns_list:
1385 reverse = False
1386 k, items = line.items()[0]
1387 if type(items) == dict:
1388 if items.has_key('reverse'):
1389 reverse = items['reverse']
1390 items = items['a']
1391 else:
1392 items = items['cname']
1393 items = [items] if type(items) != list else items
1394 for item in items:
1395 if item.startswith('IN '):
1396 wleiden_raw[k] = item
1397 elif valid_addr(item):
1398 wleiden_zone[k].append((item, reverse))
1399 else:
1400 wleiden_cname[k] = item
1401
1402 # Hack to get dynamic pool listing
1403 def chunks(l, n):
1404 return [l[i:i+n] for i in range(0, len(l), n)]
1405
1406 ntp_servers = [x[0] for x in get_nameservers()]
1407 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1408 for ntp_server in chunk:
1409 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1410
1411 details = dict()
1412 # 24 updates a day allowed
1413 details['serial'] = time.strftime('%Y%m%d%H')
1414
1415 if external:
1416 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1417 else:
1418 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1419
1420 details['master'] = dns_masters[0]
1421 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1422
1423 dns_header = '''
1424$TTL 3h
1425%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1426 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1427
1428%(ns_servers)s
1429 \n'''
1430
1431
1432 if not os.path.isdir(output_dir):
1433 os.makedirs(output_dir)
1434 details['zone'] = 'wleiden.net'
1435 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1436 f.write(dns_header % details)
1437
1438 for host,items in wleiden_zone.iteritems():
1439 for ip,reverse in items:
1440 if ip not in ['0.0.0.0']:
1441 f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
1442 for source,dest in wleiden_cname.iteritems():
1443 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1444 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1445 for source, dest in wleiden_raw.iteritems():
1446 f.write("%s.wleiden.net. %s\n" % (source, dest))
1447 f.close()
1448
1449 # Create whole bunch of specific sub arpa zones. To keep it compliant
1450 for s in range(16,32):
1451 details['zone'] = '%i.172.in-addr.arpa' % s
1452 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1453 f.write(dns_header % details)
1454
1455 #XXX: Not effient, fix to proper data structure and do checks at other
1456 # stages
1457 for host,items in wleiden_zone.iteritems():
1458 for ip,reverse in items:
1459 if not reverse:
1460 continue
1461 if valid_addr(ip):
1462 if valid_addr(ip):
1463 if int(ip.split('.')[1]) == s:
1464 rev_ip = '.'.join(reversed(ip.split('.')))
1465 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1466 f.close()
1467
1468
1469def usage():
1470 print """Usage: %(prog)s <argument>
1471Argument:
1472\tstandalone [port] = Run configurator webserver [8000]
1473\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1474\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1475\tfull-export = Generate yaml export script for heatmap.
1476\tstatic [outputdir] = Generate all config files and store on disk
1477\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1478\ttest <node> [<file>] = Receive output for certain node [all files].
1479\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1480\tlist <status> <items> = List systems which have certain status
1481
1482Arguments:
1483\t<node> = NodeName (example: HybridRick)
1484\t<file> = %(files)s
1485\t<status> = all|up|down|planned
1486\t<items> = systems|nodes|proxies
1487
1488NOTE FOR DEVELOPERS; you can test your changes like this:
1489 BEFORE any changes in this code:
1490 $ ./gformat.py static /tmp/pre
1491 AFTER the changes:
1492 $ ./gformat.py static /tmp/post
1493 VIEW differences and VERIFY all are OK:
1494 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1495""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1496 exit(0)
1497
1498
1499def is_text_request(environ=os.environ):
1500 """ Find out whether we are calling from the CLI or any text based CLI utility """
1501 try:
1502 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1503 except KeyError:
1504 return True
1505
1506def switchFormat(setting):
1507 if setting:
1508 return "YES"
1509 else:
1510 return "NO"
1511
1512def rlinput(prompt, prefill=''):
1513 import readline
1514 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1515 try:
1516 return raw_input(prompt)
1517 finally:
1518 readline.set_startup_hook()
1519
1520def fix_conflict(left, right, default='i'):
1521 while True:
1522 print "## %-30s | %-30s" % (left, right)
1523 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1524 if not c:
1525 c = default
1526
1527 if c in ['l','1']:
1528 return left
1529 elif c in ['r','2']:
1530 return right
1531 elif c in ['e', '3']:
1532 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1533 elif c in ['i', '4']:
1534 return None
1535 else:
1536 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1537
1538
1539
1540def print_cgi_response(response_headers, output):
1541 """Could we not use some kind of wsgi wrapper to make this output?"""
1542 for header in response_headers:
1543 print "%s: %s" % header
1544 print
1545 print output
1546
1547
1548def fill_cache():
1549 ''' Poor man re-loading of few cache items (the slow ones) '''
1550 for host in get_hostlist():
1551 get_yaml(host)
1552
1553
1554def reload_cache():
1555 clear_cache()
1556 fill_cache()
1557
1558
1559def main():
1560 """Hard working sub"""
1561 # Allow easy hacking using the CLI
1562 if not os.environ.has_key('PATH_INFO'):
1563 if len(sys.argv) < 2:
1564 usage()
1565
1566 if sys.argv[1] == "standalone":
1567 import SocketServer
1568 import CGIHTTPServer
1569 # Hop to the right working directory.
1570 os.chdir(os.path.dirname(__file__))
1571 try:
1572 PORT = int(sys.argv[2])
1573 except (IndexError,ValueError):
1574 PORT = 8000
1575
1576 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1577 """ Serve this CGI from the root of the webserver """
1578 def is_cgi(self):
1579 if "favicon" in self.path:
1580 return False
1581
1582 self.cgi_info = (os.path.basename(__file__), self.path)
1583 self.path = ''
1584 return True
1585 handler = MyCGIHTTPRequestHandler
1586 SocketServer.TCPServer.allow_reuse_address = True
1587 httpd = SocketServer.TCPServer(("", PORT), handler)
1588 httpd.server_name = 'localhost'
1589 httpd.server_port = PORT
1590
1591 logger.info("serving at port %s", PORT)
1592 try:
1593 httpd.serve_forever()
1594 except KeyboardInterrupt:
1595 httpd.shutdown()
1596 logger.info("All done goodbye")
1597 elif sys.argv[1] == "test":
1598 # Basic argument validation
1599 try:
1600 node = sys.argv[2]
1601 datadump = get_yaml(node)
1602 except IndexError:
1603 print "Invalid argument"
1604 exit(1)
1605 except IOError as e:
1606 print e
1607 exit(1)
1608
1609
1610 # Get files to generate
1611 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1612
1613 # Actual config generation
1614 for config in gen_files:
1615 logger.info("## Generating %s %s", node, config)
1616 print generate_config(node, config, datadump)
1617 elif sys.argv[1] == "test-cgi":
1618 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1619 os.environ['SCRIPT_NAME'] = __file__
1620 response_headers, output = process_cgi_request()
1621 print_cgi_response(response_headers, output)
1622 elif sys.argv[1] == "static":
1623 items = dict()
1624 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1625 for node in get_hostlist():
1626 items['node'] = node
1627 items['wdir'] = "%(output_dir)s/%(node)s" % items
1628 if not os.path.isdir(items['wdir']):
1629 os.makedirs(items['wdir'])
1630 datadump = get_yaml(node)
1631 for config in files:
1632 items['config'] = config
1633 logger.info("## Generating %(node)s %(config)s" % items)
1634 f = open("%(wdir)s/%(config)s" % items, "w")
1635 f.write(generate_config(node, config, datadump))
1636 f.close()
1637 elif sys.argv[1] == "wind-export":
1638 items = dict()
1639 for node in get_hostlist():
1640 datadump = get_yaml(node)
1641 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1642 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1643 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1644 VALUES (
1645 (SELECT id FROM users WHERE username = 'rvdzwet'),
1646 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1647 'Y');""" % datadump
1648 #for config in files:
1649 # items['config'] = config
1650 # print "## Generating %(node)s %(config)s" % items
1651 # f = open("%(wdir)s/%(config)s" % items, "w")
1652 # f.write(generate_config(node, config, datadump))
1653 # f.close()
1654 for node in get_hostlist():
1655 datadump = get_yaml(node)
1656 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1657 ifacedump = datadump[iface_key]
1658 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1659 ifacedump['nodename'] = datadump['nodename']
1660 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1661 ifacedump['channel'] = 0
1662 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1663 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1664 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1665 elif sys.argv[1] == "nagios-export":
1666 try:
1667 heavy_load = (sys.argv[2] == "--heavy-load")
1668 except IndexError:
1669 heavy_load = False
1670
1671 hostgroup_details = {
1672 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1673 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1674 'walphen' : 'Stichting Wireless Alphen',
1675 'westeinder' : 'Westeinder Plassen',
1676 }
1677
1678 # Convert IP to Host
1679 ip2host = {'root' : 'root'}
1680 for host in get_hostlist():
1681 datadump = get_yaml(host)
1682 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1683 for iface in datadump['autogen_iface_keys']:
1684 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1685
1686 # Find dependency tree based on output of lvrouted.mytree of nearest node
1687 parents = defaultdict(list)
1688 stack = ['root']
1689 prev_depth = 0
1690 for line in open('lvrouted.mytree').readlines():
1691 depth = line.count('\t')
1692 ip = line.strip().split()[0]
1693
1694 if prev_depth < depth:
1695 try:
1696 parents[ip2host[ip]].append(ip2host[stack[-1]])
1697 except KeyError as e:
1698 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1699 stack.append(ip)
1700 elif prev_depth > depth:
1701 stack = stack[:(depth - prev_depth)]
1702 elif prev_depth == depth:
1703 try:
1704 parents[ip2host[ip]].append(ip2host[stack[-1]])
1705 except KeyError as e:
1706 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1707
1708
1709 prev_depth = depth
1710 # Observe that some nodes has themself as parent or multiple parents
1711 # for now take only the first parent, other behaviour is yet to be explained
1712
1713
1714
1715 params = {
1716 'check_interval' : 5 if heavy_load else 120,
1717 'retry_interval' : 1 if heavy_load else 10,
1718 'max_check_attempts' : 10 if heavy_load else 6,
1719 'notification_interval': 120 if heavy_load else 240,
1720 }
1721
1722 print '''\
1723define host {
1724 name wleiden-node ; Default Node Template
1725 use generic-host ; Use the standard template as initial starting point
1726 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1727 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1728 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1729 notification_interval %(notification_interval)s
1730 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1731 check_command check-host-alive ; Default command to check FreeBSD hosts
1732 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1733}
1734
1735define service {
1736 name wleiden-service ; Default Service Template
1737 use generic-service ; Use the standard template as initial starting point
1738 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1739 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1740 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1741 notification_interval %(notification_interval)s
1742 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1743 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1744}
1745
1746# Please make sure to install:
1747# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1748#
1749# Recompile net-mgmt/nagios-plugins to support check_snmp
1750# make -C /usr/ports/net-mgmt/nagios-plugins
1751#
1752# Install net/bind-tools to allow v2/check_dns_wl to work:
1753# pkg install bind-tools
1754#
1755define command{
1756 command_name check_snmp_disk
1757 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1758}
1759
1760define command{
1761 command_name check_netsnmp_load
1762 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1763}
1764
1765define command{
1766 command_name check_netsnmp_proc
1767 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1768}
1769
1770define command{
1771 command_name check_by_ssh
1772 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1773}
1774
1775define command{
1776 command_name check_dns_wl
1777 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1778}
1779
1780define command{
1781 command_name check_snmp_uptime
1782 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1783}
1784
1785
1786# TDB: dhcp leases
1787# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1788
1789# TDB: internet status
1790# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1791
1792# TDB: Advanced local passive checks
1793# /usr/local/libexec/nagios/check_by_ssh
1794''' % params
1795
1796 print '''\
1797# Service Group, not displayed by default
1798define hostgroup {
1799 hostgroup_name srv_hybrid
1800 alias All Hybrid Nodes
1801 register 0
1802}
1803
1804define service {
1805 use wleiden-service
1806 hostgroup_name srv_hybrid
1807 service_description SSH
1808 check_command check_ssh
1809}
1810
1811define service {
1812 use wleiden-service,service-pnp
1813 hostgroup_name srv_hybrid
1814 service_description HTTP
1815 check_command check_http
1816}
1817
1818define service {
1819 use wleiden-service
1820 hostgroup_name srv_hybrid
1821 service_description DNS
1822 check_command check_dns_wl!"www.wirelessleiden.nl"
1823}
1824'''
1825
1826 if heavy_load:
1827 print '''\
1828define service {
1829 use wleiden-service
1830 hostgroup_name srv_hybrid
1831 service_description UPTIME
1832 check_command check_snmp_uptime
1833}
1834
1835#define service {
1836# use wleiden-service
1837# hostgroup_name srv_hybrid
1838# service_description NTP
1839# check_command check_ntp_peer
1840#}
1841
1842define service {
1843 use wleiden-service
1844 hostgroup_name srv_hybrid
1845 service_description LOAD
1846 check_command check_netsnmp_load
1847}
1848
1849define service {
1850 use wleiden-service
1851 hostgroup_name srv_hybrid
1852 service_description PROC
1853 check_command check_netsnmp_proc
1854}
1855
1856define service {
1857 use wleiden-service
1858 hostgroup_name srv_hybrid
1859 service_description DISK
1860 check_command check_snmp_disk
1861}
1862'''
1863 for node in get_hostlist():
1864 datadump = get_yaml(node)
1865 if not datadump['status'] == 'up':
1866 continue
1867 if not hostgroup_details.has_key(datadump['monitoring_group']):
1868 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
1869 print '''\
1870define host {
1871 use wleiden-node,host-pnp
1872 contact_groups admins
1873 host_name %(autogen_fqdn)s
1874 address %(masterip)s
1875 hostgroups srv_hybrid,%(monitoring_group)s\
1876''' % datadump
1877 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
1878 print '''\
1879 parents %(parents)s\
1880''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
1881 print '''\
1882}
1883'''
1884
1885
1886 for name,alias in hostgroup_details.iteritems():
1887 print '''\
1888define hostgroup {
1889 hostgroup_name %s
1890 alias %s
1891} ''' % (name, alias)
1892
1893
1894 elif sys.argv[1] == "full-export":
1895 hosts = {}
1896 for node in get_hostlist():
1897 datadump = get_yaml(node)
1898 hosts[datadump['nodename']] = datadump
1899 print yaml.dump(hosts)
1900
1901 elif sys.argv[1] == "dns":
1902 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
1903 elif sys.argv[1] == "cleanup":
1904 # First generate all datadumps
1905 datadumps = dict()
1906 ssid_to_node = dict()
1907 for host in get_hostlist():
1908 logger.info("# Processing: %s", host)
1909 # Set some boring default values
1910 datadump = { 'board' : 'UNKNOWN' }
1911 datadump.update(get_yaml(host))
1912 datadumps[datadump['autogen_realname']] = datadump
1913
1914 (poel, errors) = make_relations(datadumps)
1915 print "\n".join(["# WARNING: %s" % x for x in errors])
1916
1917 for host,datadump in datadumps.iteritems():
1918 try:
1919 # Convert all yes and no to boolean values
1920 def fix_boolean(dump):
1921 for key in dump.keys():
1922 if type(dump[key]) == dict:
1923 dump[key] = fix_boolean(dump[key])
1924 elif str(dump[key]).lower() in ["yes", "true"]:
1925 dump[key] = True
1926 elif str(dump[key]).lower() in ["no", "false"]:
1927 # Compass richting no (Noord Oost) is valid input
1928 if key != "compass": dump[key] = False
1929 return dump
1930 datadump = fix_boolean(datadump)
1931
1932 if datadump['rdnap_x'] and datadump['rdnap_y']:
1933 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
1934 elif datadump['latitude'] and datadump['longitude']:
1935 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
1936
1937 if datadump['nodename'].startswith('Proxy'):
1938 datadump['nodename'] = datadump['nodename'].lower()
1939
1940 for iface_key in datadump['autogen_iface_keys']:
1941 try:
1942 # All our normal wireless cards are normal APs now
1943 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
1944 datadump[iface_key]['mode'] = 'ap'
1945 # Wireless Leiden SSID have an consistent lowercase/uppercase
1946 if datadump[iface_key].has_key('ssid'):
1947 ssid = datadump[iface_key]['ssid']
1948 prefix = 'ap-WirelessLeiden-'
1949 if ssid.lower().startswith(prefix.lower()):
1950 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
1951 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
1952 datadump[iface_key]['mode'] = 'autogen-FIXME'
1953 if not datadump[iface_key].has_key('comment'):
1954 datadump[iface_key]['comment'] = 'autogen-FIXME'
1955
1956 if datadump[iface_key].has_key('ns_mac'):
1957 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
1958
1959 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
1960 datadump[iface_key] = datadump[iface_key]['desc']
1961
1962 # We are not using 802.11b anymore. OFDM is preferred over DSSS
1963 # due to better collision avoidance.
1964 if datadump[iface_key]['type'] == '11b':
1965 datadump[iface_key]['type'] = '11g'
1966
1967 # Setting 802.11g channels to de-facto standards, to avoid
1968 # un-detected sharing with other overlapping channels
1969 #
1970 # Technically we could also use channel 13 in NL, but this is not
1971 # recommended as foreign devices might not be able to select this
1972 # channel. Secondly using 1,5,9,13 instead is going to clash with
1973 # the de-facto usage of 1,6,11.
1974 #
1975 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
1976 channels_at_2400Mhz = (1,6,11)
1977 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
1978 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
1979 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
1980 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
1981
1982 # Mandatory interface keys
1983 if not datadump[iface_key].has_key('status'):
1984 datadump[iface_key]['status'] = 'planned'
1985
1986 x = datadump[iface_key]['comment']
1987 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
1988
1989 # Fixing bridge_type if none is found
1990 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
1991 if not 'bridge_type' in datadump[iface_key]:
1992 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
1993
1994 # Making sure description works
1995 if datadump[iface_key].has_key('desc'):
1996 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
1997 del datadump[iface_key]['desc']
1998 else:
1999 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2000 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2001 if response:
2002 datadump[iface_key]['comment'] = response
2003 del datadump[iface_key]['desc']
2004
2005 # Check DHCP configuration
2006 dhcp_type(datadump[iface_key])
2007
2008 # Set the compass value based on the angle between the poels
2009 if datadump[iface_key].has_key('ns_ip'):
2010 my_pool = poel[network(datadump[iface_key]['ip'])]
2011 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2012 if remote_hosts:
2013 compass_target = remote_hosts[0]
2014 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2015
2016 # Monitoring Group default
2017 if not 'monitoring_group' in datadump:
2018 datadump['monitoring_group'] = 'wleiden'
2019
2020 except Exception as e:
2021 print "# Error while processing interface %s" % iface_key
2022 raise
2023 store_yaml(datadump)
2024 except Exception as e:
2025 print "# Error while processing %s" % host
2026 raise
2027 elif sys.argv[1] == "list":
2028 use_fqdn = False
2029 if len(sys.argv) < 4 or not sys.argv[2] in ["up", "down", "planned", "all"]:
2030 usage()
2031 if sys.argv[3] == "nodes":
2032 systems = get_nodelist()
2033 elif sys.argv[3] == "proxies":
2034 systems = get_proxylist()
2035 elif sys.argv[3] == "systems":
2036 systems = get_hostlist()
2037 else:
2038 usage()
2039 if len(sys.argv) > 4:
2040 if sys.argv[4] == "fqdn":
2041 use_fqdn = True
2042 else:
2043 usage()
2044
2045 for system in systems:
2046 datadump = get_yaml(system)
2047
2048 output = datadump['autogen_fqdn'] if use_fqdn else system
2049 if sys.argv[2] == "all":
2050 print output
2051 elif datadump['status'] == sys.argv[2]:
2052 print output
2053 elif sys.argv[1] == "create":
2054 if sys.argv[2] == "network.kml":
2055 print make_network_kml.make_graph()
2056 elif sys.argv[2] == "host-ips.txt":
2057 for system in get_hostlist():
2058 datadump = get_yaml(system)
2059 ips = [datadump['masterip']]
2060 for ifkey in datadump['autogen_iface_keys']:
2061 ips.append(datadump[ifkey]['ip'].split('/')[0])
2062 print system, ' '.join(ips)
2063 elif sys.argv[2] == "host-pos.txt":
2064 for system in get_hostlist():
2065 datadump = get_yaml(system)
2066 print system, datadump['rdnap_x'], datadump['rdnap_y']
2067 elif sys.argv[2] == 'ssh_config':
2068 print '''
2069Host *.wleiden.net
2070 User root
2071
2072Host 172.16.*.*
2073 User root
2074'''
2075 for system in get_hostlist():
2076 datadump = get_yaml(system)
2077 print '''\
2078Host %s
2079 User root
2080
2081Host %s
2082 User root
2083
2084Host %s
2085 User root
2086
2087Host %s
2088 User root
2089''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2090 else:
2091 usage()
2092 else:
2093 usage()
2094 else:
2095 # Do not enable debugging for config requests as it highly clutters the output
2096 if not is_text_request():
2097 cgitb.enable()
2098 response_headers, output = process_cgi_request()
2099 print_cgi_response(response_headers, output)
2100
2101def application(environ, start_response):
2102 status = '200 OK'
2103 response_headers, output = process_cgi_request(environ)
2104 start_response(status, response_headers)
2105
2106 # Debugging only
2107 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2108 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2109 return [output]
2110
2111if __name__ == "__main__":
2112 main()
Note: See TracBrowser for help on using the repository browser.