source: genesis/tools/gformat.py@ 13315

Last change on this file since 13315 was 13315, checked in by rick, 9 years ago

Use diffrent open providers to spread risks
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 70.4 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13315 2015-07-27 21:48:51Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = []
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125def clear_cache():
126 ''' Poor mans cache implementation '''
127 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
128 datadump_cache = {}
129 interface_list_cache = {}
130 rc_conf_local_cache = {}
131 ileiden_proxies = []
132 normal_proxies = []
133 nameservers_cache = []
134
135NO_DHCP = 0
136DHCP_CLIENT = 10
137DHCP_SERVER = 20
138def dhcp_type(item):
139 if not item.has_key('dhcp'):
140 return NO_DHCP
141 elif not item['dhcp']:
142 return NO_DHCP
143 elif item['dhcp'].lower() == 'client':
144 return DHCP_CLIENT
145 else:
146 # Validation Checks
147 begin,end = map(int,item['dhcp'].split('-'))
148 if begin >= end:
149 raise ValueError("DHCP Start >= DHCP End")
150 return DHCP_SERVER
151
152def etrs2rd(lat, lon):
153 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
154 p2 = pyproj.Proj(init='EPSG:28992')
155 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
156 return (RDx, RDy)
157
158def rd2etrs(RDx, RDy):
159 p1 = pyproj.Proj(init='EPSG:28992')
160 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
161 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
162 return (lat, lon)
163
164def get_yaml(item,add_version_info=True):
165 try:
166 """ Get configuration yaml for 'item'"""
167 if datadump_cache.has_key(item):
168 return datadump_cache[item].copy()
169
170 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
171
172 # Default values
173 datadump = {
174 'autogen_revision' : 'NOTFOUND',
175 'autogen_gfile' : gfile,
176 'service_proxy_ileiden' : False,
177 }
178 f = open(gfile, 'r')
179 datadump.update(yaml.load(f,Loader=Loader))
180 if datadump['nodetype'] == 'Hybrid':
181 # Some values are defined implicitly
182 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
183 datadump['service_incoming_rdr'] = True
184 # Use some boring defaults
185 defaults = {
186 'service_proxy_normal' : False,
187 'service_accesspoint' : True,
188 'service_incoming_rdr' : False,
189 'service_concentrator' : False,
190 'monitoring_group' : 'wleiden',
191 }
192 for (key,value) in defaults.iteritems():
193 if not datadump.has_key(key):
194 datadump[key] = value
195 f.close()
196
197 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
198 if add_version_info:
199 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
200 lines = p.communicate()[0].split('\n')
201 if p.returncode == 0:
202 for line in lines:
203 if line:
204 (key, value) = line.split(': ')
205 datadump["autogen_" + key.lower().replace(' ','_')] = value
206
207 # Preformat certain needed variables for formatting and push those into special object
208 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
209
210 wlan_count=0
211 try:
212 for key in datadump['autogen_iface_keys']:
213 datadump[key]['autogen_ifbase'] = key.split('_')[1]
214 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
215 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
216 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
217 wlan_count += 1
218 else:
219 datadump[key]['autogen_ifname'] = datadump[key]['autogen_ifbase']
220 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
221 if datadump[key]['autogen_bridge'] and not 'alias' in key:
222 datadump[key]['autogen_bridge_interfaces'] = datadump[key]['members'].split()
223 except Exception as e:
224 print "# Error while processing interface %s" % key
225 raise
226
227 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
228 if dhcp_type(datadump[key]) == DHCP_SERVER]
229
230 datadump['autogen_dhcp_interfaces'] = dhcp_interfaces
231 datadump['autogen_item'] = item
232
233 datadump['autogen_realname'] = get_realname(datadump)
234 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
235 datadump['autogen_fqdn'] = datadump['autogen_realname'] + '.' + datadump['autogen_domain']
236 datadump_cache[item] = datadump.copy()
237 except Exception as e:
238 print "# Error while processing %s" % item
239 raise
240 return datadump
241
242
243def store_yaml(datadump, header=False):
244 """ Store configuration yaml for 'item'"""
245 item = datadump['autogen_item']
246 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
247
248 output = generate_wleiden_yaml(datadump, header)
249
250 f = open(gfile, 'w')
251 f.write(output)
252 f.close()
253
254
255def network(ip):
256 addr, mask = ip.split('/')
257 # Not parsing of these folks please
258 addr = parseaddr(addr)
259 mask = int(mask)
260 network = addr & ~((1 << (32 - mask)) - 1)
261 return network
262
263
264
265def make_relations(datadumps=dict()):
266 """ Process _ALL_ yaml files to get connection relations """
267 errors = []
268 poel = defaultdict(list)
269
270 if not datadumps:
271 for host in get_hostlist():
272 datadumps[host] = get_yaml(host)
273
274 for host, datadump in datadumps.iteritems():
275 try:
276 for iface_key in datadump['autogen_iface_keys']:
277 net_addr = network(datadump[iface_key]['ip'])
278 poel[net_addr] += [(host,datadump[iface_key])]
279 except (KeyError, ValueError), e:
280 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
281 continue
282 return (poel, errors)
283
284
285
286def valid_addr(addr):
287 """ Show which address is valid in which are not """
288 return str(addr).startswith('172.')
289
290def get_system_list(prefix):
291 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/%s*/wleiden.yaml" % (NODE_DIR, prefix))])
292
293get_hybridlist = lambda: get_system_list("Hybrid")
294get_nodelist = lambda: get_system_list("CNode")
295get_proxylist = lambda: get_system_list("Proxy")
296
297def get_hostlist():
298 """ Combined hosts and proxy list"""
299 return get_nodelist() + get_proxylist() + get_hybridlist()
300
301def angle_between_points(lat1,lat2,long1,long2):
302 """
303 Return Angle in radians between two GPS coordinates
304 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
305 """
306 dy = lat2 - lat1
307 dx = math.cos(lat1)*(long2 - long1)
308 angle = math.atan2(dy,dx)
309 return angle
310
311
312
313def angle_to_cd(angle):
314 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
315
316 # For easy conversion get positive degree
317 degrees = math.degrees(angle)
318 abs_degrees = 360 + degrees if degrees < 0 else degrees
319
320 # Numbers can be confusing calculate from the 4 main directions
321 p = 22.5
322 if abs_degrees < p:
323 cd = "n"
324 elif abs_degrees < (90 - p):
325 cd = "no"
326 elif abs_degrees < (90 + p):
327 cd = "o"
328 elif abs_degrees < (180 - p):
329 cd = "zo"
330 elif abs_degrees < (180 + p):
331 cd = "z"
332 elif abs_degrees < (270 - p):
333 cd = "zw"
334 elif abs_degrees < (270 + p):
335 cd = "w"
336 elif abs_degrees < (360 - p):
337 cd = "nw"
338 else:
339 cd = "n"
340 return cd
341
342
343
344def cd_between_hosts(hostA, hostB, datadumps):
345 # Using RDNAP coordinates
346 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
347 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
348 return angle_to_cd(math.atan2(dx,dy))
349
350 # GPS coordinates seems to fail somehow
351 #latA = float(datadumps[hostA]['latitude'])
352 #latB = float(datadumps[hostB]['latitude'])
353 #lonA = float(datadumps[hostA]['longitude'])
354 #lonB = float(datadumps[hostB]['longitude'])
355 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
356
357
358def generate_title(nodelist):
359 """ Main overview page """
360 items = {'root' : "." }
361 def fl(spaces, line):
362 return (' ' * spaces) + line + '\n'
363
364 output = """
365<html>
366 <head>
367 <title>Wireless leiden Configurator - GFormat</title>
368 <style type="text/css">
369 th {background-color: #999999}
370 tr:nth-child(odd) {background-color: #cccccc}
371 tr:nth-child(even) {background-color: #ffffff}
372 th, td {padding: 0.1em 1em}
373 </style>
374 </head>
375 <body>
376 <center>
377 <form type="GET" action="%(root)s">
378 <input type="hidden" name="action" value="update">
379 <input type="submit" value="Update Configuration Database (SVN)">
380 </form>
381 <table>
382 <caption><h3>Wireless Leiden Configurator</h3></caption>
383 """ % items
384
385 for node in nodelist:
386 items['node'] = node
387 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
388 for config in files:
389 items['config'] = config
390 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
391 output += fl(5, "</tr>")
392 output += """
393 </table>
394 <hr />
395 <em>%s</em>
396 </center>
397 </body>
398</html>
399 """ % __version__
400
401 return output
402
403
404
405def generate_node(node):
406 """ Print overview of all files available for node """
407 return "\n".join(files)
408
409def generate_node_overview(host):
410 """ Print overview of all files available for node """
411 datadump = get_yaml(host)
412 params = { 'host' : host }
413 output = "<em><a href='..'>Back to overview</a></em><hr />"
414 output += "<h2>Available files:</h2><ul>"
415 for cf in files:
416 params['cf'] = cf
417 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
418 output += "</ul>"
419
420 # Generate and connection listing
421 output += "<h2>Connected To:</h2><ul>"
422 (poel, errors) = make_relations()
423 for network, hosts in poel.iteritems():
424 if host in [x[0] for x in hosts]:
425 if len(hosts) == 1:
426 # Single not connected interface
427 continue
428 for remote,ifacedump in hosts:
429 if remote == host:
430 # This side of the interface
431 continue
432 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
433 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
434 output += "</ul>"
435 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
436
437 output += "<hr /><em><a href='..'>Back to overview</a></em>"
438 return output
439
440
441def generate_header(datadump, ctag="#"):
442 return """\
443%(ctag)s
444%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
445%(ctag)s
446""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
447
448
449
450def parseaddr(s):
451 """ Process IPv4 CIDR notation addr to a (binary) number """
452 f = s.split('.')
453 return (long(f[0]) << 24L) + \
454 (long(f[1]) << 16L) + \
455 (long(f[2]) << 8L) + \
456 long(f[3])
457
458
459
460def showaddr(a):
461 """ Display IPv4 addr in (dotted) CIDR notation """
462 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
463
464
465def is_member(ip, mask, canidate):
466 """ Return True if canidate is part of ip/mask block"""
467 ip_addr = parseaddr(ip)
468 ip_canidate = parseaddr(canidate)
469 mask = int(mask)
470 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
471 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
472 return ip_addr == ip_canidate
473
474
475
476def cidr2netmask(netmask):
477 """ Given a 'netmask' return corresponding CIDR """
478 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
479
480def get_network(addr, mask):
481 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
482
483
484def generate_dhcpd_conf(datadump):
485 """ Generate config file '/usr/local/etc/dhcpd.conf """
486 output = generate_header(datadump)
487 output += Template("""\
488# option definitions common to all supported networks...
489option domain-name "dhcp.{{ autogen_fqdn }}";
490
491default-lease-time 600;
492max-lease-time 7200;
493
494# Use this to enble / disable dynamic dns updates globally.
495#ddns-update-style none;
496
497# If this DHCP server is the official DHCP server for the local
498# network, the authoritative directive should be uncommented.
499authoritative;
500
501# Use this to send dhcp log messages to a different log file (you also
502# have to hack syslog.conf to complete the redirection).
503log-facility local7;
504
505#
506# Interface definitions
507#
508\n""").render(datadump)
509
510 dhcp_out = defaultdict(list)
511 for iface_key in datadump['autogen_iface_keys']:
512 ifname = datadump[iface_key]['autogen_ifname']
513 if not datadump[iface_key].has_key('comment'):
514 datadump[iface_key]['comment'] = None
515 dhcp_out[ifname].append(" ## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key])
516
517 (addr, mask) = datadump[iface_key]['ip'].split('/')
518 datadump[iface_key]['autogen_addr'] = addr
519 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
520 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
521 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
522 dhcp_out[ifname].append(" subnet %(autogen_subnet)s netmask %(autogen_netmask)s {\n ### not autoritive\n }\n" % \
523 datadump[iface_key])
524 continue
525
526 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
527 dhcp_part = ".".join(addr.split('.')[0:3])
528 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
529 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
530
531 # Assume the first 10 IPs could be used for static entries
532 if 'no_portal' in datadump:
533 fixed = 5
534 for mac in datadump['no_portal']:
535 dhcp_out[ifname].append("""\
536 host fixed-%(ifname)s-%(fixed)s {
537 hardware ethernet %(mac)s;
538 fixed-address %(prefix)s.%(fixed)s;
539 }
540""" % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed })
541 fixed += 1
542
543 dhcp_out[ifname].append("""\
544 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
545 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
546 option routers %(autogen_addr)s;
547 option domain-name-servers %(autogen_addr)s;
548 }
549""" % datadump[iface_key])
550
551 for ifname,value in dhcp_out.iteritems():
552 output += ("shared-network %s {\n" % ifname) + ''.join(value) + '}\n\n'
553 return output
554
555
556
557def generate_dnsmasq_conf(datadump):
558 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
559 output = generate_header(datadump)
560 output += Template("""\
561# DHCP server options
562dhcp-authoritative
563dhcp-fqdn
564domain=dhcp.{{ autogen_fqdn }}
565domain-needed
566expand-hosts
567log-async=100
568
569# Low memory footprint
570cache-size=10000
571
572\n""").render(datadump)
573
574 for iface_key in datadump['autogen_iface_keys']:
575 if not datadump[iface_key].has_key('comment'):
576 datadump[iface_key]['comment'] = None
577 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
578
579 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
580 output += "# not autoritive\n\n"
581 continue
582
583 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
584 (ip, cidr) = datadump[iface_key]['ip'].split('/')
585 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
586
587 dhcp_part = ".".join(ip.split('.')[0:3])
588 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
589 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
590 output += "dhcp-range=%(autogen_ifname)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
591
592 return output
593
594
595def make_interface_list(datadump):
596 if interface_list_cache.has_key(datadump['autogen_item']):
597 return (interface_list_cache[datadump['autogen_item']])
598 # lo0 configuration:
599 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
600 # - masterip is special as it needs to be assigned to at
601 # least one interface, so if not used assign to lo0
602 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
603 dhclient_if = {'lo0' : False}
604
605 # XXX: Find some way of send this output nicely
606 output = ''
607
608 masterip_used = False
609 for iface_key in datadump['autogen_iface_keys']:
610 if datadump[iface_key]['ip'].startswith(datadump['masterip']):
611 masterip_used = True
612 break
613 if not masterip_used:
614 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
615
616 for iface_key in datadump['autogen_iface_keys']:
617 ifacedump = datadump[iface_key]
618 ifname = ifacedump['autogen_ifname']
619
620 # Flag dhclient is possible
621 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
622 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
623
624 # Add interface IP to list
625 item = (ifacedump['ip'], ifacedump['comment'])
626 if addrs_list.has_key(ifname):
627 addrs_list[ifname].append(item)
628 else:
629 addrs_list[ifname] = [item]
630
631 # Alias only needs IP assignment for now, this might change if we
632 # are going to use virtual accesspoints
633 if "alias" in iface_key:
634 continue
635
636 # XXX: Might want to deduct type directly from interface name
637 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
638 # Default to station (client) mode
639 ifacedump['autogen_wlanmode'] = "sta"
640 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
641 ifacedump['autogen_wlanmode'] = "ap"
642
643 if not ifacedump.has_key('channel'):
644 if ifacedump['type'] == '11a':
645 ifacedump['channel'] = 36
646 else:
647 ifacedump['channel'] = 1
648
649 # Allow special hacks at the back like wds and stuff
650 if not ifacedump.has_key('extra'):
651 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
652 else:
653 ifacedump['autogen_extra'] = ifacedump['extra']
654
655 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
656 output += ("create_args_%(autogen_ifname)s='wlanmode %(autogen_wlanmode)s mode " +\
657 "%(type)s ssid %(ssid)s %(autogen_extra)s channel %(channel)s'\n") % ifacedump
658 output += "\n"
659
660 elif ifacedump['type'] in ['ethernet', 'eth']:
661 # No special config needed besides IP
662 if ifacedump['autogen_bridge']:
663 output += "cloned_interfaces='%(autogen_ifname)s'\n" % ifacedump
664 output += "ifconfig_%s='addm %s up'\n" % (ifacedump['autogen_ifname'], ' addm '.join(ifacedump['autogen_bridge_interfaces']))
665 for member in ifacedump['autogen_bridge_interfaces']:
666 output += "ifconfig_%s='up'\n" % member
667 output += "\n"
668 else:
669 assert False, "Unknown type " + ifacedump['type']
670
671 store = (addrs_list, dhclient_if, output)
672 interface_list_cache[datadump['autogen_item']] = store
673 return(store)
674
675
676
677def generate_rc_conf_local(datadump):
678 """ Generate configuration file '/etc/rc.conf.local' """
679 item = datadump['autogen_item']
680 if rc_conf_local_cache.has_key(item):
681 return rc_conf_local_cache[item]
682
683 if not datadump.has_key('ileiden'):
684 datadump['autogen_ileiden_enable'] = False
685 else:
686 datadump['autogen_ileiden_enable'] = datadump['ileiden']
687
688 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
689
690 if not ileiden_proxies or not normal_proxies:
691 for proxy in get_proxylist():
692 proxydump = get_yaml(proxy)
693 if proxydump['ileiden']:
694 ileiden_proxies.append(proxydump)
695 else:
696 normal_proxies.append(proxydump)
697 for host in get_hybridlist():
698 hostdump = get_yaml(host)
699 if hostdump['status'] == 'up':
700 if hostdump['service_proxy_ileiden']:
701 ileiden_proxies.append(hostdump)
702 if hostdump['service_proxy_normal']:
703 normal_proxies.append(hostdump)
704
705 datadump['autogen_ileiden_proxies'] = ileiden_proxies
706 datadump['autogen_normal_proxies'] = normal_proxies
707 datadump['autogen_ileiden_proxies_ips'] = ','.join([x['masterip'] for x in ileiden_proxies])
708 datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
709 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
710 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
711
712 output = generate_header(datadump, "#");
713 output += render_template(datadump, """\
714hostname='{{ autogen_fqdn }}'
715location='{{ location }}'
716nodetype="{{ nodetype }}"
717
718#
719# Configured listings
720#
721captive_portal_whitelist=""
722{% if nodetype == "Proxy" %}
723#
724# Proxy Configuration
725#
726{% if gateway -%}
727defaultrouter="{{ gateway }}"
728{% else -%}
729#defaultrouter="NOTSET"
730{% endif -%}
731internalif="{{ internalif }}"
732ileiden_enable="{{ autogen_ileiden_enable }}"
733gateway_enable="{{ autogen_ileiden_enable }}"
734pf_enable="yes"
735pf_rules="/etc/pf.conf"
736{% if autogen_ileiden_enable -%}
737pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
738lvrouted_enable="{{ autogen_ileiden_enable }}"
739lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
740{% else -%}
741pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
742{% endif -%}
743{% if internalroute -%}
744static_routes="wleiden"
745route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
746{% endif -%}
747
748{% elif nodetype == "Hybrid" %}
749 #
750 # Hybrid Configuration
751 #
752 list_ileiden_proxies="
753 {% for item in autogen_ileiden_proxies -%}
754 {{ "%-16s"|format(item.masterip) }} # {{ item.autogen_realname }}
755 {% endfor -%}
756 "
757 list_normal_proxies="
758 {% for item in autogen_normal_proxies -%}
759 {{ "%-16s"|format(item.masterip) }} # {{ item.autogen_realname }}
760 {% endfor -%}
761 "
762
763 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',')|default('none', true) }}"
764 externalif="{{ externalif|default('vr0', true) }}"
765 masterip="{{ masterip }}"
766
767 # Defined services
768 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
769 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
770 service_accesspoint="{{ service_accesspoint|yesorno }}"
771 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
772 service_concentrator="{{ service_concentrator|yesorno }}"
773 #
774
775 {% if service_proxy_ileiden %}
776 pf_rules="/etc/pf.hybrid.conf"
777 {% if service_concentrator %}
778 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
779 {% else %}
780 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
781 {% endif %}
782 pf_flags="$pf_flags -D publicnat=80,443"
783 lvrouted_flags="$lvrouted_flags -g"
784 {% elif service_proxy_normal or service_incoming_rdr %}
785 pf_rules="/etc/pf.hybrid.conf"
786 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
787 pf_flags="$pf_flags -D publicnat=0"
788 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
789 named_setfib="1"
790 tinyproxy_setfib="1"
791 dnsmasq_setfib="1"
792 sshd_setfib="1"
793 {% else %}
794 named_auto_forward_only="YES"
795 pf_rules="/etc/pf.node.conf"
796 pf_flags=""
797 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
798 {% endif %}
799 {% if service_concentrator %}
800 # Do mind installing certificates is NOT done automatically for security reasons
801 openvpn_enable="YES"
802 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
803 {% endif %}
804
805 {% if service_proxy_normal %}
806 tinyproxy_enable="yes"
807 {% else %}
808 pen_wrapper_enable="yes"
809 {% endif %}
810
811 {% if service_accesspoint %}
812 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
813 {% endif %}
814
815 {% if board == "ALIX2" %}
816 #
817 # ''Fat'' configuration, board has 256MB RAM
818 #
819 dnsmasq_enable="NO"
820 named_enable="YES"
821 {% if autogen_dhcp_interfaces -%}
822 dhcpd_enable="YES"
823 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
824 {% endif -%}
825 {% endif -%}
826
827 {% if gateway %}
828 defaultrouter="{{ gateway }}"
829 {% endif %}
830{% elif nodetype == "CNode" %}
831#
832# NODE iLeiden Configuration
833#
834
835# iLeiden Proxies {{ autogen_ileiden_proxies_names }}
836list_ileiden_proxies="{{ autogen_ileiden_proxies_ips }}"
837# normal Proxies {{ autogen_normal_proxies_names }}
838list_normal_proxies="{{ autogen_normal_proxies_ips }}"
839
840captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
841
842lvrouted_flags="-u -s s00p3rs3kr3t -m 28 -z $list_ileiden_proxies"
843{% endif %}
844
845#
846# Interface definitions
847#\n
848""")
849
850 (addrs_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
851 output += extra_ouput.strip() + "\n"
852
853 # Print IP address which needs to be assigned over here
854 output += "\n"
855 for iface,addrs in sorted(addrs_list.iteritems()):
856 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
857 output += "# %s || %s || %s\n" % (iface, addr, comment)
858
859 # Write DHCLIENT entry
860 if dhclient_if[iface]:
861 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
862
863 # Make sure the external address is always first as this is needed in the
864 # firewall setup
865 addrs = sorted(
866 [x for x in addrs if not '0.0.0.0' in x[0]],
867 key=lambda x: x[0].split('.')[0],
868 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
869 )
870 addr_str = " ".join([x[0] for x in addrs])
871 output += "ipv4_addrs_%s='%s'\n\n" % (iface, addr_str)
872
873 rc_conf_local_cache[datadump['autogen_item']] = output
874 return output
875
876
877
878
879def get_all_configs():
880 """ Get dict with key 'host' with all configs present """
881 configs = dict()
882 for host in get_hostlist():
883 datadump = get_yaml(host)
884 configs[host] = datadump
885 return configs
886
887
888def get_interface_keys(config):
889 """ Quick hack to get all interface keys, later stage convert this to a iterator """
890 return sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
891
892
893def get_used_ips(configs):
894 """ Return array of all IPs used in config files"""
895 ip_list = []
896 for config in configs:
897 ip_list.append(config['masterip'])
898 for iface_key in get_interface_keys(config):
899 l = config[iface_key]['ip']
900 addr, mask = l.split('/')
901 # Special case do not process
902 if valid_addr(addr):
903 ip_list.append(addr)
904 else:
905 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
906 return sorted(ip_list)
907
908
909
910def get_nameservers(max_servers=None):
911 if nameservers_cache:
912 return nameservers_cache[0:max_servers]
913
914 for host in get_hybridlist():
915 hostdump = get_yaml(host)
916 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
917 nameservers_cache.append((hostdump['masterip'], hostdump['autogen_realname']))
918 for host in get_proxylist():
919 hostdump = get_yaml(host)
920 if hostdump['status'] == 'up':
921 nameservers_cache.append((hostdump['masterip'], hostdump['autogen_realname']))
922
923 return nameservers_cache[0:max_servers]
924
925
926def generate_resolv_conf(datadump):
927 """ Generate configuration file '/etc/resolv.conf' """
928 # XXX: This should properly going to be an datastructure soon
929 datadump['autogen_header'] = generate_header(datadump, "#")
930 datadump['autogen_edge_nameservers'] = ''
931
932
933 for masterip,realname in get_nameservers():
934 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
935
936 return Template("""\
937{{ autogen_header }}
938search wleiden.net
939
940# Try local (cache) first
941nameserver 127.0.0.1
942
943{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
944nameserver 8.8.8.8 # Google Public NameServer
945nameserver 4.2.2.1 # Level3 Public NameServer
946{% else -%}
947# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
948{{ autogen_edge_nameservers }}
949{% endif -%}
950""").render(datadump)
951
952
953
954def generate_ntp_conf(datadump):
955 """ Generate configuration file '/etc/ntp.conf' """
956 # XXX: This should properly going to be an datastructure soon
957
958 datadump['autogen_header'] = generate_header(datadump, "#")
959 datadump['autogen_ntp_servers'] = ''
960 for host in get_proxylist():
961 hostdump = get_yaml(host)
962 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(autogen_realname)s\n" % hostdump
963 for host in get_hybridlist():
964 hostdump = get_yaml(host)
965 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
966 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(autogen_realname)s\n" % hostdump
967
968 return Template("""\
969{{ autogen_header }}
970
971{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
972# Machine hooked to internet.
973server 0.nl.pool.ntp.org iburst maxpoll 9
974server 1.nl.pool.ntp.org iburst maxpoll 9
975server 2.nl.pool.ntp.org iburst maxpoll 9
976server 3.nl.pool.ntp.org iburst maxpoll 9
977{% else -%}
978# Local Wireless Leiden NTP Servers.
979server 0.pool.ntp.wleiden.net iburst maxpoll 9
980server 1.pool.ntp.wleiden.net iburst maxpoll 9
981server 2.pool.ntp.wleiden.net iburst maxpoll 9
982server 3.pool.ntp.wleiden.net iburst maxpoll 9
983
984# All the configured NTP servers
985{{ autogen_ntp_servers }}
986{% endif %}
987
988# If a server loses sync with all upstream servers, NTP clients
989# no longer follow that server. The local clock can be configured
990# to provide a time source when this happens, but it should usually
991# be configured on just one server on a network. For more details see
992# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
993# The use of Orphan Mode may be preferable.
994#
995server 127.127.1.0
996fudge 127.127.1.0 stratum 10
997""").render(datadump)
998
999
1000def generate_pf_hybrid_conf_local(datadump):
1001 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1002 datadump['autogen_header'] = generate_header(datadump, "#")
1003 return Template("""\
1004{{ autogen_header }}
1005
1006# Redirect some internal facing services outside (7)
1007# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1008{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1009rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1010{% endfor -%}
1011""").render(datadump)
1012
1013def generate_motd(datadump):
1014 """ Generate configuration file '/etc/motd' """
1015 output = Template("""\
1016FreeBSD run ``service motd onestart'' to make me look normal
1017
1018 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1019 Loc: {{ location }}
1020
1021Services:
1022{% if board == "ALIX2" -%}
1023{{" -"}} Core Node ({{ board }})
1024{% else -%}
1025{{" -"}} Hulp Node ({{ board }})
1026{% endif -%}
1027{% if service_proxy_normal -%}
1028{{" -"}} Normal Proxy
1029{% endif -%}
1030{% if service_proxy_ileiden -%}
1031{{" -"}} iLeiden Proxy
1032{% endif -%}
1033{% if service_incoming_rdr -%}
1034{{" -"}} Incoming port redirects
1035{% endif %}
1036Interlinks:\n
1037""").render(datadump)
1038
1039 (addrs_list, dhclient_if, extra_ouput) = make_interface_list(datadump)
1040 # Just nasty hack to make the formatting looks nice
1041 iface_len = max(map(len,addrs_list.keys()))
1042 addr_len = max(map(len,[x[0] for x in [x[0] for x in addrs_list.values()]]))
1043 for iface,addrs in sorted(addrs_list.iteritems()):
1044 if iface in ['lo0']:
1045 continue
1046 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1047 output += " - %s || %s || %s\n" % (iface.ljust(iface_len), addr.ljust(addr_len), comment)
1048
1049 output += '\n'
1050 output += """\
1051Attached bridges:
1052"""
1053 has_item = False
1054 for iface_key in datadump['autogen_iface_keys']:
1055 ifacedump = datadump[iface_key]
1056 if ifacedump.has_key('ns_ip'):
1057 ifacedump['autogen_ns_ip'] = ifacedump['ns_ip'].split('/')[0]
1058 has_item = True
1059 output += " - %(autogen_ifname)s || %(mode)s || https://%(autogen_ns_ip)s\n" % ifacedump
1060 if not has_item:
1061 output += " - none\n"
1062
1063 return output
1064
1065
1066def format_yaml_value(value):
1067 """ Get yaml value in right syntax for outputting """
1068 if isinstance(value,str):
1069 output = '"%s"' % value
1070 else:
1071 output = value
1072 return output
1073
1074
1075
1076def format_wleiden_yaml(datadump):
1077 """ Special formatting to ensure it is editable"""
1078 output = "# Genesis config yaml style\n"
1079 output += "# vim:ts=2:et:sw=2:ai\n"
1080 output += "#\n"
1081 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1082 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1083 if key == 'rdr_rules':
1084 output += '%-10s:\n' % 'rdr_rules'
1085 for rdr_rule in datadump[key]:
1086 output += '- %s\n' % rdr_rule
1087 else:
1088 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1089
1090 output += "\n\n"
1091
1092 # Format (key, required)
1093 key_order = (
1094 ('comment', True),
1095 ('ip', True),
1096 ('desc', True),
1097 ('sdesc', True),
1098 ('mode', True),
1099 ('type', True),
1100 ('extra_type', False),
1101 ('channel', False),
1102 ('ssid', False),
1103 ('wlan_mac', False),
1104 ('dhcp', True),
1105 ('compass', False),
1106 ('distance', False),
1107 ('ns_ip', False),
1108 ('repeater_ip', False),
1109 ('bullet2_ip', False),
1110 ('ns_mac', False),
1111 ('bullet2_mac', False),
1112 ('ns_type', False),
1113 ('bridge_type', False),
1114 ('members', True),
1115 ('status', True),
1116 )
1117
1118 for iface_key in sorted(iface_keys):
1119 try:
1120 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1121 if remainder:
1122 raise KeyError("invalid keys: %s" % remainder)
1123
1124 output += "%s:\n" % iface_key
1125 for key,required in key_order:
1126 if datadump[iface_key].has_key(key):
1127 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1128 output += "\n\n"
1129 except Exception as e:
1130 print "# Error while processing interface %s" % iface_key
1131 raise
1132
1133 return output
1134
1135
1136
1137def generate_wleiden_yaml(datadump, header=True):
1138 """ Generate (petty) version of wleiden.yaml"""
1139 output = generate_header(datadump, "#") if header else ''
1140
1141 for key in datadump.keys():
1142 if key.startswith('autogen_'):
1143 del datadump[key]
1144 # Interface autogen cleanups
1145 elif type(datadump[key]) == dict:
1146 for key2 in datadump[key].keys():
1147 if key2.startswith('autogen_'):
1148 del datadump[key][key2]
1149
1150 output += format_wleiden_yaml(datadump)
1151 return output
1152
1153def generate_nanostation_config(datadump, iface, ns_type):
1154 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1155 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1156
1157 datadump.update(datadump['iface_%s' % iface])
1158
1159 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1160
1161def generate_yaml(datadump):
1162 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1163
1164
1165
1166def generate_config(node, config, datadump=None):
1167 """ Print configuration file 'config' of 'node' """
1168 output = ""
1169 try:
1170 # Load config file
1171 if datadump == None:
1172 datadump = get_yaml(node)
1173
1174 if config == 'wleiden.yaml':
1175 output += generate_wleiden_yaml(datadump)
1176 elif config == 'authorized_keys':
1177 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1178 output += f.read()
1179 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1180 # Fetch local keys if existing
1181 if os.path.exists(node_keys):
1182 output += open(node_keys, 'r').read()
1183 f.close()
1184 elif config == 'dnsmasq.conf':
1185 output += generate_dnsmasq_conf(datadump)
1186 elif config == 'dhcpd.conf':
1187 output += generate_dhcpd_conf(datadump)
1188 elif config == 'rc.conf.local':
1189 output += generate_rc_conf_local(datadump)
1190 elif config == 'resolv.conf':
1191 output += generate_resolv_conf(datadump)
1192 elif config == 'ntp.conf':
1193 output += generate_ntp_conf(datadump)
1194 elif config == 'motd':
1195 output += generate_motd(datadump)
1196 elif config == 'pf.hybrid.conf.local':
1197 output += generate_pf_hybrid_conf_local(datadump)
1198 elif config.startswith('vr'):
1199 interface, ns_type = config.strip('.yaml').split('-')
1200 output += generate_nanostation_config(datadump, interface, ns_type)
1201 else:
1202 assert False, "Config not found!"
1203 except IOError, e:
1204 output += "[ERROR] Config file not found"
1205 return output
1206
1207
1208
1209def process_cgi_request(environ=os.environ):
1210 """ When calling from CGI """
1211 response_headers = []
1212 content_type = 'text/plain'
1213
1214 # Update repository if requested
1215 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1216 if form and form.has_key("action") and "update" in form["action"]:
1217 output = "[INFO] Updating subverion, please wait...\n"
1218 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1219 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1220 output += "[INFO] All done, redirecting in 5 seconds"
1221 response_headers += [
1222 ('Refresh', '5; url=.'),
1223 ]
1224 reload_cache()
1225 else:
1226 base_uri = environ['PATH_INFO']
1227 uri = base_uri.strip('/').split('/')
1228
1229 output = "Template Holder"
1230 if base_uri.endswith('/create/network.kml'):
1231 content_type='application/vnd.google-earth.kml+xml'
1232 output = make_network_kml.make_graph()
1233 elif base_uri.endswith('/api/get/nodeplanner.json'):
1234 content_type='application/json'
1235 output = make_network_kml.make_nodeplanner_json()
1236 elif not uri[0]:
1237 if is_text_request(environ):
1238 output = '\n'.join(get_hostlist())
1239 else:
1240 content_type = 'text/html'
1241 output = generate_title(get_hostlist())
1242 elif len(uri) == 1:
1243 if is_text_request(environ):
1244 output = generate_node(uri[0])
1245 else:
1246 content_type = 'text/html'
1247 output = generate_node_overview(uri[0])
1248 elif len(uri) == 2:
1249 output = generate_config(uri[0], uri[1])
1250 else:
1251 assert False, "Invalid option"
1252
1253 # Return response
1254 response_headers += [
1255 ('Content-type', content_type),
1256 ('Content-Length', str(len(output))),
1257 ]
1258 return(response_headers, str(output))
1259
1260
1261def get_realname(datadump):
1262 # Proxy naming convention is special, as the proxy name is also included in
1263 # the nodename, when it comes to the numbered proxies.
1264 if datadump['nodetype'] == 'Proxy':
1265 realname = datadump['nodetype'] + datadump['nodename'].replace('proxy','')
1266 else:
1267 # By default the full name is listed and also a shortname CNAME for easy use.
1268 realname = datadump['nodetype'] + datadump['nodename']
1269 return(realname)
1270
1271
1272
1273def make_dns(output_dir = 'dns', external = False):
1274 items = dict()
1275
1276 # hostname is key, IP is value
1277 wleiden_zone = defaultdict(list)
1278 wleiden_cname = dict()
1279
1280 pool = dict()
1281 for node in get_hostlist():
1282 datadump = get_yaml(node)
1283
1284 # Proxy naming convention is special
1285 fqdn = datadump['autogen_realname']
1286 if datadump['nodetype'] in ['CNode', 'Hybrid']:
1287 wleiden_cname[datadump['nodename']] = fqdn
1288
1289 if datadump.has_key('rdr_host'):
1290 remote_target = datadump['rdr_host']
1291 elif datadump.has_key('remote_access') and datadump['remote_access']:
1292 remote_target = datadump['remote_access'].split(':')[0]
1293 else:
1294 remote_target = None
1295
1296 if remote_target:
1297 try:
1298 parseaddr(remote_target)
1299 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1300 except (IndexError, ValueError):
1301 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1302
1303
1304 wleiden_zone[fqdn].append((datadump['masterip'], True))
1305
1306 # Hacking to get proper DHCP IPs and hostnames
1307 for iface_key in get_interface_keys(datadump):
1308 iface_name = iface_key.replace('_','-')
1309 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1310 try:
1311 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1312 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1313 dhcp_part = ".".join(ip.split('.')[0:3])
1314 if ip != datadump['masterip']:
1315 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, False))
1316 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1317 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1318 except (AttributeError, ValueError, KeyError):
1319 # First push it into a pool, to indentify the counter-part later on
1320 addr = parseaddr(ip)
1321 cidr = int(cidr)
1322 addr = addr & ~((1 << (32 - cidr)) - 1)
1323 if pool.has_key(addr):
1324 pool[addr] += [(iface_name, fqdn, ip)]
1325 else:
1326 pool[addr] = [(iface_name, fqdn, ip)]
1327 continue
1328
1329
1330
1331 # WL uses an /29 to configure an interface. IP's are ordered like this:
1332 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1333
1334 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1335
1336 # Automatic naming convention of interlinks namely 2 + remote.lower()
1337 for (key,value) in pool.iteritems():
1338 # Make sure they are sorted from low-ip to high-ip
1339 value = sorted(value, key=lambda x: parseaddr(x[2]))
1340
1341 if len(value) == 1:
1342 (iface_name, fqdn, ip) = value[0]
1343 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1344
1345 # Device DNS names
1346 if 'cnode' in fqdn.lower():
1347 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1348 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1349
1350 elif len(value) == 2:
1351 (a_iface_name, a_fqdn, a_ip) = value[0]
1352 (b_iface_name, b_fqdn, b_ip) = value[1]
1353 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1354 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1355
1356 # Device DNS names
1357 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1358 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1359 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1360 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1361 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1362 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1363 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1364
1365 else:
1366 pool_members = [k[1] for k in value]
1367 for item in value:
1368 (iface_name, fqdn, ip) = item
1369 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1370
1371 # Include static DNS entries
1372 # XXX: Should they override the autogenerated results?
1373 # XXX: Convert input to yaml more useable.
1374 # Format:
1375 ##; this is a comment
1376 ## roomburgh=CNodeRoomburgh1
1377 ## apkerk1.CNodeVosko=172.17.176.8 ;this as well
1378 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1379
1380 # Hack to allow special entries, for development
1381 wleiden_raw = {}
1382
1383 for line in dns_list:
1384 reverse = False
1385 k, items = line.items()[0]
1386 if type(items) == dict:
1387 if items.has_key('reverse'):
1388 reverse = items['reverse']
1389 items = items['a']
1390 else:
1391 items = items['cname']
1392 items = [items] if type(items) != list else items
1393 for item in items:
1394 if item.startswith('IN '):
1395 wleiden_raw[k] = item
1396 elif valid_addr(item):
1397 wleiden_zone[k].append((item, reverse))
1398 else:
1399 wleiden_cname[k] = item
1400
1401 # Hack to get dynamic pool listing
1402 def chunks(l, n):
1403 return [l[i:i+n] for i in range(0, len(l), n)]
1404
1405 ntp_servers = [x[0] for x in get_nameservers()]
1406 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1407 for ntp_server in chunk:
1408 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1409
1410 details = dict()
1411 # 24 updates a day allowed
1412 details['serial'] = time.strftime('%Y%m%d%H')
1413
1414 if external:
1415 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1416 else:
1417 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1418
1419 details['master'] = dns_masters[0]
1420 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1421
1422 dns_header = '''
1423$TTL 3h
1424%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1425 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1426
1427%(ns_servers)s
1428 \n'''
1429
1430
1431 if not os.path.isdir(output_dir):
1432 os.makedirs(output_dir)
1433 details['zone'] = 'wleiden.net'
1434 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1435 f.write(dns_header % details)
1436
1437 for host,items in wleiden_zone.iteritems():
1438 for ip,reverse in items:
1439 if ip not in ['0.0.0.0']:
1440 f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
1441 for source,dest in wleiden_cname.iteritems():
1442 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1443 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1444 for source, dest in wleiden_raw.iteritems():
1445 f.write("%s.wleiden.net. %s\n" % (source, dest))
1446 f.close()
1447
1448 # Create whole bunch of specific sub arpa zones. To keep it compliant
1449 for s in range(16,32):
1450 details['zone'] = '%i.172.in-addr.arpa' % s
1451 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1452 f.write(dns_header % details)
1453
1454 #XXX: Not effient, fix to proper data structure and do checks at other
1455 # stages
1456 for host,items in wleiden_zone.iteritems():
1457 for ip,reverse in items:
1458 if not reverse:
1459 continue
1460 if valid_addr(ip):
1461 if valid_addr(ip):
1462 if int(ip.split('.')[1]) == s:
1463 rev_ip = '.'.join(reversed(ip.split('.')))
1464 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1465 f.close()
1466
1467
1468def usage():
1469 print """Usage: %(prog)s <argument>
1470Argument:
1471\tstandalone [port] = Run configurator webserver [8000]
1472\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1473\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1474\tfull-export = Generate yaml export script for heatmap.
1475\tstatic [outputdir] = Generate all config files and store on disk
1476\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1477\ttest <node> [<file>] = Receive output for certain node [all files].
1478\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1479\tlist <status> <items> = List systems which have certain status
1480
1481Arguments:
1482\t<node> = NodeName (example: HybridRick)
1483\t<file> = %(files)s
1484\t<status> = all|up|down|planned
1485\t<items> = systems|nodes|proxies
1486
1487NOTE FOR DEVELOPERS; you can test your changes like this:
1488 BEFORE any changes in this code:
1489 $ ./gformat.py static /tmp/pre
1490 AFTER the changes:
1491 $ ./gformat.py static /tmp/post
1492 VIEW differences and VERIFY all are OK:
1493 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1494""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1495 exit(0)
1496
1497
1498def is_text_request(environ=os.environ):
1499 """ Find out whether we are calling from the CLI or any text based CLI utility """
1500 try:
1501 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1502 except KeyError:
1503 return True
1504
1505def switchFormat(setting):
1506 if setting:
1507 return "YES"
1508 else:
1509 return "NO"
1510
1511def rlinput(prompt, prefill=''):
1512 import readline
1513 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1514 try:
1515 return raw_input(prompt)
1516 finally:
1517 readline.set_startup_hook()
1518
1519def fix_conflict(left, right, default='i'):
1520 while True:
1521 print "## %-30s | %-30s" % (left, right)
1522 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1523 if not c:
1524 c = default
1525
1526 if c in ['l','1']:
1527 return left
1528 elif c in ['r','2']:
1529 return right
1530 elif c in ['e', '3']:
1531 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1532 elif c in ['i', '4']:
1533 return None
1534 else:
1535 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1536
1537
1538
1539def print_cgi_response(response_headers, output):
1540 """Could we not use some kind of wsgi wrapper to make this output?"""
1541 for header in response_headers:
1542 print "%s: %s" % header
1543 print
1544 print output
1545
1546
1547def fill_cache():
1548 ''' Poor man re-loading of few cache items (the slow ones) '''
1549 for host in get_hostlist():
1550 get_yaml(host)
1551
1552
1553def reload_cache():
1554 clear_cache()
1555 fill_cache()
1556
1557
1558def main():
1559 """Hard working sub"""
1560 # Allow easy hacking using the CLI
1561 if not os.environ.has_key('PATH_INFO'):
1562 if len(sys.argv) < 2:
1563 usage()
1564
1565 if sys.argv[1] == "standalone":
1566 import SocketServer
1567 import CGIHTTPServer
1568 # Hop to the right working directory.
1569 os.chdir(os.path.dirname(__file__))
1570 try:
1571 PORT = int(sys.argv[2])
1572 except (IndexError,ValueError):
1573 PORT = 8000
1574
1575 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1576 """ Serve this CGI from the root of the webserver """
1577 def is_cgi(self):
1578 if "favicon" in self.path:
1579 return False
1580
1581 self.cgi_info = (os.path.basename(__file__), self.path)
1582 self.path = ''
1583 return True
1584 handler = MyCGIHTTPRequestHandler
1585 SocketServer.TCPServer.allow_reuse_address = True
1586 httpd = SocketServer.TCPServer(("", PORT), handler)
1587 httpd.server_name = 'localhost'
1588 httpd.server_port = PORT
1589
1590 logger.info("serving at port %s", PORT)
1591 try:
1592 httpd.serve_forever()
1593 except KeyboardInterrupt:
1594 httpd.shutdown()
1595 logger.info("All done goodbye")
1596 elif sys.argv[1] == "test":
1597 # Basic argument validation
1598 try:
1599 node = sys.argv[2]
1600 datadump = get_yaml(node)
1601 except IndexError:
1602 print "Invalid argument"
1603 exit(1)
1604 except IOError as e:
1605 print e
1606 exit(1)
1607
1608
1609 # Get files to generate
1610 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1611
1612 # Actual config generation
1613 for config in gen_files:
1614 logger.info("## Generating %s %s", node, config)
1615 print generate_config(node, config, datadump)
1616 elif sys.argv[1] == "test-cgi":
1617 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1618 os.environ['SCRIPT_NAME'] = __file__
1619 response_headers, output = process_cgi_request()
1620 print_cgi_response(response_headers, output)
1621 elif sys.argv[1] == "static":
1622 items = dict()
1623 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1624 for node in get_hostlist():
1625 items['node'] = node
1626 items['wdir'] = "%(output_dir)s/%(node)s" % items
1627 if not os.path.isdir(items['wdir']):
1628 os.makedirs(items['wdir'])
1629 datadump = get_yaml(node)
1630 for config in files:
1631 items['config'] = config
1632 logger.info("## Generating %(node)s %(config)s" % items)
1633 f = open("%(wdir)s/%(config)s" % items, "w")
1634 f.write(generate_config(node, config, datadump))
1635 f.close()
1636 elif sys.argv[1] == "wind-export":
1637 items = dict()
1638 for node in get_hostlist():
1639 datadump = get_yaml(node)
1640 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1641 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1642 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1643 VALUES (
1644 (SELECT id FROM users WHERE username = 'rvdzwet'),
1645 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1646 'Y');""" % datadump
1647 #for config in files:
1648 # items['config'] = config
1649 # print "## Generating %(node)s %(config)s" % items
1650 # f = open("%(wdir)s/%(config)s" % items, "w")
1651 # f.write(generate_config(node, config, datadump))
1652 # f.close()
1653 for node in get_hostlist():
1654 datadump = get_yaml(node)
1655 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1656 ifacedump = datadump[iface_key]
1657 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1658 ifacedump['nodename'] = datadump['nodename']
1659 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1660 ifacedump['channel'] = 0
1661 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1662 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1663 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1664 elif sys.argv[1] == "nagios-export":
1665 try:
1666 heavy_load = (sys.argv[2] == "--heavy-load")
1667 except IndexError:
1668 heavy_load = False
1669
1670 hostgroup_details = {
1671 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1672 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1673 'walphen' : 'Stichting Wireless Alphen',
1674 'westeinder' : 'Westeinder Plassen',
1675 }
1676
1677 # Convert IP to Host
1678 ip2host = {'root' : 'root'}
1679 for host in get_hostlist():
1680 datadump = get_yaml(host)
1681 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1682 for iface in datadump['autogen_iface_keys']:
1683 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1684
1685 # Find dependency tree based on output of lvrouted.mytree of nearest node
1686 parents = defaultdict(list)
1687 stack = ['root']
1688 prev_depth = 0
1689 for line in open('lvrouted.mytree').readlines():
1690 depth = line.count('\t')
1691 ip = line.strip().split()[0]
1692
1693 if prev_depth < depth:
1694 try:
1695 parents[ip2host[ip]].append(ip2host[stack[-1]])
1696 except KeyError as e:
1697 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1698 stack.append(ip)
1699 elif prev_depth > depth:
1700 stack = stack[:(depth - prev_depth)]
1701 elif prev_depth == depth:
1702 try:
1703 parents[ip2host[ip]].append(ip2host[stack[-1]])
1704 except KeyError as e:
1705 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1706
1707
1708 prev_depth = depth
1709 # Observe that some nodes has themself as parent or multiple parents
1710 # for now take only the first parent, other behaviour is yet to be explained
1711
1712
1713
1714 params = {
1715 'check_interval' : 5 if heavy_load else 120,
1716 'retry_interval' : 1 if heavy_load else 10,
1717 'max_check_attempts' : 10 if heavy_load else 6,
1718 'notification_interval': 120 if heavy_load else 240,
1719 }
1720
1721 print '''\
1722define host {
1723 name wleiden-node ; Default Node Template
1724 use generic-host ; Use the standard template as initial starting point
1725 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1726 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1727 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1728 notification_interval %(notification_interval)s
1729 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1730 check_command check-host-alive ; Default command to check FreeBSD hosts
1731 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1732}
1733
1734define service {
1735 name wleiden-service ; Default Service Template
1736 use generic-service ; Use the standard template as initial starting point
1737 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1738 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1739 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1740 notification_interval %(notification_interval)s
1741 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1742 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1743}
1744
1745# Please make sure to install:
1746# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1747#
1748# Recompile net-mgmt/nagios-plugins to support check_snmp
1749# make -C /usr/ports/net-mgmt/nagios-plugins
1750#
1751# Install net/bind-tools to allow v2/check_dns_wl to work:
1752# pkg install bind-tools
1753#
1754define command{
1755 command_name check_snmp_disk
1756 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1757}
1758
1759define command{
1760 command_name check_netsnmp_load
1761 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1762}
1763
1764define command{
1765 command_name check_netsnmp_proc
1766 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1767}
1768
1769define command{
1770 command_name check_by_ssh
1771 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1772}
1773
1774define command{
1775 command_name check_dns_wl
1776 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1777}
1778
1779define command{
1780 command_name check_snmp_uptime
1781 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1782}
1783
1784
1785# TDB: dhcp leases
1786# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1787
1788# TDB: internet status
1789# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1790
1791# TDB: Advanced local passive checks
1792# /usr/local/libexec/nagios/check_by_ssh
1793''' % params
1794
1795 print '''\
1796# Service Group, not displayed by default
1797define hostgroup {
1798 hostgroup_name srv_hybrid
1799 alias All Hybrid Nodes
1800 register 0
1801}
1802
1803define service {
1804 use wleiden-service
1805 hostgroup_name srv_hybrid
1806 service_description SSH
1807 check_command check_ssh
1808}
1809
1810define service {
1811 use wleiden-service,service-pnp
1812 hostgroup_name srv_hybrid
1813 service_description HTTP
1814 check_command check_http
1815}
1816
1817define service {
1818 use wleiden-service
1819 hostgroup_name srv_hybrid
1820 service_description DNS
1821 check_command check_dns_wl!"www.wirelessleiden.nl"
1822}
1823'''
1824
1825 if heavy_load:
1826 print '''\
1827define service {
1828 use wleiden-service
1829 hostgroup_name srv_hybrid
1830 service_description UPTIME
1831 check_command check_snmp_uptime
1832}
1833
1834#define service {
1835# use wleiden-service
1836# hostgroup_name srv_hybrid
1837# service_description NTP
1838# check_command check_ntp_peer
1839#}
1840
1841define service {
1842 use wleiden-service
1843 hostgroup_name srv_hybrid
1844 service_description LOAD
1845 check_command check_netsnmp_load
1846}
1847
1848define service {
1849 use wleiden-service
1850 hostgroup_name srv_hybrid
1851 service_description PROC
1852 check_command check_netsnmp_proc
1853}
1854
1855define service {
1856 use wleiden-service
1857 hostgroup_name srv_hybrid
1858 service_description DISK
1859 check_command check_snmp_disk
1860}
1861'''
1862 for node in get_hostlist():
1863 datadump = get_yaml(node)
1864 if not datadump['status'] == 'up':
1865 continue
1866 if not hostgroup_details.has_key(datadump['monitoring_group']):
1867 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
1868 print '''\
1869define host {
1870 use wleiden-node,host-pnp
1871 contact_groups admins
1872 host_name %(autogen_fqdn)s
1873 address %(masterip)s
1874 hostgroups srv_hybrid,%(monitoring_group)s\
1875''' % datadump
1876 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
1877 print '''\
1878 parents %(parents)s\
1879''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
1880 print '''\
1881}
1882'''
1883
1884
1885 for name,alias in hostgroup_details.iteritems():
1886 print '''\
1887define hostgroup {
1888 hostgroup_name %s
1889 alias %s
1890} ''' % (name, alias)
1891
1892
1893 elif sys.argv[1] == "full-export":
1894 hosts = {}
1895 for node in get_hostlist():
1896 datadump = get_yaml(node)
1897 hosts[datadump['nodename']] = datadump
1898 print yaml.dump(hosts)
1899
1900 elif sys.argv[1] == "dns":
1901 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
1902 elif sys.argv[1] == "cleanup":
1903 # First generate all datadumps
1904 datadumps = dict()
1905 ssid_to_node = dict()
1906 for host in get_hostlist():
1907 logger.info("# Processing: %s", host)
1908 # Set some boring default values
1909 datadump = { 'board' : 'UNKNOWN' }
1910 datadump.update(get_yaml(host))
1911 datadumps[datadump['autogen_realname']] = datadump
1912
1913 (poel, errors) = make_relations(datadumps)
1914 print "\n".join(["# WARNING: %s" % x for x in errors])
1915
1916 for host,datadump in datadumps.iteritems():
1917 try:
1918 # Convert all yes and no to boolean values
1919 def fix_boolean(dump):
1920 for key in dump.keys():
1921 if type(dump[key]) == dict:
1922 dump[key] = fix_boolean(dump[key])
1923 elif str(dump[key]).lower() in ["yes", "true"]:
1924 dump[key] = True
1925 elif str(dump[key]).lower() in ["no", "false"]:
1926 # Compass richting no (Noord Oost) is valid input
1927 if key != "compass": dump[key] = False
1928 return dump
1929 datadump = fix_boolean(datadump)
1930
1931 if datadump['rdnap_x'] and datadump['rdnap_y']:
1932 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
1933 elif datadump['latitude'] and datadump['longitude']:
1934 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
1935
1936 if datadump['nodename'].startswith('Proxy'):
1937 datadump['nodename'] = datadump['nodename'].lower()
1938
1939 for iface_key in datadump['autogen_iface_keys']:
1940 try:
1941 # All our normal wireless cards are normal APs now
1942 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
1943 datadump[iface_key]['mode'] = 'ap'
1944 # Wireless Leiden SSID have an consistent lowercase/uppercase
1945 if datadump[iface_key].has_key('ssid'):
1946 ssid = datadump[iface_key]['ssid']
1947 prefix = 'ap-WirelessLeiden-'
1948 if ssid.lower().startswith(prefix.lower()):
1949 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
1950 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
1951 datadump[iface_key]['mode'] = 'autogen-FIXME'
1952 if not datadump[iface_key].has_key('comment'):
1953 datadump[iface_key]['comment'] = 'autogen-FIXME'
1954
1955 if datadump[iface_key].has_key('ns_mac'):
1956 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
1957
1958 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
1959 datadump[iface_key] = datadump[iface_key]['desc']
1960
1961 # We are not using 802.11b anymore. OFDM is preferred over DSSS
1962 # due to better collision avoidance.
1963 if datadump[iface_key]['type'] == '11b':
1964 datadump[iface_key]['type'] = '11g'
1965
1966 # Setting 802.11g channels to de-facto standards, to avoid
1967 # un-detected sharing with other overlapping channels
1968 #
1969 # Technically we could also use channel 13 in NL, but this is not
1970 # recommended as foreign devices might not be able to select this
1971 # channel. Secondly using 1,5,9,13 instead is going to clash with
1972 # the de-facto usage of 1,6,11.
1973 #
1974 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
1975 channels_at_2400Mhz = (1,6,11)
1976 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
1977 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
1978 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
1979 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
1980
1981 # Mandatory interface keys
1982 if not datadump[iface_key].has_key('status'):
1983 datadump[iface_key]['status'] = 'planned'
1984
1985 x = datadump[iface_key]['comment']
1986 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
1987
1988 # Fixing bridge_type if none is found
1989 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
1990 if not 'bridge_type' in datadump[iface_key]:
1991 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
1992
1993 # Making sure description works
1994 if datadump[iface_key].has_key('desc'):
1995 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
1996 del datadump[iface_key]['desc']
1997 else:
1998 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
1999 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2000 if response:
2001 datadump[iface_key]['comment'] = response
2002 del datadump[iface_key]['desc']
2003
2004 # Check DHCP configuration
2005 dhcp_type(datadump[iface_key])
2006
2007 # Set the compass value based on the angle between the poels
2008 if datadump[iface_key].has_key('ns_ip'):
2009 my_pool = poel[network(datadump[iface_key]['ip'])]
2010 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2011 if remote_hosts:
2012 compass_target = remote_hosts[0]
2013 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2014
2015 # Monitoring Group default
2016 if not 'monitoring_group' in datadump:
2017 datadump['monitoring_group'] = 'wleiden'
2018
2019 except Exception as e:
2020 print "# Error while processing interface %s" % iface_key
2021 raise
2022 store_yaml(datadump)
2023 except Exception as e:
2024 print "# Error while processing %s" % host
2025 raise
2026 elif sys.argv[1] == "list":
2027 use_fqdn = False
2028 if len(sys.argv) < 4:
2029 usage()
2030 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2031 usage()
2032 if not sys.argv[3] in ["nodes","proxies","systems"]:
2033 usage()
2034
2035 if len(sys.argv) > 4:
2036 if sys.argv[4] == "fqdn":
2037 use_fqdn = True
2038 else:
2039 usage()
2040
2041 for system in get_hostlist():
2042 datadump = get_yaml(system)
2043 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2044 continue
2045
2046 output = datadump['autogen_fqdn'] if use_fqdn else system
2047 if sys.argv[2] == "all":
2048 print output
2049 elif datadump['status'] == sys.argv[2]:
2050 print output
2051 elif sys.argv[1] == "create":
2052 if sys.argv[2] == "network.kml":
2053 print make_network_kml.make_graph()
2054 elif sys.argv[2] == "host-ips.txt":
2055 for system in get_hostlist():
2056 datadump = get_yaml(system)
2057 ips = [datadump['masterip']]
2058 for ifkey in datadump['autogen_iface_keys']:
2059 ips.append(datadump[ifkey]['ip'].split('/')[0])
2060 print system, ' '.join(ips)
2061 elif sys.argv[2] == "host-pos.txt":
2062 for system in get_hostlist():
2063 datadump = get_yaml(system)
2064 print system, datadump['rdnap_x'], datadump['rdnap_y']
2065 elif sys.argv[2] == 'ssh_config':
2066 print '''
2067Host *.wleiden.net
2068 User root
2069
2070Host 172.16.*.*
2071 User root
2072'''
2073 for system in get_hostlist():
2074 datadump = get_yaml(system)
2075 print '''\
2076Host %s
2077 User root
2078
2079Host %s
2080 User root
2081
2082Host %s
2083 User root
2084
2085Host %s
2086 User root
2087''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2088 else:
2089 usage()
2090 else:
2091 usage()
2092 else:
2093 # Do not enable debugging for config requests as it highly clutters the output
2094 if not is_text_request():
2095 cgitb.enable()
2096 response_headers, output = process_cgi_request()
2097 print_cgi_response(response_headers, output)
2098
2099def application(environ, start_response):
2100 status = '200 OK'
2101 response_headers, output = process_cgi_request(environ)
2102 start_response(status, response_headers)
2103
2104 # Debugging only
2105 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2106 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2107 return [output]
2108
2109if __name__ == "__main__":
2110 main()
Note: See TracBrowser for help on using the repository browser.