source: genesis/tools/gformat.py@ 13734

Last change on this file since 13734 was 13734, checked in by rick, 8 years ago

Keep equal interfaces together, more easy debugging

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 75.9 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict, OrderedDict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13734 2017-01-18 22:41:57Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'unbound.wleiden.conf',
111 'wleiden.yaml',
112 ]
113
114# Global variables uses
115OK = 10
116DOWN = 20
117UNKNOWN = 90
118
119
120ileiden_proxies = OrderedDict()
121normal_proxies = []
122datadump_cache = {}
123interface_list_cache = {}
124rc_conf_local_cache = {}
125nameservers_cache = []
126relations_cache = None
127def clear_cache():
128 ''' Poor mans cache implementation '''
129 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
130 datadump_cache = {}
131 interface_list_cache = {}
132 rc_conf_local_cache = {}
133 ileiden_proxies = OrderedDict()
134 normal_proxies = []
135 nameservers_cache = []
136 relations_cache = None
137
138NO_DHCP = 0
139DHCP_CLIENT = 10
140DHCP_SERVER = 20
141def dhcp_type(item):
142 if not item.has_key('dhcp'):
143 return NO_DHCP
144 elif not item['dhcp']:
145 return NO_DHCP
146 elif item['dhcp'].lower() == 'client':
147 return DHCP_CLIENT
148 else:
149 # Validation Checks
150 begin,end = map(int,item['dhcp'].split('-'))
151 if begin >= end:
152 raise ValueError("DHCP Start >= DHCP End")
153 return DHCP_SERVER
154
155def etrs2rd(lat, lon):
156 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
157 p2 = pyproj.Proj(init='EPSG:28992')
158 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
159 return (RDx, RDy)
160
161def rd2etrs(RDx, RDy):
162 p1 = pyproj.Proj(init='EPSG:28992')
163 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
164 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
165 return (lat, lon)
166
167def get_yaml(item,add_version_info=True):
168 try:
169 """ Get configuration yaml for 'item'"""
170 if datadump_cache.has_key(item):
171 return datadump_cache[item].copy()
172
173 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
174
175 # Default values
176 datadump = {
177 'autogen_revision' : 'NOTFOUND',
178 'autogen_gfile' : gfile,
179 'service_proxy_ileiden' : False,
180 }
181 f = open(gfile, 'r')
182 datadump.update(yaml.load(f,Loader=Loader))
183 if datadump['nodetype'] == 'Hybrid':
184 # Some values are defined implicitly
185 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
186 datadump['service_incoming_rdr'] = True
187 # Use some boring defaults
188 defaults = {
189 'service_proxy_normal' : False,
190 'service_accesspoint' : True,
191 'service_incoming_rdr' : False,
192 'service_concentrator' : False,
193 'monitoring_group' : 'wleiden',
194 }
195 for (key,value) in defaults.iteritems():
196 if not datadump.has_key(key):
197 datadump[key] = value
198 f.close()
199
200 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
201 if add_version_info:
202 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
203 lines = p.communicate()[0].split('\n')
204 if p.returncode == 0:
205 for line in lines:
206 if line:
207 (key, value) = line.split(': ')
208 datadump["autogen_" + key.lower().replace(' ','_')] = value
209
210 # Preformat certain needed variables for formatting and push those into special object
211 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
212
213 wlan_count=0
214 try:
215 for key in get_interface_keys(datadump, True):
216 datadump[key]['autogen_ifbase'] = key.split('_')[1]
217 datadump[key]['autogen_vlan'] = False
218
219 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
220 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
221
222 if datadump[key].has_key('parent'):
223 if datadump[key].has_key('ip'):
224 raise ValueError("Interface bridge member cannot have IP assigned")
225 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
226 raise ValueError("Interface bridge member cannot have DHCP set")
227
228 if datadump[key].has_key('ip'):
229 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
230
231 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
232 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
233 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
234 wlan_count += 1
235 else:
236 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
237 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
238 datadump[key]['autogen_vlan'] = key.split('_')[2]
239 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
240 else:
241 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
242
243 except Exception:
244 print "# Error while processing interface %s" % key
245 raise
246
247 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
248 if dhcp_type(datadump[key]) == DHCP_SERVER]
249
250 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
251 datadump['autogen_item'] = item
252
253 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
254 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
255 datadump_cache[item] = datadump.copy()
256 except Exception:
257 print "# Error while processing %s" % item
258 raise
259 return datadump
260
261
262def store_yaml(datadump, header=False):
263 """ Store configuration yaml for 'item'"""
264 item = datadump['autogen_item']
265 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
266
267 output = generate_wleiden_yaml(datadump, header)
268
269 f = open(gfile, 'w')
270 f.write(output)
271 f.close()
272
273
274def network(ip):
275 addr, mask = ip.split('/')
276 # Not parsing of these folks please
277 addr = parseaddr(addr)
278 mask = int(mask)
279 network = addr & ~((1 << (32 - mask)) - 1)
280 return network
281
282
283
284def make_relations():
285 """ Process _ALL_ yaml files to get connection relations """
286 global relations_cache
287
288 if relations_cache:
289 return relations_cache
290
291 errors = []
292 poel = defaultdict(list)
293
294 for host in get_hostlist():
295 datadump = get_yaml(host)
296 try:
297 for iface_key in get_interface_keys(datadump):
298 net_addr = network(datadump[iface_key]['ip'])
299 poel[net_addr] += [(host,datadump[iface_key].copy())]
300 except (KeyError, ValueError), e:
301 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
302 continue
303
304 relations_cache = (poel, errors)
305 return relations_cache
306
307
308
309def valid_addr(addr):
310 """ Show which address is valid in which are not """
311 return str(addr).startswith('172.')
312
313def get_hostlist():
314 """ Combined hosts and proxy list"""
315 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
316
317def angle_between_points(lat1,lat2,long1,long2):
318 """
319 Return Angle in radians between two GPS coordinates
320 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
321 """
322 dy = lat2 - lat1
323 dx = math.cos(lat1)*(long2 - long1)
324 angle = math.atan2(dy,dx)
325 return angle
326
327
328
329def angle_to_cd(angle):
330 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
331
332 # For easy conversion get positive degree
333 degrees = math.degrees(angle)
334 abs_degrees = 360 + degrees if degrees < 0 else degrees
335
336 # Numbers can be confusing calculate from the 4 main directions
337 p = 22.5
338 if abs_degrees < p:
339 cd = "n"
340 elif abs_degrees < (90 - p):
341 cd = "no"
342 elif abs_degrees < (90 + p):
343 cd = "o"
344 elif abs_degrees < (180 - p):
345 cd = "zo"
346 elif abs_degrees < (180 + p):
347 cd = "z"
348 elif abs_degrees < (270 - p):
349 cd = "zw"
350 elif abs_degrees < (270 + p):
351 cd = "w"
352 elif abs_degrees < (360 - p):
353 cd = "nw"
354 else:
355 cd = "n"
356 return cd
357
358
359
360def cd_between_hosts(hostA, hostB, datadumps):
361 # Using RDNAP coordinates
362 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
363 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
364 return angle_to_cd(math.atan2(dx,dy))
365
366 # GPS coordinates seems to fail somehow
367 #latA = float(datadumps[hostA]['latitude'])
368 #latB = float(datadumps[hostB]['latitude'])
369 #lonA = float(datadumps[hostA]['longitude'])
370 #lonB = float(datadumps[hostB]['longitude'])
371 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
372
373
374def generate_title(nodelist):
375 """ Main overview page """
376 items = {'root' : "." }
377 def fl(spaces, line):
378 return (' ' * spaces) + line + '\n'
379
380 output = """
381<html>
382 <head>
383 <title>Wireless leiden Configurator - GFormat</title>
384 <style type="text/css">
385 th {background-color: #999999}
386 tr:nth-child(odd) {background-color: #cccccc}
387 tr:nth-child(even) {background-color: #ffffff}
388 th, td {padding: 0.1em 1em}
389 </style>
390 </head>
391 <body>
392 <center>
393 <form type="GET" action="%(root)s">
394 <input type="hidden" name="action" value="update">
395 <input type="submit" value="Update Configuration Database (SVN)">
396 </form>
397 <table>
398 <caption><h3>Wireless Leiden Configurator</h3></caption>
399 """ % items
400
401 for node in nodelist:
402 items['node'] = node
403 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
404 for config in files:
405 items['config'] = config
406 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
407 output += fl(5, "</tr>")
408 output += """
409 </table>
410 <hr />
411 <em>%s</em>
412 </center>
413 </body>
414</html>
415 """ % __version__
416
417 return output
418
419
420
421def generate_node(node):
422 """ Print overview of all files available for node """
423 return "\n".join(files)
424
425def generate_node_overview(host):
426 """ Print overview of all files available for node """
427 datadump = get_yaml(host)
428 params = { 'host' : host }
429 output = "<em><a href='..'>Back to overview</a></em><hr />"
430 output += "<h2>Available files:</h2><ul>"
431 for cf in files:
432 params['cf'] = cf
433 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
434 output += "</ul>"
435
436 # Generate and connection listing
437 output += "<h2>Connected To:</h2><ul>"
438 (poel, errors) = make_relations()
439 for network, hosts in poel.iteritems():
440 if host in [x[0] for x in hosts]:
441 if len(hosts) == 1:
442 # Single not connected interface
443 continue
444 for remote,ifacedump in hosts:
445 if remote == host:
446 # This side of the interface
447 continue
448 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
449 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
450 output += "</ul>"
451 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
452
453 output += "<hr /><em><a href='..'>Back to overview</a></em>"
454 return output
455
456
457def generate_header(datadump, ctag="#"):
458 return """\
459%(ctag)s
460%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
461%(ctag)s
462""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
463
464
465
466def parseaddr(s):
467 """ Process IPv4 CIDR notation addr to a (binary) number """
468 f = s.split('.')
469 return (long(f[0]) << 24L) + \
470 (long(f[1]) << 16L) + \
471 (long(f[2]) << 8L) + \
472 long(f[3])
473
474
475
476def showaddr(a):
477 """ Display IPv4 addr in (dotted) CIDR notation """
478 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
479
480
481def is_member(ip, mask, canidate):
482 """ Return True if canidate is part of ip/mask block"""
483 ip_addr = parseaddr(ip)
484 ip_canidate = parseaddr(canidate)
485 mask = int(mask)
486 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
487 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
488 return ip_addr == ip_canidate
489
490
491
492def cidr2netmask(netmask):
493 """ Given a 'netmask' return corresponding CIDR """
494 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
495
496def get_network(addr, mask):
497 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
498
499
500def generate_dhcpd_conf(datadump):
501 """ Generate config file '/usr/local/etc/dhcpd.conf """
502 output = generate_header(datadump)
503 output += Template("""\
504# option definitions common to all supported networks...
505option domain-name "dhcp.{{ autogen_fqdn }}";
506
507default-lease-time 600;
508max-lease-time 7200;
509
510# Use this to enble / disable dynamic dns updates globally.
511#ddns-update-style none;
512
513# If this DHCP server is the official DHCP server for the local
514# network, the authoritative directive should be uncommented.
515authoritative;
516
517# Use this to send dhcp log messages to a different log file (you also
518# have to hack syslog.conf to complete the redirection).
519log-facility local7;
520
521#
522# Interface definitions
523#
524\n\n""").render(datadump)
525
526
527 # TODO: Use textwrap.fill instead
528 def indent(text, count):
529 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
530
531 # Process DHCP blocks
532 dhcp_out = defaultdict(list)
533 for iface_key in get_interface_keys(datadump):
534 ifname = datadump[iface_key]['autogen_ifbase']
535 if not datadump[iface_key].has_key('comment'):
536 datadump[iface_key]['comment'] = None
537
538 if not datadump[iface_key].has_key('ip'):
539 continue
540
541 dhcp_out[iface_key].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
542
543 (addr, mask) = datadump[iface_key]['ip'].split('/')
544 datadump[iface_key]['autogen_addr'] = addr
545 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
546 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
547
548 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
549 dhcp_out[ifname].append(textwrap.dedent("""\
550 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
551 ### not autoritive
552 }
553 """ % datadump[iface_key]))
554 continue
555
556 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
557 dhcp_part = ".".join(addr.split('.')[0:3])
558 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
559 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
560
561 # Assume the first 10 IPs could be used for static entries
562 if 'no_portal' in datadump:
563 fixed = 5
564 for mac in datadump['no_portal']:
565 dhcp_out[ifname].append(textwrap.dedent("""\
566 host fixed-%(ifname)s-%(fixed)s {
567 hardware ethernet %(mac)s;
568 fixed-address %(prefix)s.%(fixed)s;
569 }
570 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
571 fixed += 1
572
573 dhcp_out[ifname].append(textwrap.dedent("""\
574 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
575 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
576 option routers %(autogen_addr)s;
577 option domain-name-servers %(autogen_addr)s;
578 }
579 """ % datadump[iface_key]))
580
581 # Output the blocks in groups
582 for ifname,value in sorted(dhcp_out.iteritems()):
583 if len(value) > 2:
584 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
585 else:
586 output += ''.join(value) + "\n\n"
587 return output
588
589
590
591def generate_dnsmasq_conf(datadump):
592 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
593 output = generate_header(datadump)
594 output += Template("""\
595# DHCP server options
596dhcp-authoritative
597dhcp-fqdn
598domain=dhcp.{{ autogen_fqdn }}
599domain-needed
600expand-hosts
601log-async=100
602
603# Low memory footprint
604cache-size=10000
605
606\n""").render(datadump)
607
608 for iface_key in get_interface_keys(datadump):
609 if not datadump[iface_key].has_key('comment'):
610 datadump[iface_key]['comment'] = None
611 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
612
613 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
614 output += "# not autoritive\n\n"
615 continue
616
617 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
618 (ip, cidr) = datadump[iface_key]['ip'].split('/')
619 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
620
621 dhcp_part = ".".join(ip.split('.')[0:3])
622 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
623 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
624 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
625
626 return output
627
628
629class AutoVivification(dict):
630 """Implementation of perl's autovivification feature."""
631 def __getitem__(self, item):
632 try:
633 return dict.__getitem__(self, item)
634 except KeyError:
635 value = self[item] = type(self)()
636 return value
637
638def make_interface_list(datadump):
639 if interface_list_cache.has_key(datadump['autogen_item']):
640 return (interface_list_cache[datadump['autogen_item']])
641 # lo0 configuration:
642 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
643 # - masterip is special as it needs to be assigned to at
644 # least one interface, so if not used assign to lo0
645 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
646 vlan_list = defaultdict(list)
647 bridge_list = defaultdict(list)
648 flags_if = AutoVivification()
649 dhclient_if = {'lo0' : False}
650
651 # XXX: Find some way of send this output nicely
652 output = ''
653
654 masterip_used = False
655 for iface_key in get_interface_keys(datadump):
656 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
657 masterip_used = True
658 break
659 if not masterip_used:
660 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
661
662 if 'serviceid' in datadump:
663 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
664
665 for iface_key in get_interface_keys(datadump):
666 ifacedump = datadump[iface_key]
667 ifname = ifacedump['autogen_ifname']
668
669 # If defined as vlan interface
670 if ifacedump['autogen_vlan']:
671 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
672
673 # If defined as bridge interface
674 if ifacedump['autogen_bridge_member']:
675 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
676
677 # Flag dhclient is possible
678 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
679 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
680
681 # Ethernet address
682 if ifacedump.has_key('ether'):
683 flags_if[ifname]['ether'] = ifacedump['ether']
684
685 # Add interface IP to list
686 if ifacedump.has_key('ip'):
687 item = (ifacedump['ip'], ifacedump['comment'])
688 if addrs_list.has_key(ifname):
689 addrs_list[ifname].append(item)
690 else:
691 addrs_list[ifname] = [item]
692
693 # Alias only needs IP assignment for now, this might change if we
694 # are going to use virtual accesspoints
695 if "alias" in iface_key:
696 continue
697
698 # XXX: Might want to deduct type directly from interface name
699 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
700 # Default to station (client) mode
701 ifacedump['autogen_wlanmode'] = "sta"
702 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
703 ifacedump['autogen_wlanmode'] = "ap"
704
705 if not ifacedump.has_key('channel'):
706 if ifacedump['type'] == '11a':
707 ifacedump['channel'] = 36
708 else:
709 ifacedump['channel'] = 1
710
711 # Allow special hacks at the back like wds and stuff
712 if not ifacedump.has_key('extra'):
713 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
714 else:
715 ifacedump['autogen_extra'] = ifacedump['extra']
716
717 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
718
719 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
720 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
721 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
722 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
723 output += "\n"
724
725 elif ifacedump['type'] in ['ethernet', 'eth']:
726 # No special config needed besides IP
727 pass
728 elif ifacedump['type'] in ['vlan']:
729 # VLAN member has no special configuration
730 pass
731 else:
732 assert False, "Unknown type " + ifacedump['type']
733
734 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
735 interface_list_cache[datadump['autogen_item']] = store
736 return(store)
737
738
739
740def create_proxies_list():
741 if not ileiden_proxies or not normal_proxies:
742 for i in range(1,50):
743 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
744
745 for host in get_hostlist():
746 hostdump = get_yaml(host)
747 if hostdump['status'] == 'up':
748 if hostdump['service_proxy_ileiden']:
749 ileiden_proxies[hostdump['serviceid']] = hostdump
750 if hostdump['service_proxy_normal']:
751 normal_proxies.append(hostdump)
752
753
754
755def generate_rc_conf_local(datadump):
756 """ Generate configuration file '/etc/rc.conf.local' """
757 item = datadump['autogen_item']
758 if rc_conf_local_cache.has_key(item):
759 return rc_conf_local_cache[item]
760
761 if not datadump.has_key('ileiden'):
762 datadump['autogen_ileiden_enable'] = False
763 else:
764 datadump['autogen_ileiden_enable'] = datadump['ileiden']
765
766 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
767
768 create_proxies_list()
769 datadump['autogen_ileiden_proxies'] = ileiden_proxies
770 datadump['autogen_normal_proxies'] = normal_proxies
771 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
772 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
773 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
774 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
775
776 output = generate_header(datadump, "#");
777 output += render_template(datadump, """\
778hostname='{{ autogen_fqdn }}'
779location='{{ location }}'
780nodetype="{{ nodetype }}"
781
782#
783# Configured listings
784#
785captive_portal_whitelist=""
786{% if nodetype == "Proxy" %}
787#
788# Proxy Configuration
789#
790{% if gateway and service_proxy_ileiden -%}
791defaultrouter="{{ gateway }}"
792{% else -%}
793#defaultrouter="NOTSET"
794{% endif -%}
795internalif="{{ internalif }}"
796ileiden_enable="{{ autogen_ileiden_enable }}"
797gateway_enable="{{ autogen_ileiden_enable }}"
798pf_enable="yes"
799pf_rules="/etc/pf.conf"
800{% if autogen_ileiden_enable -%}
801pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
802lvrouted_enable="{{ autogen_ileiden_enable }}"
803lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
804{% else -%}
805pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
806{% endif -%}
807{% if internalroute -%}
808static_routes="wleiden"
809route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
810{% endif -%}
811
812{% elif nodetype == "Hybrid" %}
813 #
814 # Hybrid Configuration
815 #
816 list_ileiden_proxies="
817 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
818 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
819 {% endfor -%}
820 "
821 list_normal_proxies="
822 {% for item in autogen_normal_proxies -%}
823 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
824 {% endfor -%}
825 "
826
827 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
828 externalif="{{ externalif|default('vr0', true) }}"
829 masterip="{{ masterip }}"
830
831 {% if gateway and service_proxy_ileiden %}
832 defaultrouter="{{ gateway }}"
833 {% else %}
834 #defaultrouter="NOTSET"
835 {% endif %}
836
837 #
838 # Defined services
839 #
840 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
841 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
842 service_accesspoint="{{ service_accesspoint|yesorno }}"
843 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
844 service_concentrator="{{ service_concentrator|yesorno }}"
845
846 {% if service_proxy_ileiden %}
847 pf_rules="/etc/pf.hybrid.conf"
848 {% if service_concentrator %}
849 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
850 {% else %}
851 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
852 {% endif %}
853 pf_flags="$pf_flags -D publicnat=80,443"
854 lvrouted_flags="$lvrouted_flags -g"
855 {% elif service_proxy_normal or service_incoming_rdr %}
856 pf_rules="/etc/pf.hybrid.conf"
857 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
858 pf_flags="$pf_flags -D publicnat=0"
859 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
860 named_setfib="1"
861 tinyproxy_setfib="1"
862 dnsmasq_setfib="1"
863 sshd_setfib="1"
864 {% else %}
865 named_auto_forward_only="YES"
866 pf_rules="/etc/pf.node.conf"
867 pf_flags=""
868 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
869 {% endif %}
870 {% if service_concentrator %}
871 # Do mind installing certificates is NOT done automatically for security reasons
872 openvpn_enable="YES"
873 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
874 {% endif %}
875
876 {% if service_proxy_normal %}
877 tinyproxy_enable="yes"
878 {% else %}
879 pen_wrapper_enable="yes"
880 {% endif %}
881
882 {% if service_accesspoint %}
883 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
884 {% endif %}
885
886 {% if board == "ALIX2" %}
887 #
888 # ''Fat'' configuration, board has 256MB RAM
889 #
890 dnsmasq_enable="NO"
891 named_enable="YES"
892 unbound_enable="YES"
893 {% if autogen_dhcp_interfaces -%}
894 dhcpd_enable="YES"
895 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
896 {% endif -%}
897 {% elif board == "apu1d" %}
898 #
899 # ''Fat'' configuration, board has 1024MB RAM
900 #
901 dnsmasq_enable="NO"
902 local_unbound_enable="YES"
903 {% if autogen_dhcp_interfaces -%}
904 dhcpd_enable="YES"
905 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
906 {% endif -%}
907 {% endif -%}
908{% endif %}
909
910#
911# Script variables
912#
913attached_devices="{{ autogen_attached_devices|join(' ') }}"
914neighbours="{{ autogen_neighbours|join(' ') }}"
915
916
917#
918# Interface definitions
919#\n
920""")
921
922 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
923 for iface, vlans in sorted(vlan_list.items()):
924 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
925
926 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
927 for iface in sorted(vlan_list.keys()):
928 if not iface in addrs_list.keys():
929 output += "ifconfig_%s='up'\n" % iface
930
931 output += "\n"
932
933 # Bridge configuration:
934 if bridge_list.keys():
935 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
936
937 for iface in bridge_list.keys():
938 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
939
940 # Bridge member interfaces not containing a configuration should be marked active explcitly.
941 for _,members in bridge_list.items():
942 for iface in members:
943 if not iface in addrs_list.keys():
944 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
945
946 output += "\n"
947
948 # Details like SSID
949 if extra_ouput:
950 output += extra_ouput.strip() + "\n"
951
952 # Print IP address which needs to be assigned over here
953 output += "\n"
954 for iface,addrs in sorted(addrs_list.iteritems()):
955 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
956 output += "# %s || %s || %s\n" % (iface, addr, comment)
957
958 # Write DHCLIENT entry
959 if iface in dhclient_if and dhclient_if[iface]:
960 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
961 continue
962
963 # Make sure the external address is always first as this is needed in the
964 # firewall setup
965 addrs = sorted(
966 [x for x in addrs if not '0.0.0.0' in x[0]],
967 key=lambda x: x[0].split('.')[0],
968 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
969 )
970
971 idx_offset = 0
972 # Set MAC is required
973 if flags_if[iface].has_key('ether'):
974 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
975 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
976 idx_offset += 1
977 else:
978 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
979
980 for idx, addr in enumerate(addrs[1:]):
981 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
982
983 output += "\n"
984
985 rc_conf_local_cache[datadump['autogen_item']] = output
986 return output
987
988
989
990def get_all_configs():
991 """ Get dict with key 'host' with all configs present """
992 configs = dict()
993 for host in get_hostlist():
994 datadump = get_yaml(host)
995 configs[host] = datadump
996 return configs
997
998
999def get_interface_keys(config, extra=False):
1000 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1001 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1002 if extra == False:
1003 return filter(lambda x: not "extra" in x, elems)
1004 else:
1005 return elems
1006
1007
1008def get_used_ips(configs):
1009 """ Return array of all IPs used in config files"""
1010 ip_list = []
1011 for config in configs:
1012 ip_list.append(config['masterip'])
1013 if 'serviceid' in config:
1014 ip_list.append(config['serviceid'])
1015 for iface_key in get_interface_keys(config, True):
1016 l = config[iface_key]['ip']
1017 addr, mask = l.split('/')
1018 # Special case do not process
1019 if valid_addr(addr):
1020 ip_list.append(addr)
1021 else:
1022 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1023 return sorted(ip_list)
1024
1025
1026
1027def get_nameservers(max_servers=None):
1028 if nameservers_cache:
1029 return nameservers_cache[0:max_servers]
1030
1031 for host in get_hostlist():
1032 hostdump = get_yaml(host)
1033 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1034 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
1035
1036 return nameservers_cache[0:max_servers]
1037
1038
1039def get_neighbours(datadump):
1040 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1041
1042 (poel, errors) = make_relations()
1043 table = []
1044 for iface,addrs in sorted(addrs_list.iteritems()):
1045 if iface in ['lo0']:
1046 continue
1047
1048 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1049 if not addr.startswith('172.'):
1050 # Avoid listing internet connections as pool
1051 continue
1052 for neighbour in poel[network(addr)]:
1053 if neighbour[0] != datadump['autogen_item']:
1054 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1055 return table
1056
1057
1058def get_attached_devices(datadump, url=False):
1059 table = []
1060 for iface_key in get_interface_keys(datadump, True):
1061 ifacedump = datadump[iface_key]
1062
1063 if not ifacedump.has_key('ns_ip'):
1064 continue
1065
1066 x_ip = ifacedump['ns_ip'].split('/')[0]
1067
1068 if 'mode' in ifacedump:
1069 x_mode = ifacedump['mode']
1070 else:
1071 x_mode = 'unknown'
1072
1073 if 'bridge_type' in ifacedump:
1074 device_type = ifacedump['bridge_type']
1075 else:
1076 device_type = 'Unknown'
1077
1078 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1079 return table
1080
1081
1082def generate_resolv_conf(datadump):
1083 """ Generate configuration file '/etc/resolv.conf' """
1084 # XXX: This should properly going to be an datastructure soon
1085 datadump['autogen_header'] = generate_header(datadump, "#")
1086 datadump['autogen_edge_nameservers'] = ''
1087
1088
1089 for masterip,realname in get_nameservers():
1090 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1091
1092 return Template("""\
1093{{ autogen_header }}
1094search wleiden.net
1095
1096# Try local (cache) first
1097nameserver 127.0.0.1
1098
1099{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1100nameserver 8.8.8.8 # Google Public NameServer
1101nameserver 64.6.64.6 # Verisign Public NameServer
1102{% else -%}
1103# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1104{{ autogen_edge_nameservers }}
1105{% endif -%}
1106""").render(datadump)
1107
1108
1109
1110def generate_ntp_conf(datadump):
1111 """ Generate configuration file '/etc/ntp.conf' """
1112 # XXX: This should properly going to be an datastructure soon
1113
1114 datadump['autogen_header'] = generate_header(datadump, "#")
1115 datadump['autogen_ntp_servers'] = ''
1116 for host in get_hostlist():
1117 hostdump = get_yaml(host)
1118 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1119 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1120
1121 return Template("""\
1122{{ autogen_header }}
1123
1124{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1125# Machine hooked to internet.
1126server 0.nl.pool.ntp.org iburst maxpoll 9
1127server 1.nl.pool.ntp.org iburst maxpoll 9
1128server 2.nl.pool.ntp.org iburst maxpoll 9
1129server 3.nl.pool.ntp.org iburst maxpoll 9
1130{% else -%}
1131# Local Wireless Leiden NTP Servers.
1132server 0.pool.ntp.wleiden.net iburst maxpoll 9
1133server 1.pool.ntp.wleiden.net iburst maxpoll 9
1134server 2.pool.ntp.wleiden.net iburst maxpoll 9
1135server 3.pool.ntp.wleiden.net iburst maxpoll 9
1136
1137# All the configured NTP servers
1138{{ autogen_ntp_servers }}
1139{% endif %}
1140
1141# If a server loses sync with all upstream servers, NTP clients
1142# no longer follow that server. The local clock can be configured
1143# to provide a time source when this happens, but it should usually
1144# be configured on just one server on a network. For more details see
1145# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1146# The use of Orphan Mode may be preferable.
1147#
1148server 127.127.1.0
1149fudge 127.127.1.0 stratum 10
1150""").render(datadump)
1151
1152
1153def generate_pf_hybrid_conf_local(datadump):
1154 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1155 datadump['autogen_header'] = generate_header(datadump, "#")
1156 return Template("""\
1157{{ autogen_header }}
1158
1159# Redirect some internal facing services outside (7)
1160# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1161{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1162rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1163{% endfor -%}
1164""").render(datadump)
1165
1166def generate_unbound_wleiden_conf(datadump):
1167 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1168 datadump['autogen_header'] = generate_header(datadump, "#")
1169
1170 autogen_ips = []
1171 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1172 for iface,addrs in sorted(addrs_list.iteritems()):
1173 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1174 if addr.startswith('172'):
1175 autogen_ips.append((addr.split('/')[0], comment))
1176 datadump['autogen_ips'] = autogen_ips
1177
1178 create_proxies_list()
1179 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1180 return Template("""\
1181{{ autogen_header }}
1182
1183server:
1184{%- for ip,comment in autogen_ips %}
1185 interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1186{%- endfor %}
1187
1188forward-zone:
1189 name: '.'
1190{%- if service_proxy_ileiden %}
1191 forward-addr: 8.8.8.8 # Google DNS A
1192 forward-addr: 8.8.4.4 # Google DNS B
1193 forward-addr: 208.67.222.222 # OpenDNS DNS A
1194 forward-addr: 208.67.220.220 # OpenDNS DNS B
1195{% else -%}
1196{% for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1197 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1198{%- endfor %}
1199{% endif -%}
1200""").render(datadump)
1201
1202def generate_motd(datadump):
1203 """ Generate configuration file '/etc/motd' """
1204 output = Template("""\
1205FreeBSD run ``service motd onestart'' to make me look normal
1206
1207 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1208 Loc: {{ location }}
1209
1210Services:
1211{% if board == "ALIX2" -%}
1212{{" -"}} Core Node ({{ board }})
1213{% else -%}
1214{{" -"}} Hulp Node ({{ board }})
1215{% endif -%}
1216{% if service_proxy_normal -%}
1217{{" -"}} Normal Proxy
1218{% endif -%}
1219{% if service_proxy_ileiden -%}
1220{{" -"}} iLeiden Proxy
1221{% endif -%}
1222{% if service_incoming_rdr -%}
1223{{" -"}} Incoming port redirects
1224{% endif %}
1225Interlinks:\n
1226""").render(datadump)
1227
1228
1229 def make_table(table):
1230 if not table:
1231 return " - none\n"
1232 else:
1233 lines = ""
1234 col_width = [max(len(x) for x in col) for col in zip(*table)]
1235 for row in table:
1236 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1237 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1238 return lines
1239
1240 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1241 table = []
1242 for iface,addrs in sorted(addrs_list.iteritems()):
1243 if iface in ['lo0']:
1244 continue
1245 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1246 table.append((iface, addr, comment))
1247
1248 output += make_table(table)
1249 output += '\n'
1250 output += """\
1251Attached devices:
1252"""
1253 output += make_table(get_attached_devices(datadump, url=True))
1254 output += '\n'
1255 output += """\
1256Available neighbours:
1257"""
1258 output += make_table(get_neighbours(datadump))
1259
1260 return output
1261
1262
1263def format_yaml_value(value):
1264 """ Get yaml value in right syntax for outputting """
1265 if isinstance(value,str):
1266 output = '"%s"' % value
1267 else:
1268 output = value
1269 return output
1270
1271
1272
1273def format_wleiden_yaml(datadump):
1274 """ Special formatting to ensure it is editable"""
1275 output = "# Genesis config yaml style\n"
1276 output += "# vim:ts=2:et:sw=2:ai\n"
1277 output += "#\n"
1278 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1279 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1280 if key == 'rdr_rules':
1281 output += '%-10s:\n' % 'rdr_rules'
1282 for rdr_rule in datadump[key]:
1283 output += '- %s\n' % rdr_rule
1284 else:
1285 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1286
1287 output += "\n\n"
1288
1289 # Format (key, required)
1290 key_order = (
1291 ('comment', True),
1292 ('parent', False),
1293 ('ip', False),
1294 ('ether', False),
1295 ('desc', True),
1296 ('sdesc', True),
1297 ('mode', True),
1298 ('type', True),
1299 ('extra_type', False),
1300 ('channel', False),
1301 ('ssid', False),
1302 ('wlan_mac', False),
1303 ('dhcp', True),
1304 ('compass', False),
1305 ('distance', False),
1306 ('ns_ip', False),
1307 ('repeater_ip', False),
1308 ('bullet2_ip', False),
1309 ('ns_mac', False),
1310 ('bullet2_mac', False),
1311 ('ns_type', False),
1312 ('bridge_type', False),
1313 ('status', True),
1314 )
1315
1316 for iface_key in sorted(iface_keys):
1317 try:
1318 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1319 if remainder:
1320 raise KeyError("invalid keys: %s" % remainder)
1321
1322 output += "%s:\n" % iface_key
1323 for key,required in key_order:
1324 if datadump[iface_key].has_key(key):
1325 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1326 output += "\n\n"
1327 except Exception:
1328 print "# Error while processing interface %s" % iface_key
1329 raise
1330
1331 return output
1332
1333
1334
1335def generate_wleiden_yaml(datadump, header=True):
1336 """ Generate (petty) version of wleiden.yaml"""
1337 output = generate_header(datadump, "#") if header else ''
1338
1339 for key in datadump.keys():
1340 if key.startswith('autogen_'):
1341 del datadump[key]
1342 # Interface autogen cleanups
1343 elif type(datadump[key]) == dict:
1344 for key2 in datadump[key].keys():
1345 if key2.startswith('autogen_'):
1346 del datadump[key][key2]
1347
1348 output += format_wleiden_yaml(datadump)
1349 return output
1350
1351def generate_nanostation_config(datadump, iface, ns_type):
1352 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1353 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1354
1355 datadump.update(datadump['iface_%s' % iface])
1356
1357 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1358
1359def generate_yaml(datadump):
1360 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1361
1362
1363
1364def generate_config(node, config, datadump=None):
1365 """ Print configuration file 'config' of 'node' """
1366 output = ""
1367 try:
1368 # Load config file
1369 if datadump == None:
1370 datadump = get_yaml(node)
1371
1372 if config == 'wleiden.yaml':
1373 output += generate_wleiden_yaml(datadump)
1374 elif config == 'authorized_keys':
1375 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1376 output += f.read()
1377 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1378 # Fetch local keys if existing
1379 if os.path.exists(node_keys):
1380 output += open(node_keys, 'r').read()
1381 f.close()
1382 elif config == 'dnsmasq.conf':
1383 output += generate_dnsmasq_conf(datadump)
1384 elif config == 'dhcpd.conf':
1385 output += generate_dhcpd_conf(datadump)
1386 elif config == 'rc.conf.local':
1387 output += generate_rc_conf_local(datadump)
1388 elif config == 'resolv.conf':
1389 output += generate_resolv_conf(datadump)
1390 elif config == 'ntp.conf':
1391 output += generate_ntp_conf(datadump)
1392 elif config == 'motd':
1393 output += generate_motd(datadump)
1394 elif config == 'pf.hybrid.conf.local':
1395 output += generate_pf_hybrid_conf_local(datadump)
1396 elif config == 'unbound.wleiden.conf':
1397 output += generate_unbound_wleiden_conf(datadump)
1398 elif config.startswith('vr'):
1399 interface, ns_type = config.strip('.yaml').split('-')
1400 output += generate_nanostation_config(datadump, interface, ns_type)
1401 else:
1402 assert False, "Config not found!"
1403 except IOError, e:
1404 output += "[ERROR] Config file not found"
1405 return output
1406
1407
1408
1409def process_cgi_request(environ=os.environ):
1410 """ When calling from CGI """
1411 response_headers = []
1412 content_type = 'text/plain'
1413
1414 # Update repository if requested
1415 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1416 if form and form.has_key("action") and "update" in form["action"]:
1417 output = "[INFO] Updating subverion, please wait...\n"
1418 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1419 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1420 output += "[INFO] All done, redirecting in 5 seconds"
1421 response_headers += [
1422 ('Refresh', '5; url=.'),
1423 ]
1424 reload_cache()
1425 else:
1426 base_uri = environ['PATH_INFO']
1427 uri = base_uri.strip('/').split('/')
1428
1429 output = "Template Holder"
1430 if base_uri.endswith('/create/network.kml'):
1431 content_type='application/vnd.google-earth.kml+xml'
1432 output = make_network_kml.make_graph()
1433 elif base_uri.endswith('/api/get/nodeplanner.json'):
1434 content_type='application/json'
1435 output = make_network_kml.make_nodeplanner_json()
1436 elif not uri[0]:
1437 if is_text_request(environ):
1438 output = '\n'.join(get_hostlist())
1439 else:
1440 content_type = 'text/html'
1441 output = generate_title(get_hostlist())
1442 elif len(uri) == 1:
1443 if is_text_request(environ):
1444 output = generate_node(uri[0])
1445 else:
1446 content_type = 'text/html'
1447 output = generate_node_overview(uri[0])
1448 elif len(uri) == 2:
1449 output = generate_config(uri[0], uri[1])
1450 else:
1451 assert False, "Invalid option"
1452
1453 # Return response
1454 response_headers += [
1455 ('Content-type', content_type),
1456 ('Content-Length', str(len(output))),
1457 ]
1458 return(response_headers, str(output))
1459
1460
1461def make_dns(output_dir = 'dns', external = False):
1462 items = dict()
1463
1464 # hostname is key, IP is value
1465 wleiden_zone = defaultdict(list)
1466 wleiden_cname = dict()
1467
1468 pool = dict()
1469 for node in get_hostlist():
1470 datadump = get_yaml(node)
1471
1472 fqdn = datadump['nodename']
1473
1474 if datadump.has_key('rdr_host'):
1475 remote_target = datadump['rdr_host']
1476 elif datadump.has_key('remote_access') and datadump['remote_access']:
1477 remote_target = datadump['remote_access'].split(':')[0]
1478 else:
1479 remote_target = None
1480
1481 if remote_target:
1482 try:
1483 parseaddr(remote_target)
1484 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1485 except (IndexError, ValueError):
1486 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1487
1488
1489 wleiden_zone[fqdn].append((datadump['masterip'], True))
1490
1491 # Hacking to get proper DHCP IPs and hostnames
1492 for iface_key in get_interface_keys(datadump):
1493 iface_name = iface_key.replace('_','-')
1494 if 'ip' in datadump[iface_key]:
1495 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1496 try:
1497 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1498 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1499 dhcp_part = ".".join(ip.split('.')[0:3])
1500 if ip != datadump['masterip']:
1501 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1502 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1503 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1504 except (AttributeError, ValueError, KeyError):
1505 # First push it into a pool, to indentify the counter-part later on
1506 addr = parseaddr(ip)
1507 cidr = int(cidr)
1508 addr = addr & ~((1 << (32 - cidr)) - 1)
1509 if pool.has_key(addr):
1510 pool[addr] += [(iface_name, fqdn, ip)]
1511 else:
1512 pool[addr] = [(iface_name, fqdn, ip)]
1513 continue
1514
1515
1516
1517 # WL uses an /29 to configure an interface. IP's are ordered like this:
1518 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1519
1520 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1521
1522 # Automatic naming convention of interlinks namely 2 + remote.lower()
1523 for (key,value) in pool.iteritems():
1524 # Make sure they are sorted from low-ip to high-ip
1525 value = sorted(value, key=lambda x: parseaddr(x[2]))
1526
1527 if len(value) == 1:
1528 (iface_name, fqdn, ip) = value[0]
1529 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1530
1531 # Device DNS names
1532 if 'cnode' in fqdn.lower():
1533 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1534 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1535
1536 elif len(value) == 2:
1537 (a_iface_name, a_fqdn, a_ip) = value[0]
1538 (b_iface_name, b_fqdn, b_ip) = value[1]
1539 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1540 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1541
1542 # Device DNS names
1543 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1544 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1545 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1546 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1547 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1548 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1549 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1550
1551 else:
1552 pool_members = [k[1] for k in value]
1553 for item in value:
1554 (iface_name, fqdn, ip) = item
1555 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1556
1557 # Include static DNS entries
1558 # XXX: Should they override the autogenerated results?
1559 # XXX: Convert input to yaml more useable.
1560 # Format:
1561 ##; this is a comment
1562 ## roomburgh=Roomburgh1
1563 ## apkerk1.Vosko=172.17.176.8 ;this as well
1564 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1565
1566 # Hack to allow special entries, for development
1567 wleiden_raw = {}
1568
1569 for line in dns_list:
1570 reverse = False
1571 k, items = line.items()[0]
1572 if type(items) == dict:
1573 if items.has_key('reverse'):
1574 reverse = items['reverse']
1575 items = items['a']
1576 else:
1577 items = items['cname']
1578 items = [items] if type(items) != list else items
1579 for item in items:
1580 if item.startswith('IN '):
1581 wleiden_raw[k] = item
1582 elif valid_addr(item):
1583 wleiden_zone[k].append((item, reverse))
1584 else:
1585 wleiden_cname[k] = item
1586
1587 # Hack to get dynamic pool listing
1588 def chunks(l, n):
1589 return [l[i:i+n] for i in range(0, len(l), n)]
1590
1591 ntp_servers = [x[0] for x in get_nameservers()]
1592 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1593 for ntp_server in chunk:
1594 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1595
1596 details = dict()
1597 # 24 updates a day allowed
1598 details['serial'] = time.strftime('%Y%m%d%H')
1599
1600 if external:
1601 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1602 else:
1603 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1604
1605 details['master'] = dns_masters[0]
1606 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1607
1608 dns_header = '''
1609$TTL 3h
1610%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1611 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1612
1613%(ns_servers)s
1614 \n'''
1615
1616
1617 if not os.path.isdir(output_dir):
1618 os.makedirs(output_dir)
1619 details['zone'] = 'wleiden.net'
1620 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1621 f.write(dns_header % details)
1622
1623 for host,items in wleiden_zone.iteritems():
1624 for ip,reverse in items:
1625 if ip not in ['0.0.0.0']:
1626 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1627 for source,dest in wleiden_cname.iteritems():
1628 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1629 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1630 for source, dest in wleiden_raw.iteritems():
1631 f.write("%s.wleiden.net. %s\n" % (source, dest))
1632 f.close()
1633
1634 # Create whole bunch of specific sub arpa zones. To keep it compliant
1635 for s in range(16,32):
1636 details['zone'] = '%i.172.in-addr.arpa' % s
1637 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1638 f.write(dns_header % details)
1639
1640 #XXX: Not effient, fix to proper data structure and do checks at other
1641 # stages
1642 for host,items in wleiden_zone.iteritems():
1643 for ip,reverse in items:
1644 if not reverse:
1645 continue
1646 if valid_addr(ip):
1647 if valid_addr(ip):
1648 if int(ip.split('.')[1]) == s:
1649 rev_ip = '.'.join(reversed(ip.split('.')))
1650 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1651 f.close()
1652
1653
1654def usage():
1655 print """Usage: %(prog)s <argument>
1656Argument:
1657\tcleanup = Cleanup all YAML files to specified format
1658\tstandalone [port] = Run configurator webserver [8000]
1659\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1660\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1661\tfull-export = Generate yaml export script for heatmap.
1662\tstatic [outputdir] = Generate all config files and store on disk
1663\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1664\ttest <node> [<file>] = Receive output for certain node [all files].
1665\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1666\tlist <status> <items> = List systems which have certain status
1667\tcreate network.kml = Create Network KML file for use in Google Earth
1668
1669Arguments:
1670\t<node> = NodeName (example: HybridRick)
1671\t<file> = %(files)s
1672\t<status> = all|up|down|planned
1673\t<items> = systems|nodes|proxies
1674
1675NOTE FOR DEVELOPERS; you can test your changes like this:
1676 BEFORE any changes in this code:
1677 $ ./gformat.py static /tmp/pre
1678 AFTER the changes:
1679 $ ./gformat.py static /tmp/post
1680 VIEW differences and VERIFY all are OK:
1681 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1682""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1683 exit(0)
1684
1685
1686def is_text_request(environ=os.environ):
1687 """ Find out whether we are calling from the CLI or any text based CLI utility """
1688 try:
1689 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1690 except KeyError:
1691 return True
1692
1693def switchFormat(setting):
1694 if setting:
1695 return "YES"
1696 else:
1697 return "NO"
1698
1699def rlinput(prompt, prefill=''):
1700 import readline
1701 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1702 try:
1703 return raw_input(prompt)
1704 finally:
1705 readline.set_startup_hook()
1706
1707def fix_conflict(left, right, default='i'):
1708 while True:
1709 print "## %-30s | %-30s" % (left, right)
1710 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1711 if not c:
1712 c = default
1713
1714 if c in ['l','1']:
1715 return left
1716 elif c in ['r','2']:
1717 return right
1718 elif c in ['e', '3']:
1719 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1720 elif c in ['i', '4']:
1721 return None
1722 else:
1723 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1724
1725
1726
1727def print_cgi_response(response_headers, output):
1728 """Could we not use some kind of wsgi wrapper to make this output?"""
1729 for header in response_headers:
1730 print "%s: %s" % header
1731 print
1732 print output
1733
1734
1735def fill_cache():
1736 ''' Poor man re-loading of few cache items (the slow ones) '''
1737 for host in get_hostlist():
1738 get_yaml(host)
1739
1740
1741def reload_cache():
1742 clear_cache()
1743 fill_cache()
1744
1745
1746def main():
1747 """Hard working sub"""
1748 # Allow easy hacking using the CLI
1749 if not os.environ.has_key('PATH_INFO'):
1750 if len(sys.argv) < 2:
1751 usage()
1752
1753 if sys.argv[1] == "standalone":
1754 import SocketServer
1755 import CGIHTTPServer
1756 # Hop to the right working directory.
1757 os.chdir(os.path.dirname(__file__))
1758 try:
1759 PORT = int(sys.argv[2])
1760 except (IndexError,ValueError):
1761 PORT = 8000
1762
1763 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1764 """ Serve this CGI from the root of the webserver """
1765 def is_cgi(self):
1766 if "favicon" in self.path:
1767 return False
1768
1769 self.cgi_info = (os.path.basename(__file__), self.path)
1770 self.path = ''
1771 return True
1772 handler = MyCGIHTTPRequestHandler
1773 SocketServer.TCPServer.allow_reuse_address = True
1774 httpd = SocketServer.TCPServer(("", PORT), handler)
1775 httpd.server_name = 'localhost'
1776 httpd.server_port = PORT
1777
1778 logger.info("serving at port %s", PORT)
1779 try:
1780 httpd.serve_forever()
1781 except KeyboardInterrupt:
1782 httpd.shutdown()
1783 logger.info("All done goodbye")
1784 elif sys.argv[1] == "test":
1785 # Basic argument validation
1786 try:
1787 node = sys.argv[2]
1788 except IndexError:
1789 print "Invalid argument"
1790 exit(1)
1791 except IOError as e:
1792 print e
1793 exit(1)
1794
1795 datadump = get_yaml(node)
1796
1797
1798 # Get files to generate
1799 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1800
1801 # Actual config generation
1802 for config in gen_files:
1803 logger.info("## Generating %s %s", node, config)
1804 print generate_config(node, config, datadump)
1805 elif sys.argv[1] == "test-cgi":
1806 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1807 os.environ['SCRIPT_NAME'] = __file__
1808 response_headers, output = process_cgi_request()
1809 print_cgi_response(response_headers, output)
1810 elif sys.argv[1] == "static":
1811 items = dict()
1812 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1813 for node in get_hostlist():
1814 items['node'] = node
1815 items['wdir'] = "%(output_dir)s/%(node)s" % items
1816 if not os.path.isdir(items['wdir']):
1817 os.makedirs(items['wdir'])
1818 datadump = get_yaml(node)
1819 for config in files:
1820 items['config'] = config
1821 logger.info("## Generating %(node)s %(config)s" % items)
1822 f = open("%(wdir)s/%(config)s" % items, "w")
1823 f.write(generate_config(node, config, datadump))
1824 f.close()
1825 elif sys.argv[1] == "wind-export":
1826 items = dict()
1827 for node in get_hostlist():
1828 datadump = get_yaml(node)
1829 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1830 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1831 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1832 VALUES (
1833 (SELECT id FROM users WHERE username = 'rvdzwet'),
1834 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1835 'Y');""" % datadump
1836 #for config in files:
1837 # items['config'] = config
1838 # print "## Generating %(node)s %(config)s" % items
1839 # f = open("%(wdir)s/%(config)s" % items, "w")
1840 # f.write(generate_config(node, config, datadump))
1841 # f.close()
1842 for node in get_hostlist():
1843 datadump = get_yaml(node)
1844 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1845 ifacedump = datadump[iface_key]
1846 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1847 ifacedump['nodename'] = datadump['nodename']
1848 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1849 ifacedump['channel'] = 0
1850 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1851 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1852 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1853 elif sys.argv[1] == "nagios-export":
1854 try:
1855 heavy_load = (sys.argv[2] == "--heavy-load")
1856 except IndexError:
1857 heavy_load = False
1858
1859 hostgroup_details = {
1860 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1861 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1862 'walphen' : 'Stichting Wireless Alphen',
1863 'westeinder' : 'Westeinder Plassen',
1864 }
1865
1866 # Convert IP to Host
1867 ip2host = {'root' : 'root'}
1868 for host in get_hostlist():
1869 datadump = get_yaml(host)
1870 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1871 for iface in get_interface_keys(datadump):
1872 if datadump[iface].has_key('autogen_gateway'):
1873 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1874
1875 # Find dependency tree based on output of lvrouted.mytree of nearest node
1876 parents = defaultdict(list)
1877 stack = ['root']
1878 prev_depth = 0
1879 for line in open('lvrouted.mytree').readlines():
1880 depth = line.count('\t')
1881 ip = line.strip().split()[0]
1882
1883 if prev_depth < depth:
1884 try:
1885 parents[ip2host[ip]].append(ip2host[stack[-1]])
1886 except KeyError as e:
1887 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1888 stack.append(ip)
1889 elif prev_depth > depth:
1890 stack = stack[:(depth - prev_depth)]
1891 elif prev_depth == depth:
1892 try:
1893 parents[ip2host[ip]].append(ip2host[stack[-1]])
1894 except KeyError as e:
1895 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1896
1897
1898 prev_depth = depth
1899 # Observe that some nodes has themself as parent or multiple parents
1900 # for now take only the first parent, other behaviour is yet to be explained
1901
1902
1903
1904 params = {
1905 'check_interval' : 5 if heavy_load else 120,
1906 'retry_interval' : 1 if heavy_load else 10,
1907 'max_check_attempts' : 10 if heavy_load else 6,
1908 'notification_interval': 120 if heavy_load else 240,
1909 }
1910
1911 print '''\
1912define host {
1913 name wleiden-node ; Default Node Template
1914 use generic-host ; Use the standard template as initial starting point
1915 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1916 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1917 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1918 notification_interval %(notification_interval)s
1919 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1920 check_command check-host-alive ; Default command to check FreeBSD hosts
1921 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1922}
1923
1924define service {
1925 name wleiden-service ; Default Service Template
1926 use generic-service ; Use the standard template as initial starting point
1927 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1928 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1929 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1930 notification_interval %(notification_interval)s
1931 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1932 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1933}
1934
1935# Please make sure to install:
1936# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1937#
1938# Recompile net-mgmt/nagios-plugins to support check_snmp
1939# make -C /usr/ports/net-mgmt/nagios-plugins
1940#
1941# Install net/bind-tools to allow v2/check_dns_wl to work:
1942# pkg install bind-tools
1943#
1944define command{
1945 command_name check_snmp_disk
1946 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1947}
1948
1949define command{
1950 command_name check_netsnmp_load
1951 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1952}
1953
1954define command{
1955 command_name check_netsnmp_proc
1956 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1957}
1958
1959define command{
1960 command_name check_by_ssh
1961 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1962}
1963
1964define command{
1965 command_name check_dns_wl
1966 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1967}
1968
1969define command{
1970 command_name check_snmp_uptime
1971 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1972}
1973
1974
1975# TDB: dhcp leases
1976# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1977
1978# TDB: internet status
1979# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1980
1981# TDB: Advanced local passive checks
1982# /usr/local/libexec/nagios/check_by_ssh
1983''' % params
1984
1985 print '''\
1986# Service Group, not displayed by default
1987define hostgroup {
1988 hostgroup_name srv_hybrid
1989 alias All Hybrid Nodes
1990 register 0
1991}
1992
1993define service {
1994 use wleiden-service
1995 hostgroup_name srv_hybrid
1996 service_description SSH
1997 check_command check_ssh
1998}
1999
2000define service {
2001 use wleiden-service,service-pnp
2002 hostgroup_name srv_hybrid
2003 service_description HTTP
2004 check_command check_http
2005}
2006
2007define service {
2008 use wleiden-service
2009 hostgroup_name srv_hybrid
2010 service_description DNS
2011 check_command check_dns_wl!"www.wirelessleiden.nl"
2012}
2013'''
2014
2015 if heavy_load:
2016 print '''\
2017define service {
2018 use wleiden-service
2019 hostgroup_name srv_hybrid
2020 service_description UPTIME
2021 check_command check_snmp_uptime
2022}
2023
2024#define service {
2025# use wleiden-service
2026# hostgroup_name srv_hybrid
2027# service_description NTP
2028# check_command check_ntp_peer
2029#}
2030
2031define service {
2032 use wleiden-service
2033 hostgroup_name srv_hybrid
2034 service_description LOAD
2035 check_command check_netsnmp_load
2036}
2037
2038define service {
2039 use wleiden-service
2040 hostgroup_name srv_hybrid
2041 service_description PROC
2042 check_command check_netsnmp_proc
2043}
2044
2045define service {
2046 use wleiden-service
2047 hostgroup_name srv_hybrid
2048 service_description DISK
2049 check_command check_snmp_disk
2050}
2051'''
2052 for node in get_hostlist():
2053 datadump = get_yaml(node)
2054 if not datadump['status'] == 'up':
2055 continue
2056 if not hostgroup_details.has_key(datadump['monitoring_group']):
2057 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2058 print '''\
2059define host {
2060 use wleiden-node,host-pnp
2061 contact_groups admins
2062 host_name %(autogen_fqdn)s
2063 address %(masterip)s
2064 hostgroups srv_hybrid,%(monitoring_group)s\
2065''' % datadump
2066 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2067 print '''\
2068 parents %(parents)s\
2069''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2070 print '''\
2071}
2072'''
2073
2074
2075 for name,alias in hostgroup_details.iteritems():
2076 print '''\
2077define hostgroup {
2078 hostgroup_name %s
2079 alias %s
2080} ''' % (name, alias)
2081
2082
2083 elif sys.argv[1] == "full-export":
2084 hosts = {}
2085 for node in get_hostlist():
2086 datadump = get_yaml(node)
2087 hosts[datadump['nodename']] = datadump
2088 print yaml.dump(hosts)
2089
2090 elif sys.argv[1] == "dns":
2091 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2092 elif sys.argv[1] == "cleanup":
2093 # First generate all datadumps
2094 datadumps = dict()
2095 ssid_to_node = dict()
2096 for host in get_hostlist():
2097 logger.info("# Processing: %s", host)
2098 # Set some boring default values
2099 datadump = { 'board' : 'UNKNOWN' }
2100 datadump.update(get_yaml(host))
2101 datadumps[datadump['nodename']] = datadump
2102
2103 (poel, errors) = make_relations()
2104 print "\n".join(["# WARNING: %s" % x for x in errors])
2105
2106 for host,datadump in datadumps.iteritems():
2107 try:
2108 # Convert all yes and no to boolean values
2109 def fix_boolean(dump):
2110 for key in dump.keys():
2111 if type(dump[key]) == dict:
2112 dump[key] = fix_boolean(dump[key])
2113 elif str(dump[key]).lower() in ["yes", "true"]:
2114 dump[key] = True
2115 elif str(dump[key]).lower() in ["no", "false"]:
2116 # Compass richting no (Noord Oost) is valid input
2117 if key != "compass": dump[key] = False
2118 return dump
2119 datadump = fix_boolean(datadump)
2120
2121 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2122 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2123 elif 'latitude' in datadump and 'longitude' in datadump:
2124 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2125
2126 if datadump['nodename'].startswith('Proxy'):
2127 datadump['nodename'] = datadump['nodename'].lower()
2128
2129 for iface_key in get_interface_keys(datadump):
2130 try:
2131 # All our normal wireless cards are normal APs now
2132 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2133 datadump[iface_key]['mode'] = 'ap'
2134 # Wireless Leiden SSID have an consistent lowercase/uppercase
2135 if datadump[iface_key].has_key('ssid'):
2136 ssid = datadump[iface_key]['ssid']
2137 prefix = 'ap-WirelessLeiden-'
2138 if ssid.lower().startswith(prefix.lower()):
2139 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2140 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2141 datadump[iface_key]['mode'] = 'autogen-FIXME'
2142 if not datadump[iface_key].has_key('comment'):
2143 datadump[iface_key]['comment'] = 'autogen-FIXME'
2144
2145 if datadump[iface_key].has_key('ns_mac'):
2146 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2147
2148 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2149 datadump[iface_key] = datadump[iface_key]['desc']
2150
2151 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2152 # due to better collision avoidance.
2153 if datadump[iface_key]['type'] == '11b':
2154 datadump[iface_key]['type'] = '11g'
2155
2156 # Setting 802.11g channels to de-facto standards, to avoid
2157 # un-detected sharing with other overlapping channels
2158 #
2159 # Technically we could also use channel 13 in NL, but this is not
2160 # recommended as foreign devices might not be able to select this
2161 # channel. Secondly using 1,5,9,13 instead is going to clash with
2162 # the de-facto usage of 1,6,11.
2163 #
2164 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2165 channels_at_2400Mhz = (1,6,11)
2166 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2167 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2168 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2169 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2170
2171 # Mandatory interface keys
2172 if not datadump[iface_key].has_key('status'):
2173 datadump[iface_key]['status'] = 'planned'
2174
2175 x = datadump[iface_key]['comment']
2176 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2177
2178 # Fixing bridge_type if none is found
2179 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2180 if not 'bridge_type' in datadump[iface_key]:
2181 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2182
2183 # Making sure description works
2184 if datadump[iface_key].has_key('desc'):
2185 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2186 del datadump[iface_key]['desc']
2187 else:
2188 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2189 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2190 if response:
2191 datadump[iface_key]['comment'] = response
2192 del datadump[iface_key]['desc']
2193
2194 # Check DHCP configuration
2195 dhcp_type(datadump[iface_key])
2196
2197 # Set the compass value based on the angle between the poels
2198 if datadump[iface_key].has_key('ns_ip'):
2199 my_pool = poel[network(datadump[iface_key]['ip'])]
2200 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2201 if remote_hosts:
2202 compass_target = remote_hosts[0]
2203 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2204
2205 # Monitoring Group default
2206 if not 'monitoring_group' in datadump:
2207 datadump['monitoring_group'] = 'wleiden'
2208
2209 except Exception:
2210 print "# Error while processing interface %s" % iface_key
2211 raise
2212 store_yaml(datadump)
2213 except Exception:
2214 print "# Error while processing %s" % host
2215 raise
2216 elif sys.argv[1] == "list":
2217 use_fqdn = False
2218 if len(sys.argv) < 4:
2219 usage()
2220 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2221 usage()
2222 if not sys.argv[3] in ["nodes","proxies","systems"]:
2223 usage()
2224
2225 if len(sys.argv) > 4:
2226 if sys.argv[4] == "fqdn":
2227 use_fqdn = True
2228 else:
2229 usage()
2230
2231 for system in get_hostlist():
2232 datadump = get_yaml(system)
2233 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2234 continue
2235
2236 output = datadump['autogen_fqdn'] if use_fqdn else system
2237 if sys.argv[2] == "all":
2238 print output
2239 elif datadump['status'] == sys.argv[2]:
2240 print output
2241 elif sys.argv[1] == "create":
2242 if sys.argv[2] == "network.kml":
2243 print make_network_kml.make_graph()
2244 elif sys.argv[2] == "host-ips.txt":
2245 for system in get_hostlist():
2246 datadump = get_yaml(system)
2247 ips = [datadump['masterip']]
2248 for ifkey in get_interface_keys(datadump):
2249 ips.append(datadump[ifkey]['ip'].split('/')[0])
2250 print system, ' '.join(ips)
2251 elif sys.argv[2] == "host-pos.txt":
2252 for system in get_hostlist():
2253 datadump = get_yaml(system)
2254 print system, datadump['rdnap_x'], datadump['rdnap_y']
2255 elif sys.argv[2] == 'ssh_config':
2256 print '''
2257Host *.wleiden.net
2258 User root
2259
2260Host 172.16.*.*
2261 User root
2262'''
2263 for system in get_hostlist():
2264 datadump = get_yaml(system)
2265 print '''\
2266Host %s
2267 User root
2268
2269Host %s
2270 User root
2271
2272Host %s
2273 User root
2274
2275Host %s
2276 User root
2277''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2278 else:
2279 usage()
2280 else:
2281 usage()
2282 else:
2283 # Do not enable debugging for config requests as it highly clutters the output
2284 if not is_text_request():
2285 cgitb.enable()
2286 response_headers, output = process_cgi_request()
2287 print_cgi_response(response_headers, output)
2288
2289def application(environ, start_response):
2290 status = '200 OK'
2291 response_headers, output = process_cgi_request(environ)
2292 start_response(status, response_headers)
2293
2294 # Debugging only
2295 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2296 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2297 return [output]
2298
2299if __name__ == "__main__":
2300 main()
Note: See TracBrowser for help on using the repository browser.