source: genesis/tools/gformat.py@ 13894

Last change on this file since 13894 was 13886, checked in by rick, 8 years ago

Typo
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 78.2 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# FreeBSD:
28# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
29# Fedora:
30# yum install python-yaml pyproj proj-epsg python-jinja2
31#
32# Rick van der Zwet <info@rickvanderzwet.nl>
33#
34
35# Hack to make the script directory is also threated as a module search path.
36import sys
37import os
38sys.path.append(os.path.dirname(__file__))
39
40SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
41
42import argparse
43import cgi
44import cgitb
45import copy
46import glob
47import make_network_kml
48import math
49import pyproj
50import random
51import re
52import socket
53import string
54import subprocess
55import textwrap
56import time
57import urlparse
58
59from pprint import pprint
60from collections import defaultdict, OrderedDict
61from sys import stderr
62try:
63 import yaml
64except ImportError, e:
65 print e
66 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
67 exit(1)
68
69try:
70 from yaml import CLoader as Loader
71 from yaml import CDumper as Dumper
72except ImportError:
73 from yaml import Loader, Dumper
74
75from jinja2 import Environment, Template
76def yesorno(value):
77 return "YES" if bool(value) else "NO"
78env = Environment()
79env.filters['yesorno'] = yesorno
80def render_template(datadump, template):
81 result = env.from_string(template).render(datadump)
82 # Make it look pretty to the naked eye, as jinja templates are not so
83 # friendly when it comes to whitespace formatting
84 ## Remove extra whitespace at end of line lstrip() style.
85 result = re.sub(r'\n[\ ]+','\n', result)
86 ## Include only a single newline between an definition and a comment
87 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
88 ## Remove extra newlines after single comment
89 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
90 return result
91
92import logging
93logging.basicConfig(format='# %(levelname)s: %(message)s' )
94logger = logging.getLogger()
95logger.setLevel(logging.DEBUG)
96
97
98if os.environ.has_key('CONFIGROOT'):
99 NODE_DIR = os.environ['CONFIGROOT']
100else:
101 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
102__version__ = '$Id: gformat.py 13886 2017-05-27 14:43:31Z rick $'
103
104files = [
105 'authorized_keys',
106 'dnsmasq.conf',
107 'dhcpd.conf',
108 'rc.conf.local',
109 'resolv.conf',
110 'motd',
111 'ntp.conf',
112 'pf.hybrid.conf.local',
113 'unbound.wleiden.conf',
114 'wleiden.yaml',
115 ]
116
117# Global variables uses
118OK = 10
119DOWN = 20
120UNKNOWN = 90
121
122
123ileiden_proxies = OrderedDict()
124normal_proxies = []
125datadump_cache = {}
126interface_list_cache = {}
127rc_conf_local_cache = {}
128nameservers_cache = []
129relations_cache = None
130def clear_cache():
131 ''' Poor mans cache implementation '''
132 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
133 datadump_cache = {}
134 interface_list_cache = {}
135 rc_conf_local_cache = {}
136 ileiden_proxies = OrderedDict()
137 normal_proxies = []
138 nameservers_cache = []
139 relations_cache = None
140
141NO_DHCP = 0
142DHCP_CLIENT = 10
143DHCP_SERVER = 20
144def dhcp_type(item):
145 if not item.has_key('dhcp'):
146 return NO_DHCP
147 elif not item['dhcp']:
148 return NO_DHCP
149 elif item['dhcp'].lower() == 'client':
150 return DHCP_CLIENT
151 else:
152 # Validation Checks
153 begin,end = map(int,item['dhcp'].split('-'))
154 if begin >= end:
155 raise ValueError("DHCP Start >= DHCP End")
156 return DHCP_SERVER
157
158def etrs2rd(lat, lon):
159 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
160 p2 = pyproj.Proj(init='EPSG:28992')
161 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
162 return (RDx, RDy)
163
164def rd2etrs(RDx, RDy):
165 p1 = pyproj.Proj(init='EPSG:28992')
166 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
167 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
168 return (lat, lon)
169
170def get_yaml(item,add_version_info=True):
171 try:
172 """ Get configuration yaml for 'item'"""
173 if datadump_cache.has_key(item):
174 return datadump_cache[item].copy()
175
176 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
177 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
178 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
179
180 # Default values
181 datadump = {
182 'autogen_revision' : 'NOTFOUND',
183 'autogen_gfile' : gfile,
184 'service_proxy_ileiden' : False,
185 }
186 f = open(gfile, 'r')
187 datadump.update(yaml.load(f,Loader=Loader))
188 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
189 if datadump['nodetype'] == 'Hybrid':
190 # Some values are defined implicitly
191 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
192 datadump['service_incoming_rdr'] = True
193 # Use some boring defaults
194 defaults = {
195 'service_proxy_normal' : False,
196 'service_accesspoint' : True,
197 'service_incoming_rdr' : False,
198 'service_concentrator' : False,
199 'monitoring_group' : 'wleiden',
200 }
201 for (key,value) in defaults.iteritems():
202 if not datadump.has_key(key):
203 datadump[key] = value
204 f.close()
205
206 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
207 if add_version_info:
208 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
209 lines = p.communicate()[0].split('\n')
210 if p.returncode == 0:
211 for line in lines:
212 if line:
213 (key, value) = line.split(': ')
214 datadump["autogen_" + key.lower().replace(' ','_')] = value
215
216 # Preformat certain needed variables for formatting and push those into special object
217 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
218
219 wlan_count=0
220 try:
221 for key in get_interface_keys(datadump, True):
222 datadump[key]['autogen_ifbase'] = key.split('_')[1]
223 datadump[key]['autogen_vlan'] = False
224
225 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
226 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
227
228 if datadump[key].has_key('parent'):
229 if datadump[key].has_key('ip'):
230 raise ValueError("Interface bridge member cannot have IP assigned")
231 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
232 raise ValueError("Interface bridge member cannot have DHCP set")
233
234 if datadump[key].has_key('ip'):
235 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
236
237 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
238 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
239 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
240 wlan_count += 1
241 else:
242 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
243 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
244 datadump[key]['autogen_vlan'] = key.split('_')[2]
245 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
246 else:
247 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
248
249 except Exception:
250 print "# Error while processing interface %s" % key
251 raise
252
253 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
254 if dhcp_type(datadump[key]) == DHCP_SERVER]
255
256 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
257 datadump['autogen_item'] = item
258
259 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
260 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
261 datadump_cache[item] = datadump.copy()
262 except Exception:
263 print "# Error while processing %s" % item
264 raise
265 return datadump
266
267
268def store_yaml(datadump, header=False):
269 """ Store configuration yaml for 'item'"""
270 item = datadump['autogen_item']
271 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
272
273 output = generate_wleiden_yaml(datadump, header)
274
275 f = open(gfile, 'w')
276 f.write(output)
277 f.close()
278
279
280def network(ip):
281 addr, mask = ip.split('/')
282 # Not parsing of these folks please
283 addr = parseaddr(addr)
284 mask = int(mask)
285 network = addr & ~((1 << (32 - mask)) - 1)
286 return network
287
288
289
290def make_relations():
291 """ Process _ALL_ yaml files to get connection relations """
292 global relations_cache
293
294 if relations_cache:
295 return relations_cache
296
297 errors = []
298 poel = defaultdict(list)
299
300 for host in get_hostlist():
301 datadump = get_yaml(host)
302 try:
303 for iface_key in get_interface_keys(datadump):
304 # Bridge members has no IP assigned
305 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
306 continue
307 net_addr = network(datadump[iface_key]['ip'])
308 poel[net_addr] += [(host,datadump[iface_key].copy())]
309 except (KeyError, ValueError) as e:
310 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
311 continue
312
313 relations_cache = (poel, errors)
314 return relations_cache
315
316
317
318def valid_addr(addr):
319 """ Show which address is valid in which are not """
320 return str(addr).startswith('172.')
321
322def get_hostlist():
323 """ Combined hosts and proxy list"""
324 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
325
326def angle_between_points(lat1,lat2,long1,long2):
327 """
328 Return Angle in radians between two GPS coordinates
329 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
330 """
331 dy = lat2 - lat1
332 dx = math.cos(lat1)*(long2 - long1)
333 angle = math.atan2(dy,dx)
334 return angle
335
336
337
338def angle_to_cd(angle):
339 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
340
341 # For easy conversion get positive degree
342 degrees = math.degrees(angle)
343 abs_degrees = 360 + degrees if degrees < 0 else degrees
344
345 # Numbers can be confusing calculate from the 4 main directions
346 p = 22.5
347 if abs_degrees < p:
348 cd = "n"
349 elif abs_degrees < (90 - p):
350 cd = "no"
351 elif abs_degrees < (90 + p):
352 cd = "o"
353 elif abs_degrees < (180 - p):
354 cd = "zo"
355 elif abs_degrees < (180 + p):
356 cd = "z"
357 elif abs_degrees < (270 - p):
358 cd = "zw"
359 elif abs_degrees < (270 + p):
360 cd = "w"
361 elif abs_degrees < (360 - p):
362 cd = "nw"
363 else:
364 cd = "n"
365 return cd
366
367
368
369def cd_between_hosts(hostA, hostB, datadumps):
370 # Using RDNAP coordinates
371 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
372 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
373 return angle_to_cd(math.atan2(dx,dy))
374
375 # GPS coordinates seems to fail somehow
376 #latA = float(datadumps[hostA]['latitude'])
377 #latB = float(datadumps[hostB]['latitude'])
378 #lonA = float(datadumps[hostA]['longitude'])
379 #lonB = float(datadumps[hostB]['longitude'])
380 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
381
382
383def generate_title(nodelist):
384 """ Main overview page """
385 items = {'root' : "." }
386 def fl(spaces, line):
387 return (' ' * spaces) + line + '\n'
388
389 output = """
390<html>
391 <head>
392 <title>Wireless leiden Configurator - GFormat</title>
393 <style type="text/css">
394 th {background-color: #999999}
395 tr:nth-child(odd) {background-color: #cccccc}
396 tr:nth-child(even) {background-color: #ffffff}
397 th, td {padding: 0.1em 1em}
398 </style>
399 </head>
400 <body>
401 <center>
402 <form type="GET" action="%(root)s">
403 <input type="hidden" name="action" value="update">
404 <input type="submit" value="Update Configuration Database (SVN)">
405 </form>
406 <table>
407 <caption><h3>Wireless Leiden Configurator</h3></caption>
408 """ % items
409
410 for node in nodelist:
411 items['node'] = node
412 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
413 for config in files:
414 items['config'] = config
415 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
416 output += fl(5, "</tr>")
417 output += """
418 </table>
419 <hr />
420 <em>%s</em>
421 </center>
422 </body>
423</html>
424 """ % __version__
425
426 return output
427
428
429
430def generate_node(node):
431 """ Print overview of all files available for node """
432 return "\n".join(files)
433
434def generate_node_overview(host):
435 """ Print overview of all files available for node """
436 datadump = get_yaml(host)
437 params = { 'host' : host }
438 output = "<em><a href='..'>Back to overview</a></em><hr />"
439 output += "<h2>Available files:</h2><ul>"
440 for cf in files:
441 params['cf'] = cf
442 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
443 output += "</ul>"
444
445 # Generate and connection listing
446 output += "<h2>Connected To:</h2><ul>"
447 (poel, errors) = make_relations()
448 for network, hosts in poel.iteritems():
449 if host in [x[0] for x in hosts]:
450 if len(hosts) == 1:
451 # Single not connected interface
452 continue
453 for remote,ifacedump in hosts:
454 if remote == host:
455 # This side of the interface
456 continue
457 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
458 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
459 output += "</ul>"
460 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
461
462 output += "<hr /><em><a href='..'>Back to overview</a></em>"
463 return output
464
465
466def generate_header(datadump, ctag="#"):
467 return """\
468%(ctag)s
469%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
470%(ctag)s
471""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
472
473
474
475def parseaddr(s):
476 """ Process IPv4 CIDR notation addr to a (binary) number """
477 f = s.split('.')
478 return (long(f[0]) << 24L) + \
479 (long(f[1]) << 16L) + \
480 (long(f[2]) << 8L) + \
481 long(f[3])
482
483
484
485def showaddr(a):
486 """ Display IPv4 addr in (dotted) CIDR notation """
487 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
488
489
490def is_member(ip, mask, canidate):
491 """ Return True if canidate is part of ip/mask block"""
492 ip_addr = parseaddr(ip)
493 ip_canidate = parseaddr(canidate)
494 mask = int(mask)
495 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
496 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
497 return ip_addr == ip_canidate
498
499
500
501def cidr2netmask(netmask):
502 """ Given a 'netmask' return corresponding CIDR """
503 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
504
505def get_network(addr, mask):
506 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
507
508
509def generate_dhcpd_conf(datadump):
510 """ Generate config file '/usr/local/etc/dhcpd.conf """
511 # Redundency support, in cause local DNS server is not running/responding.
512 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
513 output = generate_header(datadump)
514 output += Template("""\
515# option definitions common to all supported networks...
516option domain-name "dhcp.{{ autogen_fqdn }}";
517
518default-lease-time 600;
519max-lease-time 7200;
520
521# Use this to enble / disable dynamic dns updates globally.
522#ddns-update-style none;
523
524# If this DHCP server is the official DHCP server for the local
525# network, the authoritative directive should be uncommented.
526authoritative;
527
528# Use this to send dhcp log messages to a different log file (you also
529# have to hack syslog.conf to complete the redirection).
530log-facility local7;
531
532#
533# Interface definitions
534#
535\n\n""").render(datadump)
536
537
538 # TODO: Use textwrap.fill instead
539 def indent(text, count):
540 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
541
542 # Process DHCP blocks
543 dhcp_out = defaultdict(list)
544 for iface_key in get_interface_keys(datadump):
545 ifname = datadump[iface_key]['autogen_ifbase']
546 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
547 if not datadump[iface_key].has_key('comment'):
548 datadump[iface_key]['comment'] = None
549
550 if not datadump[iface_key].has_key('ip'):
551 continue
552
553 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
554
555 (addr, mask) = datadump[iface_key]['ip'].split('/')
556 datadump[iface_key]['autogen_addr'] = addr
557 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
558 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
559
560 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
561 dhcp_out[groupif].append(textwrap.dedent("""\
562 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
563 ### not autoritive
564 }
565 """ % datadump[iface_key]))
566 continue
567
568 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
569 dhcp_part = ".".join(addr.split('.')[0:3])
570 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
571 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
572 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
573
574 # Assume the first 10 IPs could be used for static entries
575 if 'no_portal' in datadump:
576 fixed = 5
577 for mac in datadump['no_portal']:
578 dhcp_out[groupif].append(textwrap.dedent("""\
579 host fixed-%(ifname)s-%(fixed)s {
580 hardware ethernet %(mac)s;
581 fixed-address %(prefix)s.%(fixed)s;
582 }
583 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
584 fixed += 1
585
586 if 'dhcp_fixed' in datadump[iface_key]:
587 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
588 dhcp_out[groupif].append(textwrap.dedent("""\
589 host fixed-%(host)s {
590 hardware ethernet %(mac)s;
591 fixed-address %(addr)s;
592 }
593 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
594
595
596 dhcp_out[groupif].append(textwrap.dedent("""\
597 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
598 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
599 option routers %(autogen_addr)s;
600 option domain-name-servers %(autogen_dns_servers)s;
601
602 }
603 """ % datadump[iface_key]))
604
605 # Output the blocks in groups
606 for ifname,value in sorted(dhcp_out.iteritems()):
607 if len(value) > 2:
608 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
609 else:
610 output += ''.join(value) + "\n\n"
611 return output
612
613
614
615def generate_dnsmasq_conf(datadump):
616 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
617 output = generate_header(datadump)
618 output += Template("""\
619# DHCP server options
620dhcp-authoritative
621dhcp-fqdn
622domain=dhcp.{{ autogen_fqdn }}
623domain-needed
624expand-hosts
625log-async=100
626
627# Low memory footprint
628cache-size=10000
629
630\n""").render(datadump)
631
632 for iface_key in get_interface_keys(datadump):
633 if not datadump[iface_key].has_key('comment'):
634 datadump[iface_key]['comment'] = None
635 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
636
637 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
638 output += "# not autoritive\n\n"
639 continue
640
641 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
642 (ip, cidr) = datadump[iface_key]['ip'].split('/')
643 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
644
645 dhcp_part = ".".join(ip.split('.')[0:3])
646 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
647 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
648 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
649
650 return output
651
652
653class AutoVivification(dict):
654 """Implementation of perl's autovivification feature."""
655 def __getitem__(self, item):
656 try:
657 return dict.__getitem__(self, item)
658 except KeyError:
659 value = self[item] = type(self)()
660 return value
661
662def make_interface_list(datadump):
663 if interface_list_cache.has_key(datadump['autogen_item']):
664 return (interface_list_cache[datadump['autogen_item']])
665 # lo0 configuration:
666 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
667 # - masterip is special as it needs to be assigned to at
668 # least one interface, so if not used assign to lo0
669 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
670 vlan_list = defaultdict(list)
671 bridge_list = defaultdict(list)
672 flags_if = AutoVivification()
673 dhclient_if = {'lo0' : False}
674
675 # XXX: Find some way of send this output nicely
676 output = ''
677
678 masterip_used = False
679 for iface_key in get_interface_keys(datadump):
680 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
681 masterip_used = True
682 break
683 if not masterip_used:
684 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
685
686 if 'serviceid' in datadump:
687 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
688
689 for iface_key in get_interface_keys(datadump):
690 ifacedump = datadump[iface_key]
691 ifname = ifacedump['autogen_ifname']
692
693 # If defined as vlan interface
694 if ifacedump['autogen_vlan']:
695 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
696
697 # If defined as bridge interface
698 if ifacedump['autogen_bridge_member']:
699 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
700
701 # Flag dhclient is possible
702 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
703 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
704
705 # Ethernet address
706 if ifacedump.has_key('ether'):
707 flags_if[ifname]['ether'] = ifacedump['ether']
708
709 # Add interface IP to list
710 if ifacedump.has_key('ip'):
711 item = (ifacedump['ip'], ifacedump['comment'])
712 if addrs_list.has_key(ifname):
713 addrs_list[ifname].append(item)
714 else:
715 addrs_list[ifname] = [item]
716
717 # Alias only needs IP assignment for now, this might change if we
718 # are going to use virtual accesspoints
719 if "alias" in iface_key:
720 continue
721
722 # XXX: Might want to deduct type directly from interface name
723 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
724 # Default to station (client) mode
725 ifacedump['autogen_wlanmode'] = "sta"
726 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
727 ifacedump['autogen_wlanmode'] = "ap"
728
729 if not ifacedump.has_key('channel'):
730 if ifacedump['type'] == '11a':
731 ifacedump['channel'] = 36
732 else:
733 ifacedump['channel'] = 1
734
735 # Allow special hacks at the back like wds and stuff
736 if not ifacedump.has_key('extra'):
737 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
738 else:
739 ifacedump['autogen_extra'] = ifacedump['extra']
740
741 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
742
743 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
744 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
745 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
746 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
747 output += "\n"
748
749 elif ifacedump['type'] in ['ethernet', 'eth']:
750 # No special config needed besides IP
751 pass
752 elif ifacedump['type'] in ['vlan']:
753 # VLAN member has no special configuration
754 pass
755 else:
756 assert False, "Unknown type " + ifacedump['type']
757
758 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
759 interface_list_cache[datadump['autogen_item']] = store
760 return(store)
761
762
763
764def create_proxies_list():
765 if not ileiden_proxies or not normal_proxies:
766 # Placeholder for to-be-installed proxies, this will avoid updating the all
767 # nodes to include this new machine, yet due to an unbound issue, this list
768 # has to be kept small.
769
770 for i in range(1,20):
771 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
772
773 for host in get_hostlist():
774 hostdump = get_yaml(host)
775 if hostdump['status'] == 'up':
776 if hostdump['service_proxy_ileiden']:
777 ileiden_proxies[hostdump['serviceid']] = hostdump
778 if hostdump['service_proxy_normal']:
779 normal_proxies.append(hostdump)
780
781
782
783def generate_rc_conf_local(datadump):
784 """ Generate configuration file '/etc/rc.conf.local' """
785 item = datadump['autogen_item']
786 if rc_conf_local_cache.has_key(item):
787 return rc_conf_local_cache[item]
788
789 if not datadump.has_key('ileiden'):
790 datadump['autogen_ileiden_enable'] = False
791 else:
792 datadump['autogen_ileiden_enable'] = datadump['ileiden']
793
794 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
795
796 create_proxies_list()
797 datadump['autogen_ileiden_proxies'] = ileiden_proxies
798 datadump['autogen_normal_proxies'] = normal_proxies
799 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
800 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
801 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
802 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
803
804 output = generate_header(datadump, "#");
805 output += render_template(datadump, """\
806hostname='{{ autogen_fqdn }}'
807location='{{ location }}'
808nodetype="{{ nodetype }}"
809
810#
811# Configured listings
812#
813captive_portal_whitelist=""
814{% if nodetype == "Proxy" %}
815#
816# Proxy Configuration
817#
818{% if gateway and service_proxy_ileiden -%}
819defaultrouter="{{ gateway }}"
820{% else -%}
821#defaultrouter="NOTSET"
822{% endif -%}
823internalif="{{ internalif }}"
824ileiden_enable="{{ autogen_ileiden_enable }}"
825gateway_enable="{{ autogen_ileiden_enable }}"
826pf_enable="yes"
827pf_rules="/etc/pf.conf"
828{% if autogen_ileiden_enable -%}
829pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
830lvrouted_enable="{{ autogen_ileiden_enable }}"
831lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
832{% else -%}
833pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
834{% endif -%}
835{% if internalroute -%}
836static_routes="wleiden"
837route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
838{% endif -%}
839
840{% elif nodetype == "Hybrid" %}
841 #
842 # Hybrid Configuration
843 #
844 list_ileiden_proxies="
845 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
846 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
847 {% endfor -%}
848 "
849 list_normal_proxies="
850 {% for item in autogen_normal_proxies -%}
851 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
852 {% endfor -%}
853 "
854
855 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
856 externalif="{{ externalif|default('vr0', true) }}"
857 masterip="{{ masterip }}"
858
859 {% if gateway and service_proxy_ileiden %}
860 defaultrouter="{{ gateway }}"
861 {% else %}
862 #defaultrouter="NOTSET"
863 {% endif %}
864
865 #
866 # Defined services
867 #
868 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
869 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
870 service_accesspoint="{{ service_accesspoint|yesorno }}"
871 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
872 service_concentrator="{{ service_concentrator|yesorno }}"
873
874 {% if service_proxy_ileiden %}
875 pf_rules="/etc/pf.hybrid.conf"
876 {% if service_concentrator %}
877 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
878 {% else %}
879 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
880 {% endif %}
881 pf_flags="$pf_flags -D publicnat=80,443"
882 lvrouted_flags="$lvrouted_flags -g"
883 {% elif service_proxy_normal or service_incoming_rdr %}
884 pf_rules="/etc/pf.hybrid.conf"
885 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
886 pf_flags="$pf_flags -D publicnat=0"
887 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
888 named_setfib="1"
889 tinyproxy_setfib="1"
890 dnsmasq_setfib="1"
891 sshd_setfib="1"
892 {% else %}
893 named_auto_forward_only="YES"
894 pf_rules="/etc/pf.node.conf"
895 pf_flags=""
896 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
897 {% endif %}
898 {% if service_concentrator %}
899 # Do mind installing certificates is NOT done automatically for security reasons
900 openvpn_enable="YES"
901 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
902 {% endif %}
903
904 {% if service_proxy_normal %}
905 tinyproxy_enable="yes"
906 {% else %}
907 pen_wrapper_enable="yes"
908 {% endif %}
909
910 {% if service_accesspoint %}
911 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
912 {% endif %}
913
914 {% if board == "ALIX2" %}
915 #
916 # ''Fat'' configuration, board has 256MB RAM
917 #
918 dnsmasq_enable="NO"
919 named_enable="YES"
920 unbound_enable="YES"
921 {% if autogen_dhcp_interfaces -%}
922 dhcpd_enable="YES"
923 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
924 {% endif -%}
925 {% elif board == "apu1d" %}
926 #
927 # ''Fat'' configuration, board has 1024MB RAM
928 #
929 dnsmasq_enable="NO"
930 local_unbound_enable="YES"
931 {% if autogen_dhcp_interfaces -%}
932 dhcpd_enable="YES"
933 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
934 {% endif -%}
935 {% endif -%}
936{% endif %}
937
938#
939# Script variables
940#
941attached_devices="{{ autogen_attached_devices|join(' ') }}"
942neighbours="{{ autogen_neighbours|join(' ') }}"
943
944
945#
946# Interface definitions
947#\n
948""")
949
950 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
951 for iface, vlans in sorted(vlan_list.items()):
952 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
953
954 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
955 for iface in sorted(vlan_list.keys()):
956 if not iface in addrs_list.keys():
957 output += "ifconfig_%s='up'\n" % iface
958
959 output += "\n"
960
961 # Bridge configuration:
962 if bridge_list.keys():
963 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
964
965 for iface in bridge_list.keys():
966 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
967
968 # Bridge member interfaces not containing a configuration should be marked active explcitly.
969 for _,members in bridge_list.items():
970 for iface in members:
971 if not iface in addrs_list.keys():
972 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
973
974 output += "\n"
975
976 # Details like SSID
977 if extra_ouput:
978 output += extra_ouput.strip() + "\n"
979
980 # Print IP address which needs to be assigned over here
981 output += "\n"
982 for iface,addrs in sorted(addrs_list.iteritems()):
983 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
984 output += "# %s || %s || %s\n" % (iface, addr, comment)
985
986 # Write DHCLIENT entry
987 if iface in dhclient_if and dhclient_if[iface]:
988 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
989 continue
990
991 # Make sure the external address is always first as this is needed in the
992 # firewall setup
993 addrs = sorted(
994 [x for x in addrs if not '0.0.0.0' in x[0]],
995 key=lambda x: x[0].split('.')[0],
996 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
997 )
998
999 idx_offset = 0
1000 # Set MAC is required
1001 if flags_if[iface].has_key('ether'):
1002 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1003 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1004 idx_offset += 1
1005 else:
1006 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1007
1008 for idx, addr in enumerate(addrs[1:]):
1009 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1010
1011 output += "\n"
1012
1013 rc_conf_local_cache[datadump['autogen_item']] = output
1014 return output
1015
1016
1017
1018def get_all_configs():
1019 """ Get dict with key 'host' with all configs present """
1020 configs = dict()
1021 for host in get_hostlist():
1022 datadump = get_yaml(host)
1023 configs[host] = datadump
1024 return configs
1025
1026
1027def get_interface_keys(config, extra=False):
1028 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1029 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1030 if extra == False:
1031 return filter(lambda x: not "extra" in x, elems)
1032 else:
1033 return elems
1034
1035
1036def get_used_ips(configs):
1037 """ Return array of all IPs used in config files"""
1038 ip_list = []
1039 for config in configs:
1040 ip_list.append(config['masterip'])
1041 if 'serviceid' in config:
1042 ip_list.append(config['serviceid'])
1043 for iface_key in get_interface_keys(config, True):
1044 l = config[iface_key]['ip']
1045 addr, mask = l.split('/')
1046 # Special case do not process
1047 if valid_addr(addr):
1048 ip_list.append(addr)
1049 else:
1050 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1051 return sorted(ip_list)
1052
1053
1054
1055def get_nameservers(max_servers=None):
1056 if nameservers_cache:
1057 return nameservers_cache[0:max_servers]
1058
1059 for host in get_hostlist():
1060 hostdump = get_yaml(host)
1061 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1062 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
1063
1064 return nameservers_cache[0:max_servers]
1065
1066
1067def get_neighbours(datadump):
1068 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1069
1070 (poel, errors) = make_relations()
1071 table = []
1072 for iface,addrs in sorted(addrs_list.iteritems()):
1073 if iface in ['lo0']:
1074 continue
1075
1076 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1077 if not addr.startswith('172.'):
1078 # Avoid listing internet connections as pool
1079 continue
1080 for neighbour in poel[network(addr)]:
1081 if neighbour[0] != datadump['autogen_item']:
1082 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1083 return table
1084
1085
1086def get_attached_devices(datadump, url=False):
1087 table = []
1088 for iface_key in get_interface_keys(datadump, True):
1089 ifacedump = datadump[iface_key]
1090
1091 if not ifacedump.has_key('ns_ip'):
1092 continue
1093
1094 x_ip = ifacedump['ns_ip'].split('/')[0]
1095
1096 if 'mode' in ifacedump:
1097 x_mode = ifacedump['mode']
1098 else:
1099 x_mode = 'unknown'
1100
1101 if 'bridge_type' in ifacedump:
1102 device_type = ifacedump['bridge_type']
1103 else:
1104 device_type = 'Unknown'
1105
1106 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1107 return table
1108
1109
1110def generate_resolv_conf(datadump):
1111 """ Generate configuration file '/etc/resolv.conf' """
1112 # XXX: This should properly going to be an datastructure soon
1113 datadump['autogen_header'] = generate_header(datadump, "#")
1114 datadump['autogen_edge_nameservers'] = ''
1115
1116
1117 for masterip,realname in get_nameservers():
1118 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1119
1120 return Template("""\
1121{{ autogen_header }}
1122search wleiden.net
1123
1124# Try local (cache) first
1125nameserver 127.0.0.1
1126
1127{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1128nameserver 8.8.8.8 # Google Public NameServer
1129nameserver 64.6.64.6 # Verisign Public NameServer
1130{% else -%}
1131# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1132{{ autogen_edge_nameservers }}
1133{% endif -%}
1134""").render(datadump)
1135
1136
1137
1138def generate_ntp_conf(datadump):
1139 """ Generate configuration file '/etc/ntp.conf' """
1140 # XXX: This should properly going to be an datastructure soon
1141
1142 datadump['autogen_header'] = generate_header(datadump, "#")
1143 datadump['autogen_ntp_servers'] = ''
1144 for host in get_hostlist():
1145 hostdump = get_yaml(host)
1146 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1147 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1148
1149 return Template("""\
1150{{ autogen_header }}
1151
1152{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1153# Machine hooked to internet.
1154server 0.nl.pool.ntp.org iburst maxpoll 9
1155server 1.nl.pool.ntp.org iburst maxpoll 9
1156server 2.nl.pool.ntp.org iburst maxpoll 9
1157server 3.nl.pool.ntp.org iburst maxpoll 9
1158{% else -%}
1159# Local Wireless Leiden NTP Servers.
1160server 0.pool.ntp.wleiden.net iburst maxpoll 9
1161server 1.pool.ntp.wleiden.net iburst maxpoll 9
1162server 2.pool.ntp.wleiden.net iburst maxpoll 9
1163server 3.pool.ntp.wleiden.net iburst maxpoll 9
1164
1165# All the configured NTP servers
1166{{ autogen_ntp_servers }}
1167{% endif %}
1168
1169# If a server loses sync with all upstream servers, NTP clients
1170# no longer follow that server. The local clock can be configured
1171# to provide a time source when this happens, but it should usually
1172# be configured on just one server on a network. For more details see
1173# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1174# The use of Orphan Mode may be preferable.
1175#
1176server 127.127.1.0
1177fudge 127.127.1.0 stratum 10
1178""").render(datadump)
1179
1180
1181def generate_pf_hybrid_conf_local(datadump):
1182 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1183 datadump['autogen_header'] = generate_header(datadump, "#")
1184 if datadump['service_incoming_rdr']:
1185 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1186 return Template("""\
1187{{ autogen_header }}
1188
1189# Redirect some internal facing services outside (7)
1190# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1191{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1192rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1193{% endfor -%}
1194# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1195{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1196rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1197{% endfor -%}
1198""").render(datadump)
1199
1200def generate_unbound_wleiden_conf(datadump):
1201 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1202 datadump['autogen_header'] = generate_header(datadump, "#")
1203
1204 autogen_ips = []
1205 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1206 for iface,addrs in sorted(addrs_list.iteritems()):
1207 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1208 if addr.startswith('172'):
1209 autogen_ips.append((addr.split('/')[0], comment))
1210 datadump['autogen_ips'] = autogen_ips
1211
1212 create_proxies_list()
1213 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1214 return Template("""\
1215{{ autogen_header }}
1216
1217server:
1218{%- for ip,comment in autogen_ips %}
1219 interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1220{%- endfor %}
1221
1222forward-zone:
1223 name: '.'
1224{%- if service_proxy_ileiden %}
1225 forward-addr: 8.8.8.8 # Google DNS A
1226 forward-addr: 8.8.4.4 # Google DNS B
1227 forward-addr: 208.67.222.222 # OpenDNS DNS A
1228 forward-addr: 208.67.220.220 # OpenDNS DNS B
1229{% else -%}
1230{% for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1231 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1232{%- endfor %}
1233{% endif -%}
1234""").render(datadump)
1235
1236def generate_motd(datadump):
1237 """ Generate configuration file '/etc/motd' """
1238 output = Template("""\
1239FreeBSD run ``service motd onestart'' to make me look normal
1240
1241 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1242 Loc: {{ location }}
1243
1244Services:
1245{% if board == "ALIX2" -%}
1246{{" -"}} Core Node ({{ board }})
1247{% else -%}
1248{{" -"}} Hulp Node ({{ board }})
1249{% endif -%}
1250{% if service_proxy_normal -%}
1251{{" -"}} Normal Proxy
1252{% endif -%}
1253{% if service_proxy_ileiden -%}
1254{{" -"}} iLeiden Proxy
1255{% endif -%}
1256{% if service_incoming_rdr -%}
1257{{" -"}} Incoming port redirects
1258{% endif %}
1259Interlinks:\n
1260""").render(datadump)
1261
1262
1263 def make_table(table):
1264 if not table:
1265 return " - none\n"
1266 else:
1267 lines = ""
1268 col_width = [max(len(x) for x in col) for col in zip(*table)]
1269 for row in table:
1270 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1271 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1272 return lines
1273
1274 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1275 table = []
1276 for iface,addrs in sorted(addrs_list.iteritems()):
1277 if iface in ['lo0']:
1278 continue
1279 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1280 table.append((iface, addr, comment))
1281
1282 output += make_table(table)
1283 output += '\n'
1284 output += """\
1285Attached devices:
1286"""
1287 output += make_table(get_attached_devices(datadump, url=True))
1288 output += '\n'
1289 output += """\
1290Available neighbours:
1291"""
1292 output += make_table(get_neighbours(datadump))
1293
1294 return output
1295
1296
1297def format_yaml_value(value):
1298 """ Get yaml value in right syntax for outputting """
1299 if isinstance(value,str):
1300 output = '"%s"' % value
1301 else:
1302 output = value
1303 return output
1304
1305
1306
1307def format_wleiden_yaml(datadump):
1308 """ Special formatting to ensure it is editable"""
1309 output = "# Genesis config yaml style\n"
1310 output += "# vim:ts=2:et:sw=2:ai\n"
1311 output += "#\n"
1312 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1313 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1314 if key == 'rdr_rules':
1315 output += '%-10s:\n' % 'rdr_rules'
1316 for rdr_rule in datadump[key]:
1317 output += '- %s\n' % rdr_rule
1318 else:
1319 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1320
1321 output += "\n\n"
1322
1323 # Format (key, required)
1324 key_order = (
1325 ('comment', True),
1326 ('parent', False),
1327 ('ip', False),
1328 ('ether', False),
1329 ('desc', True),
1330 ('sdesc', True),
1331 ('mode', True),
1332 ('type', True),
1333 ('extra_type', False),
1334 ('channel', False),
1335 ('ssid', False),
1336 ('wlan_mac', False),
1337 ('dhcp', True),
1338 ('dhcp_fixed', False),
1339 ('compass', False),
1340 ('distance', False),
1341 ('ns_ip', False),
1342 ('repeater_ip', False),
1343 ('bullet2_ip', False),
1344 ('ns_mac', False),
1345 ('bullet2_mac', False),
1346 ('ns_type', False),
1347 ('bridge_type', False),
1348 ('status', True),
1349 )
1350
1351 for iface_key in sorted(iface_keys):
1352 try:
1353 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1354 if remainder:
1355 raise KeyError("invalid keys: %s" % remainder)
1356
1357 output += "%s:\n" % iface_key
1358 for key,required in key_order:
1359 if datadump[iface_key].has_key(key):
1360 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1361 output += "\n\n"
1362 except Exception:
1363 print "# Error while processing interface %s" % iface_key
1364 raise
1365
1366 return output
1367
1368
1369
1370def generate_wleiden_yaml(datadump, header=True):
1371 """ Generate (petty) version of wleiden.yaml"""
1372 output = generate_header(datadump, "#") if header else ''
1373
1374 for key in datadump.keys():
1375 if key.startswith('autogen_'):
1376 del datadump[key]
1377 # Interface autogen cleanups
1378 elif type(datadump[key]) == dict:
1379 for key2 in datadump[key].keys():
1380 if key2.startswith('autogen_'):
1381 del datadump[key][key2]
1382
1383 output += format_wleiden_yaml(datadump)
1384 return output
1385
1386def generate_nanostation_config(datadump, iface, ns_type):
1387 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1388 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1389
1390 datadump.update(datadump['iface_%s' % iface])
1391
1392 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1393
1394def generate_yaml(datadump):
1395 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1396
1397
1398
1399def generate_config(node, config, datadump=None):
1400 """ Print configuration file 'config' of 'node' """
1401 output = ""
1402 try:
1403 # Load config file
1404 if datadump == None:
1405 datadump = get_yaml(node)
1406
1407 if config == 'wleiden.yaml':
1408 output += generate_wleiden_yaml(datadump)
1409 elif config == 'authorized_keys':
1410 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1411 output += f.read()
1412 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1413 # Fetch local keys if existing
1414 if os.path.exists(node_keys):
1415 output += open(node_keys, 'r').read()
1416 f.close()
1417 elif config == 'dnsmasq.conf':
1418 output += generate_dnsmasq_conf(datadump)
1419 elif config == 'dhcpd.conf':
1420 output += generate_dhcpd_conf(datadump)
1421 elif config == 'rc.conf.local':
1422 output += generate_rc_conf_local(datadump)
1423 elif config == 'resolv.conf':
1424 output += generate_resolv_conf(datadump)
1425 elif config == 'ntp.conf':
1426 output += generate_ntp_conf(datadump)
1427 elif config == 'motd':
1428 output += generate_motd(datadump)
1429 elif config == 'pf.hybrid.conf.local':
1430 output += generate_pf_hybrid_conf_local(datadump)
1431 elif config == 'unbound.wleiden.conf':
1432 output += generate_unbound_wleiden_conf(datadump)
1433 elif config.startswith('vr'):
1434 interface, ns_type = config.strip('.yaml').split('-')
1435 output += generate_nanostation_config(datadump, interface, ns_type)
1436 else:
1437 assert False, "Config not found!"
1438 except IOError, e:
1439 output += "[ERROR] Config file not found"
1440 return output
1441
1442
1443
1444def process_cgi_request(environ=os.environ):
1445 """ When calling from CGI """
1446 response_headers = []
1447 content_type = 'text/plain'
1448
1449 # Update repository if requested
1450 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1451 if form and form.has_key("action") and "update" in form["action"]:
1452 output = "[INFO] Updating subverion, please wait...\n"
1453 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1454 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1455 output += "[INFO] All done, redirecting in 5 seconds"
1456 response_headers += [
1457 ('Refresh', '5; url=.'),
1458 ]
1459 reload_cache()
1460 else:
1461 base_uri = environ['PATH_INFO']
1462 uri = base_uri.strip('/').split('/')
1463
1464 output = "Template Holder"
1465 if base_uri.endswith('/create/network.kml'):
1466 content_type='application/vnd.google-earth.kml+xml'
1467 output = make_network_kml.make_graph()
1468 elif base_uri.endswith('/api/get/nodeplanner.json'):
1469 content_type='application/json'
1470 output = make_network_kml.make_nodeplanner_json()
1471 elif not uri[0]:
1472 if is_text_request(environ):
1473 output = '\n'.join(get_hostlist())
1474 else:
1475 content_type = 'text/html'
1476 output = generate_title(get_hostlist())
1477 elif len(uri) == 1:
1478 if is_text_request(environ):
1479 output = generate_node(uri[0])
1480 else:
1481 content_type = 'text/html'
1482 output = generate_node_overview(uri[0])
1483 elif len(uri) == 2:
1484 output = generate_config(uri[0], uri[1])
1485 else:
1486 assert False, "Invalid option"
1487
1488 # Return response
1489 response_headers += [
1490 ('Content-type', content_type),
1491 ('Content-Length', str(len(output))),
1492 ]
1493 return(response_headers, str(output))
1494
1495
1496def make_dns(output_dir = 'dns', external = False):
1497 items = dict()
1498
1499 # hostname is key, IP is value
1500 wleiden_zone = defaultdict(list)
1501 wleiden_cname = dict()
1502
1503 pool = dict()
1504 for node in get_hostlist():
1505 datadump = get_yaml(node)
1506
1507 fqdn = datadump['nodename']
1508
1509 if datadump.has_key('rdr_host'):
1510 remote_target = datadump['rdr_host']
1511 elif datadump.has_key('remote_access') and datadump['remote_access']:
1512 remote_target = datadump['remote_access'].split(':')[0]
1513 else:
1514 remote_target = None
1515
1516 if remote_target:
1517 try:
1518 parseaddr(remote_target)
1519 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1520 except (IndexError, ValueError):
1521 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1522
1523
1524 wleiden_zone[fqdn].append((datadump['masterip'], True))
1525
1526 # Hacking to get proper DHCP IPs and hostnames
1527 for iface_key in get_interface_keys(datadump):
1528 iface_name = iface_key.replace('_','-')
1529 if 'ip' in datadump[iface_key]:
1530 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1531 try:
1532 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1533 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1534 dhcp_part = ".".join(ip.split('.')[0:3])
1535 if ip != datadump['masterip']:
1536 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1537 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1538 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1539 except (AttributeError, ValueError, KeyError):
1540 # First push it into a pool, to indentify the counter-part later on
1541 addr = parseaddr(ip)
1542 cidr = int(cidr)
1543 addr = addr & ~((1 << (32 - cidr)) - 1)
1544 if pool.has_key(addr):
1545 pool[addr] += [(iface_name, fqdn, ip)]
1546 else:
1547 pool[addr] = [(iface_name, fqdn, ip)]
1548 continue
1549
1550
1551
1552 # WL uses an /29 to configure an interface. IP's are ordered like this:
1553 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1554
1555 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1556
1557 # Automatic naming convention of interlinks namely 2 + remote.lower()
1558 for (key,value) in pool.iteritems():
1559 # Make sure they are sorted from low-ip to high-ip
1560 value = sorted(value, key=lambda x: parseaddr(x[2]))
1561
1562 if len(value) == 1:
1563 (iface_name, fqdn, ip) = value[0]
1564 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1565
1566 # Device DNS names
1567 if 'cnode' in fqdn.lower():
1568 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1569 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1570
1571 elif len(value) == 2:
1572 (a_iface_name, a_fqdn, a_ip) = value[0]
1573 (b_iface_name, b_fqdn, b_ip) = value[1]
1574 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1575 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1576
1577 # Device DNS names
1578 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1579 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1580 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1581 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1582 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1583 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1584 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1585
1586 else:
1587 pool_members = [k[1] for k in value]
1588 for item in value:
1589 (iface_name, fqdn, ip) = item
1590 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1591
1592 # Include static DNS entries
1593 # XXX: Should they override the autogenerated results?
1594 # XXX: Convert input to yaml more useable.
1595 # Format:
1596 ##; this is a comment
1597 ## roomburgh=Roomburgh1
1598 ## apkerk1.Vosko=172.17.176.8 ;this as well
1599 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1600
1601 # Hack to allow special entries, for development
1602 wleiden_raw = {}
1603
1604 for line in dns_list:
1605 reverse = False
1606 k, items = line.items()[0]
1607 if type(items) == dict:
1608 if items.has_key('reverse'):
1609 reverse = items['reverse']
1610 items = items['a']
1611 else:
1612 items = items['cname']
1613 items = [items] if type(items) != list else items
1614 for item in items:
1615 if item.startswith('IN '):
1616 wleiden_raw[k] = item
1617 elif valid_addr(item):
1618 wleiden_zone[k].append((item, reverse))
1619 else:
1620 wleiden_cname[k] = item
1621
1622 # Hack to get dynamic pool listing
1623 def chunks(l, n):
1624 return [l[i:i+n] for i in range(0, len(l), n)]
1625
1626 ntp_servers = [x[0] for x in get_nameservers()]
1627 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1628 for ntp_server in chunk:
1629 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1630
1631 details = dict()
1632 # 24 updates a day allowed
1633 details['serial'] = time.strftime('%Y%m%d%H')
1634
1635 if external:
1636 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1637 else:
1638 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1639
1640 details['master'] = dns_masters[0]
1641 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1642
1643 dns_header = '''
1644$TTL 3h
1645%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1646 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1647
1648%(ns_servers)s
1649 \n'''
1650
1651
1652 if not os.path.isdir(output_dir):
1653 os.makedirs(output_dir)
1654 details['zone'] = 'wleiden.net'
1655 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1656 f.write(dns_header % details)
1657
1658 for host,items in wleiden_zone.iteritems():
1659 for ip,reverse in items:
1660 if ip not in ['0.0.0.0']:
1661 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1662 for source,dest in wleiden_cname.iteritems():
1663 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1664 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1665 for source, dest in wleiden_raw.iteritems():
1666 f.write("%s.wleiden.net. %s\n" % (source, dest))
1667 f.close()
1668
1669 # Create whole bunch of specific sub arpa zones. To keep it compliant
1670 for s in range(16,32):
1671 details['zone'] = '%i.172.in-addr.arpa' % s
1672 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1673 f.write(dns_header % details)
1674
1675 #XXX: Not effient, fix to proper data structure and do checks at other
1676 # stages
1677 for host,items in wleiden_zone.iteritems():
1678 for ip,reverse in items:
1679 if not reverse:
1680 continue
1681 if valid_addr(ip):
1682 if valid_addr(ip):
1683 if int(ip.split('.')[1]) == s:
1684 rev_ip = '.'.join(reversed(ip.split('.')))
1685 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1686 f.close()
1687
1688
1689def usage():
1690 print """Usage: %(prog)s <argument>
1691Argument:
1692\tcleanup = Cleanup all YAML files to specified format
1693\tstandalone [port] = Run configurator webserver [8000]
1694\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1695\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1696\tfull-export = Generate yaml export script for heatmap.
1697\tstatic [outputdir] = Generate all config files and store on disk
1698\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1699\ttest <node> [<file>] = Receive output for certain node [all files].
1700\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1701\tlist <status> <items> = List systems which have certain status
1702\tcreate network.kml = Create Network KML file for use in Google Earth
1703
1704Arguments:
1705\t<node> = NodeName (example: HybridRick)
1706\t<file> = %(files)s
1707\t<status> = all|up|down|planned
1708\t<items> = systems|nodes|proxies
1709
1710NOTE FOR DEVELOPERS; you can test your changes like this:
1711 BEFORE any changes in this code:
1712 $ ./gformat.py static /tmp/pre
1713 AFTER the changes:
1714 $ ./gformat.py static /tmp/post
1715 VIEW differences and VERIFY all are OK:
1716 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1717""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1718 exit(0)
1719
1720
1721def is_text_request(environ=os.environ):
1722 """ Find out whether we are calling from the CLI or any text based CLI utility """
1723 try:
1724 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1725 except KeyError:
1726 return True
1727
1728def switchFormat(setting):
1729 if setting:
1730 return "YES"
1731 else:
1732 return "NO"
1733
1734def rlinput(prompt, prefill=''):
1735 import readline
1736 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1737 try:
1738 return raw_input(prompt)
1739 finally:
1740 readline.set_startup_hook()
1741
1742def fix_conflict(left, right, default='i'):
1743 while True:
1744 print "## %-30s | %-30s" % (left, right)
1745 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1746 if not c:
1747 c = default
1748
1749 if c in ['l','1']:
1750 return left
1751 elif c in ['r','2']:
1752 return right
1753 elif c in ['e', '3']:
1754 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1755 elif c in ['i', '4']:
1756 return None
1757 else:
1758 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1759
1760
1761
1762def print_cgi_response(response_headers, output):
1763 """Could we not use some kind of wsgi wrapper to make this output?"""
1764 for header in response_headers:
1765 print "%s: %s" % header
1766 print
1767 print output
1768
1769
1770def fill_cache():
1771 ''' Poor man re-loading of few cache items (the slow ones) '''
1772 for host in get_hostlist():
1773 get_yaml(host)
1774
1775
1776def reload_cache():
1777 clear_cache()
1778 fill_cache()
1779
1780
1781def main():
1782 """Hard working sub"""
1783 # Allow easy hacking using the CLI
1784 if not os.environ.has_key('PATH_INFO'):
1785 if len(sys.argv) < 2:
1786 usage()
1787
1788 if sys.argv[1] == "standalone":
1789 import SocketServer
1790 import CGIHTTPServer
1791 # Hop to the right working directory.
1792 os.chdir(os.path.dirname(__file__))
1793 try:
1794 PORT = int(sys.argv[2])
1795 except (IndexError,ValueError):
1796 PORT = 8000
1797
1798 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1799 """ Serve this CGI from the root of the webserver """
1800 def is_cgi(self):
1801 if "favicon" in self.path:
1802 return False
1803
1804 self.cgi_info = (os.path.basename(__file__), self.path)
1805 self.path = ''
1806 return True
1807 handler = MyCGIHTTPRequestHandler
1808 SocketServer.TCPServer.allow_reuse_address = True
1809 httpd = SocketServer.TCPServer(("", PORT), handler)
1810 httpd.server_name = 'localhost'
1811 httpd.server_port = PORT
1812
1813 logger.info("serving at port %s", PORT)
1814 try:
1815 httpd.serve_forever()
1816 except KeyboardInterrupt:
1817 httpd.shutdown()
1818 logger.info("All done goodbye")
1819 elif sys.argv[1] == "test":
1820 # Basic argument validation
1821 try:
1822 node = sys.argv[2]
1823 except IndexError:
1824 print "Invalid argument"
1825 exit(1)
1826 except IOError as e:
1827 print e
1828 exit(1)
1829
1830 datadump = get_yaml(node)
1831
1832
1833 # Get files to generate
1834 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1835
1836 # Actual config generation
1837 for config in gen_files:
1838 logger.info("## Generating %s %s", node, config)
1839 print generate_config(node, config, datadump)
1840 elif sys.argv[1] == "test-cgi":
1841 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1842 os.environ['SCRIPT_NAME'] = __file__
1843 response_headers, output = process_cgi_request()
1844 print_cgi_response(response_headers, output)
1845 elif sys.argv[1] == "static":
1846 items = dict()
1847 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1848 for node in get_hostlist():
1849 items['node'] = node
1850 items['wdir'] = "%(output_dir)s/%(node)s" % items
1851 if not os.path.isdir(items['wdir']):
1852 os.makedirs(items['wdir'])
1853 datadump = get_yaml(node)
1854 for config in files:
1855 items['config'] = config
1856 logger.info("## Generating %(node)s %(config)s" % items)
1857 f = open("%(wdir)s/%(config)s" % items, "w")
1858 f.write(generate_config(node, config, datadump))
1859 f.close()
1860 elif sys.argv[1] == "wind-export":
1861 items = dict()
1862 for node in get_hostlist():
1863 datadump = get_yaml(node)
1864 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1865 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1866 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1867 VALUES (
1868 (SELECT id FROM users WHERE username = 'rvdzwet'),
1869 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1870 'Y');""" % datadump
1871 #for config in files:
1872 # items['config'] = config
1873 # print "## Generating %(node)s %(config)s" % items
1874 # f = open("%(wdir)s/%(config)s" % items, "w")
1875 # f.write(generate_config(node, config, datadump))
1876 # f.close()
1877 for node in get_hostlist():
1878 datadump = get_yaml(node)
1879 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1880 ifacedump = datadump[iface_key]
1881 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1882 ifacedump['nodename'] = datadump['nodename']
1883 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1884 ifacedump['channel'] = 0
1885 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1886 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1887 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1888 elif sys.argv[1] == "nagios-export":
1889 try:
1890 heavy_load = (sys.argv[2] == "--heavy-load")
1891 except IndexError:
1892 heavy_load = False
1893
1894 hostgroup_details = {
1895 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1896 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1897 'walphen' : 'Stichting Wireless Alphen',
1898 'westeinder' : 'Westeinder Plassen',
1899 }
1900
1901 # Convert IP to Host
1902 ip2host = {'root' : 'root'}
1903 for host in get_hostlist():
1904 datadump = get_yaml(host)
1905 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1906 for iface in get_interface_keys(datadump):
1907 if datadump[iface].has_key('autogen_gateway'):
1908 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1909
1910 # Find dependency tree based on output of lvrouted.mytree of nearest node
1911 parents = defaultdict(list)
1912 stack = ['root']
1913 prev_depth = 0
1914 for line in open('lvrouted.mytree').readlines():
1915 depth = line.count('\t')
1916 ip = line.strip().split()[0]
1917
1918 if prev_depth < depth:
1919 try:
1920 parents[ip2host[ip]].append(ip2host[stack[-1]])
1921 except KeyError as e:
1922 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1923 stack.append(ip)
1924 elif prev_depth > depth:
1925 stack = stack[:(depth - prev_depth)]
1926 elif prev_depth == depth:
1927 try:
1928 parents[ip2host[ip]].append(ip2host[stack[-1]])
1929 except KeyError as e:
1930 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1931
1932
1933 prev_depth = depth
1934 # Observe that some nodes has themself as parent or multiple parents
1935 # for now take only the first parent, other behaviour is yet to be explained
1936
1937
1938
1939 params = {
1940 'check_interval' : 5 if heavy_load else 120,
1941 'retry_interval' : 1 if heavy_load else 10,
1942 'max_check_attempts' : 10 if heavy_load else 6,
1943 'notification_interval': 120 if heavy_load else 240,
1944 }
1945
1946 print '''\
1947define host {
1948 name wleiden-node ; Default Node Template
1949 use generic-host ; Use the standard template as initial starting point
1950 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1951 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1952 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1953 notification_interval %(notification_interval)s
1954 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1955 check_command check-host-alive ; Default command to check FreeBSD hosts
1956 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1957}
1958
1959define service {
1960 name wleiden-service ; Default Service Template
1961 use generic-service ; Use the standard template as initial starting point
1962 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1963 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1964 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1965 notification_interval %(notification_interval)s
1966 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1967 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1968}
1969
1970# Please make sure to install:
1971# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1972#
1973# Recompile net-mgmt/nagios-plugins to support check_snmp
1974# make -C /usr/ports/net-mgmt/nagios-plugins
1975#
1976# Install net/bind-tools to allow v2/check_dns_wl to work:
1977# pkg install bind-tools
1978#
1979define command{
1980 command_name check_snmp_disk
1981 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1982}
1983
1984define command{
1985 command_name check_netsnmp_load
1986 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1987}
1988
1989define command{
1990 command_name check_netsnmp_proc
1991 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1992}
1993
1994define command{
1995 command_name check_by_ssh
1996 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1997}
1998
1999define command{
2000 command_name check_dns_wl
2001 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2002}
2003
2004define command{
2005 command_name check_snmp_uptime
2006 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2007}
2008
2009
2010# TDB: dhcp leases
2011# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2012
2013# TDB: internet status
2014# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2015
2016# TDB: Advanced local passive checks
2017# /usr/local/libexec/nagios/check_by_ssh
2018''' % params
2019
2020 print '''\
2021# Service Group, not displayed by default
2022define hostgroup {
2023 hostgroup_name srv_hybrid
2024 alias All Hybrid Nodes
2025 register 0
2026}
2027
2028define service {
2029 use wleiden-service
2030 hostgroup_name srv_hybrid
2031 service_description SSH
2032 check_command check_ssh
2033}
2034
2035define service {
2036 use wleiden-service,service-pnp
2037 hostgroup_name srv_hybrid
2038 service_description HTTP
2039 check_command check_http
2040}
2041
2042define service {
2043 use wleiden-service
2044 hostgroup_name srv_hybrid
2045 service_description DNS
2046 check_command check_dns_wl!"www.wirelessleiden.nl"
2047}
2048'''
2049
2050 if heavy_load:
2051 print '''\
2052define service {
2053 use wleiden-service
2054 hostgroup_name srv_hybrid
2055 service_description UPTIME
2056 check_command check_snmp_uptime
2057}
2058
2059#define service {
2060# use wleiden-service
2061# hostgroup_name srv_hybrid
2062# service_description NTP
2063# check_command check_ntp_peer
2064#}
2065
2066define service {
2067 use wleiden-service
2068 hostgroup_name srv_hybrid
2069 service_description LOAD
2070 check_command check_netsnmp_load
2071}
2072
2073define service {
2074 use wleiden-service
2075 hostgroup_name srv_hybrid
2076 service_description PROC
2077 check_command check_netsnmp_proc
2078}
2079
2080define service {
2081 use wleiden-service
2082 hostgroup_name srv_hybrid
2083 service_description DISK
2084 check_command check_snmp_disk
2085}
2086'''
2087 for node in get_hostlist():
2088 datadump = get_yaml(node)
2089 if not datadump['status'] == 'up':
2090 continue
2091 if not hostgroup_details.has_key(datadump['monitoring_group']):
2092 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2093 print '''\
2094define host {
2095 use wleiden-node,host-pnp
2096 contact_groups admins
2097 host_name %(autogen_fqdn)s
2098 address %(masterip)s
2099 hostgroups srv_hybrid,%(monitoring_group)s\
2100''' % datadump
2101 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2102 print '''\
2103 parents %(parents)s\
2104''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2105 print '''\
2106}
2107'''
2108
2109
2110 for name,alias in hostgroup_details.iteritems():
2111 print '''\
2112define hostgroup {
2113 hostgroup_name %s
2114 alias %s
2115} ''' % (name, alias)
2116
2117
2118 elif sys.argv[1] == "full-export":
2119 hosts = {}
2120 for node in get_hostlist():
2121 datadump = get_yaml(node)
2122 hosts[datadump['nodename']] = datadump
2123 print yaml.dump(hosts)
2124
2125 elif sys.argv[1] == "dns":
2126 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2127 elif sys.argv[1] == "cleanup":
2128 # First generate all datadumps
2129 datadumps = dict()
2130 ssid_to_node = dict()
2131 for host in get_hostlist():
2132 logger.info("# Processing: %s", host)
2133 # Set some boring default values
2134 datadump = { 'board' : 'UNKNOWN' }
2135 datadump.update(get_yaml(host))
2136 datadumps[datadump['nodename']] = datadump
2137
2138 (poel, errors) = make_relations()
2139 print "\n".join(["# WARNING: %s" % x for x in errors])
2140
2141 for host,datadump in datadumps.iteritems():
2142 try:
2143 # Convert all yes and no to boolean values
2144 def fix_boolean(dump):
2145 for key in dump.keys():
2146 if type(dump[key]) == dict:
2147 dump[key] = fix_boolean(dump[key])
2148 elif str(dump[key]).lower() in ["yes", "true"]:
2149 dump[key] = True
2150 elif str(dump[key]).lower() in ["no", "false"]:
2151 # Compass richting no (Noord Oost) is valid input
2152 if key != "compass": dump[key] = False
2153 return dump
2154 datadump = fix_boolean(datadump)
2155
2156 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2157 if not 'latitude' in datadump and not 'longitude' in datadump:
2158 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2159 elif 'latitude' in datadump and 'longitude' in datadump:
2160 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2161 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2162 # TODO: Compare outcome of both coordinate systems and validate against each-other
2163
2164 if datadump['nodename'].startswith('Proxy'):
2165 datadump['nodename'] = datadump['nodename'].lower()
2166
2167 for iface_key in get_interface_keys(datadump):
2168 try:
2169 # All our normal wireless cards are normal APs now
2170 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2171 datadump[iface_key]['mode'] = 'ap'
2172 # Wireless Leiden SSID have an consistent lowercase/uppercase
2173 if datadump[iface_key].has_key('ssid'):
2174 ssid = datadump[iface_key]['ssid']
2175 prefix = 'ap-WirelessLeiden-'
2176 if ssid.lower().startswith(prefix.lower()):
2177 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2178 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2179 datadump[iface_key]['mode'] = 'autogen-FIXME'
2180 if not datadump[iface_key].has_key('comment'):
2181 datadump[iface_key]['comment'] = 'autogen-FIXME'
2182
2183 if datadump[iface_key].has_key('ns_mac'):
2184 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2185
2186 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2187 datadump[iface_key] = datadump[iface_key]['desc']
2188
2189 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2190 # due to better collision avoidance.
2191 if datadump[iface_key]['type'] == '11b':
2192 datadump[iface_key]['type'] = '11g'
2193
2194 # Setting 802.11g channels to de-facto standards, to avoid
2195 # un-detected sharing with other overlapping channels
2196 #
2197 # Technically we could also use channel 13 in NL, but this is not
2198 # recommended as foreign devices might not be able to select this
2199 # channel. Secondly using 1,5,9,13 instead is going to clash with
2200 # the de-facto usage of 1,6,11.
2201 #
2202 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2203 channels_at_2400Mhz = (1,6,11)
2204 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2205 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2206 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2207 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2208
2209 # Mandatory interface keys
2210 if not datadump[iface_key].has_key('status'):
2211 datadump[iface_key]['status'] = 'planned'
2212
2213 x = datadump[iface_key]['comment']
2214 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2215
2216 # Fixing bridge_type if none is found
2217 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2218 if not 'bridge_type' in datadump[iface_key]:
2219 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2220
2221 # Making sure description works
2222 if datadump[iface_key].has_key('desc'):
2223 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2224 del datadump[iface_key]['desc']
2225 else:
2226 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2227 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2228 if response:
2229 datadump[iface_key]['comment'] = response
2230 del datadump[iface_key]['desc']
2231
2232 # Check DHCP configuration
2233 dhcp_type(datadump[iface_key])
2234
2235 # Set the compass value based on the angle between the poels
2236 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2237 my_pool = poel[network(datadump[iface_key]['ip'])]
2238 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2239 if remote_hosts:
2240 compass_target = remote_hosts[0]
2241 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2242 # TODO: Compass wanted and actual direction might differ
2243
2244 # Monitoring Group default
2245 if not 'monitoring_group' in datadump:
2246 datadump['monitoring_group'] = 'wleiden'
2247
2248 except Exception:
2249 print "# Error while processing interface %s" % iface_key
2250 raise
2251 store_yaml(datadump)
2252 except Exception:
2253 print "# Error while processing %s" % host
2254 raise
2255 elif sys.argv[1] == "list":
2256 use_fqdn = False
2257 if len(sys.argv) < 4:
2258 usage()
2259 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2260 usage()
2261 if not sys.argv[3] in ["nodes","proxies","systems"]:
2262 usage()
2263
2264 if len(sys.argv) > 4:
2265 if sys.argv[4] == "fqdn":
2266 use_fqdn = True
2267 else:
2268 usage()
2269
2270 for system in get_hostlist():
2271 datadump = get_yaml(system)
2272 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2273 continue
2274
2275 output = datadump['autogen_fqdn'] if use_fqdn else system
2276 if sys.argv[2] == "all":
2277 print output
2278 elif datadump['status'] == sys.argv[2]:
2279 print output
2280 elif sys.argv[1] == "create":
2281 if sys.argv[2] == "network.kml":
2282 print make_network_kml.make_graph()
2283 elif sys.argv[2] == "host-ips.txt":
2284 for system in get_hostlist():
2285 datadump = get_yaml(system)
2286 ips = [datadump['masterip']]
2287 for ifkey in get_interface_keys(datadump):
2288 ips.append(datadump[ifkey]['ip'].split('/')[0])
2289 print system, ' '.join(ips)
2290 elif sys.argv[2] == "host-pos.txt":
2291 for system in get_hostlist():
2292 datadump = get_yaml(system)
2293 print system, datadump['rdnap_x'], datadump['rdnap_y']
2294 elif sys.argv[2] == 'ssh_config':
2295 print '''
2296Host *.wleiden.net
2297 User root
2298
2299Host 172.16.*.*
2300 User root
2301'''
2302 for system in get_hostlist():
2303 datadump = get_yaml(system)
2304 print '''\
2305Host %s
2306 User root
2307
2308Host %s
2309 User root
2310
2311Host %s
2312 User root
2313
2314Host %s
2315 User root
2316''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2317 else:
2318 usage()
2319 else:
2320 usage()
2321 else:
2322 # Do not enable debugging for config requests as it highly clutters the output
2323 if not is_text_request():
2324 cgitb.enable()
2325 response_headers, output = process_cgi_request()
2326 print_cgi_response(response_headers, output)
2327
2328def application(environ, start_response):
2329 status = '200 OK'
2330 response_headers, output = process_cgi_request(environ)
2331 start_response(status, response_headers)
2332
2333 # Debugging only
2334 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2335 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2336 return [output]
2337
2338if __name__ == "__main__":
2339 main()
Note: See TracBrowser for help on using the repository browser.