source: genesis/tools/gformat.py@ 13691

Last change on this file since 13691 was 13680, checked in by rick, 8 years ago

Control gateway routing using 'flagged' IP.

The use of specialized serviceid IP addresses for use in determining the
presence of a default gateway will allow us, to remove the IP if the gateway is
not functional, without affecting other functions of the node. The presence of
the unused entries allowes new gateways to be added without the need of
updating all configuration files.

Side-note: This is a workaround since default routing to pasted through
lvrouted using the '-g' parameter at gateways and leaving the '-z' parameter
empty other nodes. How-ever this setup is causing defaultgw 'loops' in the
network, yet need to be tested some more.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 74.1 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict, OrderedDict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13680 2017-01-04 21:41:53Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = OrderedDict()
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126def clear_cache():
127 ''' Poor mans cache implementation '''
128 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
129 datadump_cache = {}
130 interface_list_cache = {}
131 rc_conf_local_cache = {}
132 ileiden_proxies = OrderedDict()
133 normal_proxies = []
134 nameservers_cache = []
135 relations_cache = None
136
137NO_DHCP = 0
138DHCP_CLIENT = 10
139DHCP_SERVER = 20
140def dhcp_type(item):
141 if not item.has_key('dhcp'):
142 return NO_DHCP
143 elif not item['dhcp']:
144 return NO_DHCP
145 elif item['dhcp'].lower() == 'client':
146 return DHCP_CLIENT
147 else:
148 # Validation Checks
149 begin,end = map(int,item['dhcp'].split('-'))
150 if begin >= end:
151 raise ValueError("DHCP Start >= DHCP End")
152 return DHCP_SERVER
153
154def etrs2rd(lat, lon):
155 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
156 p2 = pyproj.Proj(init='EPSG:28992')
157 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
158 return (RDx, RDy)
159
160def rd2etrs(RDx, RDy):
161 p1 = pyproj.Proj(init='EPSG:28992')
162 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
163 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
164 return (lat, lon)
165
166def get_yaml(item,add_version_info=True):
167 try:
168 """ Get configuration yaml for 'item'"""
169 if datadump_cache.has_key(item):
170 return datadump_cache[item].copy()
171
172 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
173
174 # Default values
175 datadump = {
176 'autogen_revision' : 'NOTFOUND',
177 'autogen_gfile' : gfile,
178 'service_proxy_ileiden' : False,
179 }
180 f = open(gfile, 'r')
181 datadump.update(yaml.load(f,Loader=Loader))
182 if datadump['nodetype'] == 'Hybrid':
183 # Some values are defined implicitly
184 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
185 datadump['service_incoming_rdr'] = True
186 # Use some boring defaults
187 defaults = {
188 'service_proxy_normal' : False,
189 'service_accesspoint' : True,
190 'service_incoming_rdr' : False,
191 'service_concentrator' : False,
192 'monitoring_group' : 'wleiden',
193 }
194 for (key,value) in defaults.iteritems():
195 if not datadump.has_key(key):
196 datadump[key] = value
197 f.close()
198
199 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
200 if add_version_info:
201 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
202 lines = p.communicate()[0].split('\n')
203 if p.returncode == 0:
204 for line in lines:
205 if line:
206 (key, value) = line.split(': ')
207 datadump["autogen_" + key.lower().replace(' ','_')] = value
208
209 # Preformat certain needed variables for formatting and push those into special object
210 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
211
212 wlan_count=0
213 try:
214 for key in get_interface_keys(datadump, True):
215 datadump[key]['autogen_ifbase'] = key.split('_')[1]
216 datadump[key]['autogen_vlan'] = False
217
218 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
219 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
220
221 if datadump[key].has_key('ip'):
222 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
223
224 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
225 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
226 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
227 wlan_count += 1
228 else:
229 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
230 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
231 datadump[key]['autogen_vlan'] = key.split('_')[2]
232 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
233 else:
234 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
235
236 except Exception:
237 print "# Error while processing interface %s" % key
238 raise
239
240 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
241 if dhcp_type(datadump[key]) == DHCP_SERVER]
242
243 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
244 datadump['autogen_item'] = item
245
246 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
247 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
248 datadump_cache[item] = datadump.copy()
249 except Exception:
250 print "# Error while processing %s" % item
251 raise
252 return datadump
253
254
255def store_yaml(datadump, header=False):
256 """ Store configuration yaml for 'item'"""
257 item = datadump['autogen_item']
258 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
259
260 output = generate_wleiden_yaml(datadump, header)
261
262 f = open(gfile, 'w')
263 f.write(output)
264 f.close()
265
266
267def network(ip):
268 addr, mask = ip.split('/')
269 # Not parsing of these folks please
270 addr = parseaddr(addr)
271 mask = int(mask)
272 network = addr & ~((1 << (32 - mask)) - 1)
273 return network
274
275
276
277def make_relations():
278 """ Process _ALL_ yaml files to get connection relations """
279 global relations_cache
280
281 if relations_cache:
282 return relations_cache
283
284 errors = []
285 poel = defaultdict(list)
286
287 for host in get_hostlist():
288 datadump = get_yaml(host)
289 try:
290 for iface_key in get_interface_keys(datadump):
291 net_addr = network(datadump[iface_key]['ip'])
292 poel[net_addr] += [(host,datadump[iface_key].copy())]
293 except (KeyError, ValueError), e:
294 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
295 continue
296
297 relations_cache = (poel, errors)
298 return relations_cache
299
300
301
302def valid_addr(addr):
303 """ Show which address is valid in which are not """
304 return str(addr).startswith('172.')
305
306def get_hostlist():
307 """ Combined hosts and proxy list"""
308 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
309
310def angle_between_points(lat1,lat2,long1,long2):
311 """
312 Return Angle in radians between two GPS coordinates
313 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
314 """
315 dy = lat2 - lat1
316 dx = math.cos(lat1)*(long2 - long1)
317 angle = math.atan2(dy,dx)
318 return angle
319
320
321
322def angle_to_cd(angle):
323 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
324
325 # For easy conversion get positive degree
326 degrees = math.degrees(angle)
327 abs_degrees = 360 + degrees if degrees < 0 else degrees
328
329 # Numbers can be confusing calculate from the 4 main directions
330 p = 22.5
331 if abs_degrees < p:
332 cd = "n"
333 elif abs_degrees < (90 - p):
334 cd = "no"
335 elif abs_degrees < (90 + p):
336 cd = "o"
337 elif abs_degrees < (180 - p):
338 cd = "zo"
339 elif abs_degrees < (180 + p):
340 cd = "z"
341 elif abs_degrees < (270 - p):
342 cd = "zw"
343 elif abs_degrees < (270 + p):
344 cd = "w"
345 elif abs_degrees < (360 - p):
346 cd = "nw"
347 else:
348 cd = "n"
349 return cd
350
351
352
353def cd_between_hosts(hostA, hostB, datadumps):
354 # Using RDNAP coordinates
355 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
356 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
357 return angle_to_cd(math.atan2(dx,dy))
358
359 # GPS coordinates seems to fail somehow
360 #latA = float(datadumps[hostA]['latitude'])
361 #latB = float(datadumps[hostB]['latitude'])
362 #lonA = float(datadumps[hostA]['longitude'])
363 #lonB = float(datadumps[hostB]['longitude'])
364 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
365
366
367def generate_title(nodelist):
368 """ Main overview page """
369 items = {'root' : "." }
370 def fl(spaces, line):
371 return (' ' * spaces) + line + '\n'
372
373 output = """
374<html>
375 <head>
376 <title>Wireless leiden Configurator - GFormat</title>
377 <style type="text/css">
378 th {background-color: #999999}
379 tr:nth-child(odd) {background-color: #cccccc}
380 tr:nth-child(even) {background-color: #ffffff}
381 th, td {padding: 0.1em 1em}
382 </style>
383 </head>
384 <body>
385 <center>
386 <form type="GET" action="%(root)s">
387 <input type="hidden" name="action" value="update">
388 <input type="submit" value="Update Configuration Database (SVN)">
389 </form>
390 <table>
391 <caption><h3>Wireless Leiden Configurator</h3></caption>
392 """ % items
393
394 for node in nodelist:
395 items['node'] = node
396 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
397 for config in files:
398 items['config'] = config
399 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
400 output += fl(5, "</tr>")
401 output += """
402 </table>
403 <hr />
404 <em>%s</em>
405 </center>
406 </body>
407</html>
408 """ % __version__
409
410 return output
411
412
413
414def generate_node(node):
415 """ Print overview of all files available for node """
416 return "\n".join(files)
417
418def generate_node_overview(host):
419 """ Print overview of all files available for node """
420 datadump = get_yaml(host)
421 params = { 'host' : host }
422 output = "<em><a href='..'>Back to overview</a></em><hr />"
423 output += "<h2>Available files:</h2><ul>"
424 for cf in files:
425 params['cf'] = cf
426 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
427 output += "</ul>"
428
429 # Generate and connection listing
430 output += "<h2>Connected To:</h2><ul>"
431 (poel, errors) = make_relations()
432 for network, hosts in poel.iteritems():
433 if host in [x[0] for x in hosts]:
434 if len(hosts) == 1:
435 # Single not connected interface
436 continue
437 for remote,ifacedump in hosts:
438 if remote == host:
439 # This side of the interface
440 continue
441 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
442 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
443 output += "</ul>"
444 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
445
446 output += "<hr /><em><a href='..'>Back to overview</a></em>"
447 return output
448
449
450def generate_header(datadump, ctag="#"):
451 return """\
452%(ctag)s
453%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
454%(ctag)s
455""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
456
457
458
459def parseaddr(s):
460 """ Process IPv4 CIDR notation addr to a (binary) number """
461 f = s.split('.')
462 return (long(f[0]) << 24L) + \
463 (long(f[1]) << 16L) + \
464 (long(f[2]) << 8L) + \
465 long(f[3])
466
467
468
469def showaddr(a):
470 """ Display IPv4 addr in (dotted) CIDR notation """
471 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
472
473
474def is_member(ip, mask, canidate):
475 """ Return True if canidate is part of ip/mask block"""
476 ip_addr = parseaddr(ip)
477 ip_canidate = parseaddr(canidate)
478 mask = int(mask)
479 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
480 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
481 return ip_addr == ip_canidate
482
483
484
485def cidr2netmask(netmask):
486 """ Given a 'netmask' return corresponding CIDR """
487 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
488
489def get_network(addr, mask):
490 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
491
492
493def generate_dhcpd_conf(datadump):
494 """ Generate config file '/usr/local/etc/dhcpd.conf """
495 output = generate_header(datadump)
496 output += Template("""\
497# option definitions common to all supported networks...
498option domain-name "dhcp.{{ autogen_fqdn }}";
499
500default-lease-time 600;
501max-lease-time 7200;
502
503# Use this to enble / disable dynamic dns updates globally.
504#ddns-update-style none;
505
506# If this DHCP server is the official DHCP server for the local
507# network, the authoritative directive should be uncommented.
508authoritative;
509
510# Use this to send dhcp log messages to a different log file (you also
511# have to hack syslog.conf to complete the redirection).
512log-facility local7;
513
514#
515# Interface definitions
516#
517\n\n""").render(datadump)
518
519
520 # TODO: Use textwrap.fill instead
521 def indent(text, count):
522 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
523
524 dhcp_out = defaultdict(list)
525 for iface_key in get_interface_keys(datadump):
526 ifname = datadump[iface_key]['autogen_ifbase']
527 if not datadump[iface_key].has_key('comment'):
528 datadump[iface_key]['comment'] = None
529
530 if not datadump[iface_key].has_key('ip'):
531 continue
532
533 dhcp_out[iface_key].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
534
535 (addr, mask) = datadump[iface_key]['ip'].split('/')
536 datadump[iface_key]['autogen_addr'] = addr
537 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
538 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
539
540 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
541 dhcp_out[iface_key].append(textwrap.dedent("""\
542 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
543 ### not autoritive
544 }
545 """ % datadump[iface_key]))
546 continue
547
548 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
549 dhcp_part = ".".join(addr.split('.')[0:3])
550 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
551 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
552
553 # Assume the first 10 IPs could be used for static entries
554 if 'no_portal' in datadump:
555 fixed = 5
556 for mac in datadump['no_portal']:
557 dhcp_out[iface_key].append(textwrap.dedent("""\
558 host fixed-%(ifname)s-%(fixed)s {
559 hardware ethernet %(mac)s;
560 fixed-address %(prefix)s.%(fixed)s;
561 }
562 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
563 fixed += 1
564
565 dhcp_out[iface_key].append(textwrap.dedent("""\
566 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
567 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
568 option routers %(autogen_addr)s;
569 option domain-name-servers %(autogen_addr)s;
570 }
571 """ % datadump[iface_key]))
572
573 for ifname,value in dhcp_out.iteritems():
574 if len(value) > 2:
575 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
576 else:
577 output += ''.join(value) + "\n\n"
578 return output
579
580
581
582def generate_dnsmasq_conf(datadump):
583 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
584 output = generate_header(datadump)
585 output += Template("""\
586# DHCP server options
587dhcp-authoritative
588dhcp-fqdn
589domain=dhcp.{{ autogen_fqdn }}
590domain-needed
591expand-hosts
592log-async=100
593
594# Low memory footprint
595cache-size=10000
596
597\n""").render(datadump)
598
599 for iface_key in get_interface_keys(datadump):
600 if not datadump[iface_key].has_key('comment'):
601 datadump[iface_key]['comment'] = None
602 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
603
604 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
605 output += "# not autoritive\n\n"
606 continue
607
608 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
609 (ip, cidr) = datadump[iface_key]['ip'].split('/')
610 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
611
612 dhcp_part = ".".join(ip.split('.')[0:3])
613 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
614 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
615 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
616
617 return output
618
619
620class AutoVivification(dict):
621 """Implementation of perl's autovivification feature."""
622 def __getitem__(self, item):
623 try:
624 return dict.__getitem__(self, item)
625 except KeyError:
626 value = self[item] = type(self)()
627 return value
628
629def make_interface_list(datadump):
630 if interface_list_cache.has_key(datadump['autogen_item']):
631 return (interface_list_cache[datadump['autogen_item']])
632 # lo0 configuration:
633 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
634 # - masterip is special as it needs to be assigned to at
635 # least one interface, so if not used assign to lo0
636 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
637 vlan_list = defaultdict(list)
638 bridge_list = defaultdict(list)
639 flags_if = AutoVivification()
640 dhclient_if = {'lo0' : False}
641
642 # XXX: Find some way of send this output nicely
643 output = ''
644
645 masterip_used = False
646 for iface_key in get_interface_keys(datadump):
647 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
648 masterip_used = True
649 break
650 if not masterip_used:
651 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
652
653 if 'serviceid' in datadump:
654 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
655
656 for iface_key in get_interface_keys(datadump):
657 ifacedump = datadump[iface_key]
658 ifname = ifacedump['autogen_ifname']
659
660 # If defined as vlan interface
661 if ifacedump['autogen_vlan']:
662 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
663
664 # If defined as bridge interface
665 if ifacedump['autogen_bridge_member']:
666 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
667
668 # Flag dhclient is possible
669 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
670 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
671
672 # Ethernet address
673 if ifacedump.has_key('ether'):
674 flags_if[ifname]['ether'] = ifacedump['ether']
675
676 # Add interface IP to list
677 if ifacedump.has_key('ip'):
678 item = (ifacedump['ip'], ifacedump['comment'])
679 if addrs_list.has_key(ifname):
680 addrs_list[ifname].append(item)
681 else:
682 addrs_list[ifname] = [item]
683
684 # Alias only needs IP assignment for now, this might change if we
685 # are going to use virtual accesspoints
686 if "alias" in iface_key:
687 continue
688
689 # XXX: Might want to deduct type directly from interface name
690 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
691 # Default to station (client) mode
692 ifacedump['autogen_wlanmode'] = "sta"
693 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
694 ifacedump['autogen_wlanmode'] = "ap"
695
696 if not ifacedump.has_key('channel'):
697 if ifacedump['type'] == '11a':
698 ifacedump['channel'] = 36
699 else:
700 ifacedump['channel'] = 1
701
702 # Allow special hacks at the back like wds and stuff
703 if not ifacedump.has_key('extra'):
704 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
705 else:
706 ifacedump['autogen_extra'] = ifacedump['extra']
707
708 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
709
710 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
711 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
712 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
713 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
714 output += "\n"
715
716 elif ifacedump['type'] in ['ethernet', 'eth']:
717 # No special config needed besides IP
718 pass
719 elif ifacedump['type'] in ['vlan']:
720 # VLAN member has no special configuration
721 pass
722 else:
723 assert False, "Unknown type " + ifacedump['type']
724
725 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
726 interface_list_cache[datadump['autogen_item']] = store
727 return(store)
728
729
730
731def generate_rc_conf_local(datadump):
732 """ Generate configuration file '/etc/rc.conf.local' """
733 item = datadump['autogen_item']
734 if rc_conf_local_cache.has_key(item):
735 return rc_conf_local_cache[item]
736
737 if not datadump.has_key('ileiden'):
738 datadump['autogen_ileiden_enable'] = False
739 else:
740 datadump['autogen_ileiden_enable'] = datadump['ileiden']
741
742 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
743
744
745 for i in range(1,50):
746 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
747
748 if not ileiden_proxies or not normal_proxies:
749 for host in get_hostlist():
750 hostdump = get_yaml(host)
751 if hostdump['status'] == 'up':
752 if hostdump['service_proxy_ileiden']:
753 ileiden_proxies[hostdump['serviceid']] = hostdump
754 if hostdump['service_proxy_normal']:
755 normal_proxies.append(hostdump)
756
757 datadump['autogen_ileiden_proxies'] = ileiden_proxies
758 datadump['autogen_normal_proxies'] = normal_proxies
759 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
760 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
761 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
762 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
763
764 output = generate_header(datadump, "#");
765 output += render_template(datadump, """\
766hostname='{{ autogen_fqdn }}'
767location='{{ location }}'
768nodetype="{{ nodetype }}"
769
770#
771# Configured listings
772#
773captive_portal_whitelist=""
774{% if nodetype == "Proxy" %}
775#
776# Proxy Configuration
777#
778{% if gateway and service_proxy_ileiden -%}
779defaultrouter="{{ gateway }}"
780{% else -%}
781#defaultrouter="NOTSET"
782{% endif -%}
783internalif="{{ internalif }}"
784ileiden_enable="{{ autogen_ileiden_enable }}"
785gateway_enable="{{ autogen_ileiden_enable }}"
786pf_enable="yes"
787pf_rules="/etc/pf.conf"
788{% if autogen_ileiden_enable -%}
789pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
790lvrouted_enable="{{ autogen_ileiden_enable }}"
791lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
792{% else -%}
793pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
794{% endif -%}
795{% if internalroute -%}
796static_routes="wleiden"
797route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
798{% endif -%}
799
800{% elif nodetype == "Hybrid" %}
801 #
802 # Hybrid Configuration
803 #
804 list_ileiden_proxies="
805 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
806 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
807 {% endfor -%}
808 "
809 list_normal_proxies="
810 {% for item in autogen_normal_proxies -%}
811 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
812 {% endfor -%}
813 "
814
815 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
816 externalif="{{ externalif|default('vr0', true) }}"
817 masterip="{{ masterip }}"
818
819 {% if gateway and service_proxy_ileiden %}
820 defaultrouter="{{ gateway }}"
821 {% else %}
822 #defaultrouter="NOTSET"
823 {% endif %}
824
825 #
826 # Defined services
827 #
828 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
829 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
830 service_accesspoint="{{ service_accesspoint|yesorno }}"
831 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
832 service_concentrator="{{ service_concentrator|yesorno }}"
833
834 {% if service_proxy_ileiden %}
835 pf_rules="/etc/pf.hybrid.conf"
836 {% if service_concentrator %}
837 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
838 {% else %}
839 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
840 {% endif %}
841 pf_flags="$pf_flags -D publicnat=80,443"
842 lvrouted_flags="$lvrouted_flags -g"
843 {% elif service_proxy_normal or service_incoming_rdr %}
844 pf_rules="/etc/pf.hybrid.conf"
845 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
846 pf_flags="$pf_flags -D publicnat=0"
847 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
848 named_setfib="1"
849 tinyproxy_setfib="1"
850 dnsmasq_setfib="1"
851 sshd_setfib="1"
852 {% else %}
853 named_auto_forward_only="YES"
854 pf_rules="/etc/pf.node.conf"
855 pf_flags=""
856 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
857 {% endif %}
858 {% if service_concentrator %}
859 # Do mind installing certificates is NOT done automatically for security reasons
860 openvpn_enable="YES"
861 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
862 {% endif %}
863
864 {% if service_proxy_normal %}
865 tinyproxy_enable="yes"
866 {% else %}
867 pen_wrapper_enable="yes"
868 {% endif %}
869
870 {% if service_accesspoint %}
871 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
872 {% endif %}
873
874 {% if board == "ALIX2" %}
875 #
876 # ''Fat'' configuration, board has 256MB RAM
877 #
878 dnsmasq_enable="NO"
879 named_enable="YES"
880 {% if autogen_dhcp_interfaces -%}
881 dhcpd_enable="YES"
882 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
883 {% endif -%}
884 {% elif board == "apu1d" %}
885 #
886 # ''Fat'' configuration, board has 1024MB RAM
887 #
888 dnsmasq_enable="NO"
889 local_unbound_enable="YES"
890 {% if autogen_dhcp_interfaces -%}
891 dhcpd_enable="YES"
892 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
893 {% endif -%}
894 {% endif -%}
895{% endif %}
896
897#
898# Script variables
899#
900attached_devices="{{ autogen_attached_devices|join(' ') }}"
901neighbours="{{ autogen_neighbours|join(' ') }}"
902
903
904#
905# Interface definitions
906#\n
907""")
908
909 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
910 for iface, vlans in sorted(vlan_list.items()):
911 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
912
913 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
914 for iface in sorted(vlan_list.keys()):
915 if not iface in addrs_list.keys():
916 output += "ifconfig_%s='up'\n" % iface
917
918 output += "\n"
919
920 # Bridge configuration:
921 if bridge_list.keys():
922 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
923
924 for iface in bridge_list.keys():
925 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
926
927 # Bridge member interfaces not containing a configuration should be marked active explcitly.
928 for _,members in bridge_list.items():
929 for iface in members:
930 if not iface in addrs_list.keys():
931 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
932
933 output += "\n"
934
935 # Details like SSID
936 if extra_ouput:
937 output += extra_ouput.strip() + "\n"
938
939 # Print IP address which needs to be assigned over here
940 output += "\n"
941 for iface,addrs in sorted(addrs_list.iteritems()):
942 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
943 output += "# %s || %s || %s\n" % (iface, addr, comment)
944
945 # Write DHCLIENT entry
946 if iface in dhclient_if and dhclient_if[iface]:
947 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
948 continue
949
950 # Make sure the external address is always first as this is needed in the
951 # firewall setup
952 addrs = sorted(
953 [x for x in addrs if not '0.0.0.0' in x[0]],
954 key=lambda x: x[0].split('.')[0],
955 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
956 )
957
958 idx_offset = 0
959 # Set MAC is required
960 if flags_if[iface].has_key('ether'):
961 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
962 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
963 idx_offset += 1
964 else:
965 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
966
967 for idx, addr in enumerate(addrs[1:]):
968 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
969
970 output += "\n"
971
972 rc_conf_local_cache[datadump['autogen_item']] = output
973 return output
974
975
976
977def get_all_configs():
978 """ Get dict with key 'host' with all configs present """
979 configs = dict()
980 for host in get_hostlist():
981 datadump = get_yaml(host)
982 configs[host] = datadump
983 return configs
984
985
986def get_interface_keys(config, extra=False):
987 """ Quick hack to get all interface keys, later stage convert this to a iterator """
988 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
989 if extra == False:
990 return filter(lambda x: not "extra" in x, elems)
991 else:
992 return elems
993
994
995def get_used_ips(configs):
996 """ Return array of all IPs used in config files"""
997 ip_list = []
998 for config in configs:
999 ip_list.append(config['masterip'])
1000 if 'serviceid' in config:
1001 ip_list.append(config['serviceid'])
1002 for iface_key in get_interface_keys(config, True):
1003 l = config[iface_key]['ip']
1004 addr, mask = l.split('/')
1005 # Special case do not process
1006 if valid_addr(addr):
1007 ip_list.append(addr)
1008 else:
1009 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1010 return sorted(ip_list)
1011
1012
1013
1014def get_nameservers(max_servers=None):
1015 if nameservers_cache:
1016 return nameservers_cache[0:max_servers]
1017
1018 for host in get_hostlist():
1019 hostdump = get_yaml(host)
1020 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1021 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
1022
1023 return nameservers_cache[0:max_servers]
1024
1025
1026def get_neighbours(datadump):
1027 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1028
1029 (poel, errors) = make_relations()
1030 table = []
1031 for iface,addrs in sorted(addrs_list.iteritems()):
1032 if iface in ['lo0']:
1033 continue
1034
1035 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1036 if not addr.startswith('172.'):
1037 # Avoid listing internet connections as pool
1038 continue
1039 for neighbour in poel[network(addr)]:
1040 if neighbour[0] != datadump['autogen_item']:
1041 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1042 return table
1043
1044
1045def get_attached_devices(datadump, url=False):
1046 table = []
1047 for iface_key in get_interface_keys(datadump, True):
1048 ifacedump = datadump[iface_key]
1049
1050 if not ifacedump.has_key('ns_ip'):
1051 continue
1052
1053 x_ip = ifacedump['ns_ip'].split('/')[0]
1054
1055 if 'mode' in ifacedump:
1056 x_mode = ifacedump['mode']
1057 else:
1058 x_mode = 'unknown'
1059
1060 if 'bridge_type' in ifacedump:
1061 device_type = ifacedump['bridge_type']
1062 else:
1063 device_type = 'Unknown'
1064
1065 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1066 return table
1067
1068
1069def generate_resolv_conf(datadump):
1070 """ Generate configuration file '/etc/resolv.conf' """
1071 # XXX: This should properly going to be an datastructure soon
1072 datadump['autogen_header'] = generate_header(datadump, "#")
1073 datadump['autogen_edge_nameservers'] = ''
1074
1075
1076 for masterip,realname in get_nameservers():
1077 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1078
1079 return Template("""\
1080{{ autogen_header }}
1081search wleiden.net
1082
1083# Try local (cache) first
1084nameserver 127.0.0.1
1085
1086{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1087nameserver 8.8.8.8 # Google Public NameServer
1088nameserver 4.2.2.1 # Level3 Public NameServer
1089{% else -%}
1090# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1091{{ autogen_edge_nameservers }}
1092{% endif -%}
1093""").render(datadump)
1094
1095
1096
1097def generate_ntp_conf(datadump):
1098 """ Generate configuration file '/etc/ntp.conf' """
1099 # XXX: This should properly going to be an datastructure soon
1100
1101 datadump['autogen_header'] = generate_header(datadump, "#")
1102 datadump['autogen_ntp_servers'] = ''
1103 for host in get_hostlist():
1104 hostdump = get_yaml(host)
1105 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1106 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1107
1108 return Template("""\
1109{{ autogen_header }}
1110
1111{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1112# Machine hooked to internet.
1113server 0.nl.pool.ntp.org iburst maxpoll 9
1114server 1.nl.pool.ntp.org iburst maxpoll 9
1115server 2.nl.pool.ntp.org iburst maxpoll 9
1116server 3.nl.pool.ntp.org iburst maxpoll 9
1117{% else -%}
1118# Local Wireless Leiden NTP Servers.
1119server 0.pool.ntp.wleiden.net iburst maxpoll 9
1120server 1.pool.ntp.wleiden.net iburst maxpoll 9
1121server 2.pool.ntp.wleiden.net iburst maxpoll 9
1122server 3.pool.ntp.wleiden.net iburst maxpoll 9
1123
1124# All the configured NTP servers
1125{{ autogen_ntp_servers }}
1126{% endif %}
1127
1128# If a server loses sync with all upstream servers, NTP clients
1129# no longer follow that server. The local clock can be configured
1130# to provide a time source when this happens, but it should usually
1131# be configured on just one server on a network. For more details see
1132# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1133# The use of Orphan Mode may be preferable.
1134#
1135server 127.127.1.0
1136fudge 127.127.1.0 stratum 10
1137""").render(datadump)
1138
1139
1140def generate_pf_hybrid_conf_local(datadump):
1141 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1142 datadump['autogen_header'] = generate_header(datadump, "#")
1143 return Template("""\
1144{{ autogen_header }}
1145
1146# Redirect some internal facing services outside (7)
1147# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1148{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1149rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1150{% endfor -%}
1151""").render(datadump)
1152
1153def generate_motd(datadump):
1154 """ Generate configuration file '/etc/motd' """
1155 output = Template("""\
1156FreeBSD run ``service motd onestart'' to make me look normal
1157
1158 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1159 Loc: {{ location }}
1160
1161Services:
1162{% if board == "ALIX2" -%}
1163{{" -"}} Core Node ({{ board }})
1164{% else -%}
1165{{" -"}} Hulp Node ({{ board }})
1166{% endif -%}
1167{% if service_proxy_normal -%}
1168{{" -"}} Normal Proxy
1169{% endif -%}
1170{% if service_proxy_ileiden -%}
1171{{" -"}} iLeiden Proxy
1172{% endif -%}
1173{% if service_incoming_rdr -%}
1174{{" -"}} Incoming port redirects
1175{% endif %}
1176Interlinks:\n
1177""").render(datadump)
1178
1179
1180 def make_table(table):
1181 if not table:
1182 return " - none\n"
1183 else:
1184 lines = ""
1185 col_width = [max(len(x) for x in col) for col in zip(*table)]
1186 for row in table:
1187 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1188 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1189 return lines
1190
1191 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1192 table = []
1193 for iface,addrs in sorted(addrs_list.iteritems()):
1194 if iface in ['lo0']:
1195 continue
1196 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1197 table.append((iface, addr, comment))
1198
1199 output += make_table(table)
1200 output += '\n'
1201 output += """\
1202Attached devices:
1203"""
1204 output += make_table(get_attached_devices(datadump, url=True))
1205 output += '\n'
1206 output += """\
1207Available neighbours:
1208"""
1209 output += make_table(get_neighbours(datadump))
1210
1211 return output
1212
1213
1214def format_yaml_value(value):
1215 """ Get yaml value in right syntax for outputting """
1216 if isinstance(value,str):
1217 output = '"%s"' % value
1218 else:
1219 output = value
1220 return output
1221
1222
1223
1224def format_wleiden_yaml(datadump):
1225 """ Special formatting to ensure it is editable"""
1226 output = "# Genesis config yaml style\n"
1227 output += "# vim:ts=2:et:sw=2:ai\n"
1228 output += "#\n"
1229 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1230 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1231 if key == 'rdr_rules':
1232 output += '%-10s:\n' % 'rdr_rules'
1233 for rdr_rule in datadump[key]:
1234 output += '- %s\n' % rdr_rule
1235 else:
1236 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1237
1238 output += "\n\n"
1239
1240 # Format (key, required)
1241 key_order = (
1242 ('comment', True),
1243 ('parent', False),
1244 ('ip', False),
1245 ('ether', False),
1246 ('desc', True),
1247 ('sdesc', True),
1248 ('mode', True),
1249 ('type', True),
1250 ('extra_type', False),
1251 ('channel', False),
1252 ('ssid', False),
1253 ('wlan_mac', False),
1254 ('dhcp', True),
1255 ('compass', False),
1256 ('distance', False),
1257 ('ns_ip', False),
1258 ('repeater_ip', False),
1259 ('bullet2_ip', False),
1260 ('ns_mac', False),
1261 ('bullet2_mac', False),
1262 ('ns_type', False),
1263 ('bridge_type', False),
1264 ('status', True),
1265 )
1266
1267 for iface_key in sorted(iface_keys):
1268 try:
1269 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1270 if remainder:
1271 raise KeyError("invalid keys: %s" % remainder)
1272
1273 output += "%s:\n" % iface_key
1274 for key,required in key_order:
1275 if datadump[iface_key].has_key(key):
1276 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1277 output += "\n\n"
1278 except Exception:
1279 print "# Error while processing interface %s" % iface_key
1280 raise
1281
1282 return output
1283
1284
1285
1286def generate_wleiden_yaml(datadump, header=True):
1287 """ Generate (petty) version of wleiden.yaml"""
1288 output = generate_header(datadump, "#") if header else ''
1289
1290 for key in datadump.keys():
1291 if key.startswith('autogen_'):
1292 del datadump[key]
1293 # Interface autogen cleanups
1294 elif type(datadump[key]) == dict:
1295 for key2 in datadump[key].keys():
1296 if key2.startswith('autogen_'):
1297 del datadump[key][key2]
1298
1299 output += format_wleiden_yaml(datadump)
1300 return output
1301
1302def generate_nanostation_config(datadump, iface, ns_type):
1303 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1304 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1305
1306 datadump.update(datadump['iface_%s' % iface])
1307
1308 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1309
1310def generate_yaml(datadump):
1311 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1312
1313
1314
1315def generate_config(node, config, datadump=None):
1316 """ Print configuration file 'config' of 'node' """
1317 output = ""
1318 try:
1319 # Load config file
1320 if datadump == None:
1321 datadump = get_yaml(node)
1322
1323 if config == 'wleiden.yaml':
1324 output += generate_wleiden_yaml(datadump)
1325 elif config == 'authorized_keys':
1326 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1327 output += f.read()
1328 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1329 # Fetch local keys if existing
1330 if os.path.exists(node_keys):
1331 output += open(node_keys, 'r').read()
1332 f.close()
1333 elif config == 'dnsmasq.conf':
1334 output += generate_dnsmasq_conf(datadump)
1335 elif config == 'dhcpd.conf':
1336 output += generate_dhcpd_conf(datadump)
1337 elif config == 'rc.conf.local':
1338 output += generate_rc_conf_local(datadump)
1339 elif config == 'resolv.conf':
1340 output += generate_resolv_conf(datadump)
1341 elif config == 'ntp.conf':
1342 output += generate_ntp_conf(datadump)
1343 elif config == 'motd':
1344 output += generate_motd(datadump)
1345 elif config == 'pf.hybrid.conf.local':
1346 output += generate_pf_hybrid_conf_local(datadump)
1347 elif config.startswith('vr'):
1348 interface, ns_type = config.strip('.yaml').split('-')
1349 output += generate_nanostation_config(datadump, interface, ns_type)
1350 else:
1351 assert False, "Config not found!"
1352 except IOError, e:
1353 output += "[ERROR] Config file not found"
1354 return output
1355
1356
1357
1358def process_cgi_request(environ=os.environ):
1359 """ When calling from CGI """
1360 response_headers = []
1361 content_type = 'text/plain'
1362
1363 # Update repository if requested
1364 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1365 if form and form.has_key("action") and "update" in form["action"]:
1366 output = "[INFO] Updating subverion, please wait...\n"
1367 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1368 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1369 output += "[INFO] All done, redirecting in 5 seconds"
1370 response_headers += [
1371 ('Refresh', '5; url=.'),
1372 ]
1373 reload_cache()
1374 else:
1375 base_uri = environ['PATH_INFO']
1376 uri = base_uri.strip('/').split('/')
1377
1378 output = "Template Holder"
1379 if base_uri.endswith('/create/network.kml'):
1380 content_type='application/vnd.google-earth.kml+xml'
1381 output = make_network_kml.make_graph()
1382 elif base_uri.endswith('/api/get/nodeplanner.json'):
1383 content_type='application/json'
1384 output = make_network_kml.make_nodeplanner_json()
1385 elif not uri[0]:
1386 if is_text_request(environ):
1387 output = '\n'.join(get_hostlist())
1388 else:
1389 content_type = 'text/html'
1390 output = generate_title(get_hostlist())
1391 elif len(uri) == 1:
1392 if is_text_request(environ):
1393 output = generate_node(uri[0])
1394 else:
1395 content_type = 'text/html'
1396 output = generate_node_overview(uri[0])
1397 elif len(uri) == 2:
1398 output = generate_config(uri[0], uri[1])
1399 else:
1400 assert False, "Invalid option"
1401
1402 # Return response
1403 response_headers += [
1404 ('Content-type', content_type),
1405 ('Content-Length', str(len(output))),
1406 ]
1407 return(response_headers, str(output))
1408
1409
1410def make_dns(output_dir = 'dns', external = False):
1411 items = dict()
1412
1413 # hostname is key, IP is value
1414 wleiden_zone = defaultdict(list)
1415 wleiden_cname = dict()
1416
1417 pool = dict()
1418 for node in get_hostlist():
1419 datadump = get_yaml(node)
1420
1421 fqdn = datadump['nodename']
1422
1423 if datadump.has_key('rdr_host'):
1424 remote_target = datadump['rdr_host']
1425 elif datadump.has_key('remote_access') and datadump['remote_access']:
1426 remote_target = datadump['remote_access'].split(':')[0]
1427 else:
1428 remote_target = None
1429
1430 if remote_target:
1431 try:
1432 parseaddr(remote_target)
1433 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1434 except (IndexError, ValueError):
1435 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1436
1437
1438 wleiden_zone[fqdn].append((datadump['masterip'], True))
1439
1440 # Hacking to get proper DHCP IPs and hostnames
1441 for iface_key in get_interface_keys(datadump):
1442 iface_name = iface_key.replace('_','-')
1443 if 'ip' in datadump[iface_key]:
1444 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1445 try:
1446 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1447 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1448 dhcp_part = ".".join(ip.split('.')[0:3])
1449 if ip != datadump['masterip']:
1450 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1451 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1452 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1453 except (AttributeError, ValueError, KeyError):
1454 # First push it into a pool, to indentify the counter-part later on
1455 addr = parseaddr(ip)
1456 cidr = int(cidr)
1457 addr = addr & ~((1 << (32 - cidr)) - 1)
1458 if pool.has_key(addr):
1459 pool[addr] += [(iface_name, fqdn, ip)]
1460 else:
1461 pool[addr] = [(iface_name, fqdn, ip)]
1462 continue
1463
1464
1465
1466 # WL uses an /29 to configure an interface. IP's are ordered like this:
1467 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1468
1469 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1470
1471 # Automatic naming convention of interlinks namely 2 + remote.lower()
1472 for (key,value) in pool.iteritems():
1473 # Make sure they are sorted from low-ip to high-ip
1474 value = sorted(value, key=lambda x: parseaddr(x[2]))
1475
1476 if len(value) == 1:
1477 (iface_name, fqdn, ip) = value[0]
1478 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1479
1480 # Device DNS names
1481 if 'cnode' in fqdn.lower():
1482 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1483 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1484
1485 elif len(value) == 2:
1486 (a_iface_name, a_fqdn, a_ip) = value[0]
1487 (b_iface_name, b_fqdn, b_ip) = value[1]
1488 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1489 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1490
1491 # Device DNS names
1492 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1493 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1494 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1495 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1496 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1497 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1498 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1499
1500 else:
1501 pool_members = [k[1] for k in value]
1502 for item in value:
1503 (iface_name, fqdn, ip) = item
1504 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1505
1506 # Include static DNS entries
1507 # XXX: Should they override the autogenerated results?
1508 # XXX: Convert input to yaml more useable.
1509 # Format:
1510 ##; this is a comment
1511 ## roomburgh=Roomburgh1
1512 ## apkerk1.Vosko=172.17.176.8 ;this as well
1513 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1514
1515 # Hack to allow special entries, for development
1516 wleiden_raw = {}
1517
1518 for line in dns_list:
1519 reverse = False
1520 k, items = line.items()[0]
1521 if type(items) == dict:
1522 if items.has_key('reverse'):
1523 reverse = items['reverse']
1524 items = items['a']
1525 else:
1526 items = items['cname']
1527 items = [items] if type(items) != list else items
1528 for item in items:
1529 if item.startswith('IN '):
1530 wleiden_raw[k] = item
1531 elif valid_addr(item):
1532 wleiden_zone[k].append((item, reverse))
1533 else:
1534 wleiden_cname[k] = item
1535
1536 # Hack to get dynamic pool listing
1537 def chunks(l, n):
1538 return [l[i:i+n] for i in range(0, len(l), n)]
1539
1540 ntp_servers = [x[0] for x in get_nameservers()]
1541 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1542 for ntp_server in chunk:
1543 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1544
1545 details = dict()
1546 # 24 updates a day allowed
1547 details['serial'] = time.strftime('%Y%m%d%H')
1548
1549 if external:
1550 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1551 else:
1552 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1553
1554 details['master'] = dns_masters[0]
1555 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1556
1557 dns_header = '''
1558$TTL 3h
1559%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1560 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1561
1562%(ns_servers)s
1563 \n'''
1564
1565
1566 if not os.path.isdir(output_dir):
1567 os.makedirs(output_dir)
1568 details['zone'] = 'wleiden.net'
1569 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1570 f.write(dns_header % details)
1571
1572 for host,items in wleiden_zone.iteritems():
1573 for ip,reverse in items:
1574 if ip not in ['0.0.0.0']:
1575 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1576 for source,dest in wleiden_cname.iteritems():
1577 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1578 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1579 for source, dest in wleiden_raw.iteritems():
1580 f.write("%s.wleiden.net. %s\n" % (source, dest))
1581 f.close()
1582
1583 # Create whole bunch of specific sub arpa zones. To keep it compliant
1584 for s in range(16,32):
1585 details['zone'] = '%i.172.in-addr.arpa' % s
1586 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1587 f.write(dns_header % details)
1588
1589 #XXX: Not effient, fix to proper data structure and do checks at other
1590 # stages
1591 for host,items in wleiden_zone.iteritems():
1592 for ip,reverse in items:
1593 if not reverse:
1594 continue
1595 if valid_addr(ip):
1596 if valid_addr(ip):
1597 if int(ip.split('.')[1]) == s:
1598 rev_ip = '.'.join(reversed(ip.split('.')))
1599 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1600 f.close()
1601
1602
1603def usage():
1604 print """Usage: %(prog)s <argument>
1605Argument:
1606\tcleanup = Cleanup all YAML files to specified format
1607\tstandalone [port] = Run configurator webserver [8000]
1608\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1609\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1610\tfull-export = Generate yaml export script for heatmap.
1611\tstatic [outputdir] = Generate all config files and store on disk
1612\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1613\ttest <node> [<file>] = Receive output for certain node [all files].
1614\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1615\tlist <status> <items> = List systems which have certain status
1616\tcreate network.kml = Create Network KML file for use in Google Earth
1617
1618Arguments:
1619\t<node> = NodeName (example: HybridRick)
1620\t<file> = %(files)s
1621\t<status> = all|up|down|planned
1622\t<items> = systems|nodes|proxies
1623
1624NOTE FOR DEVELOPERS; you can test your changes like this:
1625 BEFORE any changes in this code:
1626 $ ./gformat.py static /tmp/pre
1627 AFTER the changes:
1628 $ ./gformat.py static /tmp/post
1629 VIEW differences and VERIFY all are OK:
1630 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1631""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1632 exit(0)
1633
1634
1635def is_text_request(environ=os.environ):
1636 """ Find out whether we are calling from the CLI or any text based CLI utility """
1637 try:
1638 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1639 except KeyError:
1640 return True
1641
1642def switchFormat(setting):
1643 if setting:
1644 return "YES"
1645 else:
1646 return "NO"
1647
1648def rlinput(prompt, prefill=''):
1649 import readline
1650 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1651 try:
1652 return raw_input(prompt)
1653 finally:
1654 readline.set_startup_hook()
1655
1656def fix_conflict(left, right, default='i'):
1657 while True:
1658 print "## %-30s | %-30s" % (left, right)
1659 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1660 if not c:
1661 c = default
1662
1663 if c in ['l','1']:
1664 return left
1665 elif c in ['r','2']:
1666 return right
1667 elif c in ['e', '3']:
1668 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1669 elif c in ['i', '4']:
1670 return None
1671 else:
1672 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1673
1674
1675
1676def print_cgi_response(response_headers, output):
1677 """Could we not use some kind of wsgi wrapper to make this output?"""
1678 for header in response_headers:
1679 print "%s: %s" % header
1680 print
1681 print output
1682
1683
1684def fill_cache():
1685 ''' Poor man re-loading of few cache items (the slow ones) '''
1686 for host in get_hostlist():
1687 get_yaml(host)
1688
1689
1690def reload_cache():
1691 clear_cache()
1692 fill_cache()
1693
1694
1695def main():
1696 """Hard working sub"""
1697 # Allow easy hacking using the CLI
1698 if not os.environ.has_key('PATH_INFO'):
1699 if len(sys.argv) < 2:
1700 usage()
1701
1702 if sys.argv[1] == "standalone":
1703 import SocketServer
1704 import CGIHTTPServer
1705 # Hop to the right working directory.
1706 os.chdir(os.path.dirname(__file__))
1707 try:
1708 PORT = int(sys.argv[2])
1709 except (IndexError,ValueError):
1710 PORT = 8000
1711
1712 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1713 """ Serve this CGI from the root of the webserver """
1714 def is_cgi(self):
1715 if "favicon" in self.path:
1716 return False
1717
1718 self.cgi_info = (os.path.basename(__file__), self.path)
1719 self.path = ''
1720 return True
1721 handler = MyCGIHTTPRequestHandler
1722 SocketServer.TCPServer.allow_reuse_address = True
1723 httpd = SocketServer.TCPServer(("", PORT), handler)
1724 httpd.server_name = 'localhost'
1725 httpd.server_port = PORT
1726
1727 logger.info("serving at port %s", PORT)
1728 try:
1729 httpd.serve_forever()
1730 except KeyboardInterrupt:
1731 httpd.shutdown()
1732 logger.info("All done goodbye")
1733 elif sys.argv[1] == "test":
1734 # Basic argument validation
1735 try:
1736 node = sys.argv[2]
1737 except IndexError:
1738 print "Invalid argument"
1739 exit(1)
1740 except IOError as e:
1741 print e
1742 exit(1)
1743
1744 datadump = get_yaml(node)
1745
1746
1747 # Get files to generate
1748 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1749
1750 # Actual config generation
1751 for config in gen_files:
1752 logger.info("## Generating %s %s", node, config)
1753 print generate_config(node, config, datadump)
1754 elif sys.argv[1] == "test-cgi":
1755 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1756 os.environ['SCRIPT_NAME'] = __file__
1757 response_headers, output = process_cgi_request()
1758 print_cgi_response(response_headers, output)
1759 elif sys.argv[1] == "static":
1760 items = dict()
1761 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1762 for node in get_hostlist():
1763 items['node'] = node
1764 items['wdir'] = "%(output_dir)s/%(node)s" % items
1765 if not os.path.isdir(items['wdir']):
1766 os.makedirs(items['wdir'])
1767 datadump = get_yaml(node)
1768 for config in files:
1769 items['config'] = config
1770 logger.info("## Generating %(node)s %(config)s" % items)
1771 f = open("%(wdir)s/%(config)s" % items, "w")
1772 f.write(generate_config(node, config, datadump))
1773 f.close()
1774 elif sys.argv[1] == "wind-export":
1775 items = dict()
1776 for node in get_hostlist():
1777 datadump = get_yaml(node)
1778 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1779 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1780 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1781 VALUES (
1782 (SELECT id FROM users WHERE username = 'rvdzwet'),
1783 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1784 'Y');""" % datadump
1785 #for config in files:
1786 # items['config'] = config
1787 # print "## Generating %(node)s %(config)s" % items
1788 # f = open("%(wdir)s/%(config)s" % items, "w")
1789 # f.write(generate_config(node, config, datadump))
1790 # f.close()
1791 for node in get_hostlist():
1792 datadump = get_yaml(node)
1793 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1794 ifacedump = datadump[iface_key]
1795 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1796 ifacedump['nodename'] = datadump['nodename']
1797 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1798 ifacedump['channel'] = 0
1799 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1800 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1801 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1802 elif sys.argv[1] == "nagios-export":
1803 try:
1804 heavy_load = (sys.argv[2] == "--heavy-load")
1805 except IndexError:
1806 heavy_load = False
1807
1808 hostgroup_details = {
1809 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1810 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1811 'walphen' : 'Stichting Wireless Alphen',
1812 'westeinder' : 'Westeinder Plassen',
1813 }
1814
1815 # Convert IP to Host
1816 ip2host = {'root' : 'root'}
1817 for host in get_hostlist():
1818 datadump = get_yaml(host)
1819 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1820 for iface in get_interface_keys(datadump):
1821 if datadump[iface].has_key('autogen_gateway'):
1822 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1823
1824 # Find dependency tree based on output of lvrouted.mytree of nearest node
1825 parents = defaultdict(list)
1826 stack = ['root']
1827 prev_depth = 0
1828 for line in open('lvrouted.mytree').readlines():
1829 depth = line.count('\t')
1830 ip = line.strip().split()[0]
1831
1832 if prev_depth < depth:
1833 try:
1834 parents[ip2host[ip]].append(ip2host[stack[-1]])
1835 except KeyError as e:
1836 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1837 stack.append(ip)
1838 elif prev_depth > depth:
1839 stack = stack[:(depth - prev_depth)]
1840 elif prev_depth == depth:
1841 try:
1842 parents[ip2host[ip]].append(ip2host[stack[-1]])
1843 except KeyError as e:
1844 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1845
1846
1847 prev_depth = depth
1848 # Observe that some nodes has themself as parent or multiple parents
1849 # for now take only the first parent, other behaviour is yet to be explained
1850
1851
1852
1853 params = {
1854 'check_interval' : 5 if heavy_load else 120,
1855 'retry_interval' : 1 if heavy_load else 10,
1856 'max_check_attempts' : 10 if heavy_load else 6,
1857 'notification_interval': 120 if heavy_load else 240,
1858 }
1859
1860 print '''\
1861define host {
1862 name wleiden-node ; Default Node Template
1863 use generic-host ; Use the standard template as initial starting point
1864 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1865 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1866 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1867 notification_interval %(notification_interval)s
1868 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1869 check_command check-host-alive ; Default command to check FreeBSD hosts
1870 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1871}
1872
1873define service {
1874 name wleiden-service ; Default Service Template
1875 use generic-service ; Use the standard template as initial starting point
1876 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1877 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1878 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1879 notification_interval %(notification_interval)s
1880 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1881 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1882}
1883
1884# Please make sure to install:
1885# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1886#
1887# Recompile net-mgmt/nagios-plugins to support check_snmp
1888# make -C /usr/ports/net-mgmt/nagios-plugins
1889#
1890# Install net/bind-tools to allow v2/check_dns_wl to work:
1891# pkg install bind-tools
1892#
1893define command{
1894 command_name check_snmp_disk
1895 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1896}
1897
1898define command{
1899 command_name check_netsnmp_load
1900 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1901}
1902
1903define command{
1904 command_name check_netsnmp_proc
1905 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1906}
1907
1908define command{
1909 command_name check_by_ssh
1910 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1911}
1912
1913define command{
1914 command_name check_dns_wl
1915 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1916}
1917
1918define command{
1919 command_name check_snmp_uptime
1920 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1921}
1922
1923
1924# TDB: dhcp leases
1925# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1926
1927# TDB: internet status
1928# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1929
1930# TDB: Advanced local passive checks
1931# /usr/local/libexec/nagios/check_by_ssh
1932''' % params
1933
1934 print '''\
1935# Service Group, not displayed by default
1936define hostgroup {
1937 hostgroup_name srv_hybrid
1938 alias All Hybrid Nodes
1939 register 0
1940}
1941
1942define service {
1943 use wleiden-service
1944 hostgroup_name srv_hybrid
1945 service_description SSH
1946 check_command check_ssh
1947}
1948
1949define service {
1950 use wleiden-service,service-pnp
1951 hostgroup_name srv_hybrid
1952 service_description HTTP
1953 check_command check_http
1954}
1955
1956define service {
1957 use wleiden-service
1958 hostgroup_name srv_hybrid
1959 service_description DNS
1960 check_command check_dns_wl!"www.wirelessleiden.nl"
1961}
1962'''
1963
1964 if heavy_load:
1965 print '''\
1966define service {
1967 use wleiden-service
1968 hostgroup_name srv_hybrid
1969 service_description UPTIME
1970 check_command check_snmp_uptime
1971}
1972
1973#define service {
1974# use wleiden-service
1975# hostgroup_name srv_hybrid
1976# service_description NTP
1977# check_command check_ntp_peer
1978#}
1979
1980define service {
1981 use wleiden-service
1982 hostgroup_name srv_hybrid
1983 service_description LOAD
1984 check_command check_netsnmp_load
1985}
1986
1987define service {
1988 use wleiden-service
1989 hostgroup_name srv_hybrid
1990 service_description PROC
1991 check_command check_netsnmp_proc
1992}
1993
1994define service {
1995 use wleiden-service
1996 hostgroup_name srv_hybrid
1997 service_description DISK
1998 check_command check_snmp_disk
1999}
2000'''
2001 for node in get_hostlist():
2002 datadump = get_yaml(node)
2003 if not datadump['status'] == 'up':
2004 continue
2005 if not hostgroup_details.has_key(datadump['monitoring_group']):
2006 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2007 print '''\
2008define host {
2009 use wleiden-node,host-pnp
2010 contact_groups admins
2011 host_name %(autogen_fqdn)s
2012 address %(masterip)s
2013 hostgroups srv_hybrid,%(monitoring_group)s\
2014''' % datadump
2015 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2016 print '''\
2017 parents %(parents)s\
2018''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2019 print '''\
2020}
2021'''
2022
2023
2024 for name,alias in hostgroup_details.iteritems():
2025 print '''\
2026define hostgroup {
2027 hostgroup_name %s
2028 alias %s
2029} ''' % (name, alias)
2030
2031
2032 elif sys.argv[1] == "full-export":
2033 hosts = {}
2034 for node in get_hostlist():
2035 datadump = get_yaml(node)
2036 hosts[datadump['nodename']] = datadump
2037 print yaml.dump(hosts)
2038
2039 elif sys.argv[1] == "dns":
2040 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2041 elif sys.argv[1] == "cleanup":
2042 # First generate all datadumps
2043 datadumps = dict()
2044 ssid_to_node = dict()
2045 for host in get_hostlist():
2046 logger.info("# Processing: %s", host)
2047 # Set some boring default values
2048 datadump = { 'board' : 'UNKNOWN' }
2049 datadump.update(get_yaml(host))
2050 datadumps[datadump['nodename']] = datadump
2051
2052 (poel, errors) = make_relations()
2053 print "\n".join(["# WARNING: %s" % x for x in errors])
2054
2055 for host,datadump in datadumps.iteritems():
2056 try:
2057 # Convert all yes and no to boolean values
2058 def fix_boolean(dump):
2059 for key in dump.keys():
2060 if type(dump[key]) == dict:
2061 dump[key] = fix_boolean(dump[key])
2062 elif str(dump[key]).lower() in ["yes", "true"]:
2063 dump[key] = True
2064 elif str(dump[key]).lower() in ["no", "false"]:
2065 # Compass richting no (Noord Oost) is valid input
2066 if key != "compass": dump[key] = False
2067 return dump
2068 datadump = fix_boolean(datadump)
2069
2070 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2071 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2072 elif 'latitude' in datadump and 'longitude' in datadump:
2073 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2074
2075 if datadump['nodename'].startswith('Proxy'):
2076 datadump['nodename'] = datadump['nodename'].lower()
2077
2078 for iface_key in get_interface_keys(datadump):
2079 try:
2080 # All our normal wireless cards are normal APs now
2081 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2082 datadump[iface_key]['mode'] = 'ap'
2083 # Wireless Leiden SSID have an consistent lowercase/uppercase
2084 if datadump[iface_key].has_key('ssid'):
2085 ssid = datadump[iface_key]['ssid']
2086 prefix = 'ap-WirelessLeiden-'
2087 if ssid.lower().startswith(prefix.lower()):
2088 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2089 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2090 datadump[iface_key]['mode'] = 'autogen-FIXME'
2091 if not datadump[iface_key].has_key('comment'):
2092 datadump[iface_key]['comment'] = 'autogen-FIXME'
2093
2094 if datadump[iface_key].has_key('ns_mac'):
2095 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2096
2097 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2098 datadump[iface_key] = datadump[iface_key]['desc']
2099
2100 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2101 # due to better collision avoidance.
2102 if datadump[iface_key]['type'] == '11b':
2103 datadump[iface_key]['type'] = '11g'
2104
2105 # Setting 802.11g channels to de-facto standards, to avoid
2106 # un-detected sharing with other overlapping channels
2107 #
2108 # Technically we could also use channel 13 in NL, but this is not
2109 # recommended as foreign devices might not be able to select this
2110 # channel. Secondly using 1,5,9,13 instead is going to clash with
2111 # the de-facto usage of 1,6,11.
2112 #
2113 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2114 channels_at_2400Mhz = (1,6,11)
2115 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2116 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2117 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2118 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2119
2120 # Mandatory interface keys
2121 if not datadump[iface_key].has_key('status'):
2122 datadump[iface_key]['status'] = 'planned'
2123
2124 x = datadump[iface_key]['comment']
2125 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2126
2127 # Fixing bridge_type if none is found
2128 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2129 if not 'bridge_type' in datadump[iface_key]:
2130 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2131
2132 # Making sure description works
2133 if datadump[iface_key].has_key('desc'):
2134 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2135 del datadump[iface_key]['desc']
2136 else:
2137 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2138 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2139 if response:
2140 datadump[iface_key]['comment'] = response
2141 del datadump[iface_key]['desc']
2142
2143 # Check DHCP configuration
2144 dhcp_type(datadump[iface_key])
2145
2146 # Set the compass value based on the angle between the poels
2147 if datadump[iface_key].has_key('ns_ip'):
2148 my_pool = poel[network(datadump[iface_key]['ip'])]
2149 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2150 if remote_hosts:
2151 compass_target = remote_hosts[0]
2152 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2153
2154 # Monitoring Group default
2155 if not 'monitoring_group' in datadump:
2156 datadump['monitoring_group'] = 'wleiden'
2157
2158 except Exception:
2159 print "# Error while processing interface %s" % iface_key
2160 raise
2161 store_yaml(datadump)
2162 except Exception:
2163 print "# Error while processing %s" % host
2164 raise
2165 elif sys.argv[1] == "list":
2166 use_fqdn = False
2167 if len(sys.argv) < 4:
2168 usage()
2169 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2170 usage()
2171 if not sys.argv[3] in ["nodes","proxies","systems"]:
2172 usage()
2173
2174 if len(sys.argv) > 4:
2175 if sys.argv[4] == "fqdn":
2176 use_fqdn = True
2177 else:
2178 usage()
2179
2180 for system in get_hostlist():
2181 datadump = get_yaml(system)
2182 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2183 continue
2184
2185 output = datadump['autogen_fqdn'] if use_fqdn else system
2186 if sys.argv[2] == "all":
2187 print output
2188 elif datadump['status'] == sys.argv[2]:
2189 print output
2190 elif sys.argv[1] == "create":
2191 if sys.argv[2] == "network.kml":
2192 print make_network_kml.make_graph()
2193 elif sys.argv[2] == "host-ips.txt":
2194 for system in get_hostlist():
2195 datadump = get_yaml(system)
2196 ips = [datadump['masterip']]
2197 for ifkey in get_interface_keys(datadump):
2198 ips.append(datadump[ifkey]['ip'].split('/')[0])
2199 print system, ' '.join(ips)
2200 elif sys.argv[2] == "host-pos.txt":
2201 for system in get_hostlist():
2202 datadump = get_yaml(system)
2203 print system, datadump['rdnap_x'], datadump['rdnap_y']
2204 elif sys.argv[2] == 'ssh_config':
2205 print '''
2206Host *.wleiden.net
2207 User root
2208
2209Host 172.16.*.*
2210 User root
2211'''
2212 for system in get_hostlist():
2213 datadump = get_yaml(system)
2214 print '''\
2215Host %s
2216 User root
2217
2218Host %s
2219 User root
2220
2221Host %s
2222 User root
2223
2224Host %s
2225 User root
2226''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2227 else:
2228 usage()
2229 else:
2230 usage()
2231 else:
2232 # Do not enable debugging for config requests as it highly clutters the output
2233 if not is_text_request():
2234 cgitb.enable()
2235 response_headers, output = process_cgi_request()
2236 print_cgi_response(response_headers, output)
2237
2238def application(environ, start_response):
2239 status = '200 OK'
2240 response_headers, output = process_cgi_request(environ)
2241 start_response(status, response_headers)
2242
2243 # Debugging only
2244 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2245 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2246 return [output]
2247
2248if __name__ == "__main__":
2249 main()
Note: See TracBrowser for help on using the repository browser.