source: genesis/tools/gformat.py@ 13676

Last change on this file since 13676 was 13676, checked in by rick, 6 years ago

Quick and dirty hack to support VLANs

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 74.0 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13676 2016-12-24 15:35:35Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = []
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126def clear_cache():
127 ''' Poor mans cache implementation '''
128 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
129 datadump_cache = {}
130 interface_list_cache = {}
131 rc_conf_local_cache = {}
132 ileiden_proxies = []
133 normal_proxies = []
134 nameservers_cache = []
135 relations_cache = None
136
137NO_DHCP = 0
138DHCP_CLIENT = 10
139DHCP_SERVER = 20
140def dhcp_type(item):
141 if not item.has_key('dhcp'):
142 return NO_DHCP
143 elif not item['dhcp']:
144 return NO_DHCP
145 elif item['dhcp'].lower() == 'client':
146 return DHCP_CLIENT
147 else:
148 # Validation Checks
149 begin,end = map(int,item['dhcp'].split('-'))
150 if begin >= end:
151 raise ValueError("DHCP Start >= DHCP End")
152 return DHCP_SERVER
153
154def etrs2rd(lat, lon):
155 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
156 p2 = pyproj.Proj(init='EPSG:28992')
157 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
158 return (RDx, RDy)
159
160def rd2etrs(RDx, RDy):
161 p1 = pyproj.Proj(init='EPSG:28992')
162 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
163 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
164 return (lat, lon)
165
166def get_yaml(item,add_version_info=True):
167 try:
168 """ Get configuration yaml for 'item'"""
169 if datadump_cache.has_key(item):
170 return datadump_cache[item].copy()
171
172 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
173
174 # Default values
175 datadump = {
176 'autogen_revision' : 'NOTFOUND',
177 'autogen_gfile' : gfile,
178 'service_proxy_ileiden' : False,
179 }
180 f = open(gfile, 'r')
181 datadump.update(yaml.load(f,Loader=Loader))
182 if datadump['nodetype'] == 'Hybrid':
183 # Some values are defined implicitly
184 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
185 datadump['service_incoming_rdr'] = True
186 # Use some boring defaults
187 defaults = {
188 'service_proxy_normal' : False,
189 'service_accesspoint' : True,
190 'service_incoming_rdr' : False,
191 'service_concentrator' : False,
192 'monitoring_group' : 'wleiden',
193 }
194 for (key,value) in defaults.iteritems():
195 if not datadump.has_key(key):
196 datadump[key] = value
197 f.close()
198
199 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
200 if add_version_info:
201 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
202 lines = p.communicate()[0].split('\n')
203 if p.returncode == 0:
204 for line in lines:
205 if line:
206 (key, value) = line.split(': ')
207 datadump["autogen_" + key.lower().replace(' ','_')] = value
208
209 # Preformat certain needed variables for formatting and push those into special object
210 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
211
212 wlan_count=0
213 try:
214 for key in get_interface_keys(datadump, True):
215 datadump[key]['autogen_ifbase'] = key.split('_')[1]
216 datadump[key]['autogen_vlan'] = False
217
218 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
219 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
220
221 if datadump[key].has_key('ip'):
222 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
223
224 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
225 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
226 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
227 wlan_count += 1
228 else:
229 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
230 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
231 datadump[key]['autogen_vlan'] = key.split('_')[2]
232 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
233 else:
234 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
235
236 except Exception:
237 print "# Error while processing interface %s" % key
238 raise
239
240 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
241 if dhcp_type(datadump[key]) == DHCP_SERVER]
242
243 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
244 datadump['autogen_item'] = item
245
246 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
247 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
248 datadump_cache[item] = datadump.copy()
249 except Exception:
250 print "# Error while processing %s" % item
251 raise
252 return datadump
253
254
255def store_yaml(datadump, header=False):
256 """ Store configuration yaml for 'item'"""
257 item = datadump['autogen_item']
258 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
259
260 output = generate_wleiden_yaml(datadump, header)
261
262 f = open(gfile, 'w')
263 f.write(output)
264 f.close()
265
266
267def network(ip):
268 addr, mask = ip.split('/')
269 # Not parsing of these folks please
270 addr = parseaddr(addr)
271 mask = int(mask)
272 network = addr & ~((1 << (32 - mask)) - 1)
273 return network
274
275
276
277def make_relations():
278 """ Process _ALL_ yaml files to get connection relations """
279 global relations_cache
280
281 if relations_cache:
282 return relations_cache
283
284 errors = []
285 poel = defaultdict(list)
286
287 for host in get_hostlist():
288 datadump = get_yaml(host)
289 try:
290 for iface_key in get_interface_keys(datadump):
291 net_addr = network(datadump[iface_key]['ip'])
292 poel[net_addr] += [(host,datadump[iface_key].copy())]
293 except (KeyError, ValueError), e:
294 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
295 continue
296
297 relations_cache = (poel, errors)
298 return relations_cache
299
300
301
302def valid_addr(addr):
303 """ Show which address is valid in which are not """
304 return str(addr).startswith('172.')
305
306def get_hostlist():
307 """ Combined hosts and proxy list"""
308 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
309
310def angle_between_points(lat1,lat2,long1,long2):
311 """
312 Return Angle in radians between two GPS coordinates
313 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
314 """
315 dy = lat2 - lat1
316 dx = math.cos(lat1)*(long2 - long1)
317 angle = math.atan2(dy,dx)
318 return angle
319
320
321
322def angle_to_cd(angle):
323 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
324
325 # For easy conversion get positive degree
326 degrees = math.degrees(angle)
327 abs_degrees = 360 + degrees if degrees < 0 else degrees
328
329 # Numbers can be confusing calculate from the 4 main directions
330 p = 22.5
331 if abs_degrees < p:
332 cd = "n"
333 elif abs_degrees < (90 - p):
334 cd = "no"
335 elif abs_degrees < (90 + p):
336 cd = "o"
337 elif abs_degrees < (180 - p):
338 cd = "zo"
339 elif abs_degrees < (180 + p):
340 cd = "z"
341 elif abs_degrees < (270 - p):
342 cd = "zw"
343 elif abs_degrees < (270 + p):
344 cd = "w"
345 elif abs_degrees < (360 - p):
346 cd = "nw"
347 else:
348 cd = "n"
349 return cd
350
351
352
353def cd_between_hosts(hostA, hostB, datadumps):
354 # Using RDNAP coordinates
355 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
356 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
357 return angle_to_cd(math.atan2(dx,dy))
358
359 # GPS coordinates seems to fail somehow
360 #latA = float(datadumps[hostA]['latitude'])
361 #latB = float(datadumps[hostB]['latitude'])
362 #lonA = float(datadumps[hostA]['longitude'])
363 #lonB = float(datadumps[hostB]['longitude'])
364 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
365
366
367def generate_title(nodelist):
368 """ Main overview page """
369 items = {'root' : "." }
370 def fl(spaces, line):
371 return (' ' * spaces) + line + '\n'
372
373 output = """
374<html>
375 <head>
376 <title>Wireless leiden Configurator - GFormat</title>
377 <style type="text/css">
378 th {background-color: #999999}
379 tr:nth-child(odd) {background-color: #cccccc}
380 tr:nth-child(even) {background-color: #ffffff}
381 th, td {padding: 0.1em 1em}
382 </style>
383 </head>
384 <body>
385 <center>
386 <form type="GET" action="%(root)s">
387 <input type="hidden" name="action" value="update">
388 <input type="submit" value="Update Configuration Database (SVN)">
389 </form>
390 <table>
391 <caption><h3>Wireless Leiden Configurator</h3></caption>
392 """ % items
393
394 for node in nodelist:
395 items['node'] = node
396 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
397 for config in files:
398 items['config'] = config
399 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
400 output += fl(5, "</tr>")
401 output += """
402 </table>
403 <hr />
404 <em>%s</em>
405 </center>
406 </body>
407</html>
408 """ % __version__
409
410 return output
411
412
413
414def generate_node(node):
415 """ Print overview of all files available for node """
416 return "\n".join(files)
417
418def generate_node_overview(host):
419 """ Print overview of all files available for node """
420 datadump = get_yaml(host)
421 params = { 'host' : host }
422 output = "<em><a href='..'>Back to overview</a></em><hr />"
423 output += "<h2>Available files:</h2><ul>"
424 for cf in files:
425 params['cf'] = cf
426 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
427 output += "</ul>"
428
429 # Generate and connection listing
430 output += "<h2>Connected To:</h2><ul>"
431 (poel, errors) = make_relations()
432 for network, hosts in poel.iteritems():
433 if host in [x[0] for x in hosts]:
434 if len(hosts) == 1:
435 # Single not connected interface
436 continue
437 for remote,ifacedump in hosts:
438 if remote == host:
439 # This side of the interface
440 continue
441 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
442 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
443 output += "</ul>"
444 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
445
446 output += "<hr /><em><a href='..'>Back to overview</a></em>"
447 return output
448
449
450def generate_header(datadump, ctag="#"):
451 return """\
452%(ctag)s
453%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
454%(ctag)s
455""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
456
457
458
459def parseaddr(s):
460 """ Process IPv4 CIDR notation addr to a (binary) number """
461 f = s.split('.')
462 return (long(f[0]) << 24L) + \
463 (long(f[1]) << 16L) + \
464 (long(f[2]) << 8L) + \
465 long(f[3])
466
467
468
469def showaddr(a):
470 """ Display IPv4 addr in (dotted) CIDR notation """
471 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
472
473
474def is_member(ip, mask, canidate):
475 """ Return True if canidate is part of ip/mask block"""
476 ip_addr = parseaddr(ip)
477 ip_canidate = parseaddr(canidate)
478 mask = int(mask)
479 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
480 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
481 return ip_addr == ip_canidate
482
483
484
485def cidr2netmask(netmask):
486 """ Given a 'netmask' return corresponding CIDR """
487 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
488
489def get_network(addr, mask):
490 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
491
492
493def generate_dhcpd_conf(datadump):
494 """ Generate config file '/usr/local/etc/dhcpd.conf """
495 output = generate_header(datadump)
496 output += Template("""\
497# option definitions common to all supported networks...
498option domain-name "dhcp.{{ autogen_fqdn }}";
499
500default-lease-time 600;
501max-lease-time 7200;
502
503# Use this to enble / disable dynamic dns updates globally.
504#ddns-update-style none;
505
506# If this DHCP server is the official DHCP server for the local
507# network, the authoritative directive should be uncommented.
508authoritative;
509
510# Use this to send dhcp log messages to a different log file (you also
511# have to hack syslog.conf to complete the redirection).
512log-facility local7;
513
514#
515# Interface definitions
516#
517\n\n""").render(datadump)
518
519
520 # TODO: Use textwrap.fill instead
521 def indent(text, count):
522 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
523
524 dhcp_out = defaultdict(list)
525 for iface_key in get_interface_keys(datadump):
526 ifname = datadump[iface_key]['autogen_ifbase']
527 if not datadump[iface_key].has_key('comment'):
528 datadump[iface_key]['comment'] = None
529
530 if not datadump[iface_key].has_key('ip'):
531 continue
532
533 dhcp_out[iface_key].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
534
535 (addr, mask) = datadump[iface_key]['ip'].split('/')
536 datadump[iface_key]['autogen_addr'] = addr
537 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
538 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
539
540 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
541 dhcp_out[iface_key].append(textwrap.dedent("""\
542 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
543 ### not autoritive
544 }
545 """ % datadump[iface_key]))
546 continue
547
548 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
549 dhcp_part = ".".join(addr.split('.')[0:3])
550 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
551 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
552
553 # Assume the first 10 IPs could be used for static entries
554 if 'no_portal' in datadump:
555 fixed = 5
556 for mac in datadump['no_portal']:
557 dhcp_out[iface_key].append(textwrap.dedent("""\
558 host fixed-%(ifname)s-%(fixed)s {
559 hardware ethernet %(mac)s;
560 fixed-address %(prefix)s.%(fixed)s;
561 }
562 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
563 fixed += 1
564
565 dhcp_out[iface_key].append(textwrap.dedent("""\
566 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
567 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
568 option routers %(autogen_addr)s;
569 option domain-name-servers %(autogen_addr)s;
570 }
571 """ % datadump[iface_key]))
572
573 for ifname,value in dhcp_out.iteritems():
574 if len(value) > 2:
575 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
576 else:
577 output += ''.join(value) + "\n\n"
578 return output
579
580
581
582def generate_dnsmasq_conf(datadump):
583 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
584 output = generate_header(datadump)
585 output += Template("""\
586# DHCP server options
587dhcp-authoritative
588dhcp-fqdn
589domain=dhcp.{{ autogen_fqdn }}
590domain-needed
591expand-hosts
592log-async=100
593
594# Low memory footprint
595cache-size=10000
596
597\n""").render(datadump)
598
599 for iface_key in get_interface_keys(datadump):
600 if not datadump[iface_key].has_key('comment'):
601 datadump[iface_key]['comment'] = None
602 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
603
604 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
605 output += "# not autoritive\n\n"
606 continue
607
608 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
609 (ip, cidr) = datadump[iface_key]['ip'].split('/')
610 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
611
612 dhcp_part = ".".join(ip.split('.')[0:3])
613 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
614 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
615 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
616
617 return output
618
619
620class AutoVivification(dict):
621 """Implementation of perl's autovivification feature."""
622 def __getitem__(self, item):
623 try:
624 return dict.__getitem__(self, item)
625 except KeyError:
626 value = self[item] = type(self)()
627 return value
628
629def make_interface_list(datadump):
630 if interface_list_cache.has_key(datadump['autogen_item']):
631 return (interface_list_cache[datadump['autogen_item']])
632 # lo0 configuration:
633 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
634 # - masterip is special as it needs to be assigned to at
635 # least one interface, so if not used assign to lo0
636 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
637 vlan_list = defaultdict(list)
638 bridge_list = defaultdict(list)
639 flags_if = AutoVivification()
640 dhclient_if = {'lo0' : False}
641
642 # XXX: Find some way of send this output nicely
643 output = ''
644
645 masterip_used = False
646 for iface_key in get_interface_keys(datadump):
647 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
648 masterip_used = True
649 break
650 if not masterip_used:
651 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
652
653 for iface_key in get_interface_keys(datadump):
654 ifacedump = datadump[iface_key]
655 ifname = ifacedump['autogen_ifname']
656
657 # If defined as vlan interface
658 if ifacedump['autogen_vlan']:
659 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
660
661 # If defined as bridge interface
662 if ifacedump['autogen_bridge_member']:
663 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
664
665 # Flag dhclient is possible
666 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
667 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
668
669 # Ethernet address
670 if ifacedump.has_key('ether'):
671 flags_if[ifname]['ether'] = ifacedump['ether']
672
673 # Add interface IP to list
674 if ifacedump.has_key('ip'):
675 item = (ifacedump['ip'], ifacedump['comment'])
676 if addrs_list.has_key(ifname):
677 addrs_list[ifname].append(item)
678 else:
679 addrs_list[ifname] = [item]
680
681 # Alias only needs IP assignment for now, this might change if we
682 # are going to use virtual accesspoints
683 if "alias" in iface_key:
684 continue
685
686 # XXX: Might want to deduct type directly from interface name
687 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
688 # Default to station (client) mode
689 ifacedump['autogen_wlanmode'] = "sta"
690 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
691 ifacedump['autogen_wlanmode'] = "ap"
692
693 if not ifacedump.has_key('channel'):
694 if ifacedump['type'] == '11a':
695 ifacedump['channel'] = 36
696 else:
697 ifacedump['channel'] = 1
698
699 # Allow special hacks at the back like wds and stuff
700 if not ifacedump.has_key('extra'):
701 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
702 else:
703 ifacedump['autogen_extra'] = ifacedump['extra']
704
705 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
706
707 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
708 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
709 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
710 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
711 output += "\n"
712
713 elif ifacedump['type'] in ['ethernet', 'eth']:
714 # No special config needed besides IP
715 pass
716 elif ifacedump['type'] in ['vlan']:
717 # VLAN member has no special configuration
718 pass
719 else:
720 assert False, "Unknown type " + ifacedump['type']
721
722 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
723 interface_list_cache[datadump['autogen_item']] = store
724 return(store)
725
726
727
728def generate_rc_conf_local(datadump):
729 """ Generate configuration file '/etc/rc.conf.local' """
730 item = datadump['autogen_item']
731 if rc_conf_local_cache.has_key(item):
732 return rc_conf_local_cache[item]
733
734 if not datadump.has_key('ileiden'):
735 datadump['autogen_ileiden_enable'] = False
736 else:
737 datadump['autogen_ileiden_enable'] = datadump['ileiden']
738
739 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
740
741 if not ileiden_proxies or not normal_proxies:
742 for host in get_hostlist():
743 hostdump = get_yaml(host)
744 if hostdump['status'] == 'up':
745 if hostdump['service_proxy_ileiden']:
746 ileiden_proxies.append(hostdump)
747 if hostdump['service_proxy_normal']:
748 normal_proxies.append(hostdump)
749
750 datadump['autogen_ileiden_proxies'] = ileiden_proxies
751 datadump['autogen_normal_proxies'] = normal_proxies
752 datadump['autogen_ileiden_proxies_ips'] = ','.join([x['masterip'] for x in ileiden_proxies])
753 datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
754 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
755 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
756 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
757 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
758
759 output = generate_header(datadump, "#");
760 output += render_template(datadump, """\
761hostname='{{ autogen_fqdn }}'
762location='{{ location }}'
763nodetype="{{ nodetype }}"
764
765#
766# Configured listings
767#
768captive_portal_whitelist=""
769{% if nodetype == "Proxy" %}
770#
771# Proxy Configuration
772#
773{% if gateway and service_proxy_ileiden -%}
774defaultrouter="{{ gateway }}"
775{% else -%}
776#defaultrouter="NOTSET"
777{% endif -%}
778internalif="{{ internalif }}"
779ileiden_enable="{{ autogen_ileiden_enable }}"
780gateway_enable="{{ autogen_ileiden_enable }}"
781pf_enable="yes"
782pf_rules="/etc/pf.conf"
783{% if autogen_ileiden_enable -%}
784pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
785lvrouted_enable="{{ autogen_ileiden_enable }}"
786lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
787{% else -%}
788pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
789{% endif -%}
790{% if internalroute -%}
791static_routes="wleiden"
792route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
793{% endif -%}
794
795{% elif nodetype == "Hybrid" %}
796 #
797 # Hybrid Configuration
798 #
799 list_ileiden_proxies="
800 {% for item in autogen_ileiden_proxies -%}
801 {{ "%-16s"|format(item.masterip) }} # {{ item.nodename }}
802 {% endfor -%}
803 "
804 list_normal_proxies="
805 {% for item in autogen_normal_proxies -%}
806 {{ "%-16s"|format(item.masterip) }} # {{ item.nodename }}
807 {% endfor -%}
808 "
809
810 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
811 externalif="{{ externalif|default('vr0', true) }}"
812 masterip="{{ masterip }}"
813
814 {% if gateway and service_proxy_ileiden %}
815 defaultrouter="{{ gateway }}"
816 {% else %}
817 #defaultrouter="NOTSET"
818 {% endif %}
819
820 #
821 # Defined services
822 #
823 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
824 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
825 service_accesspoint="{{ service_accesspoint|yesorno }}"
826 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
827 service_concentrator="{{ service_concentrator|yesorno }}"
828
829 {% if service_proxy_ileiden %}
830 pf_rules="/etc/pf.hybrid.conf"
831 {% if service_concentrator %}
832 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
833 {% else %}
834 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
835 {% endif %}
836 pf_flags="$pf_flags -D publicnat=80,443"
837 lvrouted_flags="$lvrouted_flags -g"
838 {% elif service_proxy_normal or service_incoming_rdr %}
839 pf_rules="/etc/pf.hybrid.conf"
840 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
841 pf_flags="$pf_flags -D publicnat=0"
842 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
843 named_setfib="1"
844 tinyproxy_setfib="1"
845 dnsmasq_setfib="1"
846 sshd_setfib="1"
847 {% else %}
848 named_auto_forward_only="YES"
849 pf_rules="/etc/pf.node.conf"
850 pf_flags=""
851 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
852 {% endif %}
853 {% if service_concentrator %}
854 # Do mind installing certificates is NOT done automatically for security reasons
855 openvpn_enable="YES"
856 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
857 {% endif %}
858
859 {% if service_proxy_normal %}
860 tinyproxy_enable="yes"
861 {% else %}
862 pen_wrapper_enable="yes"
863 {% endif %}
864
865 {% if service_accesspoint %}
866 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
867 {% endif %}
868
869 {% if board == "ALIX2" %}
870 #
871 # ''Fat'' configuration, board has 256MB RAM
872 #
873 dnsmasq_enable="NO"
874 named_enable="YES"
875 {% if autogen_dhcp_interfaces -%}
876 dhcpd_enable="YES"
877 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
878 {% endif -%}
879 {% elif board == "apu1d" %}
880 #
881 # ''Fat'' configuration, board has 1024MB RAM
882 #
883 dnsmasq_enable="NO"
884 local_unbound_enable="YES"
885 {% if autogen_dhcp_interfaces -%}
886 dhcpd_enable="YES"
887 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
888 {% endif -%}
889 {% endif -%}
890{% endif %}
891
892#
893# Script variables
894#
895attached_devices="{{ autogen_attached_devices|join(' ') }}"
896neighbours="{{ autogen_neighbours|join(' ') }}"
897
898
899#
900# Interface definitions
901#\n
902""")
903
904 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
905 for iface, vlans in sorted(vlan_list.items()):
906 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
907
908 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
909 for iface in sorted(vlan_list.keys()):
910 if not iface in addrs_list.keys():
911 output += "ifconfig_%s='up'\n" % iface
912
913 output += "\n"
914
915 # Bridge configuration:
916 if bridge_list.keys():
917 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
918
919 for iface in bridge_list.keys():
920 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
921
922 # Bridge member interfaces not containing a configuration should be marked active explcitly.
923 for _,members in bridge_list.items():
924 for iface in members:
925 if not iface in addrs_list.keys():
926 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
927
928 output += "\n"
929
930 # Details like SSID
931 if extra_ouput:
932 output += extra_ouput.strip() + "\n"
933
934 # Print IP address which needs to be assigned over here
935 output += "\n"
936 for iface,addrs in sorted(addrs_list.iteritems()):
937 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
938 output += "# %s || %s || %s\n" % (iface, addr, comment)
939
940 # Write DHCLIENT entry
941 if iface in dhclient_if and dhclient_if[iface]:
942 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
943 continue
944
945 # Make sure the external address is always first as this is needed in the
946 # firewall setup
947 addrs = sorted(
948 [x for x in addrs if not '0.0.0.0' in x[0]],
949 key=lambda x: x[0].split('.')[0],
950 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
951 )
952
953 idx_offset = 0
954 # Set MAC is required
955 if flags_if[iface].has_key('ether'):
956 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
957 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
958 idx_offset += 1
959 else:
960 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
961
962 for idx, addr in enumerate(addrs[1:]):
963 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
964
965 output += "\n"
966
967 rc_conf_local_cache[datadump['autogen_item']] = output
968 return output
969
970
971
972def get_all_configs():
973 """ Get dict with key 'host' with all configs present """
974 configs = dict()
975 for host in get_hostlist():
976 datadump = get_yaml(host)
977 configs[host] = datadump
978 return configs
979
980
981def get_interface_keys(config, extra=False):
982 """ Quick hack to get all interface keys, later stage convert this to a iterator """
983 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
984 if extra == False:
985 return filter(lambda x: not "extra" in x, elems)
986 else:
987 return elems
988
989
990def get_used_ips(configs):
991 """ Return array of all IPs used in config files"""
992 ip_list = []
993 for config in configs:
994 ip_list.append(config['masterip'])
995 for iface_key in get_interface_keys(config, True):
996 l = config[iface_key]['ip']
997 addr, mask = l.split('/')
998 # Special case do not process
999 if valid_addr(addr):
1000 ip_list.append(addr)
1001 else:
1002 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1003 return sorted(ip_list)
1004
1005
1006
1007def get_nameservers(max_servers=None):
1008 if nameservers_cache:
1009 return nameservers_cache[0:max_servers]
1010
1011 for host in get_hostlist():
1012 hostdump = get_yaml(host)
1013 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1014 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
1015
1016 return nameservers_cache[0:max_servers]
1017
1018
1019def get_neighbours(datadump):
1020 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1021
1022 (poel, errors) = make_relations()
1023 table = []
1024 for iface,addrs in sorted(addrs_list.iteritems()):
1025 if iface in ['lo0']:
1026 continue
1027
1028 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1029 if not addr.startswith('172.'):
1030 # Avoid listing internet connections as pool
1031 continue
1032 for neighbour in poel[network(addr)]:
1033 if neighbour[0] != datadump['autogen_item']:
1034 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1035 return table
1036
1037
1038def get_attached_devices(datadump, url=False):
1039 table = []
1040 for iface_key in get_interface_keys(datadump, True):
1041 ifacedump = datadump[iface_key]
1042
1043 if not ifacedump.has_key('ns_ip'):
1044 continue
1045
1046 x_ip = ifacedump['ns_ip'].split('/')[0]
1047
1048 if 'mode' in ifacedump:
1049 x_mode = ifacedump['mode']
1050 else:
1051 x_mode = 'unknown'
1052
1053 if 'bridge_type' in ifacedump:
1054 device_type = ifacedump['bridge_type']
1055 else:
1056 device_type = 'Unknown'
1057
1058 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1059 return table
1060
1061
1062def generate_resolv_conf(datadump):
1063 """ Generate configuration file '/etc/resolv.conf' """
1064 # XXX: This should properly going to be an datastructure soon
1065 datadump['autogen_header'] = generate_header(datadump, "#")
1066 datadump['autogen_edge_nameservers'] = ''
1067
1068
1069 for masterip,realname in get_nameservers():
1070 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1071
1072 return Template("""\
1073{{ autogen_header }}
1074search wleiden.net
1075
1076# Try local (cache) first
1077nameserver 127.0.0.1
1078
1079{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1080nameserver 8.8.8.8 # Google Public NameServer
1081nameserver 4.2.2.1 # Level3 Public NameServer
1082{% else -%}
1083# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1084{{ autogen_edge_nameservers }}
1085{% endif -%}
1086""").render(datadump)
1087
1088
1089
1090def generate_ntp_conf(datadump):
1091 """ Generate configuration file '/etc/ntp.conf' """
1092 # XXX: This should properly going to be an datastructure soon
1093
1094 datadump['autogen_header'] = generate_header(datadump, "#")
1095 datadump['autogen_ntp_servers'] = ''
1096 for host in get_hostlist():
1097 hostdump = get_yaml(host)
1098 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1099 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1100
1101 return Template("""\
1102{{ autogen_header }}
1103
1104{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1105# Machine hooked to internet.
1106server 0.nl.pool.ntp.org iburst maxpoll 9
1107server 1.nl.pool.ntp.org iburst maxpoll 9
1108server 2.nl.pool.ntp.org iburst maxpoll 9
1109server 3.nl.pool.ntp.org iburst maxpoll 9
1110{% else -%}
1111# Local Wireless Leiden NTP Servers.
1112server 0.pool.ntp.wleiden.net iburst maxpoll 9
1113server 1.pool.ntp.wleiden.net iburst maxpoll 9
1114server 2.pool.ntp.wleiden.net iburst maxpoll 9
1115server 3.pool.ntp.wleiden.net iburst maxpoll 9
1116
1117# All the configured NTP servers
1118{{ autogen_ntp_servers }}
1119{% endif %}
1120
1121# If a server loses sync with all upstream servers, NTP clients
1122# no longer follow that server. The local clock can be configured
1123# to provide a time source when this happens, but it should usually
1124# be configured on just one server on a network. For more details see
1125# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1126# The use of Orphan Mode may be preferable.
1127#
1128server 127.127.1.0
1129fudge 127.127.1.0 stratum 10
1130""").render(datadump)
1131
1132
1133def generate_pf_hybrid_conf_local(datadump):
1134 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1135 datadump['autogen_header'] = generate_header(datadump, "#")
1136 return Template("""\
1137{{ autogen_header }}
1138
1139# Redirect some internal facing services outside (7)
1140# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1141{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1142rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1143{% endfor -%}
1144""").render(datadump)
1145
1146def generate_motd(datadump):
1147 """ Generate configuration file '/etc/motd' """
1148 output = Template("""\
1149FreeBSD run ``service motd onestart'' to make me look normal
1150
1151 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1152 Loc: {{ location }}
1153
1154Services:
1155{% if board == "ALIX2" -%}
1156{{" -"}} Core Node ({{ board }})
1157{% else -%}
1158{{" -"}} Hulp Node ({{ board }})
1159{% endif -%}
1160{% if service_proxy_normal -%}
1161{{" -"}} Normal Proxy
1162{% endif -%}
1163{% if service_proxy_ileiden -%}
1164{{" -"}} iLeiden Proxy
1165{% endif -%}
1166{% if service_incoming_rdr -%}
1167{{" -"}} Incoming port redirects
1168{% endif %}
1169Interlinks:\n
1170""").render(datadump)
1171
1172
1173 def make_table(table):
1174 if not table:
1175 return " - none\n"
1176 else:
1177 lines = ""
1178 col_width = [max(len(x) for x in col) for col in zip(*table)]
1179 for row in table:
1180 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1181 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1182 return lines
1183
1184 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1185 table = []
1186 for iface,addrs in sorted(addrs_list.iteritems()):
1187 if iface in ['lo0']:
1188 continue
1189 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1190 table.append((iface, addr, comment))
1191
1192 output += make_table(table)
1193 output += '\n'
1194 output += """\
1195Attached devices:
1196"""
1197 output += make_table(get_attached_devices(datadump, url=True))
1198 output += '\n'
1199 output += """\
1200Available neighbours:
1201"""
1202 output += make_table(get_neighbours(datadump))
1203
1204 return output
1205
1206
1207def format_yaml_value(value):
1208 """ Get yaml value in right syntax for outputting """
1209 if isinstance(value,str):
1210 output = '"%s"' % value
1211 else:
1212 output = value
1213 return output
1214
1215
1216
1217def format_wleiden_yaml(datadump):
1218 """ Special formatting to ensure it is editable"""
1219 output = "# Genesis config yaml style\n"
1220 output += "# vim:ts=2:et:sw=2:ai\n"
1221 output += "#\n"
1222 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1223 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1224 if key == 'rdr_rules':
1225 output += '%-10s:\n' % 'rdr_rules'
1226 for rdr_rule in datadump[key]:
1227 output += '- %s\n' % rdr_rule
1228 else:
1229 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1230
1231 output += "\n\n"
1232
1233 # Format (key, required)
1234 key_order = (
1235 ('comment', True),
1236 ('parent', False),
1237 ('ip', False),
1238 ('ether', False),
1239 ('desc', True),
1240 ('sdesc', True),
1241 ('mode', True),
1242 ('type', True),
1243 ('extra_type', False),
1244 ('channel', False),
1245 ('ssid', False),
1246 ('wlan_mac', False),
1247 ('dhcp', True),
1248 ('compass', False),
1249 ('distance', False),
1250 ('ns_ip', False),
1251 ('repeater_ip', False),
1252 ('bullet2_ip', False),
1253 ('ns_mac', False),
1254 ('bullet2_mac', False),
1255 ('ns_type', False),
1256 ('bridge_type', False),
1257 ('status', True),
1258 )
1259
1260 for iface_key in sorted(iface_keys):
1261 try:
1262 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1263 if remainder:
1264 raise KeyError("invalid keys: %s" % remainder)
1265
1266 output += "%s:\n" % iface_key
1267 for key,required in key_order:
1268 if datadump[iface_key].has_key(key):
1269 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1270 output += "\n\n"
1271 except Exception:
1272 print "# Error while processing interface %s" % iface_key
1273 raise
1274
1275 return output
1276
1277
1278
1279def generate_wleiden_yaml(datadump, header=True):
1280 """ Generate (petty) version of wleiden.yaml"""
1281 output = generate_header(datadump, "#") if header else ''
1282
1283 for key in datadump.keys():
1284 if key.startswith('autogen_'):
1285 del datadump[key]
1286 # Interface autogen cleanups
1287 elif type(datadump[key]) == dict:
1288 for key2 in datadump[key].keys():
1289 if key2.startswith('autogen_'):
1290 del datadump[key][key2]
1291
1292 output += format_wleiden_yaml(datadump)
1293 return output
1294
1295def generate_nanostation_config(datadump, iface, ns_type):
1296 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1297 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1298
1299 datadump.update(datadump['iface_%s' % iface])
1300
1301 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1302
1303def generate_yaml(datadump):
1304 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1305
1306
1307
1308def generate_config(node, config, datadump=None):
1309 """ Print configuration file 'config' of 'node' """
1310 output = ""
1311 try:
1312 # Load config file
1313 if datadump == None:
1314 datadump = get_yaml(node)
1315
1316 if config == 'wleiden.yaml':
1317 output += generate_wleiden_yaml(datadump)
1318 elif config == 'authorized_keys':
1319 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1320 output += f.read()
1321 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1322 # Fetch local keys if existing
1323 if os.path.exists(node_keys):
1324 output += open(node_keys, 'r').read()
1325 f.close()
1326 elif config == 'dnsmasq.conf':
1327 output += generate_dnsmasq_conf(datadump)
1328 elif config == 'dhcpd.conf':
1329 output += generate_dhcpd_conf(datadump)
1330 elif config == 'rc.conf.local':
1331 output += generate_rc_conf_local(datadump)
1332 elif config == 'resolv.conf':
1333 output += generate_resolv_conf(datadump)
1334 elif config == 'ntp.conf':
1335 output += generate_ntp_conf(datadump)
1336 elif config == 'motd':
1337 output += generate_motd(datadump)
1338 elif config == 'pf.hybrid.conf.local':
1339 output += generate_pf_hybrid_conf_local(datadump)
1340 elif config.startswith('vr'):
1341 interface, ns_type = config.strip('.yaml').split('-')
1342 output += generate_nanostation_config(datadump, interface, ns_type)
1343 else:
1344 assert False, "Config not found!"
1345 except IOError, e:
1346 output += "[ERROR] Config file not found"
1347 return output
1348
1349
1350
1351def process_cgi_request(environ=os.environ):
1352 """ When calling from CGI """
1353 response_headers = []
1354 content_type = 'text/plain'
1355
1356 # Update repository if requested
1357 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1358 if form and form.has_key("action") and "update" in form["action"]:
1359 output = "[INFO] Updating subverion, please wait...\n"
1360 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1361 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1362 output += "[INFO] All done, redirecting in 5 seconds"
1363 response_headers += [
1364 ('Refresh', '5; url=.'),
1365 ]
1366 reload_cache()
1367 else:
1368 base_uri = environ['PATH_INFO']
1369 uri = base_uri.strip('/').split('/')
1370
1371 output = "Template Holder"
1372 if base_uri.endswith('/create/network.kml'):
1373 content_type='application/vnd.google-earth.kml+xml'
1374 output = make_network_kml.make_graph()
1375 elif base_uri.endswith('/api/get/nodeplanner.json'):
1376 content_type='application/json'
1377 output = make_network_kml.make_nodeplanner_json()
1378 elif not uri[0]:
1379 if is_text_request(environ):
1380 output = '\n'.join(get_hostlist())
1381 else:
1382 content_type = 'text/html'
1383 output = generate_title(get_hostlist())
1384 elif len(uri) == 1:
1385 if is_text_request(environ):
1386 output = generate_node(uri[0])
1387 else:
1388 content_type = 'text/html'
1389 output = generate_node_overview(uri[0])
1390 elif len(uri) == 2:
1391 output = generate_config(uri[0], uri[1])
1392 else:
1393 assert False, "Invalid option"
1394
1395 # Return response
1396 response_headers += [
1397 ('Content-type', content_type),
1398 ('Content-Length', str(len(output))),
1399 ]
1400 return(response_headers, str(output))
1401
1402
1403def make_dns(output_dir = 'dns', external = False):
1404 items = dict()
1405
1406 # hostname is key, IP is value
1407 wleiden_zone = defaultdict(list)
1408 wleiden_cname = dict()
1409
1410 pool = dict()
1411 for node in get_hostlist():
1412 datadump = get_yaml(node)
1413
1414 fqdn = datadump['nodename']
1415
1416 if datadump.has_key('rdr_host'):
1417 remote_target = datadump['rdr_host']
1418 elif datadump.has_key('remote_access') and datadump['remote_access']:
1419 remote_target = datadump['remote_access'].split(':')[0]
1420 else:
1421 remote_target = None
1422
1423 if remote_target:
1424 try:
1425 parseaddr(remote_target)
1426 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1427 except (IndexError, ValueError):
1428 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1429
1430
1431 wleiden_zone[fqdn].append((datadump['masterip'], True))
1432
1433 # Hacking to get proper DHCP IPs and hostnames
1434 for iface_key in get_interface_keys(datadump):
1435 iface_name = iface_key.replace('_','-')
1436 if 'ip' in datadump[iface_key]:
1437 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1438 try:
1439 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1440 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1441 dhcp_part = ".".join(ip.split('.')[0:3])
1442 if ip != datadump['masterip']:
1443 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1444 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1445 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1446 except (AttributeError, ValueError, KeyError):
1447 # First push it into a pool, to indentify the counter-part later on
1448 addr = parseaddr(ip)
1449 cidr = int(cidr)
1450 addr = addr & ~((1 << (32 - cidr)) - 1)
1451 if pool.has_key(addr):
1452 pool[addr] += [(iface_name, fqdn, ip)]
1453 else:
1454 pool[addr] = [(iface_name, fqdn, ip)]
1455 continue
1456
1457
1458
1459 # WL uses an /29 to configure an interface. IP's are ordered like this:
1460 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1461
1462 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1463
1464 # Automatic naming convention of interlinks namely 2 + remote.lower()
1465 for (key,value) in pool.iteritems():
1466 # Make sure they are sorted from low-ip to high-ip
1467 value = sorted(value, key=lambda x: parseaddr(x[2]))
1468
1469 if len(value) == 1:
1470 (iface_name, fqdn, ip) = value[0]
1471 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1472
1473 # Device DNS names
1474 if 'cnode' in fqdn.lower():
1475 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1476 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1477
1478 elif len(value) == 2:
1479 (a_iface_name, a_fqdn, a_ip) = value[0]
1480 (b_iface_name, b_fqdn, b_ip) = value[1]
1481 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1482 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1483
1484 # Device DNS names
1485 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1486 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1487 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1488 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1489 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1490 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1491 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1492
1493 else:
1494 pool_members = [k[1] for k in value]
1495 for item in value:
1496 (iface_name, fqdn, ip) = item
1497 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1498
1499 # Include static DNS entries
1500 # XXX: Should they override the autogenerated results?
1501 # XXX: Convert input to yaml more useable.
1502 # Format:
1503 ##; this is a comment
1504 ## roomburgh=Roomburgh1
1505 ## apkerk1.Vosko=172.17.176.8 ;this as well
1506 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1507
1508 # Hack to allow special entries, for development
1509 wleiden_raw = {}
1510
1511 for line in dns_list:
1512 reverse = False
1513 k, items = line.items()[0]
1514 if type(items) == dict:
1515 if items.has_key('reverse'):
1516 reverse = items['reverse']
1517 items = items['a']
1518 else:
1519 items = items['cname']
1520 items = [items] if type(items) != list else items
1521 for item in items:
1522 if item.startswith('IN '):
1523 wleiden_raw[k] = item
1524 elif valid_addr(item):
1525 wleiden_zone[k].append((item, reverse))
1526 else:
1527 wleiden_cname[k] = item
1528
1529 # Hack to get dynamic pool listing
1530 def chunks(l, n):
1531 return [l[i:i+n] for i in range(0, len(l), n)]
1532
1533 ntp_servers = [x[0] for x in get_nameservers()]
1534 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1535 for ntp_server in chunk:
1536 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1537
1538 details = dict()
1539 # 24 updates a day allowed
1540 details['serial'] = time.strftime('%Y%m%d%H')
1541
1542 if external:
1543 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1544 else:
1545 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1546
1547 details['master'] = dns_masters[0]
1548 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1549
1550 dns_header = '''
1551$TTL 3h
1552%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1553 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1554
1555%(ns_servers)s
1556 \n'''
1557
1558
1559 if not os.path.isdir(output_dir):
1560 os.makedirs(output_dir)
1561 details['zone'] = 'wleiden.net'
1562 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1563 f.write(dns_header % details)
1564
1565 for host,items in wleiden_zone.iteritems():
1566 for ip,reverse in items:
1567 if ip not in ['0.0.0.0']:
1568 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1569 for source,dest in wleiden_cname.iteritems():
1570 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1571 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1572 for source, dest in wleiden_raw.iteritems():
1573 f.write("%s.wleiden.net. %s\n" % (source, dest))
1574 f.close()
1575
1576 # Create whole bunch of specific sub arpa zones. To keep it compliant
1577 for s in range(16,32):
1578 details['zone'] = '%i.172.in-addr.arpa' % s
1579 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1580 f.write(dns_header % details)
1581
1582 #XXX: Not effient, fix to proper data structure and do checks at other
1583 # stages
1584 for host,items in wleiden_zone.iteritems():
1585 for ip,reverse in items:
1586 if not reverse:
1587 continue
1588 if valid_addr(ip):
1589 if valid_addr(ip):
1590 if int(ip.split('.')[1]) == s:
1591 rev_ip = '.'.join(reversed(ip.split('.')))
1592 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1593 f.close()
1594
1595
1596def usage():
1597 print """Usage: %(prog)s <argument>
1598Argument:
1599\tcleanup = Cleanup all YAML files to specified format
1600\tstandalone [port] = Run configurator webserver [8000]
1601\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1602\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1603\tfull-export = Generate yaml export script for heatmap.
1604\tstatic [outputdir] = Generate all config files and store on disk
1605\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1606\ttest <node> [<file>] = Receive output for certain node [all files].
1607\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1608\tlist <status> <items> = List systems which have certain status
1609\tcreate network.kml = Create Network KML file for use in Google Earth
1610
1611Arguments:
1612\t<node> = NodeName (example: HybridRick)
1613\t<file> = %(files)s
1614\t<status> = all|up|down|planned
1615\t<items> = systems|nodes|proxies
1616
1617NOTE FOR DEVELOPERS; you can test your changes like this:
1618 BEFORE any changes in this code:
1619 $ ./gformat.py static /tmp/pre
1620 AFTER the changes:
1621 $ ./gformat.py static /tmp/post
1622 VIEW differences and VERIFY all are OK:
1623 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1624""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1625 exit(0)
1626
1627
1628def is_text_request(environ=os.environ):
1629 """ Find out whether we are calling from the CLI or any text based CLI utility """
1630 try:
1631 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1632 except KeyError:
1633 return True
1634
1635def switchFormat(setting):
1636 if setting:
1637 return "YES"
1638 else:
1639 return "NO"
1640
1641def rlinput(prompt, prefill=''):
1642 import readline
1643 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1644 try:
1645 return raw_input(prompt)
1646 finally:
1647 readline.set_startup_hook()
1648
1649def fix_conflict(left, right, default='i'):
1650 while True:
1651 print "## %-30s | %-30s" % (left, right)
1652 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1653 if not c:
1654 c = default
1655
1656 if c in ['l','1']:
1657 return left
1658 elif c in ['r','2']:
1659 return right
1660 elif c in ['e', '3']:
1661 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1662 elif c in ['i', '4']:
1663 return None
1664 else:
1665 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1666
1667
1668
1669def print_cgi_response(response_headers, output):
1670 """Could we not use some kind of wsgi wrapper to make this output?"""
1671 for header in response_headers:
1672 print "%s: %s" % header
1673 print
1674 print output
1675
1676
1677def fill_cache():
1678 ''' Poor man re-loading of few cache items (the slow ones) '''
1679 for host in get_hostlist():
1680 get_yaml(host)
1681
1682
1683def reload_cache():
1684 clear_cache()
1685 fill_cache()
1686
1687
1688def main():
1689 """Hard working sub"""
1690 # Allow easy hacking using the CLI
1691 if not os.environ.has_key('PATH_INFO'):
1692 if len(sys.argv) < 2:
1693 usage()
1694
1695 if sys.argv[1] == "standalone":
1696 import SocketServer
1697 import CGIHTTPServer
1698 # Hop to the right working directory.
1699 os.chdir(os.path.dirname(__file__))
1700 try:
1701 PORT = int(sys.argv[2])
1702 except (IndexError,ValueError):
1703 PORT = 8000
1704
1705 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1706 """ Serve this CGI from the root of the webserver """
1707 def is_cgi(self):
1708 if "favicon" in self.path:
1709 return False
1710
1711 self.cgi_info = (os.path.basename(__file__), self.path)
1712 self.path = ''
1713 return True
1714 handler = MyCGIHTTPRequestHandler
1715 SocketServer.TCPServer.allow_reuse_address = True
1716 httpd = SocketServer.TCPServer(("", PORT), handler)
1717 httpd.server_name = 'localhost'
1718 httpd.server_port = PORT
1719
1720 logger.info("serving at port %s", PORT)
1721 try:
1722 httpd.serve_forever()
1723 except KeyboardInterrupt:
1724 httpd.shutdown()
1725 logger.info("All done goodbye")
1726 elif sys.argv[1] == "test":
1727 # Basic argument validation
1728 try:
1729 node = sys.argv[2]
1730 except IndexError:
1731 print "Invalid argument"
1732 exit(1)
1733 except IOError as e:
1734 print e
1735 exit(1)
1736
1737 datadump = get_yaml(node)
1738
1739
1740 # Get files to generate
1741 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1742
1743 # Actual config generation
1744 for config in gen_files:
1745 logger.info("## Generating %s %s", node, config)
1746 print generate_config(node, config, datadump)
1747 elif sys.argv[1] == "test-cgi":
1748 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1749 os.environ['SCRIPT_NAME'] = __file__
1750 response_headers, output = process_cgi_request()
1751 print_cgi_response(response_headers, output)
1752 elif sys.argv[1] == "static":
1753 items = dict()
1754 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1755 for node in get_hostlist():
1756 items['node'] = node
1757 items['wdir'] = "%(output_dir)s/%(node)s" % items
1758 if not os.path.isdir(items['wdir']):
1759 os.makedirs(items['wdir'])
1760 datadump = get_yaml(node)
1761 for config in files:
1762 items['config'] = config
1763 logger.info("## Generating %(node)s %(config)s" % items)
1764 f = open("%(wdir)s/%(config)s" % items, "w")
1765 f.write(generate_config(node, config, datadump))
1766 f.close()
1767 elif sys.argv[1] == "wind-export":
1768 items = dict()
1769 for node in get_hostlist():
1770 datadump = get_yaml(node)
1771 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1772 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1773 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1774 VALUES (
1775 (SELECT id FROM users WHERE username = 'rvdzwet'),
1776 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1777 'Y');""" % datadump
1778 #for config in files:
1779 # items['config'] = config
1780 # print "## Generating %(node)s %(config)s" % items
1781 # f = open("%(wdir)s/%(config)s" % items, "w")
1782 # f.write(generate_config(node, config, datadump))
1783 # f.close()
1784 for node in get_hostlist():
1785 datadump = get_yaml(node)
1786 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1787 ifacedump = datadump[iface_key]
1788 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1789 ifacedump['nodename'] = datadump['nodename']
1790 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1791 ifacedump['channel'] = 0
1792 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1793 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1794 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1795 elif sys.argv[1] == "nagios-export":
1796 try:
1797 heavy_load = (sys.argv[2] == "--heavy-load")
1798 except IndexError:
1799 heavy_load = False
1800
1801 hostgroup_details = {
1802 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1803 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1804 'walphen' : 'Stichting Wireless Alphen',
1805 'westeinder' : 'Westeinder Plassen',
1806 }
1807
1808 # Convert IP to Host
1809 ip2host = {'root' : 'root'}
1810 for host in get_hostlist():
1811 datadump = get_yaml(host)
1812 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1813 for iface in get_interface_keys(datadump):
1814 if datadump[iface].has_key('autogen_gateway'):
1815 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1816
1817 # Find dependency tree based on output of lvrouted.mytree of nearest node
1818 parents = defaultdict(list)
1819 stack = ['root']
1820 prev_depth = 0
1821 for line in open('lvrouted.mytree').readlines():
1822 depth = line.count('\t')
1823 ip = line.strip().split()[0]
1824
1825 if prev_depth < depth:
1826 try:
1827 parents[ip2host[ip]].append(ip2host[stack[-1]])
1828 except KeyError as e:
1829 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1830 stack.append(ip)
1831 elif prev_depth > depth:
1832 stack = stack[:(depth - prev_depth)]
1833 elif prev_depth == depth:
1834 try:
1835 parents[ip2host[ip]].append(ip2host[stack[-1]])
1836 except KeyError as e:
1837 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1838
1839
1840 prev_depth = depth
1841 # Observe that some nodes has themself as parent or multiple parents
1842 # for now take only the first parent, other behaviour is yet to be explained
1843
1844
1845
1846 params = {
1847 'check_interval' : 5 if heavy_load else 120,
1848 'retry_interval' : 1 if heavy_load else 10,
1849 'max_check_attempts' : 10 if heavy_load else 6,
1850 'notification_interval': 120 if heavy_load else 240,
1851 }
1852
1853 print '''\
1854define host {
1855 name wleiden-node ; Default Node Template
1856 use generic-host ; Use the standard template as initial starting point
1857 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1858 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1859 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1860 notification_interval %(notification_interval)s
1861 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1862 check_command check-host-alive ; Default command to check FreeBSD hosts
1863 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1864}
1865
1866define service {
1867 name wleiden-service ; Default Service Template
1868 use generic-service ; Use the standard template as initial starting point
1869 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1870 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1871 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1872 notification_interval %(notification_interval)s
1873 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1874 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1875}
1876
1877# Please make sure to install:
1878# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1879#
1880# Recompile net-mgmt/nagios-plugins to support check_snmp
1881# make -C /usr/ports/net-mgmt/nagios-plugins
1882#
1883# Install net/bind-tools to allow v2/check_dns_wl to work:
1884# pkg install bind-tools
1885#
1886define command{
1887 command_name check_snmp_disk
1888 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1889}
1890
1891define command{
1892 command_name check_netsnmp_load
1893 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1894}
1895
1896define command{
1897 command_name check_netsnmp_proc
1898 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1899}
1900
1901define command{
1902 command_name check_by_ssh
1903 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1904}
1905
1906define command{
1907 command_name check_dns_wl
1908 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1909}
1910
1911define command{
1912 command_name check_snmp_uptime
1913 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1914}
1915
1916
1917# TDB: dhcp leases
1918# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1919
1920# TDB: internet status
1921# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1922
1923# TDB: Advanced local passive checks
1924# /usr/local/libexec/nagios/check_by_ssh
1925''' % params
1926
1927 print '''\
1928# Service Group, not displayed by default
1929define hostgroup {
1930 hostgroup_name srv_hybrid
1931 alias All Hybrid Nodes
1932 register 0
1933}
1934
1935define service {
1936 use wleiden-service
1937 hostgroup_name srv_hybrid
1938 service_description SSH
1939 check_command check_ssh
1940}
1941
1942define service {
1943 use wleiden-service,service-pnp
1944 hostgroup_name srv_hybrid
1945 service_description HTTP
1946 check_command check_http
1947}
1948
1949define service {
1950 use wleiden-service
1951 hostgroup_name srv_hybrid
1952 service_description DNS
1953 check_command check_dns_wl!"www.wirelessleiden.nl"
1954}
1955'''
1956
1957 if heavy_load:
1958 print '''\
1959define service {
1960 use wleiden-service
1961 hostgroup_name srv_hybrid
1962 service_description UPTIME
1963 check_command check_snmp_uptime
1964}
1965
1966#define service {
1967# use wleiden-service
1968# hostgroup_name srv_hybrid
1969# service_description NTP
1970# check_command check_ntp_peer
1971#}
1972
1973define service {
1974 use wleiden-service
1975 hostgroup_name srv_hybrid
1976 service_description LOAD
1977 check_command check_netsnmp_load
1978}
1979
1980define service {
1981 use wleiden-service
1982 hostgroup_name srv_hybrid
1983 service_description PROC
1984 check_command check_netsnmp_proc
1985}
1986
1987define service {
1988 use wleiden-service
1989 hostgroup_name srv_hybrid
1990 service_description DISK
1991 check_command check_snmp_disk
1992}
1993'''
1994 for node in get_hostlist():
1995 datadump = get_yaml(node)
1996 if not datadump['status'] == 'up':
1997 continue
1998 if not hostgroup_details.has_key(datadump['monitoring_group']):
1999 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2000 print '''\
2001define host {
2002 use wleiden-node,host-pnp
2003 contact_groups admins
2004 host_name %(autogen_fqdn)s
2005 address %(masterip)s
2006 hostgroups srv_hybrid,%(monitoring_group)s\
2007''' % datadump
2008 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2009 print '''\
2010 parents %(parents)s\
2011''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2012 print '''\
2013}
2014'''
2015
2016
2017 for name,alias in hostgroup_details.iteritems():
2018 print '''\
2019define hostgroup {
2020 hostgroup_name %s
2021 alias %s
2022} ''' % (name, alias)
2023
2024
2025 elif sys.argv[1] == "full-export":
2026 hosts = {}
2027 for node in get_hostlist():
2028 datadump = get_yaml(node)
2029 hosts[datadump['nodename']] = datadump
2030 print yaml.dump(hosts)
2031
2032 elif sys.argv[1] == "dns":
2033 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2034 elif sys.argv[1] == "cleanup":
2035 # First generate all datadumps
2036 datadumps = dict()
2037 ssid_to_node = dict()
2038 for host in get_hostlist():
2039 logger.info("# Processing: %s", host)
2040 # Set some boring default values
2041 datadump = { 'board' : 'UNKNOWN' }
2042 datadump.update(get_yaml(host))
2043 datadumps[datadump['nodename']] = datadump
2044
2045 (poel, errors) = make_relations()
2046 print "\n".join(["# WARNING: %s" % x for x in errors])
2047
2048 for host,datadump in datadumps.iteritems():
2049 try:
2050 # Convert all yes and no to boolean values
2051 def fix_boolean(dump):
2052 for key in dump.keys():
2053 if type(dump[key]) == dict:
2054 dump[key] = fix_boolean(dump[key])
2055 elif str(dump[key]).lower() in ["yes", "true"]:
2056 dump[key] = True
2057 elif str(dump[key]).lower() in ["no", "false"]:
2058 # Compass richting no (Noord Oost) is valid input
2059 if key != "compass": dump[key] = False
2060 return dump
2061 datadump = fix_boolean(datadump)
2062
2063 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2064 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2065 elif 'latitude' in datadump and 'longitude' in datadump:
2066 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2067
2068 if datadump['nodename'].startswith('Proxy'):
2069 datadump['nodename'] = datadump['nodename'].lower()
2070
2071 for iface_key in get_interface_keys(datadump):
2072 try:
2073 # All our normal wireless cards are normal APs now
2074 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2075 datadump[iface_key]['mode'] = 'ap'
2076 # Wireless Leiden SSID have an consistent lowercase/uppercase
2077 if datadump[iface_key].has_key('ssid'):
2078 ssid = datadump[iface_key]['ssid']
2079 prefix = 'ap-WirelessLeiden-'
2080 if ssid.lower().startswith(prefix.lower()):
2081 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2082 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2083 datadump[iface_key]['mode'] = 'autogen-FIXME'
2084 if not datadump[iface_key].has_key('comment'):
2085 datadump[iface_key]['comment'] = 'autogen-FIXME'
2086
2087 if datadump[iface_key].has_key('ns_mac'):
2088 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2089
2090 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2091 datadump[iface_key] = datadump[iface_key]['desc']
2092
2093 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2094 # due to better collision avoidance.
2095 if datadump[iface_key]['type'] == '11b':
2096 datadump[iface_key]['type'] = '11g'
2097
2098 # Setting 802.11g channels to de-facto standards, to avoid
2099 # un-detected sharing with other overlapping channels
2100 #
2101 # Technically we could also use channel 13 in NL, but this is not
2102 # recommended as foreign devices might not be able to select this
2103 # channel. Secondly using 1,5,9,13 instead is going to clash with
2104 # the de-facto usage of 1,6,11.
2105 #
2106 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2107 channels_at_2400Mhz = (1,6,11)
2108 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2109 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2110 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2111 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2112
2113 # Mandatory interface keys
2114 if not datadump[iface_key].has_key('status'):
2115 datadump[iface_key]['status'] = 'planned'
2116
2117 x = datadump[iface_key]['comment']
2118 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2119
2120 # Fixing bridge_type if none is found
2121 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2122 if not 'bridge_type' in datadump[iface_key]:
2123 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2124
2125 # Making sure description works
2126 if datadump[iface_key].has_key('desc'):
2127 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2128 del datadump[iface_key]['desc']
2129 else:
2130 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2131 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2132 if response:
2133 datadump[iface_key]['comment'] = response
2134 del datadump[iface_key]['desc']
2135
2136 # Check DHCP configuration
2137 dhcp_type(datadump[iface_key])
2138
2139 # Set the compass value based on the angle between the poels
2140 if datadump[iface_key].has_key('ns_ip'):
2141 my_pool = poel[network(datadump[iface_key]['ip'])]
2142 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2143 if remote_hosts:
2144 compass_target = remote_hosts[0]
2145 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2146
2147 # Monitoring Group default
2148 if not 'monitoring_group' in datadump:
2149 datadump['monitoring_group'] = 'wleiden'
2150
2151 except Exception:
2152 print "# Error while processing interface %s" % iface_key
2153 raise
2154 store_yaml(datadump)
2155 except Exception:
2156 print "# Error while processing %s" % host
2157 raise
2158 elif sys.argv[1] == "list":
2159 use_fqdn = False
2160 if len(sys.argv) < 4:
2161 usage()
2162 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2163 usage()
2164 if not sys.argv[3] in ["nodes","proxies","systems"]:
2165 usage()
2166
2167 if len(sys.argv) > 4:
2168 if sys.argv[4] == "fqdn":
2169 use_fqdn = True
2170 else:
2171 usage()
2172
2173 for system in get_hostlist():
2174 datadump = get_yaml(system)
2175 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2176 continue
2177
2178 output = datadump['autogen_fqdn'] if use_fqdn else system
2179 if sys.argv[2] == "all":
2180 print output
2181 elif datadump['status'] == sys.argv[2]:
2182 print output
2183 elif sys.argv[1] == "create":
2184 if sys.argv[2] == "network.kml":
2185 print make_network_kml.make_graph()
2186 elif sys.argv[2] == "host-ips.txt":
2187 for system in get_hostlist():
2188 datadump = get_yaml(system)
2189 ips = [datadump['masterip']]
2190 for ifkey in get_interface_keys(datadump):
2191 ips.append(datadump[ifkey]['ip'].split('/')[0])
2192 print system, ' '.join(ips)
2193 elif sys.argv[2] == "host-pos.txt":
2194 for system in get_hostlist():
2195 datadump = get_yaml(system)
2196 print system, datadump['rdnap_x'], datadump['rdnap_y']
2197 elif sys.argv[2] == 'ssh_config':
2198 print '''
2199Host *.wleiden.net
2200 User root
2201
2202Host 172.16.*.*
2203 User root
2204'''
2205 for system in get_hostlist():
2206 datadump = get_yaml(system)
2207 print '''\
2208Host %s
2209 User root
2210
2211Host %s
2212 User root
2213
2214Host %s
2215 User root
2216
2217Host %s
2218 User root
2219''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2220 else:
2221 usage()
2222 else:
2223 usage()
2224 else:
2225 # Do not enable debugging for config requests as it highly clutters the output
2226 if not is_text_request():
2227 cgitb.enable()
2228 response_headers, output = process_cgi_request()
2229 print_cgi_response(response_headers, output)
2230
2231def application(environ, start_response):
2232 status = '200 OK'
2233 response_headers, output = process_cgi_request(environ)
2234 start_response(status, response_headers)
2235
2236 # Debugging only
2237 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2238 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2239 return [output]
2240
2241if __name__ == "__main__":
2242 main()
Note: See TracBrowser for help on using the repository browser.