source: genesis/tools/gformat.py@ 13637

Last change on this file since 13637 was 13618, checked in by rick, 8 years ago

Re-do bridge configuration to support VLAN administation.

The current bridge implementation has limits when it comes to configuring
interfaces. Since a bridge member does not have an IP yet it has connected
interface (via an VLAN switch) thus this details needs to be stored.

It is also considered to be confusing, since bridge(4) interfaces do appea
even if you did not configure them, potentially changing configurations of all
current nodes as well, which makes testing and deployment an tricky business.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 73.9 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# yum install python-yaml pyproj proj-epsg python-jinja2
28#
29# Rick van der Zwet <info@rickvanderzwet.nl>
30#
31
32# Hack to make the script directory is also threated as a module search path.
33import sys
34import os
35sys.path.append(os.path.dirname(__file__))
36
37SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
38
39import argparse
40import cgi
41import cgitb
42import copy
43import glob
44import make_network_kml
45import math
46import pyproj
47import random
48import re
49import socket
50import string
51import subprocess
52import textwrap
53import time
54import urlparse
55
56from pprint import pprint
57from collections import defaultdict
58from sys import stderr
59try:
60 import yaml
61except ImportError, e:
62 print e
63 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
64 exit(1)
65
66try:
67 from yaml import CLoader as Loader
68 from yaml import CDumper as Dumper
69except ImportError:
70 from yaml import Loader, Dumper
71
72from jinja2 import Environment, Template
73def yesorno(value):
74 return "YES" if bool(value) else "NO"
75env = Environment()
76env.filters['yesorno'] = yesorno
77def render_template(datadump, template):
78 result = env.from_string(template).render(datadump)
79 # Make it look pretty to the naked eye, as jinja templates are not so
80 # friendly when it comes to whitespace formatting
81 ## Remove extra whitespace at end of line lstrip() style.
82 result = re.sub(r'\n[\ ]+','\n', result)
83 ## Include only a single newline between an definition and a comment
84 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
85 ## Remove extra newlines after single comment
86 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
87 return result
88
89import logging
90logging.basicConfig(format='# %(levelname)s: %(message)s' )
91logger = logging.getLogger()
92logger.setLevel(logging.DEBUG)
93
94
95if os.environ.has_key('CONFIGROOT'):
96 NODE_DIR = os.environ['CONFIGROOT']
97else:
98 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
99__version__ = '$Id: gformat.py 13618 2016-08-26 09:43:15Z rick $'
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = []
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126def clear_cache():
127 ''' Poor mans cache implementation '''
128 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
129 datadump_cache = {}
130 interface_list_cache = {}
131 rc_conf_local_cache = {}
132 ileiden_proxies = []
133 normal_proxies = []
134 nameservers_cache = []
135 relations_cache = None
136
137NO_DHCP = 0
138DHCP_CLIENT = 10
139DHCP_SERVER = 20
140def dhcp_type(item):
141 if not item.has_key('dhcp'):
142 return NO_DHCP
143 elif not item['dhcp']:
144 return NO_DHCP
145 elif item['dhcp'].lower() == 'client':
146 return DHCP_CLIENT
147 else:
148 # Validation Checks
149 begin,end = map(int,item['dhcp'].split('-'))
150 if begin >= end:
151 raise ValueError("DHCP Start >= DHCP End")
152 return DHCP_SERVER
153
154def etrs2rd(lat, lon):
155 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
156 p2 = pyproj.Proj(init='EPSG:28992')
157 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
158 return (RDx, RDy)
159
160def rd2etrs(RDx, RDy):
161 p1 = pyproj.Proj(init='EPSG:28992')
162 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
163 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
164 return (lat, lon)
165
166def get_yaml(item,add_version_info=True):
167 try:
168 """ Get configuration yaml for 'item'"""
169 if datadump_cache.has_key(item):
170 return datadump_cache[item].copy()
171
172 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
173
174 # Default values
175 datadump = {
176 'autogen_revision' : 'NOTFOUND',
177 'autogen_gfile' : gfile,
178 'service_proxy_ileiden' : False,
179 }
180 f = open(gfile, 'r')
181 datadump.update(yaml.load(f,Loader=Loader))
182 if datadump['nodetype'] == 'Hybrid':
183 # Some values are defined implicitly
184 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
185 datadump['service_incoming_rdr'] = True
186 # Use some boring defaults
187 defaults = {
188 'service_proxy_normal' : False,
189 'service_accesspoint' : True,
190 'service_incoming_rdr' : False,
191 'service_concentrator' : False,
192 'monitoring_group' : 'wleiden',
193 }
194 for (key,value) in defaults.iteritems():
195 if not datadump.has_key(key):
196 datadump[key] = value
197 f.close()
198
199 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
200 if add_version_info:
201 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
202 lines = p.communicate()[0].split('\n')
203 if p.returncode == 0:
204 for line in lines:
205 if line:
206 (key, value) = line.split(': ')
207 datadump["autogen_" + key.lower().replace(' ','_')] = value
208
209 # Preformat certain needed variables for formatting and push those into special object
210 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
211
212 wlan_count=0
213 try:
214 for key in get_interface_keys(datadump, True):
215 datadump[key]['autogen_ifbase'] = key.split('_')[1]
216 datadump[key]['autogen_vlan'] = False
217
218 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
219 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
220
221 if datadump[key].has_key('ip'):
222 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
223
224 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
225 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
226 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
227 wlan_count += 1
228 else:
229 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
230 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
231 datadump[key]['autogen_vlan'] = key.split('_')[2]
232 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
233 else:
234 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
235
236 except Exception:
237 print "# Error while processing interface %s" % key
238 raise
239
240 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] \
241 if dhcp_type(datadump[key]) == DHCP_SERVER]
242
243 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in dhcp_interfaces]
244 datadump['autogen_item'] = item
245
246 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
247 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
248 datadump_cache[item] = datadump.copy()
249 except Exception:
250 print "# Error while processing %s" % item
251 raise
252 return datadump
253
254
255def store_yaml(datadump, header=False):
256 """ Store configuration yaml for 'item'"""
257 item = datadump['autogen_item']
258 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
259
260 output = generate_wleiden_yaml(datadump, header)
261
262 f = open(gfile, 'w')
263 f.write(output)
264 f.close()
265
266
267def network(ip):
268 addr, mask = ip.split('/')
269 # Not parsing of these folks please
270 addr = parseaddr(addr)
271 mask = int(mask)
272 network = addr & ~((1 << (32 - mask)) - 1)
273 return network
274
275
276
277def make_relations():
278 """ Process _ALL_ yaml files to get connection relations """
279 global relations_cache
280
281 if relations_cache:
282 return relations_cache
283
284 errors = []
285 poel = defaultdict(list)
286
287 for host in get_hostlist():
288 datadump = get_yaml(host)
289 try:
290 for iface_key in get_interface_keys(datadump):
291 net_addr = network(datadump[iface_key]['ip'])
292 poel[net_addr] += [(host,datadump[iface_key].copy())]
293 except (KeyError, ValueError), e:
294 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
295 continue
296
297 relations_cache = (poel, errors)
298 return relations_cache
299
300
301
302def valid_addr(addr):
303 """ Show which address is valid in which are not """
304 return str(addr).startswith('172.')
305
306def get_hostlist():
307 """ Combined hosts and proxy list"""
308 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
309
310def angle_between_points(lat1,lat2,long1,long2):
311 """
312 Return Angle in radians between two GPS coordinates
313 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
314 """
315 dy = lat2 - lat1
316 dx = math.cos(lat1)*(long2 - long1)
317 angle = math.atan2(dy,dx)
318 return angle
319
320
321
322def angle_to_cd(angle):
323 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
324
325 # For easy conversion get positive degree
326 degrees = math.degrees(angle)
327 abs_degrees = 360 + degrees if degrees < 0 else degrees
328
329 # Numbers can be confusing calculate from the 4 main directions
330 p = 22.5
331 if abs_degrees < p:
332 cd = "n"
333 elif abs_degrees < (90 - p):
334 cd = "no"
335 elif abs_degrees < (90 + p):
336 cd = "o"
337 elif abs_degrees < (180 - p):
338 cd = "zo"
339 elif abs_degrees < (180 + p):
340 cd = "z"
341 elif abs_degrees < (270 - p):
342 cd = "zw"
343 elif abs_degrees < (270 + p):
344 cd = "w"
345 elif abs_degrees < (360 - p):
346 cd = "nw"
347 else:
348 cd = "n"
349 return cd
350
351
352
353def cd_between_hosts(hostA, hostB, datadumps):
354 # Using RDNAP coordinates
355 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
356 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
357 return angle_to_cd(math.atan2(dx,dy))
358
359 # GPS coordinates seems to fail somehow
360 #latA = float(datadumps[hostA]['latitude'])
361 #latB = float(datadumps[hostB]['latitude'])
362 #lonA = float(datadumps[hostA]['longitude'])
363 #lonB = float(datadumps[hostB]['longitude'])
364 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
365
366
367def generate_title(nodelist):
368 """ Main overview page """
369 items = {'root' : "." }
370 def fl(spaces, line):
371 return (' ' * spaces) + line + '\n'
372
373 output = """
374<html>
375 <head>
376 <title>Wireless leiden Configurator - GFormat</title>
377 <style type="text/css">
378 th {background-color: #999999}
379 tr:nth-child(odd) {background-color: #cccccc}
380 tr:nth-child(even) {background-color: #ffffff}
381 th, td {padding: 0.1em 1em}
382 </style>
383 </head>
384 <body>
385 <center>
386 <form type="GET" action="%(root)s">
387 <input type="hidden" name="action" value="update">
388 <input type="submit" value="Update Configuration Database (SVN)">
389 </form>
390 <table>
391 <caption><h3>Wireless Leiden Configurator</h3></caption>
392 """ % items
393
394 for node in nodelist:
395 items['node'] = node
396 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
397 for config in files:
398 items['config'] = config
399 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
400 output += fl(5, "</tr>")
401 output += """
402 </table>
403 <hr />
404 <em>%s</em>
405 </center>
406 </body>
407</html>
408 """ % __version__
409
410 return output
411
412
413
414def generate_node(node):
415 """ Print overview of all files available for node """
416 return "\n".join(files)
417
418def generate_node_overview(host):
419 """ Print overview of all files available for node """
420 datadump = get_yaml(host)
421 params = { 'host' : host }
422 output = "<em><a href='..'>Back to overview</a></em><hr />"
423 output += "<h2>Available files:</h2><ul>"
424 for cf in files:
425 params['cf'] = cf
426 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
427 output += "</ul>"
428
429 # Generate and connection listing
430 output += "<h2>Connected To:</h2><ul>"
431 (poel, errors) = make_relations()
432 for network, hosts in poel.iteritems():
433 if host in [x[0] for x in hosts]:
434 if len(hosts) == 1:
435 # Single not connected interface
436 continue
437 for remote,ifacedump in hosts:
438 if remote == host:
439 # This side of the interface
440 continue
441 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
442 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
443 output += "</ul>"
444 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
445
446 output += "<hr /><em><a href='..'>Back to overview</a></em>"
447 return output
448
449
450def generate_header(datadump, ctag="#"):
451 return """\
452%(ctag)s
453%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
454%(ctag)s
455""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
456
457
458
459def parseaddr(s):
460 """ Process IPv4 CIDR notation addr to a (binary) number """
461 f = s.split('.')
462 return (long(f[0]) << 24L) + \
463 (long(f[1]) << 16L) + \
464 (long(f[2]) << 8L) + \
465 long(f[3])
466
467
468
469def showaddr(a):
470 """ Display IPv4 addr in (dotted) CIDR notation """
471 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
472
473
474def is_member(ip, mask, canidate):
475 """ Return True if canidate is part of ip/mask block"""
476 ip_addr = parseaddr(ip)
477 ip_canidate = parseaddr(canidate)
478 mask = int(mask)
479 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
480 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
481 return ip_addr == ip_canidate
482
483
484
485def cidr2netmask(netmask):
486 """ Given a 'netmask' return corresponding CIDR """
487 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
488
489def get_network(addr, mask):
490 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
491
492
493def generate_dhcpd_conf(datadump):
494 """ Generate config file '/usr/local/etc/dhcpd.conf """
495 output = generate_header(datadump)
496 output += Template("""\
497# option definitions common to all supported networks...
498option domain-name "dhcp.{{ autogen_fqdn }}";
499
500default-lease-time 600;
501max-lease-time 7200;
502
503# Use this to enble / disable dynamic dns updates globally.
504#ddns-update-style none;
505
506# If this DHCP server is the official DHCP server for the local
507# network, the authoritative directive should be uncommented.
508authoritative;
509
510# Use this to send dhcp log messages to a different log file (you also
511# have to hack syslog.conf to complete the redirection).
512log-facility local7;
513
514#
515# Interface definitions
516#
517\n\n""").render(datadump)
518
519
520 # TODO: Use textwrap.fill instead
521 def indent(text, count):
522 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
523
524 dhcp_out = defaultdict(list)
525 for iface_key in get_interface_keys(datadump):
526 ifname = datadump[iface_key]['autogen_ifbase']
527 if not datadump[iface_key].has_key('comment'):
528 datadump[iface_key]['comment'] = None
529
530 if not datadump[iface_key].has_key('ip'):
531 continue
532
533 dhcp_out[iface_key].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
534
535 (addr, mask) = datadump[iface_key]['ip'].split('/')
536 datadump[iface_key]['autogen_addr'] = addr
537 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
538 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
539
540 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
541 dhcp_out[iface_key].append(textwrap.dedent("""\
542 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
543 ### not autoritive
544 }
545 """ % datadump[iface_key]))
546 continue
547
548 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
549 dhcp_part = ".".join(addr.split('.')[0:3])
550 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
551 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
552
553 # Assume the first 10 IPs could be used for static entries
554 if 'no_portal' in datadump:
555 fixed = 5
556 for mac in datadump['no_portal']:
557 dhcp_out[iface_key].append(textwrap.dedent("""\
558 host fixed-%(ifname)s-%(fixed)s {
559 hardware ethernet %(mac)s;
560 fixed-address %(prefix)s.%(fixed)s;
561 }
562 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
563 fixed += 1
564
565 dhcp_out[iface_key].append(textwrap.dedent("""\
566 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
567 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
568 option routers %(autogen_addr)s;
569 option domain-name-servers %(autogen_addr)s;
570 }
571 """ % datadump[iface_key]))
572
573 for ifname,value in dhcp_out.iteritems():
574 if len(value) > 2:
575 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
576 else:
577 output += ''.join(value) + "\n\n"
578 return output
579
580
581
582def generate_dnsmasq_conf(datadump):
583 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
584 output = generate_header(datadump)
585 output += Template("""\
586# DHCP server options
587dhcp-authoritative
588dhcp-fqdn
589domain=dhcp.{{ autogen_fqdn }}
590domain-needed
591expand-hosts
592log-async=100
593
594# Low memory footprint
595cache-size=10000
596
597\n""").render(datadump)
598
599 for iface_key in get_interface_keys(datadump):
600 if not datadump[iface_key].has_key('comment'):
601 datadump[iface_key]['comment'] = None
602 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
603
604 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
605 output += "# not autoritive\n\n"
606 continue
607
608 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
609 (ip, cidr) = datadump[iface_key]['ip'].split('/')
610 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
611
612 dhcp_part = ".".join(ip.split('.')[0:3])
613 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
614 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
615 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
616
617 return output
618
619
620class AutoVivification(dict):
621 """Implementation of perl's autovivification feature."""
622 def __getitem__(self, item):
623 try:
624 return dict.__getitem__(self, item)
625 except KeyError:
626 value = self[item] = type(self)()
627 return value
628
629def make_interface_list(datadump):
630 if interface_list_cache.has_key(datadump['autogen_item']):
631 return (interface_list_cache[datadump['autogen_item']])
632 # lo0 configuration:
633 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
634 # - masterip is special as it needs to be assigned to at
635 # least one interface, so if not used assign to lo0
636 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
637 vlan_list = defaultdict(list)
638 bridge_list = defaultdict(list)
639 flags_if = AutoVivification()
640 dhclient_if = {'lo0' : False}
641
642 # XXX: Find some way of send this output nicely
643 output = ''
644
645 masterip_used = False
646 for iface_key in get_interface_keys(datadump):
647 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
648 masterip_used = True
649 break
650 if not masterip_used:
651 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
652
653 for iface_key in get_interface_keys(datadump):
654 ifacedump = datadump[iface_key]
655 ifname = ifacedump['autogen_ifname']
656
657 # If defined as vlan interface
658 if ifacedump['autogen_vlan']:
659 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
660
661 # If defined as bridge interface
662 if ifacedump['autogen_bridge_member']:
663 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
664
665 # Flag dhclient is possible
666 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
667 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
668
669 # Ethernet address
670 if ifacedump.has_key('ether'):
671 flags_if[ifname]['ether'] = ifacedump['ether']
672
673 # Add interface IP to list
674 if ifacedump.has_key('ip'):
675 item = (ifacedump['ip'], ifacedump['comment'])
676 if addrs_list.has_key(ifname):
677 addrs_list[ifname].append(item)
678 else:
679 addrs_list[ifname] = [item]
680
681 # Alias only needs IP assignment for now, this might change if we
682 # are going to use virtual accesspoints
683 if "alias" in iface_key:
684 continue
685
686 # XXX: Might want to deduct type directly from interface name
687 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
688 # Default to station (client) mode
689 ifacedump['autogen_wlanmode'] = "sta"
690 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
691 ifacedump['autogen_wlanmode'] = "ap"
692
693 if not ifacedump.has_key('channel'):
694 if ifacedump['type'] == '11a':
695 ifacedump['channel'] = 36
696 else:
697 ifacedump['channel'] = 1
698
699 # Allow special hacks at the back like wds and stuff
700 if not ifacedump.has_key('extra'):
701 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
702 else:
703 ifacedump['autogen_extra'] = ifacedump['extra']
704
705 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
706
707 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
708 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
709 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
710 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
711 output += "\n"
712
713 elif ifacedump['type'] in ['ethernet', 'eth']:
714 # No special config needed besides IP
715 pass
716 elif ifacedump['type'] in ['vlan']:
717 # VLAN member has no special configuration
718 pass
719 else:
720 assert False, "Unknown type " + ifacedump['type']
721
722 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
723 interface_list_cache[datadump['autogen_item']] = store
724 return(store)
725
726
727
728def generate_rc_conf_local(datadump):
729 """ Generate configuration file '/etc/rc.conf.local' """
730 item = datadump['autogen_item']
731 if rc_conf_local_cache.has_key(item):
732 return rc_conf_local_cache[item]
733
734 if not datadump.has_key('ileiden'):
735 datadump['autogen_ileiden_enable'] = False
736 else:
737 datadump['autogen_ileiden_enable'] = datadump['ileiden']
738
739 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
740
741 if not ileiden_proxies or not normal_proxies:
742 for host in get_hostlist():
743 hostdump = get_yaml(host)
744 if hostdump['status'] == 'up':
745 if hostdump['service_proxy_ileiden']:
746 ileiden_proxies.append(hostdump)
747 if hostdump['service_proxy_normal']:
748 normal_proxies.append(hostdump)
749
750 datadump['autogen_ileiden_proxies'] = ileiden_proxies
751 datadump['autogen_normal_proxies'] = normal_proxies
752 datadump['autogen_ileiden_proxies_ips'] = ','.join([x['masterip'] for x in ileiden_proxies])
753 datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
754 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
755 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
756 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
757 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
758
759 output = generate_header(datadump, "#");
760 output += render_template(datadump, """\
761hostname='{{ autogen_fqdn }}'
762location='{{ location }}'
763nodetype="{{ nodetype }}"
764
765#
766# Configured listings
767#
768captive_portal_whitelist=""
769{% if nodetype == "Proxy" %}
770#
771# Proxy Configuration
772#
773{% if gateway and service_proxy_ileiden -%}
774defaultrouter="{{ gateway }}"
775{% else -%}
776#defaultrouter="NOTSET"
777{% endif -%}
778internalif="{{ internalif }}"
779ileiden_enable="{{ autogen_ileiden_enable }}"
780gateway_enable="{{ autogen_ileiden_enable }}"
781pf_enable="yes"
782pf_rules="/etc/pf.conf"
783{% if autogen_ileiden_enable -%}
784pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
785lvrouted_enable="{{ autogen_ileiden_enable }}"
786lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
787{% else -%}
788pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
789{% endif -%}
790{% if internalroute -%}
791static_routes="wleiden"
792route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
793{% endif -%}
794
795{% elif nodetype == "Hybrid" %}
796 #
797 # Hybrid Configuration
798 #
799 list_ileiden_proxies="
800 {% for item in autogen_ileiden_proxies -%}
801 {{ "%-16s"|format(item.masterip) }} # {{ item.nodename }}
802 {% endfor -%}
803 "
804 list_normal_proxies="
805 {% for item in autogen_normal_proxies -%}
806 {{ "%-16s"|format(item.masterip) }} # {{ item.nodename }}
807 {% endfor -%}
808 "
809
810 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
811 externalif="{{ externalif|default('vr0', true) }}"
812 masterip="{{ masterip }}"
813
814 {% if gateway and service_proxy_ileiden %}
815 defaultrouter="{{ gateway }}"
816 {% else %}
817 #defaultrouter="NOTSET"
818 {% endif %}
819
820 #
821 # Defined services
822 #
823 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
824 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
825 service_accesspoint="{{ service_accesspoint|yesorno }}"
826 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
827 service_concentrator="{{ service_concentrator|yesorno }}"
828
829 {% if service_proxy_ileiden %}
830 pf_rules="/etc/pf.hybrid.conf"
831 {% if service_concentrator %}
832 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
833 {% else %}
834 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
835 {% endif %}
836 pf_flags="$pf_flags -D publicnat=80,443"
837 lvrouted_flags="$lvrouted_flags -g"
838 {% elif service_proxy_normal or service_incoming_rdr %}
839 pf_rules="/etc/pf.hybrid.conf"
840 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
841 pf_flags="$pf_flags -D publicnat=0"
842 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
843 named_setfib="1"
844 tinyproxy_setfib="1"
845 dnsmasq_setfib="1"
846 sshd_setfib="1"
847 {% else %}
848 named_auto_forward_only="YES"
849 pf_rules="/etc/pf.node.conf"
850 pf_flags=""
851 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
852 {% endif %}
853 {% if service_concentrator %}
854 # Do mind installing certificates is NOT done automatically for security reasons
855 openvpn_enable="YES"
856 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
857 {% endif %}
858
859 {% if service_proxy_normal %}
860 tinyproxy_enable="yes"
861 {% else %}
862 pen_wrapper_enable="yes"
863 {% endif %}
864
865 {% if service_accesspoint %}
866 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
867 {% endif %}
868
869 {% if board == "ALIX2" %}
870 #
871 # ''Fat'' configuration, board has 256MB RAM
872 #
873 dnsmasq_enable="NO"
874 named_enable="YES"
875 {% if autogen_dhcp_interfaces -%}
876 dhcpd_enable="YES"
877 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
878 {% endif -%}
879 {% elif board == "apu1d" %}
880 #
881 # ''Fat'' configuration, board has 1024MB RAM
882 #
883 dnsmasq_enable="NO"
884 local_unbound_enable="YES"
885 {% if autogen_dhcp_interfaces -%}
886 dhcpd_enable="YES"
887 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
888 {% endif -%}
889 {% endif -%}
890{% endif %}
891
892#
893# Script variables
894#
895attached_devices="{{ autogen_attached_devices|join(' ') }}"
896neighbours="{{ autogen_neighbours|join(' ') }}"
897
898
899#
900# Interface definitions
901#\n
902""")
903
904 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
905 for iface, vlans in vlan_list.items():
906 output += 'vlans_%s="%s"\n' % (iface, ' '.join(vlans))
907
908 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
909 for iface in vlan_list.keys():
910 if not iface in addrs_list.keys():
911 output += "ifconfig_%s='up'\n" % iface
912
913 output += "\n"
914
915 # Bridge configuration:
916 if bridge_list.keys():
917 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
918
919 for iface in bridge_list.keys():
920 output += "create_args_%s='%s'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
921
922 # Bridge member interfaces not containing a configuration should be marked active explcitly.
923 for _,members in bridge_list.items():
924 for iface in members:
925 if not iface in addrs_list.keys():
926 output += "ifconfig_%s='up'\n" % iface
927
928 output += "\n"
929
930 # Details like SSID
931 if extra_ouput:
932 output += extra_ouput.strip() + "\n"
933
934 # Print IP address which needs to be assigned over here
935 output += "\n"
936 for iface,addrs in sorted(addrs_list.iteritems()):
937 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
938 output += "# %s || %s || %s\n" % (iface, addr, comment)
939
940 # Write DHCLIENT entry
941 if iface in dhclient_if and dhclient_if[iface]:
942 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
943 continue
944
945 # Make sure the external address is always first as this is needed in the
946 # firewall setup
947 addrs = sorted(
948 [x for x in addrs if not '0.0.0.0' in x[0]],
949 key=lambda x: x[0].split('.')[0],
950 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
951 )
952
953 idx_offset = 0
954 # Set MAC is required
955 if flags_if[iface].has_key('ether'):
956 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
957 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
958 idx_offset += 1
959 else:
960 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
961
962 for idx, addr in enumerate(addrs[1:]):
963 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
964
965 output += "\n"
966
967 rc_conf_local_cache[datadump['autogen_item']] = output
968 return output
969
970
971
972def get_all_configs():
973 """ Get dict with key 'host' with all configs present """
974 configs = dict()
975 for host in get_hostlist():
976 datadump = get_yaml(host)
977 configs[host] = datadump
978 return configs
979
980
981def get_interface_keys(config, extra=False):
982 """ Quick hack to get all interface keys, later stage convert this to a iterator """
983 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
984 if extra == False:
985 return filter(lambda x: not "extra" in x, elems)
986 else:
987 return elems
988
989
990def get_used_ips(configs):
991 """ Return array of all IPs used in config files"""
992 ip_list = []
993 for config in configs:
994 ip_list.append(config['masterip'])
995 for iface_key in get_interface_keys(config, True):
996 l = config[iface_key]['ip']
997 addr, mask = l.split('/')
998 # Special case do not process
999 if valid_addr(addr):
1000 ip_list.append(addr)
1001 else:
1002 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1003 return sorted(ip_list)
1004
1005
1006
1007def get_nameservers(max_servers=None):
1008 if nameservers_cache:
1009 return nameservers_cache[0:max_servers]
1010
1011 for host in get_hostlist():
1012 hostdump = get_yaml(host)
1013 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1014 nameservers_cache.append((hostdump['masterip'], hostdump['nodename']))
1015
1016 return nameservers_cache[0:max_servers]
1017
1018
1019def get_neighbours(datadump):
1020 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1021
1022 (poel, errors) = make_relations()
1023 table = []
1024 for iface,addrs in sorted(addrs_list.iteritems()):
1025 if iface in ['lo0']:
1026 continue
1027
1028 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1029 if not addr.startswith('172.'):
1030 # Avoid listing internet connections as pool
1031 continue
1032 for neighbour in poel[network(addr)]:
1033 if neighbour[0] != datadump['autogen_item']:
1034 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1035 return table
1036
1037
1038def get_attached_devices(datadump, url=False):
1039 table = []
1040 for iface_key in get_interface_keys(datadump, True):
1041 ifacedump = datadump[iface_key]
1042
1043 if not ifacedump.has_key('ns_ip'):
1044 continue
1045
1046 x_ip = ifacedump['ns_ip'].split('/')[0]
1047
1048 if 'mode' in ifacedump:
1049 x_mode = ifacedump['mode']
1050 else:
1051 x_mode = 'unknown'
1052
1053 if 'bridge_type' in ifacedump:
1054 device_type = ifacedump['bridge_type']
1055 else:
1056 device_type = 'Unknown'
1057
1058 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1059 return table
1060
1061
1062def generate_resolv_conf(datadump):
1063 """ Generate configuration file '/etc/resolv.conf' """
1064 # XXX: This should properly going to be an datastructure soon
1065 datadump['autogen_header'] = generate_header(datadump, "#")
1066 datadump['autogen_edge_nameservers'] = ''
1067
1068
1069 for masterip,realname in get_nameservers():
1070 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1071
1072 return Template("""\
1073{{ autogen_header }}
1074search wleiden.net
1075
1076# Try local (cache) first
1077nameserver 127.0.0.1
1078
1079{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1080nameserver 8.8.8.8 # Google Public NameServer
1081nameserver 4.2.2.1 # Level3 Public NameServer
1082{% else -%}
1083# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1084{{ autogen_edge_nameservers }}
1085{% endif -%}
1086""").render(datadump)
1087
1088
1089
1090def generate_ntp_conf(datadump):
1091 """ Generate configuration file '/etc/ntp.conf' """
1092 # XXX: This should properly going to be an datastructure soon
1093
1094 datadump['autogen_header'] = generate_header(datadump, "#")
1095 datadump['autogen_ntp_servers'] = ''
1096 for host in get_hostlist():
1097 hostdump = get_yaml(host)
1098 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1099 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1100
1101 return Template("""\
1102{{ autogen_header }}
1103
1104{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1105# Machine hooked to internet.
1106server 0.nl.pool.ntp.org iburst maxpoll 9
1107server 1.nl.pool.ntp.org iburst maxpoll 9
1108server 2.nl.pool.ntp.org iburst maxpoll 9
1109server 3.nl.pool.ntp.org iburst maxpoll 9
1110{% else -%}
1111# Local Wireless Leiden NTP Servers.
1112server 0.pool.ntp.wleiden.net iburst maxpoll 9
1113server 1.pool.ntp.wleiden.net iburst maxpoll 9
1114server 2.pool.ntp.wleiden.net iburst maxpoll 9
1115server 3.pool.ntp.wleiden.net iburst maxpoll 9
1116
1117# All the configured NTP servers
1118{{ autogen_ntp_servers }}
1119{% endif %}
1120
1121# If a server loses sync with all upstream servers, NTP clients
1122# no longer follow that server. The local clock can be configured
1123# to provide a time source when this happens, but it should usually
1124# be configured on just one server on a network. For more details see
1125# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1126# The use of Orphan Mode may be preferable.
1127#
1128server 127.127.1.0
1129fudge 127.127.1.0 stratum 10
1130""").render(datadump)
1131
1132
1133def generate_pf_hybrid_conf_local(datadump):
1134 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1135 datadump['autogen_header'] = generate_header(datadump, "#")
1136 return Template("""\
1137{{ autogen_header }}
1138
1139# Redirect some internal facing services outside (7)
1140# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
1141{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
1142rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
1143{% endfor -%}
1144""").render(datadump)
1145
1146def generate_motd(datadump):
1147 """ Generate configuration file '/etc/motd' """
1148 output = Template("""\
1149FreeBSD run ``service motd onestart'' to make me look normal
1150
1151 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1152 Loc: {{ location }}
1153
1154Services:
1155{% if board == "ALIX2" -%}
1156{{" -"}} Core Node ({{ board }})
1157{% else -%}
1158{{" -"}} Hulp Node ({{ board }})
1159{% endif -%}
1160{% if service_proxy_normal -%}
1161{{" -"}} Normal Proxy
1162{% endif -%}
1163{% if service_proxy_ileiden -%}
1164{{" -"}} iLeiden Proxy
1165{% endif -%}
1166{% if service_incoming_rdr -%}
1167{{" -"}} Incoming port redirects
1168{% endif %}
1169Interlinks:\n
1170""").render(datadump)
1171
1172
1173 def make_table(table):
1174 if not table:
1175 return " - none\n"
1176 else:
1177 lines = ""
1178 col_width = [max(len(x) for x in col) for col in zip(*table)]
1179 for row in table:
1180 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1181 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1182 return lines
1183
1184 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1185 table = []
1186 for iface,addrs in sorted(addrs_list.iteritems()):
1187 if iface in ['lo0']:
1188 continue
1189 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1190 table.append((iface, addr, comment))
1191
1192 output += make_table(table)
1193 output += '\n'
1194 output += """\
1195Attached devices:
1196"""
1197 output += make_table(get_attached_devices(datadump, url=True))
1198 output += '\n'
1199 output += """\
1200Available neighbours:
1201"""
1202 output += make_table(get_neighbours(datadump))
1203
1204 return output
1205
1206
1207def format_yaml_value(value):
1208 """ Get yaml value in right syntax for outputting """
1209 if isinstance(value,str):
1210 output = '"%s"' % value
1211 else:
1212 output = value
1213 return output
1214
1215
1216
1217def format_wleiden_yaml(datadump):
1218 """ Special formatting to ensure it is editable"""
1219 output = "# Genesis config yaml style\n"
1220 output += "# vim:ts=2:et:sw=2:ai\n"
1221 output += "#\n"
1222 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1223 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1224 if key == 'rdr_rules':
1225 output += '%-10s:\n' % 'rdr_rules'
1226 for rdr_rule in datadump[key]:
1227 output += '- %s\n' % rdr_rule
1228 else:
1229 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1230
1231 output += "\n\n"
1232
1233 # Format (key, required)
1234 key_order = (
1235 ('comment', True),
1236 ('parent', False),
1237 ('ip', False),
1238 ('ether', False),
1239 ('desc', True),
1240 ('sdesc', True),
1241 ('mode', True),
1242 ('type', True),
1243 ('extra_type', False),
1244 ('channel', False),
1245 ('ssid', False),
1246 ('wlan_mac', False),
1247 ('dhcp', True),
1248 ('compass', False),
1249 ('distance', False),
1250 ('ns_ip', False),
1251 ('repeater_ip', False),
1252 ('bullet2_ip', False),
1253 ('ns_mac', False),
1254 ('bullet2_mac', False),
1255 ('ns_type', False),
1256 ('bridge_type', False),
1257 ('status', True),
1258 )
1259
1260 for iface_key in sorted(iface_keys):
1261 try:
1262 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1263 if remainder:
1264 raise KeyError("invalid keys: %s" % remainder)
1265
1266 output += "%s:\n" % iface_key
1267 for key,required in key_order:
1268 if datadump[iface_key].has_key(key):
1269 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1270 output += "\n\n"
1271 except Exception:
1272 print "# Error while processing interface %s" % iface_key
1273 raise
1274
1275 return output
1276
1277
1278
1279def generate_wleiden_yaml(datadump, header=True):
1280 """ Generate (petty) version of wleiden.yaml"""
1281 output = generate_header(datadump, "#") if header else ''
1282
1283 for key in datadump.keys():
1284 if key.startswith('autogen_'):
1285 del datadump[key]
1286 # Interface autogen cleanups
1287 elif type(datadump[key]) == dict:
1288 for key2 in datadump[key].keys():
1289 if key2.startswith('autogen_'):
1290 del datadump[key][key2]
1291
1292 output += format_wleiden_yaml(datadump)
1293 return output
1294
1295def generate_nanostation_config(datadump, iface, ns_type):
1296 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1297 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1298
1299 datadump.update(datadump['iface_%s' % iface])
1300
1301 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1302
1303def generate_yaml(datadump):
1304 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1305
1306
1307
1308def generate_config(node, config, datadump=None):
1309 """ Print configuration file 'config' of 'node' """
1310 output = ""
1311 try:
1312 # Load config file
1313 if datadump == None:
1314 datadump = get_yaml(node)
1315
1316 if config == 'wleiden.yaml':
1317 output += generate_wleiden_yaml(datadump)
1318 elif config == 'authorized_keys':
1319 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1320 output += f.read()
1321 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1322 # Fetch local keys if existing
1323 if os.path.exists(node_keys):
1324 output += open(node_keys, 'r').read()
1325 f.close()
1326 elif config == 'dnsmasq.conf':
1327 output += generate_dnsmasq_conf(datadump)
1328 elif config == 'dhcpd.conf':
1329 output += generate_dhcpd_conf(datadump)
1330 elif config == 'rc.conf.local':
1331 output += generate_rc_conf_local(datadump)
1332 elif config == 'resolv.conf':
1333 output += generate_resolv_conf(datadump)
1334 elif config == 'ntp.conf':
1335 output += generate_ntp_conf(datadump)
1336 elif config == 'motd':
1337 output += generate_motd(datadump)
1338 elif config == 'pf.hybrid.conf.local':
1339 output += generate_pf_hybrid_conf_local(datadump)
1340 elif config.startswith('vr'):
1341 interface, ns_type = config.strip('.yaml').split('-')
1342 output += generate_nanostation_config(datadump, interface, ns_type)
1343 else:
1344 assert False, "Config not found!"
1345 except IOError, e:
1346 output += "[ERROR] Config file not found"
1347 return output
1348
1349
1350
1351def process_cgi_request(environ=os.environ):
1352 """ When calling from CGI """
1353 response_headers = []
1354 content_type = 'text/plain'
1355
1356 # Update repository if requested
1357 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1358 if form and form.has_key("action") and "update" in form["action"]:
1359 output = "[INFO] Updating subverion, please wait...\n"
1360 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1361 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1362 output += "[INFO] All done, redirecting in 5 seconds"
1363 response_headers += [
1364 ('Refresh', '5; url=.'),
1365 ]
1366 reload_cache()
1367 else:
1368 base_uri = environ['PATH_INFO']
1369 uri = base_uri.strip('/').split('/')
1370
1371 output = "Template Holder"
1372 if base_uri.endswith('/create/network.kml'):
1373 content_type='application/vnd.google-earth.kml+xml'
1374 output = make_network_kml.make_graph()
1375 elif base_uri.endswith('/api/get/nodeplanner.json'):
1376 content_type='application/json'
1377 output = make_network_kml.make_nodeplanner_json()
1378 elif not uri[0]:
1379 if is_text_request(environ):
1380 output = '\n'.join(get_hostlist())
1381 else:
1382 content_type = 'text/html'
1383 output = generate_title(get_hostlist())
1384 elif len(uri) == 1:
1385 if is_text_request(environ):
1386 output = generate_node(uri[0])
1387 else:
1388 content_type = 'text/html'
1389 output = generate_node_overview(uri[0])
1390 elif len(uri) == 2:
1391 output = generate_config(uri[0], uri[1])
1392 else:
1393 assert False, "Invalid option"
1394
1395 # Return response
1396 response_headers += [
1397 ('Content-type', content_type),
1398 ('Content-Length', str(len(output))),
1399 ]
1400 return(response_headers, str(output))
1401
1402
1403def make_dns(output_dir = 'dns', external = False):
1404 items = dict()
1405
1406 # hostname is key, IP is value
1407 wleiden_zone = defaultdict(list)
1408 wleiden_cname = dict()
1409
1410 pool = dict()
1411 for node in get_hostlist():
1412 datadump = get_yaml(node)
1413
1414 fqdn = datadump['nodename']
1415
1416 if datadump.has_key('rdr_host'):
1417 remote_target = datadump['rdr_host']
1418 elif datadump.has_key('remote_access') and datadump['remote_access']:
1419 remote_target = datadump['remote_access'].split(':')[0]
1420 else:
1421 remote_target = None
1422
1423 if remote_target:
1424 try:
1425 parseaddr(remote_target)
1426 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1427 except (IndexError, ValueError):
1428 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1429
1430
1431 wleiden_zone[fqdn].append((datadump['masterip'], True))
1432
1433 # Hacking to get proper DHCP IPs and hostnames
1434 for iface_key in get_interface_keys(datadump):
1435 iface_name = iface_key.replace('_','-')
1436 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1437 try:
1438 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1439 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1440 dhcp_part = ".".join(ip.split('.')[0:3])
1441 if ip != datadump['masterip']:
1442 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, False))
1443 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1444 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1445 except (AttributeError, ValueError, KeyError):
1446 # First push it into a pool, to indentify the counter-part later on
1447 addr = parseaddr(ip)
1448 cidr = int(cidr)
1449 addr = addr & ~((1 << (32 - cidr)) - 1)
1450 if pool.has_key(addr):
1451 pool[addr] += [(iface_name, fqdn, ip)]
1452 else:
1453 pool[addr] = [(iface_name, fqdn, ip)]
1454 continue
1455
1456
1457
1458 # WL uses an /29 to configure an interface. IP's are ordered like this:
1459 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1460
1461 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1462
1463 # Automatic naming convention of interlinks namely 2 + remote.lower()
1464 for (key,value) in pool.iteritems():
1465 # Make sure they are sorted from low-ip to high-ip
1466 value = sorted(value, key=lambda x: parseaddr(x[2]))
1467
1468 if len(value) == 1:
1469 (iface_name, fqdn, ip) = value[0]
1470 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1471
1472 # Device DNS names
1473 if 'cnode' in fqdn.lower():
1474 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1475 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1476
1477 elif len(value) == 2:
1478 (a_iface_name, a_fqdn, a_ip) = value[0]
1479 (b_iface_name, b_fqdn, b_ip) = value[1]
1480 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1481 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1482
1483 # Device DNS names
1484 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1485 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1486 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1487 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1488 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1489 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1490 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1491
1492 else:
1493 pool_members = [k[1] for k in value]
1494 for item in value:
1495 (iface_name, fqdn, ip) = item
1496 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1497
1498 # Include static DNS entries
1499 # XXX: Should they override the autogenerated results?
1500 # XXX: Convert input to yaml more useable.
1501 # Format:
1502 ##; this is a comment
1503 ## roomburgh=Roomburgh1
1504 ## apkerk1.Vosko=172.17.176.8 ;this as well
1505 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1506
1507 # Hack to allow special entries, for development
1508 wleiden_raw = {}
1509
1510 for line in dns_list:
1511 reverse = False
1512 k, items = line.items()[0]
1513 if type(items) == dict:
1514 if items.has_key('reverse'):
1515 reverse = items['reverse']
1516 items = items['a']
1517 else:
1518 items = items['cname']
1519 items = [items] if type(items) != list else items
1520 for item in items:
1521 if item.startswith('IN '):
1522 wleiden_raw[k] = item
1523 elif valid_addr(item):
1524 wleiden_zone[k].append((item, reverse))
1525 else:
1526 wleiden_cname[k] = item
1527
1528 # Hack to get dynamic pool listing
1529 def chunks(l, n):
1530 return [l[i:i+n] for i in range(0, len(l), n)]
1531
1532 ntp_servers = [x[0] for x in get_nameservers()]
1533 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1534 for ntp_server in chunk:
1535 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1536
1537 details = dict()
1538 # 24 updates a day allowed
1539 details['serial'] = time.strftime('%Y%m%d%H')
1540
1541 if external:
1542 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1543 else:
1544 dns_masters = ['sunny.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1545
1546 details['master'] = dns_masters[0]
1547 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1548
1549 dns_header = '''
1550$TTL 3h
1551%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1552 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1553
1554%(ns_servers)s
1555 \n'''
1556
1557
1558 if not os.path.isdir(output_dir):
1559 os.makedirs(output_dir)
1560 details['zone'] = 'wleiden.net'
1561 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1562 f.write(dns_header % details)
1563
1564 for host,items in wleiden_zone.iteritems():
1565 for ip,reverse in items:
1566 if ip not in ['0.0.0.0']:
1567 f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
1568 for source,dest in wleiden_cname.iteritems():
1569 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1570 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1571 for source, dest in wleiden_raw.iteritems():
1572 f.write("%s.wleiden.net. %s\n" % (source, dest))
1573 f.close()
1574
1575 # Create whole bunch of specific sub arpa zones. To keep it compliant
1576 for s in range(16,32):
1577 details['zone'] = '%i.172.in-addr.arpa' % s
1578 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1579 f.write(dns_header % details)
1580
1581 #XXX: Not effient, fix to proper data structure and do checks at other
1582 # stages
1583 for host,items in wleiden_zone.iteritems():
1584 for ip,reverse in items:
1585 if not reverse:
1586 continue
1587 if valid_addr(ip):
1588 if valid_addr(ip):
1589 if int(ip.split('.')[1]) == s:
1590 rev_ip = '.'.join(reversed(ip.split('.')))
1591 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1592 f.close()
1593
1594
1595def usage():
1596 print """Usage: %(prog)s <argument>
1597Argument:
1598\tcleanup = Cleanup all YAML files to specified format
1599\tstandalone [port] = Run configurator webserver [8000]
1600\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1601\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1602\tfull-export = Generate yaml export script for heatmap.
1603\tstatic [outputdir] = Generate all config files and store on disk
1604\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1605\ttest <node> [<file>] = Receive output for certain node [all files].
1606\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1607\tlist <status> <items> = List systems which have certain status
1608\tcreate network.kml = Create Network KML file for use in Google Earth
1609
1610Arguments:
1611\t<node> = NodeName (example: HybridRick)
1612\t<file> = %(files)s
1613\t<status> = all|up|down|planned
1614\t<items> = systems|nodes|proxies
1615
1616NOTE FOR DEVELOPERS; you can test your changes like this:
1617 BEFORE any changes in this code:
1618 $ ./gformat.py static /tmp/pre
1619 AFTER the changes:
1620 $ ./gformat.py static /tmp/post
1621 VIEW differences and VERIFY all are OK:
1622 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1623""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1624 exit(0)
1625
1626
1627def is_text_request(environ=os.environ):
1628 """ Find out whether we are calling from the CLI or any text based CLI utility """
1629 try:
1630 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1631 except KeyError:
1632 return True
1633
1634def switchFormat(setting):
1635 if setting:
1636 return "YES"
1637 else:
1638 return "NO"
1639
1640def rlinput(prompt, prefill=''):
1641 import readline
1642 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1643 try:
1644 return raw_input(prompt)
1645 finally:
1646 readline.set_startup_hook()
1647
1648def fix_conflict(left, right, default='i'):
1649 while True:
1650 print "## %-30s | %-30s" % (left, right)
1651 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1652 if not c:
1653 c = default
1654
1655 if c in ['l','1']:
1656 return left
1657 elif c in ['r','2']:
1658 return right
1659 elif c in ['e', '3']:
1660 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1661 elif c in ['i', '4']:
1662 return None
1663 else:
1664 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1665
1666
1667
1668def print_cgi_response(response_headers, output):
1669 """Could we not use some kind of wsgi wrapper to make this output?"""
1670 for header in response_headers:
1671 print "%s: %s" % header
1672 print
1673 print output
1674
1675
1676def fill_cache():
1677 ''' Poor man re-loading of few cache items (the slow ones) '''
1678 for host in get_hostlist():
1679 get_yaml(host)
1680
1681
1682def reload_cache():
1683 clear_cache()
1684 fill_cache()
1685
1686
1687def main():
1688 """Hard working sub"""
1689 # Allow easy hacking using the CLI
1690 if not os.environ.has_key('PATH_INFO'):
1691 if len(sys.argv) < 2:
1692 usage()
1693
1694 if sys.argv[1] == "standalone":
1695 import SocketServer
1696 import CGIHTTPServer
1697 # Hop to the right working directory.
1698 os.chdir(os.path.dirname(__file__))
1699 try:
1700 PORT = int(sys.argv[2])
1701 except (IndexError,ValueError):
1702 PORT = 8000
1703
1704 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1705 """ Serve this CGI from the root of the webserver """
1706 def is_cgi(self):
1707 if "favicon" in self.path:
1708 return False
1709
1710 self.cgi_info = (os.path.basename(__file__), self.path)
1711 self.path = ''
1712 return True
1713 handler = MyCGIHTTPRequestHandler
1714 SocketServer.TCPServer.allow_reuse_address = True
1715 httpd = SocketServer.TCPServer(("", PORT), handler)
1716 httpd.server_name = 'localhost'
1717 httpd.server_port = PORT
1718
1719 logger.info("serving at port %s", PORT)
1720 try:
1721 httpd.serve_forever()
1722 except KeyboardInterrupt:
1723 httpd.shutdown()
1724 logger.info("All done goodbye")
1725 elif sys.argv[1] == "test":
1726 # Basic argument validation
1727 try:
1728 node = sys.argv[2]
1729 except IndexError:
1730 print "Invalid argument"
1731 exit(1)
1732 except IOError as e:
1733 print e
1734 exit(1)
1735
1736 datadump = get_yaml(node)
1737
1738
1739 # Get files to generate
1740 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1741
1742 # Actual config generation
1743 for config in gen_files:
1744 logger.info("## Generating %s %s", node, config)
1745 print generate_config(node, config, datadump)
1746 elif sys.argv[1] == "test-cgi":
1747 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1748 os.environ['SCRIPT_NAME'] = __file__
1749 response_headers, output = process_cgi_request()
1750 print_cgi_response(response_headers, output)
1751 elif sys.argv[1] == "static":
1752 items = dict()
1753 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1754 for node in get_hostlist():
1755 items['node'] = node
1756 items['wdir'] = "%(output_dir)s/%(node)s" % items
1757 if not os.path.isdir(items['wdir']):
1758 os.makedirs(items['wdir'])
1759 datadump = get_yaml(node)
1760 for config in files:
1761 items['config'] = config
1762 logger.info("## Generating %(node)s %(config)s" % items)
1763 f = open("%(wdir)s/%(config)s" % items, "w")
1764 f.write(generate_config(node, config, datadump))
1765 f.close()
1766 elif sys.argv[1] == "wind-export":
1767 items = dict()
1768 for node in get_hostlist():
1769 datadump = get_yaml(node)
1770 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1771 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1772 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1773 VALUES (
1774 (SELECT id FROM users WHERE username = 'rvdzwet'),
1775 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1776 'Y');""" % datadump
1777 #for config in files:
1778 # items['config'] = config
1779 # print "## Generating %(node)s %(config)s" % items
1780 # f = open("%(wdir)s/%(config)s" % items, "w")
1781 # f.write(generate_config(node, config, datadump))
1782 # f.close()
1783 for node in get_hostlist():
1784 datadump = get_yaml(node)
1785 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1786 ifacedump = datadump[iface_key]
1787 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1788 ifacedump['nodename'] = datadump['nodename']
1789 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1790 ifacedump['channel'] = 0
1791 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1792 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1793 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1794 elif sys.argv[1] == "nagios-export":
1795 try:
1796 heavy_load = (sys.argv[2] == "--heavy-load")
1797 except IndexError:
1798 heavy_load = False
1799
1800 hostgroup_details = {
1801 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1802 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1803 'walphen' : 'Stichting Wireless Alphen',
1804 'westeinder' : 'Westeinder Plassen',
1805 }
1806
1807 # Convert IP to Host
1808 ip2host = {'root' : 'root'}
1809 for host in get_hostlist():
1810 datadump = get_yaml(host)
1811 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1812 for iface in get_interface_keys(datadump):
1813 if datadump[iface].has_key('autogen_gateway'):
1814 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1815
1816 # Find dependency tree based on output of lvrouted.mytree of nearest node
1817 parents = defaultdict(list)
1818 stack = ['root']
1819 prev_depth = 0
1820 for line in open('lvrouted.mytree').readlines():
1821 depth = line.count('\t')
1822 ip = line.strip().split()[0]
1823
1824 if prev_depth < depth:
1825 try:
1826 parents[ip2host[ip]].append(ip2host[stack[-1]])
1827 except KeyError as e:
1828 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1829 stack.append(ip)
1830 elif prev_depth > depth:
1831 stack = stack[:(depth - prev_depth)]
1832 elif prev_depth == depth:
1833 try:
1834 parents[ip2host[ip]].append(ip2host[stack[-1]])
1835 except KeyError as e:
1836 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1837
1838
1839 prev_depth = depth
1840 # Observe that some nodes has themself as parent or multiple parents
1841 # for now take only the first parent, other behaviour is yet to be explained
1842
1843
1844
1845 params = {
1846 'check_interval' : 5 if heavy_load else 120,
1847 'retry_interval' : 1 if heavy_load else 10,
1848 'max_check_attempts' : 10 if heavy_load else 6,
1849 'notification_interval': 120 if heavy_load else 240,
1850 }
1851
1852 print '''\
1853define host {
1854 name wleiden-node ; Default Node Template
1855 use generic-host ; Use the standard template as initial starting point
1856 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1857 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1858 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1859 notification_interval %(notification_interval)s
1860 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1861 check_command check-host-alive ; Default command to check FreeBSD hosts
1862 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1863}
1864
1865define service {
1866 name wleiden-service ; Default Service Template
1867 use generic-service ; Use the standard template as initial starting point
1868 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1869 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1870 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1871 notification_interval %(notification_interval)s
1872 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1873 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1874}
1875
1876# Please make sure to install:
1877# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1878#
1879# Recompile net-mgmt/nagios-plugins to support check_snmp
1880# make -C /usr/ports/net-mgmt/nagios-plugins
1881#
1882# Install net/bind-tools to allow v2/check_dns_wl to work:
1883# pkg install bind-tools
1884#
1885define command{
1886 command_name check_snmp_disk
1887 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1888}
1889
1890define command{
1891 command_name check_netsnmp_load
1892 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
1893}
1894
1895define command{
1896 command_name check_netsnmp_proc
1897 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
1898}
1899
1900define command{
1901 command_name check_by_ssh
1902 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
1903}
1904
1905define command{
1906 command_name check_dns_wl
1907 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
1908}
1909
1910define command{
1911 command_name check_snmp_uptime
1912 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
1913}
1914
1915
1916# TDB: dhcp leases
1917# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
1918
1919# TDB: internet status
1920# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
1921
1922# TDB: Advanced local passive checks
1923# /usr/local/libexec/nagios/check_by_ssh
1924''' % params
1925
1926 print '''\
1927# Service Group, not displayed by default
1928define hostgroup {
1929 hostgroup_name srv_hybrid
1930 alias All Hybrid Nodes
1931 register 0
1932}
1933
1934define service {
1935 use wleiden-service
1936 hostgroup_name srv_hybrid
1937 service_description SSH
1938 check_command check_ssh
1939}
1940
1941define service {
1942 use wleiden-service,service-pnp
1943 hostgroup_name srv_hybrid
1944 service_description HTTP
1945 check_command check_http
1946}
1947
1948define service {
1949 use wleiden-service
1950 hostgroup_name srv_hybrid
1951 service_description DNS
1952 check_command check_dns_wl!"www.wirelessleiden.nl"
1953}
1954'''
1955
1956 if heavy_load:
1957 print '''\
1958define service {
1959 use wleiden-service
1960 hostgroup_name srv_hybrid
1961 service_description UPTIME
1962 check_command check_snmp_uptime
1963}
1964
1965#define service {
1966# use wleiden-service
1967# hostgroup_name srv_hybrid
1968# service_description NTP
1969# check_command check_ntp_peer
1970#}
1971
1972define service {
1973 use wleiden-service
1974 hostgroup_name srv_hybrid
1975 service_description LOAD
1976 check_command check_netsnmp_load
1977}
1978
1979define service {
1980 use wleiden-service
1981 hostgroup_name srv_hybrid
1982 service_description PROC
1983 check_command check_netsnmp_proc
1984}
1985
1986define service {
1987 use wleiden-service
1988 hostgroup_name srv_hybrid
1989 service_description DISK
1990 check_command check_snmp_disk
1991}
1992'''
1993 for node in get_hostlist():
1994 datadump = get_yaml(node)
1995 if not datadump['status'] == 'up':
1996 continue
1997 if not hostgroup_details.has_key(datadump['monitoring_group']):
1998 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
1999 print '''\
2000define host {
2001 use wleiden-node,host-pnp
2002 contact_groups admins
2003 host_name %(autogen_fqdn)s
2004 address %(masterip)s
2005 hostgroups srv_hybrid,%(monitoring_group)s\
2006''' % datadump
2007 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2008 print '''\
2009 parents %(parents)s\
2010''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2011 print '''\
2012}
2013'''
2014
2015
2016 for name,alias in hostgroup_details.iteritems():
2017 print '''\
2018define hostgroup {
2019 hostgroup_name %s
2020 alias %s
2021} ''' % (name, alias)
2022
2023
2024 elif sys.argv[1] == "full-export":
2025 hosts = {}
2026 for node in get_hostlist():
2027 datadump = get_yaml(node)
2028 hosts[datadump['nodename']] = datadump
2029 print yaml.dump(hosts)
2030
2031 elif sys.argv[1] == "dns":
2032 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2033 elif sys.argv[1] == "cleanup":
2034 # First generate all datadumps
2035 datadumps = dict()
2036 ssid_to_node = dict()
2037 for host in get_hostlist():
2038 logger.info("# Processing: %s", host)
2039 # Set some boring default values
2040 datadump = { 'board' : 'UNKNOWN' }
2041 datadump.update(get_yaml(host))
2042 datadumps[datadump['nodename']] = datadump
2043
2044 (poel, errors) = make_relations()
2045 print "\n".join(["# WARNING: %s" % x for x in errors])
2046
2047 for host,datadump in datadumps.iteritems():
2048 try:
2049 # Convert all yes and no to boolean values
2050 def fix_boolean(dump):
2051 for key in dump.keys():
2052 if type(dump[key]) == dict:
2053 dump[key] = fix_boolean(dump[key])
2054 elif str(dump[key]).lower() in ["yes", "true"]:
2055 dump[key] = True
2056 elif str(dump[key]).lower() in ["no", "false"]:
2057 # Compass richting no (Noord Oost) is valid input
2058 if key != "compass": dump[key] = False
2059 return dump
2060 datadump = fix_boolean(datadump)
2061
2062 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2063 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2064 elif 'latitude' in datadump and 'longitude' in datadump:
2065 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2066
2067 if datadump['nodename'].startswith('Proxy'):
2068 datadump['nodename'] = datadump['nodename'].lower()
2069
2070 for iface_key in get_interface_keys(datadump):
2071 try:
2072 # All our normal wireless cards are normal APs now
2073 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2074 datadump[iface_key]['mode'] = 'ap'
2075 # Wireless Leiden SSID have an consistent lowercase/uppercase
2076 if datadump[iface_key].has_key('ssid'):
2077 ssid = datadump[iface_key]['ssid']
2078 prefix = 'ap-WirelessLeiden-'
2079 if ssid.lower().startswith(prefix.lower()):
2080 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2081 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2082 datadump[iface_key]['mode'] = 'autogen-FIXME'
2083 if not datadump[iface_key].has_key('comment'):
2084 datadump[iface_key]['comment'] = 'autogen-FIXME'
2085
2086 if datadump[iface_key].has_key('ns_mac'):
2087 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2088
2089 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2090 datadump[iface_key] = datadump[iface_key]['desc']
2091
2092 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2093 # due to better collision avoidance.
2094 if datadump[iface_key]['type'] == '11b':
2095 datadump[iface_key]['type'] = '11g'
2096
2097 # Setting 802.11g channels to de-facto standards, to avoid
2098 # un-detected sharing with other overlapping channels
2099 #
2100 # Technically we could also use channel 13 in NL, but this is not
2101 # recommended as foreign devices might not be able to select this
2102 # channel. Secondly using 1,5,9,13 instead is going to clash with
2103 # the de-facto usage of 1,6,11.
2104 #
2105 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2106 channels_at_2400Mhz = (1,6,11)
2107 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2108 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2109 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2110 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2111
2112 # Mandatory interface keys
2113 if not datadump[iface_key].has_key('status'):
2114 datadump[iface_key]['status'] = 'planned'
2115
2116 x = datadump[iface_key]['comment']
2117 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2118
2119 # Fixing bridge_type if none is found
2120 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2121 if not 'bridge_type' in datadump[iface_key]:
2122 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2123
2124 # Making sure description works
2125 if datadump[iface_key].has_key('desc'):
2126 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2127 del datadump[iface_key]['desc']
2128 else:
2129 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2130 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2131 if response:
2132 datadump[iface_key]['comment'] = response
2133 del datadump[iface_key]['desc']
2134
2135 # Check DHCP configuration
2136 dhcp_type(datadump[iface_key])
2137
2138 # Set the compass value based on the angle between the poels
2139 if datadump[iface_key].has_key('ns_ip'):
2140 my_pool = poel[network(datadump[iface_key]['ip'])]
2141 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2142 if remote_hosts:
2143 compass_target = remote_hosts[0]
2144 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2145
2146 # Monitoring Group default
2147 if not 'monitoring_group' in datadump:
2148 datadump['monitoring_group'] = 'wleiden'
2149
2150 except Exception:
2151 print "# Error while processing interface %s" % iface_key
2152 raise
2153 store_yaml(datadump)
2154 except Exception:
2155 print "# Error while processing %s" % host
2156 raise
2157 elif sys.argv[1] == "list":
2158 use_fqdn = False
2159 if len(sys.argv) < 4:
2160 usage()
2161 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2162 usage()
2163 if not sys.argv[3] in ["nodes","proxies","systems"]:
2164 usage()
2165
2166 if len(sys.argv) > 4:
2167 if sys.argv[4] == "fqdn":
2168 use_fqdn = True
2169 else:
2170 usage()
2171
2172 for system in get_hostlist():
2173 datadump = get_yaml(system)
2174 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2175 continue
2176
2177 output = datadump['autogen_fqdn'] if use_fqdn else system
2178 if sys.argv[2] == "all":
2179 print output
2180 elif datadump['status'] == sys.argv[2]:
2181 print output
2182 elif sys.argv[1] == "create":
2183 if sys.argv[2] == "network.kml":
2184 print make_network_kml.make_graph()
2185 elif sys.argv[2] == "host-ips.txt":
2186 for system in get_hostlist():
2187 datadump = get_yaml(system)
2188 ips = [datadump['masterip']]
2189 for ifkey in get_interface_keys(datadump):
2190 ips.append(datadump[ifkey]['ip'].split('/')[0])
2191 print system, ' '.join(ips)
2192 elif sys.argv[2] == "host-pos.txt":
2193 for system in get_hostlist():
2194 datadump = get_yaml(system)
2195 print system, datadump['rdnap_x'], datadump['rdnap_y']
2196 elif sys.argv[2] == 'ssh_config':
2197 print '''
2198Host *.wleiden.net
2199 User root
2200
2201Host 172.16.*.*
2202 User root
2203'''
2204 for system in get_hostlist():
2205 datadump = get_yaml(system)
2206 print '''\
2207Host %s
2208 User root
2209
2210Host %s
2211 User root
2212
2213Host %s
2214 User root
2215
2216Host %s
2217 User root
2218''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2219 else:
2220 usage()
2221 else:
2222 usage()
2223 else:
2224 # Do not enable debugging for config requests as it highly clutters the output
2225 if not is_text_request():
2226 cgitb.enable()
2227 response_headers, output = process_cgi_request()
2228 print_cgi_response(response_headers, output)
2229
2230def application(environ, start_response):
2231 status = '200 OK'
2232 response_headers, output = process_cgi_request(environ)
2233 start_response(status, response_headers)
2234
2235 # Debugging only
2236 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2237 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2238 return [output]
2239
2240if __name__ == "__main__":
2241 main()
Note: See TracBrowser for help on using the repository browser.