source: genesis/tools/gformat.py@ 13934

Last change on this file since 13934 was 13930, checked in by rick, 8 years ago

Fix adding of interface members of bridge(4)

create_args_ is not supported for bridge creation, how-ever when using
ifconfig_ some trickery is required to support IP assignments as all needs to
be 'shifted' to aliasX interfacing.

Note alias creation is flawed by design as they all have to have unique keys,
thus we rely on _aliasX definitions for alias creation, where-as we should
parse the entries instead. Thus make_interface_list has to hack support for
bridge aliases which do require this tooling.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 79.3 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# MUCH FASTER WILL IT BE with mod_wsgi, due to caches and avoiding loading all
16# the heavy template lifting all the time.
17#
18# WSGIDaemonProcess gformat threads=25
19# WSGISocketPrefix run/wsgi
20#
21# <Directory /var/www/cgi-bin>
22# WSGIProcessGroup gformat
23# </Directory>
24# WSGIScriptAlias /hello /var/www/cgi-bin/genesis/tools/gformat.py
25#
26# Package dependencies list:
27# FreeBSD:
28# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
29# Fedora:
30# yum install python-yaml pyproj proj-epsg python-jinja2
31#
32# Rick van der Zwet <info@rickvanderzwet.nl>
33#
34
35# Hack to make the script directory is also threated as a module search path.
36import sys
37import os
38sys.path.append(os.path.dirname(__file__))
39
40SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
41
42import argparse
43import cgi
44import cgitb
45import copy
46import glob
47import make_network_kml
48import math
49import pyproj
50import random
51import re
52import socket
53import string
54import subprocess
55import textwrap
56import time
57import urlparse
58
59from pprint import pprint
60from collections import defaultdict, OrderedDict
61from sys import stderr
62try:
63 import yaml
64except ImportError, e:
65 print e
66 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
67 exit(1)
68
69try:
70 from yaml import CLoader as Loader
71 from yaml import CDumper as Dumper
72except ImportError:
73 from yaml import Loader, Dumper
74
75from jinja2 import Environment, Template
76def yesorno(value):
77 return "YES" if bool(value) else "NO"
78env = Environment()
79env.filters['yesorno'] = yesorno
80def render_template(datadump, template):
81 result = env.from_string(template).render(datadump)
82 # Make it look pretty to the naked eye, as jinja templates are not so
83 # friendly when it comes to whitespace formatting
84 ## Remove extra whitespace at end of line lstrip() style.
85 result = re.sub(r'\n[\ ]+','\n', result)
86 ## Include only a single newline between an definition and a comment
87 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
88 ## Remove extra newlines after single comment
89 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
90 return result
91
92import logging
93logging.basicConfig(format='# %(levelname)s: %(message)s' )
94logger = logging.getLogger()
95logger.setLevel(logging.DEBUG)
96
97
98if os.environ.has_key('CONFIGROOT'):
99 NODE_DIR = os.environ['CONFIGROOT']
100else:
101 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
102__version__ = '$Id: gformat.py 13930 2017-06-21 22:35:31Z rick $'
103
104files = [
105 'authorized_keys',
106 'dnsmasq.conf',
107 'dhcpd.conf',
108 'rc.conf.local',
109 'resolv.conf',
110 'motd',
111 'ntp.conf',
112 'pf.hybrid.conf.local',
113 'unbound.wleiden.conf',
114 'wleiden.yaml',
115 ]
116
117# Global variables uses
118OK = 10
119DOWN = 20
120UNKNOWN = 90
121
122
123ileiden_proxies = OrderedDict()
124normal_proxies = []
125datadump_cache = {}
126interface_list_cache = {}
127rc_conf_local_cache = {}
128nameservers_cache = []
129relations_cache = None
130def clear_cache():
131 ''' Poor mans cache implementation '''
132 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
133 datadump_cache = {}
134 interface_list_cache = {}
135 rc_conf_local_cache = {}
136 ileiden_proxies = OrderedDict()
137 normal_proxies = []
138 nameservers_cache = []
139 relations_cache = None
140
141NO_DHCP = 0
142DHCP_CLIENT = 10
143DHCP_SERVER = 20
144def dhcp_type(item):
145 if not item.has_key('dhcp'):
146 return NO_DHCP
147 elif not item['dhcp']:
148 return NO_DHCP
149 elif item['dhcp'].lower() == 'client':
150 return DHCP_CLIENT
151 else:
152 # Validation Checks
153 begin,end = map(int,item['dhcp'].split('-'))
154 if begin >= end:
155 raise ValueError("DHCP Start >= DHCP End")
156 return DHCP_SERVER
157
158def etrs2rd(lat, lon):
159 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
160 p2 = pyproj.Proj(init='EPSG:28992')
161 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
162 return (RDx, RDy)
163
164def rd2etrs(RDx, RDy):
165 p1 = pyproj.Proj(init='EPSG:28992')
166 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
167 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
168 return (lat, lon)
169
170def get_yaml(item,add_version_info=True):
171 try:
172 """ Get configuration yaml for 'item'"""
173 if datadump_cache.has_key(item):
174 return datadump_cache[item].copy()
175
176 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
177 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
178 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
179
180 # Default values
181 datadump = {
182 'autogen_revision' : 'NOTFOUND',
183 'autogen_gfile' : gfile,
184 'service_proxy_ileiden' : False,
185 }
186 f = open(gfile, 'r')
187 datadump.update(yaml.load(f,Loader=Loader))
188 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
189 if datadump['nodetype'] == 'Hybrid':
190 # Some values are defined implicitly
191 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
192 datadump['service_incoming_rdr'] = True
193 # Use some boring defaults
194 defaults = {
195 'service_proxy_normal' : False,
196 'service_accesspoint' : True,
197 'service_incoming_rdr' : False,
198 'service_concentrator' : False,
199 'monitoring_group' : 'wleiden',
200 }
201 for (key,value) in defaults.iteritems():
202 if not datadump.has_key(key):
203 datadump[key] = value
204 f.close()
205
206 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
207 if add_version_info:
208 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
209 lines = p.communicate()[0].split('\n')
210 if p.returncode == 0:
211 for line in lines:
212 if line:
213 (key, value) = line.split(': ')
214 datadump["autogen_" + key.lower().replace(' ','_')] = value
215
216 # Preformat certain needed variables for formatting and push those into special object
217 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
218
219 wlan_count=0
220 try:
221 for key in get_interface_keys(datadump, True):
222 datadump[key]['autogen_ifbase'] = key.split('_')[1]
223 datadump[key]['autogen_vlan'] = False
224
225 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
226 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
227 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
228
229 if datadump[key].has_key('parent'):
230 if datadump[key].has_key('ip'):
231 raise ValueError("Interface bridge member cannot have IP assigned")
232 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
233 raise ValueError("Interface bridge member cannot have DHCP set")
234
235 if datadump[key].has_key('ip'):
236 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
237
238 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
239 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
240 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
241 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
242 wlan_count += 1
243 else:
244 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
245 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
246 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
247 datadump[key]['autogen_vlan'] = key.split('_')[2]
248 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
249 else:
250 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
251
252 except Exception:
253 print "# Error while processing interface %s" % key
254 raise
255
256 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
257 if dhcp_type(datadump[key]) == DHCP_SERVER]
258
259 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
260 datadump['autogen_item'] = item
261
262 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
263 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
264 datadump_cache[item] = datadump.copy()
265 except Exception:
266 print "# Error while processing %s" % item
267 raise
268 return datadump
269
270
271def store_yaml(datadump, header=False):
272 """ Store configuration yaml for 'item'"""
273 item = datadump['autogen_item']
274 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
275
276 output = generate_wleiden_yaml(datadump, header)
277
278 f = open(gfile, 'w')
279 f.write(output)
280 f.close()
281
282
283def network(ip):
284 addr, mask = ip.split('/')
285 # Not parsing of these folks please
286 addr = parseaddr(addr)
287 mask = int(mask)
288 network = addr & ~((1 << (32 - mask)) - 1)
289 return network
290
291
292
293def make_relations():
294 """ Process _ALL_ yaml files to get connection relations """
295 global relations_cache
296
297 if relations_cache:
298 return relations_cache
299
300 errors = []
301 poel = defaultdict(list)
302
303 for host in get_hostlist():
304 datadump = get_yaml(host)
305 try:
306 for iface_key in get_interface_keys(datadump):
307 # Bridge members has no IP assigned
308 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
309 continue
310 net_addr = network(datadump[iface_key]['ip'])
311 poel[net_addr] += [(host,datadump[iface_key].copy())]
312 except (KeyError, ValueError) as e:
313 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
314 continue
315
316 relations_cache = (poel, errors)
317 return relations_cache
318
319
320
321def valid_addr(addr):
322 """ Show which address is valid in which are not """
323 return str(addr).startswith('172.')
324
325def get_hostlist():
326 """ Combined hosts and proxy list"""
327 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
328
329def angle_between_points(lat1,lat2,long1,long2):
330 """
331 Return Angle in radians between two GPS coordinates
332 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
333 """
334 dy = lat2 - lat1
335 dx = math.cos(lat1)*(long2 - long1)
336 angle = math.atan2(dy,dx)
337 return angle
338
339
340
341def angle_to_cd(angle):
342 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
343
344 # For easy conversion get positive degree
345 degrees = math.degrees(angle)
346 abs_degrees = 360 + degrees if degrees < 0 else degrees
347
348 # Numbers can be confusing calculate from the 4 main directions
349 p = 22.5
350 if abs_degrees < p:
351 cd = "n"
352 elif abs_degrees < (90 - p):
353 cd = "no"
354 elif abs_degrees < (90 + p):
355 cd = "o"
356 elif abs_degrees < (180 - p):
357 cd = "zo"
358 elif abs_degrees < (180 + p):
359 cd = "z"
360 elif abs_degrees < (270 - p):
361 cd = "zw"
362 elif abs_degrees < (270 + p):
363 cd = "w"
364 elif abs_degrees < (360 - p):
365 cd = "nw"
366 else:
367 cd = "n"
368 return cd
369
370
371
372def cd_between_hosts(hostA, hostB, datadumps):
373 # Using RDNAP coordinates
374 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
375 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
376 return angle_to_cd(math.atan2(dx,dy))
377
378 # GPS coordinates seems to fail somehow
379 #latA = float(datadumps[hostA]['latitude'])
380 #latB = float(datadumps[hostB]['latitude'])
381 #lonA = float(datadumps[hostA]['longitude'])
382 #lonB = float(datadumps[hostB]['longitude'])
383 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
384
385
386def generate_title(nodelist):
387 """ Main overview page """
388 items = {'root' : "." }
389 def fl(spaces, line):
390 return (' ' * spaces) + line + '\n'
391
392 output = """
393<html>
394 <head>
395 <title>Wireless leiden Configurator - GFormat</title>
396 <style type="text/css">
397 th {background-color: #999999}
398 tr:nth-child(odd) {background-color: #cccccc}
399 tr:nth-child(even) {background-color: #ffffff}
400 th, td {padding: 0.1em 1em}
401 </style>
402 </head>
403 <body>
404 <center>
405 <form type="GET" action="%(root)s">
406 <input type="hidden" name="action" value="update">
407 <input type="submit" value="Update Configuration Database (SVN)">
408 </form>
409 <table>
410 <caption><h3>Wireless Leiden Configurator</h3></caption>
411 """ % items
412
413 for node in nodelist:
414 items['node'] = node
415 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
416 for config in files:
417 items['config'] = config
418 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
419 output += fl(5, "</tr>")
420 output += """
421 </table>
422 <hr />
423 <em>%s</em>
424 </center>
425 </body>
426</html>
427 """ % __version__
428
429 return output
430
431
432
433def generate_node(node):
434 """ Print overview of all files available for node """
435 return "\n".join(files)
436
437def generate_node_overview(host):
438 """ Print overview of all files available for node """
439 datadump = get_yaml(host)
440 params = { 'host' : host }
441 output = "<em><a href='..'>Back to overview</a></em><hr />"
442 output += "<h2>Available files:</h2><ul>"
443 for cf in files:
444 params['cf'] = cf
445 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
446 output += "</ul>"
447
448 # Generate and connection listing
449 output += "<h2>Connected To:</h2><ul>"
450 (poel, errors) = make_relations()
451 for network, hosts in poel.iteritems():
452 if host in [x[0] for x in hosts]:
453 if len(hosts) == 1:
454 # Single not connected interface
455 continue
456 for remote,ifacedump in hosts:
457 if remote == host:
458 # This side of the interface
459 continue
460 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
461 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
462 output += "</ul>"
463 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
464
465 output += "<hr /><em><a href='..'>Back to overview</a></em>"
466 return output
467
468
469def generate_header(datadump, ctag="#"):
470 return """\
471%(ctag)s
472%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
473%(ctag)s
474""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
475
476
477
478def parseaddr(s):
479 """ Process IPv4 CIDR notation addr to a (binary) number """
480 f = s.split('.')
481 return (long(f[0]) << 24L) + \
482 (long(f[1]) << 16L) + \
483 (long(f[2]) << 8L) + \
484 long(f[3])
485
486
487
488def showaddr(a):
489 """ Display IPv4 addr in (dotted) CIDR notation """
490 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
491
492
493def is_member(ip, mask, canidate):
494 """ Return True if canidate is part of ip/mask block"""
495 ip_addr = parseaddr(ip)
496 ip_canidate = parseaddr(canidate)
497 mask = int(mask)
498 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
499 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
500 return ip_addr == ip_canidate
501
502
503
504def cidr2netmask(netmask):
505 """ Given a 'netmask' return corresponding CIDR """
506 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
507
508def get_network(addr, mask):
509 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
510
511
512def generate_dhcpd_conf(datadump):
513 """ Generate config file '/usr/local/etc/dhcpd.conf """
514 # Redundency support, in cause local DNS server is not running/responding.
515 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
516 output = generate_header(datadump)
517 output += Template("""\
518# option definitions common to all supported networks...
519option domain-name "dhcp.{{ autogen_fqdn }}";
520
521default-lease-time 600;
522max-lease-time 7200;
523
524# Use this to enble / disable dynamic dns updates globally.
525#ddns-update-style none;
526
527# If this DHCP server is the official DHCP server for the local
528# network, the authoritative directive should be uncommented.
529authoritative;
530
531# Use this to send dhcp log messages to a different log file (you also
532# have to hack syslog.conf to complete the redirection).
533log-facility local7;
534
535# UniFi Discovery Support
536option space ubnt;
537option ubnt.unifi-address code 1 = ip-address;
538#
539class "ubnt" {
540 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
541 option vendor-class-identifier "ubnt";
542 vendor-option-space ubnt;
543}
544
545#
546# Interface definitions
547#
548\n\n""").render(datadump)
549
550
551 # TODO: Use textwrap.fill instead
552 def indent(text, count):
553 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
554
555 # Process DHCP blocks
556 dhcp_out = defaultdict(list)
557 for iface_key in get_interface_keys(datadump):
558 ifname = datadump[iface_key]['autogen_ifbase']
559 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
560 if not datadump[iface_key].has_key('comment'):
561 datadump[iface_key]['comment'] = None
562
563 if not datadump[iface_key].has_key('ip'):
564 continue
565
566 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
567
568 (addr, mask) = datadump[iface_key]['ip'].split('/')
569 datadump[iface_key]['autogen_addr'] = addr
570 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
571 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
572
573 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
574 dhcp_out[groupif].append(textwrap.dedent("""\
575 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
576 ### not autoritive
577 }
578 """ % datadump[iface_key]))
579 continue
580
581 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
582 dhcp_part = ".".join(addr.split('.')[0:3])
583 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
584 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
585 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
586
587 # Assume the first 10 IPs could be used for static entries
588 if 'no_portal' in datadump:
589 fixed = 5
590 for mac in datadump['no_portal']:
591 dhcp_out[groupif].append(textwrap.dedent("""\
592 host fixed-%(ifname)s-%(fixed)s {
593 hardware ethernet %(mac)s;
594 fixed-address %(prefix)s.%(fixed)s;
595 }
596 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
597 fixed += 1
598
599 if 'dhcp_fixed' in datadump[iface_key]:
600 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
601 dhcp_out[groupif].append(textwrap.dedent("""\
602 host fixed-%(host)s {
603 hardware ethernet %(mac)s;
604 fixed-address %(addr)s;
605 }
606 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
607
608
609 dhcp_out[groupif].append(textwrap.dedent("""\
610 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
611 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
612 option routers %(autogen_addr)s;
613 option domain-name-servers %(autogen_dns_servers)s;
614 option ubnt.unifi-address 172.17.107.10;
615 }
616 """ % datadump[iface_key]))
617
618 # Output the blocks in groups
619 for ifname,value in sorted(dhcp_out.iteritems()):
620 if len(value) > 2:
621 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
622 else:
623 output += ''.join(value) + "\n\n"
624 return output
625
626
627
628def generate_dnsmasq_conf(datadump):
629 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
630 output = generate_header(datadump)
631 output += Template("""\
632# DHCP server options
633dhcp-authoritative
634dhcp-fqdn
635domain=dhcp.{{ autogen_fqdn }}
636domain-needed
637expand-hosts
638log-async=100
639
640# Low memory footprint
641cache-size=10000
642
643\n""").render(datadump)
644
645 for iface_key in get_interface_keys(datadump):
646 if not datadump[iface_key].has_key('comment'):
647 datadump[iface_key]['comment'] = None
648 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
649
650 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
651 output += "# not autoritive\n\n"
652 continue
653
654 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
655 (ip, cidr) = datadump[iface_key]['ip'].split('/')
656 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
657
658 dhcp_part = ".".join(ip.split('.')[0:3])
659 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
660 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
661 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
662
663 return output
664
665
666class AutoVivification(dict):
667 """Implementation of perl's autovivification feature."""
668 def __getitem__(self, item):
669 try:
670 return dict.__getitem__(self, item)
671 except KeyError:
672 value = self[item] = type(self)()
673 return value
674
675def make_interface_list(datadump):
676 if interface_list_cache.has_key(datadump['autogen_item']):
677 return (interface_list_cache[datadump['autogen_item']])
678 # lo0 configuration:
679 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
680 # - masterip is special as it needs to be assigned to at
681 # least one interface, so if not used assign to lo0
682 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
683 vlan_list = defaultdict(list)
684 bridge_list = defaultdict(list)
685 flags_if = AutoVivification()
686 dhclient_if = {'lo0' : False}
687
688 # XXX: Find some way of send this output nicely
689 output = ''
690
691 masterip_used = False
692 for iface_key in get_interface_keys(datadump):
693 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
694 masterip_used = True
695 break
696 if not masterip_used:
697 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
698
699 if 'serviceid' in datadump:
700 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
701
702 for iface_key in get_interface_keys(datadump):
703 ifacedump = datadump[iface_key]
704 if ifacedump['autogen_bridge_alias']:
705 ifname = ifacedump['autogen_ifbase']
706 else:
707 ifname = ifacedump['autogen_ifname']
708
709 # If defined as vlan interface
710 if ifacedump['autogen_vlan']:
711 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
712
713 # If defined as bridge interface
714 if ifacedump['autogen_bridge_member']:
715 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
716
717
718 # Flag dhclient is possible
719 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
720 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
721
722 # Ethernet address
723 if ifacedump.has_key('ether'):
724 flags_if[ifname]['ether'] = ifacedump['ether']
725
726 # Add interface IP to list
727 if ifacedump.has_key('ip'):
728 item = (ifacedump['ip'], ifacedump['comment'])
729 if addrs_list.has_key(ifname):
730 addrs_list[ifname].append(item)
731 else:
732 addrs_list[ifname] = [item]
733
734 # Alias only needs IP assignment for now, this might change if we
735 # are going to use virtual accesspoints
736 if "alias" in iface_key:
737 continue
738
739 # XXX: Might want to deduct type directly from interface name
740 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
741 # Default to station (client) mode
742 ifacedump['autogen_wlanmode'] = "sta"
743 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
744 ifacedump['autogen_wlanmode'] = "ap"
745
746 if not ifacedump.has_key('channel'):
747 if ifacedump['type'] == '11a':
748 ifacedump['channel'] = 36
749 else:
750 ifacedump['channel'] = 1
751
752 # Allow special hacks at the back like wds and stuff
753 if not ifacedump.has_key('extra'):
754 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
755 else:
756 ifacedump['autogen_extra'] = ifacedump['extra']
757
758 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
759
760 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
761 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
762 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
763 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
764 output += "\n"
765
766 elif ifacedump['type'] in ['ethernet', 'eth']:
767 # No special config needed besides IP
768 pass
769 elif ifacedump['type'] in ['vlan']:
770 # VLAN member has no special configuration
771 pass
772 else:
773 assert False, "Unknown type " + ifacedump['type']
774
775 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
776 interface_list_cache[datadump['autogen_item']] = store
777 return(store)
778
779
780
781def create_proxies_list():
782 if not ileiden_proxies or not normal_proxies:
783 # Placeholder for to-be-installed proxies, this will avoid updating the all
784 # nodes to include this new machine, yet due to an unbound issue, this list
785 # has to be kept small.
786
787 for i in range(1,20):
788 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
789
790 for host in get_hostlist():
791 hostdump = get_yaml(host)
792 if hostdump['status'] == 'up':
793 if hostdump['service_proxy_ileiden']:
794 ileiden_proxies[hostdump['serviceid']] = hostdump
795 if hostdump['service_proxy_normal']:
796 normal_proxies.append(hostdump)
797
798
799
800def generate_rc_conf_local(datadump):
801 """ Generate configuration file '/etc/rc.conf.local' """
802 item = datadump['autogen_item']
803 if rc_conf_local_cache.has_key(item):
804 return rc_conf_local_cache[item]
805
806 if not datadump.has_key('ileiden'):
807 datadump['autogen_ileiden_enable'] = False
808 else:
809 datadump['autogen_ileiden_enable'] = datadump['ileiden']
810
811 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
812
813 create_proxies_list()
814 datadump['autogen_ileiden_proxies'] = ileiden_proxies
815 datadump['autogen_normal_proxies'] = normal_proxies
816 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
817 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
818 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
819 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
820
821 output = generate_header(datadump, "#");
822 output += render_template(datadump, """\
823hostname='{{ autogen_fqdn }}'
824location='{{ location }}'
825nodetype="{{ nodetype }}"
826
827#
828# Configured listings
829#
830captive_portal_whitelist=""
831{% if nodetype == "Proxy" %}
832#
833# Proxy Configuration
834#
835{% if gateway and service_proxy_ileiden -%}
836defaultrouter="{{ gateway }}"
837{% else -%}
838#defaultrouter="NOTSET"
839{% endif -%}
840internalif="{{ internalif }}"
841ileiden_enable="{{ autogen_ileiden_enable }}"
842gateway_enable="{{ autogen_ileiden_enable }}"
843pf_enable="yes"
844pf_rules="/etc/pf.conf"
845{% if autogen_ileiden_enable -%}
846pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
847lvrouted_enable="{{ autogen_ileiden_enable }}"
848lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
849{% else -%}
850pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
851{% endif -%}
852{% if internalroute -%}
853static_routes="wleiden"
854route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
855{% endif -%}
856
857{% elif nodetype == "Hybrid" %}
858 #
859 # Hybrid Configuration
860 #
861 list_ileiden_proxies="
862 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
863 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
864 {% endfor -%}
865 "
866 list_normal_proxies="
867 {% for item in autogen_normal_proxies -%}
868 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
869 {% endfor -%}
870 "
871
872 {% if autogen_dhcp_interfaces -%}
873 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
874 {% else %}
875 captive_portal_interfaces="dummy"
876 {% endif %}
877 externalif="{{ externalif|default('vr0', true) }}"
878 masterip="{{ masterip }}"
879
880 {% if gateway and service_proxy_ileiden %}
881 defaultrouter="{{ gateway }}"
882 {% else %}
883 #defaultrouter="NOTSET"
884 {% endif %}
885
886 #
887 # Defined services
888 #
889 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
890 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
891 service_accesspoint="{{ service_accesspoint|yesorno }}"
892 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
893 service_concentrator="{{ service_concentrator|yesorno }}"
894
895 {% if service_proxy_ileiden %}
896 pf_rules="/etc/pf.hybrid.conf"
897 {% if service_concentrator %}
898 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
899 {% else %}
900 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
901 {% endif %}
902 pf_flags="$pf_flags -D publicnat=80,443"
903 lvrouted_flags="$lvrouted_flags -g"
904 {% elif service_proxy_normal or service_incoming_rdr %}
905 pf_rules="/etc/pf.hybrid.conf"
906 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
907 pf_flags="$pf_flags -D publicnat=0"
908 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
909 named_setfib="1"
910 tinyproxy_setfib="1"
911 dnsmasq_setfib="1"
912 sshd_setfib="1"
913 {% else %}
914 named_auto_forward_only="YES"
915 pf_rules="/etc/pf.node.conf"
916 pf_flags=""
917 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
918 {% endif %}
919 {% if service_concentrator %}
920 # Do mind installing certificates is NOT done automatically for security reasons
921 openvpn_enable="YES"
922 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
923 {% endif %}
924
925 {% if service_proxy_normal %}
926 tinyproxy_enable="yes"
927 {% else %}
928 pen_wrapper_enable="yes"
929 {% endif %}
930
931 {% if service_accesspoint %}
932 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
933 {% endif %}
934
935 {% if board == "ALIX2" or board == "net4801" %}
936 #
937 # ''Fat'' configuration, board has 256MB RAM
938 #
939 dnsmasq_enable="NO"
940 named_enable="YES"
941 unbound_enable="YES"
942 {% if autogen_dhcp_interfaces -%}
943 dhcpd_enable="YES"
944 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
945 {% endif -%}
946 {% elif board == "apu1d" %}
947 #
948 # ''Fat'' configuration, board has 1024MB RAM
949 #
950 dnsmasq_enable="NO"
951 unbound_enable="YES"
952 {% if autogen_dhcp_interfaces -%}
953 dhcpd_enable="YES"
954 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
955 {% endif -%}
956 {% endif -%}
957{% endif %}
958
959#
960# Script variables
961#
962attached_devices="{{ autogen_attached_devices|join(' ') }}"
963neighbours="{{ autogen_neighbours|join(' ') }}"
964
965
966#
967# Interface definitions
968#\n
969""")
970
971 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
972 for iface, vlans in sorted(vlan_list.items()):
973 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
974
975 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
976 for iface in sorted(vlan_list.keys()):
977 if not iface in addrs_list.keys():
978 output += "ifconfig_%s='up'\n" % iface
979
980 output += "\n"
981
982 # Bridge configuration:
983 if bridge_list.keys():
984 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
985
986 for iface in bridge_list.keys():
987 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
988
989 # Bridge member interfaces not containing a configuration should be marked active explcitly.
990 for _,members in bridge_list.items():
991 for iface in members:
992 if not iface in addrs_list.keys():
993 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
994
995 output += "\n"
996
997 # Details like SSID
998 if extra_ouput:
999 output += extra_ouput.strip() + "\n"
1000
1001 # Print IP address which needs to be assigned over here
1002 output += "\n"
1003 for iface,addrs in sorted(addrs_list.iteritems()):
1004 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1005 output += "# %s || %s || %s\n" % (iface, addr, comment)
1006
1007 # Write DHCLIENT entry
1008 if iface in dhclient_if and dhclient_if[iface]:
1009 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1010 continue
1011
1012 # Make sure the external address is always first as this is needed in the
1013 # firewall setup
1014 addrs = sorted(
1015 [x for x in addrs if not '0.0.0.0' in x[0]],
1016 key=lambda x: x[0].split('.')[0],
1017 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1018 )
1019
1020 idx_offset = 0
1021 # Set MAC is required
1022 if flags_if[iface].has_key('ether'):
1023 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1024 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1025 idx_offset += 1
1026 elif iface in bridge_list:
1027 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1028 idx_offset += 1
1029 else:
1030 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1031
1032 for idx, addr in enumerate(addrs[1:]):
1033 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1034
1035 output += "\n"
1036
1037 rc_conf_local_cache[datadump['autogen_item']] = output
1038 return output
1039
1040
1041
1042def get_all_configs():
1043 """ Get dict with key 'host' with all configs present """
1044 configs = dict()
1045 for host in get_hostlist():
1046 datadump = get_yaml(host)
1047 configs[host] = datadump
1048 return configs
1049
1050
1051def get_interface_keys(config, extra=False):
1052 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1053 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1054 if extra == False:
1055 return filter(lambda x: not "extra" in x, elems)
1056 else:
1057 return elems
1058
1059
1060def get_used_ips(configs):
1061 """ Return array of all IPs used in config files"""
1062 ip_list = []
1063 for config in configs:
1064 ip_list.append(config['masterip'])
1065 if 'serviceid' in config:
1066 ip_list.append(config['serviceid'])
1067 for iface_key in get_interface_keys(config, True):
1068 l = config[iface_key]['ip']
1069 addr, mask = l.split('/')
1070 # Special case do not process
1071 if valid_addr(addr):
1072 ip_list.append(addr)
1073 else:
1074 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1075 return sorted(ip_list)
1076
1077
1078
1079def get_nameservers(max_servers=None):
1080 if nameservers_cache:
1081 return nameservers_cache[0:max_servers]
1082
1083 for host in get_hostlist():
1084 hostdump = get_yaml(host)
1085 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1086 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1087
1088 return nameservers_cache[0:max_servers]
1089
1090
1091def get_neighbours(datadump):
1092 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1093
1094 (poel, errors) = make_relations()
1095 table = []
1096 for iface,addrs in sorted(addrs_list.iteritems()):
1097 if iface in ['lo0']:
1098 continue
1099
1100 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1101 if not addr.startswith('172.'):
1102 # Avoid listing internet connections as pool
1103 continue
1104 for neighbour in poel[network(addr)]:
1105 if neighbour[0] != datadump['autogen_item']:
1106 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1107 return table
1108
1109
1110def get_attached_devices(datadump, url=False):
1111 table = []
1112 for iface_key in get_interface_keys(datadump, True):
1113 ifacedump = datadump[iface_key]
1114
1115 if not ifacedump.has_key('ns_ip'):
1116 continue
1117
1118 x_ip = ifacedump['ns_ip'].split('/')[0]
1119
1120 if 'mode' in ifacedump:
1121 x_mode = ifacedump['mode']
1122 else:
1123 x_mode = 'unknown'
1124
1125 if 'bridge_type' in ifacedump:
1126 device_type = ifacedump['bridge_type']
1127 else:
1128 device_type = 'Unknown'
1129
1130 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1131 return table
1132
1133
1134def generate_resolv_conf(datadump):
1135 """ Generate configuration file '/etc/resolv.conf' """
1136 # XXX: This should properly going to be an datastructure soon
1137 datadump['autogen_header'] = generate_header(datadump, "#")
1138 datadump['autogen_edge_nameservers'] = ''
1139
1140
1141 for masterip,realname in get_nameservers():
1142 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1143
1144 return Template("""\
1145{{ autogen_header }}
1146search wleiden.net
1147
1148# Try local (cache) first
1149nameserver 127.0.0.1
1150
1151{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1152nameserver 8.8.8.8 # Google Public NameServer
1153nameserver 64.6.64.6 # Verisign Public NameServer
1154{% else -%}
1155# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1156{{ autogen_edge_nameservers }}
1157{% endif -%}
1158""").render(datadump)
1159
1160
1161
1162def generate_ntp_conf(datadump):
1163 """ Generate configuration file '/etc/ntp.conf' """
1164 # XXX: This should properly going to be an datastructure soon
1165
1166 datadump['autogen_header'] = generate_header(datadump, "#")
1167 datadump['autogen_ntp_servers'] = ''
1168 for host in get_hostlist():
1169 hostdump = get_yaml(host)
1170 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1171 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1172
1173 return Template("""\
1174{{ autogen_header }}
1175
1176{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1177# Machine hooked to internet.
1178server 0.nl.pool.ntp.org iburst maxpoll 9
1179server 1.nl.pool.ntp.org iburst maxpoll 9
1180server 2.nl.pool.ntp.org iburst maxpoll 9
1181server 3.nl.pool.ntp.org iburst maxpoll 9
1182{% else -%}
1183# Local Wireless Leiden NTP Servers.
1184server 0.pool.ntp.wleiden.net iburst maxpoll 9
1185server 1.pool.ntp.wleiden.net iburst maxpoll 9
1186server 2.pool.ntp.wleiden.net iburst maxpoll 9
1187server 3.pool.ntp.wleiden.net iburst maxpoll 9
1188
1189# All the configured NTP servers
1190{{ autogen_ntp_servers }}
1191{% endif %}
1192
1193# If a server loses sync with all upstream servers, NTP clients
1194# no longer follow that server. The local clock can be configured
1195# to provide a time source when this happens, but it should usually
1196# be configured on just one server on a network. For more details see
1197# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1198# The use of Orphan Mode may be preferable.
1199#
1200server 127.127.1.0
1201fudge 127.127.1.0 stratum 10
1202""").render(datadump)
1203
1204
1205def generate_pf_hybrid_conf_local(datadump):
1206 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1207 datadump['autogen_header'] = generate_header(datadump, "#")
1208 if datadump['service_incoming_rdr']:
1209 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1210 return Template("""\
1211{{ autogen_header }}
1212
1213# Redirect some internal facing services outside (7)
1214# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1215{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1216rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1217{% endfor -%}
1218# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1219{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1220rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1221{% endfor -%}
1222""").render(datadump)
1223
1224def generate_unbound_wleiden_conf(datadump):
1225 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1226 datadump['autogen_header'] = generate_header(datadump, "#")
1227
1228 autogen_ips = []
1229 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1230 for iface,addrs in sorted(addrs_list.iteritems()):
1231 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1232 if addr.startswith('172'):
1233 autogen_ips.append((addr.split('/')[0], comment))
1234 datadump['autogen_ips'] = autogen_ips
1235
1236 create_proxies_list()
1237 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1238 return Template("""\
1239{{ autogen_header }}
1240
1241server:
1242 ## Static definitions fail to start on systems with broken ue(4) interfaces
1243{%- for ip,comment in autogen_ips %}
1244 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1245{%- endfor %}
1246 ## Enabling wildcard matching as work-around
1247 interface: 0.0.0.0
1248 interface: ::0
1249
1250forward-zone:
1251 name: '.'
1252{%- if service_proxy_ileiden %}
1253 forward-addr: 8.8.8.8 # Google DNS A
1254 forward-addr: 8.8.4.4 # Google DNS B
1255 forward-addr: 208.67.222.222 # OpenDNS DNS A
1256 forward-addr: 208.67.220.220 # OpenDNS DNS B
1257{% else -%}
1258{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1259 {%- if loop.index <= 5 %}
1260 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1261 {%- endif %}
1262{%- endfor %}
1263{%- endif %}
1264""").render(datadump)
1265
1266def generate_motd(datadump):
1267 """ Generate configuration file '/etc/motd' """
1268 output = Template("""\
1269FreeBSD run ``service motd onestart'' to make me look normal
1270
1271 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1272 Loc: {{ location }}
1273
1274Services:
1275{% if board == "ALIX2" or board == "net4801" -%}
1276{{" -"}} Core Node ({{ board }})
1277{% else -%}
1278{{" -"}} Hulp Node ({{ board }})
1279{% endif -%}
1280{% if service_proxy_normal -%}
1281{{" -"}} Normal Proxy
1282{% endif -%}
1283{% if service_proxy_ileiden -%}
1284{{" -"}} iLeiden Proxy
1285{% endif -%}
1286{% if service_incoming_rdr -%}
1287{{" -"}} Incoming port redirects
1288{% endif %}
1289Interlinks:\n
1290""").render(datadump)
1291
1292
1293 def make_table(table):
1294 if not table:
1295 return " - none\n"
1296 else:
1297 lines = ""
1298 col_width = [max(len(x) for x in col) for col in zip(*table)]
1299 for row in table:
1300 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1301 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1302 return lines
1303
1304 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1305 table = []
1306 for iface,addrs in sorted(addrs_list.iteritems()):
1307 if iface in ['lo0']:
1308 continue
1309 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1310 table.append((iface, addr, comment))
1311
1312 output += make_table(table)
1313 output += '\n'
1314 output += """\
1315Attached devices:
1316"""
1317 output += make_table(get_attached_devices(datadump, url=True))
1318 output += '\n'
1319 output += """\
1320Available neighbours:
1321"""
1322 output += make_table(get_neighbours(datadump))
1323
1324 return output
1325
1326
1327def format_yaml_value(value):
1328 """ Get yaml value in right syntax for outputting """
1329 if isinstance(value,str):
1330 output = '"%s"' % value
1331 else:
1332 output = value
1333 return output
1334
1335
1336
1337def format_wleiden_yaml(datadump):
1338 """ Special formatting to ensure it is editable"""
1339 output = "# Genesis config yaml style\n"
1340 output += "# vim:ts=2:et:sw=2:ai\n"
1341 output += "#\n"
1342 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1343 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1344 if key == 'rdr_rules':
1345 output += '%-10s:\n' % 'rdr_rules'
1346 for rdr_rule in datadump[key]:
1347 output += '- %s\n' % rdr_rule
1348 else:
1349 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1350
1351 output += "\n\n"
1352
1353 # Format (key, required)
1354 key_order = (
1355 ('comment', True),
1356 ('parent', False),
1357 ('ip', False),
1358 ('ether', False),
1359 ('desc', True),
1360 ('sdesc', True),
1361 ('mode', True),
1362 ('type', True),
1363 ('extra_type', False),
1364 ('channel', False),
1365 ('ssid', False),
1366 ('wlan_mac', False),
1367 ('dhcp', True),
1368 ('dhcp_fixed', False),
1369 ('compass', False),
1370 ('distance', False),
1371 ('ns_ip', False),
1372 ('repeater_ip', False),
1373 ('bullet2_ip', False),
1374 ('ns_mac', False),
1375 ('bullet2_mac', False),
1376 ('ns_type', False),
1377 ('bridge_type', False),
1378 ('status', True),
1379 )
1380
1381 for iface_key in sorted(iface_keys):
1382 try:
1383 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1384 if remainder:
1385 raise KeyError("invalid keys: %s" % remainder)
1386
1387 output += "%s:\n" % iface_key
1388 for key,required in key_order:
1389 if datadump[iface_key].has_key(key):
1390 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1391 output += "\n\n"
1392 except Exception:
1393 print "# Error while processing interface %s" % iface_key
1394 raise
1395
1396 return output
1397
1398
1399
1400def generate_wleiden_yaml(datadump, header=True):
1401 """ Generate (petty) version of wleiden.yaml"""
1402 output = generate_header(datadump, "#") if header else ''
1403
1404 for key in datadump.keys():
1405 if key.startswith('autogen_'):
1406 del datadump[key]
1407 # Interface autogen cleanups
1408 elif type(datadump[key]) == dict:
1409 for key2 in datadump[key].keys():
1410 if key2.startswith('autogen_'):
1411 del datadump[key][key2]
1412
1413 output += format_wleiden_yaml(datadump)
1414 return output
1415
1416def generate_nanostation_config(datadump, iface, ns_type):
1417 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1418 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1419
1420 datadump.update(datadump['iface_%s' % iface])
1421
1422 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1423
1424def generate_yaml(datadump):
1425 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1426
1427
1428
1429def generate_config(node, config, datadump=None):
1430 """ Print configuration file 'config' of 'node' """
1431 output = ""
1432 try:
1433 # Load config file
1434 if datadump == None:
1435 datadump = get_yaml(node)
1436
1437 if config == 'wleiden.yaml':
1438 output += generate_wleiden_yaml(datadump)
1439 elif config == 'authorized_keys':
1440 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1441 output += f.read()
1442 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1443 # Fetch local keys if existing
1444 if os.path.exists(node_keys):
1445 output += open(node_keys, 'r').read()
1446 f.close()
1447 elif config == 'dnsmasq.conf':
1448 output += generate_dnsmasq_conf(datadump)
1449 elif config == 'dhcpd.conf':
1450 output += generate_dhcpd_conf(datadump)
1451 elif config == 'rc.conf.local':
1452 output += generate_rc_conf_local(datadump)
1453 elif config == 'resolv.conf':
1454 output += generate_resolv_conf(datadump)
1455 elif config == 'ntp.conf':
1456 output += generate_ntp_conf(datadump)
1457 elif config == 'motd':
1458 output += generate_motd(datadump)
1459 elif config == 'pf.hybrid.conf.local':
1460 output += generate_pf_hybrid_conf_local(datadump)
1461 elif config == 'unbound.wleiden.conf':
1462 output += generate_unbound_wleiden_conf(datadump)
1463 elif config.startswith('vr'):
1464 interface, ns_type = config.strip('.yaml').split('-')
1465 output += generate_nanostation_config(datadump, interface, ns_type)
1466 else:
1467 assert False, "Config not found!"
1468 except IOError, e:
1469 output += "[ERROR] Config file not found"
1470 return output
1471
1472
1473
1474def process_cgi_request(environ=os.environ):
1475 """ When calling from CGI """
1476 response_headers = []
1477 content_type = 'text/plain'
1478
1479 # Update repository if requested
1480 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1481 if form and form.has_key("action") and "update" in form["action"]:
1482 output = "[INFO] Updating subverion, please wait...\n"
1483 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1484 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1485 output += "[INFO] All done, redirecting in 5 seconds"
1486 response_headers += [
1487 ('Refresh', '5; url=.'),
1488 ]
1489 reload_cache()
1490 else:
1491 base_uri = environ['PATH_INFO']
1492 uri = base_uri.strip('/').split('/')
1493
1494 output = "Template Holder"
1495 if base_uri.endswith('/create/network.kml'):
1496 content_type='application/vnd.google-earth.kml+xml'
1497 output = make_network_kml.make_graph()
1498 elif base_uri.endswith('/api/get/nodeplanner.json'):
1499 content_type='application/json'
1500 output = make_network_kml.make_nodeplanner_json()
1501 elif not uri[0]:
1502 if is_text_request(environ):
1503 output = '\n'.join(get_hostlist())
1504 else:
1505 content_type = 'text/html'
1506 output = generate_title(get_hostlist())
1507 elif len(uri) == 1:
1508 if is_text_request(environ):
1509 output = generate_node(uri[0])
1510 else:
1511 content_type = 'text/html'
1512 output = generate_node_overview(uri[0])
1513 elif len(uri) == 2:
1514 output = generate_config(uri[0], uri[1])
1515 else:
1516 assert False, "Invalid option"
1517
1518 # Return response
1519 response_headers += [
1520 ('Content-type', content_type),
1521 ('Content-Length', str(len(output))),
1522 ]
1523 return(response_headers, str(output))
1524
1525
1526def make_dns(output_dir = 'dns', external = False):
1527 items = dict()
1528
1529 # hostname is key, IP is value
1530 wleiden_zone = defaultdict(list)
1531 wleiden_cname = dict()
1532
1533 pool = dict()
1534 for node in get_hostlist():
1535 datadump = get_yaml(node)
1536
1537 fqdn = datadump['nodename']
1538
1539 if datadump.has_key('rdr_host'):
1540 remote_target = datadump['rdr_host']
1541 elif datadump.has_key('remote_access') and datadump['remote_access']:
1542 remote_target = datadump['remote_access'].split(':')[0]
1543 else:
1544 remote_target = None
1545
1546 if remote_target:
1547 try:
1548 parseaddr(remote_target)
1549 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1550 except (IndexError, ValueError):
1551 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1552
1553
1554 wleiden_zone[fqdn].append((datadump['masterip'], True))
1555
1556 # Hacking to get proper DHCP IPs and hostnames
1557 for iface_key in get_interface_keys(datadump):
1558 iface_name = iface_key.replace('_','-')
1559 if 'ip' in datadump[iface_key]:
1560 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1561 try:
1562 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1563 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1564 dhcp_part = ".".join(ip.split('.')[0:3])
1565 if ip != datadump['masterip']:
1566 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1567 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1568 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1569 except (AttributeError, ValueError, KeyError):
1570 # First push it into a pool, to indentify the counter-part later on
1571 addr = parseaddr(ip)
1572 cidr = int(cidr)
1573 addr = addr & ~((1 << (32 - cidr)) - 1)
1574 if pool.has_key(addr):
1575 pool[addr] += [(iface_name, fqdn, ip)]
1576 else:
1577 pool[addr] = [(iface_name, fqdn, ip)]
1578 continue
1579
1580
1581
1582 # WL uses an /29 to configure an interface. IP's are ordered like this:
1583 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1584
1585 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1586
1587 # Automatic naming convention of interlinks namely 2 + remote.lower()
1588 for (key,value) in pool.iteritems():
1589 # Make sure they are sorted from low-ip to high-ip
1590 value = sorted(value, key=lambda x: parseaddr(x[2]))
1591
1592 if len(value) == 1:
1593 (iface_name, fqdn, ip) = value[0]
1594 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1595
1596 # Device DNS names
1597 if 'cnode' in fqdn.lower():
1598 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1599 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1600
1601 elif len(value) == 2:
1602 (a_iface_name, a_fqdn, a_ip) = value[0]
1603 (b_iface_name, b_fqdn, b_ip) = value[1]
1604 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1605 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1606
1607 # Device DNS names
1608 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1609 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1610 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1611 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1612 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1613 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1614 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1615
1616 else:
1617 pool_members = [k[1] for k in value]
1618 for item in value:
1619 (iface_name, fqdn, ip) = item
1620 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1621
1622 # Include static DNS entries
1623 # XXX: Should they override the autogenerated results?
1624 # XXX: Convert input to yaml more useable.
1625 # Format:
1626 ##; this is a comment
1627 ## roomburgh=Roomburgh1
1628 ## apkerk1.Vosko=172.17.176.8 ;this as well
1629 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1630
1631 # Hack to allow special entries, for development
1632 wleiden_raw = {}
1633
1634 for line in dns_list:
1635 reverse = False
1636 k, items = line.items()[0]
1637 if type(items) == dict:
1638 if items.has_key('reverse'):
1639 reverse = items['reverse']
1640 items = items['a']
1641 else:
1642 items = items['cname']
1643 items = [items] if type(items) != list else items
1644 for item in items:
1645 if item.startswith('IN '):
1646 wleiden_raw[k] = item
1647 elif valid_addr(item):
1648 wleiden_zone[k].append((item, reverse))
1649 else:
1650 wleiden_cname[k] = item
1651
1652 # Hack to get dynamic pool listing
1653 def chunks(l, n):
1654 return [l[i:i+n] for i in range(0, len(l), n)]
1655
1656 ntp_servers = [x[0] for x in get_nameservers()]
1657 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1658 for ntp_server in chunk:
1659 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1660
1661 details = dict()
1662 # 24 updates a day allowed
1663 details['serial'] = time.strftime('%Y%m%d%H')
1664
1665 if external:
1666 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1667 else:
1668 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1669
1670 details['master'] = dns_masters[0]
1671 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1672
1673 dns_header = '''
1674$TTL 3h
1675%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1676 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1677
1678%(ns_servers)s
1679 \n'''
1680
1681
1682 if not os.path.isdir(output_dir):
1683 os.makedirs(output_dir)
1684 details['zone'] = 'wleiden.net'
1685 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1686 f.write(dns_header % details)
1687
1688 for host,items in wleiden_zone.iteritems():
1689 for ip,reverse in items:
1690 if ip not in ['0.0.0.0']:
1691 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1692 for source,dest in wleiden_cname.iteritems():
1693 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1694 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1695 for source, dest in wleiden_raw.iteritems():
1696 f.write("%s.wleiden.net. %s\n" % (source, dest))
1697 f.close()
1698
1699 # Create whole bunch of specific sub arpa zones. To keep it compliant
1700 for s in range(16,32):
1701 details['zone'] = '%i.172.in-addr.arpa' % s
1702 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1703 f.write(dns_header % details)
1704
1705 #XXX: Not effient, fix to proper data structure and do checks at other
1706 # stages
1707 for host,items in wleiden_zone.iteritems():
1708 for ip,reverse in items:
1709 if not reverse:
1710 continue
1711 if valid_addr(ip):
1712 if valid_addr(ip):
1713 if int(ip.split('.')[1]) == s:
1714 rev_ip = '.'.join(reversed(ip.split('.')))
1715 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1716 f.close()
1717
1718
1719def usage():
1720 print """Usage: %(prog)s <argument>
1721Argument:
1722\tcleanup = Cleanup all YAML files to specified format
1723\tstandalone [port] = Run configurator webserver [8000]
1724\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1725\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1726\tfull-export = Generate yaml export script for heatmap.
1727\tstatic [outputdir] = Generate all config files and store on disk
1728\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1729\ttest <node> [<file>] = Receive output for certain node [all files].
1730\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1731\tlist <status> <items> = List systems which have certain status
1732\tcreate network.kml = Create Network KML file for use in Google Earth
1733
1734Arguments:
1735\t<node> = NodeName (example: HybridRick)
1736\t<file> = %(files)s
1737\t<status> = all|up|down|planned
1738\t<items> = systems|nodes|proxies
1739
1740NOTE FOR DEVELOPERS; you can test your changes like this:
1741 BEFORE any changes in this code:
1742 $ ./gformat.py static /tmp/pre
1743 AFTER the changes:
1744 $ ./gformat.py static /tmp/post
1745 VIEW differences and VERIFY all are OK:
1746 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1747""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1748 exit(0)
1749
1750
1751def is_text_request(environ=os.environ):
1752 """ Find out whether we are calling from the CLI or any text based CLI utility """
1753 try:
1754 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1755 except KeyError:
1756 return True
1757
1758def switchFormat(setting):
1759 if setting:
1760 return "YES"
1761 else:
1762 return "NO"
1763
1764def rlinput(prompt, prefill=''):
1765 import readline
1766 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1767 try:
1768 return raw_input(prompt)
1769 finally:
1770 readline.set_startup_hook()
1771
1772def fix_conflict(left, right, default='i'):
1773 while True:
1774 print "## %-30s | %-30s" % (left, right)
1775 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1776 if not c:
1777 c = default
1778
1779 if c in ['l','1']:
1780 return left
1781 elif c in ['r','2']:
1782 return right
1783 elif c in ['e', '3']:
1784 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1785 elif c in ['i', '4']:
1786 return None
1787 else:
1788 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1789
1790
1791
1792def print_cgi_response(response_headers, output):
1793 """Could we not use some kind of wsgi wrapper to make this output?"""
1794 for header in response_headers:
1795 print "%s: %s" % header
1796 print
1797 print output
1798
1799
1800def fill_cache():
1801 ''' Poor man re-loading of few cache items (the slow ones) '''
1802 for host in get_hostlist():
1803 get_yaml(host)
1804
1805
1806def reload_cache():
1807 clear_cache()
1808 fill_cache()
1809
1810
1811def main():
1812 """Hard working sub"""
1813 # Allow easy hacking using the CLI
1814 if not os.environ.has_key('PATH_INFO'):
1815 if len(sys.argv) < 2:
1816 usage()
1817
1818 if sys.argv[1] == "standalone":
1819 import SocketServer
1820 import CGIHTTPServer
1821 # Hop to the right working directory.
1822 os.chdir(os.path.dirname(__file__))
1823 try:
1824 PORT = int(sys.argv[2])
1825 except (IndexError,ValueError):
1826 PORT = 8000
1827
1828 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1829 """ Serve this CGI from the root of the webserver """
1830 def is_cgi(self):
1831 if "favicon" in self.path:
1832 return False
1833
1834 self.cgi_info = (os.path.basename(__file__), self.path)
1835 self.path = ''
1836 return True
1837 handler = MyCGIHTTPRequestHandler
1838 SocketServer.TCPServer.allow_reuse_address = True
1839 httpd = SocketServer.TCPServer(("", PORT), handler)
1840 httpd.server_name = 'localhost'
1841 httpd.server_port = PORT
1842
1843 logger.info("serving at port %s", PORT)
1844 try:
1845 httpd.serve_forever()
1846 except KeyboardInterrupt:
1847 httpd.shutdown()
1848 logger.info("All done goodbye")
1849 elif sys.argv[1] == "test":
1850 # Basic argument validation
1851 try:
1852 node = sys.argv[2]
1853 except IndexError:
1854 print "Invalid argument"
1855 exit(1)
1856 except IOError as e:
1857 print e
1858 exit(1)
1859
1860 datadump = get_yaml(node)
1861
1862
1863 # Get files to generate
1864 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1865
1866 # Actual config generation
1867 for config in gen_files:
1868 logger.info("## Generating %s %s", node, config)
1869 print generate_config(node, config, datadump)
1870 elif sys.argv[1] == "test-cgi":
1871 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1872 os.environ['SCRIPT_NAME'] = __file__
1873 response_headers, output = process_cgi_request()
1874 print_cgi_response(response_headers, output)
1875 elif sys.argv[1] == "static":
1876 items = dict()
1877 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1878 for node in get_hostlist():
1879 items['node'] = node
1880 items['wdir'] = "%(output_dir)s/%(node)s" % items
1881 if not os.path.isdir(items['wdir']):
1882 os.makedirs(items['wdir'])
1883 datadump = get_yaml(node)
1884 for config in files:
1885 items['config'] = config
1886 logger.info("## Generating %(node)s %(config)s" % items)
1887 f = open("%(wdir)s/%(config)s" % items, "w")
1888 f.write(generate_config(node, config, datadump))
1889 f.close()
1890 elif sys.argv[1] == "wind-export":
1891 items = dict()
1892 for node in get_hostlist():
1893 datadump = get_yaml(node)
1894 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1895 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1896 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1897 VALUES (
1898 (SELECT id FROM users WHERE username = 'rvdzwet'),
1899 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1900 'Y');""" % datadump
1901 #for config in files:
1902 # items['config'] = config
1903 # print "## Generating %(node)s %(config)s" % items
1904 # f = open("%(wdir)s/%(config)s" % items, "w")
1905 # f.write(generate_config(node, config, datadump))
1906 # f.close()
1907 for node in get_hostlist():
1908 datadump = get_yaml(node)
1909 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1910 ifacedump = datadump[iface_key]
1911 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1912 ifacedump['nodename'] = datadump['nodename']
1913 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1914 ifacedump['channel'] = 0
1915 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1916 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1917 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1918 elif sys.argv[1] == "nagios-export":
1919 try:
1920 heavy_load = (sys.argv[2] == "--heavy-load")
1921 except IndexError:
1922 heavy_load = False
1923
1924 hostgroup_details = {
1925 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1926 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1927 'walphen' : 'Stichting Wireless Alphen',
1928 'westeinder' : 'Westeinder Plassen',
1929 }
1930
1931 # Convert IP to Host
1932 ip2host = {'root' : 'root'}
1933 for host in get_hostlist():
1934 datadump = get_yaml(host)
1935 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1936 for iface in get_interface_keys(datadump):
1937 if datadump[iface].has_key('autogen_gateway'):
1938 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1939
1940 # Find dependency tree based on output of lvrouted.mytree of nearest node
1941 parents = defaultdict(list)
1942 stack = ['root']
1943 prev_depth = 0
1944 for line in open('lvrouted.mytree').readlines():
1945 depth = line.count('\t')
1946 ip = line.strip().split()[0]
1947
1948 if prev_depth < depth:
1949 try:
1950 parents[ip2host[ip]].append(ip2host[stack[-1]])
1951 except KeyError as e:
1952 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1953 stack.append(ip)
1954 elif prev_depth > depth:
1955 stack = stack[:(depth - prev_depth)]
1956 elif prev_depth == depth:
1957 try:
1958 parents[ip2host[ip]].append(ip2host[stack[-1]])
1959 except KeyError as e:
1960 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1961
1962
1963 prev_depth = depth
1964 # Observe that some nodes has themself as parent or multiple parents
1965 # for now take only the first parent, other behaviour is yet to be explained
1966
1967
1968
1969 params = {
1970 'check_interval' : 5 if heavy_load else 120,
1971 'retry_interval' : 1 if heavy_load else 10,
1972 'max_check_attempts' : 10 if heavy_load else 6,
1973 'notification_interval': 120 if heavy_load else 240,
1974 }
1975
1976 print '''\
1977define host {
1978 name wleiden-node ; Default Node Template
1979 use generic-host ; Use the standard template as initial starting point
1980 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1981 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1982 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1983 notification_interval %(notification_interval)s
1984 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1985 check_command check-host-alive ; Default command to check FreeBSD hosts
1986 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1987}
1988
1989define service {
1990 name wleiden-service ; Default Service Template
1991 use generic-service ; Use the standard template as initial starting point
1992 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1993 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1994 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1995 notification_interval %(notification_interval)s
1996 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1997 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1998}
1999
2000# Please make sure to install:
2001# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2002#
2003# Recompile net-mgmt/nagios-plugins to support check_snmp
2004# make -C /usr/ports/net-mgmt/nagios-plugins
2005#
2006# Install net/bind-tools to allow v2/check_dns_wl to work:
2007# pkg install bind-tools
2008#
2009define command{
2010 command_name check_snmp_disk
2011 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2012}
2013
2014define command{
2015 command_name check_netsnmp_load
2016 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2017}
2018
2019define command{
2020 command_name check_netsnmp_proc
2021 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2022}
2023
2024define command{
2025 command_name check_by_ssh
2026 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2027}
2028
2029define command{
2030 command_name check_dns_wl
2031 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2032}
2033
2034define command{
2035 command_name check_snmp_uptime
2036 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2037}
2038
2039
2040# TDB: dhcp leases
2041# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2042
2043# TDB: internet status
2044# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2045
2046# TDB: Advanced local passive checks
2047# /usr/local/libexec/nagios/check_by_ssh
2048''' % params
2049
2050 print '''\
2051# Service Group, not displayed by default
2052define hostgroup {
2053 hostgroup_name srv_hybrid
2054 alias All Hybrid Nodes
2055 register 0
2056}
2057
2058define service {
2059 use wleiden-service
2060 hostgroup_name srv_hybrid
2061 service_description SSH
2062 check_command check_ssh
2063}
2064
2065define service {
2066 use wleiden-service,service-pnp
2067 hostgroup_name srv_hybrid
2068 service_description HTTP
2069 check_command check_http
2070}
2071
2072define service {
2073 use wleiden-service
2074 hostgroup_name srv_hybrid
2075 service_description DNS
2076 check_command check_dns_wl!"www.wirelessleiden.nl"
2077}
2078'''
2079
2080 if heavy_load:
2081 print '''\
2082define service {
2083 use wleiden-service
2084 hostgroup_name srv_hybrid
2085 service_description UPTIME
2086 check_command check_snmp_uptime
2087}
2088
2089#define service {
2090# use wleiden-service
2091# hostgroup_name srv_hybrid
2092# service_description NTP
2093# check_command check_ntp_peer
2094#}
2095
2096define service {
2097 use wleiden-service
2098 hostgroup_name srv_hybrid
2099 service_description LOAD
2100 check_command check_netsnmp_load
2101}
2102
2103define service {
2104 use wleiden-service
2105 hostgroup_name srv_hybrid
2106 service_description PROC
2107 check_command check_netsnmp_proc
2108}
2109
2110define service {
2111 use wleiden-service
2112 hostgroup_name srv_hybrid
2113 service_description DISK
2114 check_command check_snmp_disk
2115}
2116'''
2117 for node in get_hostlist():
2118 datadump = get_yaml(node)
2119 if not datadump['status'] == 'up':
2120 continue
2121 if not hostgroup_details.has_key(datadump['monitoring_group']):
2122 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2123 print '''\
2124define host {
2125 use wleiden-node,host-pnp
2126 contact_groups admins
2127 host_name %(autogen_fqdn)s
2128 address %(masterip)s
2129 hostgroups srv_hybrid,%(monitoring_group)s\
2130''' % datadump
2131 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2132 print '''\
2133 parents %(parents)s\
2134''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2135 print '''\
2136}
2137'''
2138
2139
2140 for name,alias in hostgroup_details.iteritems():
2141 print '''\
2142define hostgroup {
2143 hostgroup_name %s
2144 alias %s
2145} ''' % (name, alias)
2146
2147
2148 elif sys.argv[1] == "full-export":
2149 hosts = {}
2150 for node in get_hostlist():
2151 datadump = get_yaml(node)
2152 hosts[datadump['nodename']] = datadump
2153 print yaml.dump(hosts)
2154
2155 elif sys.argv[1] == "dns":
2156 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2157 elif sys.argv[1] == "cleanup":
2158 # First generate all datadumps
2159 datadumps = dict()
2160 ssid_to_node = dict()
2161 for host in get_hostlist():
2162 logger.info("# Processing: %s", host)
2163 # Set some boring default values
2164 datadump = { 'board' : 'UNKNOWN' }
2165 datadump.update(get_yaml(host))
2166 datadumps[datadump['nodename']] = datadump
2167
2168 (poel, errors) = make_relations()
2169 print "\n".join(["# WARNING: %s" % x for x in errors])
2170
2171 for host,datadump in datadumps.iteritems():
2172 try:
2173 # Convert all yes and no to boolean values
2174 def fix_boolean(dump):
2175 for key in dump.keys():
2176 if type(dump[key]) == dict:
2177 dump[key] = fix_boolean(dump[key])
2178 elif str(dump[key]).lower() in ["yes", "true"]:
2179 dump[key] = True
2180 elif str(dump[key]).lower() in ["no", "false"]:
2181 # Compass richting no (Noord Oost) is valid input
2182 if key != "compass": dump[key] = False
2183 return dump
2184 datadump = fix_boolean(datadump)
2185
2186 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2187 if not 'latitude' in datadump and not 'longitude' in datadump:
2188 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2189 elif 'latitude' in datadump and 'longitude' in datadump:
2190 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2191 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2192 # TODO: Compare outcome of both coordinate systems and validate against each-other
2193
2194 if datadump['nodename'].startswith('Proxy'):
2195 datadump['nodename'] = datadump['nodename'].lower()
2196
2197 for iface_key in get_interface_keys(datadump):
2198 try:
2199 # All our normal wireless cards are normal APs now
2200 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2201 datadump[iface_key]['mode'] = 'ap'
2202 # Wireless Leiden SSID have an consistent lowercase/uppercase
2203 if datadump[iface_key].has_key('ssid'):
2204 ssid = datadump[iface_key]['ssid']
2205 prefix = 'ap-WirelessLeiden-'
2206 if ssid.lower().startswith(prefix.lower()):
2207 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2208 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2209 datadump[iface_key]['mode'] = 'autogen-FIXME'
2210 if not datadump[iface_key].has_key('comment'):
2211 datadump[iface_key]['comment'] = 'autogen-FIXME'
2212
2213 if datadump[iface_key].has_key('ns_mac'):
2214 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2215
2216 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2217 datadump[iface_key] = datadump[iface_key]['desc']
2218
2219 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2220 # due to better collision avoidance.
2221 if datadump[iface_key]['type'] == '11b':
2222 datadump[iface_key]['type'] = '11g'
2223
2224 # Setting 802.11g channels to de-facto standards, to avoid
2225 # un-detected sharing with other overlapping channels
2226 #
2227 # Technically we could also use channel 13 in NL, but this is not
2228 # recommended as foreign devices might not be able to select this
2229 # channel. Secondly using 1,5,9,13 instead is going to clash with
2230 # the de-facto usage of 1,6,11.
2231 #
2232 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2233 channels_at_2400Mhz = (1,6,11)
2234 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2235 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2236 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2237 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2238
2239 # Mandatory interface keys
2240 if not datadump[iface_key].has_key('status'):
2241 datadump[iface_key]['status'] = 'planned'
2242
2243 x = datadump[iface_key]['comment']
2244 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2245
2246 # Fixing bridge_type if none is found
2247 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2248 if not 'bridge_type' in datadump[iface_key]:
2249 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2250
2251 # Making sure description works
2252 if datadump[iface_key].has_key('desc'):
2253 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2254 del datadump[iface_key]['desc']
2255 else:
2256 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2257 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2258 if response:
2259 datadump[iface_key]['comment'] = response
2260 del datadump[iface_key]['desc']
2261
2262 # Check DHCP configuration
2263 dhcp_type(datadump[iface_key])
2264
2265 # Set the compass value based on the angle between the poels
2266 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2267 my_pool = poel[network(datadump[iface_key]['ip'])]
2268 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2269 if remote_hosts:
2270 compass_target = remote_hosts[0]
2271 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2272 # TODO: Compass wanted and actual direction might differ
2273
2274 # Monitoring Group default
2275 if not 'monitoring_group' in datadump:
2276 datadump['monitoring_group'] = 'wleiden'
2277
2278 except Exception:
2279 print "# Error while processing interface %s" % iface_key
2280 raise
2281 store_yaml(datadump)
2282 except Exception:
2283 print "# Error while processing %s" % host
2284 raise
2285 elif sys.argv[1] == "list":
2286 use_fqdn = False
2287 if len(sys.argv) < 4:
2288 usage()
2289 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2290 usage()
2291 if not sys.argv[3] in ["nodes","proxies","systems"]:
2292 usage()
2293
2294 if len(sys.argv) > 4:
2295 if sys.argv[4] == "fqdn":
2296 use_fqdn = True
2297 else:
2298 usage()
2299
2300 for system in get_hostlist():
2301 datadump = get_yaml(system)
2302 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2303 continue
2304
2305 output = datadump['autogen_fqdn'] if use_fqdn else system
2306 if sys.argv[2] == "all":
2307 print output
2308 elif datadump['status'] == sys.argv[2]:
2309 print output
2310 elif sys.argv[1] == "create":
2311 if sys.argv[2] == "network.kml":
2312 print make_network_kml.make_graph()
2313 elif sys.argv[2] == "host-ips.txt":
2314 for system in get_hostlist():
2315 datadump = get_yaml(system)
2316 ips = [datadump['masterip']]
2317 for ifkey in get_interface_keys(datadump):
2318 ips.append(datadump[ifkey]['ip'].split('/')[0])
2319 print system, ' '.join(ips)
2320 elif sys.argv[2] == "host-pos.txt":
2321 for system in get_hostlist():
2322 datadump = get_yaml(system)
2323 print system, datadump['rdnap_x'], datadump['rdnap_y']
2324 elif sys.argv[2] == 'ssh_config':
2325 print '''
2326Host *.wleiden.net
2327 User root
2328
2329Host 172.16.*.*
2330 User root
2331'''
2332 for system in get_hostlist():
2333 datadump = get_yaml(system)
2334 print '''\
2335Host %s
2336 User root
2337
2338Host %s
2339 User root
2340
2341Host %s
2342 User root
2343
2344Host %s
2345 User root
2346''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2347 else:
2348 usage()
2349 else:
2350 usage()
2351 else:
2352 # Do not enable debugging for config requests as it highly clutters the output
2353 if not is_text_request():
2354 cgitb.enable()
2355 response_headers, output = process_cgi_request()
2356 print_cgi_response(response_headers, output)
2357
2358def application(environ, start_response):
2359 status = '200 OK'
2360 response_headers, output = process_cgi_request(environ)
2361 start_response(status, response_headers)
2362
2363 # Debugging only
2364 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2365 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2366 return [output]
2367
2368if __name__ == "__main__":
2369 main()
Note: See TracBrowser for help on using the repository browser.