source: genesis/tools/gformat.py@ 13936

Last change on this file since 13936 was 13936, checked in by rick, 8 years ago

Add text/plain to list of accepted plain targets
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 79.5 KB
Line 
1#!/usr/local/bin/python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
35
36import argparse
37import cgi
38import cgitb
39import copy
40import glob
41import make_network_kml
42import math
43import pyproj
44import random
45import re
46import socket
47import string
48import subprocess
49import textwrap
50import time
51import urlparse
52
53from pprint import pprint
54from collections import defaultdict, OrderedDict
55from sys import stderr
56try:
57 import yaml
58except ImportError, e:
59 print e
60 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
61 exit(1)
62
63try:
64 from yaml import CLoader as Loader
65 from yaml import CDumper as Dumper
66except ImportError:
67 from yaml import Loader, Dumper
68
69from jinja2 import Environment, Template
70def yesorno(value):
71 return "YES" if bool(value) else "NO"
72env = Environment()
73env.filters['yesorno'] = yesorno
74def render_template(datadump, template):
75 result = env.from_string(template).render(datadump)
76 # Make it look pretty to the naked eye, as jinja templates are not so
77 # friendly when it comes to whitespace formatting
78 ## Remove extra whitespace at end of line lstrip() style.
79 result = re.sub(r'\n[\ ]+','\n', result)
80 ## Include only a single newline between an definition and a comment
81 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
82 ## Remove extra newlines after single comment
83 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
84 return result
85
86import logging
87logging.basicConfig(format='# %(levelname)s: %(message)s' )
88logger = logging.getLogger()
89logger.setLevel(logging.DEBUG)
90
91
92if os.environ.has_key('CONFIGROOT'):
93 NODE_DIR = os.environ['CONFIGROOT']
94else:
95 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
96__version__ = '$Id: gformat.py 13936 2017-07-06 09:00:04Z rick $'
97
98CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
99
100files = [
101 'authorized_keys',
102 'dnsmasq.conf',
103 'dhcpd.conf',
104 'rc.conf.local',
105 'resolv.conf',
106 'motd',
107 'ntp.conf',
108 'pf.hybrid.conf.local',
109 'unbound.wleiden.conf',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = OrderedDict()
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126def clear_cache():
127 ''' Poor mans cache implementation '''
128 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
129 datadump_cache = {}
130 interface_list_cache = {}
131 rc_conf_local_cache = {}
132 ileiden_proxies = OrderedDict()
133 normal_proxies = []
134 nameservers_cache = []
135 relations_cache = None
136
137NO_DHCP = 0
138DHCP_CLIENT = 10
139DHCP_SERVER = 20
140def dhcp_type(item):
141 if not item.has_key('dhcp'):
142 return NO_DHCP
143 elif not item['dhcp']:
144 return NO_DHCP
145 elif item['dhcp'].lower() == 'client':
146 return DHCP_CLIENT
147 else:
148 # Validation Checks
149 begin,end = map(int,item['dhcp'].split('-'))
150 if begin >= end:
151 raise ValueError("DHCP Start >= DHCP End")
152 return DHCP_SERVER
153
154def etrs2rd(lat, lon):
155 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
156 p2 = pyproj.Proj(init='EPSG:28992')
157 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
158 return (RDx, RDy)
159
160def rd2etrs(RDx, RDy):
161 p1 = pyproj.Proj(init='EPSG:28992')
162 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
163 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
164 return (lat, lon)
165
166def get_yaml(item,add_version_info=True):
167 try:
168 """ Get configuration yaml for 'item'"""
169 if datadump_cache.has_key(item):
170 return datadump_cache[item].copy()
171
172 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
173 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
174 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
175
176 # Default values
177 datadump = {
178 'autogen_revision' : 'NOTFOUND',
179 'autogen_gfile' : gfile,
180 'service_proxy_ileiden' : False,
181 }
182 f = open(gfile, 'r')
183 datadump.update(yaml.load(f,Loader=Loader))
184 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
185 if datadump['nodetype'] == 'Hybrid':
186 # Some values are defined implicitly
187 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
188 datadump['service_incoming_rdr'] = True
189 # Use some boring defaults
190 defaults = {
191 'service_proxy_normal' : False,
192 'service_accesspoint' : True,
193 'service_incoming_rdr' : False,
194 'service_concentrator' : False,
195 'monitoring_group' : 'wleiden',
196 }
197 for (key,value) in defaults.iteritems():
198 if not datadump.has_key(key):
199 datadump[key] = value
200 f.close()
201
202 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
203 if add_version_info:
204 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
205 lines = p.communicate()[0].split('\n')
206 if p.returncode == 0:
207 for line in lines:
208 if line:
209 (key, value) = line.split(': ')
210 datadump["autogen_" + key.lower().replace(' ','_')] = value
211
212 # Preformat certain needed variables for formatting and push those into special object
213 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
214
215 wlan_count=0
216 try:
217 for key in get_interface_keys(datadump, True):
218 datadump[key]['autogen_ifbase'] = key.split('_')[1]
219 datadump[key]['autogen_vlan'] = False
220
221 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
222 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
223 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
224
225 if datadump[key].has_key('parent'):
226 if datadump[key].has_key('ip'):
227 raise ValueError("Interface bridge member cannot have IP assigned")
228 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
229 raise ValueError("Interface bridge member cannot have DHCP set")
230
231 if datadump[key].has_key('ip'):
232 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
233
234 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
235 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
236 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
237 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
238 wlan_count += 1
239 else:
240 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
241 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
242 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
243 datadump[key]['autogen_vlan'] = key.split('_')[2]
244 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
245 else:
246 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
247
248 except Exception:
249 print "# Error while processing interface %s" % key
250 raise
251
252 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
253 if dhcp_type(datadump[key]) == DHCP_SERVER]
254
255 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
256 datadump['autogen_item'] = item
257
258 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
259 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
260 datadump_cache[item] = datadump.copy()
261 except Exception:
262 print "# Error while processing %s" % item
263 raise
264 return datadump
265
266
267def store_yaml(datadump, header=False):
268 """ Store configuration yaml for 'item'"""
269 item = datadump['autogen_item']
270 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
271
272 output = generate_wleiden_yaml(datadump, header)
273
274 f = open(gfile, 'w')
275 f.write(output)
276 f.close()
277
278
279def network(ip):
280 addr, mask = ip.split('/')
281 # Not parsing of these folks please
282 addr = parseaddr(addr)
283 mask = int(mask)
284 network = addr & ~((1 << (32 - mask)) - 1)
285 return network
286
287
288
289def make_relations():
290 """ Process _ALL_ yaml files to get connection relations """
291 global relations_cache
292
293 if relations_cache:
294 return relations_cache
295
296 errors = []
297 poel = defaultdict(list)
298
299 for host in get_hostlist():
300 datadump = get_yaml(host)
301 try:
302 for iface_key in get_interface_keys(datadump):
303 # Bridge members has no IP assigned
304 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
305 continue
306 net_addr = network(datadump[iface_key]['ip'])
307 poel[net_addr] += [(host,datadump[iface_key].copy())]
308 except (KeyError, ValueError) as e:
309 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
310 continue
311
312 relations_cache = (poel, errors)
313 return relations_cache
314
315
316
317def valid_addr(addr):
318 """ Show which address is valid in which are not """
319 return str(addr).startswith('172.')
320
321def get_hostlist():
322 """ Combined hosts and proxy list"""
323 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
324
325def angle_between_points(lat1,lat2,long1,long2):
326 """
327 Return Angle in radians between two GPS coordinates
328 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
329 """
330 dy = lat2 - lat1
331 dx = math.cos(lat1)*(long2 - long1)
332 angle = math.atan2(dy,dx)
333 return angle
334
335
336
337def angle_to_cd(angle):
338 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
339
340 # For easy conversion get positive degree
341 degrees = math.degrees(angle)
342 abs_degrees = 360 + degrees if degrees < 0 else degrees
343
344 # Numbers can be confusing calculate from the 4 main directions
345 p = 22.5
346 if abs_degrees < p:
347 cd = "n"
348 elif abs_degrees < (90 - p):
349 cd = "no"
350 elif abs_degrees < (90 + p):
351 cd = "o"
352 elif abs_degrees < (180 - p):
353 cd = "zo"
354 elif abs_degrees < (180 + p):
355 cd = "z"
356 elif abs_degrees < (270 - p):
357 cd = "zw"
358 elif abs_degrees < (270 + p):
359 cd = "w"
360 elif abs_degrees < (360 - p):
361 cd = "nw"
362 else:
363 cd = "n"
364 return cd
365
366
367
368def cd_between_hosts(hostA, hostB, datadumps):
369 # Using RDNAP coordinates
370 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
371 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
372 return angle_to_cd(math.atan2(dx,dy))
373
374 # GPS coordinates seems to fail somehow
375 #latA = float(datadumps[hostA]['latitude'])
376 #latB = float(datadumps[hostB]['latitude'])
377 #lonA = float(datadumps[hostA]['longitude'])
378 #lonB = float(datadumps[hostB]['longitude'])
379 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
380
381
382def generate_title(nodelist):
383 """ Main overview page """
384 items = {'root' : "." }
385 def fl(spaces, line):
386 return (' ' * spaces) + line + '\n'
387
388 output = """
389<html>
390 <head>
391 <title>Wireless leiden Configurator - GFormat</title>
392 <style type="text/css">
393 th {background-color: #999999}
394 tr:nth-child(odd) {background-color: #cccccc}
395 tr:nth-child(even) {background-color: #ffffff}
396 th, td {padding: 0.1em 1em}
397 </style>
398 </head>
399 <body>
400 <center>
401 <form type="GET" action="%(root)s">
402 <input type="hidden" name="action" value="update">
403 <input type="submit" value="Update Configuration Database (SVN)">
404 </form>
405 <table>
406 <caption><h3>Wireless Leiden Configurator</h3></caption>
407 """ % items
408
409 for node in nodelist:
410 items['node'] = node
411 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
412 for config in files:
413 items['config'] = config
414 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
415 output += fl(5, "</tr>")
416 output += """
417 </table>
418 <hr />
419 <em>%s</em>
420 </center>
421 </body>
422</html>
423 """ % __version__
424
425 return output
426
427
428
429def generate_node(node):
430 """ Print overview of all files available for node """
431 return "\n".join(files)
432
433def generate_node_overview(host, datadump=False):
434 """ Print overview of all files available for node """
435 if not datadump:
436 datadump = get_yaml(host)
437 params = { 'host' : host }
438 output = "<em><a href='..'>Back to overview</a></em><hr />"
439 output += "<h2>Available files:</h2><ul>"
440 for cf in files:
441 params['cf'] = cf
442 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
443 output += "</ul>"
444
445 # Generate and connection listing
446 output += "<h2>Connected To:</h2><ul>"
447 (poel, errors) = make_relations()
448 for network, hosts in poel.iteritems():
449 if host in [x[0] for x in hosts]:
450 if len(hosts) == 1:
451 # Single not connected interface
452 continue
453 for remote,ifacedump in hosts:
454 if remote == host:
455 # This side of the interface
456 continue
457 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
458 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
459 output += "</ul>"
460 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
461
462 output += "<hr /><em><a href='..'>Back to overview</a></em>"
463 return output
464
465
466def generate_header(datadump, ctag="#"):
467 return """\
468%(ctag)s
469%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
470%(ctag)s
471""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
472
473
474
475def parseaddr(s):
476 """ Process IPv4 CIDR notation addr to a (binary) number """
477 f = s.split('.')
478 return (long(f[0]) << 24L) + \
479 (long(f[1]) << 16L) + \
480 (long(f[2]) << 8L) + \
481 long(f[3])
482
483
484
485def showaddr(a):
486 """ Display IPv4 addr in (dotted) CIDR notation """
487 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
488
489
490def is_member(ip, mask, canidate):
491 """ Return True if canidate is part of ip/mask block"""
492 ip_addr = parseaddr(ip)
493 ip_canidate = parseaddr(canidate)
494 mask = int(mask)
495 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
496 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
497 return ip_addr == ip_canidate
498
499
500
501def cidr2netmask(netmask):
502 """ Given a 'netmask' return corresponding CIDR """
503 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
504
505def get_network(addr, mask):
506 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
507
508
509def generate_dhcpd_conf(datadump):
510 """ Generate config file '/usr/local/etc/dhcpd.conf """
511 # Redundency support, in cause local DNS server is not running/responding.
512 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
513 output = generate_header(datadump)
514 output += Template("""\
515# option definitions common to all supported networks...
516option domain-name "dhcp.{{ autogen_fqdn }}";
517
518default-lease-time 600;
519max-lease-time 7200;
520
521# Use this to enble / disable dynamic dns updates globally.
522#ddns-update-style none;
523
524# If this DHCP server is the official DHCP server for the local
525# network, the authoritative directive should be uncommented.
526authoritative;
527
528# Use this to send dhcp log messages to a different log file (you also
529# have to hack syslog.conf to complete the redirection).
530log-facility local7;
531
532# UniFi Discovery Support
533option space ubnt;
534option ubnt.unifi-address code 1 = ip-address;
535#
536class "ubnt" {
537 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
538 option vendor-class-identifier "ubnt";
539 vendor-option-space ubnt;
540}
541
542#
543# Interface definitions
544#
545\n\n""").render(datadump)
546
547
548 # TODO: Use textwrap.fill instead
549 def indent(text, count):
550 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
551
552 # Process DHCP blocks
553 dhcp_out = defaultdict(list)
554 for iface_key in get_interface_keys(datadump):
555 ifname = datadump[iface_key]['autogen_ifbase']
556 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
557 if not datadump[iface_key].has_key('comment'):
558 datadump[iface_key]['comment'] = None
559
560 if not datadump[iface_key].has_key('ip'):
561 continue
562
563 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
564
565 (addr, mask) = datadump[iface_key]['ip'].split('/')
566 datadump[iface_key]['autogen_addr'] = addr
567 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
568 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
569
570 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
571 dhcp_out[groupif].append(textwrap.dedent("""\
572 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
573 ### not autoritive
574 }
575 """ % datadump[iface_key]))
576 continue
577
578 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
579 dhcp_part = ".".join(addr.split('.')[0:3])
580 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
581 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
582 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
583
584 # Assume the first 10 IPs could be used for static entries
585 if 'no_portal' in datadump:
586 fixed = 5
587 for mac in datadump['no_portal']:
588 dhcp_out[groupif].append(textwrap.dedent("""\
589 host fixed-%(ifname)s-%(fixed)s {
590 hardware ethernet %(mac)s;
591 fixed-address %(prefix)s.%(fixed)s;
592 }
593 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
594 fixed += 1
595
596 if 'dhcp_fixed' in datadump[iface_key]:
597 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
598 dhcp_out[groupif].append(textwrap.dedent("""\
599 host fixed-%(host)s {
600 hardware ethernet %(mac)s;
601 fixed-address %(addr)s;
602 }
603 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
604
605
606 dhcp_out[groupif].append(textwrap.dedent("""\
607 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
608 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
609 option routers %(autogen_addr)s;
610 option domain-name-servers %(autogen_dns_servers)s;
611 option ubnt.unifi-address 172.17.107.10;
612 }
613 """ % datadump[iface_key]))
614
615 # Output the blocks in groups
616 for ifname,value in sorted(dhcp_out.iteritems()):
617 if len(value) > 2:
618 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
619 else:
620 output += ''.join(value) + "\n\n"
621 return output
622
623
624
625def generate_dnsmasq_conf(datadump):
626 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
627 output = generate_header(datadump)
628 output += Template("""\
629# DHCP server options
630dhcp-authoritative
631dhcp-fqdn
632domain=dhcp.{{ autogen_fqdn }}
633domain-needed
634expand-hosts
635log-async=100
636
637# Low memory footprint
638cache-size=10000
639
640\n""").render(datadump)
641
642 for iface_key in get_interface_keys(datadump):
643 if not datadump[iface_key].has_key('comment'):
644 datadump[iface_key]['comment'] = None
645 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
646
647 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
648 output += "# not autoritive\n\n"
649 continue
650
651 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
652 (ip, cidr) = datadump[iface_key]['ip'].split('/')
653 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
654
655 dhcp_part = ".".join(ip.split('.')[0:3])
656 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
657 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
658 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
659
660 return output
661
662
663class AutoVivification(dict):
664 """Implementation of perl's autovivification feature."""
665 def __getitem__(self, item):
666 try:
667 return dict.__getitem__(self, item)
668 except KeyError:
669 value = self[item] = type(self)()
670 return value
671
672def make_interface_list(datadump):
673 if interface_list_cache.has_key(datadump['autogen_item']):
674 return (interface_list_cache[datadump['autogen_item']])
675 # lo0 configuration:
676 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
677 # - masterip is special as it needs to be assigned to at
678 # least one interface, so if not used assign to lo0
679 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
680 vlan_list = defaultdict(list)
681 bridge_list = defaultdict(list)
682 flags_if = AutoVivification()
683 dhclient_if = {'lo0' : False}
684
685 # XXX: Find some way of send this output nicely
686 output = ''
687
688 masterip_used = False
689 for iface_key in get_interface_keys(datadump):
690 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
691 masterip_used = True
692 break
693 if not masterip_used:
694 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
695
696 if 'serviceid' in datadump:
697 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
698
699 for iface_key in get_interface_keys(datadump):
700 ifacedump = datadump[iface_key]
701 if ifacedump['autogen_bridge_alias']:
702 ifname = ifacedump['autogen_ifbase']
703 else:
704 ifname = ifacedump['autogen_ifname']
705
706 # If defined as vlan interface
707 if ifacedump['autogen_vlan']:
708 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
709
710 # If defined as bridge interface
711 if ifacedump['autogen_bridge_member']:
712 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
713
714
715 # Flag dhclient is possible
716 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
717 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
718
719 # Ethernet address
720 if ifacedump.has_key('ether'):
721 flags_if[ifname]['ether'] = ifacedump['ether']
722
723 # Add interface IP to list
724 if ifacedump.has_key('ip'):
725 item = (ifacedump['ip'], ifacedump['comment'])
726 if addrs_list.has_key(ifname):
727 addrs_list[ifname].append(item)
728 else:
729 addrs_list[ifname] = [item]
730
731 # Alias only needs IP assignment for now, this might change if we
732 # are going to use virtual accesspoints
733 if "alias" in iface_key:
734 continue
735
736 # XXX: Might want to deduct type directly from interface name
737 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
738 # Default to station (client) mode
739 ifacedump['autogen_wlanmode'] = "sta"
740 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
741 ifacedump['autogen_wlanmode'] = "ap"
742
743 if not ifacedump.has_key('channel'):
744 if ifacedump['type'] == '11a':
745 ifacedump['channel'] = 36
746 else:
747 ifacedump['channel'] = 1
748
749 # Allow special hacks at the back like wds and stuff
750 if not ifacedump.has_key('extra'):
751 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
752 else:
753 ifacedump['autogen_extra'] = ifacedump['extra']
754
755 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
756
757 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
758 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
759 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
760 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
761 output += "\n"
762
763 elif ifacedump['type'] in ['ethernet', 'eth']:
764 # No special config needed besides IP
765 pass
766 elif ifacedump['type'] in ['vlan']:
767 # VLAN member has no special configuration
768 pass
769 else:
770 assert False, "Unknown type " + ifacedump['type']
771
772 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
773 interface_list_cache[datadump['autogen_item']] = store
774 return(store)
775
776
777
778def create_proxies_list():
779 if not ileiden_proxies or not normal_proxies:
780 # Placeholder for to-be-installed proxies, this will avoid updating the all
781 # nodes to include this new machine, yet due to an unbound issue, this list
782 # has to be kept small.
783
784 for i in range(1,20):
785 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
786
787 for host in get_hostlist():
788 hostdump = get_yaml(host)
789 if hostdump['status'] == 'up':
790 if hostdump['service_proxy_ileiden']:
791 ileiden_proxies[hostdump['serviceid']] = hostdump
792 if hostdump['service_proxy_normal']:
793 normal_proxies.append(hostdump)
794
795
796
797def generate_rc_conf_local(datadump):
798 """ Generate configuration file '/etc/rc.conf.local' """
799 item = datadump['autogen_item']
800 if rc_conf_local_cache.has_key(item):
801 return rc_conf_local_cache[item]
802
803 if not datadump.has_key('ileiden'):
804 datadump['autogen_ileiden_enable'] = False
805 else:
806 datadump['autogen_ileiden_enable'] = datadump['ileiden']
807
808 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
809
810 create_proxies_list()
811 datadump['autogen_ileiden_proxies'] = ileiden_proxies
812 datadump['autogen_normal_proxies'] = normal_proxies
813 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
814 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
815 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
816 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
817
818 output = generate_header(datadump, "#");
819 output += render_template(datadump, """\
820hostname='{{ autogen_fqdn }}'
821location='{{ location }}'
822nodetype="{{ nodetype }}"
823
824#
825# Configured listings
826#
827captive_portal_whitelist=""
828{% if nodetype == "Proxy" %}
829#
830# Proxy Configuration
831#
832{% if gateway and service_proxy_ileiden -%}
833defaultrouter="{{ gateway }}"
834{% else -%}
835#defaultrouter="NOTSET"
836{% endif -%}
837internalif="{{ internalif }}"
838ileiden_enable="{{ autogen_ileiden_enable }}"
839gateway_enable="{{ autogen_ileiden_enable }}"
840pf_enable="yes"
841pf_rules="/etc/pf.conf"
842{% if autogen_ileiden_enable -%}
843pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
844lvrouted_enable="{{ autogen_ileiden_enable }}"
845lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
846{% else -%}
847pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
848{% endif -%}
849{% if internalroute -%}
850static_routes="wleiden"
851route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
852{% endif -%}
853
854{% elif nodetype == "Hybrid" %}
855 #
856 # Hybrid Configuration
857 #
858 list_ileiden_proxies="
859 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
860 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
861 {% endfor -%}
862 "
863 list_normal_proxies="
864 {% for item in autogen_normal_proxies -%}
865 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
866 {% endfor -%}
867 "
868
869 {% if autogen_dhcp_interfaces -%}
870 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
871 {% else %}
872 captive_portal_interfaces="dummy"
873 {% endif %}
874 externalif="{{ externalif|default('vr0', true) }}"
875 masterip="{{ masterip }}"
876
877 {% if gateway and service_proxy_ileiden %}
878 defaultrouter="{{ gateway }}"
879 {% else %}
880 #defaultrouter="NOTSET"
881 {% endif %}
882
883 #
884 # Defined services
885 #
886 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
887 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
888 service_accesspoint="{{ service_accesspoint|yesorno }}"
889 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
890 service_concentrator="{{ service_concentrator|yesorno }}"
891
892 {% if service_proxy_ileiden %}
893 pf_rules="/etc/pf.hybrid.conf"
894 {% if service_concentrator %}
895 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
896 {% else %}
897 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
898 {% endif %}
899 pf_flags="$pf_flags -D publicnat=80,443"
900 lvrouted_flags="$lvrouted_flags -g"
901 {% elif service_proxy_normal or service_incoming_rdr %}
902 pf_rules="/etc/pf.hybrid.conf"
903 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
904 pf_flags="$pf_flags -D publicnat=0"
905 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
906 named_setfib="1"
907 tinyproxy_setfib="1"
908 dnsmasq_setfib="1"
909 sshd_setfib="1"
910 {% else %}
911 named_auto_forward_only="YES"
912 pf_rules="/etc/pf.node.conf"
913 pf_flags=""
914 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
915 {% endif %}
916 {% if service_concentrator %}
917 # Do mind installing certificates is NOT done automatically for security reasons
918 openvpn_enable="YES"
919 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
920 {% endif %}
921
922 {% if service_proxy_normal %}
923 tinyproxy_enable="yes"
924 {% else %}
925 pen_wrapper_enable="yes"
926 {% endif %}
927
928 {% if service_accesspoint %}
929 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
930 {% endif %}
931
932 {% if board == "ALIX2" or board == "net4801" %}
933 #
934 # ''Fat'' configuration, board has 256MB RAM
935 #
936 dnsmasq_enable="NO"
937 named_enable="YES"
938 unbound_enable="YES"
939 {% if autogen_dhcp_interfaces -%}
940 dhcpd_enable="YES"
941 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
942 {% endif -%}
943 {% elif board == "apu1d" %}
944 #
945 # ''Fat'' configuration, board has 1024MB RAM
946 #
947 dnsmasq_enable="NO"
948 unbound_enable="YES"
949 {% if autogen_dhcp_interfaces -%}
950 dhcpd_enable="YES"
951 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
952 {% endif -%}
953 {% endif -%}
954{% endif %}
955
956#
957# Script variables
958#
959attached_devices="{{ autogen_attached_devices|join(' ') }}"
960neighbours="{{ autogen_neighbours|join(' ') }}"
961
962
963#
964# Interface definitions
965#\n
966""")
967
968 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
969 for iface, vlans in sorted(vlan_list.items()):
970 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
971
972 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
973 for iface in sorted(vlan_list.keys()):
974 if not iface in addrs_list.keys():
975 output += "ifconfig_%s='up'\n" % iface
976
977 output += "\n"
978
979 # Bridge configuration:
980 if bridge_list.keys():
981 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
982
983 for iface in bridge_list.keys():
984 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
985
986 # Bridge member interfaces not containing a configuration should be marked active explcitly.
987 for _,members in bridge_list.items():
988 for iface in members:
989 if not iface in addrs_list.keys():
990 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
991
992 output += "\n"
993
994 # Details like SSID
995 if extra_ouput:
996 output += extra_ouput.strip() + "\n"
997
998 # Print IP address which needs to be assigned over here
999 output += "\n"
1000 for iface,addrs in sorted(addrs_list.iteritems()):
1001 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1002 output += "# %s || %s || %s\n" % (iface, addr, comment)
1003
1004 # Write DHCLIENT entry
1005 if iface in dhclient_if and dhclient_if[iface]:
1006 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1007 continue
1008
1009 # Make sure the external address is always first as this is needed in the
1010 # firewall setup
1011 addrs = sorted(
1012 [x for x in addrs if not '0.0.0.0' in x[0]],
1013 key=lambda x: x[0].split('.')[0],
1014 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1015 )
1016
1017 idx_offset = 0
1018 # Set MAC is required
1019 if flags_if[iface].has_key('ether'):
1020 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1021 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1022 idx_offset += 1
1023 elif iface in bridge_list:
1024 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1025 idx_offset += 1
1026 else:
1027 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1028
1029 for idx, addr in enumerate(addrs[1:]):
1030 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1031
1032 output += "\n"
1033
1034 rc_conf_local_cache[datadump['autogen_item']] = output
1035 return output
1036
1037
1038
1039def get_all_configs():
1040 """ Get dict with key 'host' with all configs present """
1041 configs = dict()
1042 for host in get_hostlist():
1043 datadump = get_yaml(host)
1044 configs[host] = datadump
1045 return configs
1046
1047
1048def get_interface_keys(config, extra=False):
1049 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1050 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1051 if extra == False:
1052 return filter(lambda x: not "extra" in x, elems)
1053 else:
1054 return elems
1055
1056
1057def get_used_ips(configs):
1058 """ Return array of all IPs used in config files"""
1059 ip_list = []
1060 for config in configs:
1061 ip_list.append(config['masterip'])
1062 if 'serviceid' in config:
1063 ip_list.append(config['serviceid'])
1064 for iface_key in get_interface_keys(config, True):
1065 l = config[iface_key]['ip']
1066 addr, mask = l.split('/')
1067 # Special case do not process
1068 if valid_addr(addr):
1069 ip_list.append(addr)
1070 else:
1071 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1072 return sorted(ip_list)
1073
1074
1075
1076def get_nameservers(max_servers=None):
1077 if nameservers_cache:
1078 return nameservers_cache[0:max_servers]
1079
1080 for host in get_hostlist():
1081 hostdump = get_yaml(host)
1082 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1083 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1084
1085 return nameservers_cache[0:max_servers]
1086
1087
1088def get_neighbours(datadump):
1089 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1090
1091 (poel, errors) = make_relations()
1092 table = []
1093 for iface,addrs in sorted(addrs_list.iteritems()):
1094 if iface in ['lo0']:
1095 continue
1096
1097 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1098 if not addr.startswith('172.'):
1099 # Avoid listing internet connections as pool
1100 continue
1101 for neighbour in poel[network(addr)]:
1102 if neighbour[0] != datadump['autogen_item']:
1103 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1104 return table
1105
1106
1107def get_attached_devices(datadump, url=False):
1108 table = []
1109 for iface_key in get_interface_keys(datadump, True):
1110 ifacedump = datadump[iface_key]
1111
1112 if not ifacedump.has_key('ns_ip'):
1113 continue
1114
1115 x_ip = ifacedump['ns_ip'].split('/')[0]
1116
1117 if 'mode' in ifacedump:
1118 x_mode = ifacedump['mode']
1119 else:
1120 x_mode = 'unknown'
1121
1122 if 'bridge_type' in ifacedump:
1123 device_type = ifacedump['bridge_type']
1124 else:
1125 device_type = 'Unknown'
1126
1127 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1128 return table
1129
1130
1131def generate_resolv_conf(datadump):
1132 """ Generate configuration file '/etc/resolv.conf' """
1133 # XXX: This should properly going to be an datastructure soon
1134 datadump['autogen_header'] = generate_header(datadump, "#")
1135 datadump['autogen_edge_nameservers'] = ''
1136
1137
1138 for masterip,realname in get_nameservers():
1139 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1140
1141 return Template("""\
1142{{ autogen_header }}
1143search wleiden.net
1144
1145# Try local (cache) first
1146nameserver 127.0.0.1
1147
1148{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1149nameserver 8.8.8.8 # Google Public NameServer
1150nameserver 64.6.64.6 # Verisign Public NameServer
1151{% else -%}
1152# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1153{{ autogen_edge_nameservers }}
1154{% endif -%}
1155""").render(datadump)
1156
1157
1158
1159def generate_ntp_conf(datadump):
1160 """ Generate configuration file '/etc/ntp.conf' """
1161 # XXX: This should properly going to be an datastructure soon
1162
1163 datadump['autogen_header'] = generate_header(datadump, "#")
1164 datadump['autogen_ntp_servers'] = ''
1165 for host in get_hostlist():
1166 hostdump = get_yaml(host)
1167 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1168 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1169
1170 return Template("""\
1171{{ autogen_header }}
1172
1173{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1174# Machine hooked to internet.
1175server 0.nl.pool.ntp.org iburst maxpoll 9
1176server 1.nl.pool.ntp.org iburst maxpoll 9
1177server 2.nl.pool.ntp.org iburst maxpoll 9
1178server 3.nl.pool.ntp.org iburst maxpoll 9
1179{% else -%}
1180# Local Wireless Leiden NTP Servers.
1181server 0.pool.ntp.wleiden.net iburst maxpoll 9
1182server 1.pool.ntp.wleiden.net iburst maxpoll 9
1183server 2.pool.ntp.wleiden.net iburst maxpoll 9
1184server 3.pool.ntp.wleiden.net iburst maxpoll 9
1185
1186# All the configured NTP servers
1187{{ autogen_ntp_servers }}
1188{% endif %}
1189
1190# If a server loses sync with all upstream servers, NTP clients
1191# no longer follow that server. The local clock can be configured
1192# to provide a time source when this happens, but it should usually
1193# be configured on just one server on a network. For more details see
1194# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1195# The use of Orphan Mode may be preferable.
1196#
1197server 127.127.1.0
1198fudge 127.127.1.0 stratum 10
1199""").render(datadump)
1200
1201
1202def generate_pf_hybrid_conf_local(datadump):
1203 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1204 datadump['autogen_header'] = generate_header(datadump, "#")
1205 if datadump['service_incoming_rdr']:
1206 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1207 return Template("""\
1208{{ autogen_header }}
1209
1210# Redirect some internal facing services outside (7)
1211# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1212{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1213rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1214{% endfor -%}
1215# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1216{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1217rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1218{% endfor -%}
1219""").render(datadump)
1220
1221def generate_unbound_wleiden_conf(datadump):
1222 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1223 datadump['autogen_header'] = generate_header(datadump, "#")
1224
1225 autogen_ips = []
1226 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1227 for iface,addrs in sorted(addrs_list.iteritems()):
1228 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1229 if addr.startswith('172'):
1230 autogen_ips.append((addr.split('/')[0], comment))
1231 datadump['autogen_ips'] = autogen_ips
1232
1233 create_proxies_list()
1234 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1235 return Template("""\
1236{{ autogen_header }}
1237
1238server:
1239 ## Static definitions fail to start on systems with broken ue(4) interfaces
1240{%- for ip,comment in autogen_ips %}
1241 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1242{%- endfor %}
1243 ## Enabling wildcard matching as work-around
1244 interface: 0.0.0.0
1245 interface: ::0
1246
1247forward-zone:
1248 name: '.'
1249{%- if service_proxy_ileiden %}
1250 forward-addr: 8.8.8.8 # Google DNS A
1251 forward-addr: 8.8.4.4 # Google DNS B
1252 forward-addr: 208.67.222.222 # OpenDNS DNS A
1253 forward-addr: 208.67.220.220 # OpenDNS DNS B
1254{% else -%}
1255{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1256 {%- if loop.index <= 5 %}
1257 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1258 {%- endif %}
1259{%- endfor %}
1260{%- endif %}
1261""").render(datadump)
1262
1263def generate_motd(datadump):
1264 """ Generate configuration file '/etc/motd' """
1265 output = Template("""\
1266FreeBSD run ``service motd onestart'' to make me look normal
1267
1268 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1269 Loc: {{ location }}
1270
1271Services:
1272{% if board == "ALIX2" or board == "net4801" -%}
1273{{" -"}} Core Node ({{ board }})
1274{% else -%}
1275{{" -"}} Hulp Node ({{ board }})
1276{% endif -%}
1277{% if service_proxy_normal -%}
1278{{" -"}} Normal Proxy
1279{% endif -%}
1280{% if service_proxy_ileiden -%}
1281{{" -"}} iLeiden Proxy
1282{% endif -%}
1283{% if service_incoming_rdr -%}
1284{{" -"}} Incoming port redirects
1285{% endif %}
1286Interlinks:\n
1287""").render(datadump)
1288
1289
1290 def make_table(table):
1291 if not table:
1292 return " - none\n"
1293 else:
1294 lines = ""
1295 col_width = [max(len(x) for x in col) for col in zip(*table)]
1296 for row in table:
1297 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1298 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1299 return lines
1300
1301 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1302 table = []
1303 for iface,addrs in sorted(addrs_list.iteritems()):
1304 if iface in ['lo0']:
1305 continue
1306 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1307 table.append((iface, addr, comment))
1308
1309 output += make_table(table)
1310 output += '\n'
1311 output += """\
1312Attached devices:
1313"""
1314 output += make_table(get_attached_devices(datadump, url=True))
1315 output += '\n'
1316 output += """\
1317Available neighbours:
1318"""
1319 output += make_table(get_neighbours(datadump))
1320
1321 return output
1322
1323
1324def format_yaml_value(value):
1325 """ Get yaml value in right syntax for outputting """
1326 if isinstance(value,str):
1327 output = '"%s"' % value
1328 else:
1329 output = value
1330 return output
1331
1332
1333
1334def format_wleiden_yaml(datadump):
1335 """ Special formatting to ensure it is editable"""
1336 output = "# Genesis config yaml style\n"
1337 output += "# vim:ts=2:et:sw=2:ai\n"
1338 output += "#\n"
1339 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1340 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1341 if key == 'rdr_rules':
1342 output += '%-10s:\n' % 'rdr_rules'
1343 for rdr_rule in datadump[key]:
1344 output += '- %s\n' % rdr_rule
1345 else:
1346 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1347
1348 output += "\n\n"
1349
1350 # Format (key, required)
1351 key_order = (
1352 ('comment', True),
1353 ('parent', False),
1354 ('ip', False),
1355 ('ether', False),
1356 ('desc', True),
1357 ('sdesc', True),
1358 ('mode', True),
1359 ('type', True),
1360 ('extra_type', False),
1361 ('channel', False),
1362 ('ssid', False),
1363 ('wlan_mac', False),
1364 ('dhcp', True),
1365 ('dhcp_fixed', False),
1366 ('compass', False),
1367 ('distance', False),
1368 ('ns_ip', False),
1369 ('repeater_ip', False),
1370 ('bullet2_ip', False),
1371 ('ns_mac', False),
1372 ('bullet2_mac', False),
1373 ('ns_type', False),
1374 ('bridge_type', False),
1375 ('status', True),
1376 )
1377
1378 for iface_key in sorted(iface_keys):
1379 try:
1380 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1381 if remainder:
1382 raise KeyError("invalid keys: %s" % remainder)
1383
1384 output += "%s:\n" % iface_key
1385 for key,required in key_order:
1386 if datadump[iface_key].has_key(key):
1387 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1388 output += "\n\n"
1389 except Exception:
1390 print "# Error while processing interface %s" % iface_key
1391 raise
1392
1393 return output
1394
1395
1396
1397def generate_wleiden_yaml(datadump, header=True):
1398 """ Generate (petty) version of wleiden.yaml"""
1399 output = generate_header(datadump, "#") if header else ''
1400
1401 for key in datadump.keys():
1402 if key.startswith('autogen_'):
1403 del datadump[key]
1404 # Interface autogen cleanups
1405 elif type(datadump[key]) == dict:
1406 for key2 in datadump[key].keys():
1407 if key2.startswith('autogen_'):
1408 del datadump[key][key2]
1409
1410 output += format_wleiden_yaml(datadump)
1411 return output
1412
1413def generate_nanostation_config(datadump, iface, ns_type):
1414 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1415 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1416
1417 datadump.update(datadump['iface_%s' % iface])
1418
1419 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1420
1421def generate_yaml(datadump):
1422 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1423
1424
1425
1426def generate_config(node, config, datadump=None):
1427 """ Print configuration file 'config' of 'node' """
1428 output = ""
1429 try:
1430 # Load config file
1431 if datadump == None:
1432 datadump = get_yaml(node)
1433
1434 if config == 'wleiden.yaml':
1435 output += generate_wleiden_yaml(datadump)
1436 elif config == 'authorized_keys':
1437 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1438 output += f.read()
1439 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1440 # Fetch local keys if existing
1441 if os.path.exists(node_keys):
1442 output += open(node_keys, 'r').read()
1443 f.close()
1444 elif config == 'dnsmasq.conf':
1445 output += generate_dnsmasq_conf(datadump)
1446 elif config == 'dhcpd.conf':
1447 output += generate_dhcpd_conf(datadump)
1448 elif config == 'rc.conf.local':
1449 output += generate_rc_conf_local(datadump)
1450 elif config == 'resolv.conf':
1451 output += generate_resolv_conf(datadump)
1452 elif config == 'ntp.conf':
1453 output += generate_ntp_conf(datadump)
1454 elif config == 'motd':
1455 output += generate_motd(datadump)
1456 elif config == 'pf.hybrid.conf.local':
1457 output += generate_pf_hybrid_conf_local(datadump)
1458 elif config == 'unbound.wleiden.conf':
1459 output += generate_unbound_wleiden_conf(datadump)
1460 elif config.startswith('vr'):
1461 interface, ns_type = config.strip('.yaml').split('-')
1462 output += generate_nanostation_config(datadump, interface, ns_type)
1463 else:
1464 assert False, "Config not found!"
1465 except IOError, e:
1466 output += "[ERROR] Config file not found"
1467 return output
1468
1469
1470
1471def process_cgi_request(environ=os.environ):
1472 """ When calling from CGI """
1473 response_headers = []
1474 content_type = 'text/plain'
1475
1476 # Update repository if requested
1477 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1478 if form and form.has_key("action") and "update" in form["action"]:
1479 output = "[INFO] Updating subverion, please wait...\n"
1480 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1481 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1482 output += "[INFO] All done, redirecting in 5 seconds"
1483 response_headers += [
1484 ('Refresh', '5; url=.'),
1485 ]
1486 reload_cache()
1487 else:
1488 base_uri = environ['REQUEST_URI']
1489 uri = base_uri.strip('/').split('/')[1:]
1490
1491 output = "Template Holder"
1492 if base_uri.endswith('/create/network.kml'):
1493 content_type='application/vnd.google-earth.kml+xml'
1494 output = make_network_kml.make_graph()
1495 elif base_uri.endswith('/api/get/nodeplanner.json'):
1496 content_type='application/json'
1497 output = make_network_kml.make_nodeplanner_json()
1498 elif not uri:
1499 if is_text_request(environ):
1500 output = '\n'.join(get_hostlist())
1501 else:
1502 content_type = 'text/html'
1503 output = generate_title(get_hostlist())
1504 elif len(uri) == 1:
1505 if is_text_request(environ):
1506 output = generate_node(uri[0])
1507 else:
1508 content_type = 'text/html'
1509 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1510 elif len(uri) == 2:
1511 output = generate_config(uri[0], uri[1])
1512 else:
1513 assert False, "Invalid option"
1514
1515 # Return response
1516 response_headers += [
1517 ('Content-type', content_type),
1518 ('Content-Length', str(len(output))),
1519 ]
1520 return(response_headers, str(output))
1521
1522
1523def make_dns(output_dir = 'dns', external = False):
1524 items = dict()
1525
1526 # hostname is key, IP is value
1527 wleiden_zone = defaultdict(list)
1528 wleiden_cname = dict()
1529
1530 pool = dict()
1531 for node in get_hostlist():
1532 datadump = get_yaml(node)
1533
1534 fqdn = datadump['nodename']
1535
1536 if datadump.has_key('rdr_host'):
1537 remote_target = datadump['rdr_host']
1538 elif datadump.has_key('remote_access') and datadump['remote_access']:
1539 remote_target = datadump['remote_access'].split(':')[0]
1540 else:
1541 remote_target = None
1542
1543 if remote_target:
1544 try:
1545 parseaddr(remote_target)
1546 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1547 except (IndexError, ValueError):
1548 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1549
1550
1551 wleiden_zone[fqdn].append((datadump['masterip'], True))
1552
1553 # Hacking to get proper DHCP IPs and hostnames
1554 for iface_key in get_interface_keys(datadump):
1555 iface_name = iface_key.replace('_','-')
1556 if 'ip' in datadump[iface_key]:
1557 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1558 try:
1559 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1560 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1561 dhcp_part = ".".join(ip.split('.')[0:3])
1562 if ip != datadump['masterip']:
1563 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1564 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1565 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1566 except (AttributeError, ValueError, KeyError):
1567 # First push it into a pool, to indentify the counter-part later on
1568 addr = parseaddr(ip)
1569 cidr = int(cidr)
1570 addr = addr & ~((1 << (32 - cidr)) - 1)
1571 if pool.has_key(addr):
1572 pool[addr] += [(iface_name, fqdn, ip)]
1573 else:
1574 pool[addr] = [(iface_name, fqdn, ip)]
1575 continue
1576
1577
1578
1579 # WL uses an /29 to configure an interface. IP's are ordered like this:
1580 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1581
1582 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1583
1584 # Automatic naming convention of interlinks namely 2 + remote.lower()
1585 for (key,value) in pool.iteritems():
1586 # Make sure they are sorted from low-ip to high-ip
1587 value = sorted(value, key=lambda x: parseaddr(x[2]))
1588
1589 if len(value) == 1:
1590 (iface_name, fqdn, ip) = value[0]
1591 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1592
1593 # Device DNS names
1594 if 'cnode' in fqdn.lower():
1595 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1596 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1597
1598 elif len(value) == 2:
1599 (a_iface_name, a_fqdn, a_ip) = value[0]
1600 (b_iface_name, b_fqdn, b_ip) = value[1]
1601 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1602 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1603
1604 # Device DNS names
1605 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1606 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1607 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1608 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1609 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1610 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1611 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1612
1613 else:
1614 pool_members = [k[1] for k in value]
1615 for item in value:
1616 (iface_name, fqdn, ip) = item
1617 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1618
1619 # Include static DNS entries
1620 # XXX: Should they override the autogenerated results?
1621 # XXX: Convert input to yaml more useable.
1622 # Format:
1623 ##; this is a comment
1624 ## roomburgh=Roomburgh1
1625 ## apkerk1.Vosko=172.17.176.8 ;this as well
1626 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1627
1628 # Hack to allow special entries, for development
1629 wleiden_raw = {}
1630
1631 for line in dns_list:
1632 reverse = False
1633 k, items = line.items()[0]
1634 if type(items) == dict:
1635 if items.has_key('reverse'):
1636 reverse = items['reverse']
1637 items = items['a']
1638 else:
1639 items = items['cname']
1640 items = [items] if type(items) != list else items
1641 for item in items:
1642 if item.startswith('IN '):
1643 wleiden_raw[k] = item
1644 elif valid_addr(item):
1645 wleiden_zone[k].append((item, reverse))
1646 else:
1647 wleiden_cname[k] = item
1648
1649 # Hack to get dynamic pool listing
1650 def chunks(l, n):
1651 return [l[i:i+n] for i in range(0, len(l), n)]
1652
1653 ntp_servers = [x[0] for x in get_nameservers()]
1654 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1655 for ntp_server in chunk:
1656 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1657
1658 details = dict()
1659 # 24 updates a day allowed
1660 details['serial'] = time.strftime('%Y%m%d%H')
1661
1662 if external:
1663 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1664 else:
1665 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1666
1667 details['master'] = dns_masters[0]
1668 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1669
1670 dns_header = '''
1671$TTL 3h
1672%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1673 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1674
1675%(ns_servers)s
1676 \n'''
1677
1678
1679 if not os.path.isdir(output_dir):
1680 os.makedirs(output_dir)
1681 details['zone'] = 'wleiden.net'
1682 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1683 f.write(dns_header % details)
1684
1685 for host,items in wleiden_zone.iteritems():
1686 for ip,reverse in items:
1687 if ip not in ['0.0.0.0']:
1688 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1689 for source,dest in wleiden_cname.iteritems():
1690 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1691 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1692 for source, dest in wleiden_raw.iteritems():
1693 f.write("%s.wleiden.net. %s\n" % (source, dest))
1694 f.close()
1695
1696 # Create whole bunch of specific sub arpa zones. To keep it compliant
1697 for s in range(16,32):
1698 details['zone'] = '%i.172.in-addr.arpa' % s
1699 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1700 f.write(dns_header % details)
1701
1702 #XXX: Not effient, fix to proper data structure and do checks at other
1703 # stages
1704 for host,items in wleiden_zone.iteritems():
1705 for ip,reverse in items:
1706 if not reverse:
1707 continue
1708 if valid_addr(ip):
1709 if valid_addr(ip):
1710 if int(ip.split('.')[1]) == s:
1711 rev_ip = '.'.join(reversed(ip.split('.')))
1712 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1713 f.close()
1714
1715
1716def usage():
1717 print """Usage: %(prog)s <argument>
1718Argument:
1719\tcleanup = Cleanup all YAML files to specified format
1720\tstandalone [port] = Run configurator webserver [8000]
1721\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1722\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1723\tfull-export = Generate yaml export script for heatmap.
1724\tstatic [outputdir] = Generate all config files and store on disk
1725\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1726\ttest <node> [<file>] = Receive output for certain node [all files].
1727\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1728\tlist <status> <items> = List systems which have certain status
1729\tcreate network.kml = Create Network KML file for use in Google Earth
1730
1731Arguments:
1732\t<node> = NodeName (example: HybridRick)
1733\t<file> = %(files)s
1734\t<status> = all|up|down|planned
1735\t<items> = systems|nodes|proxies
1736
1737NOTE FOR DEVELOPERS; you can test your changes like this:
1738 BEFORE any changes in this code:
1739 $ ./gformat.py static /tmp/pre
1740 AFTER the changes:
1741 $ ./gformat.py static /tmp/post
1742 VIEW differences and VERIFY all are OK:
1743 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1744""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1745 exit(0)
1746
1747
1748def is_text_request(environ=os.environ):
1749 """ Find out whether we are calling from the CLI or any text based CLI utility """
1750 if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1751 return True
1752
1753 if 'HTTP_USER_AGENT' in environ:
1754 return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1755 else:
1756 return False
1757
1758
1759def switchFormat(setting):
1760 if setting:
1761 return "YES"
1762 else:
1763 return "NO"
1764
1765def rlinput(prompt, prefill=''):
1766 import readline
1767 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1768 try:
1769 return raw_input(prompt)
1770 finally:
1771 readline.set_startup_hook()
1772
1773def fix_conflict(left, right, default='i'):
1774 while True:
1775 print "## %-30s | %-30s" % (left, right)
1776 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1777 if not c:
1778 c = default
1779
1780 if c in ['l','1']:
1781 return left
1782 elif c in ['r','2']:
1783 return right
1784 elif c in ['e', '3']:
1785 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1786 elif c in ['i', '4']:
1787 return None
1788 else:
1789 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1790
1791
1792
1793def print_cgi_response(response_headers, output):
1794 """Could we not use some kind of wsgi wrapper to make this output?"""
1795 for header in response_headers:
1796 print "%s: %s" % header
1797 print
1798 print output
1799
1800
1801def fill_cache():
1802 ''' Poor man re-loading of few cache items (the slow ones) '''
1803 for host in get_hostlist():
1804 get_yaml(host)
1805
1806
1807def reload_cache():
1808 clear_cache()
1809 fill_cache()
1810
1811
1812def main():
1813 """Hard working sub"""
1814 # Allow easy hacking using the CLI
1815 if not os.environ.has_key('REQUEST_URI'):
1816 if len(sys.argv) < 2:
1817 usage()
1818
1819 if sys.argv[1] == "standalone":
1820 import SocketServer
1821 import CGIHTTPServer
1822 # Hop to the right working directory.
1823 os.chdir(os.path.dirname(__file__))
1824 try:
1825 PORT = int(sys.argv[2])
1826 except (IndexError,ValueError):
1827 PORT = 8000
1828
1829 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1830 """ Serve this CGI from the root of the webserver """
1831 def is_cgi(self):
1832 if "favicon" in self.path:
1833 return False
1834
1835 self.cgi_info = (os.path.basename(__file__), self.path)
1836 self.path = ''
1837 return True
1838 handler = MyCGIHTTPRequestHandler
1839 SocketServer.TCPServer.allow_reuse_address = True
1840 httpd = SocketServer.TCPServer(("", PORT), handler)
1841 httpd.server_name = 'localhost'
1842 httpd.server_port = PORT
1843
1844 logger.info("serving at port %s", PORT)
1845 try:
1846 httpd.serve_forever()
1847 except KeyboardInterrupt:
1848 httpd.shutdown()
1849 logger.info("All done goodbye")
1850 elif sys.argv[1] == "test":
1851 # Basic argument validation
1852 try:
1853 node = sys.argv[2]
1854 except IndexError:
1855 print "Invalid argument"
1856 exit(1)
1857 except IOError as e:
1858 print e
1859 exit(1)
1860
1861 datadump = get_yaml(node)
1862
1863
1864 # Get files to generate
1865 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1866
1867 # Actual config generation
1868 for config in gen_files:
1869 logger.info("## Generating %s %s", node, config)
1870 print generate_config(node, config, datadump)
1871 elif sys.argv[1] == "test-cgi":
1872 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1873 os.environ['SCRIPT_NAME'] = __file__
1874 response_headers, output = process_cgi_request()
1875 print_cgi_response(response_headers, output)
1876 elif sys.argv[1] == "static":
1877 items = dict()
1878 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1879 for node in get_hostlist():
1880 items['node'] = node
1881 items['wdir'] = "%(output_dir)s/%(node)s" % items
1882 if not os.path.isdir(items['wdir']):
1883 os.makedirs(items['wdir'])
1884 datadump = get_yaml(node)
1885 f = open("%(wdir)s/index.html" % items, "w")
1886 f.write(generate_node_overview(items['node'], datadump))
1887 f.close()
1888 for config in files:
1889 items['config'] = config
1890 logger.info("## Generating %(node)s %(config)s" % items)
1891 f = open("%(wdir)s/%(config)s" % items, "w")
1892 f.write(generate_config(node, config, datadump))
1893 f.close()
1894 elif sys.argv[1] == "wind-export":
1895 items = dict()
1896 for node in get_hostlist():
1897 datadump = get_yaml(node)
1898 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1899 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1900 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1901 VALUES (
1902 (SELECT id FROM users WHERE username = 'rvdzwet'),
1903 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1904 'Y');""" % datadump
1905 #for config in files:
1906 # items['config'] = config
1907 # print "## Generating %(node)s %(config)s" % items
1908 # f = open("%(wdir)s/%(config)s" % items, "w")
1909 # f.write(generate_config(node, config, datadump))
1910 # f.close()
1911 for node in get_hostlist():
1912 datadump = get_yaml(node)
1913 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1914 ifacedump = datadump[iface_key]
1915 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1916 ifacedump['nodename'] = datadump['nodename']
1917 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1918 ifacedump['channel'] = 0
1919 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1920 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1921 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1922 elif sys.argv[1] == "nagios-export":
1923 try:
1924 heavy_load = (sys.argv[2] == "--heavy-load")
1925 except IndexError:
1926 heavy_load = False
1927
1928 hostgroup_details = {
1929 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1930 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1931 'walphen' : 'Stichting Wireless Alphen',
1932 'westeinder' : 'Westeinder Plassen',
1933 }
1934
1935 # Convert IP to Host
1936 ip2host = {'root' : 'root'}
1937 for host in get_hostlist():
1938 datadump = get_yaml(host)
1939 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1940 for iface in get_interface_keys(datadump):
1941 if datadump[iface].has_key('autogen_gateway'):
1942 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1943
1944 # Find dependency tree based on output of lvrouted.mytree of nearest node
1945 parents = defaultdict(list)
1946 stack = ['root']
1947 prev_depth = 0
1948 for line in open('lvrouted.mytree').readlines():
1949 depth = line.count('\t')
1950 ip = line.strip().split()[0]
1951
1952 if prev_depth < depth:
1953 try:
1954 parents[ip2host[ip]].append(ip2host[stack[-1]])
1955 except KeyError as e:
1956 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1957 stack.append(ip)
1958 elif prev_depth > depth:
1959 stack = stack[:(depth - prev_depth)]
1960 elif prev_depth == depth:
1961 try:
1962 parents[ip2host[ip]].append(ip2host[stack[-1]])
1963 except KeyError as e:
1964 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1965
1966
1967 prev_depth = depth
1968 # Observe that some nodes has themself as parent or multiple parents
1969 # for now take only the first parent, other behaviour is yet to be explained
1970
1971
1972
1973 params = {
1974 'check_interval' : 5 if heavy_load else 120,
1975 'retry_interval' : 1 if heavy_load else 10,
1976 'max_check_attempts' : 10 if heavy_load else 6,
1977 'notification_interval': 120 if heavy_load else 240,
1978 }
1979
1980 print '''\
1981define host {
1982 name wleiden-node ; Default Node Template
1983 use generic-host ; Use the standard template as initial starting point
1984 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1985 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1986 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1987 notification_interval %(notification_interval)s
1988 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1989 check_command check-host-alive ; Default command to check FreeBSD hosts
1990 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1991}
1992
1993define service {
1994 name wleiden-service ; Default Service Template
1995 use generic-service ; Use the standard template as initial starting point
1996 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1997 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1998 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1999 notification_interval %(notification_interval)s
2000 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2001 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2002}
2003
2004# Please make sure to install:
2005# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2006#
2007# Recompile net-mgmt/nagios-plugins to support check_snmp
2008# make -C /usr/ports/net-mgmt/nagios-plugins
2009#
2010# Install net/bind-tools to allow v2/check_dns_wl to work:
2011# pkg install bind-tools
2012#
2013define command{
2014 command_name check_snmp_disk
2015 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2016}
2017
2018define command{
2019 command_name check_netsnmp_load
2020 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2021}
2022
2023define command{
2024 command_name check_netsnmp_proc
2025 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2026}
2027
2028define command{
2029 command_name check_by_ssh
2030 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2031}
2032
2033define command{
2034 command_name check_dns_wl
2035 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2036}
2037
2038define command{
2039 command_name check_snmp_uptime
2040 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2041}
2042
2043
2044# TDB: dhcp leases
2045# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2046
2047# TDB: internet status
2048# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2049
2050# TDB: Advanced local passive checks
2051# /usr/local/libexec/nagios/check_by_ssh
2052''' % params
2053
2054 print '''\
2055# Service Group, not displayed by default
2056define hostgroup {
2057 hostgroup_name srv_hybrid
2058 alias All Hybrid Nodes
2059 register 0
2060}
2061
2062define service {
2063 use wleiden-service
2064 hostgroup_name srv_hybrid
2065 service_description SSH
2066 check_command check_ssh
2067}
2068
2069define service {
2070 use wleiden-service,service-pnp
2071 hostgroup_name srv_hybrid
2072 service_description HTTP
2073 check_command check_http
2074}
2075
2076define service {
2077 use wleiden-service
2078 hostgroup_name srv_hybrid
2079 service_description DNS
2080 check_command check_dns_wl!"www.wirelessleiden.nl"
2081}
2082'''
2083
2084 if heavy_load:
2085 print '''\
2086define service {
2087 use wleiden-service
2088 hostgroup_name srv_hybrid
2089 service_description UPTIME
2090 check_command check_snmp_uptime
2091}
2092
2093#define service {
2094# use wleiden-service
2095# hostgroup_name srv_hybrid
2096# service_description NTP
2097# check_command check_ntp_peer
2098#}
2099
2100define service {
2101 use wleiden-service
2102 hostgroup_name srv_hybrid
2103 service_description LOAD
2104 check_command check_netsnmp_load
2105}
2106
2107define service {
2108 use wleiden-service
2109 hostgroup_name srv_hybrid
2110 service_description PROC
2111 check_command check_netsnmp_proc
2112}
2113
2114define service {
2115 use wleiden-service
2116 hostgroup_name srv_hybrid
2117 service_description DISK
2118 check_command check_snmp_disk
2119}
2120'''
2121 for node in get_hostlist():
2122 datadump = get_yaml(node)
2123 if not datadump['status'] == 'up':
2124 continue
2125 if not hostgroup_details.has_key(datadump['monitoring_group']):
2126 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2127 print '''\
2128define host {
2129 use wleiden-node,host-pnp
2130 contact_groups admins
2131 host_name %(autogen_fqdn)s
2132 address %(masterip)s
2133 hostgroups srv_hybrid,%(monitoring_group)s\
2134''' % datadump
2135 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2136 print '''\
2137 parents %(parents)s\
2138''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2139 print '''\
2140}
2141'''
2142
2143
2144 for name,alias in hostgroup_details.iteritems():
2145 print '''\
2146define hostgroup {
2147 hostgroup_name %s
2148 alias %s
2149} ''' % (name, alias)
2150
2151
2152 elif sys.argv[1] == "full-export":
2153 hosts = {}
2154 for node in get_hostlist():
2155 datadump = get_yaml(node)
2156 hosts[datadump['nodename']] = datadump
2157 print yaml.dump(hosts)
2158
2159 elif sys.argv[1] == "dns":
2160 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2161 elif sys.argv[1] == "cleanup":
2162 # First generate all datadumps
2163 datadumps = dict()
2164 ssid_to_node = dict()
2165 for host in get_hostlist():
2166 logger.info("# Processing: %s", host)
2167 # Set some boring default values
2168 datadump = { 'board' : 'UNKNOWN' }
2169 datadump.update(get_yaml(host))
2170 datadumps[datadump['nodename']] = datadump
2171
2172 (poel, errors) = make_relations()
2173 print "\n".join(["# WARNING: %s" % x for x in errors])
2174
2175 for host,datadump in datadumps.iteritems():
2176 try:
2177 # Convert all yes and no to boolean values
2178 def fix_boolean(dump):
2179 for key in dump.keys():
2180 if type(dump[key]) == dict:
2181 dump[key] = fix_boolean(dump[key])
2182 elif str(dump[key]).lower() in ["yes", "true"]:
2183 dump[key] = True
2184 elif str(dump[key]).lower() in ["no", "false"]:
2185 # Compass richting no (Noord Oost) is valid input
2186 if key != "compass": dump[key] = False
2187 return dump
2188 datadump = fix_boolean(datadump)
2189
2190 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2191 if not 'latitude' in datadump and not 'longitude' in datadump:
2192 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2193 elif 'latitude' in datadump and 'longitude' in datadump:
2194 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2195 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2196 # TODO: Compare outcome of both coordinate systems and validate against each-other
2197
2198 if datadump['nodename'].startswith('Proxy'):
2199 datadump['nodename'] = datadump['nodename'].lower()
2200
2201 for iface_key in get_interface_keys(datadump):
2202 try:
2203 # All our normal wireless cards are normal APs now
2204 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2205 datadump[iface_key]['mode'] = 'ap'
2206 # Wireless Leiden SSID have an consistent lowercase/uppercase
2207 if datadump[iface_key].has_key('ssid'):
2208 ssid = datadump[iface_key]['ssid']
2209 prefix = 'ap-WirelessLeiden-'
2210 if ssid.lower().startswith(prefix.lower()):
2211 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2212 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2213 datadump[iface_key]['mode'] = 'autogen-FIXME'
2214 if not datadump[iface_key].has_key('comment'):
2215 datadump[iface_key]['comment'] = 'autogen-FIXME'
2216
2217 if datadump[iface_key].has_key('ns_mac'):
2218 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2219
2220 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2221 datadump[iface_key] = datadump[iface_key]['desc']
2222
2223 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2224 # due to better collision avoidance.
2225 if datadump[iface_key]['type'] == '11b':
2226 datadump[iface_key]['type'] = '11g'
2227
2228 # Setting 802.11g channels to de-facto standards, to avoid
2229 # un-detected sharing with other overlapping channels
2230 #
2231 # Technically we could also use channel 13 in NL, but this is not
2232 # recommended as foreign devices might not be able to select this
2233 # channel. Secondly using 1,5,9,13 instead is going to clash with
2234 # the de-facto usage of 1,6,11.
2235 #
2236 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2237 channels_at_2400Mhz = (1,6,11)
2238 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2239 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2240 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2241 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2242
2243 # Mandatory interface keys
2244 if not datadump[iface_key].has_key('status'):
2245 datadump[iface_key]['status'] = 'planned'
2246
2247 x = datadump[iface_key]['comment']
2248 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2249
2250 # Fixing bridge_type if none is found
2251 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2252 if not 'bridge_type' in datadump[iface_key]:
2253 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2254
2255 # Making sure description works
2256 if datadump[iface_key].has_key('desc'):
2257 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2258 del datadump[iface_key]['desc']
2259 else:
2260 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2261 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2262 if response:
2263 datadump[iface_key]['comment'] = response
2264 del datadump[iface_key]['desc']
2265
2266 # Check DHCP configuration
2267 dhcp_type(datadump[iface_key])
2268
2269 # Set the compass value based on the angle between the poels
2270 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2271 my_pool = poel[network(datadump[iface_key]['ip'])]
2272 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2273 if remote_hosts:
2274 compass_target = remote_hosts[0]
2275 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2276 # TODO: Compass wanted and actual direction might differ
2277
2278 # Monitoring Group default
2279 if not 'monitoring_group' in datadump:
2280 datadump['monitoring_group'] = 'wleiden'
2281
2282 except Exception:
2283 print "# Error while processing interface %s" % iface_key
2284 raise
2285 store_yaml(datadump)
2286 except Exception:
2287 print "# Error while processing %s" % host
2288 raise
2289 elif sys.argv[1] == "list":
2290 use_fqdn = False
2291 if len(sys.argv) < 4:
2292 usage()
2293 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2294 usage()
2295 if not sys.argv[3] in ["nodes","proxies","systems"]:
2296 usage()
2297
2298 if len(sys.argv) > 4:
2299 if sys.argv[4] == "fqdn":
2300 use_fqdn = True
2301 else:
2302 usage()
2303
2304 for system in get_hostlist():
2305 datadump = get_yaml(system)
2306 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2307 continue
2308
2309 output = datadump['autogen_fqdn'] if use_fqdn else system
2310 if sys.argv[2] == "all":
2311 print output
2312 elif datadump['status'] == sys.argv[2]:
2313 print output
2314 elif sys.argv[1] == "create":
2315 if sys.argv[2] == "network.kml":
2316 print make_network_kml.make_graph()
2317 elif sys.argv[2] == "host-ips.txt":
2318 for system in get_hostlist():
2319 datadump = get_yaml(system)
2320 ips = [datadump['masterip']]
2321 for ifkey in get_interface_keys(datadump):
2322 ips.append(datadump[ifkey]['ip'].split('/')[0])
2323 print system, ' '.join(ips)
2324 elif sys.argv[2] == "host-pos.txt":
2325 for system in get_hostlist():
2326 datadump = get_yaml(system)
2327 print system, datadump['rdnap_x'], datadump['rdnap_y']
2328 elif sys.argv[2] == 'ssh_config':
2329 print '''
2330Host *.wleiden.net
2331 User root
2332
2333Host 172.16.*.*
2334 User root
2335'''
2336 for system in get_hostlist():
2337 datadump = get_yaml(system)
2338 print '''\
2339Host %s
2340 User root
2341
2342Host %s
2343 User root
2344
2345Host %s
2346 User root
2347
2348Host %s
2349 User root
2350''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2351 else:
2352 usage()
2353 else:
2354 usage()
2355 else:
2356 # Do not enable debugging for config requests as it highly clutters the output
2357 if not is_text_request():
2358 cgitb.enable()
2359 response_headers, output = process_cgi_request()
2360 print_cgi_response(response_headers, output)
2361
2362def application(environ, start_response):
2363 status = '200 OK'
2364 response_headers, output = process_cgi_request(environ)
2365 start_response(status, response_headers)
2366
2367 # Debugging only
2368 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2369 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2370 return [output]
2371
2372if __name__ == "__main__":
2373 main()
Note: See TracBrowser for help on using the repository browser.