source: genesis/tools/gformat.py@ 14170

Last change on this file since 14170 was 14170, checked in by rick, 7 years ago

Add custom prefix support for config URI

/wleiden/config/<path> now also support automatically
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 80.6 KB
Line 
1#!/usr/local/bin/python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
35SVNVERSION = filter(os.path.isfile, ('/usr/local/bin/svnversion', '/usr/bin/svnversion'))[0]
36
37import argparse
38import cgi
39import cgitb
40import copy
41import glob
42import make_network_kml
43import math
44import pyproj
45import random
46import re
47import socket
48import string
49import subprocess
50import textwrap
51import time
52import traceback
53import urlparse
54
55from pprint import pprint
56from collections import defaultdict, OrderedDict
57from sys import stderr
58try:
59 import yaml
60except ImportError, e:
61 print e
62 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
63 exit(1)
64
65try:
66 from yaml import CLoader as Loader
67 from yaml import CDumper as Dumper
68except ImportError:
69 from yaml import Loader, Dumper
70
71from jinja2 import Environment, Template
72def yesorno(value):
73 return "YES" if bool(value) else "NO"
74env = Environment()
75env.filters['yesorno'] = yesorno
76def render_template(datadump, template):
77 result = env.from_string(template).render(datadump)
78 # Make it look pretty to the naked eye, as jinja templates are not so
79 # friendly when it comes to whitespace formatting
80 ## Remove extra whitespace at end of line lstrip() style.
81 result = re.sub(r'\n[\ ]+','\n', result)
82 ## Include only a single newline between an definition and a comment
83 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
84 ## Remove extra newlines after single comment
85 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
86 return result
87
88import logging
89logging.basicConfig(format='# %(levelname)s: %(message)s' )
90logger = logging.getLogger()
91logger.setLevel(logging.DEBUG)
92
93
94if os.environ.has_key('CONFIGROOT'):
95 NODE_DIR = os.environ['CONFIGROOT']
96else:
97 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
98__version__ = '$Id: gformat.py 14170 2018-05-22 19:46:18Z rick $'
99
100CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
101
102files = [
103 'authorized_keys',
104 'dnsmasq.conf',
105 'dhcpd.conf',
106 'rc.conf.local',
107 'resolv.conf',
108 'motd',
109 'ntp.conf',
110 'pf.hybrid.conf.local',
111 'unbound.wleiden.conf',
112 'wleiden.yaml',
113 ]
114
115# Global variables uses
116OK = 10
117DOWN = 20
118UNKNOWN = 90
119
120
121ileiden_proxies = OrderedDict()
122normal_proxies = []
123datadump_cache = {}
124interface_list_cache = {}
125rc_conf_local_cache = {}
126nameservers_cache = []
127relations_cache = None
128
129NO_DHCP = 0
130DHCP_CLIENT = 10
131DHCP_SERVER = 20
132def dhcp_type(item):
133 if not item.has_key('dhcp'):
134 return NO_DHCP
135 elif not item['dhcp']:
136 return NO_DHCP
137 elif item['dhcp'].lower() == 'client':
138 return DHCP_CLIENT
139 else:
140 # Validation Checks
141 begin,end = map(int,item['dhcp'].split('-'))
142 if begin >= end:
143 raise ValueError("DHCP Start >= DHCP End")
144 return DHCP_SERVER
145
146def etrs2rd(lat, lon):
147 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
148 p2 = pyproj.Proj(init='EPSG:28992')
149 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
150 return (RDx, RDy)
151
152def rd2etrs(RDx, RDy):
153 p1 = pyproj.Proj(init='EPSG:28992')
154 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
155 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
156 return (lat, lon)
157
158def get_yaml(item,add_version_info=True):
159 try:
160 """ Get configuration yaml for 'item'"""
161 if datadump_cache.has_key(item):
162 return datadump_cache[item].copy()
163
164 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
165 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
166 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
167
168 # Default values
169 datadump = {
170 'autogen_revision' : 'NOTFOUND',
171 'autogen_gfile' : gfile,
172 'service_proxy_ileiden' : False,
173 }
174 f = open(gfile, 'r')
175 datadump.update(yaml.load(f,Loader=Loader))
176 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
177 if datadump['nodetype'] == 'Hybrid':
178 # Some values are defined implicitly
179 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
180 datadump['service_incoming_rdr'] = True
181 # Use some boring defaults
182 defaults = {
183 'service_proxy_normal' : False,
184 'service_accesspoint' : True,
185 'service_incoming_rdr' : False,
186 'service_concentrator' : False,
187 'monitoring_group' : 'wleiden',
188 }
189 for (key,value) in defaults.iteritems():
190 if not datadump.has_key(key):
191 datadump[key] = value
192 f.close()
193
194 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
195 if add_version_info:
196 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
197 lines = p.communicate()[0].split('\n')
198 if p.returncode == 0:
199 for line in lines:
200 if line:
201 (key, value) = line.split(': ')
202 datadump["autogen_" + key.lower().replace(' ','_')] = value
203
204 # Preformat certain needed variables for formatting and push those into special object
205 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
206
207 wlan_count=0
208 try:
209 for key in get_interface_keys(datadump, True):
210 datadump[key]['autogen_ifbase'] = key.split('_')[1]
211 datadump[key]['autogen_vlan'] = False
212 datadump[key]['autogen_vlan_alias'] = False
213
214 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
215 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
216 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
217
218 if datadump[key].has_key('parent'):
219 if datadump[key].has_key('ip'):
220 raise ValueError("Interface bridge member cannot have IP assigned")
221 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
222 raise ValueError("Interface bridge member cannot have DHCP set")
223
224 if datadump[key].has_key('ip'):
225 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
226
227 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
228 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
229 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
230 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
231 wlan_count += 1
232 else:
233 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
234 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
235 datadump[key]['autogen_if_dhcp'] = '.'.join(key.split('_')[1:3])
236 datadump[key]['autogen_vlan'] = key.split('_')[2]
237 datadump[key]['autogen_vlan_alias'] = '_alias' in key
238 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
239 else:
240 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
241 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
242
243 except Exception as exc:
244 exc.args = ("# Error while processing interface %s" % key,) + exc.args
245 raise
246
247 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
248 if dhcp_type(datadump[key]) == DHCP_SERVER]
249
250 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
251 datadump['autogen_item'] = item
252
253 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
254 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
255 datadump_cache[item] = datadump.copy()
256 except Exception as exc:
257 exc.args = ("# Error while processing %s" % item,) + exc.args
258 raise
259 return datadump
260
261
262def store_yaml(datadump, header=False):
263 """ Store configuration yaml for 'item'"""
264 item = datadump['autogen_item']
265 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
266
267 output = generate_wleiden_yaml(datadump, header)
268
269 f = open(gfile, 'w')
270 f.write(output)
271 f.close()
272
273
274def network(ip):
275 addr, mask = ip.split('/')
276 # Not parsing of these folks please
277 addr = parseaddr(addr)
278 mask = int(mask)
279 network = addr & ~((1 << (32 - mask)) - 1)
280 return network
281
282
283
284def make_relations():
285 """ Process _ALL_ yaml files to get connection relations """
286 global relations_cache
287
288 if relations_cache:
289 return relations_cache
290
291 errors = []
292 poel = defaultdict(list)
293
294 for host in get_hostlist():
295 datadump = get_yaml(host)
296 try:
297 for iface_key in get_interface_keys(datadump):
298 # Bridge members has no IP assigned
299 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
300 continue
301 net_addr = network(datadump[iface_key]['ip'])
302 poel[net_addr] += [(host,datadump[iface_key].copy())]
303 except (KeyError, ValueError) as e:
304 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
305 continue
306
307 relations_cache = (poel, errors)
308 return relations_cache
309
310
311
312def valid_addr(addr):
313 """ Show which address is valid in which are not """
314 return str(addr).startswith('172.')
315
316def get_hostlist():
317 """ Combined hosts and proxy list"""
318 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
319
320def angle_between_points(lat1,lat2,long1,long2):
321 """
322 Return Angle in radians between two GPS coordinates
323 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
324 """
325 dy = lat2 - lat1
326 dx = math.cos(lat1)*(long2 - long1)
327 angle = math.atan2(dy,dx)
328 return angle
329
330
331
332def angle_to_cd(angle):
333 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
334
335 # For easy conversion get positive degree
336 degrees = math.degrees(angle)
337 abs_degrees = 360 + degrees if degrees < 0 else degrees
338
339 # Numbers can be confusing calculate from the 4 main directions
340 p = 22.5
341 if abs_degrees < p:
342 cd = "n"
343 elif abs_degrees < (90 - p):
344 cd = "no"
345 elif abs_degrees < (90 + p):
346 cd = "o"
347 elif abs_degrees < (180 - p):
348 cd = "zo"
349 elif abs_degrees < (180 + p):
350 cd = "z"
351 elif abs_degrees < (270 - p):
352 cd = "zw"
353 elif abs_degrees < (270 + p):
354 cd = "w"
355 elif abs_degrees < (360 - p):
356 cd = "nw"
357 else:
358 cd = "n"
359 return cd
360
361
362
363def cd_between_hosts(hostA, hostB, datadumps):
364 # Using RDNAP coordinates
365 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
366 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
367 return angle_to_cd(math.atan2(dx,dy))
368
369 # GPS coordinates seems to fail somehow
370 #latA = float(datadumps[hostA]['latitude'])
371 #latB = float(datadumps[hostB]['latitude'])
372 #lonA = float(datadumps[hostA]['longitude'])
373 #lonB = float(datadumps[hostB]['longitude'])
374 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
375
376
377def generate_title(nodelist):
378 """ Main overview page """
379 items = { \
380 'root' : ".",
381 'version' : subprocess.Popen([SVNVERSION, "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
382 }
383 def fl(spaces, line):
384 return (' ' * spaces) + line + '\n'
385
386 output = """
387<html>
388 <head>
389 <title>Wireless leiden Configurator - GFormat</title>
390 <style type="text/css">
391 th {background-color: #999999}
392 tr:nth-child(odd) {background-color: #cccccc}
393 tr:nth-child(even) {background-color: #ffffff}
394 th, td {padding: 0.1em 1em}
395 </style>
396 </head>
397 <body>
398 <center>
399 <form type="GET" action="%(root)s">
400 <input type="hidden" name="action" value="update">
401 <input type="submit" value="Update Configuration Database (SVN)">
402 </form>
403 <table>
404 <caption><h3>Wireless Leiden Configurator - Revision %(version)s</h3></caption>
405 """ % items
406
407 for node in nodelist:
408 items['node'] = node
409 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
410 for config in files:
411 items['config'] = config
412 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
413 output += fl(5, "</tr>")
414 output += """
415 </table>
416 <hr />
417 <em>%s</em>
418 </center>
419 </body>
420</html>
421 """ % __version__
422
423 return output
424
425
426
427def generate_node(node):
428 """ Print overview of all files available for node """
429 return "\n".join(files)
430
431def generate_node_overview(host, datadump=False):
432 """ Print overview of all files available for node """
433 if not datadump:
434 datadump = get_yaml(host)
435 params = { 'host' : host }
436 output = "<em><a href='..'>Back to overview</a></em><hr />"
437 output += "<h2>Available files:</h2><ul>"
438 for cf in files:
439 params['cf'] = cf
440 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
441 output += "</ul>"
442
443 # Generate and connection listing
444 output += "<h2>Connected To:</h2><ul>"
445 (poel, errors) = make_relations()
446 for network, hosts in poel.iteritems():
447 if host in [x[0] for x in hosts]:
448 if len(hosts) == 1:
449 # Single not connected interface
450 continue
451 for remote,ifacedump in hosts:
452 if remote == host:
453 # This side of the interface
454 continue
455 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
456 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
457 output += "</ul>"
458 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
459
460 output += "<hr /><em><a href='..'>Back to overview</a></em>"
461 return output
462
463
464def generate_header(datadump, ctag="#"):
465 return """\
466%(ctag)s
467%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
468%(ctag)s
469""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
470
471
472
473def parseaddr(s):
474 """ Process IPv4 CIDR notation addr to a (binary) number """
475 f = s.split('.')
476 return (long(f[0]) << 24L) + \
477 (long(f[1]) << 16L) + \
478 (long(f[2]) << 8L) + \
479 long(f[3])
480
481
482
483def showaddr(a):
484 """ Display IPv4 addr in (dotted) CIDR notation """
485 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
486
487
488def is_member(ip, mask, canidate):
489 """ Return True if canidate is part of ip/mask block"""
490 ip_addr = parseaddr(ip)
491 ip_canidate = parseaddr(canidate)
492 mask = int(mask)
493 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
494 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
495 return ip_addr == ip_canidate
496
497
498
499def cidr2netmask(netmask):
500 """ Given a 'netmask' return corresponding CIDR """
501 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
502
503def get_network(addr, mask):
504 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
505
506
507def generate_dhcpd_conf(datadump):
508 """ Generate config file '/usr/local/etc/dhcpd.conf """
509 # Redundency support, in cause local DNS server is not running/responding.
510 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
511 output = generate_header(datadump)
512 output += Template("""\
513# option definitions common to all supported networks...
514option domain-name "dhcp.{{ autogen_fqdn }}";
515
516default-lease-time 600;
517max-lease-time 7200;
518
519# Use this to enble / disable dynamic dns updates globally.
520#ddns-update-style none;
521
522# If this DHCP server is the official DHCP server for the local
523# network, the authoritative directive should be uncommented.
524authoritative;
525
526# Use this to send dhcp log messages to a different log file (you also
527# have to hack syslog.conf to complete the redirection).
528log-facility local7;
529
530# UniFi Discovery Support
531option space ubnt;
532option ubnt.unifi-address code 1 = ip-address;
533#
534class "ubnt" {
535 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
536 option vendor-class-identifier "ubnt";
537 vendor-option-space ubnt;
538}
539
540#
541# Interface definitions
542#
543\n\n""").render(datadump)
544
545
546 # TODO: Use textwrap.fill instead
547 def indent(text, count):
548 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
549
550 # Process DHCP blocks
551 dhcp_out = defaultdict(list)
552 for iface_key in get_interface_keys(datadump):
553 ifname = datadump[iface_key]['autogen_ifbase']
554 groupif = datadump[iface_key]['autogen_if_dhcp']
555 if not datadump[iface_key].has_key('comment'):
556 datadump[iface_key]['comment'] = None
557
558 if not datadump[iface_key].has_key('ip'):
559 continue
560
561 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
562
563 (addr, mask) = datadump[iface_key]['ip'].split('/')
564 datadump[iface_key]['autogen_addr'] = addr
565 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
566 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
567
568 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
569 dhcp_out[groupif].append(textwrap.dedent("""\
570 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
571 ### not autoritive
572 }
573 """ % datadump[iface_key]))
574 continue
575
576 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
577 dhcp_part = ".".join(addr.split('.')[0:3])
578 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
579 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
580 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
581
582 # Assume the first 10 IPs could be used for static entries
583 if 'no_portal' in datadump:
584 fixed = 5
585 for mac in datadump['no_portal']:
586 dhcp_out[groupif].append(textwrap.dedent("""\
587 host fixed-%(ifname)s-%(fixed)s {
588 hardware ethernet %(mac)s;
589 fixed-address %(prefix)s.%(fixed)s;
590 }
591 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
592 fixed += 1
593
594 if 'dhcp_fixed' in datadump[iface_key]:
595 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
596 dhcp_out[groupif].append(textwrap.dedent("""\
597 host fixed-%(host)s {
598 hardware ethernet %(mac)s;
599 fixed-address %(addr)s;
600 }
601 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
602
603
604 dhcp_out[groupif].append(textwrap.dedent("""\
605 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
606 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
607 option routers %(autogen_addr)s;
608 option domain-name-servers %(autogen_dns_servers)s;
609 option ubnt.unifi-address 172.17.107.10;
610 }
611 """ % datadump[iface_key]))
612
613 # Output the blocks in groups
614 for ifname,value in sorted(dhcp_out.iteritems()):
615 output += ("shared-network \"%s\" {\n" % ifname) + indent(''.join(value), 2).rstrip() + '\n}\n\n'
616 return output
617
618
619
620def generate_dnsmasq_conf(datadump):
621 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
622 output = generate_header(datadump)
623 output += Template("""\
624# DHCP server options
625dhcp-authoritative
626dhcp-fqdn
627domain=dhcp.{{ autogen_fqdn }}
628domain-needed
629expand-hosts
630log-async=100
631
632# Low memory footprint
633cache-size=10000
634
635\n""").render(datadump)
636
637 for iface_key in get_interface_keys(datadump):
638 if not datadump[iface_key].has_key('comment'):
639 datadump[iface_key]['comment'] = None
640 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
641
642 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
643 output += "# not autoritive\n\n"
644 continue
645
646 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
647 (ip, cidr) = datadump[iface_key]['ip'].split('/')
648 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
649
650 dhcp_part = ".".join(ip.split('.')[0:3])
651 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
652 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
653 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
654
655 return output
656
657
658class AutoVivification(dict):
659 """Implementation of perl's autovivification feature."""
660 def __getitem__(self, item):
661 try:
662 return dict.__getitem__(self, item)
663 except KeyError:
664 value = self[item] = type(self)()
665 return value
666
667def make_interface_list(datadump):
668 if interface_list_cache.has_key(datadump['autogen_item']):
669 return (interface_list_cache[datadump['autogen_item']])
670 # lo0 configuration:
671 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
672 # - masterip is special as it needs to be assigned to at
673 # least one interface, so if not used assign to lo0
674 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
675 vlan_list = defaultdict(list)
676 bridge_list = defaultdict(list)
677 flags_if = AutoVivification()
678 dhclient_if = {'lo0' : False}
679
680 # XXX: Find some way of send this output nicely
681 output = ''
682
683 masterip_used = False
684 for iface_key in get_interface_keys(datadump):
685 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
686 masterip_used = True
687 break
688 if not masterip_used:
689 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
690
691 if 'serviceid' in datadump:
692 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
693
694 for iface_key in get_interface_keys(datadump):
695 ifacedump = datadump[iface_key]
696 if ifacedump['autogen_bridge_alias']:
697 ifname = ifacedump['autogen_ifbase']
698 else:
699 ifname = ifacedump['autogen_ifname']
700
701 # If defined as vlan interface
702 if ifacedump['autogen_vlan']:
703 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
704
705 # If defined as bridge interface
706 if ifacedump['autogen_bridge_member']:
707 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
708
709
710 # Flag dhclient is possible
711 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
712 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
713
714 # Ethernet address
715 if ifacedump.has_key('ether'):
716 flags_if[ifname]['ether'] = ifacedump['ether']
717
718 # Add interface IP to list
719 if ifacedump.has_key('ip'):
720 item = (ifacedump['ip'], ifacedump['comment'])
721 if addrs_list.has_key(ifname):
722 addrs_list[ifname].append(item)
723 else:
724 addrs_list[ifname] = [item]
725
726 # Alias only needs IP assignment for now, this might change if we
727 # are going to use virtual accesspoints
728 if "alias" in iface_key:
729 continue
730
731 # XXX: Might want to deduct type directly from interface name
732 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
733 # Default to station (client) mode
734 ifacedump['autogen_wlanmode'] = "sta"
735 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
736 ifacedump['autogen_wlanmode'] = "ap"
737
738 if not ifacedump.has_key('channel'):
739 if ifacedump['type'] == '11a':
740 ifacedump['channel'] = 36
741 else:
742 ifacedump['channel'] = 1
743
744 # Allow special hacks at the back like wds and stuff
745 if not ifacedump.has_key('extra'):
746 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
747 else:
748 ifacedump['autogen_extra'] = ifacedump['extra']
749
750 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
751
752 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
753 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
754 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
755 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
756 output += "\n"
757
758 elif ifacedump['type'] in ['ethernet', 'eth']:
759 # No special config needed besides IP
760 pass
761 elif ifacedump['type'] in ['vlan']:
762 # VLAN member has no special configuration
763 pass
764 else:
765 assert False, "Unknown type " + ifacedump['type']
766
767 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
768 interface_list_cache[datadump['autogen_item']] = store
769 return(store)
770
771
772
773def create_proxies_list():
774 if not ileiden_proxies or not normal_proxies:
775 # Placeholder for to-be-installed proxies, this will avoid updating the all
776 # nodes to include this new machine, yet due to an unbound issue, this list
777 # has to be kept small.
778
779 for i in range(1,20):
780 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
781
782 for host in get_hostlist():
783 hostdump = get_yaml(host)
784 if hostdump['status'] == 'up':
785 if hostdump['service_proxy_ileiden']:
786 ileiden_proxies[hostdump['serviceid']] = hostdump
787 if hostdump['service_proxy_normal']:
788 normal_proxies.append(hostdump)
789
790
791
792def generate_rc_conf_local(datadump):
793 """ Generate configuration file '/etc/rc.conf.local' """
794 item = datadump['autogen_item']
795 if rc_conf_local_cache.has_key(item):
796 return rc_conf_local_cache[item]
797
798 if not datadump.has_key('ileiden'):
799 datadump['autogen_ileiden_enable'] = False
800 else:
801 datadump['autogen_ileiden_enable'] = datadump['ileiden']
802
803 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
804
805 create_proxies_list()
806 datadump['autogen_ileiden_proxies'] = ileiden_proxies
807 datadump['autogen_normal_proxies'] = normal_proxies
808 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
809 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
810 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
811 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
812
813 output = generate_header(datadump, "#");
814 output += render_template(datadump, """\
815hostname='{{ autogen_fqdn }}'
816location='{{ location }}'
817nodetype="{{ nodetype }}"
818
819#
820# Configured listings
821#
822captive_portal_whitelist=""
823{% if nodetype == "Proxy" %}
824#
825# Proxy Configuration
826#
827{% if gateway and service_proxy_ileiden -%}
828defaultrouter="{{ gateway }}"
829{% else -%}
830#defaultrouter="NOTSET"
831{% endif -%}
832internalif="{{ internalif }}"
833ileiden_enable="{{ autogen_ileiden_enable }}"
834gateway_enable="{{ autogen_ileiden_enable }}"
835pf_enable="yes"
836pf_rules="/etc/pf.conf"
837{% if autogen_ileiden_enable -%}
838pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
839lvrouted_enable="{{ autogen_ileiden_enable }}"
840lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
841{% else -%}
842pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
843{% endif -%}
844{% if internalroute -%}
845static_routes="wleiden"
846route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
847{% endif -%}
848
849{% elif nodetype == "Hybrid" %}
850 #
851 # Hybrid Configuration
852 #
853 list_ileiden_proxies="
854 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
855 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
856 {% endfor -%}
857 "
858 list_normal_proxies="
859 {% for item in autogen_normal_proxies -%}
860 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
861 {% endfor -%}
862 "
863
864 {% if autogen_dhcp_interfaces -%}
865 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
866 {% else %}
867 captive_portal_interfaces="dummy"
868 {% endif %}
869 externalif="{{ externalif|default('vr0', true) }}"
870 masterip="{{ masterip }}"
871
872 {% if gateway and service_proxy_ileiden %}
873 defaultrouter="{{ gateway }}"
874 {% else %}
875 #defaultrouter="NOTSET"
876 {% endif %}
877
878 #
879 # Defined services
880 #
881 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
882 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
883 service_accesspoint="{{ service_accesspoint|yesorno }}"
884 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
885 service_concentrator="{{ service_concentrator|yesorno }}"
886
887 {% if service_proxy_ileiden %}
888 pf_rules="/etc/pf.hybrid.conf"
889 {% if service_concentrator %}
890 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
891 {% else %}
892 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
893 {% endif %}
894 pf_flags="$pf_flags -D publicnat=80,443"
895 lvrouted_flags="$lvrouted_flags -g"
896 {% elif service_proxy_normal or service_incoming_rdr %}
897 pf_rules="/etc/pf.hybrid.conf"
898 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
899 pf_flags="$pf_flags -D publicnat=0"
900 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
901 named_setfib="1"
902 tinyproxy_setfib="1"
903 dnsmasq_setfib="1"
904 sshd_setfib="1"
905 {% else %}
906 named_auto_forward_only="YES"
907 pf_rules="/etc/pf.node.conf"
908 pf_flags=""
909 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
910 {% endif %}
911 {% if service_concentrator %}
912 # Do mind installing certificates is NOT done automatically for security reasons
913 openvpn_enable="YES"
914 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
915 {% endif %}
916
917 {% if service_proxy_normal %}
918 tinyproxy_enable="yes"
919 {% else %}
920 pen_wrapper_enable="yes"
921 {% endif %}
922
923 {% if service_accesspoint %}
924 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
925 {% endif %}
926
927 {% if board == "ALIX2" or board == "net4801" %}
928 #
929 # ''Fat'' configuration, board has 256MB RAM
930 #
931 dnsmasq_enable="NO"
932 named_enable="YES"
933 unbound_enable="YES"
934 {% if autogen_dhcp_interfaces -%}
935 dhcpd_enable="YES"
936 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
937 {% endif -%}
938 {% elif board == "apu1d" %}
939 #
940 # ''Fat'' configuration, board has 1024MB RAM
941 #
942 dnsmasq_enable="NO"
943 unbound_enable="YES"
944 {% if autogen_dhcp_interfaces -%}
945 dhcpd_enable="YES"
946 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
947 {% endif -%}
948 {% endif -%}
949{% endif %}
950
951#
952# Script variables
953#
954attached_devices="{{ autogen_attached_devices|join(' ') }}"
955neighbours="{{ autogen_neighbours|join(' ') }}"
956
957
958#
959# Interface definitions
960#\n
961""")
962
963 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
964 for iface, vlans in sorted(vlan_list.items()):
965 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
966
967 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
968 for iface in sorted(vlan_list.keys()):
969 if not iface in addrs_list.keys():
970 output += "ifconfig_%s='up'\n" % iface
971
972 output += "\n"
973
974 # Bridge configuration:
975 if bridge_list.keys():
976 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
977
978 for iface in bridge_list.keys():
979 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
980
981 # Bridge member interfaces not containing a configuration should be marked active explcitly.
982 for _,members in bridge_list.items():
983 for iface in members:
984 if not iface in addrs_list.keys():
985 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
986
987 output += "\n"
988
989 # Details like SSID
990 if extra_ouput:
991 output += extra_ouput.strip() + "\n"
992
993 # Print IP address which needs to be assigned over here
994 output += "\n"
995 for iface,addrs in sorted(addrs_list.iteritems()):
996 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
997 output += "# %s || %s || %s\n" % (iface, addr, comment)
998
999 # Write DHCLIENT entry
1000 if iface in dhclient_if and dhclient_if[iface]:
1001 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1002 continue
1003
1004 # Make sure the external address is always first as this is needed in the
1005 # firewall setup
1006 addrs = sorted(
1007 [x for x in addrs if not '0.0.0.0' in x[0]],
1008 key=lambda x: x[0].split('.')[0],
1009 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1010 )
1011
1012 idx_offset = 0
1013 # Set MAC is required
1014 if flags_if[iface].has_key('ether'):
1015 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1016 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1017 idx_offset += 1
1018 elif iface in bridge_list:
1019 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1020 idx_offset += 1
1021 else:
1022 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1023
1024 for idx, addr in enumerate(addrs[1:]):
1025 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1026
1027 output += "\n"
1028
1029 rc_conf_local_cache[datadump['autogen_item']] = output
1030 return output
1031
1032
1033
1034def get_all_configs():
1035 """ Get dict with key 'host' with all configs present """
1036 configs = dict()
1037 for host in get_hostlist():
1038 datadump = get_yaml(host)
1039 configs[host] = datadump
1040 return configs
1041
1042
1043def get_interface_keys(config, extra=False):
1044 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1045 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1046 if extra == False:
1047 return filter(lambda x: not "extra" in x, elems)
1048 else:
1049 return elems
1050
1051
1052def get_used_ips(configs):
1053 """ Return array of all IPs used in config files"""
1054 ip_list = []
1055 for config in configs:
1056 ip_list.append(config['masterip'])
1057 if 'serviceid' in config:
1058 ip_list.append(config['serviceid'])
1059 for iface_key in get_interface_keys(config, True):
1060 l = config[iface_key]['ip']
1061 addr, mask = l.split('/')
1062 # Special case do not process
1063 if valid_addr(addr):
1064 ip_list.append(addr)
1065 else:
1066 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1067 return sorted(ip_list)
1068
1069
1070
1071def get_nameservers(max_servers=None):
1072 if nameservers_cache:
1073 return nameservers_cache[0:max_servers]
1074
1075 for host in get_hostlist():
1076 hostdump = get_yaml(host)
1077 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1078 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1079
1080 return nameservers_cache[0:max_servers]
1081
1082
1083def get_neighbours(datadump):
1084 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1085
1086 (poel, errors) = make_relations()
1087 table = []
1088 for iface,addrs in sorted(addrs_list.iteritems()):
1089 if iface in ['lo0']:
1090 continue
1091
1092 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1093 if not addr.startswith('172.'):
1094 # Avoid listing internet connections as pool
1095 continue
1096 for neighbour in poel[network(addr)]:
1097 if neighbour[0] != datadump['autogen_item']:
1098 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1099 return table
1100
1101
1102def get_attached_devices(datadump, url=False):
1103 table = []
1104 for iface_key in get_interface_keys(datadump, True):
1105 ifacedump = datadump[iface_key]
1106
1107 if not ifacedump.has_key('ns_ip'):
1108 continue
1109
1110 x_ip = ifacedump['ns_ip'].split('/')[0]
1111
1112 if 'mode' in ifacedump:
1113 x_mode = ifacedump['mode']
1114 else:
1115 x_mode = 'unknown'
1116
1117 if 'bridge_type' in ifacedump:
1118 device_type = ifacedump['bridge_type']
1119 else:
1120 device_type = 'Unknown'
1121
1122 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1123 return table
1124
1125
1126def generate_resolv_conf(datadump):
1127 """ Generate configuration file '/etc/resolv.conf' """
1128 # XXX: This should properly going to be an datastructure soon
1129 datadump['autogen_header'] = generate_header(datadump, "#")
1130 datadump['autogen_edge_nameservers'] = ''
1131
1132
1133 for masterip,realname in get_nameservers():
1134 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1135
1136 return Template("""\
1137{{ autogen_header }}
1138search wleiden.net
1139
1140# Try local (cache) first
1141nameserver 127.0.0.1
1142
1143{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1144nameserver 8.8.8.8 # Google Public NameServer
1145nameserver 64.6.64.6 # Verisign Public NameServer
1146{% else -%}
1147# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1148{{ autogen_edge_nameservers }}
1149{% endif -%}
1150""").render(datadump)
1151
1152
1153
1154def generate_ntp_conf(datadump):
1155 """ Generate configuration file '/etc/ntp.conf' """
1156 # XXX: This should properly going to be an datastructure soon
1157
1158 datadump['autogen_header'] = generate_header(datadump, "#")
1159 datadump['autogen_ntp_servers'] = ''
1160 for host in get_hostlist():
1161 hostdump = get_yaml(host)
1162 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1163 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1164
1165 return Template("""\
1166{{ autogen_header }}
1167
1168{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1169# Machine hooked to internet.
1170server 0.nl.pool.ntp.org iburst maxpoll 9
1171server 1.nl.pool.ntp.org iburst maxpoll 9
1172server 2.nl.pool.ntp.org iburst maxpoll 9
1173server 3.nl.pool.ntp.org iburst maxpoll 9
1174{% else -%}
1175# Local Wireless Leiden NTP Servers.
1176server 0.pool.ntp.wleiden.net iburst maxpoll 9
1177server 1.pool.ntp.wleiden.net iburst maxpoll 9
1178server 2.pool.ntp.wleiden.net iburst maxpoll 9
1179server 3.pool.ntp.wleiden.net iburst maxpoll 9
1180
1181# All the configured NTP servers
1182{{ autogen_ntp_servers }}
1183{% endif %}
1184
1185# If a server loses sync with all upstream servers, NTP clients
1186# no longer follow that server. The local clock can be configured
1187# to provide a time source when this happens, but it should usually
1188# be configured on just one server on a network. For more details see
1189# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1190# The use of Orphan Mode may be preferable.
1191#
1192server 127.127.1.0
1193fudge 127.127.1.0 stratum 10
1194""").render(datadump)
1195
1196
1197def generate_pf_hybrid_conf_local(datadump):
1198 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1199 datadump['autogen_header'] = generate_header(datadump, "#")
1200 if datadump['service_incoming_rdr']:
1201 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1202 return Template("""\
1203{{ autogen_header }}
1204
1205# Redirect some internal facing services outside (7)
1206# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1207{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1208rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1209{% endfor -%}
1210# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1211{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1212rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1213{% endfor -%}
1214""").render(datadump)
1215
1216def generate_unbound_wleiden_conf(datadump):
1217 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1218 datadump['autogen_header'] = generate_header(datadump, "#")
1219
1220 autogen_ips = []
1221 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1222 for iface,addrs in sorted(addrs_list.iteritems()):
1223 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1224 if addr.startswith('172'):
1225 autogen_ips.append((addr.split('/')[0], comment))
1226 datadump['autogen_ips'] = autogen_ips
1227
1228 create_proxies_list()
1229 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1230 return Template("""\
1231{{ autogen_header }}
1232
1233server:
1234 ## Static definitions fail to start on systems with broken ue(4) interfaces
1235{%- for ip,comment in autogen_ips %}
1236 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1237{%- endfor %}
1238 ## Enabling wildcard matching as work-around
1239 interface: 0.0.0.0
1240 interface: ::0
1241
1242forward-zone:
1243 name: '.'
1244{%- if service_proxy_ileiden %}
1245 forward-addr: 8.8.8.8 # Google DNS A
1246 forward-addr: 8.8.4.4 # Google DNS B
1247 forward-addr: 208.67.222.222 # OpenDNS DNS A
1248 forward-addr: 208.67.220.220 # OpenDNS DNS B
1249{% else -%}
1250{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1251 {%- if loop.index <= 5 %}
1252 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1253 {%- endif %}
1254{%- endfor %}
1255{%- endif %}
1256""").render(datadump)
1257
1258def generate_motd(datadump):
1259 """ Generate configuration file '/etc/motd' """
1260 output = Template("""\
1261FreeBSD run ``service motd onestart'' to make me look normal
1262
1263 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1264 Loc: {{ location }}
1265
1266Services:
1267{% if board == "ALIX2" or board == "net4801" -%}
1268{{" -"}} Core Node ({{ board }})
1269{% else -%}
1270{{" -"}} Hulp Node ({{ board }})
1271{% endif -%}
1272{% if service_proxy_normal -%}
1273{{" -"}} Normal Proxy
1274{% endif -%}
1275{% if service_proxy_ileiden -%}
1276{{" -"}} iLeiden Proxy
1277{% endif -%}
1278{% if service_incoming_rdr -%}
1279{{" -"}} Incoming port redirects
1280{% endif %}
1281Interlinks:\n
1282""").render(datadump)
1283
1284
1285 def make_table(table):
1286 if not table:
1287 return " - none\n"
1288 else:
1289 lines = ""
1290 col_width = [max(len(x) for x in col) for col in zip(*table)]
1291 for row in table:
1292 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1293 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1294 return lines
1295
1296 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1297 table = []
1298 for iface,addrs in sorted(addrs_list.iteritems()):
1299 if iface in ['lo0']:
1300 continue
1301 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1302 table.append((iface, addr, comment))
1303
1304 output += make_table(table)
1305 output += '\n'
1306 output += """\
1307Attached devices:
1308"""
1309 output += make_table(get_attached_devices(datadump, url=True))
1310 output += '\n'
1311 output += """\
1312Available neighbours:
1313"""
1314 output += make_table(get_neighbours(datadump))
1315
1316 return output
1317
1318
1319def format_yaml_value(value):
1320 """ Get yaml value in right syntax for outputting """
1321 if isinstance(value,str):
1322 output = '"%s"' % value
1323 else:
1324 output = value
1325 return output
1326
1327
1328
1329def format_wleiden_yaml(datadump):
1330 """ Special formatting to ensure it is editable"""
1331 output = "# Genesis config yaml style\n"
1332 output += "# vim:ts=2:et:sw=2:ai\n"
1333 output += "#\n"
1334 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1335 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1336 if key == 'rdr_rules':
1337 output += '%-10s:\n' % 'rdr_rules'
1338 for rdr_rule in datadump[key]:
1339 output += '- %s\n' % rdr_rule
1340 else:
1341 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1342
1343 output += "\n\n"
1344
1345 # Format (key, required)
1346 key_order = (
1347 ('comment', True),
1348 ('parent', False),
1349 ('ip', False),
1350 ('ipv6', False),
1351 ('ether', False),
1352 ('desc', True),
1353 ('sdesc', True),
1354 ('mode', True),
1355 ('type', True),
1356 ('extra_type', False),
1357 ('channel', False),
1358 ('ssid', False),
1359 ('wlan_mac', False),
1360 ('dhcp', True),
1361 ('dhcp_fixed', False),
1362 ('compass', False),
1363 ('distance', False),
1364 ('ns_ip', False),
1365 ('repeater_ip', False),
1366 ('bullet2_ip', False),
1367 ('ns_mac', False),
1368 ('bullet2_mac', False),
1369 ('ns_type', False),
1370 ('bridge_type', False),
1371 ('encrypted', False),
1372 ('status', True),
1373 )
1374
1375 for iface_key in sorted(iface_keys):
1376 try:
1377 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1378 if remainder:
1379 raise KeyError("invalid keys: %s" % remainder)
1380
1381 output += "%s:\n" % iface_key
1382 for key,required in key_order:
1383 if datadump[iface_key].has_key(key):
1384 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1385 output += "\n\n"
1386 except Exception as exc:
1387 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
1388 raise
1389
1390 return output
1391
1392
1393
1394def generate_wleiden_yaml(datadump, header=True):
1395 """ Generate (petty) version of wleiden.yaml"""
1396 output = generate_header(datadump, "#") if header else ''
1397
1398 for key in datadump.keys():
1399 if key.startswith('autogen_'):
1400 del datadump[key]
1401 # Interface autogen cleanups
1402 elif type(datadump[key]) == dict:
1403 for key2 in datadump[key].keys():
1404 if key2.startswith('autogen_'):
1405 del datadump[key][key2]
1406
1407 output += format_wleiden_yaml(datadump)
1408 return output
1409
1410def generate_nanostation_config(datadump, iface, ns_type):
1411 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1412 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1413
1414 datadump.update(datadump['iface_%s' % iface])
1415
1416 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1417
1418def generate_yaml(datadump):
1419 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1420
1421
1422
1423def generate_config(node, config, datadump=None):
1424 """ Print configuration file 'config' of 'node' """
1425 output = ""
1426 try:
1427 # Load config file
1428 if datadump == None:
1429 datadump = get_yaml(node)
1430
1431 if config == 'wleiden.yaml':
1432 output += generate_wleiden_yaml(datadump)
1433 elif config == 'authorized_keys':
1434 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1435 output += f.read()
1436 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1437 # Fetch local keys if existing
1438 if os.path.exists(node_keys):
1439 output += open(node_keys, 'r').read()
1440 f.close()
1441 elif config == 'dnsmasq.conf':
1442 output += generate_dnsmasq_conf(datadump)
1443 elif config == 'dhcpd.conf':
1444 output += generate_dhcpd_conf(datadump)
1445 elif config == 'rc.conf.local':
1446 output += generate_rc_conf_local(datadump)
1447 elif config == 'resolv.conf':
1448 output += generate_resolv_conf(datadump)
1449 elif config == 'ntp.conf':
1450 output += generate_ntp_conf(datadump)
1451 elif config == 'motd':
1452 output += generate_motd(datadump)
1453 elif config == 'pf.hybrid.conf.local':
1454 output += generate_pf_hybrid_conf_local(datadump)
1455 elif config == 'unbound.wleiden.conf':
1456 output += generate_unbound_wleiden_conf(datadump)
1457 elif config.startswith('vr'):
1458 interface, ns_type = config.strip('.yaml').split('-')
1459 output += generate_nanostation_config(datadump, interface, ns_type)
1460 else:
1461 assert False, "Config not found!"
1462 except IOError, e:
1463 output += "[ERROR] Config file not found"
1464 return output
1465
1466
1467
1468def generate_static(output_dir, logging=True):
1469 items = {'output_dir' : output_dir}
1470 for node in get_hostlist():
1471 items['node'] = node
1472 items['wdir'] = "%(output_dir)s/%(node)s" % items
1473 if not os.path.isdir(items['wdir']):
1474 os.makedirs(items['wdir'])
1475 datadump = get_yaml(node)
1476 f = open("%(wdir)s/index.html" % items, "w")
1477 f.write(generate_node_overview(items['node'], datadump))
1478 f.close()
1479 for config in files:
1480 items['config'] = config
1481 if logging: logger.info("## Generating %(node)s %(config)s" % items)
1482 f = open("%(wdir)s/%(config)s" % items, "w")
1483 f.write(generate_config(node, config, datadump))
1484 f.close()
1485
1486
1487
1488def process_cgi_request(environ=os.environ):
1489 """ When calling from CGI """
1490 response_headers = []
1491 content_type = 'text/plain'
1492
1493 # Update repository if requested
1494 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1495 if form and form.has_key("action") and "update" in form["action"]:
1496 refresh_rate = 5
1497 output = "[INFO] Updating subverion, please wait...\n"
1498 old_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1499 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1500 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1501 new_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1502 if old_version != new_version:
1503 try:
1504 generate_static(CACHE_DIR, False)
1505 except:
1506 output += traceback.format_exc()
1507 refresh_rate = 120
1508 pass
1509 output += "[INFO] All done, redirecting in %s seconds" % refresh_rate
1510 response_headers += [
1511 ('Refresh', '%s; url=.' % refresh_rate),
1512 ]
1513 else:
1514 # Bootstap cache directory if none exists
1515 if not glob.glob(os.path.join(CACHE_DIR,'*','index.html')):
1516 generate_static(CACHE_DIR, False)
1517
1518 # URL must be of format <prefix>/config/<path>
1519 base_uri = environ['REQUEST_URI']
1520 uri = filter(None, base_uri.strip('/').split('/config')[1].strip('/').split('/'))
1521
1522 output = "Template Holder"
1523 if base_uri.endswith('/create/network.kml'):
1524 content_type='application/vnd.google-earth.kml+xml'
1525 output = make_network_kml.make_graph()
1526 elif base_uri.endswith('/api/get/nodeplanner.json'):
1527 content_type='application/json'
1528 output = make_network_kml.make_nodeplanner_json()
1529 elif not uri:
1530 if is_text_request(environ):
1531 output = '\n'.join(get_hostlist())
1532 else:
1533 content_type = 'text/html'
1534 output = generate_title(get_hostlist())
1535 elif len(uri) == 1:
1536 if is_text_request(environ):
1537 output = generate_node(uri[0])
1538 else:
1539 content_type = 'text/html'
1540 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1541 elif len(uri) == 2:
1542 output = generate_config(uri[0], uri[1])
1543 else:
1544 assert False, "Invalid option"
1545
1546 # Return response
1547 response_headers += [
1548 ('Content-type', content_type),
1549 ('Content-Length', str(len(output))),
1550 ]
1551 return(response_headers, str(output))
1552
1553
1554def make_dns(output_dir = 'dns', external = False):
1555 items = dict()
1556
1557 # hostname is key, IP is value
1558 wleiden_zone = defaultdict(list)
1559 wleiden_cname = dict()
1560
1561 pool = dict()
1562 for node in get_hostlist():
1563 datadump = get_yaml(node)
1564
1565 fqdn = datadump['nodename']
1566
1567 if datadump.has_key('rdr_host'):
1568 remote_target = datadump['rdr_host']
1569 elif datadump.has_key('remote_access') and datadump['remote_access']:
1570 remote_target = datadump['remote_access'].split(':')[0]
1571 else:
1572 remote_target = None
1573
1574 if remote_target:
1575 try:
1576 parseaddr(remote_target)
1577 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1578 except (IndexError, ValueError):
1579 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1580
1581
1582 wleiden_zone[fqdn].append((datadump['masterip'], True))
1583
1584 # Hacking to get proper DHCP IPs and hostnames
1585 for iface_key in get_interface_keys(datadump):
1586 iface_name = iface_key.replace('_','-')
1587 if 'ip' in datadump[iface_key]:
1588 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1589 try:
1590 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1591 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1592 dhcp_part = ".".join(ip.split('.')[0:3])
1593 if ip != datadump['masterip']:
1594 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1595 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1596 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1597 except (AttributeError, ValueError, KeyError):
1598 # First push it into a pool, to indentify the counter-part later on
1599 addr = parseaddr(ip)
1600 cidr = int(cidr)
1601 addr = addr & ~((1 << (32 - cidr)) - 1)
1602 if pool.has_key(addr):
1603 pool[addr] += [(iface_name, fqdn, ip)]
1604 else:
1605 pool[addr] = [(iface_name, fqdn, ip)]
1606 continue
1607
1608
1609
1610 # WL uses an /29 to configure an interface. IP's are ordered like this:
1611 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1612
1613 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1614
1615 # Automatic naming convention of interlinks namely 2 + remote.lower()
1616 for (key,value) in pool.iteritems():
1617 # Make sure they are sorted from low-ip to high-ip
1618 value = sorted(value, key=lambda x: parseaddr(x[2]))
1619
1620 if len(value) == 1:
1621 (iface_name, fqdn, ip) = value[0]
1622 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1623
1624 # Device DNS names
1625 if 'cnode' in fqdn.lower():
1626 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1627 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1628
1629 elif len(value) == 2:
1630 (a_iface_name, a_fqdn, a_ip) = value[0]
1631 (b_iface_name, b_fqdn, b_ip) = value[1]
1632 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1633 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1634
1635 # Device DNS names
1636 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1637 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1638 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1639 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1640 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1641 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1642 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1643
1644 else:
1645 pool_members = [k[1] for k in value]
1646 for item in value:
1647 (iface_name, fqdn, ip) = item
1648 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1649
1650 # Include static DNS entries
1651 # XXX: Should they override the autogenerated results?
1652 # XXX: Convert input to yaml more useable.
1653 # Format:
1654 ##; this is a comment
1655 ## roomburgh=Roomburgh1
1656 ## apkerk1.Vosko=172.17.176.8 ;this as well
1657 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1658
1659 # Hack to allow special entries, for development
1660 wleiden_raw = {}
1661
1662 for line in dns_list:
1663 reverse = False
1664 k, items = line.items()[0]
1665 if type(items) == dict:
1666 if items.has_key('reverse'):
1667 reverse = items['reverse']
1668 items = items['a']
1669 else:
1670 items = items['cname']
1671 items = [items] if type(items) != list else items
1672 for item in items:
1673 if item.startswith('IN '):
1674 wleiden_raw[k] = item
1675 elif valid_addr(item):
1676 wleiden_zone[k].append((item, reverse))
1677 else:
1678 wleiden_cname[k] = item
1679
1680 # Hack to get dynamic pool listing
1681 def chunks(l, n):
1682 return [l[i:i+n] for i in range(0, len(l), n)]
1683
1684 ntp_servers = [x[0] for x in get_nameservers()]
1685 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1686 for ntp_server in chunk:
1687 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1688
1689 details = dict()
1690 # 24 updates a day allowed
1691 details['serial'] = time.strftime('%Y%m%d%H')
1692
1693 if external:
1694 dns_masters = ['ns1.vanderzwet.net', 'ns1.transip.net', 'ns2.transip.eu']
1695 else:
1696 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1697
1698 details['master'] = dns_masters[0]
1699 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1700
1701 dns_header = '''
1702$TTL 3h
1703%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1704 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1705
1706%(ns_servers)s
1707 \n'''
1708
1709
1710 if not os.path.isdir(output_dir):
1711 os.makedirs(output_dir)
1712 details['zone'] = 'wleiden.net'
1713 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1714 f.write(dns_header % details)
1715
1716 for host,items in wleiden_zone.iteritems():
1717 for ip,reverse in items:
1718 if ip not in ['0.0.0.0']:
1719 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1720 for source,dest in wleiden_cname.iteritems():
1721 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1722 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1723 for source, dest in wleiden_raw.iteritems():
1724 f.write("%s.wleiden.net. %s\n" % (source, dest))
1725 f.close()
1726
1727 # Create whole bunch of specific sub arpa zones. To keep it compliant
1728 for s in range(16,32):
1729 details['zone'] = '%i.172.in-addr.arpa' % s
1730 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1731 f.write(dns_header % details)
1732
1733 #XXX: Not effient, fix to proper data structure and do checks at other
1734 # stages
1735 for host,items in wleiden_zone.iteritems():
1736 for ip,reverse in items:
1737 if not reverse:
1738 continue
1739 if valid_addr(ip):
1740 if valid_addr(ip):
1741 if int(ip.split('.')[1]) == s:
1742 rev_ip = '.'.join(reversed(ip.split('.')))
1743 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1744 f.close()
1745
1746
1747def usage():
1748 print """Usage: %(prog)s <argument>
1749Argument:
1750\tcleanup = Cleanup all YAML files to specified format
1751\tstandalone [port] = Run configurator webserver [8000]
1752\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1753\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1754\tfull-export = Generate yaml export script for heatmap.
1755\tstatic [outputdir] = Generate all config files and store on disk
1756\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1757\ttest <node> [<file>] = Receive output for certain node [all files].
1758\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1759\tlist <status> <items> = List systems which have certain status
1760\tcreate network.kml = Create Network KML file for use in Google Earth
1761
1762Arguments:
1763\t<node> = NodeName (example: HybridRick)
1764\t<file> = %(files)s
1765\t<status> = all|up|down|planned
1766\t<items> = systems|nodes|proxies
1767
1768NOTE FOR DEVELOPERS; you can test your changes like this:
1769 BEFORE any changes in this code:
1770 $ ./gformat.py static /tmp/pre
1771 AFTER the changes:
1772 $ ./gformat.py static /tmp/post
1773 VIEW differences and VERIFY all are OK:
1774 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1775""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1776 exit(0)
1777
1778
1779def is_text_request(environ=os.environ):
1780 """ Find out whether we are calling from the CLI or any text based CLI utility """
1781 if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1782 return True
1783
1784 if 'HTTP_USER_AGENT' in environ:
1785 return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1786 else:
1787 return False
1788
1789
1790def switchFormat(setting):
1791 if setting:
1792 return "YES"
1793 else:
1794 return "NO"
1795
1796def rlinput(prompt, prefill=''):
1797 import readline
1798 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1799 try:
1800 return raw_input(prompt)
1801 finally:
1802 readline.set_startup_hook()
1803
1804def fix_conflict(left, right, default='i'):
1805 while True:
1806 print "## %-30s | %-30s" % (left, right)
1807 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1808 if not c:
1809 c = default
1810
1811 if c in ['l','1']:
1812 return left
1813 elif c in ['r','2']:
1814 return right
1815 elif c in ['e', '3']:
1816 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1817 elif c in ['i', '4']:
1818 return None
1819 else:
1820 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1821
1822
1823
1824def print_cgi_response(response_headers, output):
1825 for header in response_headers:
1826 print "%s: %s" % header
1827 print
1828 print output
1829
1830
1831
1832def main():
1833 """Hard working sub"""
1834 # Allow easy hacking using the CLI
1835 if not os.environ.has_key('REQUEST_URI'):
1836 if len(sys.argv) < 2:
1837 usage()
1838
1839 if sys.argv[1] == "standalone":
1840 import SocketServer
1841 import CGIHTTPServer
1842 # Hop to the right working directory.
1843 os.chdir(os.path.dirname(__file__))
1844 try:
1845 PORT = int(sys.argv[2])
1846 except (IndexError,ValueError):
1847 PORT = 8000
1848
1849 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1850 """ Serve this CGI from the root of the webserver """
1851 def is_cgi(self):
1852 if "favicon" in self.path:
1853 return False
1854
1855 self.cgi_info = (os.path.basename(__file__), self.path)
1856 self.path = ''
1857 return True
1858 handler = MyCGIHTTPRequestHandler
1859 SocketServer.TCPServer.allow_reuse_address = True
1860 httpd = SocketServer.TCPServer(("", PORT), handler)
1861 httpd.server_name = 'localhost'
1862 httpd.server_port = PORT
1863
1864 logger.info("serving at port %s", PORT)
1865 try:
1866 httpd.serve_forever()
1867 except KeyboardInterrupt:
1868 httpd.shutdown()
1869 logger.info("All done goodbye")
1870 elif sys.argv[1] == "test":
1871 # Basic argument validation
1872 try:
1873 node = sys.argv[2]
1874 except IndexError:
1875 print "Invalid argument"
1876 exit(1)
1877 except IOError as e:
1878 print e
1879 exit(1)
1880
1881 datadump = get_yaml(node)
1882
1883
1884 # Get files to generate
1885 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1886
1887 # Actual config generation
1888 for config in gen_files:
1889 logger.info("## Generating %s %s", node, config)
1890 print generate_config(node, config, datadump)
1891 elif sys.argv[1] == "test-cgi":
1892 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1893 os.environ['SCRIPT_NAME'] = __file__
1894 response_headers, output = process_cgi_request()
1895 print_cgi_response(response_headers, output)
1896 elif sys.argv[1] == "static":
1897 generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
1898 elif sys.argv[1] == "wind-export":
1899 items = dict()
1900 for node in get_hostlist():
1901 datadump = get_yaml(node)
1902 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1903 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1904 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1905 VALUES (
1906 (SELECT id FROM users WHERE username = 'rvdzwet'),
1907 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1908 'Y');""" % datadump
1909 #for config in files:
1910 # items['config'] = config
1911 # print "## Generating %(node)s %(config)s" % items
1912 # f = open("%(wdir)s/%(config)s" % items, "w")
1913 # f.write(generate_config(node, config, datadump))
1914 # f.close()
1915 for node in get_hostlist():
1916 datadump = get_yaml(node)
1917 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1918 ifacedump = datadump[iface_key]
1919 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1920 ifacedump['nodename'] = datadump['nodename']
1921 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1922 ifacedump['channel'] = 0
1923 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1924 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1925 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1926 elif sys.argv[1] == "smokeping-export":
1927 for host in get_hostlist():
1928 datadump = get_yaml(host)
1929 if datadump.get('service_proxy_normal', False) or datadump.get('service_proxy_ileiden', False):
1930 print textwrap.dedent("""\
1931 ++ wleiden-gw-%(nodename)s
1932 menu = %(nodename)s.gw
1933 title = Wireless Leiden gateway %(nodename)s.gw.wleiden.net.
1934 host = %(nodename)s.gw.wleiden.net.
1935 """ % datadump)
1936 elif sys.argv[1] == "nagios-export":
1937 try:
1938 heavy_load = (sys.argv[2] == "--heavy-load")
1939 except IndexError:
1940 heavy_load = False
1941
1942 hostgroup_details = {
1943 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1944 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1945 'walphen' : 'Stichting Wireless Alphen',
1946 'westeinder' : 'Westeinder Plassen',
1947 }
1948
1949 # Convert IP to Host
1950 ip2host = {'root' : 'root'}
1951 for host in get_hostlist():
1952 datadump = get_yaml(host)
1953 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1954 for iface in get_interface_keys(datadump):
1955 if datadump[iface].has_key('autogen_gateway'):
1956 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1957
1958 # Find dependency tree based on output of lvrouted.mytree of nearest node
1959 parents = defaultdict(list)
1960 stack = ['root']
1961 prev_depth = 0
1962 if os.path.isfile('lvrouted.mytree'):
1963 for line in open('lvrouted.mytree').readlines():
1964 depth = line.count('\t')
1965 ip = line.strip().split()[0]
1966
1967 if prev_depth < depth:
1968 try:
1969 parents[ip2host[ip]].append(ip2host[stack[-1]])
1970 except KeyError as e:
1971 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1972 stack.append(ip)
1973 elif prev_depth > depth:
1974 stack = stack[:(depth - prev_depth)]
1975 elif prev_depth == depth:
1976 try:
1977 parents[ip2host[ip]].append(ip2host[stack[-1]])
1978 except KeyError as e:
1979 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1980
1981
1982 prev_depth = depth
1983 # Observe that some nodes has themself as parent or multiple parents
1984 # for now take only the first parent, other behaviour is yet to be explained
1985
1986
1987
1988 params = {
1989 'check_interval' : 5 if heavy_load else 120,
1990 'retry_interval' : 1 if heavy_load else 10,
1991 'max_check_attempts' : 10 if heavy_load else 6,
1992 'notification_interval': 120 if heavy_load else 240,
1993 }
1994
1995 print '''\
1996define host {
1997 name wleiden-node ; Default Node Template
1998 use generic-host ; Use the standard template as initial starting point
1999 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2000 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2001 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2002 notification_interval %(notification_interval)s
2003 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2004 check_command check-host-alive ; Default command to check FreeBSD hosts
2005 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2006}
2007
2008define service {
2009 name wleiden-service ; Default Service Template
2010 use generic-service ; Use the standard template as initial starting point
2011 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2012 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2013 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2014 notification_interval %(notification_interval)s
2015 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2016 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2017}
2018
2019# Please make sure to install:
2020# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2021#
2022# Recompile net-mgmt/nagios-plugins to support check_snmp
2023# make -C /usr/ports/net-mgmt/nagios-plugins
2024#
2025# Install net/bind-tools to allow v2/check_dns_wl to work:
2026# pkg install bind-tools
2027#
2028define command{
2029 command_name check_snmp_disk
2030 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2031}
2032
2033define command{
2034 command_name check_netsnmp_load
2035 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2036}
2037
2038define command{
2039 command_name check_netsnmp_proc
2040 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2041}
2042
2043define command{
2044 command_name check_by_ssh
2045 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2046}
2047
2048define command{
2049 command_name check_dns_wl
2050 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2051}
2052
2053define command{
2054 command_name check_snmp_uptime
2055 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2056}
2057
2058
2059# TDB: dhcp leases
2060# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2061
2062# TDB: internet status
2063# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2064
2065# TDB: Advanced local passive checks
2066# /usr/local/libexec/nagios/check_by_ssh
2067''' % params
2068
2069 print '''\
2070# Service Group, not displayed by default
2071define hostgroup {
2072 hostgroup_name srv_hybrid
2073 alias All Hybrid Nodes
2074 register 0
2075}
2076
2077define service {
2078 use wleiden-service
2079 hostgroup_name srv_hybrid
2080 service_description SSH
2081 check_command check_ssh
2082}
2083
2084define service {
2085 use wleiden-service,service-pnp
2086 hostgroup_name srv_hybrid
2087 service_description HTTP
2088 check_command check_http
2089}
2090
2091define service {
2092 use wleiden-service
2093 hostgroup_name srv_hybrid
2094 service_description DNS
2095 check_command check_dns_wl!"www.wirelessleiden.nl"
2096}
2097'''
2098
2099 if heavy_load:
2100 print '''\
2101define service {
2102 use wleiden-service
2103 hostgroup_name srv_hybrid
2104 service_description UPTIME
2105 check_command check_snmp_uptime
2106}
2107
2108#define service {
2109# use wleiden-service
2110# hostgroup_name srv_hybrid
2111# service_description NTP
2112# check_command check_ntp_peer
2113#}
2114
2115define service {
2116 use wleiden-service
2117 hostgroup_name srv_hybrid
2118 service_description LOAD
2119 check_command check_netsnmp_load
2120}
2121
2122define service {
2123 use wleiden-service
2124 hostgroup_name srv_hybrid
2125 service_description PROC
2126 check_command check_netsnmp_proc
2127}
2128
2129define service {
2130 use wleiden-service
2131 hostgroup_name srv_hybrid
2132 service_description DISK
2133 check_command check_snmp_disk
2134}
2135'''
2136 for node in get_hostlist():
2137 datadump = get_yaml(node)
2138 if not datadump['status'] == 'up':
2139 continue
2140 if not hostgroup_details.has_key(datadump['monitoring_group']):
2141 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2142 print '''\
2143define host {
2144 use wleiden-node,host-pnp
2145 contact_groups admins
2146 host_name %(autogen_fqdn)s
2147 address %(masterip)s
2148 hostgroups srv_hybrid,%(monitoring_group)s\
2149''' % datadump
2150 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2151 print '''\
2152 parents %(parents)s\
2153''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2154 print '''\
2155}
2156'''
2157
2158
2159 for name,alias in hostgroup_details.iteritems():
2160 print '''\
2161define hostgroup {
2162 hostgroup_name %s
2163 alias %s
2164} ''' % (name, alias)
2165
2166
2167 elif sys.argv[1] == "full-export":
2168 hosts = {}
2169 for node in get_hostlist():
2170 datadump = get_yaml(node)
2171 hosts[datadump['nodename']] = datadump
2172 print yaml.dump(hosts)
2173
2174 elif sys.argv[1] == "dns":
2175 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2176 elif sys.argv[1] == "cleanup":
2177 # First generate all datadumps
2178 datadumps = dict()
2179 ssid_to_node = dict()
2180 for host in get_hostlist():
2181 logger.info("# Processing: %s", host)
2182 # Set some boring default values
2183 datadump = { 'board' : 'UNKNOWN' }
2184 datadump.update(get_yaml(host))
2185 datadumps[datadump['nodename']] = datadump
2186
2187 (poel, errors) = make_relations()
2188 print "\n".join(["# WARNING: %s" % x for x in errors])
2189
2190 for host,datadump in datadumps.iteritems():
2191 try:
2192 # Convert all yes and no to boolean values
2193 def fix_boolean(dump):
2194 for key in dump.keys():
2195 if type(dump[key]) == dict:
2196 dump[key] = fix_boolean(dump[key])
2197 elif str(dump[key]).lower() in ["yes", "true"]:
2198 dump[key] = True
2199 elif str(dump[key]).lower() in ["no", "false"]:
2200 # Compass richting no (Noord Oost) is valid input
2201 if key != "compass": dump[key] = False
2202 return dump
2203 datadump = fix_boolean(datadump)
2204
2205 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2206 if not 'latitude' in datadump and not 'longitude' in datadump:
2207 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2208 elif 'latitude' in datadump and 'longitude' in datadump:
2209 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2210 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2211 # TODO: Compare outcome of both coordinate systems and validate against each-other
2212
2213 if datadump['nodename'].startswith('Proxy'):
2214 datadump['nodename'] = datadump['nodename'].lower()
2215
2216 for iface_key in get_interface_keys(datadump):
2217 try:
2218 # All our normal wireless cards are normal APs now
2219 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2220 datadump[iface_key]['mode'] = 'ap'
2221 # Wireless Leiden SSID have an consistent lowercase/uppercase
2222 if datadump[iface_key].has_key('ssid'):
2223 ssid = datadump[iface_key]['ssid']
2224 prefix = 'ap-WirelessLeiden-'
2225 if ssid.lower().startswith(prefix.lower()):
2226 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2227 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2228 datadump[iface_key]['mode'] = 'autogen-FIXME'
2229 if not datadump[iface_key].has_key('comment'):
2230 datadump[iface_key]['comment'] = 'autogen-FIXME'
2231
2232 if datadump[iface_key].has_key('ns_mac'):
2233 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2234
2235 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2236 datadump[iface_key] = datadump[iface_key]['desc']
2237
2238 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2239 # due to better collision avoidance.
2240 if datadump[iface_key]['type'] == '11b':
2241 datadump[iface_key]['type'] = '11g'
2242
2243 # Setting 802.11g channels to de-facto standards, to avoid
2244 # un-detected sharing with other overlapping channels
2245 #
2246 # Technically we could also use channel 13 in NL, but this is not
2247 # recommended as foreign devices might not be able to select this
2248 # channel. Secondly using 1,5,9,13 instead is going to clash with
2249 # the de-facto usage of 1,6,11.
2250 #
2251 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2252 channels_at_2400Mhz = (1,6,11)
2253 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2254 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2255 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2256 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2257
2258 # Mandatory interface keys
2259 if not datadump[iface_key].has_key('status'):
2260 datadump[iface_key]['status'] = 'planned'
2261
2262 x = datadump[iface_key]['comment']
2263 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2264
2265 # Fixing bridge_type if none is found
2266 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2267 if not 'bridge_type' in datadump[iface_key]:
2268 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2269
2270 # Making sure description works
2271 if datadump[iface_key].has_key('desc'):
2272 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2273 del datadump[iface_key]['desc']
2274 else:
2275 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2276 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2277 if response:
2278 datadump[iface_key]['comment'] = response
2279 del datadump[iface_key]['desc']
2280
2281 # Check DHCP configuration
2282 dhcp_type(datadump[iface_key])
2283
2284 # Set the compass value based on the angle between the poels
2285 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2286 my_pool = poel[network(datadump[iface_key]['ip'])]
2287 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2288 if remote_hosts:
2289 compass_target = remote_hosts[0]
2290 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2291 # TODO: Compass wanted and actual direction might differ
2292
2293 # Monitoring Group default
2294 if not 'monitoring_group' in datadump:
2295 datadump['monitoring_group'] = 'wleiden'
2296
2297 except Exception as exc:
2298 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
2299 raise
2300 store_yaml(datadump)
2301 except Exception as exc:
2302 exc.args = ("# Error while processing %s" % host,) + exc.args
2303 raise
2304 elif sys.argv[1] == "list":
2305 use_fqdn = False
2306 if len(sys.argv) < 4:
2307 usage()
2308 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2309 usage()
2310 if not sys.argv[3] in ["nodes","proxies","systems"]:
2311 usage()
2312
2313 if len(sys.argv) > 4:
2314 if sys.argv[4] == "fqdn":
2315 use_fqdn = True
2316 else:
2317 usage()
2318
2319 for system in get_hostlist():
2320 datadump = get_yaml(system)
2321 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2322 continue
2323
2324 output = datadump['autogen_fqdn'] if use_fqdn else system
2325 if sys.argv[2] == "all":
2326 print output
2327 elif datadump['status'] == sys.argv[2]:
2328 print output
2329 elif sys.argv[1] == "create":
2330 if sys.argv[2] == "network.kml":
2331 print make_network_kml.make_graph()
2332 elif sys.argv[2] == "host-ips.txt":
2333 for system in get_hostlist():
2334 datadump = get_yaml(system)
2335 ips = [datadump['masterip']]
2336 for ifkey in get_interface_keys(datadump):
2337 ips.append(datadump[ifkey]['ip'].split('/')[0])
2338 print system, ' '.join(ips)
2339 elif sys.argv[2] == "host-pos.txt":
2340 for system in get_hostlist():
2341 datadump = get_yaml(system)
2342 print system, datadump['rdnap_x'], datadump['rdnap_y']
2343 elif sys.argv[2] == 'ssh_config':
2344 print '''
2345Host *.wleiden.net
2346 User root
2347
2348Host 172.16.*.*
2349 User root
2350'''
2351 for system in get_hostlist():
2352 datadump = get_yaml(system)
2353 print '''\
2354Host %s
2355 User root
2356
2357Host %s
2358 User root
2359
2360Host %s
2361 User root
2362
2363Host %s
2364 User root
2365''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2366 else:
2367 usage()
2368 else:
2369 usage()
2370 else:
2371 # Do not enable debugging for config requests as it highly clutters the output
2372 if not is_text_request():
2373 cgitb.enable()
2374
2375 try:
2376 response_headers, output = process_cgi_request()
2377 except:
2378 print ''
2379 print ''
2380 raise
2381
2382 print_cgi_response(response_headers, output)
2383
2384if __name__ == "__main__":
2385 main()
Note: See TracBrowser for help on using the repository browser.