source: genesis/tools/gformat.py@ 13984

Last change on this file since 13984 was 13984, checked in by www, 7 years ago

Add version number of repository, to detect whether an update is usefull at all.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 78.9 KB
Line 
1#!/usr/local/bin/python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
35SVNVERSION = filter(os.path.isfile, ('/usr/local/bin/svnversion', '/usr/bin/svnversion'))[0]
36
37import argparse
38import cgi
39import cgitb
40import copy
41import glob
42import make_network_kml
43import math
44import pyproj
45import random
46import re
47import socket
48import string
49import subprocess
50import textwrap
51import time
52import urlparse
53
54from pprint import pprint
55from collections import defaultdict, OrderedDict
56from sys import stderr
57try:
58 import yaml
59except ImportError, e:
60 print e
61 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
62 exit(1)
63
64try:
65 from yaml import CLoader as Loader
66 from yaml import CDumper as Dumper
67except ImportError:
68 from yaml import Loader, Dumper
69
70from jinja2 import Environment, Template
71def yesorno(value):
72 return "YES" if bool(value) else "NO"
73env = Environment()
74env.filters['yesorno'] = yesorno
75def render_template(datadump, template):
76 result = env.from_string(template).render(datadump)
77 # Make it look pretty to the naked eye, as jinja templates are not so
78 # friendly when it comes to whitespace formatting
79 ## Remove extra whitespace at end of line lstrip() style.
80 result = re.sub(r'\n[\ ]+','\n', result)
81 ## Include only a single newline between an definition and a comment
82 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
83 ## Remove extra newlines after single comment
84 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
85 return result
86
87import logging
88logging.basicConfig(format='# %(levelname)s: %(message)s' )
89logger = logging.getLogger()
90logger.setLevel(logging.DEBUG)
91
92
93if os.environ.has_key('CONFIGROOT'):
94 NODE_DIR = os.environ['CONFIGROOT']
95else:
96 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
97__version__ = '$Id: gformat.py 13984 2017-09-22 18:05:41Z www $'
98
99CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
100
101files = [
102 'authorized_keys',
103 'dnsmasq.conf',
104 'dhcpd.conf',
105 'rc.conf.local',
106 'resolv.conf',
107 'motd',
108 'ntp.conf',
109 'pf.hybrid.conf.local',
110 'unbound.wleiden.conf',
111 'wleiden.yaml',
112 ]
113
114# Global variables uses
115OK = 10
116DOWN = 20
117UNKNOWN = 90
118
119
120ileiden_proxies = OrderedDict()
121normal_proxies = []
122datadump_cache = {}
123interface_list_cache = {}
124rc_conf_local_cache = {}
125nameservers_cache = []
126relations_cache = None
127
128NO_DHCP = 0
129DHCP_CLIENT = 10
130DHCP_SERVER = 20
131def dhcp_type(item):
132 if not item.has_key('dhcp'):
133 return NO_DHCP
134 elif not item['dhcp']:
135 return NO_DHCP
136 elif item['dhcp'].lower() == 'client':
137 return DHCP_CLIENT
138 else:
139 # Validation Checks
140 begin,end = map(int,item['dhcp'].split('-'))
141 if begin >= end:
142 raise ValueError("DHCP Start >= DHCP End")
143 return DHCP_SERVER
144
145def etrs2rd(lat, lon):
146 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
147 p2 = pyproj.Proj(init='EPSG:28992')
148 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
149 return (RDx, RDy)
150
151def rd2etrs(RDx, RDy):
152 p1 = pyproj.Proj(init='EPSG:28992')
153 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
154 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
155 return (lat, lon)
156
157def get_yaml(item,add_version_info=True):
158 try:
159 """ Get configuration yaml for 'item'"""
160 if datadump_cache.has_key(item):
161 return datadump_cache[item].copy()
162
163 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
164 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
165 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
166
167 # Default values
168 datadump = {
169 'autogen_revision' : 'NOTFOUND',
170 'autogen_gfile' : gfile,
171 'service_proxy_ileiden' : False,
172 }
173 f = open(gfile, 'r')
174 datadump.update(yaml.load(f,Loader=Loader))
175 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
176 if datadump['nodetype'] == 'Hybrid':
177 # Some values are defined implicitly
178 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
179 datadump['service_incoming_rdr'] = True
180 # Use some boring defaults
181 defaults = {
182 'service_proxy_normal' : False,
183 'service_accesspoint' : True,
184 'service_incoming_rdr' : False,
185 'service_concentrator' : False,
186 'monitoring_group' : 'wleiden',
187 }
188 for (key,value) in defaults.iteritems():
189 if not datadump.has_key(key):
190 datadump[key] = value
191 f.close()
192
193 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
194 if add_version_info:
195 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
196 lines = p.communicate()[0].split('\n')
197 if p.returncode == 0:
198 for line in lines:
199 if line:
200 (key, value) = line.split(': ')
201 datadump["autogen_" + key.lower().replace(' ','_')] = value
202
203 # Preformat certain needed variables for formatting and push those into special object
204 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
205
206 wlan_count=0
207 try:
208 for key in get_interface_keys(datadump, True):
209 datadump[key]['autogen_ifbase'] = key.split('_')[1]
210 datadump[key]['autogen_vlan'] = False
211
212 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
213 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
214 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
215
216 if datadump[key].has_key('parent'):
217 if datadump[key].has_key('ip'):
218 raise ValueError("Interface bridge member cannot have IP assigned")
219 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
220 raise ValueError("Interface bridge member cannot have DHCP set")
221
222 if datadump[key].has_key('ip'):
223 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
224
225 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
226 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
227 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
228 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
229 wlan_count += 1
230 else:
231 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
232 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
233 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
234 datadump[key]['autogen_vlan'] = key.split('_')[2]
235 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
236 else:
237 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
238
239 except Exception:
240 print "# Error while processing interface %s" % key
241 raise
242
243 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
244 if dhcp_type(datadump[key]) == DHCP_SERVER]
245
246 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
247 datadump['autogen_item'] = item
248
249 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
250 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
251 datadump_cache[item] = datadump.copy()
252 except Exception:
253 print "# Error while processing %s" % item
254 raise
255 return datadump
256
257
258def store_yaml(datadump, header=False):
259 """ Store configuration yaml for 'item'"""
260 item = datadump['autogen_item']
261 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
262
263 output = generate_wleiden_yaml(datadump, header)
264
265 f = open(gfile, 'w')
266 f.write(output)
267 f.close()
268
269
270def network(ip):
271 addr, mask = ip.split('/')
272 # Not parsing of these folks please
273 addr = parseaddr(addr)
274 mask = int(mask)
275 network = addr & ~((1 << (32 - mask)) - 1)
276 return network
277
278
279
280def make_relations():
281 """ Process _ALL_ yaml files to get connection relations """
282 global relations_cache
283
284 if relations_cache:
285 return relations_cache
286
287 errors = []
288 poel = defaultdict(list)
289
290 for host in get_hostlist():
291 datadump = get_yaml(host)
292 try:
293 for iface_key in get_interface_keys(datadump):
294 # Bridge members has no IP assigned
295 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
296 continue
297 net_addr = network(datadump[iface_key]['ip'])
298 poel[net_addr] += [(host,datadump[iface_key].copy())]
299 except (KeyError, ValueError) as e:
300 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
301 continue
302
303 relations_cache = (poel, errors)
304 return relations_cache
305
306
307
308def valid_addr(addr):
309 """ Show which address is valid in which are not """
310 return str(addr).startswith('172.')
311
312def get_hostlist():
313 """ Combined hosts and proxy list"""
314 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
315
316def angle_between_points(lat1,lat2,long1,long2):
317 """
318 Return Angle in radians between two GPS coordinates
319 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
320 """
321 dy = lat2 - lat1
322 dx = math.cos(lat1)*(long2 - long1)
323 angle = math.atan2(dy,dx)
324 return angle
325
326
327
328def angle_to_cd(angle):
329 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
330
331 # For easy conversion get positive degree
332 degrees = math.degrees(angle)
333 abs_degrees = 360 + degrees if degrees < 0 else degrees
334
335 # Numbers can be confusing calculate from the 4 main directions
336 p = 22.5
337 if abs_degrees < p:
338 cd = "n"
339 elif abs_degrees < (90 - p):
340 cd = "no"
341 elif abs_degrees < (90 + p):
342 cd = "o"
343 elif abs_degrees < (180 - p):
344 cd = "zo"
345 elif abs_degrees < (180 + p):
346 cd = "z"
347 elif abs_degrees < (270 - p):
348 cd = "zw"
349 elif abs_degrees < (270 + p):
350 cd = "w"
351 elif abs_degrees < (360 - p):
352 cd = "nw"
353 else:
354 cd = "n"
355 return cd
356
357
358
359def cd_between_hosts(hostA, hostB, datadumps):
360 # Using RDNAP coordinates
361 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
362 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
363 return angle_to_cd(math.atan2(dx,dy))
364
365 # GPS coordinates seems to fail somehow
366 #latA = float(datadumps[hostA]['latitude'])
367 #latB = float(datadumps[hostB]['latitude'])
368 #lonA = float(datadumps[hostA]['longitude'])
369 #lonB = float(datadumps[hostB]['longitude'])
370 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
371
372
373def generate_title(nodelist):
374 """ Main overview page """
375 items = { \
376 'root' : ".",
377 'version' : subprocess.Popen([SVNVERSION, "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
378 }
379 def fl(spaces, line):
380 return (' ' * spaces) + line + '\n'
381
382 output = """
383<html>
384 <head>
385 <title>Wireless leiden Configurator - GFormat</title>
386 <style type="text/css">
387 th {background-color: #999999}
388 tr:nth-child(odd) {background-color: #cccccc}
389 tr:nth-child(even) {background-color: #ffffff}
390 th, td {padding: 0.1em 1em}
391 </style>
392 </head>
393 <body>
394 <center>
395 <form type="GET" action="%(root)s">
396 <input type="hidden" name="action" value="update">
397 <input type="submit" value="Update Configuration Database (SVN)">
398 </form>
399 <table>
400 <caption><h3>Wireless Leiden Configurator - Revision %(version)s</h3></caption>
401 """ % items
402
403 for node in nodelist:
404 items['node'] = node
405 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
406 for config in files:
407 items['config'] = config
408 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
409 output += fl(5, "</tr>")
410 output += """
411 </table>
412 <hr />
413 <em>%s</em>
414 </center>
415 </body>
416</html>
417 """ % __version__
418
419 return output
420
421
422
423def generate_node(node):
424 """ Print overview of all files available for node """
425 return "\n".join(files)
426
427def generate_node_overview(host, datadump=False):
428 """ Print overview of all files available for node """
429 if not datadump:
430 datadump = get_yaml(host)
431 params = { 'host' : host }
432 output = "<em><a href='..'>Back to overview</a></em><hr />"
433 output += "<h2>Available files:</h2><ul>"
434 for cf in files:
435 params['cf'] = cf
436 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
437 output += "</ul>"
438
439 # Generate and connection listing
440 output += "<h2>Connected To:</h2><ul>"
441 (poel, errors) = make_relations()
442 for network, hosts in poel.iteritems():
443 if host in [x[0] for x in hosts]:
444 if len(hosts) == 1:
445 # Single not connected interface
446 continue
447 for remote,ifacedump in hosts:
448 if remote == host:
449 # This side of the interface
450 continue
451 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
452 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
453 output += "</ul>"
454 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
455
456 output += "<hr /><em><a href='..'>Back to overview</a></em>"
457 return output
458
459
460def generate_header(datadump, ctag="#"):
461 return """\
462%(ctag)s
463%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
464%(ctag)s
465""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
466
467
468
469def parseaddr(s):
470 """ Process IPv4 CIDR notation addr to a (binary) number """
471 f = s.split('.')
472 return (long(f[0]) << 24L) + \
473 (long(f[1]) << 16L) + \
474 (long(f[2]) << 8L) + \
475 long(f[3])
476
477
478
479def showaddr(a):
480 """ Display IPv4 addr in (dotted) CIDR notation """
481 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
482
483
484def is_member(ip, mask, canidate):
485 """ Return True if canidate is part of ip/mask block"""
486 ip_addr = parseaddr(ip)
487 ip_canidate = parseaddr(canidate)
488 mask = int(mask)
489 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
490 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
491 return ip_addr == ip_canidate
492
493
494
495def cidr2netmask(netmask):
496 """ Given a 'netmask' return corresponding CIDR """
497 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
498
499def get_network(addr, mask):
500 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
501
502
503def generate_dhcpd_conf(datadump):
504 """ Generate config file '/usr/local/etc/dhcpd.conf """
505 # Redundency support, in cause local DNS server is not running/responding.
506 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
507 output = generate_header(datadump)
508 output += Template("""\
509# option definitions common to all supported networks...
510option domain-name "dhcp.{{ autogen_fqdn }}";
511
512default-lease-time 600;
513max-lease-time 7200;
514
515# Use this to enble / disable dynamic dns updates globally.
516#ddns-update-style none;
517
518# If this DHCP server is the official DHCP server for the local
519# network, the authoritative directive should be uncommented.
520authoritative;
521
522# Use this to send dhcp log messages to a different log file (you also
523# have to hack syslog.conf to complete the redirection).
524log-facility local7;
525
526# UniFi Discovery Support
527option space ubnt;
528option ubnt.unifi-address code 1 = ip-address;
529#
530class "ubnt" {
531 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
532 option vendor-class-identifier "ubnt";
533 vendor-option-space ubnt;
534}
535
536#
537# Interface definitions
538#
539\n\n""").render(datadump)
540
541
542 # TODO: Use textwrap.fill instead
543 def indent(text, count):
544 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
545
546 # Process DHCP blocks
547 dhcp_out = defaultdict(list)
548 for iface_key in get_interface_keys(datadump):
549 ifname = datadump[iface_key]['autogen_ifbase']
550 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
551 if not datadump[iface_key].has_key('comment'):
552 datadump[iface_key]['comment'] = None
553
554 if not datadump[iface_key].has_key('ip'):
555 continue
556
557 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
558
559 (addr, mask) = datadump[iface_key]['ip'].split('/')
560 datadump[iface_key]['autogen_addr'] = addr
561 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
562 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
563
564 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
565 dhcp_out[groupif].append(textwrap.dedent("""\
566 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
567 ### not autoritive
568 }
569 """ % datadump[iface_key]))
570 continue
571
572 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
573 dhcp_part = ".".join(addr.split('.')[0:3])
574 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
575 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
576 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
577
578 # Assume the first 10 IPs could be used for static entries
579 if 'no_portal' in datadump:
580 fixed = 5
581 for mac in datadump['no_portal']:
582 dhcp_out[groupif].append(textwrap.dedent("""\
583 host fixed-%(ifname)s-%(fixed)s {
584 hardware ethernet %(mac)s;
585 fixed-address %(prefix)s.%(fixed)s;
586 }
587 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
588 fixed += 1
589
590 if 'dhcp_fixed' in datadump[iface_key]:
591 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
592 dhcp_out[groupif].append(textwrap.dedent("""\
593 host fixed-%(host)s {
594 hardware ethernet %(mac)s;
595 fixed-address %(addr)s;
596 }
597 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
598
599
600 dhcp_out[groupif].append(textwrap.dedent("""\
601 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
602 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
603 option routers %(autogen_addr)s;
604 option domain-name-servers %(autogen_dns_servers)s;
605 option ubnt.unifi-address 172.17.107.10;
606 }
607 """ % datadump[iface_key]))
608
609 # Output the blocks in groups
610 for ifname,value in sorted(dhcp_out.iteritems()):
611 if len(value) > 2:
612 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
613 else:
614 output += ''.join(value) + "\n\n"
615 return output
616
617
618
619def generate_dnsmasq_conf(datadump):
620 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
621 output = generate_header(datadump)
622 output += Template("""\
623# DHCP server options
624dhcp-authoritative
625dhcp-fqdn
626domain=dhcp.{{ autogen_fqdn }}
627domain-needed
628expand-hosts
629log-async=100
630
631# Low memory footprint
632cache-size=10000
633
634\n""").render(datadump)
635
636 for iface_key in get_interface_keys(datadump):
637 if not datadump[iface_key].has_key('comment'):
638 datadump[iface_key]['comment'] = None
639 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
640
641 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
642 output += "# not autoritive\n\n"
643 continue
644
645 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
646 (ip, cidr) = datadump[iface_key]['ip'].split('/')
647 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
648
649 dhcp_part = ".".join(ip.split('.')[0:3])
650 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
651 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
652 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
653
654 return output
655
656
657class AutoVivification(dict):
658 """Implementation of perl's autovivification feature."""
659 def __getitem__(self, item):
660 try:
661 return dict.__getitem__(self, item)
662 except KeyError:
663 value = self[item] = type(self)()
664 return value
665
666def make_interface_list(datadump):
667 if interface_list_cache.has_key(datadump['autogen_item']):
668 return (interface_list_cache[datadump['autogen_item']])
669 # lo0 configuration:
670 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
671 # - masterip is special as it needs to be assigned to at
672 # least one interface, so if not used assign to lo0
673 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
674 vlan_list = defaultdict(list)
675 bridge_list = defaultdict(list)
676 flags_if = AutoVivification()
677 dhclient_if = {'lo0' : False}
678
679 # XXX: Find some way of send this output nicely
680 output = ''
681
682 masterip_used = False
683 for iface_key in get_interface_keys(datadump):
684 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
685 masterip_used = True
686 break
687 if not masterip_used:
688 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
689
690 if 'serviceid' in datadump:
691 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
692
693 for iface_key in get_interface_keys(datadump):
694 ifacedump = datadump[iface_key]
695 if ifacedump['autogen_bridge_alias']:
696 ifname = ifacedump['autogen_ifbase']
697 else:
698 ifname = ifacedump['autogen_ifname']
699
700 # If defined as vlan interface
701 if ifacedump['autogen_vlan']:
702 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
703
704 # If defined as bridge interface
705 if ifacedump['autogen_bridge_member']:
706 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
707
708
709 # Flag dhclient is possible
710 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
711 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
712
713 # Ethernet address
714 if ifacedump.has_key('ether'):
715 flags_if[ifname]['ether'] = ifacedump['ether']
716
717 # Add interface IP to list
718 if ifacedump.has_key('ip'):
719 item = (ifacedump['ip'], ifacedump['comment'])
720 if addrs_list.has_key(ifname):
721 addrs_list[ifname].append(item)
722 else:
723 addrs_list[ifname] = [item]
724
725 # Alias only needs IP assignment for now, this might change if we
726 # are going to use virtual accesspoints
727 if "alias" in iface_key:
728 continue
729
730 # XXX: Might want to deduct type directly from interface name
731 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
732 # Default to station (client) mode
733 ifacedump['autogen_wlanmode'] = "sta"
734 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
735 ifacedump['autogen_wlanmode'] = "ap"
736
737 if not ifacedump.has_key('channel'):
738 if ifacedump['type'] == '11a':
739 ifacedump['channel'] = 36
740 else:
741 ifacedump['channel'] = 1
742
743 # Allow special hacks at the back like wds and stuff
744 if not ifacedump.has_key('extra'):
745 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
746 else:
747 ifacedump['autogen_extra'] = ifacedump['extra']
748
749 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
750
751 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
752 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
753 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
754 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
755 output += "\n"
756
757 elif ifacedump['type'] in ['ethernet', 'eth']:
758 # No special config needed besides IP
759 pass
760 elif ifacedump['type'] in ['vlan']:
761 # VLAN member has no special configuration
762 pass
763 else:
764 assert False, "Unknown type " + ifacedump['type']
765
766 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
767 interface_list_cache[datadump['autogen_item']] = store
768 return(store)
769
770
771
772def create_proxies_list():
773 if not ileiden_proxies or not normal_proxies:
774 # Placeholder for to-be-installed proxies, this will avoid updating the all
775 # nodes to include this new machine, yet due to an unbound issue, this list
776 # has to be kept small.
777
778 for i in range(1,20):
779 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
780
781 for host in get_hostlist():
782 hostdump = get_yaml(host)
783 if hostdump['status'] == 'up':
784 if hostdump['service_proxy_ileiden']:
785 ileiden_proxies[hostdump['serviceid']] = hostdump
786 if hostdump['service_proxy_normal']:
787 normal_proxies.append(hostdump)
788
789
790
791def generate_rc_conf_local(datadump):
792 """ Generate configuration file '/etc/rc.conf.local' """
793 item = datadump['autogen_item']
794 if rc_conf_local_cache.has_key(item):
795 return rc_conf_local_cache[item]
796
797 if not datadump.has_key('ileiden'):
798 datadump['autogen_ileiden_enable'] = False
799 else:
800 datadump['autogen_ileiden_enable'] = datadump['ileiden']
801
802 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
803
804 create_proxies_list()
805 datadump['autogen_ileiden_proxies'] = ileiden_proxies
806 datadump['autogen_normal_proxies'] = normal_proxies
807 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
808 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
809 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
810 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
811
812 output = generate_header(datadump, "#");
813 output += render_template(datadump, """\
814hostname='{{ autogen_fqdn }}'
815location='{{ location }}'
816nodetype="{{ nodetype }}"
817
818#
819# Configured listings
820#
821captive_portal_whitelist=""
822{% if nodetype == "Proxy" %}
823#
824# Proxy Configuration
825#
826{% if gateway and service_proxy_ileiden -%}
827defaultrouter="{{ gateway }}"
828{% else -%}
829#defaultrouter="NOTSET"
830{% endif -%}
831internalif="{{ internalif }}"
832ileiden_enable="{{ autogen_ileiden_enable }}"
833gateway_enable="{{ autogen_ileiden_enable }}"
834pf_enable="yes"
835pf_rules="/etc/pf.conf"
836{% if autogen_ileiden_enable -%}
837pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
838lvrouted_enable="{{ autogen_ileiden_enable }}"
839lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
840{% else -%}
841pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
842{% endif -%}
843{% if internalroute -%}
844static_routes="wleiden"
845route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
846{% endif -%}
847
848{% elif nodetype == "Hybrid" %}
849 #
850 # Hybrid Configuration
851 #
852 list_ileiden_proxies="
853 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
854 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
855 {% endfor -%}
856 "
857 list_normal_proxies="
858 {% for item in autogen_normal_proxies -%}
859 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
860 {% endfor -%}
861 "
862
863 {% if autogen_dhcp_interfaces -%}
864 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
865 {% else %}
866 captive_portal_interfaces="dummy"
867 {% endif %}
868 externalif="{{ externalif|default('vr0', true) }}"
869 masterip="{{ masterip }}"
870
871 {% if gateway and service_proxy_ileiden %}
872 defaultrouter="{{ gateway }}"
873 {% else %}
874 #defaultrouter="NOTSET"
875 {% endif %}
876
877 #
878 # Defined services
879 #
880 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
881 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
882 service_accesspoint="{{ service_accesspoint|yesorno }}"
883 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
884 service_concentrator="{{ service_concentrator|yesorno }}"
885
886 {% if service_proxy_ileiden %}
887 pf_rules="/etc/pf.hybrid.conf"
888 {% if service_concentrator %}
889 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
890 {% else %}
891 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
892 {% endif %}
893 pf_flags="$pf_flags -D publicnat=80,443"
894 lvrouted_flags="$lvrouted_flags -g"
895 {% elif service_proxy_normal or service_incoming_rdr %}
896 pf_rules="/etc/pf.hybrid.conf"
897 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
898 pf_flags="$pf_flags -D publicnat=0"
899 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
900 named_setfib="1"
901 tinyproxy_setfib="1"
902 dnsmasq_setfib="1"
903 sshd_setfib="1"
904 {% else %}
905 named_auto_forward_only="YES"
906 pf_rules="/etc/pf.node.conf"
907 pf_flags=""
908 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
909 {% endif %}
910 {% if service_concentrator %}
911 # Do mind installing certificates is NOT done automatically for security reasons
912 openvpn_enable="YES"
913 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
914 {% endif %}
915
916 {% if service_proxy_normal %}
917 tinyproxy_enable="yes"
918 {% else %}
919 pen_wrapper_enable="yes"
920 {% endif %}
921
922 {% if service_accesspoint %}
923 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
924 {% endif %}
925
926 {% if board == "ALIX2" or board == "net4801" %}
927 #
928 # ''Fat'' configuration, board has 256MB RAM
929 #
930 dnsmasq_enable="NO"
931 named_enable="YES"
932 unbound_enable="YES"
933 {% if autogen_dhcp_interfaces -%}
934 dhcpd_enable="YES"
935 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
936 {% endif -%}
937 {% elif board == "apu1d" %}
938 #
939 # ''Fat'' configuration, board has 1024MB RAM
940 #
941 dnsmasq_enable="NO"
942 unbound_enable="YES"
943 {% if autogen_dhcp_interfaces -%}
944 dhcpd_enable="YES"
945 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
946 {% endif -%}
947 {% endif -%}
948{% endif %}
949
950#
951# Script variables
952#
953attached_devices="{{ autogen_attached_devices|join(' ') }}"
954neighbours="{{ autogen_neighbours|join(' ') }}"
955
956
957#
958# Interface definitions
959#\n
960""")
961
962 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
963 for iface, vlans in sorted(vlan_list.items()):
964 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
965
966 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
967 for iface in sorted(vlan_list.keys()):
968 if not iface in addrs_list.keys():
969 output += "ifconfig_%s='up'\n" % iface
970
971 output += "\n"
972
973 # Bridge configuration:
974 if bridge_list.keys():
975 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
976
977 for iface in bridge_list.keys():
978 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
979
980 # Bridge member interfaces not containing a configuration should be marked active explcitly.
981 for _,members in bridge_list.items():
982 for iface in members:
983 if not iface in addrs_list.keys():
984 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
985
986 output += "\n"
987
988 # Details like SSID
989 if extra_ouput:
990 output += extra_ouput.strip() + "\n"
991
992 # Print IP address which needs to be assigned over here
993 output += "\n"
994 for iface,addrs in sorted(addrs_list.iteritems()):
995 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
996 output += "# %s || %s || %s\n" % (iface, addr, comment)
997
998 # Write DHCLIENT entry
999 if iface in dhclient_if and dhclient_if[iface]:
1000 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1001 continue
1002
1003 # Make sure the external address is always first as this is needed in the
1004 # firewall setup
1005 addrs = sorted(
1006 [x for x in addrs if not '0.0.0.0' in x[0]],
1007 key=lambda x: x[0].split('.')[0],
1008 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1009 )
1010
1011 idx_offset = 0
1012 # Set MAC is required
1013 if flags_if[iface].has_key('ether'):
1014 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1015 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1016 idx_offset += 1
1017 elif iface in bridge_list:
1018 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1019 idx_offset += 1
1020 else:
1021 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1022
1023 for idx, addr in enumerate(addrs[1:]):
1024 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1025
1026 output += "\n"
1027
1028 rc_conf_local_cache[datadump['autogen_item']] = output
1029 return output
1030
1031
1032
1033def get_all_configs():
1034 """ Get dict with key 'host' with all configs present """
1035 configs = dict()
1036 for host in get_hostlist():
1037 datadump = get_yaml(host)
1038 configs[host] = datadump
1039 return configs
1040
1041
1042def get_interface_keys(config, extra=False):
1043 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1044 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1045 if extra == False:
1046 return filter(lambda x: not "extra" in x, elems)
1047 else:
1048 return elems
1049
1050
1051def get_used_ips(configs):
1052 """ Return array of all IPs used in config files"""
1053 ip_list = []
1054 for config in configs:
1055 ip_list.append(config['masterip'])
1056 if 'serviceid' in config:
1057 ip_list.append(config['serviceid'])
1058 for iface_key in get_interface_keys(config, True):
1059 l = config[iface_key]['ip']
1060 addr, mask = l.split('/')
1061 # Special case do not process
1062 if valid_addr(addr):
1063 ip_list.append(addr)
1064 else:
1065 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1066 return sorted(ip_list)
1067
1068
1069
1070def get_nameservers(max_servers=None):
1071 if nameservers_cache:
1072 return nameservers_cache[0:max_servers]
1073
1074 for host in get_hostlist():
1075 hostdump = get_yaml(host)
1076 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1077 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1078
1079 return nameservers_cache[0:max_servers]
1080
1081
1082def get_neighbours(datadump):
1083 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1084
1085 (poel, errors) = make_relations()
1086 table = []
1087 for iface,addrs in sorted(addrs_list.iteritems()):
1088 if iface in ['lo0']:
1089 continue
1090
1091 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1092 if not addr.startswith('172.'):
1093 # Avoid listing internet connections as pool
1094 continue
1095 for neighbour in poel[network(addr)]:
1096 if neighbour[0] != datadump['autogen_item']:
1097 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1098 return table
1099
1100
1101def get_attached_devices(datadump, url=False):
1102 table = []
1103 for iface_key in get_interface_keys(datadump, True):
1104 ifacedump = datadump[iface_key]
1105
1106 if not ifacedump.has_key('ns_ip'):
1107 continue
1108
1109 x_ip = ifacedump['ns_ip'].split('/')[0]
1110
1111 if 'mode' in ifacedump:
1112 x_mode = ifacedump['mode']
1113 else:
1114 x_mode = 'unknown'
1115
1116 if 'bridge_type' in ifacedump:
1117 device_type = ifacedump['bridge_type']
1118 else:
1119 device_type = 'Unknown'
1120
1121 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1122 return table
1123
1124
1125def generate_resolv_conf(datadump):
1126 """ Generate configuration file '/etc/resolv.conf' """
1127 # XXX: This should properly going to be an datastructure soon
1128 datadump['autogen_header'] = generate_header(datadump, "#")
1129 datadump['autogen_edge_nameservers'] = ''
1130
1131
1132 for masterip,realname in get_nameservers():
1133 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1134
1135 return Template("""\
1136{{ autogen_header }}
1137search wleiden.net
1138
1139# Try local (cache) first
1140nameserver 127.0.0.1
1141
1142{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1143nameserver 8.8.8.8 # Google Public NameServer
1144nameserver 64.6.64.6 # Verisign Public NameServer
1145{% else -%}
1146# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1147{{ autogen_edge_nameservers }}
1148{% endif -%}
1149""").render(datadump)
1150
1151
1152
1153def generate_ntp_conf(datadump):
1154 """ Generate configuration file '/etc/ntp.conf' """
1155 # XXX: This should properly going to be an datastructure soon
1156
1157 datadump['autogen_header'] = generate_header(datadump, "#")
1158 datadump['autogen_ntp_servers'] = ''
1159 for host in get_hostlist():
1160 hostdump = get_yaml(host)
1161 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1162 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1163
1164 return Template("""\
1165{{ autogen_header }}
1166
1167{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1168# Machine hooked to internet.
1169server 0.nl.pool.ntp.org iburst maxpoll 9
1170server 1.nl.pool.ntp.org iburst maxpoll 9
1171server 2.nl.pool.ntp.org iburst maxpoll 9
1172server 3.nl.pool.ntp.org iburst maxpoll 9
1173{% else -%}
1174# Local Wireless Leiden NTP Servers.
1175server 0.pool.ntp.wleiden.net iburst maxpoll 9
1176server 1.pool.ntp.wleiden.net iburst maxpoll 9
1177server 2.pool.ntp.wleiden.net iburst maxpoll 9
1178server 3.pool.ntp.wleiden.net iburst maxpoll 9
1179
1180# All the configured NTP servers
1181{{ autogen_ntp_servers }}
1182{% endif %}
1183
1184# If a server loses sync with all upstream servers, NTP clients
1185# no longer follow that server. The local clock can be configured
1186# to provide a time source when this happens, but it should usually
1187# be configured on just one server on a network. For more details see
1188# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1189# The use of Orphan Mode may be preferable.
1190#
1191server 127.127.1.0
1192fudge 127.127.1.0 stratum 10
1193""").render(datadump)
1194
1195
1196def generate_pf_hybrid_conf_local(datadump):
1197 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1198 datadump['autogen_header'] = generate_header(datadump, "#")
1199 if datadump['service_incoming_rdr']:
1200 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1201 return Template("""\
1202{{ autogen_header }}
1203
1204# Redirect some internal facing services outside (7)
1205# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1206{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1207rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1208{% endfor -%}
1209# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1210{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1211rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1212{% endfor -%}
1213""").render(datadump)
1214
1215def generate_unbound_wleiden_conf(datadump):
1216 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1217 datadump['autogen_header'] = generate_header(datadump, "#")
1218
1219 autogen_ips = []
1220 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1221 for iface,addrs in sorted(addrs_list.iteritems()):
1222 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1223 if addr.startswith('172'):
1224 autogen_ips.append((addr.split('/')[0], comment))
1225 datadump['autogen_ips'] = autogen_ips
1226
1227 create_proxies_list()
1228 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1229 return Template("""\
1230{{ autogen_header }}
1231
1232server:
1233 ## Static definitions fail to start on systems with broken ue(4) interfaces
1234{%- for ip,comment in autogen_ips %}
1235 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1236{%- endfor %}
1237 ## Enabling wildcard matching as work-around
1238 interface: 0.0.0.0
1239 interface: ::0
1240
1241forward-zone:
1242 name: '.'
1243{%- if service_proxy_ileiden %}
1244 forward-addr: 8.8.8.8 # Google DNS A
1245 forward-addr: 8.8.4.4 # Google DNS B
1246 forward-addr: 208.67.222.222 # OpenDNS DNS A
1247 forward-addr: 208.67.220.220 # OpenDNS DNS B
1248{% else -%}
1249{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1250 {%- if loop.index <= 5 %}
1251 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1252 {%- endif %}
1253{%- endfor %}
1254{%- endif %}
1255""").render(datadump)
1256
1257def generate_motd(datadump):
1258 """ Generate configuration file '/etc/motd' """
1259 output = Template("""\
1260FreeBSD run ``service motd onestart'' to make me look normal
1261
1262 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1263 Loc: {{ location }}
1264
1265Services:
1266{% if board == "ALIX2" or board == "net4801" -%}
1267{{" -"}} Core Node ({{ board }})
1268{% else -%}
1269{{" -"}} Hulp Node ({{ board }})
1270{% endif -%}
1271{% if service_proxy_normal -%}
1272{{" -"}} Normal Proxy
1273{% endif -%}
1274{% if service_proxy_ileiden -%}
1275{{" -"}} iLeiden Proxy
1276{% endif -%}
1277{% if service_incoming_rdr -%}
1278{{" -"}} Incoming port redirects
1279{% endif %}
1280Interlinks:\n
1281""").render(datadump)
1282
1283
1284 def make_table(table):
1285 if not table:
1286 return " - none\n"
1287 else:
1288 lines = ""
1289 col_width = [max(len(x) for x in col) for col in zip(*table)]
1290 for row in table:
1291 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1292 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1293 return lines
1294
1295 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1296 table = []
1297 for iface,addrs in sorted(addrs_list.iteritems()):
1298 if iface in ['lo0']:
1299 continue
1300 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1301 table.append((iface, addr, comment))
1302
1303 output += make_table(table)
1304 output += '\n'
1305 output += """\
1306Attached devices:
1307"""
1308 output += make_table(get_attached_devices(datadump, url=True))
1309 output += '\n'
1310 output += """\
1311Available neighbours:
1312"""
1313 output += make_table(get_neighbours(datadump))
1314
1315 return output
1316
1317
1318def format_yaml_value(value):
1319 """ Get yaml value in right syntax for outputting """
1320 if isinstance(value,str):
1321 output = '"%s"' % value
1322 else:
1323 output = value
1324 return output
1325
1326
1327
1328def format_wleiden_yaml(datadump):
1329 """ Special formatting to ensure it is editable"""
1330 output = "# Genesis config yaml style\n"
1331 output += "# vim:ts=2:et:sw=2:ai\n"
1332 output += "#\n"
1333 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1334 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1335 if key == 'rdr_rules':
1336 output += '%-10s:\n' % 'rdr_rules'
1337 for rdr_rule in datadump[key]:
1338 output += '- %s\n' % rdr_rule
1339 else:
1340 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1341
1342 output += "\n\n"
1343
1344 # Format (key, required)
1345 key_order = (
1346 ('comment', True),
1347 ('parent', False),
1348 ('ip', False),
1349 ('ether', False),
1350 ('desc', True),
1351 ('sdesc', True),
1352 ('mode', True),
1353 ('type', True),
1354 ('extra_type', False),
1355 ('channel', False),
1356 ('ssid', False),
1357 ('wlan_mac', False),
1358 ('dhcp', True),
1359 ('dhcp_fixed', False),
1360 ('compass', False),
1361 ('distance', False),
1362 ('ns_ip', False),
1363 ('repeater_ip', False),
1364 ('bullet2_ip', False),
1365 ('ns_mac', False),
1366 ('bullet2_mac', False),
1367 ('ns_type', False),
1368 ('bridge_type', False),
1369 ('status', True),
1370 )
1371
1372 for iface_key in sorted(iface_keys):
1373 try:
1374 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1375 if remainder:
1376 raise KeyError("invalid keys: %s" % remainder)
1377
1378 output += "%s:\n" % iface_key
1379 for key,required in key_order:
1380 if datadump[iface_key].has_key(key):
1381 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1382 output += "\n\n"
1383 except Exception:
1384 print "# Error while processing interface %s" % iface_key
1385 raise
1386
1387 return output
1388
1389
1390
1391def generate_wleiden_yaml(datadump, header=True):
1392 """ Generate (petty) version of wleiden.yaml"""
1393 output = generate_header(datadump, "#") if header else ''
1394
1395 for key in datadump.keys():
1396 if key.startswith('autogen_'):
1397 del datadump[key]
1398 # Interface autogen cleanups
1399 elif type(datadump[key]) == dict:
1400 for key2 in datadump[key].keys():
1401 if key2.startswith('autogen_'):
1402 del datadump[key][key2]
1403
1404 output += format_wleiden_yaml(datadump)
1405 return output
1406
1407def generate_nanostation_config(datadump, iface, ns_type):
1408 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1409 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1410
1411 datadump.update(datadump['iface_%s' % iface])
1412
1413 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1414
1415def generate_yaml(datadump):
1416 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1417
1418
1419
1420def generate_config(node, config, datadump=None):
1421 """ Print configuration file 'config' of 'node' """
1422 output = ""
1423 try:
1424 # Load config file
1425 if datadump == None:
1426 datadump = get_yaml(node)
1427
1428 if config == 'wleiden.yaml':
1429 output += generate_wleiden_yaml(datadump)
1430 elif config == 'authorized_keys':
1431 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1432 output += f.read()
1433 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1434 # Fetch local keys if existing
1435 if os.path.exists(node_keys):
1436 output += open(node_keys, 'r').read()
1437 f.close()
1438 elif config == 'dnsmasq.conf':
1439 output += generate_dnsmasq_conf(datadump)
1440 elif config == 'dhcpd.conf':
1441 output += generate_dhcpd_conf(datadump)
1442 elif config == 'rc.conf.local':
1443 output += generate_rc_conf_local(datadump)
1444 elif config == 'resolv.conf':
1445 output += generate_resolv_conf(datadump)
1446 elif config == 'ntp.conf':
1447 output += generate_ntp_conf(datadump)
1448 elif config == 'motd':
1449 output += generate_motd(datadump)
1450 elif config == 'pf.hybrid.conf.local':
1451 output += generate_pf_hybrid_conf_local(datadump)
1452 elif config == 'unbound.wleiden.conf':
1453 output += generate_unbound_wleiden_conf(datadump)
1454 elif config.startswith('vr'):
1455 interface, ns_type = config.strip('.yaml').split('-')
1456 output += generate_nanostation_config(datadump, interface, ns_type)
1457 else:
1458 assert False, "Config not found!"
1459 except IOError, e:
1460 output += "[ERROR] Config file not found"
1461 return output
1462
1463
1464
1465def generate_static(output_dir, logging=True):
1466 items = {'output_dir' : output_dir}
1467 for node in get_hostlist():
1468 items['node'] = node
1469 items['wdir'] = "%(output_dir)s/%(node)s" % items
1470 if not os.path.isdir(items['wdir']):
1471 os.makedirs(items['wdir'])
1472 datadump = get_yaml(node)
1473 f = open("%(wdir)s/index.html" % items, "w")
1474 f.write(generate_node_overview(items['node'], datadump))
1475 f.close()
1476 for config in files:
1477 items['config'] = config
1478 if logging: logger.info("## Generating %(node)s %(config)s" % items)
1479 f = open("%(wdir)s/%(config)s" % items, "w")
1480 f.write(generate_config(node, config, datadump))
1481 f.close()
1482
1483
1484
1485def process_cgi_request(environ=os.environ):
1486 """ When calling from CGI """
1487 response_headers = []
1488 content_type = 'text/plain'
1489
1490 # Update repository if requested
1491 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1492 if form and form.has_key("action") and "update" in form["action"]:
1493 output = "[INFO] Updating subverion, please wait...\n"
1494 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1495 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1496 generate_static(CACHE_DIR, False)
1497 output += "[INFO] All done, redirecting in 5 seconds"
1498 response_headers += [
1499 ('Refresh', '5; url=.'),
1500 ]
1501 else:
1502 base_uri = environ['REQUEST_URI']
1503 uri = base_uri.strip('/').split('/')[1:]
1504
1505 output = "Template Holder"
1506 if base_uri.endswith('/create/network.kml'):
1507 content_type='application/vnd.google-earth.kml+xml'
1508 output = make_network_kml.make_graph()
1509 elif base_uri.endswith('/api/get/nodeplanner.json'):
1510 content_type='application/json'
1511 output = make_network_kml.make_nodeplanner_json()
1512 elif not uri:
1513 if is_text_request(environ):
1514 output = '\n'.join(get_hostlist())
1515 else:
1516 content_type = 'text/html'
1517 output = generate_title(get_hostlist())
1518 elif len(uri) == 1:
1519 if is_text_request(environ):
1520 output = generate_node(uri[0])
1521 else:
1522 content_type = 'text/html'
1523 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1524 elif len(uri) == 2:
1525 output = generate_config(uri[0], uri[1])
1526 else:
1527 assert False, "Invalid option"
1528
1529 # Return response
1530 response_headers += [
1531 ('Content-type', content_type),
1532 ('Content-Length', str(len(output))),
1533 ]
1534 return(response_headers, str(output))
1535
1536
1537def make_dns(output_dir = 'dns', external = False):
1538 items = dict()
1539
1540 # hostname is key, IP is value
1541 wleiden_zone = defaultdict(list)
1542 wleiden_cname = dict()
1543
1544 pool = dict()
1545 for node in get_hostlist():
1546 datadump = get_yaml(node)
1547
1548 fqdn = datadump['nodename']
1549
1550 if datadump.has_key('rdr_host'):
1551 remote_target = datadump['rdr_host']
1552 elif datadump.has_key('remote_access') and datadump['remote_access']:
1553 remote_target = datadump['remote_access'].split(':')[0]
1554 else:
1555 remote_target = None
1556
1557 if remote_target:
1558 try:
1559 parseaddr(remote_target)
1560 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1561 except (IndexError, ValueError):
1562 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1563
1564
1565 wleiden_zone[fqdn].append((datadump['masterip'], True))
1566
1567 # Hacking to get proper DHCP IPs and hostnames
1568 for iface_key in get_interface_keys(datadump):
1569 iface_name = iface_key.replace('_','-')
1570 if 'ip' in datadump[iface_key]:
1571 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1572 try:
1573 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1574 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1575 dhcp_part = ".".join(ip.split('.')[0:3])
1576 if ip != datadump['masterip']:
1577 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1578 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1579 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1580 except (AttributeError, ValueError, KeyError):
1581 # First push it into a pool, to indentify the counter-part later on
1582 addr = parseaddr(ip)
1583 cidr = int(cidr)
1584 addr = addr & ~((1 << (32 - cidr)) - 1)
1585 if pool.has_key(addr):
1586 pool[addr] += [(iface_name, fqdn, ip)]
1587 else:
1588 pool[addr] = [(iface_name, fqdn, ip)]
1589 continue
1590
1591
1592
1593 # WL uses an /29 to configure an interface. IP's are ordered like this:
1594 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1595
1596 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1597
1598 # Automatic naming convention of interlinks namely 2 + remote.lower()
1599 for (key,value) in pool.iteritems():
1600 # Make sure they are sorted from low-ip to high-ip
1601 value = sorted(value, key=lambda x: parseaddr(x[2]))
1602
1603 if len(value) == 1:
1604 (iface_name, fqdn, ip) = value[0]
1605 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1606
1607 # Device DNS names
1608 if 'cnode' in fqdn.lower():
1609 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1610 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1611
1612 elif len(value) == 2:
1613 (a_iface_name, a_fqdn, a_ip) = value[0]
1614 (b_iface_name, b_fqdn, b_ip) = value[1]
1615 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1616 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1617
1618 # Device DNS names
1619 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1620 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1621 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1622 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1623 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1624 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1625 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1626
1627 else:
1628 pool_members = [k[1] for k in value]
1629 for item in value:
1630 (iface_name, fqdn, ip) = item
1631 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1632
1633 # Include static DNS entries
1634 # XXX: Should they override the autogenerated results?
1635 # XXX: Convert input to yaml more useable.
1636 # Format:
1637 ##; this is a comment
1638 ## roomburgh=Roomburgh1
1639 ## apkerk1.Vosko=172.17.176.8 ;this as well
1640 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1641
1642 # Hack to allow special entries, for development
1643 wleiden_raw = {}
1644
1645 for line in dns_list:
1646 reverse = False
1647 k, items = line.items()[0]
1648 if type(items) == dict:
1649 if items.has_key('reverse'):
1650 reverse = items['reverse']
1651 items = items['a']
1652 else:
1653 items = items['cname']
1654 items = [items] if type(items) != list else items
1655 for item in items:
1656 if item.startswith('IN '):
1657 wleiden_raw[k] = item
1658 elif valid_addr(item):
1659 wleiden_zone[k].append((item, reverse))
1660 else:
1661 wleiden_cname[k] = item
1662
1663 # Hack to get dynamic pool listing
1664 def chunks(l, n):
1665 return [l[i:i+n] for i in range(0, len(l), n)]
1666
1667 ntp_servers = [x[0] for x in get_nameservers()]
1668 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1669 for ntp_server in chunk:
1670 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1671
1672 details = dict()
1673 # 24 updates a day allowed
1674 details['serial'] = time.strftime('%Y%m%d%H')
1675
1676 if external:
1677 dns_masters = ['ns1.vanderzwet.net', 'ns1.transip.net', 'ns2.transip.eu']
1678 else:
1679 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1680
1681 details['master'] = dns_masters[0]
1682 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1683
1684 dns_header = '''
1685$TTL 3h
1686%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1687 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1688
1689%(ns_servers)s
1690 \n'''
1691
1692
1693 if not os.path.isdir(output_dir):
1694 os.makedirs(output_dir)
1695 details['zone'] = 'wleiden.net'
1696 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1697 f.write(dns_header % details)
1698
1699 for host,items in wleiden_zone.iteritems():
1700 for ip,reverse in items:
1701 if ip not in ['0.0.0.0']:
1702 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1703 for source,dest in wleiden_cname.iteritems():
1704 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1705 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1706 for source, dest in wleiden_raw.iteritems():
1707 f.write("%s.wleiden.net. %s\n" % (source, dest))
1708 f.close()
1709
1710 # Create whole bunch of specific sub arpa zones. To keep it compliant
1711 for s in range(16,32):
1712 details['zone'] = '%i.172.in-addr.arpa' % s
1713 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1714 f.write(dns_header % details)
1715
1716 #XXX: Not effient, fix to proper data structure and do checks at other
1717 # stages
1718 for host,items in wleiden_zone.iteritems():
1719 for ip,reverse in items:
1720 if not reverse:
1721 continue
1722 if valid_addr(ip):
1723 if valid_addr(ip):
1724 if int(ip.split('.')[1]) == s:
1725 rev_ip = '.'.join(reversed(ip.split('.')))
1726 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1727 f.close()
1728
1729
1730def usage():
1731 print """Usage: %(prog)s <argument>
1732Argument:
1733\tcleanup = Cleanup all YAML files to specified format
1734\tstandalone [port] = Run configurator webserver [8000]
1735\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1736\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1737\tfull-export = Generate yaml export script for heatmap.
1738\tstatic [outputdir] = Generate all config files and store on disk
1739\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1740\ttest <node> [<file>] = Receive output for certain node [all files].
1741\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1742\tlist <status> <items> = List systems which have certain status
1743\tcreate network.kml = Create Network KML file for use in Google Earth
1744
1745Arguments:
1746\t<node> = NodeName (example: HybridRick)
1747\t<file> = %(files)s
1748\t<status> = all|up|down|planned
1749\t<items> = systems|nodes|proxies
1750
1751NOTE FOR DEVELOPERS; you can test your changes like this:
1752 BEFORE any changes in this code:
1753 $ ./gformat.py static /tmp/pre
1754 AFTER the changes:
1755 $ ./gformat.py static /tmp/post
1756 VIEW differences and VERIFY all are OK:
1757 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1758""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1759 exit(0)
1760
1761
1762def is_text_request(environ=os.environ):
1763 """ Find out whether we are calling from the CLI or any text based CLI utility """
1764 if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1765 return True
1766
1767 if 'HTTP_USER_AGENT' in environ:
1768 return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1769 else:
1770 return False
1771
1772
1773def switchFormat(setting):
1774 if setting:
1775 return "YES"
1776 else:
1777 return "NO"
1778
1779def rlinput(prompt, prefill=''):
1780 import readline
1781 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1782 try:
1783 return raw_input(prompt)
1784 finally:
1785 readline.set_startup_hook()
1786
1787def fix_conflict(left, right, default='i'):
1788 while True:
1789 print "## %-30s | %-30s" % (left, right)
1790 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1791 if not c:
1792 c = default
1793
1794 if c in ['l','1']:
1795 return left
1796 elif c in ['r','2']:
1797 return right
1798 elif c in ['e', '3']:
1799 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1800 elif c in ['i', '4']:
1801 return None
1802 else:
1803 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1804
1805
1806
1807def print_cgi_response(response_headers, output):
1808 for header in response_headers:
1809 print "%s: %s" % header
1810 print
1811 print output
1812
1813
1814
1815def main():
1816 """Hard working sub"""
1817 # Allow easy hacking using the CLI
1818 if not os.environ.has_key('REQUEST_URI'):
1819 if len(sys.argv) < 2:
1820 usage()
1821
1822 if sys.argv[1] == "standalone":
1823 import SocketServer
1824 import CGIHTTPServer
1825 # Hop to the right working directory.
1826 os.chdir(os.path.dirname(__file__))
1827 try:
1828 PORT = int(sys.argv[2])
1829 except (IndexError,ValueError):
1830 PORT = 8000
1831
1832 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1833 """ Serve this CGI from the root of the webserver """
1834 def is_cgi(self):
1835 if "favicon" in self.path:
1836 return False
1837
1838 self.cgi_info = (os.path.basename(__file__), self.path)
1839 self.path = ''
1840 return True
1841 handler = MyCGIHTTPRequestHandler
1842 SocketServer.TCPServer.allow_reuse_address = True
1843 httpd = SocketServer.TCPServer(("", PORT), handler)
1844 httpd.server_name = 'localhost'
1845 httpd.server_port = PORT
1846
1847 logger.info("serving at port %s", PORT)
1848 try:
1849 httpd.serve_forever()
1850 except KeyboardInterrupt:
1851 httpd.shutdown()
1852 logger.info("All done goodbye")
1853 elif sys.argv[1] == "test":
1854 # Basic argument validation
1855 try:
1856 node = sys.argv[2]
1857 except IndexError:
1858 print "Invalid argument"
1859 exit(1)
1860 except IOError as e:
1861 print e
1862 exit(1)
1863
1864 datadump = get_yaml(node)
1865
1866
1867 # Get files to generate
1868 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1869
1870 # Actual config generation
1871 for config in gen_files:
1872 logger.info("## Generating %s %s", node, config)
1873 print generate_config(node, config, datadump)
1874 elif sys.argv[1] == "test-cgi":
1875 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1876 os.environ['SCRIPT_NAME'] = __file__
1877 response_headers, output = process_cgi_request()
1878 print_cgi_response(response_headers, output)
1879 elif sys.argv[1] == "static":
1880 generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
1881 elif sys.argv[1] == "wind-export":
1882 items = dict()
1883 for node in get_hostlist():
1884 datadump = get_yaml(node)
1885 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1886 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1887 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1888 VALUES (
1889 (SELECT id FROM users WHERE username = 'rvdzwet'),
1890 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1891 'Y');""" % datadump
1892 #for config in files:
1893 # items['config'] = config
1894 # print "## Generating %(node)s %(config)s" % items
1895 # f = open("%(wdir)s/%(config)s" % items, "w")
1896 # f.write(generate_config(node, config, datadump))
1897 # f.close()
1898 for node in get_hostlist():
1899 datadump = get_yaml(node)
1900 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1901 ifacedump = datadump[iface_key]
1902 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1903 ifacedump['nodename'] = datadump['nodename']
1904 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1905 ifacedump['channel'] = 0
1906 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1907 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1908 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1909 elif sys.argv[1] == "nagios-export":
1910 try:
1911 heavy_load = (sys.argv[2] == "--heavy-load")
1912 except IndexError:
1913 heavy_load = False
1914
1915 hostgroup_details = {
1916 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1917 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1918 'walphen' : 'Stichting Wireless Alphen',
1919 'westeinder' : 'Westeinder Plassen',
1920 }
1921
1922 # Convert IP to Host
1923 ip2host = {'root' : 'root'}
1924 for host in get_hostlist():
1925 datadump = get_yaml(host)
1926 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1927 for iface in get_interface_keys(datadump):
1928 if datadump[iface].has_key('autogen_gateway'):
1929 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1930
1931 # Find dependency tree based on output of lvrouted.mytree of nearest node
1932 parents = defaultdict(list)
1933 stack = ['root']
1934 prev_depth = 0
1935 for line in open('lvrouted.mytree').readlines():
1936 depth = line.count('\t')
1937 ip = line.strip().split()[0]
1938
1939 if prev_depth < depth:
1940 try:
1941 parents[ip2host[ip]].append(ip2host[stack[-1]])
1942 except KeyError as e:
1943 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1944 stack.append(ip)
1945 elif prev_depth > depth:
1946 stack = stack[:(depth - prev_depth)]
1947 elif prev_depth == depth:
1948 try:
1949 parents[ip2host[ip]].append(ip2host[stack[-1]])
1950 except KeyError as e:
1951 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1952
1953
1954 prev_depth = depth
1955 # Observe that some nodes has themself as parent or multiple parents
1956 # for now take only the first parent, other behaviour is yet to be explained
1957
1958
1959
1960 params = {
1961 'check_interval' : 5 if heavy_load else 120,
1962 'retry_interval' : 1 if heavy_load else 10,
1963 'max_check_attempts' : 10 if heavy_load else 6,
1964 'notification_interval': 120 if heavy_load else 240,
1965 }
1966
1967 print '''\
1968define host {
1969 name wleiden-node ; Default Node Template
1970 use generic-host ; Use the standard template as initial starting point
1971 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1972 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1973 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1974 notification_interval %(notification_interval)s
1975 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1976 check_command check-host-alive ; Default command to check FreeBSD hosts
1977 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1978}
1979
1980define service {
1981 name wleiden-service ; Default Service Template
1982 use generic-service ; Use the standard template as initial starting point
1983 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1984 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1985 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1986 notification_interval %(notification_interval)s
1987 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1988 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1989}
1990
1991# Please make sure to install:
1992# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1993#
1994# Recompile net-mgmt/nagios-plugins to support check_snmp
1995# make -C /usr/ports/net-mgmt/nagios-plugins
1996#
1997# Install net/bind-tools to allow v2/check_dns_wl to work:
1998# pkg install bind-tools
1999#
2000define command{
2001 command_name check_snmp_disk
2002 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2003}
2004
2005define command{
2006 command_name check_netsnmp_load
2007 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2008}
2009
2010define command{
2011 command_name check_netsnmp_proc
2012 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2013}
2014
2015define command{
2016 command_name check_by_ssh
2017 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2018}
2019
2020define command{
2021 command_name check_dns_wl
2022 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2023}
2024
2025define command{
2026 command_name check_snmp_uptime
2027 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2028}
2029
2030
2031# TDB: dhcp leases
2032# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2033
2034# TDB: internet status
2035# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2036
2037# TDB: Advanced local passive checks
2038# /usr/local/libexec/nagios/check_by_ssh
2039''' % params
2040
2041 print '''\
2042# Service Group, not displayed by default
2043define hostgroup {
2044 hostgroup_name srv_hybrid
2045 alias All Hybrid Nodes
2046 register 0
2047}
2048
2049define service {
2050 use wleiden-service
2051 hostgroup_name srv_hybrid
2052 service_description SSH
2053 check_command check_ssh
2054}
2055
2056define service {
2057 use wleiden-service,service-pnp
2058 hostgroup_name srv_hybrid
2059 service_description HTTP
2060 check_command check_http
2061}
2062
2063define service {
2064 use wleiden-service
2065 hostgroup_name srv_hybrid
2066 service_description DNS
2067 check_command check_dns_wl!"www.wirelessleiden.nl"
2068}
2069'''
2070
2071 if heavy_load:
2072 print '''\
2073define service {
2074 use wleiden-service
2075 hostgroup_name srv_hybrid
2076 service_description UPTIME
2077 check_command check_snmp_uptime
2078}
2079
2080#define service {
2081# use wleiden-service
2082# hostgroup_name srv_hybrid
2083# service_description NTP
2084# check_command check_ntp_peer
2085#}
2086
2087define service {
2088 use wleiden-service
2089 hostgroup_name srv_hybrid
2090 service_description LOAD
2091 check_command check_netsnmp_load
2092}
2093
2094define service {
2095 use wleiden-service
2096 hostgroup_name srv_hybrid
2097 service_description PROC
2098 check_command check_netsnmp_proc
2099}
2100
2101define service {
2102 use wleiden-service
2103 hostgroup_name srv_hybrid
2104 service_description DISK
2105 check_command check_snmp_disk
2106}
2107'''
2108 for node in get_hostlist():
2109 datadump = get_yaml(node)
2110 if not datadump['status'] == 'up':
2111 continue
2112 if not hostgroup_details.has_key(datadump['monitoring_group']):
2113 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2114 print '''\
2115define host {
2116 use wleiden-node,host-pnp
2117 contact_groups admins
2118 host_name %(autogen_fqdn)s
2119 address %(masterip)s
2120 hostgroups srv_hybrid,%(monitoring_group)s\
2121''' % datadump
2122 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2123 print '''\
2124 parents %(parents)s\
2125''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2126 print '''\
2127}
2128'''
2129
2130
2131 for name,alias in hostgroup_details.iteritems():
2132 print '''\
2133define hostgroup {
2134 hostgroup_name %s
2135 alias %s
2136} ''' % (name, alias)
2137
2138
2139 elif sys.argv[1] == "full-export":
2140 hosts = {}
2141 for node in get_hostlist():
2142 datadump = get_yaml(node)
2143 hosts[datadump['nodename']] = datadump
2144 print yaml.dump(hosts)
2145
2146 elif sys.argv[1] == "dns":
2147 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2148 elif sys.argv[1] == "cleanup":
2149 # First generate all datadumps
2150 datadumps = dict()
2151 ssid_to_node = dict()
2152 for host in get_hostlist():
2153 logger.info("# Processing: %s", host)
2154 # Set some boring default values
2155 datadump = { 'board' : 'UNKNOWN' }
2156 datadump.update(get_yaml(host))
2157 datadumps[datadump['nodename']] = datadump
2158
2159 (poel, errors) = make_relations()
2160 print "\n".join(["# WARNING: %s" % x for x in errors])
2161
2162 for host,datadump in datadumps.iteritems():
2163 try:
2164 # Convert all yes and no to boolean values
2165 def fix_boolean(dump):
2166 for key in dump.keys():
2167 if type(dump[key]) == dict:
2168 dump[key] = fix_boolean(dump[key])
2169 elif str(dump[key]).lower() in ["yes", "true"]:
2170 dump[key] = True
2171 elif str(dump[key]).lower() in ["no", "false"]:
2172 # Compass richting no (Noord Oost) is valid input
2173 if key != "compass": dump[key] = False
2174 return dump
2175 datadump = fix_boolean(datadump)
2176
2177 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2178 if not 'latitude' in datadump and not 'longitude' in datadump:
2179 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2180 elif 'latitude' in datadump and 'longitude' in datadump:
2181 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2182 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2183 # TODO: Compare outcome of both coordinate systems and validate against each-other
2184
2185 if datadump['nodename'].startswith('Proxy'):
2186 datadump['nodename'] = datadump['nodename'].lower()
2187
2188 for iface_key in get_interface_keys(datadump):
2189 try:
2190 # All our normal wireless cards are normal APs now
2191 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2192 datadump[iface_key]['mode'] = 'ap'
2193 # Wireless Leiden SSID have an consistent lowercase/uppercase
2194 if datadump[iface_key].has_key('ssid'):
2195 ssid = datadump[iface_key]['ssid']
2196 prefix = 'ap-WirelessLeiden-'
2197 if ssid.lower().startswith(prefix.lower()):
2198 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2199 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2200 datadump[iface_key]['mode'] = 'autogen-FIXME'
2201 if not datadump[iface_key].has_key('comment'):
2202 datadump[iface_key]['comment'] = 'autogen-FIXME'
2203
2204 if datadump[iface_key].has_key('ns_mac'):
2205 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2206
2207 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2208 datadump[iface_key] = datadump[iface_key]['desc']
2209
2210 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2211 # due to better collision avoidance.
2212 if datadump[iface_key]['type'] == '11b':
2213 datadump[iface_key]['type'] = '11g'
2214
2215 # Setting 802.11g channels to de-facto standards, to avoid
2216 # un-detected sharing with other overlapping channels
2217 #
2218 # Technically we could also use channel 13 in NL, but this is not
2219 # recommended as foreign devices might not be able to select this
2220 # channel. Secondly using 1,5,9,13 instead is going to clash with
2221 # the de-facto usage of 1,6,11.
2222 #
2223 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2224 channels_at_2400Mhz = (1,6,11)
2225 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2226 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2227 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2228 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2229
2230 # Mandatory interface keys
2231 if not datadump[iface_key].has_key('status'):
2232 datadump[iface_key]['status'] = 'planned'
2233
2234 x = datadump[iface_key]['comment']
2235 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2236
2237 # Fixing bridge_type if none is found
2238 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2239 if not 'bridge_type' in datadump[iface_key]:
2240 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2241
2242 # Making sure description works
2243 if datadump[iface_key].has_key('desc'):
2244 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2245 del datadump[iface_key]['desc']
2246 else:
2247 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2248 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2249 if response:
2250 datadump[iface_key]['comment'] = response
2251 del datadump[iface_key]['desc']
2252
2253 # Check DHCP configuration
2254 dhcp_type(datadump[iface_key])
2255
2256 # Set the compass value based on the angle between the poels
2257 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2258 my_pool = poel[network(datadump[iface_key]['ip'])]
2259 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2260 if remote_hosts:
2261 compass_target = remote_hosts[0]
2262 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2263 # TODO: Compass wanted and actual direction might differ
2264
2265 # Monitoring Group default
2266 if not 'monitoring_group' in datadump:
2267 datadump['monitoring_group'] = 'wleiden'
2268
2269 except Exception:
2270 print "# Error while processing interface %s" % iface_key
2271 raise
2272 store_yaml(datadump)
2273 except Exception:
2274 print "# Error while processing %s" % host
2275 raise
2276 elif sys.argv[1] == "list":
2277 use_fqdn = False
2278 if len(sys.argv) < 4:
2279 usage()
2280 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2281 usage()
2282 if not sys.argv[3] in ["nodes","proxies","systems"]:
2283 usage()
2284
2285 if len(sys.argv) > 4:
2286 if sys.argv[4] == "fqdn":
2287 use_fqdn = True
2288 else:
2289 usage()
2290
2291 for system in get_hostlist():
2292 datadump = get_yaml(system)
2293 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2294 continue
2295
2296 output = datadump['autogen_fqdn'] if use_fqdn else system
2297 if sys.argv[2] == "all":
2298 print output
2299 elif datadump['status'] == sys.argv[2]:
2300 print output
2301 elif sys.argv[1] == "create":
2302 if sys.argv[2] == "network.kml":
2303 print make_network_kml.make_graph()
2304 elif sys.argv[2] == "host-ips.txt":
2305 for system in get_hostlist():
2306 datadump = get_yaml(system)
2307 ips = [datadump['masterip']]
2308 for ifkey in get_interface_keys(datadump):
2309 ips.append(datadump[ifkey]['ip'].split('/')[0])
2310 print system, ' '.join(ips)
2311 elif sys.argv[2] == "host-pos.txt":
2312 for system in get_hostlist():
2313 datadump = get_yaml(system)
2314 print system, datadump['rdnap_x'], datadump['rdnap_y']
2315 elif sys.argv[2] == 'ssh_config':
2316 print '''
2317Host *.wleiden.net
2318 User root
2319
2320Host 172.16.*.*
2321 User root
2322'''
2323 for system in get_hostlist():
2324 datadump = get_yaml(system)
2325 print '''\
2326Host %s
2327 User root
2328
2329Host %s
2330 User root
2331
2332Host %s
2333 User root
2334
2335Host %s
2336 User root
2337''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2338 else:
2339 usage()
2340 else:
2341 usage()
2342 else:
2343 # Do not enable debugging for config requests as it highly clutters the output
2344 if not is_text_request():
2345 cgitb.enable()
2346 response_headers, output = process_cgi_request()
2347 print_cgi_response(response_headers, output)
2348
2349if __name__ == "__main__":
2350 main()
Note: See TracBrowser for help on using the repository browser.