source: genesis/tools/gformat.py@ 13935

Last change on this file since 13935 was 13935, checked in by www, 7 years ago

Add caching support for CGI output

WSCGI does not work well with internal structures and multiple calls, reverting
to standaard cache using a file-based pre-generated backend powered by Apache24
rewriting.

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 79.4 KB
Line 
1#!/usr/local/bin/python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
35
36import argparse
37import cgi
38import cgitb
39import copy
40import glob
41import make_network_kml
42import math
43import pyproj
44import random
45import re
46import socket
47import string
48import subprocess
49import textwrap
50import time
51import urlparse
52
53from pprint import pprint
54from collections import defaultdict, OrderedDict
55from sys import stderr
56try:
57 import yaml
58except ImportError, e:
59 print e
60 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
61 exit(1)
62
63try:
64 from yaml import CLoader as Loader
65 from yaml import CDumper as Dumper
66except ImportError:
67 from yaml import Loader, Dumper
68
69from jinja2 import Environment, Template
70def yesorno(value):
71 return "YES" if bool(value) else "NO"
72env = Environment()
73env.filters['yesorno'] = yesorno
74def render_template(datadump, template):
75 result = env.from_string(template).render(datadump)
76 # Make it look pretty to the naked eye, as jinja templates are not so
77 # friendly when it comes to whitespace formatting
78 ## Remove extra whitespace at end of line lstrip() style.
79 result = re.sub(r'\n[\ ]+','\n', result)
80 ## Include only a single newline between an definition and a comment
81 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
82 ## Remove extra newlines after single comment
83 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
84 return result
85
86import logging
87logging.basicConfig(format='# %(levelname)s: %(message)s' )
88logger = logging.getLogger()
89logger.setLevel(logging.DEBUG)
90
91
92if os.environ.has_key('CONFIGROOT'):
93 NODE_DIR = os.environ['CONFIGROOT']
94else:
95 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
96__version__ = '$Id: gformat.py 13935 2017-07-06 08:56:40Z www $'
97
98CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
99
100files = [
101 'authorized_keys',
102 'dnsmasq.conf',
103 'dhcpd.conf',
104 'rc.conf.local',
105 'resolv.conf',
106 'motd',
107 'ntp.conf',
108 'pf.hybrid.conf.local',
109 'unbound.wleiden.conf',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = OrderedDict()
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126def clear_cache():
127 ''' Poor mans cache implementation '''
128 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
129 datadump_cache = {}
130 interface_list_cache = {}
131 rc_conf_local_cache = {}
132 ileiden_proxies = OrderedDict()
133 normal_proxies = []
134 nameservers_cache = []
135 relations_cache = None
136
137NO_DHCP = 0
138DHCP_CLIENT = 10
139DHCP_SERVER = 20
140def dhcp_type(item):
141 if not item.has_key('dhcp'):
142 return NO_DHCP
143 elif not item['dhcp']:
144 return NO_DHCP
145 elif item['dhcp'].lower() == 'client':
146 return DHCP_CLIENT
147 else:
148 # Validation Checks
149 begin,end = map(int,item['dhcp'].split('-'))
150 if begin >= end:
151 raise ValueError("DHCP Start >= DHCP End")
152 return DHCP_SERVER
153
154def etrs2rd(lat, lon):
155 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
156 p2 = pyproj.Proj(init='EPSG:28992')
157 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
158 return (RDx, RDy)
159
160def rd2etrs(RDx, RDy):
161 p1 = pyproj.Proj(init='EPSG:28992')
162 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
163 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
164 return (lat, lon)
165
166def get_yaml(item,add_version_info=True):
167 try:
168 """ Get configuration yaml for 'item'"""
169 if datadump_cache.has_key(item):
170 return datadump_cache[item].copy()
171
172 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
173 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
174 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
175
176 # Default values
177 datadump = {
178 'autogen_revision' : 'NOTFOUND',
179 'autogen_gfile' : gfile,
180 'service_proxy_ileiden' : False,
181 }
182 f = open(gfile, 'r')
183 datadump.update(yaml.load(f,Loader=Loader))
184 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
185 if datadump['nodetype'] == 'Hybrid':
186 # Some values are defined implicitly
187 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
188 datadump['service_incoming_rdr'] = True
189 # Use some boring defaults
190 defaults = {
191 'service_proxy_normal' : False,
192 'service_accesspoint' : True,
193 'service_incoming_rdr' : False,
194 'service_concentrator' : False,
195 'monitoring_group' : 'wleiden',
196 }
197 for (key,value) in defaults.iteritems():
198 if not datadump.has_key(key):
199 datadump[key] = value
200 f.close()
201
202 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
203 if add_version_info:
204 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
205 lines = p.communicate()[0].split('\n')
206 if p.returncode == 0:
207 for line in lines:
208 if line:
209 (key, value) = line.split(': ')
210 datadump["autogen_" + key.lower().replace(' ','_')] = value
211
212 # Preformat certain needed variables for formatting and push those into special object
213 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
214
215 wlan_count=0
216 try:
217 for key in get_interface_keys(datadump, True):
218 datadump[key]['autogen_ifbase'] = key.split('_')[1]
219 datadump[key]['autogen_vlan'] = False
220
221 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
222 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
223 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
224
225 if datadump[key].has_key('parent'):
226 if datadump[key].has_key('ip'):
227 raise ValueError("Interface bridge member cannot have IP assigned")
228 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
229 raise ValueError("Interface bridge member cannot have DHCP set")
230
231 if datadump[key].has_key('ip'):
232 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
233
234 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
235 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
236 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
237 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
238 wlan_count += 1
239 else:
240 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
241 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
242 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
243 datadump[key]['autogen_vlan'] = key.split('_')[2]
244 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
245 else:
246 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
247
248 except Exception:
249 print "# Error while processing interface %s" % key
250 raise
251
252 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
253 if dhcp_type(datadump[key]) == DHCP_SERVER]
254
255 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
256 datadump['autogen_item'] = item
257
258 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
259 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
260 datadump_cache[item] = datadump.copy()
261 except Exception:
262 print "# Error while processing %s" % item
263 raise
264 return datadump
265
266
267def store_yaml(datadump, header=False):
268 """ Store configuration yaml for 'item'"""
269 item = datadump['autogen_item']
270 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
271
272 output = generate_wleiden_yaml(datadump, header)
273
274 f = open(gfile, 'w')
275 f.write(output)
276 f.close()
277
278
279def network(ip):
280 addr, mask = ip.split('/')
281 # Not parsing of these folks please
282 addr = parseaddr(addr)
283 mask = int(mask)
284 network = addr & ~((1 << (32 - mask)) - 1)
285 return network
286
287
288
289def make_relations():
290 """ Process _ALL_ yaml files to get connection relations """
291 global relations_cache
292
293 if relations_cache:
294 return relations_cache
295
296 errors = []
297 poel = defaultdict(list)
298
299 for host in get_hostlist():
300 datadump = get_yaml(host)
301 try:
302 for iface_key in get_interface_keys(datadump):
303 # Bridge members has no IP assigned
304 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
305 continue
306 net_addr = network(datadump[iface_key]['ip'])
307 poel[net_addr] += [(host,datadump[iface_key].copy())]
308 except (KeyError, ValueError) as e:
309 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
310 continue
311
312 relations_cache = (poel, errors)
313 return relations_cache
314
315
316
317def valid_addr(addr):
318 """ Show which address is valid in which are not """
319 return str(addr).startswith('172.')
320
321def get_hostlist():
322 """ Combined hosts and proxy list"""
323 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
324
325def angle_between_points(lat1,lat2,long1,long2):
326 """
327 Return Angle in radians between two GPS coordinates
328 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
329 """
330 dy = lat2 - lat1
331 dx = math.cos(lat1)*(long2 - long1)
332 angle = math.atan2(dy,dx)
333 return angle
334
335
336
337def angle_to_cd(angle):
338 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
339
340 # For easy conversion get positive degree
341 degrees = math.degrees(angle)
342 abs_degrees = 360 + degrees if degrees < 0 else degrees
343
344 # Numbers can be confusing calculate from the 4 main directions
345 p = 22.5
346 if abs_degrees < p:
347 cd = "n"
348 elif abs_degrees < (90 - p):
349 cd = "no"
350 elif abs_degrees < (90 + p):
351 cd = "o"
352 elif abs_degrees < (180 - p):
353 cd = "zo"
354 elif abs_degrees < (180 + p):
355 cd = "z"
356 elif abs_degrees < (270 - p):
357 cd = "zw"
358 elif abs_degrees < (270 + p):
359 cd = "w"
360 elif abs_degrees < (360 - p):
361 cd = "nw"
362 else:
363 cd = "n"
364 return cd
365
366
367
368def cd_between_hosts(hostA, hostB, datadumps):
369 # Using RDNAP coordinates
370 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
371 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
372 return angle_to_cd(math.atan2(dx,dy))
373
374 # GPS coordinates seems to fail somehow
375 #latA = float(datadumps[hostA]['latitude'])
376 #latB = float(datadumps[hostB]['latitude'])
377 #lonA = float(datadumps[hostA]['longitude'])
378 #lonB = float(datadumps[hostB]['longitude'])
379 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
380
381
382def generate_title(nodelist):
383 """ Main overview page """
384 items = {'root' : "." }
385 def fl(spaces, line):
386 return (' ' * spaces) + line + '\n'
387
388 output = """
389<html>
390 <head>
391 <title>Wireless leiden Configurator - GFormat</title>
392 <style type="text/css">
393 th {background-color: #999999}
394 tr:nth-child(odd) {background-color: #cccccc}
395 tr:nth-child(even) {background-color: #ffffff}
396 th, td {padding: 0.1em 1em}
397 </style>
398 </head>
399 <body>
400 <center>
401 <form type="GET" action="%(root)s">
402 <input type="hidden" name="action" value="update">
403 <input type="submit" value="Update Configuration Database (SVN)">
404 </form>
405 <table>
406 <caption><h3>Wireless Leiden Configurator</h3></caption>
407 """ % items
408
409 for node in nodelist:
410 items['node'] = node
411 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
412 for config in files:
413 items['config'] = config
414 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
415 output += fl(5, "</tr>")
416 output += """
417 </table>
418 <hr />
419 <em>%s</em>
420 </center>
421 </body>
422</html>
423 """ % __version__
424
425 return output
426
427
428
429def generate_node(node):
430 """ Print overview of all files available for node """
431 return "\n".join(files)
432
433def generate_node_overview(host, datadump=False):
434 """ Print overview of all files available for node """
435 if not datadump:
436 datadump = get_yaml(host)
437 params = { 'host' : host }
438 output = "<em><a href='..'>Back to overview</a></em><hr />"
439 output += "<h2>Available files:</h2><ul>"
440 for cf in files:
441 params['cf'] = cf
442 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
443 output += "</ul>"
444
445 # Generate and connection listing
446 output += "<h2>Connected To:</h2><ul>"
447 (poel, errors) = make_relations()
448 for network, hosts in poel.iteritems():
449 if host in [x[0] for x in hosts]:
450 if len(hosts) == 1:
451 # Single not connected interface
452 continue
453 for remote,ifacedump in hosts:
454 if remote == host:
455 # This side of the interface
456 continue
457 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
458 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
459 output += "</ul>"
460 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
461
462 output += "<hr /><em><a href='..'>Back to overview</a></em>"
463 return output
464
465
466def generate_header(datadump, ctag="#"):
467 return """\
468%(ctag)s
469%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
470%(ctag)s
471""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
472
473
474
475def parseaddr(s):
476 """ Process IPv4 CIDR notation addr to a (binary) number """
477 f = s.split('.')
478 return (long(f[0]) << 24L) + \
479 (long(f[1]) << 16L) + \
480 (long(f[2]) << 8L) + \
481 long(f[3])
482
483
484
485def showaddr(a):
486 """ Display IPv4 addr in (dotted) CIDR notation """
487 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
488
489
490def is_member(ip, mask, canidate):
491 """ Return True if canidate is part of ip/mask block"""
492 ip_addr = parseaddr(ip)
493 ip_canidate = parseaddr(canidate)
494 mask = int(mask)
495 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
496 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
497 return ip_addr == ip_canidate
498
499
500
501def cidr2netmask(netmask):
502 """ Given a 'netmask' return corresponding CIDR """
503 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
504
505def get_network(addr, mask):
506 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
507
508
509def generate_dhcpd_conf(datadump):
510 """ Generate config file '/usr/local/etc/dhcpd.conf """
511 # Redundency support, in cause local DNS server is not running/responding.
512 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
513 output = generate_header(datadump)
514 output += Template("""\
515# option definitions common to all supported networks...
516option domain-name "dhcp.{{ autogen_fqdn }}";
517
518default-lease-time 600;
519max-lease-time 7200;
520
521# Use this to enble / disable dynamic dns updates globally.
522#ddns-update-style none;
523
524# If this DHCP server is the official DHCP server for the local
525# network, the authoritative directive should be uncommented.
526authoritative;
527
528# Use this to send dhcp log messages to a different log file (you also
529# have to hack syslog.conf to complete the redirection).
530log-facility local7;
531
532# UniFi Discovery Support
533option space ubnt;
534option ubnt.unifi-address code 1 = ip-address;
535#
536class "ubnt" {
537 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
538 option vendor-class-identifier "ubnt";
539 vendor-option-space ubnt;
540}
541
542#
543# Interface definitions
544#
545\n\n""").render(datadump)
546
547
548 # TODO: Use textwrap.fill instead
549 def indent(text, count):
550 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
551
552 # Process DHCP blocks
553 dhcp_out = defaultdict(list)
554 for iface_key in get_interface_keys(datadump):
555 ifname = datadump[iface_key]['autogen_ifbase']
556 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
557 if not datadump[iface_key].has_key('comment'):
558 datadump[iface_key]['comment'] = None
559
560 if not datadump[iface_key].has_key('ip'):
561 continue
562
563 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
564
565 (addr, mask) = datadump[iface_key]['ip'].split('/')
566 datadump[iface_key]['autogen_addr'] = addr
567 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
568 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
569
570 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
571 dhcp_out[groupif].append(textwrap.dedent("""\
572 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
573 ### not autoritive
574 }
575 """ % datadump[iface_key]))
576 continue
577
578 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
579 dhcp_part = ".".join(addr.split('.')[0:3])
580 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
581 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
582 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
583
584 # Assume the first 10 IPs could be used for static entries
585 if 'no_portal' in datadump:
586 fixed = 5
587 for mac in datadump['no_portal']:
588 dhcp_out[groupif].append(textwrap.dedent("""\
589 host fixed-%(ifname)s-%(fixed)s {
590 hardware ethernet %(mac)s;
591 fixed-address %(prefix)s.%(fixed)s;
592 }
593 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
594 fixed += 1
595
596 if 'dhcp_fixed' in datadump[iface_key]:
597 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
598 dhcp_out[groupif].append(textwrap.dedent("""\
599 host fixed-%(host)s {
600 hardware ethernet %(mac)s;
601 fixed-address %(addr)s;
602 }
603 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
604
605
606 dhcp_out[groupif].append(textwrap.dedent("""\
607 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
608 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
609 option routers %(autogen_addr)s;
610 option domain-name-servers %(autogen_dns_servers)s;
611 option ubnt.unifi-address 172.17.107.10;
612 }
613 """ % datadump[iface_key]))
614
615 # Output the blocks in groups
616 for ifname,value in sorted(dhcp_out.iteritems()):
617 if len(value) > 2:
618 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
619 else:
620 output += ''.join(value) + "\n\n"
621 return output
622
623
624
625def generate_dnsmasq_conf(datadump):
626 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
627 output = generate_header(datadump)
628 output += Template("""\
629# DHCP server options
630dhcp-authoritative
631dhcp-fqdn
632domain=dhcp.{{ autogen_fqdn }}
633domain-needed
634expand-hosts
635log-async=100
636
637# Low memory footprint
638cache-size=10000
639
640\n""").render(datadump)
641
642 for iface_key in get_interface_keys(datadump):
643 if not datadump[iface_key].has_key('comment'):
644 datadump[iface_key]['comment'] = None
645 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
646
647 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
648 output += "# not autoritive\n\n"
649 continue
650
651 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
652 (ip, cidr) = datadump[iface_key]['ip'].split('/')
653 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
654
655 dhcp_part = ".".join(ip.split('.')[0:3])
656 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
657 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
658 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
659
660 return output
661
662
663class AutoVivification(dict):
664 """Implementation of perl's autovivification feature."""
665 def __getitem__(self, item):
666 try:
667 return dict.__getitem__(self, item)
668 except KeyError:
669 value = self[item] = type(self)()
670 return value
671
672def make_interface_list(datadump):
673 if interface_list_cache.has_key(datadump['autogen_item']):
674 return (interface_list_cache[datadump['autogen_item']])
675 # lo0 configuration:
676 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
677 # - masterip is special as it needs to be assigned to at
678 # least one interface, so if not used assign to lo0
679 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
680 vlan_list = defaultdict(list)
681 bridge_list = defaultdict(list)
682 flags_if = AutoVivification()
683 dhclient_if = {'lo0' : False}
684
685 # XXX: Find some way of send this output nicely
686 output = ''
687
688 masterip_used = False
689 for iface_key in get_interface_keys(datadump):
690 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
691 masterip_used = True
692 break
693 if not masterip_used:
694 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
695
696 if 'serviceid' in datadump:
697 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
698
699 for iface_key in get_interface_keys(datadump):
700 ifacedump = datadump[iface_key]
701 if ifacedump['autogen_bridge_alias']:
702 ifname = ifacedump['autogen_ifbase']
703 else:
704 ifname = ifacedump['autogen_ifname']
705
706 # If defined as vlan interface
707 if ifacedump['autogen_vlan']:
708 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
709
710 # If defined as bridge interface
711 if ifacedump['autogen_bridge_member']:
712 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
713
714
715 # Flag dhclient is possible
716 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
717 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
718
719 # Ethernet address
720 if ifacedump.has_key('ether'):
721 flags_if[ifname]['ether'] = ifacedump['ether']
722
723 # Add interface IP to list
724 if ifacedump.has_key('ip'):
725 item = (ifacedump['ip'], ifacedump['comment'])
726 if addrs_list.has_key(ifname):
727 addrs_list[ifname].append(item)
728 else:
729 addrs_list[ifname] = [item]
730
731 # Alias only needs IP assignment for now, this might change if we
732 # are going to use virtual accesspoints
733 if "alias" in iface_key:
734 continue
735
736 # XXX: Might want to deduct type directly from interface name
737 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
738 # Default to station (client) mode
739 ifacedump['autogen_wlanmode'] = "sta"
740 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
741 ifacedump['autogen_wlanmode'] = "ap"
742
743 if not ifacedump.has_key('channel'):
744 if ifacedump['type'] == '11a':
745 ifacedump['channel'] = 36
746 else:
747 ifacedump['channel'] = 1
748
749 # Allow special hacks at the back like wds and stuff
750 if not ifacedump.has_key('extra'):
751 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
752 else:
753 ifacedump['autogen_extra'] = ifacedump['extra']
754
755 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
756
757 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
758 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
759 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
760 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
761 output += "\n"
762
763 elif ifacedump['type'] in ['ethernet', 'eth']:
764 # No special config needed besides IP
765 pass
766 elif ifacedump['type'] in ['vlan']:
767 # VLAN member has no special configuration
768 pass
769 else:
770 assert False, "Unknown type " + ifacedump['type']
771
772 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
773 interface_list_cache[datadump['autogen_item']] = store
774 return(store)
775
776
777
778def create_proxies_list():
779 if not ileiden_proxies or not normal_proxies:
780 # Placeholder for to-be-installed proxies, this will avoid updating the all
781 # nodes to include this new machine, yet due to an unbound issue, this list
782 # has to be kept small.
783
784 for i in range(1,20):
785 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
786
787 for host in get_hostlist():
788 hostdump = get_yaml(host)
789 if hostdump['status'] == 'up':
790 if hostdump['service_proxy_ileiden']:
791 ileiden_proxies[hostdump['serviceid']] = hostdump
792 if hostdump['service_proxy_normal']:
793 normal_proxies.append(hostdump)
794
795
796
797def generate_rc_conf_local(datadump):
798 """ Generate configuration file '/etc/rc.conf.local' """
799 item = datadump['autogen_item']
800 if rc_conf_local_cache.has_key(item):
801 return rc_conf_local_cache[item]
802
803 if not datadump.has_key('ileiden'):
804 datadump['autogen_ileiden_enable'] = False
805 else:
806 datadump['autogen_ileiden_enable'] = datadump['ileiden']
807
808 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
809
810 create_proxies_list()
811 datadump['autogen_ileiden_proxies'] = ileiden_proxies
812 datadump['autogen_normal_proxies'] = normal_proxies
813 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
814 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
815 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
816 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
817
818 output = generate_header(datadump, "#");
819 output += render_template(datadump, """\
820hostname='{{ autogen_fqdn }}'
821location='{{ location }}'
822nodetype="{{ nodetype }}"
823
824#
825# Configured listings
826#
827captive_portal_whitelist=""
828{% if nodetype == "Proxy" %}
829#
830# Proxy Configuration
831#
832{% if gateway and service_proxy_ileiden -%}
833defaultrouter="{{ gateway }}"
834{% else -%}
835#defaultrouter="NOTSET"
836{% endif -%}
837internalif="{{ internalif }}"
838ileiden_enable="{{ autogen_ileiden_enable }}"
839gateway_enable="{{ autogen_ileiden_enable }}"
840pf_enable="yes"
841pf_rules="/etc/pf.conf"
842{% if autogen_ileiden_enable -%}
843pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
844lvrouted_enable="{{ autogen_ileiden_enable }}"
845lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
846{% else -%}
847pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
848{% endif -%}
849{% if internalroute -%}
850static_routes="wleiden"
851route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
852{% endif -%}
853
854{% elif nodetype == "Hybrid" %}
855 #
856 # Hybrid Configuration
857 #
858 list_ileiden_proxies="
859 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
860 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
861 {% endfor -%}
862 "
863 list_normal_proxies="
864 {% for item in autogen_normal_proxies -%}
865 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
866 {% endfor -%}
867 "
868
869 {% if autogen_dhcp_interfaces -%}
870 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
871 {% else %}
872 captive_portal_interfaces="dummy"
873 {% endif %}
874 externalif="{{ externalif|default('vr0', true) }}"
875 masterip="{{ masterip }}"
876
877 {% if gateway and service_proxy_ileiden %}
878 defaultrouter="{{ gateway }}"
879 {% else %}
880 #defaultrouter="NOTSET"
881 {% endif %}
882
883 #
884 # Defined services
885 #
886 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
887 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
888 service_accesspoint="{{ service_accesspoint|yesorno }}"
889 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
890 service_concentrator="{{ service_concentrator|yesorno }}"
891
892 {% if service_proxy_ileiden %}
893 pf_rules="/etc/pf.hybrid.conf"
894 {% if service_concentrator %}
895 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
896 {% else %}
897 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
898 {% endif %}
899 pf_flags="$pf_flags -D publicnat=80,443"
900 lvrouted_flags="$lvrouted_flags -g"
901 {% elif service_proxy_normal or service_incoming_rdr %}
902 pf_rules="/etc/pf.hybrid.conf"
903 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
904 pf_flags="$pf_flags -D publicnat=0"
905 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
906 named_setfib="1"
907 tinyproxy_setfib="1"
908 dnsmasq_setfib="1"
909 sshd_setfib="1"
910 {% else %}
911 named_auto_forward_only="YES"
912 pf_rules="/etc/pf.node.conf"
913 pf_flags=""
914 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
915 {% endif %}
916 {% if service_concentrator %}
917 # Do mind installing certificates is NOT done automatically for security reasons
918 openvpn_enable="YES"
919 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
920 {% endif %}
921
922 {% if service_proxy_normal %}
923 tinyproxy_enable="yes"
924 {% else %}
925 pen_wrapper_enable="yes"
926 {% endif %}
927
928 {% if service_accesspoint %}
929 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
930 {% endif %}
931
932 {% if board == "ALIX2" or board == "net4801" %}
933 #
934 # ''Fat'' configuration, board has 256MB RAM
935 #
936 dnsmasq_enable="NO"
937 named_enable="YES"
938 unbound_enable="YES"
939 {% if autogen_dhcp_interfaces -%}
940 dhcpd_enable="YES"
941 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
942 {% endif -%}
943 {% elif board == "apu1d" %}
944 #
945 # ''Fat'' configuration, board has 1024MB RAM
946 #
947 dnsmasq_enable="NO"
948 unbound_enable="YES"
949 {% if autogen_dhcp_interfaces -%}
950 dhcpd_enable="YES"
951 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
952 {% endif -%}
953 {% endif -%}
954{% endif %}
955
956#
957# Script variables
958#
959attached_devices="{{ autogen_attached_devices|join(' ') }}"
960neighbours="{{ autogen_neighbours|join(' ') }}"
961
962
963#
964# Interface definitions
965#\n
966""")
967
968 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
969 for iface, vlans in sorted(vlan_list.items()):
970 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
971
972 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
973 for iface in sorted(vlan_list.keys()):
974 if not iface in addrs_list.keys():
975 output += "ifconfig_%s='up'\n" % iface
976
977 output += "\n"
978
979 # Bridge configuration:
980 if bridge_list.keys():
981 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
982
983 for iface in bridge_list.keys():
984 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
985
986 # Bridge member interfaces not containing a configuration should be marked active explcitly.
987 for _,members in bridge_list.items():
988 for iface in members:
989 if not iface in addrs_list.keys():
990 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
991
992 output += "\n"
993
994 # Details like SSID
995 if extra_ouput:
996 output += extra_ouput.strip() + "\n"
997
998 # Print IP address which needs to be assigned over here
999 output += "\n"
1000 for iface,addrs in sorted(addrs_list.iteritems()):
1001 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1002 output += "# %s || %s || %s\n" % (iface, addr, comment)
1003
1004 # Write DHCLIENT entry
1005 if iface in dhclient_if and dhclient_if[iface]:
1006 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1007 continue
1008
1009 # Make sure the external address is always first as this is needed in the
1010 # firewall setup
1011 addrs = sorted(
1012 [x for x in addrs if not '0.0.0.0' in x[0]],
1013 key=lambda x: x[0].split('.')[0],
1014 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1015 )
1016
1017 idx_offset = 0
1018 # Set MAC is required
1019 if flags_if[iface].has_key('ether'):
1020 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1021 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1022 idx_offset += 1
1023 elif iface in bridge_list:
1024 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1025 idx_offset += 1
1026 else:
1027 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1028
1029 for idx, addr in enumerate(addrs[1:]):
1030 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1031
1032 output += "\n"
1033
1034 rc_conf_local_cache[datadump['autogen_item']] = output
1035 return output
1036
1037
1038
1039def get_all_configs():
1040 """ Get dict with key 'host' with all configs present """
1041 configs = dict()
1042 for host in get_hostlist():
1043 datadump = get_yaml(host)
1044 configs[host] = datadump
1045 return configs
1046
1047
1048def get_interface_keys(config, extra=False):
1049 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1050 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1051 if extra == False:
1052 return filter(lambda x: not "extra" in x, elems)
1053 else:
1054 return elems
1055
1056
1057def get_used_ips(configs):
1058 """ Return array of all IPs used in config files"""
1059 ip_list = []
1060 for config in configs:
1061 ip_list.append(config['masterip'])
1062 if 'serviceid' in config:
1063 ip_list.append(config['serviceid'])
1064 for iface_key in get_interface_keys(config, True):
1065 l = config[iface_key]['ip']
1066 addr, mask = l.split('/')
1067 # Special case do not process
1068 if valid_addr(addr):
1069 ip_list.append(addr)
1070 else:
1071 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1072 return sorted(ip_list)
1073
1074
1075
1076def get_nameservers(max_servers=None):
1077 if nameservers_cache:
1078 return nameservers_cache[0:max_servers]
1079
1080 for host in get_hostlist():
1081 hostdump = get_yaml(host)
1082 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1083 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1084
1085 return nameservers_cache[0:max_servers]
1086
1087
1088def get_neighbours(datadump):
1089 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1090
1091 (poel, errors) = make_relations()
1092 table = []
1093 for iface,addrs in sorted(addrs_list.iteritems()):
1094 if iface in ['lo0']:
1095 continue
1096
1097 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1098 if not addr.startswith('172.'):
1099 # Avoid listing internet connections as pool
1100 continue
1101 for neighbour in poel[network(addr)]:
1102 if neighbour[0] != datadump['autogen_item']:
1103 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1104 return table
1105
1106
1107def get_attached_devices(datadump, url=False):
1108 table = []
1109 for iface_key in get_interface_keys(datadump, True):
1110 ifacedump = datadump[iface_key]
1111
1112 if not ifacedump.has_key('ns_ip'):
1113 continue
1114
1115 x_ip = ifacedump['ns_ip'].split('/')[0]
1116
1117 if 'mode' in ifacedump:
1118 x_mode = ifacedump['mode']
1119 else:
1120 x_mode = 'unknown'
1121
1122 if 'bridge_type' in ifacedump:
1123 device_type = ifacedump['bridge_type']
1124 else:
1125 device_type = 'Unknown'
1126
1127 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1128 return table
1129
1130
1131def generate_resolv_conf(datadump):
1132 """ Generate configuration file '/etc/resolv.conf' """
1133 # XXX: This should properly going to be an datastructure soon
1134 datadump['autogen_header'] = generate_header(datadump, "#")
1135 datadump['autogen_edge_nameservers'] = ''
1136
1137
1138 for masterip,realname in get_nameservers():
1139 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1140
1141 return Template("""\
1142{{ autogen_header }}
1143search wleiden.net
1144
1145# Try local (cache) first
1146nameserver 127.0.0.1
1147
1148{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1149nameserver 8.8.8.8 # Google Public NameServer
1150nameserver 64.6.64.6 # Verisign Public NameServer
1151{% else -%}
1152# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1153{{ autogen_edge_nameservers }}
1154{% endif -%}
1155""").render(datadump)
1156
1157
1158
1159def generate_ntp_conf(datadump):
1160 """ Generate configuration file '/etc/ntp.conf' """
1161 # XXX: This should properly going to be an datastructure soon
1162
1163 datadump['autogen_header'] = generate_header(datadump, "#")
1164 datadump['autogen_ntp_servers'] = ''
1165 for host in get_hostlist():
1166 hostdump = get_yaml(host)
1167 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1168 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1169
1170 return Template("""\
1171{{ autogen_header }}
1172
1173{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1174# Machine hooked to internet.
1175server 0.nl.pool.ntp.org iburst maxpoll 9
1176server 1.nl.pool.ntp.org iburst maxpoll 9
1177server 2.nl.pool.ntp.org iburst maxpoll 9
1178server 3.nl.pool.ntp.org iburst maxpoll 9
1179{% else -%}
1180# Local Wireless Leiden NTP Servers.
1181server 0.pool.ntp.wleiden.net iburst maxpoll 9
1182server 1.pool.ntp.wleiden.net iburst maxpoll 9
1183server 2.pool.ntp.wleiden.net iburst maxpoll 9
1184server 3.pool.ntp.wleiden.net iburst maxpoll 9
1185
1186# All the configured NTP servers
1187{{ autogen_ntp_servers }}
1188{% endif %}
1189
1190# If a server loses sync with all upstream servers, NTP clients
1191# no longer follow that server. The local clock can be configured
1192# to provide a time source when this happens, but it should usually
1193# be configured on just one server on a network. For more details see
1194# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1195# The use of Orphan Mode may be preferable.
1196#
1197server 127.127.1.0
1198fudge 127.127.1.0 stratum 10
1199""").render(datadump)
1200
1201
1202def generate_pf_hybrid_conf_local(datadump):
1203 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1204 datadump['autogen_header'] = generate_header(datadump, "#")
1205 if datadump['service_incoming_rdr']:
1206 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1207 return Template("""\
1208{{ autogen_header }}
1209
1210# Redirect some internal facing services outside (7)
1211# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1212{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1213rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1214{% endfor -%}
1215# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1216{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1217rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1218{% endfor -%}
1219""").render(datadump)
1220
1221def generate_unbound_wleiden_conf(datadump):
1222 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1223 datadump['autogen_header'] = generate_header(datadump, "#")
1224
1225 autogen_ips = []
1226 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1227 for iface,addrs in sorted(addrs_list.iteritems()):
1228 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1229 if addr.startswith('172'):
1230 autogen_ips.append((addr.split('/')[0], comment))
1231 datadump['autogen_ips'] = autogen_ips
1232
1233 create_proxies_list()
1234 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1235 return Template("""\
1236{{ autogen_header }}
1237
1238server:
1239 ## Static definitions fail to start on systems with broken ue(4) interfaces
1240{%- for ip,comment in autogen_ips %}
1241 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1242{%- endfor %}
1243 ## Enabling wildcard matching as work-around
1244 interface: 0.0.0.0
1245 interface: ::0
1246
1247forward-zone:
1248 name: '.'
1249{%- if service_proxy_ileiden %}
1250 forward-addr: 8.8.8.8 # Google DNS A
1251 forward-addr: 8.8.4.4 # Google DNS B
1252 forward-addr: 208.67.222.222 # OpenDNS DNS A
1253 forward-addr: 208.67.220.220 # OpenDNS DNS B
1254{% else -%}
1255{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1256 {%- if loop.index <= 5 %}
1257 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1258 {%- endif %}
1259{%- endfor %}
1260{%- endif %}
1261""").render(datadump)
1262
1263def generate_motd(datadump):
1264 """ Generate configuration file '/etc/motd' """
1265 output = Template("""\
1266FreeBSD run ``service motd onestart'' to make me look normal
1267
1268 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1269 Loc: {{ location }}
1270
1271Services:
1272{% if board == "ALIX2" or board == "net4801" -%}
1273{{" -"}} Core Node ({{ board }})
1274{% else -%}
1275{{" -"}} Hulp Node ({{ board }})
1276{% endif -%}
1277{% if service_proxy_normal -%}
1278{{" -"}} Normal Proxy
1279{% endif -%}
1280{% if service_proxy_ileiden -%}
1281{{" -"}} iLeiden Proxy
1282{% endif -%}
1283{% if service_incoming_rdr -%}
1284{{" -"}} Incoming port redirects
1285{% endif %}
1286Interlinks:\n
1287""").render(datadump)
1288
1289
1290 def make_table(table):
1291 if not table:
1292 return " - none\n"
1293 else:
1294 lines = ""
1295 col_width = [max(len(x) for x in col) for col in zip(*table)]
1296 for row in table:
1297 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1298 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1299 return lines
1300
1301 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1302 table = []
1303 for iface,addrs in sorted(addrs_list.iteritems()):
1304 if iface in ['lo0']:
1305 continue
1306 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1307 table.append((iface, addr, comment))
1308
1309 output += make_table(table)
1310 output += '\n'
1311 output += """\
1312Attached devices:
1313"""
1314 output += make_table(get_attached_devices(datadump, url=True))
1315 output += '\n'
1316 output += """\
1317Available neighbours:
1318"""
1319 output += make_table(get_neighbours(datadump))
1320
1321 return output
1322
1323
1324def format_yaml_value(value):
1325 """ Get yaml value in right syntax for outputting """
1326 if isinstance(value,str):
1327 output = '"%s"' % value
1328 else:
1329 output = value
1330 return output
1331
1332
1333
1334def format_wleiden_yaml(datadump):
1335 """ Special formatting to ensure it is editable"""
1336 output = "# Genesis config yaml style\n"
1337 output += "# vim:ts=2:et:sw=2:ai\n"
1338 output += "#\n"
1339 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1340 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1341 if key == 'rdr_rules':
1342 output += '%-10s:\n' % 'rdr_rules'
1343 for rdr_rule in datadump[key]:
1344 output += '- %s\n' % rdr_rule
1345 else:
1346 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1347
1348 output += "\n\n"
1349
1350 # Format (key, required)
1351 key_order = (
1352 ('comment', True),
1353 ('parent', False),
1354 ('ip', False),
1355 ('ether', False),
1356 ('desc', True),
1357 ('sdesc', True),
1358 ('mode', True),
1359 ('type', True),
1360 ('extra_type', False),
1361 ('channel', False),
1362 ('ssid', False),
1363 ('wlan_mac', False),
1364 ('dhcp', True),
1365 ('dhcp_fixed', False),
1366 ('compass', False),
1367 ('distance', False),
1368 ('ns_ip', False),
1369 ('repeater_ip', False),
1370 ('bullet2_ip', False),
1371 ('ns_mac', False),
1372 ('bullet2_mac', False),
1373 ('ns_type', False),
1374 ('bridge_type', False),
1375 ('status', True),
1376 )
1377
1378 for iface_key in sorted(iface_keys):
1379 try:
1380 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1381 if remainder:
1382 raise KeyError("invalid keys: %s" % remainder)
1383
1384 output += "%s:\n" % iface_key
1385 for key,required in key_order:
1386 if datadump[iface_key].has_key(key):
1387 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1388 output += "\n\n"
1389 except Exception:
1390 print "# Error while processing interface %s" % iface_key
1391 raise
1392
1393 return output
1394
1395
1396
1397def generate_wleiden_yaml(datadump, header=True):
1398 """ Generate (petty) version of wleiden.yaml"""
1399 output = generate_header(datadump, "#") if header else ''
1400
1401 for key in datadump.keys():
1402 if key.startswith('autogen_'):
1403 del datadump[key]
1404 # Interface autogen cleanups
1405 elif type(datadump[key]) == dict:
1406 for key2 in datadump[key].keys():
1407 if key2.startswith('autogen_'):
1408 del datadump[key][key2]
1409
1410 output += format_wleiden_yaml(datadump)
1411 return output
1412
1413def generate_nanostation_config(datadump, iface, ns_type):
1414 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1415 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1416
1417 datadump.update(datadump['iface_%s' % iface])
1418
1419 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1420
1421def generate_yaml(datadump):
1422 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1423
1424
1425
1426def generate_config(node, config, datadump=None):
1427 """ Print configuration file 'config' of 'node' """
1428 output = ""
1429 try:
1430 # Load config file
1431 if datadump == None:
1432 datadump = get_yaml(node)
1433
1434 if config == 'wleiden.yaml':
1435 output += generate_wleiden_yaml(datadump)
1436 elif config == 'authorized_keys':
1437 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1438 output += f.read()
1439 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1440 # Fetch local keys if existing
1441 if os.path.exists(node_keys):
1442 output += open(node_keys, 'r').read()
1443 f.close()
1444 elif config == 'dnsmasq.conf':
1445 output += generate_dnsmasq_conf(datadump)
1446 elif config == 'dhcpd.conf':
1447 output += generate_dhcpd_conf(datadump)
1448 elif config == 'rc.conf.local':
1449 output += generate_rc_conf_local(datadump)
1450 elif config == 'resolv.conf':
1451 output += generate_resolv_conf(datadump)
1452 elif config == 'ntp.conf':
1453 output += generate_ntp_conf(datadump)
1454 elif config == 'motd':
1455 output += generate_motd(datadump)
1456 elif config == 'pf.hybrid.conf.local':
1457 output += generate_pf_hybrid_conf_local(datadump)
1458 elif config == 'unbound.wleiden.conf':
1459 output += generate_unbound_wleiden_conf(datadump)
1460 elif config.startswith('vr'):
1461 interface, ns_type = config.strip('.yaml').split('-')
1462 output += generate_nanostation_config(datadump, interface, ns_type)
1463 else:
1464 assert False, "Config not found!"
1465 except IOError, e:
1466 output += "[ERROR] Config file not found"
1467 return output
1468
1469
1470
1471def process_cgi_request(environ=os.environ):
1472 """ When calling from CGI """
1473 response_headers = []
1474 content_type = 'text/plain'
1475
1476 # Update repository if requested
1477 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1478 if form and form.has_key("action") and "update" in form["action"]:
1479 output = "[INFO] Updating subverion, please wait...\n"
1480 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1481 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1482 output += "[INFO] All done, redirecting in 5 seconds"
1483 response_headers += [
1484 ('Refresh', '5; url=.'),
1485 ]
1486 reload_cache()
1487 else:
1488 base_uri = environ['REQUEST_URI']
1489 uri = base_uri.strip('/').split('/')[1:]
1490
1491 output = "Template Holder"
1492 if base_uri.endswith('/create/network.kml'):
1493 content_type='application/vnd.google-earth.kml+xml'
1494 output = make_network_kml.make_graph()
1495 elif base_uri.endswith('/api/get/nodeplanner.json'):
1496 content_type='application/json'
1497 output = make_network_kml.make_nodeplanner_json()
1498 elif not uri:
1499 if is_text_request(environ):
1500 output = '\n'.join(get_hostlist())
1501 else:
1502 content_type = 'text/html'
1503 output = generate_title(get_hostlist())
1504 elif len(uri) == 1:
1505 if is_text_request(environ):
1506 output = generate_node(uri[0])
1507 else:
1508 content_type = 'text/html'
1509 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1510 elif len(uri) == 2:
1511 output = generate_config(uri[0], uri[1])
1512 else:
1513 assert False, "Invalid option"
1514
1515 # Return response
1516 response_headers += [
1517 ('Content-type', content_type),
1518 ('Content-Length', str(len(output))),
1519 ]
1520 return(response_headers, str(output))
1521
1522
1523def make_dns(output_dir = 'dns', external = False):
1524 items = dict()
1525
1526 # hostname is key, IP is value
1527 wleiden_zone = defaultdict(list)
1528 wleiden_cname = dict()
1529
1530 pool = dict()
1531 for node in get_hostlist():
1532 datadump = get_yaml(node)
1533
1534 fqdn = datadump['nodename']
1535
1536 if datadump.has_key('rdr_host'):
1537 remote_target = datadump['rdr_host']
1538 elif datadump.has_key('remote_access') and datadump['remote_access']:
1539 remote_target = datadump['remote_access'].split(':')[0]
1540 else:
1541 remote_target = None
1542
1543 if remote_target:
1544 try:
1545 parseaddr(remote_target)
1546 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1547 except (IndexError, ValueError):
1548 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1549
1550
1551 wleiden_zone[fqdn].append((datadump['masterip'], True))
1552
1553 # Hacking to get proper DHCP IPs and hostnames
1554 for iface_key in get_interface_keys(datadump):
1555 iface_name = iface_key.replace('_','-')
1556 if 'ip' in datadump[iface_key]:
1557 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1558 try:
1559 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1560 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1561 dhcp_part = ".".join(ip.split('.')[0:3])
1562 if ip != datadump['masterip']:
1563 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1564 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1565 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1566 except (AttributeError, ValueError, KeyError):
1567 # First push it into a pool, to indentify the counter-part later on
1568 addr = parseaddr(ip)
1569 cidr = int(cidr)
1570 addr = addr & ~((1 << (32 - cidr)) - 1)
1571 if pool.has_key(addr):
1572 pool[addr] += [(iface_name, fqdn, ip)]
1573 else:
1574 pool[addr] = [(iface_name, fqdn, ip)]
1575 continue
1576
1577
1578
1579 # WL uses an /29 to configure an interface. IP's are ordered like this:
1580 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1581
1582 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1583
1584 # Automatic naming convention of interlinks namely 2 + remote.lower()
1585 for (key,value) in pool.iteritems():
1586 # Make sure they are sorted from low-ip to high-ip
1587 value = sorted(value, key=lambda x: parseaddr(x[2]))
1588
1589 if len(value) == 1:
1590 (iface_name, fqdn, ip) = value[0]
1591 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1592
1593 # Device DNS names
1594 if 'cnode' in fqdn.lower():
1595 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1596 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1597
1598 elif len(value) == 2:
1599 (a_iface_name, a_fqdn, a_ip) = value[0]
1600 (b_iface_name, b_fqdn, b_ip) = value[1]
1601 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1602 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1603
1604 # Device DNS names
1605 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1606 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1607 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1608 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1609 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1610 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1611 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1612
1613 else:
1614 pool_members = [k[1] for k in value]
1615 for item in value:
1616 (iface_name, fqdn, ip) = item
1617 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1618
1619 # Include static DNS entries
1620 # XXX: Should they override the autogenerated results?
1621 # XXX: Convert input to yaml more useable.
1622 # Format:
1623 ##; this is a comment
1624 ## roomburgh=Roomburgh1
1625 ## apkerk1.Vosko=172.17.176.8 ;this as well
1626 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1627
1628 # Hack to allow special entries, for development
1629 wleiden_raw = {}
1630
1631 for line in dns_list:
1632 reverse = False
1633 k, items = line.items()[0]
1634 if type(items) == dict:
1635 if items.has_key('reverse'):
1636 reverse = items['reverse']
1637 items = items['a']
1638 else:
1639 items = items['cname']
1640 items = [items] if type(items) != list else items
1641 for item in items:
1642 if item.startswith('IN '):
1643 wleiden_raw[k] = item
1644 elif valid_addr(item):
1645 wleiden_zone[k].append((item, reverse))
1646 else:
1647 wleiden_cname[k] = item
1648
1649 # Hack to get dynamic pool listing
1650 def chunks(l, n):
1651 return [l[i:i+n] for i in range(0, len(l), n)]
1652
1653 ntp_servers = [x[0] for x in get_nameservers()]
1654 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1655 for ntp_server in chunk:
1656 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1657
1658 details = dict()
1659 # 24 updates a day allowed
1660 details['serial'] = time.strftime('%Y%m%d%H')
1661
1662 if external:
1663 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1664 else:
1665 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1666
1667 details['master'] = dns_masters[0]
1668 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1669
1670 dns_header = '''
1671$TTL 3h
1672%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1673 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1674
1675%(ns_servers)s
1676 \n'''
1677
1678
1679 if not os.path.isdir(output_dir):
1680 os.makedirs(output_dir)
1681 details['zone'] = 'wleiden.net'
1682 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1683 f.write(dns_header % details)
1684
1685 for host,items in wleiden_zone.iteritems():
1686 for ip,reverse in items:
1687 if ip not in ['0.0.0.0']:
1688 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1689 for source,dest in wleiden_cname.iteritems():
1690 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1691 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1692 for source, dest in wleiden_raw.iteritems():
1693 f.write("%s.wleiden.net. %s\n" % (source, dest))
1694 f.close()
1695
1696 # Create whole bunch of specific sub arpa zones. To keep it compliant
1697 for s in range(16,32):
1698 details['zone'] = '%i.172.in-addr.arpa' % s
1699 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1700 f.write(dns_header % details)
1701
1702 #XXX: Not effient, fix to proper data structure and do checks at other
1703 # stages
1704 for host,items in wleiden_zone.iteritems():
1705 for ip,reverse in items:
1706 if not reverse:
1707 continue
1708 if valid_addr(ip):
1709 if valid_addr(ip):
1710 if int(ip.split('.')[1]) == s:
1711 rev_ip = '.'.join(reversed(ip.split('.')))
1712 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1713 f.close()
1714
1715
1716def usage():
1717 print """Usage: %(prog)s <argument>
1718Argument:
1719\tcleanup = Cleanup all YAML files to specified format
1720\tstandalone [port] = Run configurator webserver [8000]
1721\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1722\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1723\tfull-export = Generate yaml export script for heatmap.
1724\tstatic [outputdir] = Generate all config files and store on disk
1725\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1726\ttest <node> [<file>] = Receive output for certain node [all files].
1727\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1728\tlist <status> <items> = List systems which have certain status
1729\tcreate network.kml = Create Network KML file for use in Google Earth
1730
1731Arguments:
1732\t<node> = NodeName (example: HybridRick)
1733\t<file> = %(files)s
1734\t<status> = all|up|down|planned
1735\t<items> = systems|nodes|proxies
1736
1737NOTE FOR DEVELOPERS; you can test your changes like this:
1738 BEFORE any changes in this code:
1739 $ ./gformat.py static /tmp/pre
1740 AFTER the changes:
1741 $ ./gformat.py static /tmp/post
1742 VIEW differences and VERIFY all are OK:
1743 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1744""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1745 exit(0)
1746
1747
1748def is_text_request(environ=os.environ):
1749 """ Find out whether we are calling from the CLI or any text based CLI utility """
1750 try:
1751 return environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1752 except KeyError:
1753 return True
1754
1755def switchFormat(setting):
1756 if setting:
1757 return "YES"
1758 else:
1759 return "NO"
1760
1761def rlinput(prompt, prefill=''):
1762 import readline
1763 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1764 try:
1765 return raw_input(prompt)
1766 finally:
1767 readline.set_startup_hook()
1768
1769def fix_conflict(left, right, default='i'):
1770 while True:
1771 print "## %-30s | %-30s" % (left, right)
1772 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1773 if not c:
1774 c = default
1775
1776 if c in ['l','1']:
1777 return left
1778 elif c in ['r','2']:
1779 return right
1780 elif c in ['e', '3']:
1781 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1782 elif c in ['i', '4']:
1783 return None
1784 else:
1785 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1786
1787
1788
1789def print_cgi_response(response_headers, output):
1790 """Could we not use some kind of wsgi wrapper to make this output?"""
1791 for header in response_headers:
1792 print "%s: %s" % header
1793 print
1794 print output
1795
1796
1797def fill_cache():
1798 ''' Poor man re-loading of few cache items (the slow ones) '''
1799 for host in get_hostlist():
1800 get_yaml(host)
1801
1802
1803def reload_cache():
1804 clear_cache()
1805 fill_cache()
1806
1807
1808def main():
1809 """Hard working sub"""
1810 # Allow easy hacking using the CLI
1811 if not os.environ.has_key('REQUEST_URI'):
1812 if len(sys.argv) < 2:
1813 usage()
1814
1815 if sys.argv[1] == "standalone":
1816 import SocketServer
1817 import CGIHTTPServer
1818 # Hop to the right working directory.
1819 os.chdir(os.path.dirname(__file__))
1820 try:
1821 PORT = int(sys.argv[2])
1822 except (IndexError,ValueError):
1823 PORT = 8000
1824
1825 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1826 """ Serve this CGI from the root of the webserver """
1827 def is_cgi(self):
1828 if "favicon" in self.path:
1829 return False
1830
1831 self.cgi_info = (os.path.basename(__file__), self.path)
1832 self.path = ''
1833 return True
1834 handler = MyCGIHTTPRequestHandler
1835 SocketServer.TCPServer.allow_reuse_address = True
1836 httpd = SocketServer.TCPServer(("", PORT), handler)
1837 httpd.server_name = 'localhost'
1838 httpd.server_port = PORT
1839
1840 logger.info("serving at port %s", PORT)
1841 try:
1842 httpd.serve_forever()
1843 except KeyboardInterrupt:
1844 httpd.shutdown()
1845 logger.info("All done goodbye")
1846 elif sys.argv[1] == "test":
1847 # Basic argument validation
1848 try:
1849 node = sys.argv[2]
1850 except IndexError:
1851 print "Invalid argument"
1852 exit(1)
1853 except IOError as e:
1854 print e
1855 exit(1)
1856
1857 datadump = get_yaml(node)
1858
1859
1860 # Get files to generate
1861 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1862
1863 # Actual config generation
1864 for config in gen_files:
1865 logger.info("## Generating %s %s", node, config)
1866 print generate_config(node, config, datadump)
1867 elif sys.argv[1] == "test-cgi":
1868 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1869 os.environ['SCRIPT_NAME'] = __file__
1870 response_headers, output = process_cgi_request()
1871 print_cgi_response(response_headers, output)
1872 elif sys.argv[1] == "static":
1873 items = dict()
1874 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1875 for node in get_hostlist():
1876 items['node'] = node
1877 items['wdir'] = "%(output_dir)s/%(node)s" % items
1878 if not os.path.isdir(items['wdir']):
1879 os.makedirs(items['wdir'])
1880 datadump = get_yaml(node)
1881 f = open("%(wdir)s/index.html" % items, "w")
1882 f.write(generate_node_overview(items['node'], datadump))
1883 f.close()
1884 for config in files:
1885 items['config'] = config
1886 logger.info("## Generating %(node)s %(config)s" % items)
1887 f = open("%(wdir)s/%(config)s" % items, "w")
1888 f.write(generate_config(node, config, datadump))
1889 f.close()
1890 elif sys.argv[1] == "wind-export":
1891 items = dict()
1892 for node in get_hostlist():
1893 datadump = get_yaml(node)
1894 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1895 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1896 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1897 VALUES (
1898 (SELECT id FROM users WHERE username = 'rvdzwet'),
1899 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1900 'Y');""" % datadump
1901 #for config in files:
1902 # items['config'] = config
1903 # print "## Generating %(node)s %(config)s" % items
1904 # f = open("%(wdir)s/%(config)s" % items, "w")
1905 # f.write(generate_config(node, config, datadump))
1906 # f.close()
1907 for node in get_hostlist():
1908 datadump = get_yaml(node)
1909 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1910 ifacedump = datadump[iface_key]
1911 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1912 ifacedump['nodename'] = datadump['nodename']
1913 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1914 ifacedump['channel'] = 0
1915 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1916 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1917 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1918 elif sys.argv[1] == "nagios-export":
1919 try:
1920 heavy_load = (sys.argv[2] == "--heavy-load")
1921 except IndexError:
1922 heavy_load = False
1923
1924 hostgroup_details = {
1925 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1926 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1927 'walphen' : 'Stichting Wireless Alphen',
1928 'westeinder' : 'Westeinder Plassen',
1929 }
1930
1931 # Convert IP to Host
1932 ip2host = {'root' : 'root'}
1933 for host in get_hostlist():
1934 datadump = get_yaml(host)
1935 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1936 for iface in get_interface_keys(datadump):
1937 if datadump[iface].has_key('autogen_gateway'):
1938 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1939
1940 # Find dependency tree based on output of lvrouted.mytree of nearest node
1941 parents = defaultdict(list)
1942 stack = ['root']
1943 prev_depth = 0
1944 for line in open('lvrouted.mytree').readlines():
1945 depth = line.count('\t')
1946 ip = line.strip().split()[0]
1947
1948 if prev_depth < depth:
1949 try:
1950 parents[ip2host[ip]].append(ip2host[stack[-1]])
1951 except KeyError as e:
1952 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1953 stack.append(ip)
1954 elif prev_depth > depth:
1955 stack = stack[:(depth - prev_depth)]
1956 elif prev_depth == depth:
1957 try:
1958 parents[ip2host[ip]].append(ip2host[stack[-1]])
1959 except KeyError as e:
1960 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1961
1962
1963 prev_depth = depth
1964 # Observe that some nodes has themself as parent or multiple parents
1965 # for now take only the first parent, other behaviour is yet to be explained
1966
1967
1968
1969 params = {
1970 'check_interval' : 5 if heavy_load else 120,
1971 'retry_interval' : 1 if heavy_load else 10,
1972 'max_check_attempts' : 10 if heavy_load else 6,
1973 'notification_interval': 120 if heavy_load else 240,
1974 }
1975
1976 print '''\
1977define host {
1978 name wleiden-node ; Default Node Template
1979 use generic-host ; Use the standard template as initial starting point
1980 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1981 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1982 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1983 notification_interval %(notification_interval)s
1984 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1985 check_command check-host-alive ; Default command to check FreeBSD hosts
1986 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1987}
1988
1989define service {
1990 name wleiden-service ; Default Service Template
1991 use generic-service ; Use the standard template as initial starting point
1992 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1993 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1994 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1995 notification_interval %(notification_interval)s
1996 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1997 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1998}
1999
2000# Please make sure to install:
2001# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2002#
2003# Recompile net-mgmt/nagios-plugins to support check_snmp
2004# make -C /usr/ports/net-mgmt/nagios-plugins
2005#
2006# Install net/bind-tools to allow v2/check_dns_wl to work:
2007# pkg install bind-tools
2008#
2009define command{
2010 command_name check_snmp_disk
2011 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2012}
2013
2014define command{
2015 command_name check_netsnmp_load
2016 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2017}
2018
2019define command{
2020 command_name check_netsnmp_proc
2021 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2022}
2023
2024define command{
2025 command_name check_by_ssh
2026 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2027}
2028
2029define command{
2030 command_name check_dns_wl
2031 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2032}
2033
2034define command{
2035 command_name check_snmp_uptime
2036 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2037}
2038
2039
2040# TDB: dhcp leases
2041# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2042
2043# TDB: internet status
2044# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2045
2046# TDB: Advanced local passive checks
2047# /usr/local/libexec/nagios/check_by_ssh
2048''' % params
2049
2050 print '''\
2051# Service Group, not displayed by default
2052define hostgroup {
2053 hostgroup_name srv_hybrid
2054 alias All Hybrid Nodes
2055 register 0
2056}
2057
2058define service {
2059 use wleiden-service
2060 hostgroup_name srv_hybrid
2061 service_description SSH
2062 check_command check_ssh
2063}
2064
2065define service {
2066 use wleiden-service,service-pnp
2067 hostgroup_name srv_hybrid
2068 service_description HTTP
2069 check_command check_http
2070}
2071
2072define service {
2073 use wleiden-service
2074 hostgroup_name srv_hybrid
2075 service_description DNS
2076 check_command check_dns_wl!"www.wirelessleiden.nl"
2077}
2078'''
2079
2080 if heavy_load:
2081 print '''\
2082define service {
2083 use wleiden-service
2084 hostgroup_name srv_hybrid
2085 service_description UPTIME
2086 check_command check_snmp_uptime
2087}
2088
2089#define service {
2090# use wleiden-service
2091# hostgroup_name srv_hybrid
2092# service_description NTP
2093# check_command check_ntp_peer
2094#}
2095
2096define service {
2097 use wleiden-service
2098 hostgroup_name srv_hybrid
2099 service_description LOAD
2100 check_command check_netsnmp_load
2101}
2102
2103define service {
2104 use wleiden-service
2105 hostgroup_name srv_hybrid
2106 service_description PROC
2107 check_command check_netsnmp_proc
2108}
2109
2110define service {
2111 use wleiden-service
2112 hostgroup_name srv_hybrid
2113 service_description DISK
2114 check_command check_snmp_disk
2115}
2116'''
2117 for node in get_hostlist():
2118 datadump = get_yaml(node)
2119 if not datadump['status'] == 'up':
2120 continue
2121 if not hostgroup_details.has_key(datadump['monitoring_group']):
2122 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2123 print '''\
2124define host {
2125 use wleiden-node,host-pnp
2126 contact_groups admins
2127 host_name %(autogen_fqdn)s
2128 address %(masterip)s
2129 hostgroups srv_hybrid,%(monitoring_group)s\
2130''' % datadump
2131 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2132 print '''\
2133 parents %(parents)s\
2134''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2135 print '''\
2136}
2137'''
2138
2139
2140 for name,alias in hostgroup_details.iteritems():
2141 print '''\
2142define hostgroup {
2143 hostgroup_name %s
2144 alias %s
2145} ''' % (name, alias)
2146
2147
2148 elif sys.argv[1] == "full-export":
2149 hosts = {}
2150 for node in get_hostlist():
2151 datadump = get_yaml(node)
2152 hosts[datadump['nodename']] = datadump
2153 print yaml.dump(hosts)
2154
2155 elif sys.argv[1] == "dns":
2156 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2157 elif sys.argv[1] == "cleanup":
2158 # First generate all datadumps
2159 datadumps = dict()
2160 ssid_to_node = dict()
2161 for host in get_hostlist():
2162 logger.info("# Processing: %s", host)
2163 # Set some boring default values
2164 datadump = { 'board' : 'UNKNOWN' }
2165 datadump.update(get_yaml(host))
2166 datadumps[datadump['nodename']] = datadump
2167
2168 (poel, errors) = make_relations()
2169 print "\n".join(["# WARNING: %s" % x for x in errors])
2170
2171 for host,datadump in datadumps.iteritems():
2172 try:
2173 # Convert all yes and no to boolean values
2174 def fix_boolean(dump):
2175 for key in dump.keys():
2176 if type(dump[key]) == dict:
2177 dump[key] = fix_boolean(dump[key])
2178 elif str(dump[key]).lower() in ["yes", "true"]:
2179 dump[key] = True
2180 elif str(dump[key]).lower() in ["no", "false"]:
2181 # Compass richting no (Noord Oost) is valid input
2182 if key != "compass": dump[key] = False
2183 return dump
2184 datadump = fix_boolean(datadump)
2185
2186 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2187 if not 'latitude' in datadump and not 'longitude' in datadump:
2188 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2189 elif 'latitude' in datadump and 'longitude' in datadump:
2190 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2191 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2192 # TODO: Compare outcome of both coordinate systems and validate against each-other
2193
2194 if datadump['nodename'].startswith('Proxy'):
2195 datadump['nodename'] = datadump['nodename'].lower()
2196
2197 for iface_key in get_interface_keys(datadump):
2198 try:
2199 # All our normal wireless cards are normal APs now
2200 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2201 datadump[iface_key]['mode'] = 'ap'
2202 # Wireless Leiden SSID have an consistent lowercase/uppercase
2203 if datadump[iface_key].has_key('ssid'):
2204 ssid = datadump[iface_key]['ssid']
2205 prefix = 'ap-WirelessLeiden-'
2206 if ssid.lower().startswith(prefix.lower()):
2207 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2208 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2209 datadump[iface_key]['mode'] = 'autogen-FIXME'
2210 if not datadump[iface_key].has_key('comment'):
2211 datadump[iface_key]['comment'] = 'autogen-FIXME'
2212
2213 if datadump[iface_key].has_key('ns_mac'):
2214 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2215
2216 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2217 datadump[iface_key] = datadump[iface_key]['desc']
2218
2219 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2220 # due to better collision avoidance.
2221 if datadump[iface_key]['type'] == '11b':
2222 datadump[iface_key]['type'] = '11g'
2223
2224 # Setting 802.11g channels to de-facto standards, to avoid
2225 # un-detected sharing with other overlapping channels
2226 #
2227 # Technically we could also use channel 13 in NL, but this is not
2228 # recommended as foreign devices might not be able to select this
2229 # channel. Secondly using 1,5,9,13 instead is going to clash with
2230 # the de-facto usage of 1,6,11.
2231 #
2232 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2233 channels_at_2400Mhz = (1,6,11)
2234 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2235 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2236 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2237 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2238
2239 # Mandatory interface keys
2240 if not datadump[iface_key].has_key('status'):
2241 datadump[iface_key]['status'] = 'planned'
2242
2243 x = datadump[iface_key]['comment']
2244 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2245
2246 # Fixing bridge_type if none is found
2247 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2248 if not 'bridge_type' in datadump[iface_key]:
2249 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2250
2251 # Making sure description works
2252 if datadump[iface_key].has_key('desc'):
2253 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2254 del datadump[iface_key]['desc']
2255 else:
2256 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2257 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2258 if response:
2259 datadump[iface_key]['comment'] = response
2260 del datadump[iface_key]['desc']
2261
2262 # Check DHCP configuration
2263 dhcp_type(datadump[iface_key])
2264
2265 # Set the compass value based on the angle between the poels
2266 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2267 my_pool = poel[network(datadump[iface_key]['ip'])]
2268 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2269 if remote_hosts:
2270 compass_target = remote_hosts[0]
2271 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2272 # TODO: Compass wanted and actual direction might differ
2273
2274 # Monitoring Group default
2275 if not 'monitoring_group' in datadump:
2276 datadump['monitoring_group'] = 'wleiden'
2277
2278 except Exception:
2279 print "# Error while processing interface %s" % iface_key
2280 raise
2281 store_yaml(datadump)
2282 except Exception:
2283 print "# Error while processing %s" % host
2284 raise
2285 elif sys.argv[1] == "list":
2286 use_fqdn = False
2287 if len(sys.argv) < 4:
2288 usage()
2289 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2290 usage()
2291 if not sys.argv[3] in ["nodes","proxies","systems"]:
2292 usage()
2293
2294 if len(sys.argv) > 4:
2295 if sys.argv[4] == "fqdn":
2296 use_fqdn = True
2297 else:
2298 usage()
2299
2300 for system in get_hostlist():
2301 datadump = get_yaml(system)
2302 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2303 continue
2304
2305 output = datadump['autogen_fqdn'] if use_fqdn else system
2306 if sys.argv[2] == "all":
2307 print output
2308 elif datadump['status'] == sys.argv[2]:
2309 print output
2310 elif sys.argv[1] == "create":
2311 if sys.argv[2] == "network.kml":
2312 print make_network_kml.make_graph()
2313 elif sys.argv[2] == "host-ips.txt":
2314 for system in get_hostlist():
2315 datadump = get_yaml(system)
2316 ips = [datadump['masterip']]
2317 for ifkey in get_interface_keys(datadump):
2318 ips.append(datadump[ifkey]['ip'].split('/')[0])
2319 print system, ' '.join(ips)
2320 elif sys.argv[2] == "host-pos.txt":
2321 for system in get_hostlist():
2322 datadump = get_yaml(system)
2323 print system, datadump['rdnap_x'], datadump['rdnap_y']
2324 elif sys.argv[2] == 'ssh_config':
2325 print '''
2326Host *.wleiden.net
2327 User root
2328
2329Host 172.16.*.*
2330 User root
2331'''
2332 for system in get_hostlist():
2333 datadump = get_yaml(system)
2334 print '''\
2335Host %s
2336 User root
2337
2338Host %s
2339 User root
2340
2341Host %s
2342 User root
2343
2344Host %s
2345 User root
2346''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2347 else:
2348 usage()
2349 else:
2350 usage()
2351 else:
2352 # Do not enable debugging for config requests as it highly clutters the output
2353 if not is_text_request():
2354 cgitb.enable()
2355 response_headers, output = process_cgi_request()
2356 print_cgi_response(response_headers, output)
2357
2358def application(environ, start_response):
2359 status = '200 OK'
2360 response_headers, output = process_cgi_request(environ)
2361 start_response(status, response_headers)
2362
2363 # Debugging only
2364 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2365 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2366 return [output]
2367
2368if __name__ == "__main__":
2369 main()
Note: See TracBrowser for help on using the repository browser.