source: genesis/tools/gformat.py@ 13961

Last change on this file since 13961 was 13940, checked in by rick, 7 years ago

Update nameservers, backup moved to TransIP
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 78.6 KB
Line 
1#!/usr/local/bin/python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
35
36import argparse
37import cgi
38import cgitb
39import copy
40import glob
41import make_network_kml
42import math
43import pyproj
44import random
45import re
46import socket
47import string
48import subprocess
49import textwrap
50import time
51import urlparse
52
53from pprint import pprint
54from collections import defaultdict, OrderedDict
55from sys import stderr
56try:
57 import yaml
58except ImportError, e:
59 print e
60 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
61 exit(1)
62
63try:
64 from yaml import CLoader as Loader
65 from yaml import CDumper as Dumper
66except ImportError:
67 from yaml import Loader, Dumper
68
69from jinja2 import Environment, Template
70def yesorno(value):
71 return "YES" if bool(value) else "NO"
72env = Environment()
73env.filters['yesorno'] = yesorno
74def render_template(datadump, template):
75 result = env.from_string(template).render(datadump)
76 # Make it look pretty to the naked eye, as jinja templates are not so
77 # friendly when it comes to whitespace formatting
78 ## Remove extra whitespace at end of line lstrip() style.
79 result = re.sub(r'\n[\ ]+','\n', result)
80 ## Include only a single newline between an definition and a comment
81 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
82 ## Remove extra newlines after single comment
83 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
84 return result
85
86import logging
87logging.basicConfig(format='# %(levelname)s: %(message)s' )
88logger = logging.getLogger()
89logger.setLevel(logging.DEBUG)
90
91
92if os.environ.has_key('CONFIGROOT'):
93 NODE_DIR = os.environ['CONFIGROOT']
94else:
95 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
96__version__ = '$Id: gformat.py 13940 2017-07-18 09:36:02Z rick $'
97
98CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
99
100files = [
101 'authorized_keys',
102 'dnsmasq.conf',
103 'dhcpd.conf',
104 'rc.conf.local',
105 'resolv.conf',
106 'motd',
107 'ntp.conf',
108 'pf.hybrid.conf.local',
109 'unbound.wleiden.conf',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = OrderedDict()
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126
127NO_DHCP = 0
128DHCP_CLIENT = 10
129DHCP_SERVER = 20
130def dhcp_type(item):
131 if not item.has_key('dhcp'):
132 return NO_DHCP
133 elif not item['dhcp']:
134 return NO_DHCP
135 elif item['dhcp'].lower() == 'client':
136 return DHCP_CLIENT
137 else:
138 # Validation Checks
139 begin,end = map(int,item['dhcp'].split('-'))
140 if begin >= end:
141 raise ValueError("DHCP Start >= DHCP End")
142 return DHCP_SERVER
143
144def etrs2rd(lat, lon):
145 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
146 p2 = pyproj.Proj(init='EPSG:28992')
147 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
148 return (RDx, RDy)
149
150def rd2etrs(RDx, RDy):
151 p1 = pyproj.Proj(init='EPSG:28992')
152 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
153 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
154 return (lat, lon)
155
156def get_yaml(item,add_version_info=True):
157 try:
158 """ Get configuration yaml for 'item'"""
159 if datadump_cache.has_key(item):
160 return datadump_cache[item].copy()
161
162 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
163 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
164 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
165
166 # Default values
167 datadump = {
168 'autogen_revision' : 'NOTFOUND',
169 'autogen_gfile' : gfile,
170 'service_proxy_ileiden' : False,
171 }
172 f = open(gfile, 'r')
173 datadump.update(yaml.load(f,Loader=Loader))
174 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
175 if datadump['nodetype'] == 'Hybrid':
176 # Some values are defined implicitly
177 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
178 datadump['service_incoming_rdr'] = True
179 # Use some boring defaults
180 defaults = {
181 'service_proxy_normal' : False,
182 'service_accesspoint' : True,
183 'service_incoming_rdr' : False,
184 'service_concentrator' : False,
185 'monitoring_group' : 'wleiden',
186 }
187 for (key,value) in defaults.iteritems():
188 if not datadump.has_key(key):
189 datadump[key] = value
190 f.close()
191
192 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
193 if add_version_info:
194 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
195 lines = p.communicate()[0].split('\n')
196 if p.returncode == 0:
197 for line in lines:
198 if line:
199 (key, value) = line.split(': ')
200 datadump["autogen_" + key.lower().replace(' ','_')] = value
201
202 # Preformat certain needed variables for formatting and push those into special object
203 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
204
205 wlan_count=0
206 try:
207 for key in get_interface_keys(datadump, True):
208 datadump[key]['autogen_ifbase'] = key.split('_')[1]
209 datadump[key]['autogen_vlan'] = False
210
211 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
212 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
213 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
214
215 if datadump[key].has_key('parent'):
216 if datadump[key].has_key('ip'):
217 raise ValueError("Interface bridge member cannot have IP assigned")
218 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
219 raise ValueError("Interface bridge member cannot have DHCP set")
220
221 if datadump[key].has_key('ip'):
222 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
223
224 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
225 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
226 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
227 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
228 wlan_count += 1
229 else:
230 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
231 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
232 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
233 datadump[key]['autogen_vlan'] = key.split('_')[2]
234 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
235 else:
236 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
237
238 except Exception:
239 print "# Error while processing interface %s" % key
240 raise
241
242 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
243 if dhcp_type(datadump[key]) == DHCP_SERVER]
244
245 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
246 datadump['autogen_item'] = item
247
248 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
249 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
250 datadump_cache[item] = datadump.copy()
251 except Exception:
252 print "# Error while processing %s" % item
253 raise
254 return datadump
255
256
257def store_yaml(datadump, header=False):
258 """ Store configuration yaml for 'item'"""
259 item = datadump['autogen_item']
260 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
261
262 output = generate_wleiden_yaml(datadump, header)
263
264 f = open(gfile, 'w')
265 f.write(output)
266 f.close()
267
268
269def network(ip):
270 addr, mask = ip.split('/')
271 # Not parsing of these folks please
272 addr = parseaddr(addr)
273 mask = int(mask)
274 network = addr & ~((1 << (32 - mask)) - 1)
275 return network
276
277
278
279def make_relations():
280 """ Process _ALL_ yaml files to get connection relations """
281 global relations_cache
282
283 if relations_cache:
284 return relations_cache
285
286 errors = []
287 poel = defaultdict(list)
288
289 for host in get_hostlist():
290 datadump = get_yaml(host)
291 try:
292 for iface_key in get_interface_keys(datadump):
293 # Bridge members has no IP assigned
294 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
295 continue
296 net_addr = network(datadump[iface_key]['ip'])
297 poel[net_addr] += [(host,datadump[iface_key].copy())]
298 except (KeyError, ValueError) as e:
299 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
300 continue
301
302 relations_cache = (poel, errors)
303 return relations_cache
304
305
306
307def valid_addr(addr):
308 """ Show which address is valid in which are not """
309 return str(addr).startswith('172.')
310
311def get_hostlist():
312 """ Combined hosts and proxy list"""
313 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
314
315def angle_between_points(lat1,lat2,long1,long2):
316 """
317 Return Angle in radians between two GPS coordinates
318 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
319 """
320 dy = lat2 - lat1
321 dx = math.cos(lat1)*(long2 - long1)
322 angle = math.atan2(dy,dx)
323 return angle
324
325
326
327def angle_to_cd(angle):
328 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
329
330 # For easy conversion get positive degree
331 degrees = math.degrees(angle)
332 abs_degrees = 360 + degrees if degrees < 0 else degrees
333
334 # Numbers can be confusing calculate from the 4 main directions
335 p = 22.5
336 if abs_degrees < p:
337 cd = "n"
338 elif abs_degrees < (90 - p):
339 cd = "no"
340 elif abs_degrees < (90 + p):
341 cd = "o"
342 elif abs_degrees < (180 - p):
343 cd = "zo"
344 elif abs_degrees < (180 + p):
345 cd = "z"
346 elif abs_degrees < (270 - p):
347 cd = "zw"
348 elif abs_degrees < (270 + p):
349 cd = "w"
350 elif abs_degrees < (360 - p):
351 cd = "nw"
352 else:
353 cd = "n"
354 return cd
355
356
357
358def cd_between_hosts(hostA, hostB, datadumps):
359 # Using RDNAP coordinates
360 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
361 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
362 return angle_to_cd(math.atan2(dx,dy))
363
364 # GPS coordinates seems to fail somehow
365 #latA = float(datadumps[hostA]['latitude'])
366 #latB = float(datadumps[hostB]['latitude'])
367 #lonA = float(datadumps[hostA]['longitude'])
368 #lonB = float(datadumps[hostB]['longitude'])
369 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
370
371
372def generate_title(nodelist):
373 """ Main overview page """
374 items = {'root' : "." }
375 def fl(spaces, line):
376 return (' ' * spaces) + line + '\n'
377
378 output = """
379<html>
380 <head>
381 <title>Wireless leiden Configurator - GFormat</title>
382 <style type="text/css">
383 th {background-color: #999999}
384 tr:nth-child(odd) {background-color: #cccccc}
385 tr:nth-child(even) {background-color: #ffffff}
386 th, td {padding: 0.1em 1em}
387 </style>
388 </head>
389 <body>
390 <center>
391 <form type="GET" action="%(root)s">
392 <input type="hidden" name="action" value="update">
393 <input type="submit" value="Update Configuration Database (SVN)">
394 </form>
395 <table>
396 <caption><h3>Wireless Leiden Configurator</h3></caption>
397 """ % items
398
399 for node in nodelist:
400 items['node'] = node
401 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
402 for config in files:
403 items['config'] = config
404 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
405 output += fl(5, "</tr>")
406 output += """
407 </table>
408 <hr />
409 <em>%s</em>
410 </center>
411 </body>
412</html>
413 """ % __version__
414
415 return output
416
417
418
419def generate_node(node):
420 """ Print overview of all files available for node """
421 return "\n".join(files)
422
423def generate_node_overview(host, datadump=False):
424 """ Print overview of all files available for node """
425 if not datadump:
426 datadump = get_yaml(host)
427 params = { 'host' : host }
428 output = "<em><a href='..'>Back to overview</a></em><hr />"
429 output += "<h2>Available files:</h2><ul>"
430 for cf in files:
431 params['cf'] = cf
432 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
433 output += "</ul>"
434
435 # Generate and connection listing
436 output += "<h2>Connected To:</h2><ul>"
437 (poel, errors) = make_relations()
438 for network, hosts in poel.iteritems():
439 if host in [x[0] for x in hosts]:
440 if len(hosts) == 1:
441 # Single not connected interface
442 continue
443 for remote,ifacedump in hosts:
444 if remote == host:
445 # This side of the interface
446 continue
447 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
448 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
449 output += "</ul>"
450 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
451
452 output += "<hr /><em><a href='..'>Back to overview</a></em>"
453 return output
454
455
456def generate_header(datadump, ctag="#"):
457 return """\
458%(ctag)s
459%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
460%(ctag)s
461""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
462
463
464
465def parseaddr(s):
466 """ Process IPv4 CIDR notation addr to a (binary) number """
467 f = s.split('.')
468 return (long(f[0]) << 24L) + \
469 (long(f[1]) << 16L) + \
470 (long(f[2]) << 8L) + \
471 long(f[3])
472
473
474
475def showaddr(a):
476 """ Display IPv4 addr in (dotted) CIDR notation """
477 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
478
479
480def is_member(ip, mask, canidate):
481 """ Return True if canidate is part of ip/mask block"""
482 ip_addr = parseaddr(ip)
483 ip_canidate = parseaddr(canidate)
484 mask = int(mask)
485 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
486 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
487 return ip_addr == ip_canidate
488
489
490
491def cidr2netmask(netmask):
492 """ Given a 'netmask' return corresponding CIDR """
493 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
494
495def get_network(addr, mask):
496 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
497
498
499def generate_dhcpd_conf(datadump):
500 """ Generate config file '/usr/local/etc/dhcpd.conf """
501 # Redundency support, in cause local DNS server is not running/responding.
502 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
503 output = generate_header(datadump)
504 output += Template("""\
505# option definitions common to all supported networks...
506option domain-name "dhcp.{{ autogen_fqdn }}";
507
508default-lease-time 600;
509max-lease-time 7200;
510
511# Use this to enble / disable dynamic dns updates globally.
512#ddns-update-style none;
513
514# If this DHCP server is the official DHCP server for the local
515# network, the authoritative directive should be uncommented.
516authoritative;
517
518# Use this to send dhcp log messages to a different log file (you also
519# have to hack syslog.conf to complete the redirection).
520log-facility local7;
521
522# UniFi Discovery Support
523option space ubnt;
524option ubnt.unifi-address code 1 = ip-address;
525#
526class "ubnt" {
527 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
528 option vendor-class-identifier "ubnt";
529 vendor-option-space ubnt;
530}
531
532#
533# Interface definitions
534#
535\n\n""").render(datadump)
536
537
538 # TODO: Use textwrap.fill instead
539 def indent(text, count):
540 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
541
542 # Process DHCP blocks
543 dhcp_out = defaultdict(list)
544 for iface_key in get_interface_keys(datadump):
545 ifname = datadump[iface_key]['autogen_ifbase']
546 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
547 if not datadump[iface_key].has_key('comment'):
548 datadump[iface_key]['comment'] = None
549
550 if not datadump[iface_key].has_key('ip'):
551 continue
552
553 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
554
555 (addr, mask) = datadump[iface_key]['ip'].split('/')
556 datadump[iface_key]['autogen_addr'] = addr
557 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
558 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
559
560 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
561 dhcp_out[groupif].append(textwrap.dedent("""\
562 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
563 ### not autoritive
564 }
565 """ % datadump[iface_key]))
566 continue
567
568 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
569 dhcp_part = ".".join(addr.split('.')[0:3])
570 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
571 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
572 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
573
574 # Assume the first 10 IPs could be used for static entries
575 if 'no_portal' in datadump:
576 fixed = 5
577 for mac in datadump['no_portal']:
578 dhcp_out[groupif].append(textwrap.dedent("""\
579 host fixed-%(ifname)s-%(fixed)s {
580 hardware ethernet %(mac)s;
581 fixed-address %(prefix)s.%(fixed)s;
582 }
583 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
584 fixed += 1
585
586 if 'dhcp_fixed' in datadump[iface_key]:
587 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
588 dhcp_out[groupif].append(textwrap.dedent("""\
589 host fixed-%(host)s {
590 hardware ethernet %(mac)s;
591 fixed-address %(addr)s;
592 }
593 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
594
595
596 dhcp_out[groupif].append(textwrap.dedent("""\
597 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
598 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
599 option routers %(autogen_addr)s;
600 option domain-name-servers %(autogen_dns_servers)s;
601 option ubnt.unifi-address 172.17.107.10;
602 }
603 """ % datadump[iface_key]))
604
605 # Output the blocks in groups
606 for ifname,value in sorted(dhcp_out.iteritems()):
607 if len(value) > 2:
608 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
609 else:
610 output += ''.join(value) + "\n\n"
611 return output
612
613
614
615def generate_dnsmasq_conf(datadump):
616 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
617 output = generate_header(datadump)
618 output += Template("""\
619# DHCP server options
620dhcp-authoritative
621dhcp-fqdn
622domain=dhcp.{{ autogen_fqdn }}
623domain-needed
624expand-hosts
625log-async=100
626
627# Low memory footprint
628cache-size=10000
629
630\n""").render(datadump)
631
632 for iface_key in get_interface_keys(datadump):
633 if not datadump[iface_key].has_key('comment'):
634 datadump[iface_key]['comment'] = None
635 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
636
637 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
638 output += "# not autoritive\n\n"
639 continue
640
641 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
642 (ip, cidr) = datadump[iface_key]['ip'].split('/')
643 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
644
645 dhcp_part = ".".join(ip.split('.')[0:3])
646 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
647 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
648 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
649
650 return output
651
652
653class AutoVivification(dict):
654 """Implementation of perl's autovivification feature."""
655 def __getitem__(self, item):
656 try:
657 return dict.__getitem__(self, item)
658 except KeyError:
659 value = self[item] = type(self)()
660 return value
661
662def make_interface_list(datadump):
663 if interface_list_cache.has_key(datadump['autogen_item']):
664 return (interface_list_cache[datadump['autogen_item']])
665 # lo0 configuration:
666 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
667 # - masterip is special as it needs to be assigned to at
668 # least one interface, so if not used assign to lo0
669 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
670 vlan_list = defaultdict(list)
671 bridge_list = defaultdict(list)
672 flags_if = AutoVivification()
673 dhclient_if = {'lo0' : False}
674
675 # XXX: Find some way of send this output nicely
676 output = ''
677
678 masterip_used = False
679 for iface_key in get_interface_keys(datadump):
680 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
681 masterip_used = True
682 break
683 if not masterip_used:
684 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
685
686 if 'serviceid' in datadump:
687 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
688
689 for iface_key in get_interface_keys(datadump):
690 ifacedump = datadump[iface_key]
691 if ifacedump['autogen_bridge_alias']:
692 ifname = ifacedump['autogen_ifbase']
693 else:
694 ifname = ifacedump['autogen_ifname']
695
696 # If defined as vlan interface
697 if ifacedump['autogen_vlan']:
698 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
699
700 # If defined as bridge interface
701 if ifacedump['autogen_bridge_member']:
702 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
703
704
705 # Flag dhclient is possible
706 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
707 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
708
709 # Ethernet address
710 if ifacedump.has_key('ether'):
711 flags_if[ifname]['ether'] = ifacedump['ether']
712
713 # Add interface IP to list
714 if ifacedump.has_key('ip'):
715 item = (ifacedump['ip'], ifacedump['comment'])
716 if addrs_list.has_key(ifname):
717 addrs_list[ifname].append(item)
718 else:
719 addrs_list[ifname] = [item]
720
721 # Alias only needs IP assignment for now, this might change if we
722 # are going to use virtual accesspoints
723 if "alias" in iface_key:
724 continue
725
726 # XXX: Might want to deduct type directly from interface name
727 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
728 # Default to station (client) mode
729 ifacedump['autogen_wlanmode'] = "sta"
730 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
731 ifacedump['autogen_wlanmode'] = "ap"
732
733 if not ifacedump.has_key('channel'):
734 if ifacedump['type'] == '11a':
735 ifacedump['channel'] = 36
736 else:
737 ifacedump['channel'] = 1
738
739 # Allow special hacks at the back like wds and stuff
740 if not ifacedump.has_key('extra'):
741 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
742 else:
743 ifacedump['autogen_extra'] = ifacedump['extra']
744
745 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
746
747 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
748 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
749 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
750 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
751 output += "\n"
752
753 elif ifacedump['type'] in ['ethernet', 'eth']:
754 # No special config needed besides IP
755 pass
756 elif ifacedump['type'] in ['vlan']:
757 # VLAN member has no special configuration
758 pass
759 else:
760 assert False, "Unknown type " + ifacedump['type']
761
762 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
763 interface_list_cache[datadump['autogen_item']] = store
764 return(store)
765
766
767
768def create_proxies_list():
769 if not ileiden_proxies or not normal_proxies:
770 # Placeholder for to-be-installed proxies, this will avoid updating the all
771 # nodes to include this new machine, yet due to an unbound issue, this list
772 # has to be kept small.
773
774 for i in range(1,20):
775 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
776
777 for host in get_hostlist():
778 hostdump = get_yaml(host)
779 if hostdump['status'] == 'up':
780 if hostdump['service_proxy_ileiden']:
781 ileiden_proxies[hostdump['serviceid']] = hostdump
782 if hostdump['service_proxy_normal']:
783 normal_proxies.append(hostdump)
784
785
786
787def generate_rc_conf_local(datadump):
788 """ Generate configuration file '/etc/rc.conf.local' """
789 item = datadump['autogen_item']
790 if rc_conf_local_cache.has_key(item):
791 return rc_conf_local_cache[item]
792
793 if not datadump.has_key('ileiden'):
794 datadump['autogen_ileiden_enable'] = False
795 else:
796 datadump['autogen_ileiden_enable'] = datadump['ileiden']
797
798 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
799
800 create_proxies_list()
801 datadump['autogen_ileiden_proxies'] = ileiden_proxies
802 datadump['autogen_normal_proxies'] = normal_proxies
803 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
804 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
805 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
806 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
807
808 output = generate_header(datadump, "#");
809 output += render_template(datadump, """\
810hostname='{{ autogen_fqdn }}'
811location='{{ location }}'
812nodetype="{{ nodetype }}"
813
814#
815# Configured listings
816#
817captive_portal_whitelist=""
818{% if nodetype == "Proxy" %}
819#
820# Proxy Configuration
821#
822{% if gateway and service_proxy_ileiden -%}
823defaultrouter="{{ gateway }}"
824{% else -%}
825#defaultrouter="NOTSET"
826{% endif -%}
827internalif="{{ internalif }}"
828ileiden_enable="{{ autogen_ileiden_enable }}"
829gateway_enable="{{ autogen_ileiden_enable }}"
830pf_enable="yes"
831pf_rules="/etc/pf.conf"
832{% if autogen_ileiden_enable -%}
833pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
834lvrouted_enable="{{ autogen_ileiden_enable }}"
835lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
836{% else -%}
837pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
838{% endif -%}
839{% if internalroute -%}
840static_routes="wleiden"
841route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
842{% endif -%}
843
844{% elif nodetype == "Hybrid" %}
845 #
846 # Hybrid Configuration
847 #
848 list_ileiden_proxies="
849 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
850 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
851 {% endfor -%}
852 "
853 list_normal_proxies="
854 {% for item in autogen_normal_proxies -%}
855 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
856 {% endfor -%}
857 "
858
859 {% if autogen_dhcp_interfaces -%}
860 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
861 {% else %}
862 captive_portal_interfaces="dummy"
863 {% endif %}
864 externalif="{{ externalif|default('vr0', true) }}"
865 masterip="{{ masterip }}"
866
867 {% if gateway and service_proxy_ileiden %}
868 defaultrouter="{{ gateway }}"
869 {% else %}
870 #defaultrouter="NOTSET"
871 {% endif %}
872
873 #
874 # Defined services
875 #
876 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
877 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
878 service_accesspoint="{{ service_accesspoint|yesorno }}"
879 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
880 service_concentrator="{{ service_concentrator|yesorno }}"
881
882 {% if service_proxy_ileiden %}
883 pf_rules="/etc/pf.hybrid.conf"
884 {% if service_concentrator %}
885 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
886 {% else %}
887 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
888 {% endif %}
889 pf_flags="$pf_flags -D publicnat=80,443"
890 lvrouted_flags="$lvrouted_flags -g"
891 {% elif service_proxy_normal or service_incoming_rdr %}
892 pf_rules="/etc/pf.hybrid.conf"
893 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
894 pf_flags="$pf_flags -D publicnat=0"
895 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
896 named_setfib="1"
897 tinyproxy_setfib="1"
898 dnsmasq_setfib="1"
899 sshd_setfib="1"
900 {% else %}
901 named_auto_forward_only="YES"
902 pf_rules="/etc/pf.node.conf"
903 pf_flags=""
904 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
905 {% endif %}
906 {% if service_concentrator %}
907 # Do mind installing certificates is NOT done automatically for security reasons
908 openvpn_enable="YES"
909 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
910 {% endif %}
911
912 {% if service_proxy_normal %}
913 tinyproxy_enable="yes"
914 {% else %}
915 pen_wrapper_enable="yes"
916 {% endif %}
917
918 {% if service_accesspoint %}
919 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
920 {% endif %}
921
922 {% if board == "ALIX2" or board == "net4801" %}
923 #
924 # ''Fat'' configuration, board has 256MB RAM
925 #
926 dnsmasq_enable="NO"
927 named_enable="YES"
928 unbound_enable="YES"
929 {% if autogen_dhcp_interfaces -%}
930 dhcpd_enable="YES"
931 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
932 {% endif -%}
933 {% elif board == "apu1d" %}
934 #
935 # ''Fat'' configuration, board has 1024MB RAM
936 #
937 dnsmasq_enable="NO"
938 unbound_enable="YES"
939 {% if autogen_dhcp_interfaces -%}
940 dhcpd_enable="YES"
941 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
942 {% endif -%}
943 {% endif -%}
944{% endif %}
945
946#
947# Script variables
948#
949attached_devices="{{ autogen_attached_devices|join(' ') }}"
950neighbours="{{ autogen_neighbours|join(' ') }}"
951
952
953#
954# Interface definitions
955#\n
956""")
957
958 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
959 for iface, vlans in sorted(vlan_list.items()):
960 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
961
962 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
963 for iface in sorted(vlan_list.keys()):
964 if not iface in addrs_list.keys():
965 output += "ifconfig_%s='up'\n" % iface
966
967 output += "\n"
968
969 # Bridge configuration:
970 if bridge_list.keys():
971 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
972
973 for iface in bridge_list.keys():
974 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
975
976 # Bridge member interfaces not containing a configuration should be marked active explcitly.
977 for _,members in bridge_list.items():
978 for iface in members:
979 if not iface in addrs_list.keys():
980 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
981
982 output += "\n"
983
984 # Details like SSID
985 if extra_ouput:
986 output += extra_ouput.strip() + "\n"
987
988 # Print IP address which needs to be assigned over here
989 output += "\n"
990 for iface,addrs in sorted(addrs_list.iteritems()):
991 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
992 output += "# %s || %s || %s\n" % (iface, addr, comment)
993
994 # Write DHCLIENT entry
995 if iface in dhclient_if and dhclient_if[iface]:
996 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
997 continue
998
999 # Make sure the external address is always first as this is needed in the
1000 # firewall setup
1001 addrs = sorted(
1002 [x for x in addrs if not '0.0.0.0' in x[0]],
1003 key=lambda x: x[0].split('.')[0],
1004 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1005 )
1006
1007 idx_offset = 0
1008 # Set MAC is required
1009 if flags_if[iface].has_key('ether'):
1010 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1011 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1012 idx_offset += 1
1013 elif iface in bridge_list:
1014 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1015 idx_offset += 1
1016 else:
1017 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1018
1019 for idx, addr in enumerate(addrs[1:]):
1020 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1021
1022 output += "\n"
1023
1024 rc_conf_local_cache[datadump['autogen_item']] = output
1025 return output
1026
1027
1028
1029def get_all_configs():
1030 """ Get dict with key 'host' with all configs present """
1031 configs = dict()
1032 for host in get_hostlist():
1033 datadump = get_yaml(host)
1034 configs[host] = datadump
1035 return configs
1036
1037
1038def get_interface_keys(config, extra=False):
1039 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1040 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1041 if extra == False:
1042 return filter(lambda x: not "extra" in x, elems)
1043 else:
1044 return elems
1045
1046
1047def get_used_ips(configs):
1048 """ Return array of all IPs used in config files"""
1049 ip_list = []
1050 for config in configs:
1051 ip_list.append(config['masterip'])
1052 if 'serviceid' in config:
1053 ip_list.append(config['serviceid'])
1054 for iface_key in get_interface_keys(config, True):
1055 l = config[iface_key]['ip']
1056 addr, mask = l.split('/')
1057 # Special case do not process
1058 if valid_addr(addr):
1059 ip_list.append(addr)
1060 else:
1061 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1062 return sorted(ip_list)
1063
1064
1065
1066def get_nameservers(max_servers=None):
1067 if nameservers_cache:
1068 return nameservers_cache[0:max_servers]
1069
1070 for host in get_hostlist():
1071 hostdump = get_yaml(host)
1072 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1073 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1074
1075 return nameservers_cache[0:max_servers]
1076
1077
1078def get_neighbours(datadump):
1079 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1080
1081 (poel, errors) = make_relations()
1082 table = []
1083 for iface,addrs in sorted(addrs_list.iteritems()):
1084 if iface in ['lo0']:
1085 continue
1086
1087 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1088 if not addr.startswith('172.'):
1089 # Avoid listing internet connections as pool
1090 continue
1091 for neighbour in poel[network(addr)]:
1092 if neighbour[0] != datadump['autogen_item']:
1093 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1094 return table
1095
1096
1097def get_attached_devices(datadump, url=False):
1098 table = []
1099 for iface_key in get_interface_keys(datadump, True):
1100 ifacedump = datadump[iface_key]
1101
1102 if not ifacedump.has_key('ns_ip'):
1103 continue
1104
1105 x_ip = ifacedump['ns_ip'].split('/')[0]
1106
1107 if 'mode' in ifacedump:
1108 x_mode = ifacedump['mode']
1109 else:
1110 x_mode = 'unknown'
1111
1112 if 'bridge_type' in ifacedump:
1113 device_type = ifacedump['bridge_type']
1114 else:
1115 device_type = 'Unknown'
1116
1117 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1118 return table
1119
1120
1121def generate_resolv_conf(datadump):
1122 """ Generate configuration file '/etc/resolv.conf' """
1123 # XXX: This should properly going to be an datastructure soon
1124 datadump['autogen_header'] = generate_header(datadump, "#")
1125 datadump['autogen_edge_nameservers'] = ''
1126
1127
1128 for masterip,realname in get_nameservers():
1129 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1130
1131 return Template("""\
1132{{ autogen_header }}
1133search wleiden.net
1134
1135# Try local (cache) first
1136nameserver 127.0.0.1
1137
1138{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1139nameserver 8.8.8.8 # Google Public NameServer
1140nameserver 64.6.64.6 # Verisign Public NameServer
1141{% else -%}
1142# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1143{{ autogen_edge_nameservers }}
1144{% endif -%}
1145""").render(datadump)
1146
1147
1148
1149def generate_ntp_conf(datadump):
1150 """ Generate configuration file '/etc/ntp.conf' """
1151 # XXX: This should properly going to be an datastructure soon
1152
1153 datadump['autogen_header'] = generate_header(datadump, "#")
1154 datadump['autogen_ntp_servers'] = ''
1155 for host in get_hostlist():
1156 hostdump = get_yaml(host)
1157 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1158 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1159
1160 return Template("""\
1161{{ autogen_header }}
1162
1163{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1164# Machine hooked to internet.
1165server 0.nl.pool.ntp.org iburst maxpoll 9
1166server 1.nl.pool.ntp.org iburst maxpoll 9
1167server 2.nl.pool.ntp.org iburst maxpoll 9
1168server 3.nl.pool.ntp.org iburst maxpoll 9
1169{% else -%}
1170# Local Wireless Leiden NTP Servers.
1171server 0.pool.ntp.wleiden.net iburst maxpoll 9
1172server 1.pool.ntp.wleiden.net iburst maxpoll 9
1173server 2.pool.ntp.wleiden.net iburst maxpoll 9
1174server 3.pool.ntp.wleiden.net iburst maxpoll 9
1175
1176# All the configured NTP servers
1177{{ autogen_ntp_servers }}
1178{% endif %}
1179
1180# If a server loses sync with all upstream servers, NTP clients
1181# no longer follow that server. The local clock can be configured
1182# to provide a time source when this happens, but it should usually
1183# be configured on just one server on a network. For more details see
1184# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1185# The use of Orphan Mode may be preferable.
1186#
1187server 127.127.1.0
1188fudge 127.127.1.0 stratum 10
1189""").render(datadump)
1190
1191
1192def generate_pf_hybrid_conf_local(datadump):
1193 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1194 datadump['autogen_header'] = generate_header(datadump, "#")
1195 if datadump['service_incoming_rdr']:
1196 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1197 return Template("""\
1198{{ autogen_header }}
1199
1200# Redirect some internal facing services outside (7)
1201# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1202{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1203rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1204{% endfor -%}
1205# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1206{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1207rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1208{% endfor -%}
1209""").render(datadump)
1210
1211def generate_unbound_wleiden_conf(datadump):
1212 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1213 datadump['autogen_header'] = generate_header(datadump, "#")
1214
1215 autogen_ips = []
1216 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1217 for iface,addrs in sorted(addrs_list.iteritems()):
1218 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1219 if addr.startswith('172'):
1220 autogen_ips.append((addr.split('/')[0], comment))
1221 datadump['autogen_ips'] = autogen_ips
1222
1223 create_proxies_list()
1224 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1225 return Template("""\
1226{{ autogen_header }}
1227
1228server:
1229 ## Static definitions fail to start on systems with broken ue(4) interfaces
1230{%- for ip,comment in autogen_ips %}
1231 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1232{%- endfor %}
1233 ## Enabling wildcard matching as work-around
1234 interface: 0.0.0.0
1235 interface: ::0
1236
1237forward-zone:
1238 name: '.'
1239{%- if service_proxy_ileiden %}
1240 forward-addr: 8.8.8.8 # Google DNS A
1241 forward-addr: 8.8.4.4 # Google DNS B
1242 forward-addr: 208.67.222.222 # OpenDNS DNS A
1243 forward-addr: 208.67.220.220 # OpenDNS DNS B
1244{% else -%}
1245{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1246 {%- if loop.index <= 5 %}
1247 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1248 {%- endif %}
1249{%- endfor %}
1250{%- endif %}
1251""").render(datadump)
1252
1253def generate_motd(datadump):
1254 """ Generate configuration file '/etc/motd' """
1255 output = Template("""\
1256FreeBSD run ``service motd onestart'' to make me look normal
1257
1258 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1259 Loc: {{ location }}
1260
1261Services:
1262{% if board == "ALIX2" or board == "net4801" -%}
1263{{" -"}} Core Node ({{ board }})
1264{% else -%}
1265{{" -"}} Hulp Node ({{ board }})
1266{% endif -%}
1267{% if service_proxy_normal -%}
1268{{" -"}} Normal Proxy
1269{% endif -%}
1270{% if service_proxy_ileiden -%}
1271{{" -"}} iLeiden Proxy
1272{% endif -%}
1273{% if service_incoming_rdr -%}
1274{{" -"}} Incoming port redirects
1275{% endif %}
1276Interlinks:\n
1277""").render(datadump)
1278
1279
1280 def make_table(table):
1281 if not table:
1282 return " - none\n"
1283 else:
1284 lines = ""
1285 col_width = [max(len(x) for x in col) for col in zip(*table)]
1286 for row in table:
1287 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1288 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1289 return lines
1290
1291 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1292 table = []
1293 for iface,addrs in sorted(addrs_list.iteritems()):
1294 if iface in ['lo0']:
1295 continue
1296 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1297 table.append((iface, addr, comment))
1298
1299 output += make_table(table)
1300 output += '\n'
1301 output += """\
1302Attached devices:
1303"""
1304 output += make_table(get_attached_devices(datadump, url=True))
1305 output += '\n'
1306 output += """\
1307Available neighbours:
1308"""
1309 output += make_table(get_neighbours(datadump))
1310
1311 return output
1312
1313
1314def format_yaml_value(value):
1315 """ Get yaml value in right syntax for outputting """
1316 if isinstance(value,str):
1317 output = '"%s"' % value
1318 else:
1319 output = value
1320 return output
1321
1322
1323
1324def format_wleiden_yaml(datadump):
1325 """ Special formatting to ensure it is editable"""
1326 output = "# Genesis config yaml style\n"
1327 output += "# vim:ts=2:et:sw=2:ai\n"
1328 output += "#\n"
1329 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1330 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1331 if key == 'rdr_rules':
1332 output += '%-10s:\n' % 'rdr_rules'
1333 for rdr_rule in datadump[key]:
1334 output += '- %s\n' % rdr_rule
1335 else:
1336 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1337
1338 output += "\n\n"
1339
1340 # Format (key, required)
1341 key_order = (
1342 ('comment', True),
1343 ('parent', False),
1344 ('ip', False),
1345 ('ether', False),
1346 ('desc', True),
1347 ('sdesc', True),
1348 ('mode', True),
1349 ('type', True),
1350 ('extra_type', False),
1351 ('channel', False),
1352 ('ssid', False),
1353 ('wlan_mac', False),
1354 ('dhcp', True),
1355 ('dhcp_fixed', False),
1356 ('compass', False),
1357 ('distance', False),
1358 ('ns_ip', False),
1359 ('repeater_ip', False),
1360 ('bullet2_ip', False),
1361 ('ns_mac', False),
1362 ('bullet2_mac', False),
1363 ('ns_type', False),
1364 ('bridge_type', False),
1365 ('status', True),
1366 )
1367
1368 for iface_key in sorted(iface_keys):
1369 try:
1370 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1371 if remainder:
1372 raise KeyError("invalid keys: %s" % remainder)
1373
1374 output += "%s:\n" % iface_key
1375 for key,required in key_order:
1376 if datadump[iface_key].has_key(key):
1377 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1378 output += "\n\n"
1379 except Exception:
1380 print "# Error while processing interface %s" % iface_key
1381 raise
1382
1383 return output
1384
1385
1386
1387def generate_wleiden_yaml(datadump, header=True):
1388 """ Generate (petty) version of wleiden.yaml"""
1389 output = generate_header(datadump, "#") if header else ''
1390
1391 for key in datadump.keys():
1392 if key.startswith('autogen_'):
1393 del datadump[key]
1394 # Interface autogen cleanups
1395 elif type(datadump[key]) == dict:
1396 for key2 in datadump[key].keys():
1397 if key2.startswith('autogen_'):
1398 del datadump[key][key2]
1399
1400 output += format_wleiden_yaml(datadump)
1401 return output
1402
1403def generate_nanostation_config(datadump, iface, ns_type):
1404 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1405 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1406
1407 datadump.update(datadump['iface_%s' % iface])
1408
1409 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1410
1411def generate_yaml(datadump):
1412 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1413
1414
1415
1416def generate_config(node, config, datadump=None):
1417 """ Print configuration file 'config' of 'node' """
1418 output = ""
1419 try:
1420 # Load config file
1421 if datadump == None:
1422 datadump = get_yaml(node)
1423
1424 if config == 'wleiden.yaml':
1425 output += generate_wleiden_yaml(datadump)
1426 elif config == 'authorized_keys':
1427 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1428 output += f.read()
1429 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1430 # Fetch local keys if existing
1431 if os.path.exists(node_keys):
1432 output += open(node_keys, 'r').read()
1433 f.close()
1434 elif config == 'dnsmasq.conf':
1435 output += generate_dnsmasq_conf(datadump)
1436 elif config == 'dhcpd.conf':
1437 output += generate_dhcpd_conf(datadump)
1438 elif config == 'rc.conf.local':
1439 output += generate_rc_conf_local(datadump)
1440 elif config == 'resolv.conf':
1441 output += generate_resolv_conf(datadump)
1442 elif config == 'ntp.conf':
1443 output += generate_ntp_conf(datadump)
1444 elif config == 'motd':
1445 output += generate_motd(datadump)
1446 elif config == 'pf.hybrid.conf.local':
1447 output += generate_pf_hybrid_conf_local(datadump)
1448 elif config == 'unbound.wleiden.conf':
1449 output += generate_unbound_wleiden_conf(datadump)
1450 elif config.startswith('vr'):
1451 interface, ns_type = config.strip('.yaml').split('-')
1452 output += generate_nanostation_config(datadump, interface, ns_type)
1453 else:
1454 assert False, "Config not found!"
1455 except IOError, e:
1456 output += "[ERROR] Config file not found"
1457 return output
1458
1459
1460
1461def generate_static(output_dir, logging=True):
1462 items = {'output_dir' : output_dir}
1463 for node in get_hostlist():
1464 items['node'] = node
1465 items['wdir'] = "%(output_dir)s/%(node)s" % items
1466 if not os.path.isdir(items['wdir']):
1467 os.makedirs(items['wdir'])
1468 datadump = get_yaml(node)
1469 f = open("%(wdir)s/index.html" % items, "w")
1470 f.write(generate_node_overview(items['node'], datadump))
1471 f.close()
1472 for config in files:
1473 items['config'] = config
1474 if logging: logger.info("## Generating %(node)s %(config)s" % items)
1475 f = open("%(wdir)s/%(config)s" % items, "w")
1476 f.write(generate_config(node, config, datadump))
1477 f.close()
1478
1479
1480
1481def process_cgi_request(environ=os.environ):
1482 """ When calling from CGI """
1483 response_headers = []
1484 content_type = 'text/plain'
1485
1486 # Update repository if requested
1487 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1488 if form and form.has_key("action") and "update" in form["action"]:
1489 output = "[INFO] Updating subverion, please wait...\n"
1490 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1491 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1492 generate_static(CACHE_DIR, False)
1493 output += "[INFO] All done, redirecting in 5 seconds"
1494 response_headers += [
1495 ('Refresh', '5; url=.'),
1496 ]
1497 else:
1498 base_uri = environ['REQUEST_URI']
1499 uri = base_uri.strip('/').split('/')[1:]
1500
1501 output = "Template Holder"
1502 if base_uri.endswith('/create/network.kml'):
1503 content_type='application/vnd.google-earth.kml+xml'
1504 output = make_network_kml.make_graph()
1505 elif base_uri.endswith('/api/get/nodeplanner.json'):
1506 content_type='application/json'
1507 output = make_network_kml.make_nodeplanner_json()
1508 elif not uri:
1509 if is_text_request(environ):
1510 output = '\n'.join(get_hostlist())
1511 else:
1512 content_type = 'text/html'
1513 output = generate_title(get_hostlist())
1514 elif len(uri) == 1:
1515 if is_text_request(environ):
1516 output = generate_node(uri[0])
1517 else:
1518 content_type = 'text/html'
1519 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1520 elif len(uri) == 2:
1521 output = generate_config(uri[0], uri[1])
1522 else:
1523 assert False, "Invalid option"
1524
1525 # Return response
1526 response_headers += [
1527 ('Content-type', content_type),
1528 ('Content-Length', str(len(output))),
1529 ]
1530 return(response_headers, str(output))
1531
1532
1533def make_dns(output_dir = 'dns', external = False):
1534 items = dict()
1535
1536 # hostname is key, IP is value
1537 wleiden_zone = defaultdict(list)
1538 wleiden_cname = dict()
1539
1540 pool = dict()
1541 for node in get_hostlist():
1542 datadump = get_yaml(node)
1543
1544 fqdn = datadump['nodename']
1545
1546 if datadump.has_key('rdr_host'):
1547 remote_target = datadump['rdr_host']
1548 elif datadump.has_key('remote_access') and datadump['remote_access']:
1549 remote_target = datadump['remote_access'].split(':')[0]
1550 else:
1551 remote_target = None
1552
1553 if remote_target:
1554 try:
1555 parseaddr(remote_target)
1556 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1557 except (IndexError, ValueError):
1558 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1559
1560
1561 wleiden_zone[fqdn].append((datadump['masterip'], True))
1562
1563 # Hacking to get proper DHCP IPs and hostnames
1564 for iface_key in get_interface_keys(datadump):
1565 iface_name = iface_key.replace('_','-')
1566 if 'ip' in datadump[iface_key]:
1567 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1568 try:
1569 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1570 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1571 dhcp_part = ".".join(ip.split('.')[0:3])
1572 if ip != datadump['masterip']:
1573 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1574 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1575 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1576 except (AttributeError, ValueError, KeyError):
1577 # First push it into a pool, to indentify the counter-part later on
1578 addr = parseaddr(ip)
1579 cidr = int(cidr)
1580 addr = addr & ~((1 << (32 - cidr)) - 1)
1581 if pool.has_key(addr):
1582 pool[addr] += [(iface_name, fqdn, ip)]
1583 else:
1584 pool[addr] = [(iface_name, fqdn, ip)]
1585 continue
1586
1587
1588
1589 # WL uses an /29 to configure an interface. IP's are ordered like this:
1590 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1591
1592 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1593
1594 # Automatic naming convention of interlinks namely 2 + remote.lower()
1595 for (key,value) in pool.iteritems():
1596 # Make sure they are sorted from low-ip to high-ip
1597 value = sorted(value, key=lambda x: parseaddr(x[2]))
1598
1599 if len(value) == 1:
1600 (iface_name, fqdn, ip) = value[0]
1601 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1602
1603 # Device DNS names
1604 if 'cnode' in fqdn.lower():
1605 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1606 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1607
1608 elif len(value) == 2:
1609 (a_iface_name, a_fqdn, a_ip) = value[0]
1610 (b_iface_name, b_fqdn, b_ip) = value[1]
1611 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1612 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1613
1614 # Device DNS names
1615 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1616 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1617 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1618 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1619 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1620 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1621 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1622
1623 else:
1624 pool_members = [k[1] for k in value]
1625 for item in value:
1626 (iface_name, fqdn, ip) = item
1627 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1628
1629 # Include static DNS entries
1630 # XXX: Should they override the autogenerated results?
1631 # XXX: Convert input to yaml more useable.
1632 # Format:
1633 ##; this is a comment
1634 ## roomburgh=Roomburgh1
1635 ## apkerk1.Vosko=172.17.176.8 ;this as well
1636 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1637
1638 # Hack to allow special entries, for development
1639 wleiden_raw = {}
1640
1641 for line in dns_list:
1642 reverse = False
1643 k, items = line.items()[0]
1644 if type(items) == dict:
1645 if items.has_key('reverse'):
1646 reverse = items['reverse']
1647 items = items['a']
1648 else:
1649 items = items['cname']
1650 items = [items] if type(items) != list else items
1651 for item in items:
1652 if item.startswith('IN '):
1653 wleiden_raw[k] = item
1654 elif valid_addr(item):
1655 wleiden_zone[k].append((item, reverse))
1656 else:
1657 wleiden_cname[k] = item
1658
1659 # Hack to get dynamic pool listing
1660 def chunks(l, n):
1661 return [l[i:i+n] for i in range(0, len(l), n)]
1662
1663 ntp_servers = [x[0] for x in get_nameservers()]
1664 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1665 for ntp_server in chunk:
1666 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1667
1668 details = dict()
1669 # 24 updates a day allowed
1670 details['serial'] = time.strftime('%Y%m%d%H')
1671
1672 if external:
1673 dns_masters = ['ns1.vanderzwet.net', 'ns1.transip.net', 'ns2.transip.eu']
1674 else:
1675 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1676
1677 details['master'] = dns_masters[0]
1678 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1679
1680 dns_header = '''
1681$TTL 3h
1682%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1683 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1684
1685%(ns_servers)s
1686 \n'''
1687
1688
1689 if not os.path.isdir(output_dir):
1690 os.makedirs(output_dir)
1691 details['zone'] = 'wleiden.net'
1692 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1693 f.write(dns_header % details)
1694
1695 for host,items in wleiden_zone.iteritems():
1696 for ip,reverse in items:
1697 if ip not in ['0.0.0.0']:
1698 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1699 for source,dest in wleiden_cname.iteritems():
1700 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1701 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1702 for source, dest in wleiden_raw.iteritems():
1703 f.write("%s.wleiden.net. %s\n" % (source, dest))
1704 f.close()
1705
1706 # Create whole bunch of specific sub arpa zones. To keep it compliant
1707 for s in range(16,32):
1708 details['zone'] = '%i.172.in-addr.arpa' % s
1709 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1710 f.write(dns_header % details)
1711
1712 #XXX: Not effient, fix to proper data structure and do checks at other
1713 # stages
1714 for host,items in wleiden_zone.iteritems():
1715 for ip,reverse in items:
1716 if not reverse:
1717 continue
1718 if valid_addr(ip):
1719 if valid_addr(ip):
1720 if int(ip.split('.')[1]) == s:
1721 rev_ip = '.'.join(reversed(ip.split('.')))
1722 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1723 f.close()
1724
1725
1726def usage():
1727 print """Usage: %(prog)s <argument>
1728Argument:
1729\tcleanup = Cleanup all YAML files to specified format
1730\tstandalone [port] = Run configurator webserver [8000]
1731\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1732\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1733\tfull-export = Generate yaml export script for heatmap.
1734\tstatic [outputdir] = Generate all config files and store on disk
1735\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1736\ttest <node> [<file>] = Receive output for certain node [all files].
1737\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1738\tlist <status> <items> = List systems which have certain status
1739\tcreate network.kml = Create Network KML file for use in Google Earth
1740
1741Arguments:
1742\t<node> = NodeName (example: HybridRick)
1743\t<file> = %(files)s
1744\t<status> = all|up|down|planned
1745\t<items> = systems|nodes|proxies
1746
1747NOTE FOR DEVELOPERS; you can test your changes like this:
1748 BEFORE any changes in this code:
1749 $ ./gformat.py static /tmp/pre
1750 AFTER the changes:
1751 $ ./gformat.py static /tmp/post
1752 VIEW differences and VERIFY all are OK:
1753 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1754""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1755 exit(0)
1756
1757
1758def is_text_request(environ=os.environ):
1759 """ Find out whether we are calling from the CLI or any text based CLI utility """
1760 if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1761 return True
1762
1763 if 'HTTP_USER_AGENT' in environ:
1764 return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1765 else:
1766 return False
1767
1768
1769def switchFormat(setting):
1770 if setting:
1771 return "YES"
1772 else:
1773 return "NO"
1774
1775def rlinput(prompt, prefill=''):
1776 import readline
1777 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1778 try:
1779 return raw_input(prompt)
1780 finally:
1781 readline.set_startup_hook()
1782
1783def fix_conflict(left, right, default='i'):
1784 while True:
1785 print "## %-30s | %-30s" % (left, right)
1786 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1787 if not c:
1788 c = default
1789
1790 if c in ['l','1']:
1791 return left
1792 elif c in ['r','2']:
1793 return right
1794 elif c in ['e', '3']:
1795 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1796 elif c in ['i', '4']:
1797 return None
1798 else:
1799 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1800
1801
1802
1803def print_cgi_response(response_headers, output):
1804 for header in response_headers:
1805 print "%s: %s" % header
1806 print
1807 print output
1808
1809
1810
1811def main():
1812 """Hard working sub"""
1813 # Allow easy hacking using the CLI
1814 if not os.environ.has_key('REQUEST_URI'):
1815 if len(sys.argv) < 2:
1816 usage()
1817
1818 if sys.argv[1] == "standalone":
1819 import SocketServer
1820 import CGIHTTPServer
1821 # Hop to the right working directory.
1822 os.chdir(os.path.dirname(__file__))
1823 try:
1824 PORT = int(sys.argv[2])
1825 except (IndexError,ValueError):
1826 PORT = 8000
1827
1828 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1829 """ Serve this CGI from the root of the webserver """
1830 def is_cgi(self):
1831 if "favicon" in self.path:
1832 return False
1833
1834 self.cgi_info = (os.path.basename(__file__), self.path)
1835 self.path = ''
1836 return True
1837 handler = MyCGIHTTPRequestHandler
1838 SocketServer.TCPServer.allow_reuse_address = True
1839 httpd = SocketServer.TCPServer(("", PORT), handler)
1840 httpd.server_name = 'localhost'
1841 httpd.server_port = PORT
1842
1843 logger.info("serving at port %s", PORT)
1844 try:
1845 httpd.serve_forever()
1846 except KeyboardInterrupt:
1847 httpd.shutdown()
1848 logger.info("All done goodbye")
1849 elif sys.argv[1] == "test":
1850 # Basic argument validation
1851 try:
1852 node = sys.argv[2]
1853 except IndexError:
1854 print "Invalid argument"
1855 exit(1)
1856 except IOError as e:
1857 print e
1858 exit(1)
1859
1860 datadump = get_yaml(node)
1861
1862
1863 # Get files to generate
1864 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1865
1866 # Actual config generation
1867 for config in gen_files:
1868 logger.info("## Generating %s %s", node, config)
1869 print generate_config(node, config, datadump)
1870 elif sys.argv[1] == "test-cgi":
1871 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1872 os.environ['SCRIPT_NAME'] = __file__
1873 response_headers, output = process_cgi_request()
1874 print_cgi_response(response_headers, output)
1875 elif sys.argv[1] == "static":
1876 generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
1877 elif sys.argv[1] == "wind-export":
1878 items = dict()
1879 for node in get_hostlist():
1880 datadump = get_yaml(node)
1881 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1882 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1883 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1884 VALUES (
1885 (SELECT id FROM users WHERE username = 'rvdzwet'),
1886 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1887 'Y');""" % datadump
1888 #for config in files:
1889 # items['config'] = config
1890 # print "## Generating %(node)s %(config)s" % items
1891 # f = open("%(wdir)s/%(config)s" % items, "w")
1892 # f.write(generate_config(node, config, datadump))
1893 # f.close()
1894 for node in get_hostlist():
1895 datadump = get_yaml(node)
1896 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1897 ifacedump = datadump[iface_key]
1898 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1899 ifacedump['nodename'] = datadump['nodename']
1900 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1901 ifacedump['channel'] = 0
1902 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1903 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1904 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1905 elif sys.argv[1] == "nagios-export":
1906 try:
1907 heavy_load = (sys.argv[2] == "--heavy-load")
1908 except IndexError:
1909 heavy_load = False
1910
1911 hostgroup_details = {
1912 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1913 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1914 'walphen' : 'Stichting Wireless Alphen',
1915 'westeinder' : 'Westeinder Plassen',
1916 }
1917
1918 # Convert IP to Host
1919 ip2host = {'root' : 'root'}
1920 for host in get_hostlist():
1921 datadump = get_yaml(host)
1922 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1923 for iface in get_interface_keys(datadump):
1924 if datadump[iface].has_key('autogen_gateway'):
1925 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1926
1927 # Find dependency tree based on output of lvrouted.mytree of nearest node
1928 parents = defaultdict(list)
1929 stack = ['root']
1930 prev_depth = 0
1931 for line in open('lvrouted.mytree').readlines():
1932 depth = line.count('\t')
1933 ip = line.strip().split()[0]
1934
1935 if prev_depth < depth:
1936 try:
1937 parents[ip2host[ip]].append(ip2host[stack[-1]])
1938 except KeyError as e:
1939 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1940 stack.append(ip)
1941 elif prev_depth > depth:
1942 stack = stack[:(depth - prev_depth)]
1943 elif prev_depth == depth:
1944 try:
1945 parents[ip2host[ip]].append(ip2host[stack[-1]])
1946 except KeyError as e:
1947 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1948
1949
1950 prev_depth = depth
1951 # Observe that some nodes has themself as parent or multiple parents
1952 # for now take only the first parent, other behaviour is yet to be explained
1953
1954
1955
1956 params = {
1957 'check_interval' : 5 if heavy_load else 120,
1958 'retry_interval' : 1 if heavy_load else 10,
1959 'max_check_attempts' : 10 if heavy_load else 6,
1960 'notification_interval': 120 if heavy_load else 240,
1961 }
1962
1963 print '''\
1964define host {
1965 name wleiden-node ; Default Node Template
1966 use generic-host ; Use the standard template as initial starting point
1967 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1968 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1969 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1970 notification_interval %(notification_interval)s
1971 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1972 check_command check-host-alive ; Default command to check FreeBSD hosts
1973 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1974}
1975
1976define service {
1977 name wleiden-service ; Default Service Template
1978 use generic-service ; Use the standard template as initial starting point
1979 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1980 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1981 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1982 notification_interval %(notification_interval)s
1983 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1984 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1985}
1986
1987# Please make sure to install:
1988# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
1989#
1990# Recompile net-mgmt/nagios-plugins to support check_snmp
1991# make -C /usr/ports/net-mgmt/nagios-plugins
1992#
1993# Install net/bind-tools to allow v2/check_dns_wl to work:
1994# pkg install bind-tools
1995#
1996define command{
1997 command_name check_snmp_disk
1998 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
1999}
2000
2001define command{
2002 command_name check_netsnmp_load
2003 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2004}
2005
2006define command{
2007 command_name check_netsnmp_proc
2008 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2009}
2010
2011define command{
2012 command_name check_by_ssh
2013 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2014}
2015
2016define command{
2017 command_name check_dns_wl
2018 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2019}
2020
2021define command{
2022 command_name check_snmp_uptime
2023 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2024}
2025
2026
2027# TDB: dhcp leases
2028# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2029
2030# TDB: internet status
2031# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2032
2033# TDB: Advanced local passive checks
2034# /usr/local/libexec/nagios/check_by_ssh
2035''' % params
2036
2037 print '''\
2038# Service Group, not displayed by default
2039define hostgroup {
2040 hostgroup_name srv_hybrid
2041 alias All Hybrid Nodes
2042 register 0
2043}
2044
2045define service {
2046 use wleiden-service
2047 hostgroup_name srv_hybrid
2048 service_description SSH
2049 check_command check_ssh
2050}
2051
2052define service {
2053 use wleiden-service,service-pnp
2054 hostgroup_name srv_hybrid
2055 service_description HTTP
2056 check_command check_http
2057}
2058
2059define service {
2060 use wleiden-service
2061 hostgroup_name srv_hybrid
2062 service_description DNS
2063 check_command check_dns_wl!"www.wirelessleiden.nl"
2064}
2065'''
2066
2067 if heavy_load:
2068 print '''\
2069define service {
2070 use wleiden-service
2071 hostgroup_name srv_hybrid
2072 service_description UPTIME
2073 check_command check_snmp_uptime
2074}
2075
2076#define service {
2077# use wleiden-service
2078# hostgroup_name srv_hybrid
2079# service_description NTP
2080# check_command check_ntp_peer
2081#}
2082
2083define service {
2084 use wleiden-service
2085 hostgroup_name srv_hybrid
2086 service_description LOAD
2087 check_command check_netsnmp_load
2088}
2089
2090define service {
2091 use wleiden-service
2092 hostgroup_name srv_hybrid
2093 service_description PROC
2094 check_command check_netsnmp_proc
2095}
2096
2097define service {
2098 use wleiden-service
2099 hostgroup_name srv_hybrid
2100 service_description DISK
2101 check_command check_snmp_disk
2102}
2103'''
2104 for node in get_hostlist():
2105 datadump = get_yaml(node)
2106 if not datadump['status'] == 'up':
2107 continue
2108 if not hostgroup_details.has_key(datadump['monitoring_group']):
2109 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2110 print '''\
2111define host {
2112 use wleiden-node,host-pnp
2113 contact_groups admins
2114 host_name %(autogen_fqdn)s
2115 address %(masterip)s
2116 hostgroups srv_hybrid,%(monitoring_group)s\
2117''' % datadump
2118 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2119 print '''\
2120 parents %(parents)s\
2121''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2122 print '''\
2123}
2124'''
2125
2126
2127 for name,alias in hostgroup_details.iteritems():
2128 print '''\
2129define hostgroup {
2130 hostgroup_name %s
2131 alias %s
2132} ''' % (name, alias)
2133
2134
2135 elif sys.argv[1] == "full-export":
2136 hosts = {}
2137 for node in get_hostlist():
2138 datadump = get_yaml(node)
2139 hosts[datadump['nodename']] = datadump
2140 print yaml.dump(hosts)
2141
2142 elif sys.argv[1] == "dns":
2143 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2144 elif sys.argv[1] == "cleanup":
2145 # First generate all datadumps
2146 datadumps = dict()
2147 ssid_to_node = dict()
2148 for host in get_hostlist():
2149 logger.info("# Processing: %s", host)
2150 # Set some boring default values
2151 datadump = { 'board' : 'UNKNOWN' }
2152 datadump.update(get_yaml(host))
2153 datadumps[datadump['nodename']] = datadump
2154
2155 (poel, errors) = make_relations()
2156 print "\n".join(["# WARNING: %s" % x for x in errors])
2157
2158 for host,datadump in datadumps.iteritems():
2159 try:
2160 # Convert all yes and no to boolean values
2161 def fix_boolean(dump):
2162 for key in dump.keys():
2163 if type(dump[key]) == dict:
2164 dump[key] = fix_boolean(dump[key])
2165 elif str(dump[key]).lower() in ["yes", "true"]:
2166 dump[key] = True
2167 elif str(dump[key]).lower() in ["no", "false"]:
2168 # Compass richting no (Noord Oost) is valid input
2169 if key != "compass": dump[key] = False
2170 return dump
2171 datadump = fix_boolean(datadump)
2172
2173 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2174 if not 'latitude' in datadump and not 'longitude' in datadump:
2175 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2176 elif 'latitude' in datadump and 'longitude' in datadump:
2177 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2178 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2179 # TODO: Compare outcome of both coordinate systems and validate against each-other
2180
2181 if datadump['nodename'].startswith('Proxy'):
2182 datadump['nodename'] = datadump['nodename'].lower()
2183
2184 for iface_key in get_interface_keys(datadump):
2185 try:
2186 # All our normal wireless cards are normal APs now
2187 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2188 datadump[iface_key]['mode'] = 'ap'
2189 # Wireless Leiden SSID have an consistent lowercase/uppercase
2190 if datadump[iface_key].has_key('ssid'):
2191 ssid = datadump[iface_key]['ssid']
2192 prefix = 'ap-WirelessLeiden-'
2193 if ssid.lower().startswith(prefix.lower()):
2194 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2195 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2196 datadump[iface_key]['mode'] = 'autogen-FIXME'
2197 if not datadump[iface_key].has_key('comment'):
2198 datadump[iface_key]['comment'] = 'autogen-FIXME'
2199
2200 if datadump[iface_key].has_key('ns_mac'):
2201 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2202
2203 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2204 datadump[iface_key] = datadump[iface_key]['desc']
2205
2206 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2207 # due to better collision avoidance.
2208 if datadump[iface_key]['type'] == '11b':
2209 datadump[iface_key]['type'] = '11g'
2210
2211 # Setting 802.11g channels to de-facto standards, to avoid
2212 # un-detected sharing with other overlapping channels
2213 #
2214 # Technically we could also use channel 13 in NL, but this is not
2215 # recommended as foreign devices might not be able to select this
2216 # channel. Secondly using 1,5,9,13 instead is going to clash with
2217 # the de-facto usage of 1,6,11.
2218 #
2219 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2220 channels_at_2400Mhz = (1,6,11)
2221 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2222 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2223 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2224 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2225
2226 # Mandatory interface keys
2227 if not datadump[iface_key].has_key('status'):
2228 datadump[iface_key]['status'] = 'planned'
2229
2230 x = datadump[iface_key]['comment']
2231 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2232
2233 # Fixing bridge_type if none is found
2234 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2235 if not 'bridge_type' in datadump[iface_key]:
2236 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2237
2238 # Making sure description works
2239 if datadump[iface_key].has_key('desc'):
2240 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2241 del datadump[iface_key]['desc']
2242 else:
2243 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2244 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2245 if response:
2246 datadump[iface_key]['comment'] = response
2247 del datadump[iface_key]['desc']
2248
2249 # Check DHCP configuration
2250 dhcp_type(datadump[iface_key])
2251
2252 # Set the compass value based on the angle between the poels
2253 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2254 my_pool = poel[network(datadump[iface_key]['ip'])]
2255 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2256 if remote_hosts:
2257 compass_target = remote_hosts[0]
2258 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2259 # TODO: Compass wanted and actual direction might differ
2260
2261 # Monitoring Group default
2262 if not 'monitoring_group' in datadump:
2263 datadump['monitoring_group'] = 'wleiden'
2264
2265 except Exception:
2266 print "# Error while processing interface %s" % iface_key
2267 raise
2268 store_yaml(datadump)
2269 except Exception:
2270 print "# Error while processing %s" % host
2271 raise
2272 elif sys.argv[1] == "list":
2273 use_fqdn = False
2274 if len(sys.argv) < 4:
2275 usage()
2276 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2277 usage()
2278 if not sys.argv[3] in ["nodes","proxies","systems"]:
2279 usage()
2280
2281 if len(sys.argv) > 4:
2282 if sys.argv[4] == "fqdn":
2283 use_fqdn = True
2284 else:
2285 usage()
2286
2287 for system in get_hostlist():
2288 datadump = get_yaml(system)
2289 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2290 continue
2291
2292 output = datadump['autogen_fqdn'] if use_fqdn else system
2293 if sys.argv[2] == "all":
2294 print output
2295 elif datadump['status'] == sys.argv[2]:
2296 print output
2297 elif sys.argv[1] == "create":
2298 if sys.argv[2] == "network.kml":
2299 print make_network_kml.make_graph()
2300 elif sys.argv[2] == "host-ips.txt":
2301 for system in get_hostlist():
2302 datadump = get_yaml(system)
2303 ips = [datadump['masterip']]
2304 for ifkey in get_interface_keys(datadump):
2305 ips.append(datadump[ifkey]['ip'].split('/')[0])
2306 print system, ' '.join(ips)
2307 elif sys.argv[2] == "host-pos.txt":
2308 for system in get_hostlist():
2309 datadump = get_yaml(system)
2310 print system, datadump['rdnap_x'], datadump['rdnap_y']
2311 elif sys.argv[2] == 'ssh_config':
2312 print '''
2313Host *.wleiden.net
2314 User root
2315
2316Host 172.16.*.*
2317 User root
2318'''
2319 for system in get_hostlist():
2320 datadump = get_yaml(system)
2321 print '''\
2322Host %s
2323 User root
2324
2325Host %s
2326 User root
2327
2328Host %s
2329 User root
2330
2331Host %s
2332 User root
2333''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2334 else:
2335 usage()
2336 else:
2337 usage()
2338 else:
2339 # Do not enable debugging for config requests as it highly clutters the output
2340 if not is_text_request():
2341 cgitb.enable()
2342 response_headers, output = process_cgi_request()
2343 print_cgi_response(response_headers, output)
2344
2345if __name__ == "__main__":
2346 main()
Note: See TracBrowser for help on using the repository browser.