source: genesis/tools/gformat.py@ 14213

Last change on this file since 14213 was 14206, checked in by rick, 6 years ago

Add option for whitelisting services on proxy

To avoid clutter on the network a non-proxy service also requires
this list to be present. This could how-ever yield to unwanted results if the
proxy dynamically changes to one which have a lesser set of features allowed.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 80.7 KB
Line 
1#!/usr/local/bin/python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
35SVNVERSION = filter(os.path.isfile, ('/usr/local/bin/svnversion', '/usr/bin/svnversion'))[0]
36
37import argparse
38import cgi
39import cgitb
40import copy
41import glob
42import make_network_kml
43import math
44import pyproj
45import random
46import re
47import socket
48import string
49import subprocess
50import textwrap
51import time
52import traceback
53import urlparse
54
55from pprint import pprint
56from collections import defaultdict, OrderedDict
57from sys import stderr
58try:
59 import yaml
60except ImportError, e:
61 print e
62 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
63 exit(1)
64
65try:
66 from yaml import CLoader as Loader
67 from yaml import CDumper as Dumper
68except ImportError:
69 from yaml import Loader, Dumper
70
71from jinja2 import Environment, Template
72def yesorno(value):
73 return "YES" if bool(value) else "NO"
74env = Environment()
75env.filters['yesorno'] = yesorno
76def render_template(datadump, template):
77 result = env.from_string(template).render(datadump)
78 # Make it look pretty to the naked eye, as jinja templates are not so
79 # friendly when it comes to whitespace formatting
80 ## Remove extra whitespace at end of line lstrip() style.
81 result = re.sub(r'\n[\ ]+','\n', result)
82 ## Include only a single newline between an definition and a comment
83 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
84 ## Remove extra newlines after single comment
85 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
86 return result
87
88import logging
89logging.basicConfig(format='# %(levelname)s: %(message)s' )
90logger = logging.getLogger()
91logger.setLevel(logging.DEBUG)
92
93
94if os.environ.has_key('CONFIGROOT'):
95 NODE_DIR = os.environ['CONFIGROOT']
96else:
97 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
98__version__ = '$Id: gformat.py 14206 2018-08-29 19:03:25Z rick $'
99
100CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
101
102files = [
103 'authorized_keys',
104 'dnsmasq.conf',
105 'dhcpd.conf',
106 'rc.conf.local',
107 'resolv.conf',
108 'motd',
109 'ntp.conf',
110 'pf.hybrid.conf.local',
111 'unbound.wleiden.conf',
112 'wleiden.yaml',
113 ]
114
115# Global variables uses
116OK = 10
117DOWN = 20
118UNKNOWN = 90
119
120
121ileiden_proxies = OrderedDict()
122normal_proxies = []
123datadump_cache = {}
124interface_list_cache = {}
125rc_conf_local_cache = {}
126nameservers_cache = []
127relations_cache = None
128
129NO_DHCP = 0
130DHCP_CLIENT = 10
131DHCP_SERVER = 20
132def dhcp_type(item):
133 if not item.has_key('dhcp'):
134 return NO_DHCP
135 elif not item['dhcp']:
136 return NO_DHCP
137 elif item['dhcp'].lower() == 'client':
138 return DHCP_CLIENT
139 else:
140 # Validation Checks
141 begin,end = map(int,item['dhcp'].split('-'))
142 if begin >= end:
143 raise ValueError("DHCP Start >= DHCP End")
144 return DHCP_SERVER
145
146def etrs2rd(lat, lon):
147 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
148 p2 = pyproj.Proj(init='EPSG:28992')
149 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
150 return (RDx, RDy)
151
152def rd2etrs(RDx, RDy):
153 p1 = pyproj.Proj(init='EPSG:28992')
154 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
155 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
156 return (lat, lon)
157
158def get_yaml(item,add_version_info=True):
159 try:
160 """ Get configuration yaml for 'item'"""
161 if datadump_cache.has_key(item):
162 return datadump_cache[item].copy()
163
164 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
165 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
166 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
167
168 # Default values
169 datadump = {
170 'autogen_revision' : 'NOTFOUND',
171 'autogen_gfile' : gfile,
172 'service_proxy_ileiden' : False,
173 'publicnat' : ['http', 'https'],
174 }
175 f = open(gfile, 'r')
176 datadump.update(yaml.load(f,Loader=Loader))
177 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
178 if datadump['nodetype'] == 'Hybrid':
179 # Some values are defined implicitly
180 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
181 datadump['service_incoming_rdr'] = True
182 # Use some boring defaults
183 defaults = {
184 'service_proxy_normal' : False,
185 'service_accesspoint' : True,
186 'service_incoming_rdr' : False,
187 'service_concentrator' : False,
188 'monitoring_group' : 'wleiden',
189 }
190 for (key,value) in defaults.iteritems():
191 if not datadump.has_key(key):
192 datadump[key] = value
193 f.close()
194
195 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
196 if add_version_info:
197 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
198 lines = p.communicate()[0].split('\n')
199 if p.returncode == 0:
200 for line in lines:
201 if line:
202 (key, value) = line.split(': ')
203 datadump["autogen_" + key.lower().replace(' ','_')] = value
204
205 # Preformat certain needed variables for formatting and push those into special object
206 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
207
208 wlan_count=0
209 try:
210 for key in get_interface_keys(datadump, True):
211 datadump[key]['autogen_ifbase'] = key.split('_')[1]
212 datadump[key]['autogen_vlan'] = False
213 datadump[key]['autogen_vlan_alias'] = False
214
215 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
216 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
217 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
218
219 if datadump[key].has_key('parent'):
220 if datadump[key].has_key('ip'):
221 raise ValueError("Interface bridge member cannot have IP assigned")
222 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
223 raise ValueError("Interface bridge member cannot have DHCP set")
224
225 if datadump[key].has_key('ip'):
226 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
227
228 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
229 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
230 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
231 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
232 wlan_count += 1
233 else:
234 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
235 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
236 datadump[key]['autogen_if_dhcp'] = '.'.join(key.split('_')[1:3])
237 datadump[key]['autogen_vlan'] = key.split('_')[2]
238 datadump[key]['autogen_vlan_alias'] = '_alias' in key
239 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
240 else:
241 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
242 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
243
244 except Exception as exc:
245 exc.args = ("# Error while processing interface %s" % key,) + exc.args
246 raise
247
248 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
249 if dhcp_type(datadump[key]) == DHCP_SERVER]
250
251 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
252 datadump['autogen_item'] = item
253
254 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
255 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
256 datadump_cache[item] = datadump.copy()
257 except Exception as exc:
258 exc.args = ("# Error while processing %s" % item,) + exc.args
259 raise
260 return datadump
261
262
263def store_yaml(datadump, header=False):
264 """ Store configuration yaml for 'item'"""
265 item = datadump['autogen_item']
266 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
267
268 output = generate_wleiden_yaml(datadump, header)
269
270 f = open(gfile, 'w')
271 f.write(output)
272 f.close()
273
274
275def network(ip):
276 addr, mask = ip.split('/')
277 # Not parsing of these folks please
278 addr = parseaddr(addr)
279 mask = int(mask)
280 network = addr & ~((1 << (32 - mask)) - 1)
281 return network
282
283
284
285def make_relations():
286 """ Process _ALL_ yaml files to get connection relations """
287 global relations_cache
288
289 if relations_cache:
290 return relations_cache
291
292 errors = []
293 poel = defaultdict(list)
294
295 for host in get_hostlist():
296 datadump = get_yaml(host)
297 try:
298 for iface_key in get_interface_keys(datadump):
299 # Bridge members has no IP assigned
300 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
301 continue
302 net_addr = network(datadump[iface_key]['ip'])
303 poel[net_addr] += [(host,datadump[iface_key].copy())]
304 except (KeyError, ValueError) as e:
305 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
306 continue
307
308 relations_cache = (poel, errors)
309 return relations_cache
310
311
312
313def valid_addr(addr):
314 """ Show which address is valid in which are not """
315 return str(addr).startswith('172.')
316
317def get_hostlist():
318 """ Combined hosts and proxy list"""
319 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
320
321def angle_between_points(lat1,lat2,long1,long2):
322 """
323 Return Angle in radians between two GPS coordinates
324 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
325 """
326 dy = lat2 - lat1
327 dx = math.cos(lat1)*(long2 - long1)
328 angle = math.atan2(dy,dx)
329 return angle
330
331
332
333def angle_to_cd(angle):
334 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
335
336 # For easy conversion get positive degree
337 degrees = math.degrees(angle)
338 abs_degrees = 360 + degrees if degrees < 0 else degrees
339
340 # Numbers can be confusing calculate from the 4 main directions
341 p = 22.5
342 if abs_degrees < p:
343 cd = "n"
344 elif abs_degrees < (90 - p):
345 cd = "no"
346 elif abs_degrees < (90 + p):
347 cd = "o"
348 elif abs_degrees < (180 - p):
349 cd = "zo"
350 elif abs_degrees < (180 + p):
351 cd = "z"
352 elif abs_degrees < (270 - p):
353 cd = "zw"
354 elif abs_degrees < (270 + p):
355 cd = "w"
356 elif abs_degrees < (360 - p):
357 cd = "nw"
358 else:
359 cd = "n"
360 return cd
361
362
363
364def cd_between_hosts(hostA, hostB, datadumps):
365 # Using RDNAP coordinates
366 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
367 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
368 return angle_to_cd(math.atan2(dx,dy))
369
370 # GPS coordinates seems to fail somehow
371 #latA = float(datadumps[hostA]['latitude'])
372 #latB = float(datadumps[hostB]['latitude'])
373 #lonA = float(datadumps[hostA]['longitude'])
374 #lonB = float(datadumps[hostB]['longitude'])
375 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
376
377
378def generate_title(nodelist):
379 """ Main overview page """
380 items = { \
381 'root' : ".",
382 'version' : subprocess.Popen([SVNVERSION, "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
383 }
384 def fl(spaces, line):
385 return (' ' * spaces) + line + '\n'
386
387 output = """
388<html>
389 <head>
390 <title>Wireless leiden Configurator - GFormat</title>
391 <style type="text/css">
392 th {background-color: #999999}
393 tr:nth-child(odd) {background-color: #cccccc}
394 tr:nth-child(even) {background-color: #ffffff}
395 th, td {padding: 0.1em 1em}
396 </style>
397 </head>
398 <body>
399 <center>
400 <form type="GET" action="%(root)s">
401 <input type="hidden" name="action" value="update">
402 <input type="submit" value="Update Configuration Database (SVN)">
403 </form>
404 <table>
405 <caption><h3>Wireless Leiden Configurator - Revision %(version)s</h3></caption>
406 """ % items
407
408 for node in nodelist:
409 items['node'] = node
410 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
411 for config in files:
412 items['config'] = config
413 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
414 output += fl(5, "</tr>")
415 output += """
416 </table>
417 <hr />
418 <em>%s</em>
419 </center>
420 </body>
421</html>
422 """ % __version__
423
424 return output
425
426
427
428def generate_node(node):
429 """ Print overview of all files available for node """
430 return "\n".join(files)
431
432def generate_node_overview(host, datadump=False):
433 """ Print overview of all files available for node """
434 if not datadump:
435 datadump = get_yaml(host)
436 params = { 'host' : host }
437 output = "<em><a href='..'>Back to overview</a></em><hr />"
438 output += "<h2>Available files:</h2><ul>"
439 for cf in files:
440 params['cf'] = cf
441 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
442 output += "</ul>"
443
444 # Generate and connection listing
445 output += "<h2>Connected To:</h2><ul>"
446 (poel, errors) = make_relations()
447 for network, hosts in poel.iteritems():
448 if host in [x[0] for x in hosts]:
449 if len(hosts) == 1:
450 # Single not connected interface
451 continue
452 for remote,ifacedump in hosts:
453 if remote == host:
454 # This side of the interface
455 continue
456 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
457 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
458 output += "</ul>"
459 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
460
461 output += "<hr /><em><a href='..'>Back to overview</a></em>"
462 return output
463
464
465def generate_header(datadump, ctag="#"):
466 return """\
467%(ctag)s
468%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
469%(ctag)s
470""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
471
472
473
474def parseaddr(s):
475 """ Process IPv4 CIDR notation addr to a (binary) number """
476 f = s.split('.')
477 return (long(f[0]) << 24L) + \
478 (long(f[1]) << 16L) + \
479 (long(f[2]) << 8L) + \
480 long(f[3])
481
482
483
484def showaddr(a):
485 """ Display IPv4 addr in (dotted) CIDR notation """
486 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
487
488
489def is_member(ip, mask, canidate):
490 """ Return True if canidate is part of ip/mask block"""
491 ip_addr = parseaddr(ip)
492 ip_canidate = parseaddr(canidate)
493 mask = int(mask)
494 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
495 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
496 return ip_addr == ip_canidate
497
498
499
500def cidr2netmask(netmask):
501 """ Given a 'netmask' return corresponding CIDR """
502 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
503
504def get_network(addr, mask):
505 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
506
507
508def generate_dhcpd_conf(datadump):
509 """ Generate config file '/usr/local/etc/dhcpd.conf """
510 # Redundency support, in cause local DNS server is not running/responding.
511 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
512 output = generate_header(datadump)
513 output += Template("""\
514# option definitions common to all supported networks...
515option domain-name "dhcp.{{ autogen_fqdn }}";
516
517default-lease-time 600;
518max-lease-time 7200;
519
520# Use this to enble / disable dynamic dns updates globally.
521#ddns-update-style none;
522
523# If this DHCP server is the official DHCP server for the local
524# network, the authoritative directive should be uncommented.
525authoritative;
526
527# Use this to send dhcp log messages to a different log file (you also
528# have to hack syslog.conf to complete the redirection).
529log-facility local7;
530
531# UniFi Discovery Support
532option space ubnt;
533option ubnt.unifi-address code 1 = ip-address;
534#
535class "ubnt" {
536 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
537 option vendor-class-identifier "ubnt";
538 vendor-option-space ubnt;
539}
540
541#
542# Interface definitions
543#
544\n\n""").render(datadump)
545
546
547 # TODO: Use textwrap.fill instead
548 def indent(text, count):
549 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
550
551 # Process DHCP blocks
552 dhcp_out = defaultdict(list)
553 for iface_key in get_interface_keys(datadump):
554 ifname = datadump[iface_key]['autogen_ifbase']
555 groupif = datadump[iface_key]['autogen_if_dhcp']
556 if not datadump[iface_key].has_key('comment'):
557 datadump[iface_key]['comment'] = None
558
559 if not datadump[iface_key].has_key('ip'):
560 continue
561
562 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
563
564 (addr, mask) = datadump[iface_key]['ip'].split('/')
565 datadump[iface_key]['autogen_addr'] = addr
566 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
567 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
568
569 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
570 dhcp_out[groupif].append(textwrap.dedent("""\
571 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
572 ### not autoritive
573 }
574 """ % datadump[iface_key]))
575 continue
576
577 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
578 dhcp_part = ".".join(addr.split('.')[0:3])
579 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
580 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
581 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
582
583 # Assume the first 10 IPs could be used for static entries
584 if 'no_portal' in datadump:
585 fixed = 5
586 for mac in datadump['no_portal']:
587 dhcp_out[groupif].append(textwrap.dedent("""\
588 host fixed-%(ifname)s-%(fixed)s {
589 hardware ethernet %(mac)s;
590 fixed-address %(prefix)s.%(fixed)s;
591 }
592 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
593 fixed += 1
594
595 if 'dhcp_fixed' in datadump[iface_key]:
596 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
597 dhcp_out[groupif].append(textwrap.dedent("""\
598 host fixed-%(host)s {
599 hardware ethernet %(mac)s;
600 fixed-address %(addr)s;
601 }
602 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
603
604
605 dhcp_out[groupif].append(textwrap.dedent("""\
606 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
607 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
608 option routers %(autogen_addr)s;
609 option domain-name-servers %(autogen_dns_servers)s;
610 option ubnt.unifi-address 172.17.107.10;
611 }
612 """ % datadump[iface_key]))
613
614 # Output the blocks in groups
615 for ifname,value in sorted(dhcp_out.iteritems()):
616 output += ("shared-network \"%s\" {\n" % ifname) + indent(''.join(value), 2).rstrip() + '\n}\n\n'
617 return output
618
619
620
621def generate_dnsmasq_conf(datadump):
622 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
623 output = generate_header(datadump)
624 output += Template("""\
625# DHCP server options
626dhcp-authoritative
627dhcp-fqdn
628domain=dhcp.{{ autogen_fqdn }}
629domain-needed
630expand-hosts
631log-async=100
632
633# Low memory footprint
634cache-size=10000
635
636\n""").render(datadump)
637
638 for iface_key in get_interface_keys(datadump):
639 if not datadump[iface_key].has_key('comment'):
640 datadump[iface_key]['comment'] = None
641 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
642
643 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
644 output += "# not autoritive\n\n"
645 continue
646
647 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
648 (ip, cidr) = datadump[iface_key]['ip'].split('/')
649 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
650
651 dhcp_part = ".".join(ip.split('.')[0:3])
652 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
653 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
654 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
655
656 return output
657
658
659class AutoVivification(dict):
660 """Implementation of perl's autovivification feature."""
661 def __getitem__(self, item):
662 try:
663 return dict.__getitem__(self, item)
664 except KeyError:
665 value = self[item] = type(self)()
666 return value
667
668def make_interface_list(datadump):
669 if interface_list_cache.has_key(datadump['autogen_item']):
670 return (interface_list_cache[datadump['autogen_item']])
671 # lo0 configuration:
672 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
673 # - masterip is special as it needs to be assigned to at
674 # least one interface, so if not used assign to lo0
675 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
676 vlan_list = defaultdict(list)
677 bridge_list = defaultdict(list)
678 flags_if = AutoVivification()
679 dhclient_if = {'lo0' : False}
680
681 # XXX: Find some way of send this output nicely
682 output = ''
683
684 masterip_used = False
685 for iface_key in get_interface_keys(datadump):
686 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
687 masterip_used = True
688 break
689 if not masterip_used:
690 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
691
692 if 'serviceid' in datadump:
693 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
694
695 for iface_key in get_interface_keys(datadump):
696 ifacedump = datadump[iface_key]
697 if ifacedump['autogen_bridge_alias']:
698 ifname = ifacedump['autogen_ifbase']
699 else:
700 ifname = ifacedump['autogen_ifname']
701
702 # If defined as vlan interface
703 if ifacedump['autogen_vlan']:
704 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
705
706 # If defined as bridge interface
707 if ifacedump['autogen_bridge_member']:
708 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
709
710
711 # Flag dhclient is possible
712 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
713 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
714
715 # Ethernet address
716 if ifacedump.has_key('ether'):
717 flags_if[ifname]['ether'] = ifacedump['ether']
718
719 # Add interface IP to list
720 if ifacedump.has_key('ip'):
721 item = (ifacedump['ip'], ifacedump['comment'])
722 if addrs_list.has_key(ifname):
723 addrs_list[ifname].append(item)
724 else:
725 addrs_list[ifname] = [item]
726
727 # Alias only needs IP assignment for now, this might change if we
728 # are going to use virtual accesspoints
729 if "alias" in iface_key:
730 continue
731
732 # XXX: Might want to deduct type directly from interface name
733 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
734 # Default to station (client) mode
735 ifacedump['autogen_wlanmode'] = "sta"
736 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
737 ifacedump['autogen_wlanmode'] = "ap"
738
739 if not ifacedump.has_key('channel'):
740 if ifacedump['type'] == '11a':
741 ifacedump['channel'] = 36
742 else:
743 ifacedump['channel'] = 1
744
745 # Allow special hacks at the back like wds and stuff
746 if not ifacedump.has_key('extra'):
747 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
748 else:
749 ifacedump['autogen_extra'] = ifacedump['extra']
750
751 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
752
753 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
754 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
755 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
756 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
757 output += "\n"
758
759 elif ifacedump['type'] in ['ethernet', 'eth']:
760 # No special config needed besides IP
761 pass
762 elif ifacedump['type'] in ['vlan']:
763 # VLAN member has no special configuration
764 pass
765 else:
766 assert False, "Unknown type " + ifacedump['type']
767
768 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
769 interface_list_cache[datadump['autogen_item']] = store
770 return(store)
771
772
773
774def create_proxies_list():
775 if not ileiden_proxies or not normal_proxies:
776 # Placeholder for to-be-installed proxies, this will avoid updating the all
777 # nodes to include this new machine, yet due to an unbound issue, this list
778 # has to be kept small.
779
780 for i in range(1,20):
781 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
782
783 for host in get_hostlist():
784 hostdump = get_yaml(host)
785 if hostdump['status'] == 'up':
786 if hostdump['service_proxy_ileiden']:
787 ileiden_proxies[hostdump['serviceid']] = hostdump
788 if hostdump['service_proxy_normal']:
789 normal_proxies.append(hostdump)
790
791
792
793def generate_rc_conf_local(datadump):
794 """ Generate configuration file '/etc/rc.conf.local' """
795 item = datadump['autogen_item']
796 if rc_conf_local_cache.has_key(item):
797 return rc_conf_local_cache[item]
798
799 if not datadump.has_key('ileiden'):
800 datadump['autogen_ileiden_enable'] = False
801 else:
802 datadump['autogen_ileiden_enable'] = datadump['ileiden']
803
804 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
805
806 create_proxies_list()
807 datadump['autogen_ileiden_proxies'] = ileiden_proxies
808 datadump['autogen_normal_proxies'] = normal_proxies
809 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
810 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
811 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
812 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
813
814 output = generate_header(datadump, "#");
815 output += render_template(datadump, """\
816hostname='{{ autogen_fqdn }}'
817location='{{ location }}'
818nodetype="{{ nodetype }}"
819
820#
821# Configured listings
822#
823captive_portal_whitelist=""
824{% if nodetype == "Proxy" %}
825#
826# Proxy Configuration
827#
828{% if gateway and service_proxy_ileiden -%}
829defaultrouter="{{ gateway }}"
830{% else -%}
831#defaultrouter="NOTSET"
832{% endif -%}
833internalif="{{ internalif }}"
834ileiden_enable="{{ autogen_ileiden_enable }}"
835gateway_enable="{{ autogen_ileiden_enable }}"
836pf_enable="yes"
837pf_rules="/etc/pf.conf"
838{% if autogen_ileiden_enable -%}
839pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={{ publicnat|join(',') }} -D ileiden_ports={{ publicnat|join(',') }}"
840lvrouted_enable="{{ autogen_ileiden_enable }}"
841lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
842{% else -%}
843pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
844{% endif -%}
845{% if internalroute -%}
846static_routes="wleiden"
847route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
848{% endif -%}
849
850{% elif nodetype == "Hybrid" %}
851 #
852 # Hybrid Configuration
853 #
854 list_ileiden_proxies="
855 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
856 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
857 {% endfor -%}
858 "
859 list_normal_proxies="
860 {% for item in autogen_normal_proxies -%}
861 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
862 {% endfor -%}
863 "
864
865 {% if autogen_dhcp_interfaces -%}
866 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
867 {% else %}
868 captive_portal_interfaces="dummy"
869 {% endif %}
870 externalif="{{ externalif|default('vr0', true) }}"
871 masterip="{{ masterip }}"
872
873 {% if gateway and service_proxy_ileiden %}
874 defaultrouter="{{ gateway }}"
875 {% else %}
876 #defaultrouter="NOTSET"
877 {% endif %}
878
879 #
880 # Defined services
881 #
882 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
883 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
884 service_accesspoint="{{ service_accesspoint|yesorno }}"
885 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
886 service_concentrator="{{ service_concentrator|yesorno }}"
887
888 {% if service_proxy_ileiden %}
889 pf_rules="/etc/pf.hybrid.conf"
890 {% if service_concentrator %}
891 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
892 {% else %}
893 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
894 {% endif %}
895 pf_flags="$pf_flags -D publicnat={{ publicnat|join(',') }}"
896 lvrouted_flags="$lvrouted_flags -g"
897 {% elif service_proxy_normal or service_incoming_rdr %}
898 pf_rules="/etc/pf.hybrid.conf"
899 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
900 pf_flags="$pf_flags -D publicnat=0"
901 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
902 named_setfib="1"
903 tinyproxy_setfib="1"
904 dnsmasq_setfib="1"
905 sshd_setfib="1"
906 {% else %}
907 named_auto_forward_only="YES"
908 pf_rules="/etc/pf.node.conf"
909 pf_flags="-D ileiden_ports={{ publicnat|join(',') }}"
910 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
911 {% endif %}
912 {% if service_concentrator %}
913 # Do mind installing certificates is NOT done automatically for security reasons
914 openvpn_enable="YES"
915 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
916 {% endif %}
917
918 {% if service_proxy_normal %}
919 tinyproxy_enable="yes"
920 {% else %}
921 pen_wrapper_enable="yes"
922 {% endif %}
923
924 {% if service_accesspoint %}
925 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
926 {% endif %}
927
928 {% if board == "ALIX2" or board == "net4801" %}
929 #
930 # ''Fat'' configuration, board has 256MB RAM
931 #
932 dnsmasq_enable="NO"
933 named_enable="YES"
934 unbound_enable="YES"
935 {% if autogen_dhcp_interfaces -%}
936 dhcpd_enable="YES"
937 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
938 {% endif -%}
939 {% elif board == "apu1d" %}
940 #
941 # ''Fat'' configuration, board has 1024MB RAM
942 #
943 dnsmasq_enable="NO"
944 unbound_enable="YES"
945 {% if autogen_dhcp_interfaces -%}
946 dhcpd_enable="YES"
947 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
948 {% endif -%}
949 {% endif -%}
950{% endif %}
951
952#
953# Script variables
954#
955attached_devices="{{ autogen_attached_devices|join(' ') }}"
956neighbours="{{ autogen_neighbours|join(' ') }}"
957
958
959#
960# Interface definitions
961#\n
962""")
963
964 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
965 for iface, vlans in sorted(vlan_list.items()):
966 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
967
968 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
969 for iface in sorted(vlan_list.keys()):
970 if not iface in addrs_list.keys():
971 output += "ifconfig_%s='up'\n" % iface
972
973 output += "\n"
974
975 # Bridge configuration:
976 if bridge_list.keys():
977 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
978
979 for iface in bridge_list.keys():
980 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
981
982 # Bridge member interfaces not containing a configuration should be marked active explcitly.
983 for _,members in bridge_list.items():
984 for iface in members:
985 if not iface in addrs_list.keys():
986 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
987
988 output += "\n"
989
990 # Details like SSID
991 if extra_ouput:
992 output += extra_ouput.strip() + "\n"
993
994 # Print IP address which needs to be assigned over here
995 output += "\n"
996 for iface,addrs in sorted(addrs_list.iteritems()):
997 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
998 output += "# %s || %s || %s\n" % (iface, addr, comment)
999
1000 # Write DHCLIENT entry
1001 if iface in dhclient_if and dhclient_if[iface]:
1002 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1003 continue
1004
1005 # Make sure the external address is always first as this is needed in the
1006 # firewall setup
1007 addrs = sorted(
1008 [x for x in addrs if not '0.0.0.0' in x[0]],
1009 key=lambda x: x[0].split('.')[0],
1010 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1011 )
1012
1013 idx_offset = 0
1014 # Set MAC is required
1015 if flags_if[iface].has_key('ether'):
1016 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1017 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1018 idx_offset += 1
1019 elif iface in bridge_list:
1020 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1021 idx_offset += 1
1022 else:
1023 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1024
1025 for idx, addr in enumerate(addrs[1:]):
1026 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1027
1028 output += "\n"
1029
1030 rc_conf_local_cache[datadump['autogen_item']] = output
1031 return output
1032
1033
1034
1035def get_all_configs():
1036 """ Get dict with key 'host' with all configs present """
1037 configs = dict()
1038 for host in get_hostlist():
1039 datadump = get_yaml(host)
1040 configs[host] = datadump
1041 return configs
1042
1043
1044def get_interface_keys(config, extra=False):
1045 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1046 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1047 if extra == False:
1048 return filter(lambda x: not "extra" in x, elems)
1049 else:
1050 return elems
1051
1052
1053def get_used_ips(configs):
1054 """ Return array of all IPs used in config files"""
1055 ip_list = []
1056 for config in configs:
1057 ip_list.append(config['masterip'])
1058 if 'serviceid' in config:
1059 ip_list.append(config['serviceid'])
1060 for iface_key in get_interface_keys(config, True):
1061 l = config[iface_key]['ip']
1062 addr, mask = l.split('/')
1063 # Special case do not process
1064 if valid_addr(addr):
1065 ip_list.append(addr)
1066 else:
1067 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1068 return sorted(ip_list)
1069
1070
1071
1072def get_nameservers(max_servers=None):
1073 if nameservers_cache:
1074 return nameservers_cache[0:max_servers]
1075
1076 for host in get_hostlist():
1077 hostdump = get_yaml(host)
1078 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1079 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1080
1081 return nameservers_cache[0:max_servers]
1082
1083
1084def get_neighbours(datadump):
1085 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1086
1087 (poel, errors) = make_relations()
1088 table = []
1089 for iface,addrs in sorted(addrs_list.iteritems()):
1090 if iface in ['lo0']:
1091 continue
1092
1093 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1094 if not addr.startswith('172.'):
1095 # Avoid listing internet connections as pool
1096 continue
1097 for neighbour in poel[network(addr)]:
1098 if neighbour[0] != datadump['autogen_item']:
1099 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1100 return table
1101
1102
1103def get_attached_devices(datadump, url=False):
1104 table = []
1105 for iface_key in get_interface_keys(datadump, True):
1106 ifacedump = datadump[iface_key]
1107
1108 if not ifacedump.has_key('ns_ip'):
1109 continue
1110
1111 x_ip = ifacedump['ns_ip'].split('/')[0]
1112
1113 if 'mode' in ifacedump:
1114 x_mode = ifacedump['mode']
1115 else:
1116 x_mode = 'unknown'
1117
1118 if 'bridge_type' in ifacedump:
1119 device_type = ifacedump['bridge_type']
1120 else:
1121 device_type = 'Unknown'
1122
1123 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1124 return table
1125
1126
1127def generate_resolv_conf(datadump):
1128 """ Generate configuration file '/etc/resolv.conf' """
1129 # XXX: This should properly going to be an datastructure soon
1130 datadump['autogen_header'] = generate_header(datadump, "#")
1131 datadump['autogen_edge_nameservers'] = ''
1132
1133
1134 for masterip,realname in get_nameservers():
1135 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1136
1137 return Template("""\
1138{{ autogen_header }}
1139search wleiden.net
1140
1141# Try local (cache) first
1142nameserver 127.0.0.1
1143
1144{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1145nameserver 8.8.8.8 # Google Public NameServer
1146nameserver 64.6.64.6 # Verisign Public NameServer
1147{% else -%}
1148# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1149{{ autogen_edge_nameservers }}
1150{% endif -%}
1151""").render(datadump)
1152
1153
1154
1155def generate_ntp_conf(datadump):
1156 """ Generate configuration file '/etc/ntp.conf' """
1157 # XXX: This should properly going to be an datastructure soon
1158
1159 datadump['autogen_header'] = generate_header(datadump, "#")
1160 datadump['autogen_ntp_servers'] = ''
1161 for host in get_hostlist():
1162 hostdump = get_yaml(host)
1163 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1164 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1165
1166 return Template("""\
1167{{ autogen_header }}
1168
1169{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1170# Machine hooked to internet.
1171server 0.nl.pool.ntp.org iburst maxpoll 9
1172server 1.nl.pool.ntp.org iburst maxpoll 9
1173server 2.nl.pool.ntp.org iburst maxpoll 9
1174server 3.nl.pool.ntp.org iburst maxpoll 9
1175{% else -%}
1176# Local Wireless Leiden NTP Servers.
1177server 0.pool.ntp.wleiden.net iburst maxpoll 9
1178server 1.pool.ntp.wleiden.net iburst maxpoll 9
1179server 2.pool.ntp.wleiden.net iburst maxpoll 9
1180server 3.pool.ntp.wleiden.net iburst maxpoll 9
1181
1182# All the configured NTP servers
1183{{ autogen_ntp_servers }}
1184{% endif %}
1185
1186# If a server loses sync with all upstream servers, NTP clients
1187# no longer follow that server. The local clock can be configured
1188# to provide a time source when this happens, but it should usually
1189# be configured on just one server on a network. For more details see
1190# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1191# The use of Orphan Mode may be preferable.
1192#
1193server 127.127.1.0
1194fudge 127.127.1.0 stratum 10
1195""").render(datadump)
1196
1197
1198def generate_pf_hybrid_conf_local(datadump):
1199 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1200 datadump['autogen_header'] = generate_header(datadump, "#")
1201 if datadump['service_incoming_rdr']:
1202 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1203 return Template("""\
1204{{ autogen_header }}
1205
1206# Redirect some internal facing services outside (7)
1207# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1208{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1209rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1210{% endfor -%}
1211# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1212{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1213rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1214{% endfor -%}
1215""").render(datadump)
1216
1217def generate_unbound_wleiden_conf(datadump):
1218 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1219 datadump['autogen_header'] = generate_header(datadump, "#")
1220
1221 autogen_ips = []
1222 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1223 for iface,addrs in sorted(addrs_list.iteritems()):
1224 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1225 if addr.startswith('172'):
1226 autogen_ips.append((addr.split('/')[0], comment))
1227 datadump['autogen_ips'] = autogen_ips
1228
1229 create_proxies_list()
1230 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1231 return Template("""\
1232{{ autogen_header }}
1233
1234server:
1235 ## Static definitions fail to start on systems with broken ue(4) interfaces
1236{%- for ip,comment in autogen_ips %}
1237 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1238{%- endfor %}
1239 ## Enabling wildcard matching as work-around
1240 interface: 0.0.0.0
1241 interface: ::0
1242
1243forward-zone:
1244 name: '.'
1245{%- if service_proxy_ileiden %}
1246 forward-addr: 8.8.8.8 # Google DNS A
1247 forward-addr: 8.8.4.4 # Google DNS B
1248 forward-addr: 208.67.222.222 # OpenDNS DNS A
1249 forward-addr: 208.67.220.220 # OpenDNS DNS B
1250{% else -%}
1251{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1252 {%- if loop.index <= 5 %}
1253 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1254 {%- endif %}
1255{%- endfor %}
1256{%- endif %}
1257""").render(datadump)
1258
1259def generate_motd(datadump):
1260 """ Generate configuration file '/etc/motd' """
1261 output = Template("""\
1262FreeBSD run ``service motd onestart'' to make me look normal
1263
1264 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1265 Loc: {{ location }}
1266
1267Services:
1268{% if board == "ALIX2" or board == "net4801" -%}
1269{{" -"}} Core Node ({{ board }})
1270{% else -%}
1271{{" -"}} Hulp Node ({{ board }})
1272{% endif -%}
1273{% if service_proxy_normal -%}
1274{{" -"}} Normal Proxy
1275{% endif -%}
1276{% if service_proxy_ileiden -%}
1277{{" -"}} iLeiden Proxy
1278{% endif -%}
1279{% if service_incoming_rdr -%}
1280{{" -"}} Incoming port redirects
1281{% endif %}
1282Interlinks:\n
1283""").render(datadump)
1284
1285
1286 def make_table(table):
1287 if not table:
1288 return " - none\n"
1289 else:
1290 lines = ""
1291 col_width = [max(len(x) for x in col) for col in zip(*table)]
1292 for row in table:
1293 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1294 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1295 return lines
1296
1297 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1298 table = []
1299 for iface,addrs in sorted(addrs_list.iteritems()):
1300 if iface in ['lo0']:
1301 continue
1302 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1303 table.append((iface, addr, comment))
1304
1305 output += make_table(table)
1306 output += '\n'
1307 output += """\
1308Attached devices:
1309"""
1310 output += make_table(get_attached_devices(datadump, url=True))
1311 output += '\n'
1312 output += """\
1313Available neighbours:
1314"""
1315 output += make_table(get_neighbours(datadump))
1316
1317 return output
1318
1319
1320def format_yaml_value(value):
1321 """ Get yaml value in right syntax for outputting """
1322 if isinstance(value,str):
1323 output = '"%s"' % value
1324 else:
1325 output = value
1326 return output
1327
1328
1329
1330def format_wleiden_yaml(datadump):
1331 """ Special formatting to ensure it is editable"""
1332 output = "# Genesis config yaml style\n"
1333 output += "# vim:ts=2:et:sw=2:ai\n"
1334 output += "#\n"
1335 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1336 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1337 if key == 'rdr_rules':
1338 output += '%-10s:\n' % 'rdr_rules'
1339 for rdr_rule in datadump[key]:
1340 output += '- %s\n' % rdr_rule
1341 else:
1342 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1343
1344 output += "\n\n"
1345
1346 # Format (key, required)
1347 key_order = (
1348 ('comment', True),
1349 ('parent', False),
1350 ('ip', False),
1351 ('ipv6', False),
1352 ('ether', False),
1353 ('desc', True),
1354 ('sdesc', True),
1355 ('mode', True),
1356 ('type', True),
1357 ('extra_type', False),
1358 ('channel', False),
1359 ('ssid', False),
1360 ('wlan_mac', False),
1361 ('dhcp', True),
1362 ('dhcp_fixed', False),
1363 ('compass', False),
1364 ('distance', False),
1365 ('ns_ip', False),
1366 ('repeater_ip', False),
1367 ('bullet2_ip', False),
1368 ('ns_mac', False),
1369 ('bullet2_mac', False),
1370 ('ns_type', False),
1371 ('bridge_type', False),
1372 ('encrypted', False),
1373 ('status', True),
1374 )
1375
1376 for iface_key in sorted(iface_keys):
1377 try:
1378 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1379 if remainder:
1380 raise KeyError("invalid keys: %s" % remainder)
1381
1382 output += "%s:\n" % iface_key
1383 for key,required in key_order:
1384 if datadump[iface_key].has_key(key):
1385 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1386 output += "\n\n"
1387 except Exception as exc:
1388 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
1389 raise
1390
1391 return output
1392
1393
1394
1395def generate_wleiden_yaml(datadump, header=True):
1396 """ Generate (petty) version of wleiden.yaml"""
1397 output = generate_header(datadump, "#") if header else ''
1398
1399 for key in datadump.keys():
1400 if key.startswith('autogen_'):
1401 del datadump[key]
1402 # Interface autogen cleanups
1403 elif type(datadump[key]) == dict:
1404 for key2 in datadump[key].keys():
1405 if key2.startswith('autogen_'):
1406 del datadump[key][key2]
1407
1408 output += format_wleiden_yaml(datadump)
1409 return output
1410
1411def generate_nanostation_config(datadump, iface, ns_type):
1412 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1413 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1414
1415 datadump.update(datadump['iface_%s' % iface])
1416
1417 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1418
1419def generate_yaml(datadump):
1420 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1421
1422
1423
1424def generate_config(node, config, datadump=None):
1425 """ Print configuration file 'config' of 'node' """
1426 output = ""
1427 try:
1428 # Load config file
1429 if datadump == None:
1430 datadump = get_yaml(node)
1431
1432 if config == 'wleiden.yaml':
1433 output += generate_wleiden_yaml(datadump)
1434 elif config == 'authorized_keys':
1435 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1436 output += f.read()
1437 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1438 # Fetch local keys if existing
1439 if os.path.exists(node_keys):
1440 output += open(node_keys, 'r').read()
1441 f.close()
1442 elif config == 'dnsmasq.conf':
1443 output += generate_dnsmasq_conf(datadump)
1444 elif config == 'dhcpd.conf':
1445 output += generate_dhcpd_conf(datadump)
1446 elif config == 'rc.conf.local':
1447 output += generate_rc_conf_local(datadump)
1448 elif config == 'resolv.conf':
1449 output += generate_resolv_conf(datadump)
1450 elif config == 'ntp.conf':
1451 output += generate_ntp_conf(datadump)
1452 elif config == 'motd':
1453 output += generate_motd(datadump)
1454 elif config == 'pf.hybrid.conf.local':
1455 output += generate_pf_hybrid_conf_local(datadump)
1456 elif config == 'unbound.wleiden.conf':
1457 output += generate_unbound_wleiden_conf(datadump)
1458 elif config.startswith('vr'):
1459 interface, ns_type = config.strip('.yaml').split('-')
1460 output += generate_nanostation_config(datadump, interface, ns_type)
1461 else:
1462 assert False, "Config not found!"
1463 except IOError, e:
1464 output += "[ERROR] Config file not found"
1465 return output
1466
1467
1468
1469def generate_static(output_dir, logging=True):
1470 items = {'output_dir' : output_dir}
1471 for node in get_hostlist():
1472 items['node'] = node
1473 items['wdir'] = "%(output_dir)s/%(node)s" % items
1474 if not os.path.isdir(items['wdir']):
1475 os.makedirs(items['wdir'])
1476 datadump = get_yaml(node)
1477 f = open("%(wdir)s/index.html" % items, "w")
1478 f.write(generate_node_overview(items['node'], datadump))
1479 f.close()
1480 for config in files:
1481 items['config'] = config
1482 if logging: logger.info("## Generating %(node)s %(config)s" % items)
1483 f = open("%(wdir)s/%(config)s" % items, "w")
1484 f.write(generate_config(node, config, datadump))
1485 f.close()
1486
1487
1488
1489def process_cgi_request(environ=os.environ):
1490 """ When calling from CGI """
1491 response_headers = []
1492 content_type = 'text/plain'
1493
1494 # Update repository if requested
1495 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1496 if form and form.has_key("action") and "update" in form["action"]:
1497 refresh_rate = 5
1498 output = "[INFO] Updating subverion, please wait...\n"
1499 old_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1500 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1501 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1502 new_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1503 if old_version != new_version:
1504 try:
1505 generate_static(CACHE_DIR, False)
1506 except:
1507 output += traceback.format_exc()
1508 refresh_rate = 120
1509 pass
1510 output += "[INFO] All done, redirecting in %s seconds" % refresh_rate
1511 response_headers += [
1512 ('Refresh', '%s; url=.' % refresh_rate),
1513 ]
1514 else:
1515 # Bootstap cache directory if none exists
1516 if not glob.glob(os.path.join(CACHE_DIR,'*','index.html')):
1517 generate_static(CACHE_DIR, False)
1518
1519 # URL must be of format <prefix>/config/<path>
1520 base_uri = environ['REQUEST_URI']
1521 uri = filter(None, base_uri.split('/config/')[1].strip('/').split('/'))
1522
1523 output = "Template Holder"
1524 if base_uri.endswith('/create/network.kml'):
1525 content_type='application/vnd.google-earth.kml+xml'
1526 output = make_network_kml.make_graph()
1527 elif base_uri.endswith('/api/get/nodeplanner.json'):
1528 content_type='application/json'
1529 output = make_network_kml.make_nodeplanner_json()
1530 elif not uri:
1531 if is_text_request(environ):
1532 output = '\n'.join(get_hostlist())
1533 else:
1534 content_type = 'text/html'
1535 output = generate_title(get_hostlist())
1536 elif len(uri) == 1:
1537 if is_text_request(environ):
1538 output = generate_node(uri[0])
1539 else:
1540 content_type = 'text/html'
1541 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1542 elif len(uri) == 2:
1543 output = generate_config(uri[0], uri[1])
1544 else:
1545 assert False, "Invalid option"
1546
1547 # Return response
1548 response_headers += [
1549 ('Content-type', content_type),
1550 ('Content-Length', str(len(output))),
1551 ]
1552 return(response_headers, str(output))
1553
1554
1555def make_dns(output_dir = 'dns', external = False):
1556 items = dict()
1557
1558 # hostname is key, IP is value
1559 wleiden_zone = defaultdict(list)
1560 wleiden_cname = dict()
1561
1562 pool = dict()
1563 for node in get_hostlist():
1564 datadump = get_yaml(node)
1565
1566 fqdn = datadump['nodename']
1567
1568 if datadump.has_key('rdr_host'):
1569 remote_target = datadump['rdr_host']
1570 elif datadump.has_key('remote_access') and datadump['remote_access']:
1571 remote_target = datadump['remote_access'].split(':')[0]
1572 else:
1573 remote_target = None
1574
1575 if remote_target:
1576 try:
1577 parseaddr(remote_target)
1578 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1579 except (IndexError, ValueError):
1580 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1581
1582
1583 wleiden_zone[fqdn].append((datadump['masterip'], True))
1584
1585 # Hacking to get proper DHCP IPs and hostnames
1586 for iface_key in get_interface_keys(datadump):
1587 iface_name = iface_key.replace('_','-')
1588 if 'ip' in datadump[iface_key]:
1589 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1590 try:
1591 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1592 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1593 dhcp_part = ".".join(ip.split('.')[0:3])
1594 if ip != datadump['masterip']:
1595 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1596 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1597 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1598 except (AttributeError, ValueError, KeyError):
1599 # First push it into a pool, to indentify the counter-part later on
1600 addr = parseaddr(ip)
1601 cidr = int(cidr)
1602 addr = addr & ~((1 << (32 - cidr)) - 1)
1603 if pool.has_key(addr):
1604 pool[addr] += [(iface_name, fqdn, ip)]
1605 else:
1606 pool[addr] = [(iface_name, fqdn, ip)]
1607 continue
1608
1609
1610
1611 # WL uses an /29 to configure an interface. IP's are ordered like this:
1612 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1613
1614 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1615
1616 # Automatic naming convention of interlinks namely 2 + remote.lower()
1617 for (key,value) in pool.iteritems():
1618 # Make sure they are sorted from low-ip to high-ip
1619 value = sorted(value, key=lambda x: parseaddr(x[2]))
1620
1621 if len(value) == 1:
1622 (iface_name, fqdn, ip) = value[0]
1623 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1624
1625 # Device DNS names
1626 if 'cnode' in fqdn.lower():
1627 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1628 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1629
1630 elif len(value) == 2:
1631 (a_iface_name, a_fqdn, a_ip) = value[0]
1632 (b_iface_name, b_fqdn, b_ip) = value[1]
1633 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1634 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1635
1636 # Device DNS names
1637 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1638 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1639 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1640 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1641 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1642 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1643 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1644
1645 else:
1646 pool_members = [k[1] for k in value]
1647 for item in value:
1648 (iface_name, fqdn, ip) = item
1649 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1650
1651 # Include static DNS entries
1652 # XXX: Should they override the autogenerated results?
1653 # XXX: Convert input to yaml more useable.
1654 # Format:
1655 ##; this is a comment
1656 ## roomburgh=Roomburgh1
1657 ## apkerk1.Vosko=172.17.176.8 ;this as well
1658 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1659
1660 # Hack to allow special entries, for development
1661 wleiden_raw = {}
1662
1663 for line in dns_list:
1664 reverse = False
1665 k, items = line.items()[0]
1666 if type(items) == dict:
1667 if items.has_key('reverse'):
1668 reverse = items['reverse']
1669 items = items['a']
1670 else:
1671 items = items['cname']
1672 items = [items] if type(items) != list else items
1673 for item in items:
1674 if item.startswith('IN '):
1675 wleiden_raw[k] = item
1676 elif valid_addr(item):
1677 wleiden_zone[k].append((item, reverse))
1678 else:
1679 wleiden_cname[k] = item
1680
1681 # Hack to get dynamic pool listing
1682 def chunks(l, n):
1683 return [l[i:i+n] for i in range(0, len(l), n)]
1684
1685 ntp_servers = [x[0] for x in get_nameservers()]
1686 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1687 for ntp_server in chunk:
1688 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1689
1690 details = dict()
1691 # 24 updates a day allowed
1692 details['serial'] = time.strftime('%Y%m%d%H')
1693
1694 if external:
1695 dns_masters = ['ns1.vanderzwet.net', 'ns1.anywi.com']
1696 else:
1697 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1698
1699 details['master'] = dns_masters[0]
1700 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1701
1702 dns_header = '''
1703$TTL 3h
1704%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 3h )
1705 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1706
1707%(ns_servers)s
1708 \n'''
1709
1710
1711 if not os.path.isdir(output_dir):
1712 os.makedirs(output_dir)
1713 details['zone'] = 'wleiden.net'
1714 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1715 f.write(dns_header % details)
1716
1717 for host,items in wleiden_zone.iteritems():
1718 for ip,reverse in items:
1719 if ip not in ['0.0.0.0']:
1720 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1721 for source,dest in wleiden_cname.iteritems():
1722 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1723 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1724 for source, dest in wleiden_raw.iteritems():
1725 f.write("%s.wleiden.net. %s\n" % (source, dest))
1726 f.close()
1727
1728 # Create whole bunch of specific sub arpa zones. To keep it compliant
1729 for s in range(16,32):
1730 details['zone'] = '%i.172.in-addr.arpa' % s
1731 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1732 f.write(dns_header % details)
1733
1734 #XXX: Not effient, fix to proper data structure and do checks at other
1735 # stages
1736 for host,items in wleiden_zone.iteritems():
1737 for ip,reverse in items:
1738 if not reverse:
1739 continue
1740 if valid_addr(ip):
1741 if valid_addr(ip):
1742 if int(ip.split('.')[1]) == s:
1743 rev_ip = '.'.join(reversed(ip.split('.')))
1744 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1745 f.close()
1746
1747
1748def usage():
1749 print """Usage: %(prog)s <argument>
1750Argument:
1751\tcleanup = Cleanup all YAML files to specified format
1752\tstandalone [port] = Run configurator webserver [8000]
1753\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1754\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1755\tfull-export = Generate yaml export script for heatmap.
1756\tstatic [outputdir] = Generate all config files and store on disk
1757\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1758\ttest <node> [<file>] = Receive output for certain node [all files].
1759\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1760\tlist <status> <items> = List systems which have certain status
1761\tcreate network.kml = Create Network KML file for use in Google Earth
1762
1763Arguments:
1764\t<node> = NodeName (example: HybridRick)
1765\t<file> = %(files)s
1766\t<status> = all|up|down|planned
1767\t<items> = systems|nodes|proxies
1768
1769NOTE FOR DEVELOPERS; you can test your changes like this:
1770 BEFORE any changes in this code:
1771 $ ./gformat.py static /tmp/pre
1772 AFTER the changes:
1773 $ ./gformat.py static /tmp/post
1774 VIEW differences and VERIFY all are OK:
1775 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1776""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1777 exit(0)
1778
1779
1780def is_text_request(environ=os.environ):
1781 """ Find out whether we are calling from the CLI or any text based CLI utility """
1782 if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1783 return True
1784
1785 if 'HTTP_USER_AGENT' in environ:
1786 return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1787 else:
1788 return False
1789
1790
1791def switchFormat(setting):
1792 if setting:
1793 return "YES"
1794 else:
1795 return "NO"
1796
1797def rlinput(prompt, prefill=''):
1798 import readline
1799 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1800 try:
1801 return raw_input(prompt)
1802 finally:
1803 readline.set_startup_hook()
1804
1805def fix_conflict(left, right, default='i'):
1806 while True:
1807 print "## %-30s | %-30s" % (left, right)
1808 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1809 if not c:
1810 c = default
1811
1812 if c in ['l','1']:
1813 return left
1814 elif c in ['r','2']:
1815 return right
1816 elif c in ['e', '3']:
1817 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1818 elif c in ['i', '4']:
1819 return None
1820 else:
1821 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1822
1823
1824
1825def print_cgi_response(response_headers, output):
1826 for header in response_headers:
1827 print "%s: %s" % header
1828 print
1829 print output
1830
1831
1832
1833def main():
1834 """Hard working sub"""
1835 # Allow easy hacking using the CLI
1836 if not os.environ.has_key('REQUEST_URI'):
1837 if len(sys.argv) < 2:
1838 usage()
1839
1840 if sys.argv[1] == "standalone":
1841 import SocketServer
1842 import CGIHTTPServer
1843 # Hop to the right working directory.
1844 os.chdir(os.path.dirname(__file__))
1845 try:
1846 PORT = int(sys.argv[2])
1847 except (IndexError,ValueError):
1848 PORT = 8000
1849
1850 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1851 """ Serve this CGI from the root of the webserver """
1852 def is_cgi(self):
1853 if "favicon" in self.path:
1854 return False
1855
1856 self.cgi_info = (os.path.basename(__file__), self.path)
1857 self.path = ''
1858 return True
1859 handler = MyCGIHTTPRequestHandler
1860 SocketServer.TCPServer.allow_reuse_address = True
1861 httpd = SocketServer.TCPServer(("", PORT), handler)
1862 httpd.server_name = 'localhost'
1863 httpd.server_port = PORT
1864
1865 logger.info("serving at port %s", PORT)
1866 try:
1867 httpd.serve_forever()
1868 except KeyboardInterrupt:
1869 httpd.shutdown()
1870 logger.info("All done goodbye")
1871 elif sys.argv[1] == "test":
1872 # Basic argument validation
1873 try:
1874 node = sys.argv[2]
1875 except IndexError:
1876 print "Invalid argument"
1877 exit(1)
1878 except IOError as e:
1879 print e
1880 exit(1)
1881
1882 datadump = get_yaml(node)
1883
1884
1885 # Get files to generate
1886 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1887
1888 # Actual config generation
1889 for config in gen_files:
1890 logger.info("## Generating %s %s", node, config)
1891 print generate_config(node, config, datadump)
1892 elif sys.argv[1] == "test-cgi":
1893 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1894 os.environ['SCRIPT_NAME'] = __file__
1895 response_headers, output = process_cgi_request()
1896 print_cgi_response(response_headers, output)
1897 elif sys.argv[1] == "static":
1898 generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
1899 elif sys.argv[1] == "wind-export":
1900 items = dict()
1901 for node in get_hostlist():
1902 datadump = get_yaml(node)
1903 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1904 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1905 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1906 VALUES (
1907 (SELECT id FROM users WHERE username = 'rvdzwet'),
1908 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1909 'Y');""" % datadump
1910 #for config in files:
1911 # items['config'] = config
1912 # print "## Generating %(node)s %(config)s" % items
1913 # f = open("%(wdir)s/%(config)s" % items, "w")
1914 # f.write(generate_config(node, config, datadump))
1915 # f.close()
1916 for node in get_hostlist():
1917 datadump = get_yaml(node)
1918 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1919 ifacedump = datadump[iface_key]
1920 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1921 ifacedump['nodename'] = datadump['nodename']
1922 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1923 ifacedump['channel'] = 0
1924 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1925 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1926 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1927 elif sys.argv[1] == "smokeping-export":
1928 for host in get_hostlist():
1929 datadump = get_yaml(host)
1930 if datadump.get('service_proxy_normal', False) or datadump.get('service_proxy_ileiden', False):
1931 print textwrap.dedent("""\
1932 ++ wleiden-gw-%(nodename)s
1933 menu = %(nodename)s.gw
1934 title = Wireless Leiden gateway %(nodename)s.gw.wleiden.net.
1935 host = %(nodename)s.gw.wleiden.net.
1936 """ % datadump)
1937 elif sys.argv[1] == "nagios-export":
1938 try:
1939 heavy_load = (sys.argv[2] == "--heavy-load")
1940 except IndexError:
1941 heavy_load = False
1942
1943 hostgroup_details = {
1944 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1945 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1946 'walphen' : 'Stichting Wireless Alphen',
1947 'westeinder' : 'Westeinder Plassen',
1948 }
1949
1950 # Convert IP to Host
1951 ip2host = {'root' : 'root'}
1952 for host in get_hostlist():
1953 datadump = get_yaml(host)
1954 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1955 for iface in get_interface_keys(datadump):
1956 if datadump[iface].has_key('autogen_gateway'):
1957 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1958
1959 # Find dependency tree based on output of lvrouted.mytree of nearest node
1960 parents = defaultdict(list)
1961 stack = ['root']
1962 prev_depth = 0
1963 if os.path.isfile('lvrouted.mytree'):
1964 for line in open('lvrouted.mytree').readlines():
1965 depth = line.count('\t')
1966 ip = line.strip().split()[0]
1967
1968 if prev_depth < depth:
1969 try:
1970 parents[ip2host[ip]].append(ip2host[stack[-1]])
1971 except KeyError as e:
1972 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1973 stack.append(ip)
1974 elif prev_depth > depth:
1975 stack = stack[:(depth - prev_depth)]
1976 elif prev_depth == depth:
1977 try:
1978 parents[ip2host[ip]].append(ip2host[stack[-1]])
1979 except KeyError as e:
1980 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1981
1982
1983 prev_depth = depth
1984 # Observe that some nodes has themself as parent or multiple parents
1985 # for now take only the first parent, other behaviour is yet to be explained
1986
1987
1988
1989 params = {
1990 'check_interval' : 5 if heavy_load else 120,
1991 'retry_interval' : 1 if heavy_load else 10,
1992 'max_check_attempts' : 10 if heavy_load else 6,
1993 'notification_interval': 120 if heavy_load else 240,
1994 }
1995
1996 print '''\
1997define host {
1998 name wleiden-node ; Default Node Template
1999 use generic-host ; Use the standard template as initial starting point
2000 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2001 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2002 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2003 notification_interval %(notification_interval)s
2004 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2005 check_command check-host-alive ; Default command to check FreeBSD hosts
2006 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2007}
2008
2009define service {
2010 name wleiden-service ; Default Service Template
2011 use generic-service ; Use the standard template as initial starting point
2012 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2013 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2014 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2015 notification_interval %(notification_interval)s
2016 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2017 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2018}
2019
2020# Please make sure to install:
2021# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2022#
2023# Recompile net-mgmt/nagios-plugins to support check_snmp
2024# make -C /usr/ports/net-mgmt/nagios-plugins
2025#
2026# Install net/bind-tools to allow v2/check_dns_wl to work:
2027# pkg install bind-tools
2028#
2029define command{
2030 command_name check_snmp_disk
2031 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2032}
2033
2034define command{
2035 command_name check_netsnmp_load
2036 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2037}
2038
2039define command{
2040 command_name check_netsnmp_proc
2041 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2042}
2043
2044define command{
2045 command_name check_by_ssh
2046 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2047}
2048
2049define command{
2050 command_name check_dns_wl
2051 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2052}
2053
2054define command{
2055 command_name check_snmp_uptime
2056 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2057}
2058
2059
2060# TDB: dhcp leases
2061# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2062
2063# TDB: internet status
2064# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2065
2066# TDB: Advanced local passive checks
2067# /usr/local/libexec/nagios/check_by_ssh
2068''' % params
2069
2070 print '''\
2071# Service Group, not displayed by default
2072define hostgroup {
2073 hostgroup_name srv_hybrid
2074 alias All Hybrid Nodes
2075 register 0
2076}
2077
2078define service {
2079 use wleiden-service
2080 hostgroup_name srv_hybrid
2081 service_description SSH
2082 check_command check_ssh
2083}
2084
2085define service {
2086 use wleiden-service,service-pnp
2087 hostgroup_name srv_hybrid
2088 service_description HTTP
2089 check_command check_http
2090}
2091
2092define service {
2093 use wleiden-service
2094 hostgroup_name srv_hybrid
2095 service_description DNS
2096 check_command check_dns_wl!"www.wirelessleiden.nl"
2097}
2098'''
2099
2100 if heavy_load:
2101 print '''\
2102define service {
2103 use wleiden-service
2104 hostgroup_name srv_hybrid
2105 service_description UPTIME
2106 check_command check_snmp_uptime
2107}
2108
2109#define service {
2110# use wleiden-service
2111# hostgroup_name srv_hybrid
2112# service_description NTP
2113# check_command check_ntp_peer
2114#}
2115
2116define service {
2117 use wleiden-service
2118 hostgroup_name srv_hybrid
2119 service_description LOAD
2120 check_command check_netsnmp_load
2121}
2122
2123define service {
2124 use wleiden-service
2125 hostgroup_name srv_hybrid
2126 service_description PROC
2127 check_command check_netsnmp_proc
2128}
2129
2130define service {
2131 use wleiden-service
2132 hostgroup_name srv_hybrid
2133 service_description DISK
2134 check_command check_snmp_disk
2135}
2136'''
2137 for node in get_hostlist():
2138 datadump = get_yaml(node)
2139 if not datadump['status'] == 'up':
2140 continue
2141 if not hostgroup_details.has_key(datadump['monitoring_group']):
2142 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2143 print '''\
2144define host {
2145 use wleiden-node,host-pnp
2146 contact_groups admins
2147 host_name %(autogen_fqdn)s
2148 address %(masterip)s
2149 hostgroups srv_hybrid,%(monitoring_group)s\
2150''' % datadump
2151 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2152 print '''\
2153 parents %(parents)s\
2154''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2155 print '''\
2156}
2157'''
2158
2159
2160 for name,alias in hostgroup_details.iteritems():
2161 print '''\
2162define hostgroup {
2163 hostgroup_name %s
2164 alias %s
2165} ''' % (name, alias)
2166
2167
2168 elif sys.argv[1] == "full-export":
2169 hosts = {}
2170 for node in get_hostlist():
2171 datadump = get_yaml(node)
2172 hosts[datadump['nodename']] = datadump
2173 print yaml.dump(hosts)
2174
2175 elif sys.argv[1] == "dns":
2176 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2177 elif sys.argv[1] == "cleanup":
2178 # First generate all datadumps
2179 datadumps = dict()
2180 ssid_to_node = dict()
2181 for host in get_hostlist():
2182 logger.info("# Processing: %s", host)
2183 # Set some boring default values
2184 datadump = { 'board' : 'UNKNOWN' }
2185 datadump.update(get_yaml(host))
2186 datadumps[datadump['nodename']] = datadump
2187
2188 (poel, errors) = make_relations()
2189 print "\n".join(["# WARNING: %s" % x for x in errors])
2190
2191 for host,datadump in datadumps.iteritems():
2192 try:
2193 # Convert all yes and no to boolean values
2194 def fix_boolean(dump):
2195 for key in dump.keys():
2196 if type(dump[key]) == dict:
2197 dump[key] = fix_boolean(dump[key])
2198 elif str(dump[key]).lower() in ["yes", "true"]:
2199 dump[key] = True
2200 elif str(dump[key]).lower() in ["no", "false"]:
2201 # Compass richting no (Noord Oost) is valid input
2202 if key != "compass": dump[key] = False
2203 return dump
2204 datadump = fix_boolean(datadump)
2205
2206 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2207 if not 'latitude' in datadump and not 'longitude' in datadump:
2208 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2209 elif 'latitude' in datadump and 'longitude' in datadump:
2210 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2211 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2212 # TODO: Compare outcome of both coordinate systems and validate against each-other
2213
2214 if datadump['nodename'].startswith('Proxy'):
2215 datadump['nodename'] = datadump['nodename'].lower()
2216
2217 for iface_key in get_interface_keys(datadump):
2218 try:
2219 # All our normal wireless cards are normal APs now
2220 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2221 datadump[iface_key]['mode'] = 'ap'
2222 # Wireless Leiden SSID have an consistent lowercase/uppercase
2223 if datadump[iface_key].has_key('ssid'):
2224 ssid = datadump[iface_key]['ssid']
2225 prefix = 'ap-WirelessLeiden-'
2226 if ssid.lower().startswith(prefix.lower()):
2227 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2228 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2229 datadump[iface_key]['mode'] = 'autogen-FIXME'
2230 if not datadump[iface_key].has_key('comment'):
2231 datadump[iface_key]['comment'] = 'autogen-FIXME'
2232
2233 if datadump[iface_key].has_key('ns_mac'):
2234 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2235
2236 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2237 datadump[iface_key] = datadump[iface_key]['desc']
2238
2239 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2240 # due to better collision avoidance.
2241 if datadump[iface_key]['type'] == '11b':
2242 datadump[iface_key]['type'] = '11g'
2243
2244 # Setting 802.11g channels to de-facto standards, to avoid
2245 # un-detected sharing with other overlapping channels
2246 #
2247 # Technically we could also use channel 13 in NL, but this is not
2248 # recommended as foreign devices might not be able to select this
2249 # channel. Secondly using 1,5,9,13 instead is going to clash with
2250 # the de-facto usage of 1,6,11.
2251 #
2252 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2253 channels_at_2400Mhz = (1,6,11)
2254 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2255 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2256 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2257 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2258
2259 # Mandatory interface keys
2260 if not datadump[iface_key].has_key('status'):
2261 datadump[iface_key]['status'] = 'planned'
2262
2263 x = datadump[iface_key]['comment']
2264 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2265
2266 # Fixing bridge_type if none is found
2267 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2268 if not 'bridge_type' in datadump[iface_key]:
2269 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2270
2271 # Making sure description works
2272 if datadump[iface_key].has_key('desc'):
2273 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2274 del datadump[iface_key]['desc']
2275 else:
2276 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2277 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2278 if response:
2279 datadump[iface_key]['comment'] = response
2280 del datadump[iface_key]['desc']
2281
2282 # Check DHCP configuration
2283 dhcp_type(datadump[iface_key])
2284
2285 # Set the compass value based on the angle between the poels
2286 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2287 my_pool = poel[network(datadump[iface_key]['ip'])]
2288 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2289 if remote_hosts:
2290 compass_target = remote_hosts[0]
2291 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2292 # TODO: Compass wanted and actual direction might differ
2293
2294 # Monitoring Group default
2295 if not 'monitoring_group' in datadump:
2296 datadump['monitoring_group'] = 'wleiden'
2297
2298 except Exception as exc:
2299 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
2300 raise
2301 store_yaml(datadump)
2302 except Exception as exc:
2303 exc.args = ("# Error while processing %s" % host,) + exc.args
2304 raise
2305 elif sys.argv[1] == "list":
2306 use_fqdn = False
2307 if len(sys.argv) < 4:
2308 usage()
2309 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2310 usage()
2311 if not sys.argv[3] in ["nodes","proxies","systems"]:
2312 usage()
2313
2314 if len(sys.argv) > 4:
2315 if sys.argv[4] == "fqdn":
2316 use_fqdn = True
2317 else:
2318 usage()
2319
2320 for system in get_hostlist():
2321 datadump = get_yaml(system)
2322 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2323 continue
2324
2325 output = datadump['autogen_fqdn'] if use_fqdn else system
2326 if sys.argv[2] == "all":
2327 print output
2328 elif datadump['status'] == sys.argv[2]:
2329 print output
2330 elif sys.argv[1] == "create":
2331 if sys.argv[2] == "network.kml":
2332 print make_network_kml.make_graph()
2333 elif sys.argv[2] == "host-ips.txt":
2334 for system in get_hostlist():
2335 datadump = get_yaml(system)
2336 ips = [datadump['masterip']]
2337 for ifkey in get_interface_keys(datadump):
2338 ips.append(datadump[ifkey]['ip'].split('/')[0])
2339 print system, ' '.join(ips)
2340 elif sys.argv[2] == "host-pos.txt":
2341 for system in get_hostlist():
2342 datadump = get_yaml(system)
2343 print system, datadump['rdnap_x'], datadump['rdnap_y']
2344 elif sys.argv[2] == 'ssh_config':
2345 print '''
2346Host *.wleiden.net
2347 User root
2348
2349Host 172.16.*.*
2350 User root
2351'''
2352 for system in get_hostlist():
2353 datadump = get_yaml(system)
2354 print '''\
2355Host %s
2356 User root
2357
2358Host %s
2359 User root
2360
2361Host %s
2362 User root
2363
2364Host %s
2365 User root
2366''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2367 else:
2368 usage()
2369 else:
2370 usage()
2371 else:
2372 # Do not enable debugging for config requests as it highly clutters the output
2373 if not is_text_request():
2374 cgitb.enable()
2375
2376 try:
2377 response_headers, output = process_cgi_request()
2378 except:
2379 print ''
2380 print ''
2381 raise
2382
2383 print_cgi_response(response_headers, output)
2384
2385if __name__ == "__main__":
2386 main()
Note: See TracBrowser for help on using the repository browser.