source: genesis/tools/gformat.py@ 14030

Last change on this file since 14030 was 14024, checked in by rick, 7 years ago

Add PoC of using inbound IPv6 at Dwars

New Ziggo connections do not offer Ipv4 by default.
Bonus; no more difficult port forwarding, only one IPv6 port filtering rule in
Ziggo router.
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 79.2 KB
Line 
1#!/usr/local/bin/python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
35SVNVERSION = filter(os.path.isfile, ('/usr/local/bin/svnversion', '/usr/bin/svnversion'))[0]
36
37import argparse
38import cgi
39import cgitb
40import copy
41import glob
42import make_network_kml
43import math
44import pyproj
45import random
46import re
47import socket
48import string
49import subprocess
50import textwrap
51import time
52import traceback
53import urlparse
54
55from pprint import pprint
56from collections import defaultdict, OrderedDict
57from sys import stderr
58try:
59 import yaml
60except ImportError, e:
61 print e
62 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
63 exit(1)
64
65try:
66 from yaml import CLoader as Loader
67 from yaml import CDumper as Dumper
68except ImportError:
69 from yaml import Loader, Dumper
70
71from jinja2 import Environment, Template
72def yesorno(value):
73 return "YES" if bool(value) else "NO"
74env = Environment()
75env.filters['yesorno'] = yesorno
76def render_template(datadump, template):
77 result = env.from_string(template).render(datadump)
78 # Make it look pretty to the naked eye, as jinja templates are not so
79 # friendly when it comes to whitespace formatting
80 ## Remove extra whitespace at end of line lstrip() style.
81 result = re.sub(r'\n[\ ]+','\n', result)
82 ## Include only a single newline between an definition and a comment
83 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
84 ## Remove extra newlines after single comment
85 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
86 return result
87
88import logging
89logging.basicConfig(format='# %(levelname)s: %(message)s' )
90logger = logging.getLogger()
91logger.setLevel(logging.DEBUG)
92
93
94if os.environ.has_key('CONFIGROOT'):
95 NODE_DIR = os.environ['CONFIGROOT']
96else:
97 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
98__version__ = '$Id: gformat.py 14024 2018-01-20 14:10:14Z rick $'
99
100CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
101
102files = [
103 'authorized_keys',
104 'dnsmasq.conf',
105 'dhcpd.conf',
106 'rc.conf.local',
107 'resolv.conf',
108 'motd',
109 'ntp.conf',
110 'pf.hybrid.conf.local',
111 'unbound.wleiden.conf',
112 'wleiden.yaml',
113 ]
114
115# Global variables uses
116OK = 10
117DOWN = 20
118UNKNOWN = 90
119
120
121ileiden_proxies = OrderedDict()
122normal_proxies = []
123datadump_cache = {}
124interface_list_cache = {}
125rc_conf_local_cache = {}
126nameservers_cache = []
127relations_cache = None
128
129NO_DHCP = 0
130DHCP_CLIENT = 10
131DHCP_SERVER = 20
132def dhcp_type(item):
133 if not item.has_key('dhcp'):
134 return NO_DHCP
135 elif not item['dhcp']:
136 return NO_DHCP
137 elif item['dhcp'].lower() == 'client':
138 return DHCP_CLIENT
139 else:
140 # Validation Checks
141 begin,end = map(int,item['dhcp'].split('-'))
142 if begin >= end:
143 raise ValueError("DHCP Start >= DHCP End")
144 return DHCP_SERVER
145
146def etrs2rd(lat, lon):
147 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
148 p2 = pyproj.Proj(init='EPSG:28992')
149 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
150 return (RDx, RDy)
151
152def rd2etrs(RDx, RDy):
153 p1 = pyproj.Proj(init='EPSG:28992')
154 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
155 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
156 return (lat, lon)
157
158def get_yaml(item,add_version_info=True):
159 try:
160 """ Get configuration yaml for 'item'"""
161 if datadump_cache.has_key(item):
162 return datadump_cache[item].copy()
163
164 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
165 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
166 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
167
168 # Default values
169 datadump = {
170 'autogen_revision' : 'NOTFOUND',
171 'autogen_gfile' : gfile,
172 'service_proxy_ileiden' : False,
173 }
174 f = open(gfile, 'r')
175 datadump.update(yaml.load(f,Loader=Loader))
176 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
177 if datadump['nodetype'] == 'Hybrid':
178 # Some values are defined implicitly
179 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
180 datadump['service_incoming_rdr'] = True
181 # Use some boring defaults
182 defaults = {
183 'service_proxy_normal' : False,
184 'service_accesspoint' : True,
185 'service_incoming_rdr' : False,
186 'service_concentrator' : False,
187 'monitoring_group' : 'wleiden',
188 }
189 for (key,value) in defaults.iteritems():
190 if not datadump.has_key(key):
191 datadump[key] = value
192 f.close()
193
194 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
195 if add_version_info:
196 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
197 lines = p.communicate()[0].split('\n')
198 if p.returncode == 0:
199 for line in lines:
200 if line:
201 (key, value) = line.split(': ')
202 datadump["autogen_" + key.lower().replace(' ','_')] = value
203
204 # Preformat certain needed variables for formatting and push those into special object
205 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
206
207 wlan_count=0
208 try:
209 for key in get_interface_keys(datadump, True):
210 datadump[key]['autogen_ifbase'] = key.split('_')[1]
211 datadump[key]['autogen_vlan'] = False
212
213 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
214 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
215 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
216
217 if datadump[key].has_key('parent'):
218 if datadump[key].has_key('ip'):
219 raise ValueError("Interface bridge member cannot have IP assigned")
220 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
221 raise ValueError("Interface bridge member cannot have DHCP set")
222
223 if datadump[key].has_key('ip'):
224 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
225
226 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
227 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
228 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
229 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
230 wlan_count += 1
231 else:
232 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
233 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
234 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
235 datadump[key]['autogen_vlan'] = key.split('_')[2]
236 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
237 else:
238 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
239
240 except Exception as exc:
241 exc.args = ("# Error while processing interface %s" % key,) + exc.args
242 raise
243
244 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
245 if dhcp_type(datadump[key]) == DHCP_SERVER]
246
247 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
248 datadump['autogen_item'] = item
249
250 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
251 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
252 datadump_cache[item] = datadump.copy()
253 except Exception as exc:
254 exc.args = ("# Error while processing %s" % item,) + exc.args
255 raise
256 return datadump
257
258
259def store_yaml(datadump, header=False):
260 """ Store configuration yaml for 'item'"""
261 item = datadump['autogen_item']
262 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
263
264 output = generate_wleiden_yaml(datadump, header)
265
266 f = open(gfile, 'w')
267 f.write(output)
268 f.close()
269
270
271def network(ip):
272 addr, mask = ip.split('/')
273 # Not parsing of these folks please
274 addr = parseaddr(addr)
275 mask = int(mask)
276 network = addr & ~((1 << (32 - mask)) - 1)
277 return network
278
279
280
281def make_relations():
282 """ Process _ALL_ yaml files to get connection relations """
283 global relations_cache
284
285 if relations_cache:
286 return relations_cache
287
288 errors = []
289 poel = defaultdict(list)
290
291 for host in get_hostlist():
292 datadump = get_yaml(host)
293 try:
294 for iface_key in get_interface_keys(datadump):
295 # Bridge members has no IP assigned
296 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
297 continue
298 net_addr = network(datadump[iface_key]['ip'])
299 poel[net_addr] += [(host,datadump[iface_key].copy())]
300 except (KeyError, ValueError) as e:
301 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
302 continue
303
304 relations_cache = (poel, errors)
305 return relations_cache
306
307
308
309def valid_addr(addr):
310 """ Show which address is valid in which are not """
311 return str(addr).startswith('172.')
312
313def get_hostlist():
314 """ Combined hosts and proxy list"""
315 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
316
317def angle_between_points(lat1,lat2,long1,long2):
318 """
319 Return Angle in radians between two GPS coordinates
320 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
321 """
322 dy = lat2 - lat1
323 dx = math.cos(lat1)*(long2 - long1)
324 angle = math.atan2(dy,dx)
325 return angle
326
327
328
329def angle_to_cd(angle):
330 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
331
332 # For easy conversion get positive degree
333 degrees = math.degrees(angle)
334 abs_degrees = 360 + degrees if degrees < 0 else degrees
335
336 # Numbers can be confusing calculate from the 4 main directions
337 p = 22.5
338 if abs_degrees < p:
339 cd = "n"
340 elif abs_degrees < (90 - p):
341 cd = "no"
342 elif abs_degrees < (90 + p):
343 cd = "o"
344 elif abs_degrees < (180 - p):
345 cd = "zo"
346 elif abs_degrees < (180 + p):
347 cd = "z"
348 elif abs_degrees < (270 - p):
349 cd = "zw"
350 elif abs_degrees < (270 + p):
351 cd = "w"
352 elif abs_degrees < (360 - p):
353 cd = "nw"
354 else:
355 cd = "n"
356 return cd
357
358
359
360def cd_between_hosts(hostA, hostB, datadumps):
361 # Using RDNAP coordinates
362 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
363 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
364 return angle_to_cd(math.atan2(dx,dy))
365
366 # GPS coordinates seems to fail somehow
367 #latA = float(datadumps[hostA]['latitude'])
368 #latB = float(datadumps[hostB]['latitude'])
369 #lonA = float(datadumps[hostA]['longitude'])
370 #lonB = float(datadumps[hostB]['longitude'])
371 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
372
373
374def generate_title(nodelist):
375 """ Main overview page """
376 items = { \
377 'root' : ".",
378 'version' : subprocess.Popen([SVNVERSION, "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
379 }
380 def fl(spaces, line):
381 return (' ' * spaces) + line + '\n'
382
383 output = """
384<html>
385 <head>
386 <title>Wireless leiden Configurator - GFormat</title>
387 <style type="text/css">
388 th {background-color: #999999}
389 tr:nth-child(odd) {background-color: #cccccc}
390 tr:nth-child(even) {background-color: #ffffff}
391 th, td {padding: 0.1em 1em}
392 </style>
393 </head>
394 <body>
395 <center>
396 <form type="GET" action="%(root)s">
397 <input type="hidden" name="action" value="update">
398 <input type="submit" value="Update Configuration Database (SVN)">
399 </form>
400 <table>
401 <caption><h3>Wireless Leiden Configurator - Revision %(version)s</h3></caption>
402 """ % items
403
404 for node in nodelist:
405 items['node'] = node
406 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
407 for config in files:
408 items['config'] = config
409 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
410 output += fl(5, "</tr>")
411 output += """
412 </table>
413 <hr />
414 <em>%s</em>
415 </center>
416 </body>
417</html>
418 """ % __version__
419
420 return output
421
422
423
424def generate_node(node):
425 """ Print overview of all files available for node """
426 return "\n".join(files)
427
428def generate_node_overview(host, datadump=False):
429 """ Print overview of all files available for node """
430 if not datadump:
431 datadump = get_yaml(host)
432 params = { 'host' : host }
433 output = "<em><a href='..'>Back to overview</a></em><hr />"
434 output += "<h2>Available files:</h2><ul>"
435 for cf in files:
436 params['cf'] = cf
437 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
438 output += "</ul>"
439
440 # Generate and connection listing
441 output += "<h2>Connected To:</h2><ul>"
442 (poel, errors) = make_relations()
443 for network, hosts in poel.iteritems():
444 if host in [x[0] for x in hosts]:
445 if len(hosts) == 1:
446 # Single not connected interface
447 continue
448 for remote,ifacedump in hosts:
449 if remote == host:
450 # This side of the interface
451 continue
452 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
453 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
454 output += "</ul>"
455 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
456
457 output += "<hr /><em><a href='..'>Back to overview</a></em>"
458 return output
459
460
461def generate_header(datadump, ctag="#"):
462 return """\
463%(ctag)s
464%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
465%(ctag)s
466""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
467
468
469
470def parseaddr(s):
471 """ Process IPv4 CIDR notation addr to a (binary) number """
472 f = s.split('.')
473 return (long(f[0]) << 24L) + \
474 (long(f[1]) << 16L) + \
475 (long(f[2]) << 8L) + \
476 long(f[3])
477
478
479
480def showaddr(a):
481 """ Display IPv4 addr in (dotted) CIDR notation """
482 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
483
484
485def is_member(ip, mask, canidate):
486 """ Return True if canidate is part of ip/mask block"""
487 ip_addr = parseaddr(ip)
488 ip_canidate = parseaddr(canidate)
489 mask = int(mask)
490 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
491 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
492 return ip_addr == ip_canidate
493
494
495
496def cidr2netmask(netmask):
497 """ Given a 'netmask' return corresponding CIDR """
498 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
499
500def get_network(addr, mask):
501 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
502
503
504def generate_dhcpd_conf(datadump):
505 """ Generate config file '/usr/local/etc/dhcpd.conf """
506 # Redundency support, in cause local DNS server is not running/responding.
507 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
508 output = generate_header(datadump)
509 output += Template("""\
510# option definitions common to all supported networks...
511option domain-name "dhcp.{{ autogen_fqdn }}";
512
513default-lease-time 600;
514max-lease-time 7200;
515
516# Use this to enble / disable dynamic dns updates globally.
517#ddns-update-style none;
518
519# If this DHCP server is the official DHCP server for the local
520# network, the authoritative directive should be uncommented.
521authoritative;
522
523# Use this to send dhcp log messages to a different log file (you also
524# have to hack syslog.conf to complete the redirection).
525log-facility local7;
526
527# UniFi Discovery Support
528option space ubnt;
529option ubnt.unifi-address code 1 = ip-address;
530#
531class "ubnt" {
532 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
533 option vendor-class-identifier "ubnt";
534 vendor-option-space ubnt;
535}
536
537#
538# Interface definitions
539#
540\n\n""").render(datadump)
541
542
543 # TODO: Use textwrap.fill instead
544 def indent(text, count):
545 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
546
547 # Process DHCP blocks
548 dhcp_out = defaultdict(list)
549 for iface_key in get_interface_keys(datadump):
550 ifname = datadump[iface_key]['autogen_ifbase']
551 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
552 if not datadump[iface_key].has_key('comment'):
553 datadump[iface_key]['comment'] = None
554
555 if not datadump[iface_key].has_key('ip'):
556 continue
557
558 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
559
560 (addr, mask) = datadump[iface_key]['ip'].split('/')
561 datadump[iface_key]['autogen_addr'] = addr
562 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
563 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
564
565 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
566 dhcp_out[groupif].append(textwrap.dedent("""\
567 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
568 ### not autoritive
569 }
570 """ % datadump[iface_key]))
571 continue
572
573 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
574 dhcp_part = ".".join(addr.split('.')[0:3])
575 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
576 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
577 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
578
579 # Assume the first 10 IPs could be used for static entries
580 if 'no_portal' in datadump:
581 fixed = 5
582 for mac in datadump['no_portal']:
583 dhcp_out[groupif].append(textwrap.dedent("""\
584 host fixed-%(ifname)s-%(fixed)s {
585 hardware ethernet %(mac)s;
586 fixed-address %(prefix)s.%(fixed)s;
587 }
588 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
589 fixed += 1
590
591 if 'dhcp_fixed' in datadump[iface_key]:
592 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
593 dhcp_out[groupif].append(textwrap.dedent("""\
594 host fixed-%(host)s {
595 hardware ethernet %(mac)s;
596 fixed-address %(addr)s;
597 }
598 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
599
600
601 dhcp_out[groupif].append(textwrap.dedent("""\
602 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
603 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
604 option routers %(autogen_addr)s;
605 option domain-name-servers %(autogen_dns_servers)s;
606 option ubnt.unifi-address 172.17.107.10;
607 }
608 """ % datadump[iface_key]))
609
610 # Output the blocks in groups
611 for ifname,value in sorted(dhcp_out.iteritems()):
612 if len(value) > 2:
613 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
614 else:
615 output += ''.join(value) + "\n\n"
616 return output
617
618
619
620def generate_dnsmasq_conf(datadump):
621 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
622 output = generate_header(datadump)
623 output += Template("""\
624# DHCP server options
625dhcp-authoritative
626dhcp-fqdn
627domain=dhcp.{{ autogen_fqdn }}
628domain-needed
629expand-hosts
630log-async=100
631
632# Low memory footprint
633cache-size=10000
634
635\n""").render(datadump)
636
637 for iface_key in get_interface_keys(datadump):
638 if not datadump[iface_key].has_key('comment'):
639 datadump[iface_key]['comment'] = None
640 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
641
642 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
643 output += "# not autoritive\n\n"
644 continue
645
646 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
647 (ip, cidr) = datadump[iface_key]['ip'].split('/')
648 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
649
650 dhcp_part = ".".join(ip.split('.')[0:3])
651 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
652 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
653 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
654
655 return output
656
657
658class AutoVivification(dict):
659 """Implementation of perl's autovivification feature."""
660 def __getitem__(self, item):
661 try:
662 return dict.__getitem__(self, item)
663 except KeyError:
664 value = self[item] = type(self)()
665 return value
666
667def make_interface_list(datadump):
668 if interface_list_cache.has_key(datadump['autogen_item']):
669 return (interface_list_cache[datadump['autogen_item']])
670 # lo0 configuration:
671 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
672 # - masterip is special as it needs to be assigned to at
673 # least one interface, so if not used assign to lo0
674 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
675 vlan_list = defaultdict(list)
676 bridge_list = defaultdict(list)
677 flags_if = AutoVivification()
678 dhclient_if = {'lo0' : False}
679
680 # XXX: Find some way of send this output nicely
681 output = ''
682
683 masterip_used = False
684 for iface_key in get_interface_keys(datadump):
685 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
686 masterip_used = True
687 break
688 if not masterip_used:
689 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
690
691 if 'serviceid' in datadump:
692 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
693
694 for iface_key in get_interface_keys(datadump):
695 ifacedump = datadump[iface_key]
696 if ifacedump['autogen_bridge_alias']:
697 ifname = ifacedump['autogen_ifbase']
698 else:
699 ifname = ifacedump['autogen_ifname']
700
701 # If defined as vlan interface
702 if ifacedump['autogen_vlan']:
703 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
704
705 # If defined as bridge interface
706 if ifacedump['autogen_bridge_member']:
707 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
708
709
710 # Flag dhclient is possible
711 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
712 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
713
714 # Ethernet address
715 if ifacedump.has_key('ether'):
716 flags_if[ifname]['ether'] = ifacedump['ether']
717
718 # Add interface IP to list
719 if ifacedump.has_key('ip'):
720 item = (ifacedump['ip'], ifacedump['comment'])
721 if addrs_list.has_key(ifname):
722 addrs_list[ifname].append(item)
723 else:
724 addrs_list[ifname] = [item]
725
726 # Alias only needs IP assignment for now, this might change if we
727 # are going to use virtual accesspoints
728 if "alias" in iface_key:
729 continue
730
731 # XXX: Might want to deduct type directly from interface name
732 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
733 # Default to station (client) mode
734 ifacedump['autogen_wlanmode'] = "sta"
735 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
736 ifacedump['autogen_wlanmode'] = "ap"
737
738 if not ifacedump.has_key('channel'):
739 if ifacedump['type'] == '11a':
740 ifacedump['channel'] = 36
741 else:
742 ifacedump['channel'] = 1
743
744 # Allow special hacks at the back like wds and stuff
745 if not ifacedump.has_key('extra'):
746 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
747 else:
748 ifacedump['autogen_extra'] = ifacedump['extra']
749
750 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
751
752 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
753 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
754 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
755 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
756 output += "\n"
757
758 elif ifacedump['type'] in ['ethernet', 'eth']:
759 # No special config needed besides IP
760 pass
761 elif ifacedump['type'] in ['vlan']:
762 # VLAN member has no special configuration
763 pass
764 else:
765 assert False, "Unknown type " + ifacedump['type']
766
767 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
768 interface_list_cache[datadump['autogen_item']] = store
769 return(store)
770
771
772
773def create_proxies_list():
774 if not ileiden_proxies or not normal_proxies:
775 # Placeholder for to-be-installed proxies, this will avoid updating the all
776 # nodes to include this new machine, yet due to an unbound issue, this list
777 # has to be kept small.
778
779 for i in range(1,20):
780 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
781
782 for host in get_hostlist():
783 hostdump = get_yaml(host)
784 if hostdump['status'] == 'up':
785 if hostdump['service_proxy_ileiden']:
786 ileiden_proxies[hostdump['serviceid']] = hostdump
787 if hostdump['service_proxy_normal']:
788 normal_proxies.append(hostdump)
789
790
791
792def generate_rc_conf_local(datadump):
793 """ Generate configuration file '/etc/rc.conf.local' """
794 item = datadump['autogen_item']
795 if rc_conf_local_cache.has_key(item):
796 return rc_conf_local_cache[item]
797
798 if not datadump.has_key('ileiden'):
799 datadump['autogen_ileiden_enable'] = False
800 else:
801 datadump['autogen_ileiden_enable'] = datadump['ileiden']
802
803 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
804
805 create_proxies_list()
806 datadump['autogen_ileiden_proxies'] = ileiden_proxies
807 datadump['autogen_normal_proxies'] = normal_proxies
808 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
809 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
810 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
811 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
812
813 output = generate_header(datadump, "#");
814 output += render_template(datadump, """\
815hostname='{{ autogen_fqdn }}'
816location='{{ location }}'
817nodetype="{{ nodetype }}"
818
819#
820# Configured listings
821#
822captive_portal_whitelist=""
823{% if nodetype == "Proxy" %}
824#
825# Proxy Configuration
826#
827{% if gateway and service_proxy_ileiden -%}
828defaultrouter="{{ gateway }}"
829{% else -%}
830#defaultrouter="NOTSET"
831{% endif -%}
832internalif="{{ internalif }}"
833ileiden_enable="{{ autogen_ileiden_enable }}"
834gateway_enable="{{ autogen_ileiden_enable }}"
835pf_enable="yes"
836pf_rules="/etc/pf.conf"
837{% if autogen_ileiden_enable -%}
838pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
839lvrouted_enable="{{ autogen_ileiden_enable }}"
840lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
841{% else -%}
842pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
843{% endif -%}
844{% if internalroute -%}
845static_routes="wleiden"
846route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
847{% endif -%}
848
849{% elif nodetype == "Hybrid" %}
850 #
851 # Hybrid Configuration
852 #
853 list_ileiden_proxies="
854 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
855 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
856 {% endfor -%}
857 "
858 list_normal_proxies="
859 {% for item in autogen_normal_proxies -%}
860 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
861 {% endfor -%}
862 "
863
864 {% if autogen_dhcp_interfaces -%}
865 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
866 {% else %}
867 captive_portal_interfaces="dummy"
868 {% endif %}
869 externalif="{{ externalif|default('vr0', true) }}"
870 masterip="{{ masterip }}"
871
872 {% if gateway and service_proxy_ileiden %}
873 defaultrouter="{{ gateway }}"
874 {% else %}
875 #defaultrouter="NOTSET"
876 {% endif %}
877
878 #
879 # Defined services
880 #
881 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
882 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
883 service_accesspoint="{{ service_accesspoint|yesorno }}"
884 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
885 service_concentrator="{{ service_concentrator|yesorno }}"
886
887 {% if service_proxy_ileiden %}
888 pf_rules="/etc/pf.hybrid.conf"
889 {% if service_concentrator %}
890 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
891 {% else %}
892 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
893 {% endif %}
894 pf_flags="$pf_flags -D publicnat=80,443"
895 lvrouted_flags="$lvrouted_flags -g"
896 {% elif service_proxy_normal or service_incoming_rdr %}
897 pf_rules="/etc/pf.hybrid.conf"
898 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
899 pf_flags="$pf_flags -D publicnat=0"
900 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
901 named_setfib="1"
902 tinyproxy_setfib="1"
903 dnsmasq_setfib="1"
904 sshd_setfib="1"
905 {% else %}
906 named_auto_forward_only="YES"
907 pf_rules="/etc/pf.node.conf"
908 pf_flags=""
909 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
910 {% endif %}
911 {% if service_concentrator %}
912 # Do mind installing certificates is NOT done automatically for security reasons
913 openvpn_enable="YES"
914 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
915 {% endif %}
916
917 {% if service_proxy_normal %}
918 tinyproxy_enable="yes"
919 {% else %}
920 pen_wrapper_enable="yes"
921 {% endif %}
922
923 {% if service_accesspoint %}
924 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
925 {% endif %}
926
927 {% if board == "ALIX2" or board == "net4801" %}
928 #
929 # ''Fat'' configuration, board has 256MB RAM
930 #
931 dnsmasq_enable="NO"
932 named_enable="YES"
933 unbound_enable="YES"
934 {% if autogen_dhcp_interfaces -%}
935 dhcpd_enable="YES"
936 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
937 {% endif -%}
938 {% elif board == "apu1d" %}
939 #
940 # ''Fat'' configuration, board has 1024MB RAM
941 #
942 dnsmasq_enable="NO"
943 unbound_enable="YES"
944 {% if autogen_dhcp_interfaces -%}
945 dhcpd_enable="YES"
946 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
947 {% endif -%}
948 {% endif -%}
949{% endif %}
950
951#
952# Script variables
953#
954attached_devices="{{ autogen_attached_devices|join(' ') }}"
955neighbours="{{ autogen_neighbours|join(' ') }}"
956
957
958#
959# Interface definitions
960#\n
961""")
962
963 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
964 for iface, vlans in sorted(vlan_list.items()):
965 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
966
967 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
968 for iface in sorted(vlan_list.keys()):
969 if not iface in addrs_list.keys():
970 output += "ifconfig_%s='up'\n" % iface
971
972 output += "\n"
973
974 # Bridge configuration:
975 if bridge_list.keys():
976 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
977
978 for iface in bridge_list.keys():
979 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
980
981 # Bridge member interfaces not containing a configuration should be marked active explcitly.
982 for _,members in bridge_list.items():
983 for iface in members:
984 if not iface in addrs_list.keys():
985 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
986
987 output += "\n"
988
989 # Details like SSID
990 if extra_ouput:
991 output += extra_ouput.strip() + "\n"
992
993 # Print IP address which needs to be assigned over here
994 output += "\n"
995 for iface,addrs in sorted(addrs_list.iteritems()):
996 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
997 output += "# %s || %s || %s\n" % (iface, addr, comment)
998
999 # Write DHCLIENT entry
1000 if iface in dhclient_if and dhclient_if[iface]:
1001 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1002 continue
1003
1004 # Make sure the external address is always first as this is needed in the
1005 # firewall setup
1006 addrs = sorted(
1007 [x for x in addrs if not '0.0.0.0' in x[0]],
1008 key=lambda x: x[0].split('.')[0],
1009 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1010 )
1011
1012 idx_offset = 0
1013 # Set MAC is required
1014 if flags_if[iface].has_key('ether'):
1015 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1016 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1017 idx_offset += 1
1018 elif iface in bridge_list:
1019 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1020 idx_offset += 1
1021 else:
1022 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1023
1024 for idx, addr in enumerate(addrs[1:]):
1025 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1026
1027 output += "\n"
1028
1029 rc_conf_local_cache[datadump['autogen_item']] = output
1030 return output
1031
1032
1033
1034def get_all_configs():
1035 """ Get dict with key 'host' with all configs present """
1036 configs = dict()
1037 for host in get_hostlist():
1038 datadump = get_yaml(host)
1039 configs[host] = datadump
1040 return configs
1041
1042
1043def get_interface_keys(config, extra=False):
1044 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1045 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1046 if extra == False:
1047 return filter(lambda x: not "extra" in x, elems)
1048 else:
1049 return elems
1050
1051
1052def get_used_ips(configs):
1053 """ Return array of all IPs used in config files"""
1054 ip_list = []
1055 for config in configs:
1056 ip_list.append(config['masterip'])
1057 if 'serviceid' in config:
1058 ip_list.append(config['serviceid'])
1059 for iface_key in get_interface_keys(config, True):
1060 l = config[iface_key]['ip']
1061 addr, mask = l.split('/')
1062 # Special case do not process
1063 if valid_addr(addr):
1064 ip_list.append(addr)
1065 else:
1066 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1067 return sorted(ip_list)
1068
1069
1070
1071def get_nameservers(max_servers=None):
1072 if nameservers_cache:
1073 return nameservers_cache[0:max_servers]
1074
1075 for host in get_hostlist():
1076 hostdump = get_yaml(host)
1077 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1078 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1079
1080 return nameservers_cache[0:max_servers]
1081
1082
1083def get_neighbours(datadump):
1084 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1085
1086 (poel, errors) = make_relations()
1087 table = []
1088 for iface,addrs in sorted(addrs_list.iteritems()):
1089 if iface in ['lo0']:
1090 continue
1091
1092 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1093 if not addr.startswith('172.'):
1094 # Avoid listing internet connections as pool
1095 continue
1096 for neighbour in poel[network(addr)]:
1097 if neighbour[0] != datadump['autogen_item']:
1098 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1099 return table
1100
1101
1102def get_attached_devices(datadump, url=False):
1103 table = []
1104 for iface_key in get_interface_keys(datadump, True):
1105 ifacedump = datadump[iface_key]
1106
1107 if not ifacedump.has_key('ns_ip'):
1108 continue
1109
1110 x_ip = ifacedump['ns_ip'].split('/')[0]
1111
1112 if 'mode' in ifacedump:
1113 x_mode = ifacedump['mode']
1114 else:
1115 x_mode = 'unknown'
1116
1117 if 'bridge_type' in ifacedump:
1118 device_type = ifacedump['bridge_type']
1119 else:
1120 device_type = 'Unknown'
1121
1122 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1123 return table
1124
1125
1126def generate_resolv_conf(datadump):
1127 """ Generate configuration file '/etc/resolv.conf' """
1128 # XXX: This should properly going to be an datastructure soon
1129 datadump['autogen_header'] = generate_header(datadump, "#")
1130 datadump['autogen_edge_nameservers'] = ''
1131
1132
1133 for masterip,realname in get_nameservers():
1134 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1135
1136 return Template("""\
1137{{ autogen_header }}
1138search wleiden.net
1139
1140# Try local (cache) first
1141nameserver 127.0.0.1
1142
1143{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1144nameserver 8.8.8.8 # Google Public NameServer
1145nameserver 64.6.64.6 # Verisign Public NameServer
1146{% else -%}
1147# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1148{{ autogen_edge_nameservers }}
1149{% endif -%}
1150""").render(datadump)
1151
1152
1153
1154def generate_ntp_conf(datadump):
1155 """ Generate configuration file '/etc/ntp.conf' """
1156 # XXX: This should properly going to be an datastructure soon
1157
1158 datadump['autogen_header'] = generate_header(datadump, "#")
1159 datadump['autogen_ntp_servers'] = ''
1160 for host in get_hostlist():
1161 hostdump = get_yaml(host)
1162 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1163 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1164
1165 return Template("""\
1166{{ autogen_header }}
1167
1168{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1169# Machine hooked to internet.
1170server 0.nl.pool.ntp.org iburst maxpoll 9
1171server 1.nl.pool.ntp.org iburst maxpoll 9
1172server 2.nl.pool.ntp.org iburst maxpoll 9
1173server 3.nl.pool.ntp.org iburst maxpoll 9
1174{% else -%}
1175# Local Wireless Leiden NTP Servers.
1176server 0.pool.ntp.wleiden.net iburst maxpoll 9
1177server 1.pool.ntp.wleiden.net iburst maxpoll 9
1178server 2.pool.ntp.wleiden.net iburst maxpoll 9
1179server 3.pool.ntp.wleiden.net iburst maxpoll 9
1180
1181# All the configured NTP servers
1182{{ autogen_ntp_servers }}
1183{% endif %}
1184
1185# If a server loses sync with all upstream servers, NTP clients
1186# no longer follow that server. The local clock can be configured
1187# to provide a time source when this happens, but it should usually
1188# be configured on just one server on a network. For more details see
1189# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1190# The use of Orphan Mode may be preferable.
1191#
1192server 127.127.1.0
1193fudge 127.127.1.0 stratum 10
1194""").render(datadump)
1195
1196
1197def generate_pf_hybrid_conf_local(datadump):
1198 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1199 datadump['autogen_header'] = generate_header(datadump, "#")
1200 if datadump['service_incoming_rdr']:
1201 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1202 return Template("""\
1203{{ autogen_header }}
1204
1205# Redirect some internal facing services outside (7)
1206# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1207{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1208rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1209{% endfor -%}
1210# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1211{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1212rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1213{% endfor -%}
1214""").render(datadump)
1215
1216def generate_unbound_wleiden_conf(datadump):
1217 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1218 datadump['autogen_header'] = generate_header(datadump, "#")
1219
1220 autogen_ips = []
1221 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1222 for iface,addrs in sorted(addrs_list.iteritems()):
1223 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1224 if addr.startswith('172'):
1225 autogen_ips.append((addr.split('/')[0], comment))
1226 datadump['autogen_ips'] = autogen_ips
1227
1228 create_proxies_list()
1229 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1230 return Template("""\
1231{{ autogen_header }}
1232
1233server:
1234 ## Static definitions fail to start on systems with broken ue(4) interfaces
1235{%- for ip,comment in autogen_ips %}
1236 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1237{%- endfor %}
1238 ## Enabling wildcard matching as work-around
1239 interface: 0.0.0.0
1240 interface: ::0
1241
1242forward-zone:
1243 name: '.'
1244{%- if service_proxy_ileiden %}
1245 forward-addr: 8.8.8.8 # Google DNS A
1246 forward-addr: 8.8.4.4 # Google DNS B
1247 forward-addr: 208.67.222.222 # OpenDNS DNS A
1248 forward-addr: 208.67.220.220 # OpenDNS DNS B
1249{% else -%}
1250{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1251 {%- if loop.index <= 5 %}
1252 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1253 {%- endif %}
1254{%- endfor %}
1255{%- endif %}
1256""").render(datadump)
1257
1258def generate_motd(datadump):
1259 """ Generate configuration file '/etc/motd' """
1260 output = Template("""\
1261FreeBSD run ``service motd onestart'' to make me look normal
1262
1263 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1264 Loc: {{ location }}
1265
1266Services:
1267{% if board == "ALIX2" or board == "net4801" -%}
1268{{" -"}} Core Node ({{ board }})
1269{% else -%}
1270{{" -"}} Hulp Node ({{ board }})
1271{% endif -%}
1272{% if service_proxy_normal -%}
1273{{" -"}} Normal Proxy
1274{% endif -%}
1275{% if service_proxy_ileiden -%}
1276{{" -"}} iLeiden Proxy
1277{% endif -%}
1278{% if service_incoming_rdr -%}
1279{{" -"}} Incoming port redirects
1280{% endif %}
1281Interlinks:\n
1282""").render(datadump)
1283
1284
1285 def make_table(table):
1286 if not table:
1287 return " - none\n"
1288 else:
1289 lines = ""
1290 col_width = [max(len(x) for x in col) for col in zip(*table)]
1291 for row in table:
1292 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1293 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1294 return lines
1295
1296 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1297 table = []
1298 for iface,addrs in sorted(addrs_list.iteritems()):
1299 if iface in ['lo0']:
1300 continue
1301 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1302 table.append((iface, addr, comment))
1303
1304 output += make_table(table)
1305 output += '\n'
1306 output += """\
1307Attached devices:
1308"""
1309 output += make_table(get_attached_devices(datadump, url=True))
1310 output += '\n'
1311 output += """\
1312Available neighbours:
1313"""
1314 output += make_table(get_neighbours(datadump))
1315
1316 return output
1317
1318
1319def format_yaml_value(value):
1320 """ Get yaml value in right syntax for outputting """
1321 if isinstance(value,str):
1322 output = '"%s"' % value
1323 else:
1324 output = value
1325 return output
1326
1327
1328
1329def format_wleiden_yaml(datadump):
1330 """ Special formatting to ensure it is editable"""
1331 output = "# Genesis config yaml style\n"
1332 output += "# vim:ts=2:et:sw=2:ai\n"
1333 output += "#\n"
1334 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1335 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1336 if key == 'rdr_rules':
1337 output += '%-10s:\n' % 'rdr_rules'
1338 for rdr_rule in datadump[key]:
1339 output += '- %s\n' % rdr_rule
1340 else:
1341 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1342
1343 output += "\n\n"
1344
1345 # Format (key, required)
1346 key_order = (
1347 ('comment', True),
1348 ('parent', False),
1349 ('ip', False),
1350 ('ipv6', False),
1351 ('ether', False),
1352 ('desc', True),
1353 ('sdesc', True),
1354 ('mode', True),
1355 ('type', True),
1356 ('extra_type', False),
1357 ('channel', False),
1358 ('ssid', False),
1359 ('wlan_mac', False),
1360 ('dhcp', True),
1361 ('dhcp_fixed', False),
1362 ('compass', False),
1363 ('distance', False),
1364 ('ns_ip', False),
1365 ('repeater_ip', False),
1366 ('bullet2_ip', False),
1367 ('ns_mac', False),
1368 ('bullet2_mac', False),
1369 ('ns_type', False),
1370 ('bridge_type', False),
1371 ('status', True),
1372 )
1373
1374 for iface_key in sorted(iface_keys):
1375 try:
1376 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1377 if remainder:
1378 raise KeyError("invalid keys: %s" % remainder)
1379
1380 output += "%s:\n" % iface_key
1381 for key,required in key_order:
1382 if datadump[iface_key].has_key(key):
1383 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1384 output += "\n\n"
1385 except Exception as exc:
1386 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
1387 raise
1388
1389 return output
1390
1391
1392
1393def generate_wleiden_yaml(datadump, header=True):
1394 """ Generate (petty) version of wleiden.yaml"""
1395 output = generate_header(datadump, "#") if header else ''
1396
1397 for key in datadump.keys():
1398 if key.startswith('autogen_'):
1399 del datadump[key]
1400 # Interface autogen cleanups
1401 elif type(datadump[key]) == dict:
1402 for key2 in datadump[key].keys():
1403 if key2.startswith('autogen_'):
1404 del datadump[key][key2]
1405
1406 output += format_wleiden_yaml(datadump)
1407 return output
1408
1409def generate_nanostation_config(datadump, iface, ns_type):
1410 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1411 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1412
1413 datadump.update(datadump['iface_%s' % iface])
1414
1415 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1416
1417def generate_yaml(datadump):
1418 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1419
1420
1421
1422def generate_config(node, config, datadump=None):
1423 """ Print configuration file 'config' of 'node' """
1424 output = ""
1425 try:
1426 # Load config file
1427 if datadump == None:
1428 datadump = get_yaml(node)
1429
1430 if config == 'wleiden.yaml':
1431 output += generate_wleiden_yaml(datadump)
1432 elif config == 'authorized_keys':
1433 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1434 output += f.read()
1435 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1436 # Fetch local keys if existing
1437 if os.path.exists(node_keys):
1438 output += open(node_keys, 'r').read()
1439 f.close()
1440 elif config == 'dnsmasq.conf':
1441 output += generate_dnsmasq_conf(datadump)
1442 elif config == 'dhcpd.conf':
1443 output += generate_dhcpd_conf(datadump)
1444 elif config == 'rc.conf.local':
1445 output += generate_rc_conf_local(datadump)
1446 elif config == 'resolv.conf':
1447 output += generate_resolv_conf(datadump)
1448 elif config == 'ntp.conf':
1449 output += generate_ntp_conf(datadump)
1450 elif config == 'motd':
1451 output += generate_motd(datadump)
1452 elif config == 'pf.hybrid.conf.local':
1453 output += generate_pf_hybrid_conf_local(datadump)
1454 elif config == 'unbound.wleiden.conf':
1455 output += generate_unbound_wleiden_conf(datadump)
1456 elif config.startswith('vr'):
1457 interface, ns_type = config.strip('.yaml').split('-')
1458 output += generate_nanostation_config(datadump, interface, ns_type)
1459 else:
1460 assert False, "Config not found!"
1461 except IOError, e:
1462 output += "[ERROR] Config file not found"
1463 return output
1464
1465
1466
1467def generate_static(output_dir, logging=True):
1468 items = {'output_dir' : output_dir}
1469 for node in get_hostlist():
1470 items['node'] = node
1471 items['wdir'] = "%(output_dir)s/%(node)s" % items
1472 if not os.path.isdir(items['wdir']):
1473 os.makedirs(items['wdir'])
1474 datadump = get_yaml(node)
1475 f = open("%(wdir)s/index.html" % items, "w")
1476 f.write(generate_node_overview(items['node'], datadump))
1477 f.close()
1478 for config in files:
1479 items['config'] = config
1480 if logging: logger.info("## Generating %(node)s %(config)s" % items)
1481 f = open("%(wdir)s/%(config)s" % items, "w")
1482 f.write(generate_config(node, config, datadump))
1483 f.close()
1484
1485
1486
1487def process_cgi_request(environ=os.environ):
1488 """ When calling from CGI """
1489 response_headers = []
1490 content_type = 'text/plain'
1491
1492 # Update repository if requested
1493 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1494 if form and form.has_key("action") and "update" in form["action"]:
1495 refresh_rate = 5
1496 output = "[INFO] Updating subverion, please wait...\n"
1497 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1498 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1499 try:
1500 generate_static(CACHE_DIR, False)
1501 except:
1502 output += traceback.format_exc()
1503 refresh_rate = 120
1504 pass
1505 output += "[INFO] All done, redirecting in %s seconds" % refresh_rate
1506 response_headers += [
1507 ('Refresh', '%s; url=.' % refresh_rate),
1508 ]
1509 else:
1510 base_uri = environ['REQUEST_URI']
1511 uri = base_uri.strip('/').split('/')[1:]
1512
1513 output = "Template Holder"
1514 if base_uri.endswith('/create/network.kml'):
1515 content_type='application/vnd.google-earth.kml+xml'
1516 output = make_network_kml.make_graph()
1517 elif base_uri.endswith('/api/get/nodeplanner.json'):
1518 content_type='application/json'
1519 output = make_network_kml.make_nodeplanner_json()
1520 elif not uri:
1521 if is_text_request(environ):
1522 output = '\n'.join(get_hostlist())
1523 else:
1524 content_type = 'text/html'
1525 output = generate_title(get_hostlist())
1526 elif len(uri) == 1:
1527 if is_text_request(environ):
1528 output = generate_node(uri[0])
1529 else:
1530 content_type = 'text/html'
1531 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1532 elif len(uri) == 2:
1533 output = generate_config(uri[0], uri[1])
1534 else:
1535 assert False, "Invalid option"
1536
1537 # Return response
1538 response_headers += [
1539 ('Content-type', content_type),
1540 ('Content-Length', str(len(output))),
1541 ]
1542 return(response_headers, str(output))
1543
1544
1545def make_dns(output_dir = 'dns', external = False):
1546 items = dict()
1547
1548 # hostname is key, IP is value
1549 wleiden_zone = defaultdict(list)
1550 wleiden_cname = dict()
1551
1552 pool = dict()
1553 for node in get_hostlist():
1554 datadump = get_yaml(node)
1555
1556 fqdn = datadump['nodename']
1557
1558 if datadump.has_key('rdr_host'):
1559 remote_target = datadump['rdr_host']
1560 elif datadump.has_key('remote_access') and datadump['remote_access']:
1561 remote_target = datadump['remote_access'].split(':')[0]
1562 else:
1563 remote_target = None
1564
1565 if remote_target:
1566 try:
1567 parseaddr(remote_target)
1568 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1569 except (IndexError, ValueError):
1570 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1571
1572
1573 wleiden_zone[fqdn].append((datadump['masterip'], True))
1574
1575 # Hacking to get proper DHCP IPs and hostnames
1576 for iface_key in get_interface_keys(datadump):
1577 iface_name = iface_key.replace('_','-')
1578 if 'ip' in datadump[iface_key]:
1579 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1580 try:
1581 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1582 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1583 dhcp_part = ".".join(ip.split('.')[0:3])
1584 if ip != datadump['masterip']:
1585 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1586 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1587 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1588 except (AttributeError, ValueError, KeyError):
1589 # First push it into a pool, to indentify the counter-part later on
1590 addr = parseaddr(ip)
1591 cidr = int(cidr)
1592 addr = addr & ~((1 << (32 - cidr)) - 1)
1593 if pool.has_key(addr):
1594 pool[addr] += [(iface_name, fqdn, ip)]
1595 else:
1596 pool[addr] = [(iface_name, fqdn, ip)]
1597 continue
1598
1599
1600
1601 # WL uses an /29 to configure an interface. IP's are ordered like this:
1602 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1603
1604 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1605
1606 # Automatic naming convention of interlinks namely 2 + remote.lower()
1607 for (key,value) in pool.iteritems():
1608 # Make sure they are sorted from low-ip to high-ip
1609 value = sorted(value, key=lambda x: parseaddr(x[2]))
1610
1611 if len(value) == 1:
1612 (iface_name, fqdn, ip) = value[0]
1613 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1614
1615 # Device DNS names
1616 if 'cnode' in fqdn.lower():
1617 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1618 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1619
1620 elif len(value) == 2:
1621 (a_iface_name, a_fqdn, a_ip) = value[0]
1622 (b_iface_name, b_fqdn, b_ip) = value[1]
1623 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1624 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1625
1626 # Device DNS names
1627 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1628 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1629 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1630 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1631 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1632 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1633 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1634
1635 else:
1636 pool_members = [k[1] for k in value]
1637 for item in value:
1638 (iface_name, fqdn, ip) = item
1639 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1640
1641 # Include static DNS entries
1642 # XXX: Should they override the autogenerated results?
1643 # XXX: Convert input to yaml more useable.
1644 # Format:
1645 ##; this is a comment
1646 ## roomburgh=Roomburgh1
1647 ## apkerk1.Vosko=172.17.176.8 ;this as well
1648 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1649
1650 # Hack to allow special entries, for development
1651 wleiden_raw = {}
1652
1653 for line in dns_list:
1654 reverse = False
1655 k, items = line.items()[0]
1656 if type(items) == dict:
1657 if items.has_key('reverse'):
1658 reverse = items['reverse']
1659 items = items['a']
1660 else:
1661 items = items['cname']
1662 items = [items] if type(items) != list else items
1663 for item in items:
1664 if item.startswith('IN '):
1665 wleiden_raw[k] = item
1666 elif valid_addr(item):
1667 wleiden_zone[k].append((item, reverse))
1668 else:
1669 wleiden_cname[k] = item
1670
1671 # Hack to get dynamic pool listing
1672 def chunks(l, n):
1673 return [l[i:i+n] for i in range(0, len(l), n)]
1674
1675 ntp_servers = [x[0] for x in get_nameservers()]
1676 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1677 for ntp_server in chunk:
1678 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1679
1680 details = dict()
1681 # 24 updates a day allowed
1682 details['serial'] = time.strftime('%Y%m%d%H')
1683
1684 if external:
1685 dns_masters = ['ns1.vanderzwet.net', 'ns1.transip.net', 'ns2.transip.eu']
1686 else:
1687 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1688
1689 details['master'] = dns_masters[0]
1690 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1691
1692 dns_header = '''
1693$TTL 3h
1694%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1695 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1696
1697%(ns_servers)s
1698 \n'''
1699
1700
1701 if not os.path.isdir(output_dir):
1702 os.makedirs(output_dir)
1703 details['zone'] = 'wleiden.net'
1704 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1705 f.write(dns_header % details)
1706
1707 for host,items in wleiden_zone.iteritems():
1708 for ip,reverse in items:
1709 if ip not in ['0.0.0.0']:
1710 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1711 for source,dest in wleiden_cname.iteritems():
1712 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1713 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1714 for source, dest in wleiden_raw.iteritems():
1715 f.write("%s.wleiden.net. %s\n" % (source, dest))
1716 f.close()
1717
1718 # Create whole bunch of specific sub arpa zones. To keep it compliant
1719 for s in range(16,32):
1720 details['zone'] = '%i.172.in-addr.arpa' % s
1721 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1722 f.write(dns_header % details)
1723
1724 #XXX: Not effient, fix to proper data structure and do checks at other
1725 # stages
1726 for host,items in wleiden_zone.iteritems():
1727 for ip,reverse in items:
1728 if not reverse:
1729 continue
1730 if valid_addr(ip):
1731 if valid_addr(ip):
1732 if int(ip.split('.')[1]) == s:
1733 rev_ip = '.'.join(reversed(ip.split('.')))
1734 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1735 f.close()
1736
1737
1738def usage():
1739 print """Usage: %(prog)s <argument>
1740Argument:
1741\tcleanup = Cleanup all YAML files to specified format
1742\tstandalone [port] = Run configurator webserver [8000]
1743\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1744\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1745\tfull-export = Generate yaml export script for heatmap.
1746\tstatic [outputdir] = Generate all config files and store on disk
1747\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1748\ttest <node> [<file>] = Receive output for certain node [all files].
1749\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1750\tlist <status> <items> = List systems which have certain status
1751\tcreate network.kml = Create Network KML file for use in Google Earth
1752
1753Arguments:
1754\t<node> = NodeName (example: HybridRick)
1755\t<file> = %(files)s
1756\t<status> = all|up|down|planned
1757\t<items> = systems|nodes|proxies
1758
1759NOTE FOR DEVELOPERS; you can test your changes like this:
1760 BEFORE any changes in this code:
1761 $ ./gformat.py static /tmp/pre
1762 AFTER the changes:
1763 $ ./gformat.py static /tmp/post
1764 VIEW differences and VERIFY all are OK:
1765 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1766""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1767 exit(0)
1768
1769
1770def is_text_request(environ=os.environ):
1771 """ Find out whether we are calling from the CLI or any text based CLI utility """
1772 if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1773 return True
1774
1775 if 'HTTP_USER_AGENT' in environ:
1776 return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1777 else:
1778 return False
1779
1780
1781def switchFormat(setting):
1782 if setting:
1783 return "YES"
1784 else:
1785 return "NO"
1786
1787def rlinput(prompt, prefill=''):
1788 import readline
1789 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1790 try:
1791 return raw_input(prompt)
1792 finally:
1793 readline.set_startup_hook()
1794
1795def fix_conflict(left, right, default='i'):
1796 while True:
1797 print "## %-30s | %-30s" % (left, right)
1798 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1799 if not c:
1800 c = default
1801
1802 if c in ['l','1']:
1803 return left
1804 elif c in ['r','2']:
1805 return right
1806 elif c in ['e', '3']:
1807 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1808 elif c in ['i', '4']:
1809 return None
1810 else:
1811 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1812
1813
1814
1815def print_cgi_response(response_headers, output):
1816 for header in response_headers:
1817 print "%s: %s" % header
1818 print
1819 print output
1820
1821
1822
1823def main():
1824 """Hard working sub"""
1825 # Allow easy hacking using the CLI
1826 if not os.environ.has_key('REQUEST_URI'):
1827 if len(sys.argv) < 2:
1828 usage()
1829
1830 if sys.argv[1] == "standalone":
1831 import SocketServer
1832 import CGIHTTPServer
1833 # Hop to the right working directory.
1834 os.chdir(os.path.dirname(__file__))
1835 try:
1836 PORT = int(sys.argv[2])
1837 except (IndexError,ValueError):
1838 PORT = 8000
1839
1840 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1841 """ Serve this CGI from the root of the webserver """
1842 def is_cgi(self):
1843 if "favicon" in self.path:
1844 return False
1845
1846 self.cgi_info = (os.path.basename(__file__), self.path)
1847 self.path = ''
1848 return True
1849 handler = MyCGIHTTPRequestHandler
1850 SocketServer.TCPServer.allow_reuse_address = True
1851 httpd = SocketServer.TCPServer(("", PORT), handler)
1852 httpd.server_name = 'localhost'
1853 httpd.server_port = PORT
1854
1855 logger.info("serving at port %s", PORT)
1856 try:
1857 httpd.serve_forever()
1858 except KeyboardInterrupt:
1859 httpd.shutdown()
1860 logger.info("All done goodbye")
1861 elif sys.argv[1] == "test":
1862 # Basic argument validation
1863 try:
1864 node = sys.argv[2]
1865 except IndexError:
1866 print "Invalid argument"
1867 exit(1)
1868 except IOError as e:
1869 print e
1870 exit(1)
1871
1872 datadump = get_yaml(node)
1873
1874
1875 # Get files to generate
1876 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1877
1878 # Actual config generation
1879 for config in gen_files:
1880 logger.info("## Generating %s %s", node, config)
1881 print generate_config(node, config, datadump)
1882 elif sys.argv[1] == "test-cgi":
1883 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1884 os.environ['SCRIPT_NAME'] = __file__
1885 response_headers, output = process_cgi_request()
1886 print_cgi_response(response_headers, output)
1887 elif sys.argv[1] == "static":
1888 generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
1889 elif sys.argv[1] == "wind-export":
1890 items = dict()
1891 for node in get_hostlist():
1892 datadump = get_yaml(node)
1893 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1894 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1895 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1896 VALUES (
1897 (SELECT id FROM users WHERE username = 'rvdzwet'),
1898 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1899 'Y');""" % datadump
1900 #for config in files:
1901 # items['config'] = config
1902 # print "## Generating %(node)s %(config)s" % items
1903 # f = open("%(wdir)s/%(config)s" % items, "w")
1904 # f.write(generate_config(node, config, datadump))
1905 # f.close()
1906 for node in get_hostlist():
1907 datadump = get_yaml(node)
1908 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1909 ifacedump = datadump[iface_key]
1910 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1911 ifacedump['nodename'] = datadump['nodename']
1912 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1913 ifacedump['channel'] = 0
1914 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1915 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1916 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1917 elif sys.argv[1] == "nagios-export":
1918 try:
1919 heavy_load = (sys.argv[2] == "--heavy-load")
1920 except IndexError:
1921 heavy_load = False
1922
1923 hostgroup_details = {
1924 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1925 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1926 'walphen' : 'Stichting Wireless Alphen',
1927 'westeinder' : 'Westeinder Plassen',
1928 }
1929
1930 # Convert IP to Host
1931 ip2host = {'root' : 'root'}
1932 for host in get_hostlist():
1933 datadump = get_yaml(host)
1934 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1935 for iface in get_interface_keys(datadump):
1936 if datadump[iface].has_key('autogen_gateway'):
1937 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1938
1939 # Find dependency tree based on output of lvrouted.mytree of nearest node
1940 parents = defaultdict(list)
1941 stack = ['root']
1942 prev_depth = 0
1943 for line in open('lvrouted.mytree').readlines():
1944 depth = line.count('\t')
1945 ip = line.strip().split()[0]
1946
1947 if prev_depth < depth:
1948 try:
1949 parents[ip2host[ip]].append(ip2host[stack[-1]])
1950 except KeyError as e:
1951 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1952 stack.append(ip)
1953 elif prev_depth > depth:
1954 stack = stack[:(depth - prev_depth)]
1955 elif prev_depth == depth:
1956 try:
1957 parents[ip2host[ip]].append(ip2host[stack[-1]])
1958 except KeyError as e:
1959 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1960
1961
1962 prev_depth = depth
1963 # Observe that some nodes has themself as parent or multiple parents
1964 # for now take only the first parent, other behaviour is yet to be explained
1965
1966
1967
1968 params = {
1969 'check_interval' : 5 if heavy_load else 120,
1970 'retry_interval' : 1 if heavy_load else 10,
1971 'max_check_attempts' : 10 if heavy_load else 6,
1972 'notification_interval': 120 if heavy_load else 240,
1973 }
1974
1975 print '''\
1976define host {
1977 name wleiden-node ; Default Node Template
1978 use generic-host ; Use the standard template as initial starting point
1979 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1980 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1981 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1982 notification_interval %(notification_interval)s
1983 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1984 check_command check-host-alive ; Default command to check FreeBSD hosts
1985 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1986}
1987
1988define service {
1989 name wleiden-service ; Default Service Template
1990 use generic-service ; Use the standard template as initial starting point
1991 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1992 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1993 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1994 notification_interval %(notification_interval)s
1995 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1996 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1997}
1998
1999# Please make sure to install:
2000# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2001#
2002# Recompile net-mgmt/nagios-plugins to support check_snmp
2003# make -C /usr/ports/net-mgmt/nagios-plugins
2004#
2005# Install net/bind-tools to allow v2/check_dns_wl to work:
2006# pkg install bind-tools
2007#
2008define command{
2009 command_name check_snmp_disk
2010 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2011}
2012
2013define command{
2014 command_name check_netsnmp_load
2015 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2016}
2017
2018define command{
2019 command_name check_netsnmp_proc
2020 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2021}
2022
2023define command{
2024 command_name check_by_ssh
2025 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2026}
2027
2028define command{
2029 command_name check_dns_wl
2030 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2031}
2032
2033define command{
2034 command_name check_snmp_uptime
2035 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2036}
2037
2038
2039# TDB: dhcp leases
2040# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2041
2042# TDB: internet status
2043# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2044
2045# TDB: Advanced local passive checks
2046# /usr/local/libexec/nagios/check_by_ssh
2047''' % params
2048
2049 print '''\
2050# Service Group, not displayed by default
2051define hostgroup {
2052 hostgroup_name srv_hybrid
2053 alias All Hybrid Nodes
2054 register 0
2055}
2056
2057define service {
2058 use wleiden-service
2059 hostgroup_name srv_hybrid
2060 service_description SSH
2061 check_command check_ssh
2062}
2063
2064define service {
2065 use wleiden-service,service-pnp
2066 hostgroup_name srv_hybrid
2067 service_description HTTP
2068 check_command check_http
2069}
2070
2071define service {
2072 use wleiden-service
2073 hostgroup_name srv_hybrid
2074 service_description DNS
2075 check_command check_dns_wl!"www.wirelessleiden.nl"
2076}
2077'''
2078
2079 if heavy_load:
2080 print '''\
2081define service {
2082 use wleiden-service
2083 hostgroup_name srv_hybrid
2084 service_description UPTIME
2085 check_command check_snmp_uptime
2086}
2087
2088#define service {
2089# use wleiden-service
2090# hostgroup_name srv_hybrid
2091# service_description NTP
2092# check_command check_ntp_peer
2093#}
2094
2095define service {
2096 use wleiden-service
2097 hostgroup_name srv_hybrid
2098 service_description LOAD
2099 check_command check_netsnmp_load
2100}
2101
2102define service {
2103 use wleiden-service
2104 hostgroup_name srv_hybrid
2105 service_description PROC
2106 check_command check_netsnmp_proc
2107}
2108
2109define service {
2110 use wleiden-service
2111 hostgroup_name srv_hybrid
2112 service_description DISK
2113 check_command check_snmp_disk
2114}
2115'''
2116 for node in get_hostlist():
2117 datadump = get_yaml(node)
2118 if not datadump['status'] == 'up':
2119 continue
2120 if not hostgroup_details.has_key(datadump['monitoring_group']):
2121 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2122 print '''\
2123define host {
2124 use wleiden-node,host-pnp
2125 contact_groups admins
2126 host_name %(autogen_fqdn)s
2127 address %(masterip)s
2128 hostgroups srv_hybrid,%(monitoring_group)s\
2129''' % datadump
2130 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2131 print '''\
2132 parents %(parents)s\
2133''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2134 print '''\
2135}
2136'''
2137
2138
2139 for name,alias in hostgroup_details.iteritems():
2140 print '''\
2141define hostgroup {
2142 hostgroup_name %s
2143 alias %s
2144} ''' % (name, alias)
2145
2146
2147 elif sys.argv[1] == "full-export":
2148 hosts = {}
2149 for node in get_hostlist():
2150 datadump = get_yaml(node)
2151 hosts[datadump['nodename']] = datadump
2152 print yaml.dump(hosts)
2153
2154 elif sys.argv[1] == "dns":
2155 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2156 elif sys.argv[1] == "cleanup":
2157 # First generate all datadumps
2158 datadumps = dict()
2159 ssid_to_node = dict()
2160 for host in get_hostlist():
2161 logger.info("# Processing: %s", host)
2162 # Set some boring default values
2163 datadump = { 'board' : 'UNKNOWN' }
2164 datadump.update(get_yaml(host))
2165 datadumps[datadump['nodename']] = datadump
2166
2167 (poel, errors) = make_relations()
2168 print "\n".join(["# WARNING: %s" % x for x in errors])
2169
2170 for host,datadump in datadumps.iteritems():
2171 try:
2172 # Convert all yes and no to boolean values
2173 def fix_boolean(dump):
2174 for key in dump.keys():
2175 if type(dump[key]) == dict:
2176 dump[key] = fix_boolean(dump[key])
2177 elif str(dump[key]).lower() in ["yes", "true"]:
2178 dump[key] = True
2179 elif str(dump[key]).lower() in ["no", "false"]:
2180 # Compass richting no (Noord Oost) is valid input
2181 if key != "compass": dump[key] = False
2182 return dump
2183 datadump = fix_boolean(datadump)
2184
2185 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2186 if not 'latitude' in datadump and not 'longitude' in datadump:
2187 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2188 elif 'latitude' in datadump and 'longitude' in datadump:
2189 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2190 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2191 # TODO: Compare outcome of both coordinate systems and validate against each-other
2192
2193 if datadump['nodename'].startswith('Proxy'):
2194 datadump['nodename'] = datadump['nodename'].lower()
2195
2196 for iface_key in get_interface_keys(datadump):
2197 try:
2198 # All our normal wireless cards are normal APs now
2199 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2200 datadump[iface_key]['mode'] = 'ap'
2201 # Wireless Leiden SSID have an consistent lowercase/uppercase
2202 if datadump[iface_key].has_key('ssid'):
2203 ssid = datadump[iface_key]['ssid']
2204 prefix = 'ap-WirelessLeiden-'
2205 if ssid.lower().startswith(prefix.lower()):
2206 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2207 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2208 datadump[iface_key]['mode'] = 'autogen-FIXME'
2209 if not datadump[iface_key].has_key('comment'):
2210 datadump[iface_key]['comment'] = 'autogen-FIXME'
2211
2212 if datadump[iface_key].has_key('ns_mac'):
2213 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2214
2215 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2216 datadump[iface_key] = datadump[iface_key]['desc']
2217
2218 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2219 # due to better collision avoidance.
2220 if datadump[iface_key]['type'] == '11b':
2221 datadump[iface_key]['type'] = '11g'
2222
2223 # Setting 802.11g channels to de-facto standards, to avoid
2224 # un-detected sharing with other overlapping channels
2225 #
2226 # Technically we could also use channel 13 in NL, but this is not
2227 # recommended as foreign devices might not be able to select this
2228 # channel. Secondly using 1,5,9,13 instead is going to clash with
2229 # the de-facto usage of 1,6,11.
2230 #
2231 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2232 channels_at_2400Mhz = (1,6,11)
2233 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2234 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2235 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2236 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2237
2238 # Mandatory interface keys
2239 if not datadump[iface_key].has_key('status'):
2240 datadump[iface_key]['status'] = 'planned'
2241
2242 x = datadump[iface_key]['comment']
2243 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2244
2245 # Fixing bridge_type if none is found
2246 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2247 if not 'bridge_type' in datadump[iface_key]:
2248 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2249
2250 # Making sure description works
2251 if datadump[iface_key].has_key('desc'):
2252 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2253 del datadump[iface_key]['desc']
2254 else:
2255 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2256 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2257 if response:
2258 datadump[iface_key]['comment'] = response
2259 del datadump[iface_key]['desc']
2260
2261 # Check DHCP configuration
2262 dhcp_type(datadump[iface_key])
2263
2264 # Set the compass value based on the angle between the poels
2265 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2266 my_pool = poel[network(datadump[iface_key]['ip'])]
2267 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2268 if remote_hosts:
2269 compass_target = remote_hosts[0]
2270 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2271 # TODO: Compass wanted and actual direction might differ
2272
2273 # Monitoring Group default
2274 if not 'monitoring_group' in datadump:
2275 datadump['monitoring_group'] = 'wleiden'
2276
2277 except Exception as exc:
2278 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
2279 raise
2280 store_yaml(datadump)
2281 except Exception as exc:
2282 exc.args = ("# Error while processing %s" % host,) + exc.args
2283 raise
2284 elif sys.argv[1] == "list":
2285 use_fqdn = False
2286 if len(sys.argv) < 4:
2287 usage()
2288 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2289 usage()
2290 if not sys.argv[3] in ["nodes","proxies","systems"]:
2291 usage()
2292
2293 if len(sys.argv) > 4:
2294 if sys.argv[4] == "fqdn":
2295 use_fqdn = True
2296 else:
2297 usage()
2298
2299 for system in get_hostlist():
2300 datadump = get_yaml(system)
2301 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2302 continue
2303
2304 output = datadump['autogen_fqdn'] if use_fqdn else system
2305 if sys.argv[2] == "all":
2306 print output
2307 elif datadump['status'] == sys.argv[2]:
2308 print output
2309 elif sys.argv[1] == "create":
2310 if sys.argv[2] == "network.kml":
2311 print make_network_kml.make_graph()
2312 elif sys.argv[2] == "host-ips.txt":
2313 for system in get_hostlist():
2314 datadump = get_yaml(system)
2315 ips = [datadump['masterip']]
2316 for ifkey in get_interface_keys(datadump):
2317 ips.append(datadump[ifkey]['ip'].split('/')[0])
2318 print system, ' '.join(ips)
2319 elif sys.argv[2] == "host-pos.txt":
2320 for system in get_hostlist():
2321 datadump = get_yaml(system)
2322 print system, datadump['rdnap_x'], datadump['rdnap_y']
2323 elif sys.argv[2] == 'ssh_config':
2324 print '''
2325Host *.wleiden.net
2326 User root
2327
2328Host 172.16.*.*
2329 User root
2330'''
2331 for system in get_hostlist():
2332 datadump = get_yaml(system)
2333 print '''\
2334Host %s
2335 User root
2336
2337Host %s
2338 User root
2339
2340Host %s
2341 User root
2342
2343Host %s
2344 User root
2345''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2346 else:
2347 usage()
2348 else:
2349 usage()
2350 else:
2351 # Do not enable debugging for config requests as it highly clutters the output
2352 if not is_text_request():
2353 cgitb.enable()
2354 response_headers, output = process_cgi_request()
2355 print_cgi_response(response_headers, output)
2356
2357if __name__ == "__main__":
2358 main()
Note: See TracBrowser for help on using the repository browser.