source: genesis/tools/gformat.py@ 13937

Last change on this file since 13937 was 13937, checked in by rick, 7 years ago

Add static files generating on update
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 79.6 KB
Line 
1#!/usr/local/bin/python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
35
36import argparse
37import cgi
38import cgitb
39import copy
40import glob
41import make_network_kml
42import math
43import pyproj
44import random
45import re
46import socket
47import string
48import subprocess
49import textwrap
50import time
51import urlparse
52
53from pprint import pprint
54from collections import defaultdict, OrderedDict
55from sys import stderr
56try:
57 import yaml
58except ImportError, e:
59 print e
60 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
61 exit(1)
62
63try:
64 from yaml import CLoader as Loader
65 from yaml import CDumper as Dumper
66except ImportError:
67 from yaml import Loader, Dumper
68
69from jinja2 import Environment, Template
70def yesorno(value):
71 return "YES" if bool(value) else "NO"
72env = Environment()
73env.filters['yesorno'] = yesorno
74def render_template(datadump, template):
75 result = env.from_string(template).render(datadump)
76 # Make it look pretty to the naked eye, as jinja templates are not so
77 # friendly when it comes to whitespace formatting
78 ## Remove extra whitespace at end of line lstrip() style.
79 result = re.sub(r'\n[\ ]+','\n', result)
80 ## Include only a single newline between an definition and a comment
81 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
82 ## Remove extra newlines after single comment
83 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
84 return result
85
86import logging
87logging.basicConfig(format='# %(levelname)s: %(message)s' )
88logger = logging.getLogger()
89logger.setLevel(logging.DEBUG)
90
91
92if os.environ.has_key('CONFIGROOT'):
93 NODE_DIR = os.environ['CONFIGROOT']
94else:
95 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
96__version__ = '$Id: gformat.py 13937 2017-07-06 09:10:48Z rick $'
97
98CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
99
100files = [
101 'authorized_keys',
102 'dnsmasq.conf',
103 'dhcpd.conf',
104 'rc.conf.local',
105 'resolv.conf',
106 'motd',
107 'ntp.conf',
108 'pf.hybrid.conf.local',
109 'unbound.wleiden.conf',
110 'wleiden.yaml',
111 ]
112
113# Global variables uses
114OK = 10
115DOWN = 20
116UNKNOWN = 90
117
118
119ileiden_proxies = OrderedDict()
120normal_proxies = []
121datadump_cache = {}
122interface_list_cache = {}
123rc_conf_local_cache = {}
124nameservers_cache = []
125relations_cache = None
126def clear_cache():
127 ''' Poor mans cache implementation '''
128 global datadump_cache, interface_list_cache, rc_conf_local_cache, ileiden_proxies, normal_proxies, nameservers_cache
129 datadump_cache = {}
130 interface_list_cache = {}
131 rc_conf_local_cache = {}
132 ileiden_proxies = OrderedDict()
133 normal_proxies = []
134 nameservers_cache = []
135 relations_cache = None
136
137NO_DHCP = 0
138DHCP_CLIENT = 10
139DHCP_SERVER = 20
140def dhcp_type(item):
141 if not item.has_key('dhcp'):
142 return NO_DHCP
143 elif not item['dhcp']:
144 return NO_DHCP
145 elif item['dhcp'].lower() == 'client':
146 return DHCP_CLIENT
147 else:
148 # Validation Checks
149 begin,end = map(int,item['dhcp'].split('-'))
150 if begin >= end:
151 raise ValueError("DHCP Start >= DHCP End")
152 return DHCP_SERVER
153
154def etrs2rd(lat, lon):
155 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
156 p2 = pyproj.Proj(init='EPSG:28992')
157 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
158 return (RDx, RDy)
159
160def rd2etrs(RDx, RDy):
161 p1 = pyproj.Proj(init='EPSG:28992')
162 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
163 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
164 return (lat, lon)
165
166def get_yaml(item,add_version_info=True):
167 try:
168 """ Get configuration yaml for 'item'"""
169 if datadump_cache.has_key(item):
170 return datadump_cache[item].copy()
171
172 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
173 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
174 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
175
176 # Default values
177 datadump = {
178 'autogen_revision' : 'NOTFOUND',
179 'autogen_gfile' : gfile,
180 'service_proxy_ileiden' : False,
181 }
182 f = open(gfile, 'r')
183 datadump.update(yaml.load(f,Loader=Loader))
184 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
185 if datadump['nodetype'] == 'Hybrid':
186 # Some values are defined implicitly
187 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
188 datadump['service_incoming_rdr'] = True
189 # Use some boring defaults
190 defaults = {
191 'service_proxy_normal' : False,
192 'service_accesspoint' : True,
193 'service_incoming_rdr' : False,
194 'service_concentrator' : False,
195 'monitoring_group' : 'wleiden',
196 }
197 for (key,value) in defaults.iteritems():
198 if not datadump.has_key(key):
199 datadump[key] = value
200 f.close()
201
202 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
203 if add_version_info:
204 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
205 lines = p.communicate()[0].split('\n')
206 if p.returncode == 0:
207 for line in lines:
208 if line:
209 (key, value) = line.split(': ')
210 datadump["autogen_" + key.lower().replace(' ','_')] = value
211
212 # Preformat certain needed variables for formatting and push those into special object
213 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
214
215 wlan_count=0
216 try:
217 for key in get_interface_keys(datadump, True):
218 datadump[key]['autogen_ifbase'] = key.split('_')[1]
219 datadump[key]['autogen_vlan'] = False
220
221 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
222 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
223 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
224
225 if datadump[key].has_key('parent'):
226 if datadump[key].has_key('ip'):
227 raise ValueError("Interface bridge member cannot have IP assigned")
228 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
229 raise ValueError("Interface bridge member cannot have DHCP set")
230
231 if datadump[key].has_key('ip'):
232 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
233
234 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
235 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
236 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
237 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
238 wlan_count += 1
239 else:
240 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
241 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
242 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
243 datadump[key]['autogen_vlan'] = key.split('_')[2]
244 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
245 else:
246 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
247
248 except Exception:
249 print "# Error while processing interface %s" % key
250 raise
251
252 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
253 if dhcp_type(datadump[key]) == DHCP_SERVER]
254
255 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
256 datadump['autogen_item'] = item
257
258 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
259 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
260 datadump_cache[item] = datadump.copy()
261 except Exception:
262 print "# Error while processing %s" % item
263 raise
264 return datadump
265
266
267def store_yaml(datadump, header=False):
268 """ Store configuration yaml for 'item'"""
269 item = datadump['autogen_item']
270 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
271
272 output = generate_wleiden_yaml(datadump, header)
273
274 f = open(gfile, 'w')
275 f.write(output)
276 f.close()
277
278
279def network(ip):
280 addr, mask = ip.split('/')
281 # Not parsing of these folks please
282 addr = parseaddr(addr)
283 mask = int(mask)
284 network = addr & ~((1 << (32 - mask)) - 1)
285 return network
286
287
288
289def make_relations():
290 """ Process _ALL_ yaml files to get connection relations """
291 global relations_cache
292
293 if relations_cache:
294 return relations_cache
295
296 errors = []
297 poel = defaultdict(list)
298
299 for host in get_hostlist():
300 datadump = get_yaml(host)
301 try:
302 for iface_key in get_interface_keys(datadump):
303 # Bridge members has no IP assigned
304 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
305 continue
306 net_addr = network(datadump[iface_key]['ip'])
307 poel[net_addr] += [(host,datadump[iface_key].copy())]
308 except (KeyError, ValueError) as e:
309 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
310 continue
311
312 relations_cache = (poel, errors)
313 return relations_cache
314
315
316
317def valid_addr(addr):
318 """ Show which address is valid in which are not """
319 return str(addr).startswith('172.')
320
321def get_hostlist():
322 """ Combined hosts and proxy list"""
323 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
324
325def angle_between_points(lat1,lat2,long1,long2):
326 """
327 Return Angle in radians between two GPS coordinates
328 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
329 """
330 dy = lat2 - lat1
331 dx = math.cos(lat1)*(long2 - long1)
332 angle = math.atan2(dy,dx)
333 return angle
334
335
336
337def angle_to_cd(angle):
338 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
339
340 # For easy conversion get positive degree
341 degrees = math.degrees(angle)
342 abs_degrees = 360 + degrees if degrees < 0 else degrees
343
344 # Numbers can be confusing calculate from the 4 main directions
345 p = 22.5
346 if abs_degrees < p:
347 cd = "n"
348 elif abs_degrees < (90 - p):
349 cd = "no"
350 elif abs_degrees < (90 + p):
351 cd = "o"
352 elif abs_degrees < (180 - p):
353 cd = "zo"
354 elif abs_degrees < (180 + p):
355 cd = "z"
356 elif abs_degrees < (270 - p):
357 cd = "zw"
358 elif abs_degrees < (270 + p):
359 cd = "w"
360 elif abs_degrees < (360 - p):
361 cd = "nw"
362 else:
363 cd = "n"
364 return cd
365
366
367
368def cd_between_hosts(hostA, hostB, datadumps):
369 # Using RDNAP coordinates
370 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
371 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
372 return angle_to_cd(math.atan2(dx,dy))
373
374 # GPS coordinates seems to fail somehow
375 #latA = float(datadumps[hostA]['latitude'])
376 #latB = float(datadumps[hostB]['latitude'])
377 #lonA = float(datadumps[hostA]['longitude'])
378 #lonB = float(datadumps[hostB]['longitude'])
379 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
380
381
382def generate_title(nodelist):
383 """ Main overview page """
384 items = {'root' : "." }
385 def fl(spaces, line):
386 return (' ' * spaces) + line + '\n'
387
388 output = """
389<html>
390 <head>
391 <title>Wireless leiden Configurator - GFormat</title>
392 <style type="text/css">
393 th {background-color: #999999}
394 tr:nth-child(odd) {background-color: #cccccc}
395 tr:nth-child(even) {background-color: #ffffff}
396 th, td {padding: 0.1em 1em}
397 </style>
398 </head>
399 <body>
400 <center>
401 <form type="GET" action="%(root)s">
402 <input type="hidden" name="action" value="update">
403 <input type="submit" value="Update Configuration Database (SVN)">
404 </form>
405 <table>
406 <caption><h3>Wireless Leiden Configurator</h3></caption>
407 """ % items
408
409 for node in nodelist:
410 items['node'] = node
411 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
412 for config in files:
413 items['config'] = config
414 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
415 output += fl(5, "</tr>")
416 output += """
417 </table>
418 <hr />
419 <em>%s</em>
420 </center>
421 </body>
422</html>
423 """ % __version__
424
425 return output
426
427
428
429def generate_node(node):
430 """ Print overview of all files available for node """
431 return "\n".join(files)
432
433def generate_node_overview(host, datadump=False):
434 """ Print overview of all files available for node """
435 if not datadump:
436 datadump = get_yaml(host)
437 params = { 'host' : host }
438 output = "<em><a href='..'>Back to overview</a></em><hr />"
439 output += "<h2>Available files:</h2><ul>"
440 for cf in files:
441 params['cf'] = cf
442 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
443 output += "</ul>"
444
445 # Generate and connection listing
446 output += "<h2>Connected To:</h2><ul>"
447 (poel, errors) = make_relations()
448 for network, hosts in poel.iteritems():
449 if host in [x[0] for x in hosts]:
450 if len(hosts) == 1:
451 # Single not connected interface
452 continue
453 for remote,ifacedump in hosts:
454 if remote == host:
455 # This side of the interface
456 continue
457 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
458 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
459 output += "</ul>"
460 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
461
462 output += "<hr /><em><a href='..'>Back to overview</a></em>"
463 return output
464
465
466def generate_header(datadump, ctag="#"):
467 return """\
468%(ctag)s
469%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
470%(ctag)s
471""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
472
473
474
475def parseaddr(s):
476 """ Process IPv4 CIDR notation addr to a (binary) number """
477 f = s.split('.')
478 return (long(f[0]) << 24L) + \
479 (long(f[1]) << 16L) + \
480 (long(f[2]) << 8L) + \
481 long(f[3])
482
483
484
485def showaddr(a):
486 """ Display IPv4 addr in (dotted) CIDR notation """
487 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
488
489
490def is_member(ip, mask, canidate):
491 """ Return True if canidate is part of ip/mask block"""
492 ip_addr = parseaddr(ip)
493 ip_canidate = parseaddr(canidate)
494 mask = int(mask)
495 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
496 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
497 return ip_addr == ip_canidate
498
499
500
501def cidr2netmask(netmask):
502 """ Given a 'netmask' return corresponding CIDR """
503 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
504
505def get_network(addr, mask):
506 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
507
508
509def generate_dhcpd_conf(datadump):
510 """ Generate config file '/usr/local/etc/dhcpd.conf """
511 # Redundency support, in cause local DNS server is not running/responding.
512 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
513 output = generate_header(datadump)
514 output += Template("""\
515# option definitions common to all supported networks...
516option domain-name "dhcp.{{ autogen_fqdn }}";
517
518default-lease-time 600;
519max-lease-time 7200;
520
521# Use this to enble / disable dynamic dns updates globally.
522#ddns-update-style none;
523
524# If this DHCP server is the official DHCP server for the local
525# network, the authoritative directive should be uncommented.
526authoritative;
527
528# Use this to send dhcp log messages to a different log file (you also
529# have to hack syslog.conf to complete the redirection).
530log-facility local7;
531
532# UniFi Discovery Support
533option space ubnt;
534option ubnt.unifi-address code 1 = ip-address;
535#
536class "ubnt" {
537 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
538 option vendor-class-identifier "ubnt";
539 vendor-option-space ubnt;
540}
541
542#
543# Interface definitions
544#
545\n\n""").render(datadump)
546
547
548 # TODO: Use textwrap.fill instead
549 def indent(text, count):
550 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
551
552 # Process DHCP blocks
553 dhcp_out = defaultdict(list)
554 for iface_key in get_interface_keys(datadump):
555 ifname = datadump[iface_key]['autogen_ifbase']
556 groupif = ifname if not datadump[iface_key]['autogen_vlan'] else iface_key
557 if not datadump[iface_key].has_key('comment'):
558 datadump[iface_key]['comment'] = None
559
560 if not datadump[iface_key].has_key('ip'):
561 continue
562
563 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
564
565 (addr, mask) = datadump[iface_key]['ip'].split('/')
566 datadump[iface_key]['autogen_addr'] = addr
567 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
568 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
569
570 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
571 dhcp_out[groupif].append(textwrap.dedent("""\
572 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
573 ### not autoritive
574 }
575 """ % datadump[iface_key]))
576 continue
577
578 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
579 dhcp_part = ".".join(addr.split('.')[0:3])
580 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
581 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
582 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
583
584 # Assume the first 10 IPs could be used for static entries
585 if 'no_portal' in datadump:
586 fixed = 5
587 for mac in datadump['no_portal']:
588 dhcp_out[groupif].append(textwrap.dedent("""\
589 host fixed-%(ifname)s-%(fixed)s {
590 hardware ethernet %(mac)s;
591 fixed-address %(prefix)s.%(fixed)s;
592 }
593 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
594 fixed += 1
595
596 if 'dhcp_fixed' in datadump[iface_key]:
597 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
598 dhcp_out[groupif].append(textwrap.dedent("""\
599 host fixed-%(host)s {
600 hardware ethernet %(mac)s;
601 fixed-address %(addr)s;
602 }
603 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
604
605
606 dhcp_out[groupif].append(textwrap.dedent("""\
607 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
608 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
609 option routers %(autogen_addr)s;
610 option domain-name-servers %(autogen_dns_servers)s;
611 option ubnt.unifi-address 172.17.107.10;
612 }
613 """ % datadump[iface_key]))
614
615 # Output the blocks in groups
616 for ifname,value in sorted(dhcp_out.iteritems()):
617 if len(value) > 2:
618 output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2) + '\n}\n\n'
619 else:
620 output += ''.join(value) + "\n\n"
621 return output
622
623
624
625def generate_dnsmasq_conf(datadump):
626 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
627 output = generate_header(datadump)
628 output += Template("""\
629# DHCP server options
630dhcp-authoritative
631dhcp-fqdn
632domain=dhcp.{{ autogen_fqdn }}
633domain-needed
634expand-hosts
635log-async=100
636
637# Low memory footprint
638cache-size=10000
639
640\n""").render(datadump)
641
642 for iface_key in get_interface_keys(datadump):
643 if not datadump[iface_key].has_key('comment'):
644 datadump[iface_key]['comment'] = None
645 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
646
647 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
648 output += "# not autoritive\n\n"
649 continue
650
651 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
652 (ip, cidr) = datadump[iface_key]['ip'].split('/')
653 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
654
655 dhcp_part = ".".join(ip.split('.')[0:3])
656 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
657 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
658 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
659
660 return output
661
662
663class AutoVivification(dict):
664 """Implementation of perl's autovivification feature."""
665 def __getitem__(self, item):
666 try:
667 return dict.__getitem__(self, item)
668 except KeyError:
669 value = self[item] = type(self)()
670 return value
671
672def make_interface_list(datadump):
673 if interface_list_cache.has_key(datadump['autogen_item']):
674 return (interface_list_cache[datadump['autogen_item']])
675 # lo0 configuration:
676 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
677 # - masterip is special as it needs to be assigned to at
678 # least one interface, so if not used assign to lo0
679 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
680 vlan_list = defaultdict(list)
681 bridge_list = defaultdict(list)
682 flags_if = AutoVivification()
683 dhclient_if = {'lo0' : False}
684
685 # XXX: Find some way of send this output nicely
686 output = ''
687
688 masterip_used = False
689 for iface_key in get_interface_keys(datadump):
690 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
691 masterip_used = True
692 break
693 if not masterip_used:
694 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
695
696 if 'serviceid' in datadump:
697 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
698
699 for iface_key in get_interface_keys(datadump):
700 ifacedump = datadump[iface_key]
701 if ifacedump['autogen_bridge_alias']:
702 ifname = ifacedump['autogen_ifbase']
703 else:
704 ifname = ifacedump['autogen_ifname']
705
706 # If defined as vlan interface
707 if ifacedump['autogen_vlan']:
708 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
709
710 # If defined as bridge interface
711 if ifacedump['autogen_bridge_member']:
712 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
713
714
715 # Flag dhclient is possible
716 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
717 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
718
719 # Ethernet address
720 if ifacedump.has_key('ether'):
721 flags_if[ifname]['ether'] = ifacedump['ether']
722
723 # Add interface IP to list
724 if ifacedump.has_key('ip'):
725 item = (ifacedump['ip'], ifacedump['comment'])
726 if addrs_list.has_key(ifname):
727 addrs_list[ifname].append(item)
728 else:
729 addrs_list[ifname] = [item]
730
731 # Alias only needs IP assignment for now, this might change if we
732 # are going to use virtual accesspoints
733 if "alias" in iface_key:
734 continue
735
736 # XXX: Might want to deduct type directly from interface name
737 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
738 # Default to station (client) mode
739 ifacedump['autogen_wlanmode'] = "sta"
740 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
741 ifacedump['autogen_wlanmode'] = "ap"
742
743 if not ifacedump.has_key('channel'):
744 if ifacedump['type'] == '11a':
745 ifacedump['channel'] = 36
746 else:
747 ifacedump['channel'] = 1
748
749 # Allow special hacks at the back like wds and stuff
750 if not ifacedump.has_key('extra'):
751 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
752 else:
753 ifacedump['autogen_extra'] = ifacedump['extra']
754
755 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
756
757 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
758 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
759 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
760 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
761 output += "\n"
762
763 elif ifacedump['type'] in ['ethernet', 'eth']:
764 # No special config needed besides IP
765 pass
766 elif ifacedump['type'] in ['vlan']:
767 # VLAN member has no special configuration
768 pass
769 else:
770 assert False, "Unknown type " + ifacedump['type']
771
772 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
773 interface_list_cache[datadump['autogen_item']] = store
774 return(store)
775
776
777
778def create_proxies_list():
779 if not ileiden_proxies or not normal_proxies:
780 # Placeholder for to-be-installed proxies, this will avoid updating the all
781 # nodes to include this new machine, yet due to an unbound issue, this list
782 # has to be kept small.
783
784 for i in range(1,20):
785 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
786
787 for host in get_hostlist():
788 hostdump = get_yaml(host)
789 if hostdump['status'] == 'up':
790 if hostdump['service_proxy_ileiden']:
791 ileiden_proxies[hostdump['serviceid']] = hostdump
792 if hostdump['service_proxy_normal']:
793 normal_proxies.append(hostdump)
794
795
796
797def generate_rc_conf_local(datadump):
798 """ Generate configuration file '/etc/rc.conf.local' """
799 item = datadump['autogen_item']
800 if rc_conf_local_cache.has_key(item):
801 return rc_conf_local_cache[item]
802
803 if not datadump.has_key('ileiden'):
804 datadump['autogen_ileiden_enable'] = False
805 else:
806 datadump['autogen_ileiden_enable'] = datadump['ileiden']
807
808 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
809
810 create_proxies_list()
811 datadump['autogen_ileiden_proxies'] = ileiden_proxies
812 datadump['autogen_normal_proxies'] = normal_proxies
813 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
814 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
815 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
816 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
817
818 output = generate_header(datadump, "#");
819 output += render_template(datadump, """\
820hostname='{{ autogen_fqdn }}'
821location='{{ location }}'
822nodetype="{{ nodetype }}"
823
824#
825# Configured listings
826#
827captive_portal_whitelist=""
828{% if nodetype == "Proxy" %}
829#
830# Proxy Configuration
831#
832{% if gateway and service_proxy_ileiden -%}
833defaultrouter="{{ gateway }}"
834{% else -%}
835#defaultrouter="NOTSET"
836{% endif -%}
837internalif="{{ internalif }}"
838ileiden_enable="{{ autogen_ileiden_enable }}"
839gateway_enable="{{ autogen_ileiden_enable }}"
840pf_enable="yes"
841pf_rules="/etc/pf.conf"
842{% if autogen_ileiden_enable -%}
843pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
844lvrouted_enable="{{ autogen_ileiden_enable }}"
845lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
846{% else -%}
847pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
848{% endif -%}
849{% if internalroute -%}
850static_routes="wleiden"
851route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
852{% endif -%}
853
854{% elif nodetype == "Hybrid" %}
855 #
856 # Hybrid Configuration
857 #
858 list_ileiden_proxies="
859 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
860 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
861 {% endfor -%}
862 "
863 list_normal_proxies="
864 {% for item in autogen_normal_proxies -%}
865 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
866 {% endfor -%}
867 "
868
869 {% if autogen_dhcp_interfaces -%}
870 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
871 {% else %}
872 captive_portal_interfaces="dummy"
873 {% endif %}
874 externalif="{{ externalif|default('vr0', true) }}"
875 masterip="{{ masterip }}"
876
877 {% if gateway and service_proxy_ileiden %}
878 defaultrouter="{{ gateway }}"
879 {% else %}
880 #defaultrouter="NOTSET"
881 {% endif %}
882
883 #
884 # Defined services
885 #
886 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
887 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
888 service_accesspoint="{{ service_accesspoint|yesorno }}"
889 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
890 service_concentrator="{{ service_concentrator|yesorno }}"
891
892 {% if service_proxy_ileiden %}
893 pf_rules="/etc/pf.hybrid.conf"
894 {% if service_concentrator %}
895 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
896 {% else %}
897 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
898 {% endif %}
899 pf_flags="$pf_flags -D publicnat=80,443"
900 lvrouted_flags="$lvrouted_flags -g"
901 {% elif service_proxy_normal or service_incoming_rdr %}
902 pf_rules="/etc/pf.hybrid.conf"
903 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
904 pf_flags="$pf_flags -D publicnat=0"
905 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
906 named_setfib="1"
907 tinyproxy_setfib="1"
908 dnsmasq_setfib="1"
909 sshd_setfib="1"
910 {% else %}
911 named_auto_forward_only="YES"
912 pf_rules="/etc/pf.node.conf"
913 pf_flags=""
914 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
915 {% endif %}
916 {% if service_concentrator %}
917 # Do mind installing certificates is NOT done automatically for security reasons
918 openvpn_enable="YES"
919 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
920 {% endif %}
921
922 {% if service_proxy_normal %}
923 tinyproxy_enable="yes"
924 {% else %}
925 pen_wrapper_enable="yes"
926 {% endif %}
927
928 {% if service_accesspoint %}
929 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
930 {% endif %}
931
932 {% if board == "ALIX2" or board == "net4801" %}
933 #
934 # ''Fat'' configuration, board has 256MB RAM
935 #
936 dnsmasq_enable="NO"
937 named_enable="YES"
938 unbound_enable="YES"
939 {% if autogen_dhcp_interfaces -%}
940 dhcpd_enable="YES"
941 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
942 {% endif -%}
943 {% elif board == "apu1d" %}
944 #
945 # ''Fat'' configuration, board has 1024MB RAM
946 #
947 dnsmasq_enable="NO"
948 unbound_enable="YES"
949 {% if autogen_dhcp_interfaces -%}
950 dhcpd_enable="YES"
951 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
952 {% endif -%}
953 {% endif -%}
954{% endif %}
955
956#
957# Script variables
958#
959attached_devices="{{ autogen_attached_devices|join(' ') }}"
960neighbours="{{ autogen_neighbours|join(' ') }}"
961
962
963#
964# Interface definitions
965#\n
966""")
967
968 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
969 for iface, vlans in sorted(vlan_list.items()):
970 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
971
972 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
973 for iface in sorted(vlan_list.keys()):
974 if not iface in addrs_list.keys():
975 output += "ifconfig_%s='up'\n" % iface
976
977 output += "\n"
978
979 # Bridge configuration:
980 if bridge_list.keys():
981 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
982
983 for iface in bridge_list.keys():
984 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
985
986 # Bridge member interfaces not containing a configuration should be marked active explcitly.
987 for _,members in bridge_list.items():
988 for iface in members:
989 if not iface in addrs_list.keys():
990 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
991
992 output += "\n"
993
994 # Details like SSID
995 if extra_ouput:
996 output += extra_ouput.strip() + "\n"
997
998 # Print IP address which needs to be assigned over here
999 output += "\n"
1000 for iface,addrs in sorted(addrs_list.iteritems()):
1001 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1002 output += "# %s || %s || %s\n" % (iface, addr, comment)
1003
1004 # Write DHCLIENT entry
1005 if iface in dhclient_if and dhclient_if[iface]:
1006 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1007 continue
1008
1009 # Make sure the external address is always first as this is needed in the
1010 # firewall setup
1011 addrs = sorted(
1012 [x for x in addrs if not '0.0.0.0' in x[0]],
1013 key=lambda x: x[0].split('.')[0],
1014 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1015 )
1016
1017 idx_offset = 0
1018 # Set MAC is required
1019 if flags_if[iface].has_key('ether'):
1020 output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1021 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1022 idx_offset += 1
1023 elif iface in bridge_list:
1024 output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1025 idx_offset += 1
1026 else:
1027 output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1028
1029 for idx, addr in enumerate(addrs[1:]):
1030 output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1031
1032 output += "\n"
1033
1034 rc_conf_local_cache[datadump['autogen_item']] = output
1035 return output
1036
1037
1038
1039def get_all_configs():
1040 """ Get dict with key 'host' with all configs present """
1041 configs = dict()
1042 for host in get_hostlist():
1043 datadump = get_yaml(host)
1044 configs[host] = datadump
1045 return configs
1046
1047
1048def get_interface_keys(config, extra=False):
1049 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1050 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1051 if extra == False:
1052 return filter(lambda x: not "extra" in x, elems)
1053 else:
1054 return elems
1055
1056
1057def get_used_ips(configs):
1058 """ Return array of all IPs used in config files"""
1059 ip_list = []
1060 for config in configs:
1061 ip_list.append(config['masterip'])
1062 if 'serviceid' in config:
1063 ip_list.append(config['serviceid'])
1064 for iface_key in get_interface_keys(config, True):
1065 l = config[iface_key]['ip']
1066 addr, mask = l.split('/')
1067 # Special case do not process
1068 if valid_addr(addr):
1069 ip_list.append(addr)
1070 else:
1071 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1072 return sorted(ip_list)
1073
1074
1075
1076def get_nameservers(max_servers=None):
1077 if nameservers_cache:
1078 return nameservers_cache[0:max_servers]
1079
1080 for host in get_hostlist():
1081 hostdump = get_yaml(host)
1082 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1083 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1084
1085 return nameservers_cache[0:max_servers]
1086
1087
1088def get_neighbours(datadump):
1089 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1090
1091 (poel, errors) = make_relations()
1092 table = []
1093 for iface,addrs in sorted(addrs_list.iteritems()):
1094 if iface in ['lo0']:
1095 continue
1096
1097 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1098 if not addr.startswith('172.'):
1099 # Avoid listing internet connections as pool
1100 continue
1101 for neighbour in poel[network(addr)]:
1102 if neighbour[0] != datadump['autogen_item']:
1103 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1104 return table
1105
1106
1107def get_attached_devices(datadump, url=False):
1108 table = []
1109 for iface_key in get_interface_keys(datadump, True):
1110 ifacedump = datadump[iface_key]
1111
1112 if not ifacedump.has_key('ns_ip'):
1113 continue
1114
1115 x_ip = ifacedump['ns_ip'].split('/')[0]
1116
1117 if 'mode' in ifacedump:
1118 x_mode = ifacedump['mode']
1119 else:
1120 x_mode = 'unknown'
1121
1122 if 'bridge_type' in ifacedump:
1123 device_type = ifacedump['bridge_type']
1124 else:
1125 device_type = 'Unknown'
1126
1127 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1128 return table
1129
1130
1131def generate_resolv_conf(datadump):
1132 """ Generate configuration file '/etc/resolv.conf' """
1133 # XXX: This should properly going to be an datastructure soon
1134 datadump['autogen_header'] = generate_header(datadump, "#")
1135 datadump['autogen_edge_nameservers'] = ''
1136
1137
1138 for masterip,realname in get_nameservers():
1139 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1140
1141 return Template("""\
1142{{ autogen_header }}
1143search wleiden.net
1144
1145# Try local (cache) first
1146nameserver 127.0.0.1
1147
1148{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1149nameserver 8.8.8.8 # Google Public NameServer
1150nameserver 64.6.64.6 # Verisign Public NameServer
1151{% else -%}
1152# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1153{{ autogen_edge_nameservers }}
1154{% endif -%}
1155""").render(datadump)
1156
1157
1158
1159def generate_ntp_conf(datadump):
1160 """ Generate configuration file '/etc/ntp.conf' """
1161 # XXX: This should properly going to be an datastructure soon
1162
1163 datadump['autogen_header'] = generate_header(datadump, "#")
1164 datadump['autogen_ntp_servers'] = ''
1165 for host in get_hostlist():
1166 hostdump = get_yaml(host)
1167 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1168 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1169
1170 return Template("""\
1171{{ autogen_header }}
1172
1173{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1174# Machine hooked to internet.
1175server 0.nl.pool.ntp.org iburst maxpoll 9
1176server 1.nl.pool.ntp.org iburst maxpoll 9
1177server 2.nl.pool.ntp.org iburst maxpoll 9
1178server 3.nl.pool.ntp.org iburst maxpoll 9
1179{% else -%}
1180# Local Wireless Leiden NTP Servers.
1181server 0.pool.ntp.wleiden.net iburst maxpoll 9
1182server 1.pool.ntp.wleiden.net iburst maxpoll 9
1183server 2.pool.ntp.wleiden.net iburst maxpoll 9
1184server 3.pool.ntp.wleiden.net iburst maxpoll 9
1185
1186# All the configured NTP servers
1187{{ autogen_ntp_servers }}
1188{% endif %}
1189
1190# If a server loses sync with all upstream servers, NTP clients
1191# no longer follow that server. The local clock can be configured
1192# to provide a time source when this happens, but it should usually
1193# be configured on just one server on a network. For more details see
1194# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1195# The use of Orphan Mode may be preferable.
1196#
1197server 127.127.1.0
1198fudge 127.127.1.0 stratum 10
1199""").render(datadump)
1200
1201
1202def generate_pf_hybrid_conf_local(datadump):
1203 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1204 datadump['autogen_header'] = generate_header(datadump, "#")
1205 if datadump['service_incoming_rdr']:
1206 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1207 return Template("""\
1208{{ autogen_header }}
1209
1210# Redirect some internal facing services outside (7)
1211# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1212{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1213rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1214{% endfor -%}
1215# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1216{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1217rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1218{% endfor -%}
1219""").render(datadump)
1220
1221def generate_unbound_wleiden_conf(datadump):
1222 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1223 datadump['autogen_header'] = generate_header(datadump, "#")
1224
1225 autogen_ips = []
1226 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1227 for iface,addrs in sorted(addrs_list.iteritems()):
1228 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1229 if addr.startswith('172'):
1230 autogen_ips.append((addr.split('/')[0], comment))
1231 datadump['autogen_ips'] = autogen_ips
1232
1233 create_proxies_list()
1234 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1235 return Template("""\
1236{{ autogen_header }}
1237
1238server:
1239 ## Static definitions fail to start on systems with broken ue(4) interfaces
1240{%- for ip,comment in autogen_ips %}
1241 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1242{%- endfor %}
1243 ## Enabling wildcard matching as work-around
1244 interface: 0.0.0.0
1245 interface: ::0
1246
1247forward-zone:
1248 name: '.'
1249{%- if service_proxy_ileiden %}
1250 forward-addr: 8.8.8.8 # Google DNS A
1251 forward-addr: 8.8.4.4 # Google DNS B
1252 forward-addr: 208.67.222.222 # OpenDNS DNS A
1253 forward-addr: 208.67.220.220 # OpenDNS DNS B
1254{% else -%}
1255{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1256 {%- if loop.index <= 5 %}
1257 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1258 {%- endif %}
1259{%- endfor %}
1260{%- endif %}
1261""").render(datadump)
1262
1263def generate_motd(datadump):
1264 """ Generate configuration file '/etc/motd' """
1265 output = Template("""\
1266FreeBSD run ``service motd onestart'' to make me look normal
1267
1268 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1269 Loc: {{ location }}
1270
1271Services:
1272{% if board == "ALIX2" or board == "net4801" -%}
1273{{" -"}} Core Node ({{ board }})
1274{% else -%}
1275{{" -"}} Hulp Node ({{ board }})
1276{% endif -%}
1277{% if service_proxy_normal -%}
1278{{" -"}} Normal Proxy
1279{% endif -%}
1280{% if service_proxy_ileiden -%}
1281{{" -"}} iLeiden Proxy
1282{% endif -%}
1283{% if service_incoming_rdr -%}
1284{{" -"}} Incoming port redirects
1285{% endif %}
1286Interlinks:\n
1287""").render(datadump)
1288
1289
1290 def make_table(table):
1291 if not table:
1292 return " - none\n"
1293 else:
1294 lines = ""
1295 col_width = [max(len(x) for x in col) for col in zip(*table)]
1296 for row in table:
1297 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1298 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1299 return lines
1300
1301 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1302 table = []
1303 for iface,addrs in sorted(addrs_list.iteritems()):
1304 if iface in ['lo0']:
1305 continue
1306 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1307 table.append((iface, addr, comment))
1308
1309 output += make_table(table)
1310 output += '\n'
1311 output += """\
1312Attached devices:
1313"""
1314 output += make_table(get_attached_devices(datadump, url=True))
1315 output += '\n'
1316 output += """\
1317Available neighbours:
1318"""
1319 output += make_table(get_neighbours(datadump))
1320
1321 return output
1322
1323
1324def format_yaml_value(value):
1325 """ Get yaml value in right syntax for outputting """
1326 if isinstance(value,str):
1327 output = '"%s"' % value
1328 else:
1329 output = value
1330 return output
1331
1332
1333
1334def format_wleiden_yaml(datadump):
1335 """ Special formatting to ensure it is editable"""
1336 output = "# Genesis config yaml style\n"
1337 output += "# vim:ts=2:et:sw=2:ai\n"
1338 output += "#\n"
1339 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1340 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1341 if key == 'rdr_rules':
1342 output += '%-10s:\n' % 'rdr_rules'
1343 for rdr_rule in datadump[key]:
1344 output += '- %s\n' % rdr_rule
1345 else:
1346 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1347
1348 output += "\n\n"
1349
1350 # Format (key, required)
1351 key_order = (
1352 ('comment', True),
1353 ('parent', False),
1354 ('ip', False),
1355 ('ether', False),
1356 ('desc', True),
1357 ('sdesc', True),
1358 ('mode', True),
1359 ('type', True),
1360 ('extra_type', False),
1361 ('channel', False),
1362 ('ssid', False),
1363 ('wlan_mac', False),
1364 ('dhcp', True),
1365 ('dhcp_fixed', False),
1366 ('compass', False),
1367 ('distance', False),
1368 ('ns_ip', False),
1369 ('repeater_ip', False),
1370 ('bullet2_ip', False),
1371 ('ns_mac', False),
1372 ('bullet2_mac', False),
1373 ('ns_type', False),
1374 ('bridge_type', False),
1375 ('status', True),
1376 )
1377
1378 for iface_key in sorted(iface_keys):
1379 try:
1380 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1381 if remainder:
1382 raise KeyError("invalid keys: %s" % remainder)
1383
1384 output += "%s:\n" % iface_key
1385 for key,required in key_order:
1386 if datadump[iface_key].has_key(key):
1387 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1388 output += "\n\n"
1389 except Exception:
1390 print "# Error while processing interface %s" % iface_key
1391 raise
1392
1393 return output
1394
1395
1396
1397def generate_wleiden_yaml(datadump, header=True):
1398 """ Generate (petty) version of wleiden.yaml"""
1399 output = generate_header(datadump, "#") if header else ''
1400
1401 for key in datadump.keys():
1402 if key.startswith('autogen_'):
1403 del datadump[key]
1404 # Interface autogen cleanups
1405 elif type(datadump[key]) == dict:
1406 for key2 in datadump[key].keys():
1407 if key2.startswith('autogen_'):
1408 del datadump[key][key2]
1409
1410 output += format_wleiden_yaml(datadump)
1411 return output
1412
1413def generate_nanostation_config(datadump, iface, ns_type):
1414 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1415 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1416
1417 datadump.update(datadump['iface_%s' % iface])
1418
1419 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1420
1421def generate_yaml(datadump):
1422 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1423
1424
1425
1426def generate_config(node, config, datadump=None):
1427 """ Print configuration file 'config' of 'node' """
1428 output = ""
1429 try:
1430 # Load config file
1431 if datadump == None:
1432 datadump = get_yaml(node)
1433
1434 if config == 'wleiden.yaml':
1435 output += generate_wleiden_yaml(datadump)
1436 elif config == 'authorized_keys':
1437 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1438 output += f.read()
1439 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1440 # Fetch local keys if existing
1441 if os.path.exists(node_keys):
1442 output += open(node_keys, 'r').read()
1443 f.close()
1444 elif config == 'dnsmasq.conf':
1445 output += generate_dnsmasq_conf(datadump)
1446 elif config == 'dhcpd.conf':
1447 output += generate_dhcpd_conf(datadump)
1448 elif config == 'rc.conf.local':
1449 output += generate_rc_conf_local(datadump)
1450 elif config == 'resolv.conf':
1451 output += generate_resolv_conf(datadump)
1452 elif config == 'ntp.conf':
1453 output += generate_ntp_conf(datadump)
1454 elif config == 'motd':
1455 output += generate_motd(datadump)
1456 elif config == 'pf.hybrid.conf.local':
1457 output += generate_pf_hybrid_conf_local(datadump)
1458 elif config == 'unbound.wleiden.conf':
1459 output += generate_unbound_wleiden_conf(datadump)
1460 elif config.startswith('vr'):
1461 interface, ns_type = config.strip('.yaml').split('-')
1462 output += generate_nanostation_config(datadump, interface, ns_type)
1463 else:
1464 assert False, "Config not found!"
1465 except IOError, e:
1466 output += "[ERROR] Config file not found"
1467 return output
1468
1469
1470
1471def generate_static(output_dir, logging=True):
1472 items = {'output_dir' : output_dir}
1473 for node in get_hostlist():
1474 items['node'] = node
1475 items['wdir'] = "%(output_dir)s/%(node)s" % items
1476 if not os.path.isdir(items['wdir']):
1477 os.makedirs(items['wdir'])
1478 datadump = get_yaml(node)
1479 f = open("%(wdir)s/index.html" % items, "w")
1480 f.write(generate_node_overview(items['node'], datadump))
1481 f.close()
1482 for config in files:
1483 items['config'] = config
1484 if logging: logger.info("## Generating %(node)s %(config)s" % items)
1485 f = open("%(wdir)s/%(config)s" % items, "w")
1486 f.write(generate_config(node, config, datadump))
1487 f.close()
1488
1489
1490
1491def process_cgi_request(environ=os.environ):
1492 """ When calling from CGI """
1493 response_headers = []
1494 content_type = 'text/plain'
1495
1496 # Update repository if requested
1497 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1498 if form and form.has_key("action") and "update" in form["action"]:
1499 output = "[INFO] Updating subverion, please wait...\n"
1500 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1501 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1502 generate_static(CACHE_DIR, False)
1503 output += "[INFO] All done, redirecting in 5 seconds"
1504 response_headers += [
1505 ('Refresh', '5; url=.'),
1506 ]
1507 reload_cache()
1508 else:
1509 base_uri = environ['REQUEST_URI']
1510 uri = base_uri.strip('/').split('/')[1:]
1511
1512 output = "Template Holder"
1513 if base_uri.endswith('/create/network.kml'):
1514 content_type='application/vnd.google-earth.kml+xml'
1515 output = make_network_kml.make_graph()
1516 elif base_uri.endswith('/api/get/nodeplanner.json'):
1517 content_type='application/json'
1518 output = make_network_kml.make_nodeplanner_json()
1519 elif not uri:
1520 if is_text_request(environ):
1521 output = '\n'.join(get_hostlist())
1522 else:
1523 content_type = 'text/html'
1524 output = generate_title(get_hostlist())
1525 elif len(uri) == 1:
1526 if is_text_request(environ):
1527 output = generate_node(uri[0])
1528 else:
1529 content_type = 'text/html'
1530 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1531 elif len(uri) == 2:
1532 output = generate_config(uri[0], uri[1])
1533 else:
1534 assert False, "Invalid option"
1535
1536 # Return response
1537 response_headers += [
1538 ('Content-type', content_type),
1539 ('Content-Length', str(len(output))),
1540 ]
1541 return(response_headers, str(output))
1542
1543
1544def make_dns(output_dir = 'dns', external = False):
1545 items = dict()
1546
1547 # hostname is key, IP is value
1548 wleiden_zone = defaultdict(list)
1549 wleiden_cname = dict()
1550
1551 pool = dict()
1552 for node in get_hostlist():
1553 datadump = get_yaml(node)
1554
1555 fqdn = datadump['nodename']
1556
1557 if datadump.has_key('rdr_host'):
1558 remote_target = datadump['rdr_host']
1559 elif datadump.has_key('remote_access') and datadump['remote_access']:
1560 remote_target = datadump['remote_access'].split(':')[0]
1561 else:
1562 remote_target = None
1563
1564 if remote_target:
1565 try:
1566 parseaddr(remote_target)
1567 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1568 except (IndexError, ValueError):
1569 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1570
1571
1572 wleiden_zone[fqdn].append((datadump['masterip'], True))
1573
1574 # Hacking to get proper DHCP IPs and hostnames
1575 for iface_key in get_interface_keys(datadump):
1576 iface_name = iface_key.replace('_','-')
1577 if 'ip' in datadump[iface_key]:
1578 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1579 try:
1580 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1581 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1582 dhcp_part = ".".join(ip.split('.')[0:3])
1583 if ip != datadump['masterip']:
1584 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1585 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1586 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1587 except (AttributeError, ValueError, KeyError):
1588 # First push it into a pool, to indentify the counter-part later on
1589 addr = parseaddr(ip)
1590 cidr = int(cidr)
1591 addr = addr & ~((1 << (32 - cidr)) - 1)
1592 if pool.has_key(addr):
1593 pool[addr] += [(iface_name, fqdn, ip)]
1594 else:
1595 pool[addr] = [(iface_name, fqdn, ip)]
1596 continue
1597
1598
1599
1600 # WL uses an /29 to configure an interface. IP's are ordered like this:
1601 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1602
1603 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1604
1605 # Automatic naming convention of interlinks namely 2 + remote.lower()
1606 for (key,value) in pool.iteritems():
1607 # Make sure they are sorted from low-ip to high-ip
1608 value = sorted(value, key=lambda x: parseaddr(x[2]))
1609
1610 if len(value) == 1:
1611 (iface_name, fqdn, ip) = value[0]
1612 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1613
1614 # Device DNS names
1615 if 'cnode' in fqdn.lower():
1616 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1617 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1618
1619 elif len(value) == 2:
1620 (a_iface_name, a_fqdn, a_ip) = value[0]
1621 (b_iface_name, b_fqdn, b_ip) = value[1]
1622 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1623 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1624
1625 # Device DNS names
1626 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1627 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1628 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1629 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1630 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1631 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1632 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1633
1634 else:
1635 pool_members = [k[1] for k in value]
1636 for item in value:
1637 (iface_name, fqdn, ip) = item
1638 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1639
1640 # Include static DNS entries
1641 # XXX: Should they override the autogenerated results?
1642 # XXX: Convert input to yaml more useable.
1643 # Format:
1644 ##; this is a comment
1645 ## roomburgh=Roomburgh1
1646 ## apkerk1.Vosko=172.17.176.8 ;this as well
1647 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1648
1649 # Hack to allow special entries, for development
1650 wleiden_raw = {}
1651
1652 for line in dns_list:
1653 reverse = False
1654 k, items = line.items()[0]
1655 if type(items) == dict:
1656 if items.has_key('reverse'):
1657 reverse = items['reverse']
1658 items = items['a']
1659 else:
1660 items = items['cname']
1661 items = [items] if type(items) != list else items
1662 for item in items:
1663 if item.startswith('IN '):
1664 wleiden_raw[k] = item
1665 elif valid_addr(item):
1666 wleiden_zone[k].append((item, reverse))
1667 else:
1668 wleiden_cname[k] = item
1669
1670 # Hack to get dynamic pool listing
1671 def chunks(l, n):
1672 return [l[i:i+n] for i in range(0, len(l), n)]
1673
1674 ntp_servers = [x[0] for x in get_nameservers()]
1675 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1676 for ntp_server in chunk:
1677 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1678
1679 details = dict()
1680 # 24 updates a day allowed
1681 details['serial'] = time.strftime('%Y%m%d%H')
1682
1683 if external:
1684 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1685 else:
1686 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1687
1688 details['master'] = dns_masters[0]
1689 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1690
1691 dns_header = '''
1692$TTL 3h
1693%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1694 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1695
1696%(ns_servers)s
1697 \n'''
1698
1699
1700 if not os.path.isdir(output_dir):
1701 os.makedirs(output_dir)
1702 details['zone'] = 'wleiden.net'
1703 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1704 f.write(dns_header % details)
1705
1706 for host,items in wleiden_zone.iteritems():
1707 for ip,reverse in items:
1708 if ip not in ['0.0.0.0']:
1709 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1710 for source,dest in wleiden_cname.iteritems():
1711 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1712 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1713 for source, dest in wleiden_raw.iteritems():
1714 f.write("%s.wleiden.net. %s\n" % (source, dest))
1715 f.close()
1716
1717 # Create whole bunch of specific sub arpa zones. To keep it compliant
1718 for s in range(16,32):
1719 details['zone'] = '%i.172.in-addr.arpa' % s
1720 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1721 f.write(dns_header % details)
1722
1723 #XXX: Not effient, fix to proper data structure and do checks at other
1724 # stages
1725 for host,items in wleiden_zone.iteritems():
1726 for ip,reverse in items:
1727 if not reverse:
1728 continue
1729 if valid_addr(ip):
1730 if valid_addr(ip):
1731 if int(ip.split('.')[1]) == s:
1732 rev_ip = '.'.join(reversed(ip.split('.')))
1733 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1734 f.close()
1735
1736
1737def usage():
1738 print """Usage: %(prog)s <argument>
1739Argument:
1740\tcleanup = Cleanup all YAML files to specified format
1741\tstandalone [port] = Run configurator webserver [8000]
1742\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1743\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1744\tfull-export = Generate yaml export script for heatmap.
1745\tstatic [outputdir] = Generate all config files and store on disk
1746\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1747\ttest <node> [<file>] = Receive output for certain node [all files].
1748\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1749\tlist <status> <items> = List systems which have certain status
1750\tcreate network.kml = Create Network KML file for use in Google Earth
1751
1752Arguments:
1753\t<node> = NodeName (example: HybridRick)
1754\t<file> = %(files)s
1755\t<status> = all|up|down|planned
1756\t<items> = systems|nodes|proxies
1757
1758NOTE FOR DEVELOPERS; you can test your changes like this:
1759 BEFORE any changes in this code:
1760 $ ./gformat.py static /tmp/pre
1761 AFTER the changes:
1762 $ ./gformat.py static /tmp/post
1763 VIEW differences and VERIFY all are OK:
1764 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1765""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1766 exit(0)
1767
1768
1769def is_text_request(environ=os.environ):
1770 """ Find out whether we are calling from the CLI or any text based CLI utility """
1771 if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1772 return True
1773
1774 if 'HTTP_USER_AGENT' in environ:
1775 return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1776 else:
1777 return False
1778
1779
1780def switchFormat(setting):
1781 if setting:
1782 return "YES"
1783 else:
1784 return "NO"
1785
1786def rlinput(prompt, prefill=''):
1787 import readline
1788 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1789 try:
1790 return raw_input(prompt)
1791 finally:
1792 readline.set_startup_hook()
1793
1794def fix_conflict(left, right, default='i'):
1795 while True:
1796 print "## %-30s | %-30s" % (left, right)
1797 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1798 if not c:
1799 c = default
1800
1801 if c in ['l','1']:
1802 return left
1803 elif c in ['r','2']:
1804 return right
1805 elif c in ['e', '3']:
1806 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1807 elif c in ['i', '4']:
1808 return None
1809 else:
1810 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1811
1812
1813
1814def print_cgi_response(response_headers, output):
1815 """Could we not use some kind of wsgi wrapper to make this output?"""
1816 for header in response_headers:
1817 print "%s: %s" % header
1818 print
1819 print output
1820
1821
1822def fill_cache():
1823 ''' Poor man re-loading of few cache items (the slow ones) '''
1824 for host in get_hostlist():
1825 get_yaml(host)
1826
1827
1828def reload_cache():
1829 clear_cache()
1830 fill_cache()
1831
1832
1833def main():
1834 """Hard working sub"""
1835 # Allow easy hacking using the CLI
1836 if not os.environ.has_key('REQUEST_URI'):
1837 if len(sys.argv) < 2:
1838 usage()
1839
1840 if sys.argv[1] == "standalone":
1841 import SocketServer
1842 import CGIHTTPServer
1843 # Hop to the right working directory.
1844 os.chdir(os.path.dirname(__file__))
1845 try:
1846 PORT = int(sys.argv[2])
1847 except (IndexError,ValueError):
1848 PORT = 8000
1849
1850 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1851 """ Serve this CGI from the root of the webserver """
1852 def is_cgi(self):
1853 if "favicon" in self.path:
1854 return False
1855
1856 self.cgi_info = (os.path.basename(__file__), self.path)
1857 self.path = ''
1858 return True
1859 handler = MyCGIHTTPRequestHandler
1860 SocketServer.TCPServer.allow_reuse_address = True
1861 httpd = SocketServer.TCPServer(("", PORT), handler)
1862 httpd.server_name = 'localhost'
1863 httpd.server_port = PORT
1864
1865 logger.info("serving at port %s", PORT)
1866 try:
1867 httpd.serve_forever()
1868 except KeyboardInterrupt:
1869 httpd.shutdown()
1870 logger.info("All done goodbye")
1871 elif sys.argv[1] == "test":
1872 # Basic argument validation
1873 try:
1874 node = sys.argv[2]
1875 except IndexError:
1876 print "Invalid argument"
1877 exit(1)
1878 except IOError as e:
1879 print e
1880 exit(1)
1881
1882 datadump = get_yaml(node)
1883
1884
1885 # Get files to generate
1886 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1887
1888 # Actual config generation
1889 for config in gen_files:
1890 logger.info("## Generating %s %s", node, config)
1891 print generate_config(node, config, datadump)
1892 elif sys.argv[1] == "test-cgi":
1893 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1894 os.environ['SCRIPT_NAME'] = __file__
1895 response_headers, output = process_cgi_request()
1896 print_cgi_response(response_headers, output)
1897 elif sys.argv[1] == "static":
1898 generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
1899 elif sys.argv[1] == "wind-export":
1900 items = dict()
1901 for node in get_hostlist():
1902 datadump = get_yaml(node)
1903 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1904 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1905 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1906 VALUES (
1907 (SELECT id FROM users WHERE username = 'rvdzwet'),
1908 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1909 'Y');""" % datadump
1910 #for config in files:
1911 # items['config'] = config
1912 # print "## Generating %(node)s %(config)s" % items
1913 # f = open("%(wdir)s/%(config)s" % items, "w")
1914 # f.write(generate_config(node, config, datadump))
1915 # f.close()
1916 for node in get_hostlist():
1917 datadump = get_yaml(node)
1918 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1919 ifacedump = datadump[iface_key]
1920 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1921 ifacedump['nodename'] = datadump['nodename']
1922 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1923 ifacedump['channel'] = 0
1924 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1925 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1926 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1927 elif sys.argv[1] == "nagios-export":
1928 try:
1929 heavy_load = (sys.argv[2] == "--heavy-load")
1930 except IndexError:
1931 heavy_load = False
1932
1933 hostgroup_details = {
1934 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1935 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1936 'walphen' : 'Stichting Wireless Alphen',
1937 'westeinder' : 'Westeinder Plassen',
1938 }
1939
1940 # Convert IP to Host
1941 ip2host = {'root' : 'root'}
1942 for host in get_hostlist():
1943 datadump = get_yaml(host)
1944 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1945 for iface in get_interface_keys(datadump):
1946 if datadump[iface].has_key('autogen_gateway'):
1947 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1948
1949 # Find dependency tree based on output of lvrouted.mytree of nearest node
1950 parents = defaultdict(list)
1951 stack = ['root']
1952 prev_depth = 0
1953 for line in open('lvrouted.mytree').readlines():
1954 depth = line.count('\t')
1955 ip = line.strip().split()[0]
1956
1957 if prev_depth < depth:
1958 try:
1959 parents[ip2host[ip]].append(ip2host[stack[-1]])
1960 except KeyError as e:
1961 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1962 stack.append(ip)
1963 elif prev_depth > depth:
1964 stack = stack[:(depth - prev_depth)]
1965 elif prev_depth == depth:
1966 try:
1967 parents[ip2host[ip]].append(ip2host[stack[-1]])
1968 except KeyError as e:
1969 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1970
1971
1972 prev_depth = depth
1973 # Observe that some nodes has themself as parent or multiple parents
1974 # for now take only the first parent, other behaviour is yet to be explained
1975
1976
1977
1978 params = {
1979 'check_interval' : 5 if heavy_load else 120,
1980 'retry_interval' : 1 if heavy_load else 10,
1981 'max_check_attempts' : 10 if heavy_load else 6,
1982 'notification_interval': 120 if heavy_load else 240,
1983 }
1984
1985 print '''\
1986define host {
1987 name wleiden-node ; Default Node Template
1988 use generic-host ; Use the standard template as initial starting point
1989 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1990 check_interval %(check_interval)s ; Actively check the host every 5 minutes
1991 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1992 notification_interval %(notification_interval)s
1993 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1994 check_command check-host-alive ; Default command to check FreeBSD hosts
1995 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1996}
1997
1998define service {
1999 name wleiden-service ; Default Service Template
2000 use generic-service ; Use the standard template as initial starting point
2001 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2002 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2003 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2004 notification_interval %(notification_interval)s
2005 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2006 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2007}
2008
2009# Please make sure to install:
2010# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2011#
2012# Recompile net-mgmt/nagios-plugins to support check_snmp
2013# make -C /usr/ports/net-mgmt/nagios-plugins
2014#
2015# Install net/bind-tools to allow v2/check_dns_wl to work:
2016# pkg install bind-tools
2017#
2018define command{
2019 command_name check_snmp_disk
2020 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2021}
2022
2023define command{
2024 command_name check_netsnmp_load
2025 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2026}
2027
2028define command{
2029 command_name check_netsnmp_proc
2030 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2031}
2032
2033define command{
2034 command_name check_by_ssh
2035 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2036}
2037
2038define command{
2039 command_name check_dns_wl
2040 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2041}
2042
2043define command{
2044 command_name check_snmp_uptime
2045 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2046}
2047
2048
2049# TDB: dhcp leases
2050# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2051
2052# TDB: internet status
2053# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2054
2055# TDB: Advanced local passive checks
2056# /usr/local/libexec/nagios/check_by_ssh
2057''' % params
2058
2059 print '''\
2060# Service Group, not displayed by default
2061define hostgroup {
2062 hostgroup_name srv_hybrid
2063 alias All Hybrid Nodes
2064 register 0
2065}
2066
2067define service {
2068 use wleiden-service
2069 hostgroup_name srv_hybrid
2070 service_description SSH
2071 check_command check_ssh
2072}
2073
2074define service {
2075 use wleiden-service,service-pnp
2076 hostgroup_name srv_hybrid
2077 service_description HTTP
2078 check_command check_http
2079}
2080
2081define service {
2082 use wleiden-service
2083 hostgroup_name srv_hybrid
2084 service_description DNS
2085 check_command check_dns_wl!"www.wirelessleiden.nl"
2086}
2087'''
2088
2089 if heavy_load:
2090 print '''\
2091define service {
2092 use wleiden-service
2093 hostgroup_name srv_hybrid
2094 service_description UPTIME
2095 check_command check_snmp_uptime
2096}
2097
2098#define service {
2099# use wleiden-service
2100# hostgroup_name srv_hybrid
2101# service_description NTP
2102# check_command check_ntp_peer
2103#}
2104
2105define service {
2106 use wleiden-service
2107 hostgroup_name srv_hybrid
2108 service_description LOAD
2109 check_command check_netsnmp_load
2110}
2111
2112define service {
2113 use wleiden-service
2114 hostgroup_name srv_hybrid
2115 service_description PROC
2116 check_command check_netsnmp_proc
2117}
2118
2119define service {
2120 use wleiden-service
2121 hostgroup_name srv_hybrid
2122 service_description DISK
2123 check_command check_snmp_disk
2124}
2125'''
2126 for node in get_hostlist():
2127 datadump = get_yaml(node)
2128 if not datadump['status'] == 'up':
2129 continue
2130 if not hostgroup_details.has_key(datadump['monitoring_group']):
2131 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2132 print '''\
2133define host {
2134 use wleiden-node,host-pnp
2135 contact_groups admins
2136 host_name %(autogen_fqdn)s
2137 address %(masterip)s
2138 hostgroups srv_hybrid,%(monitoring_group)s\
2139''' % datadump
2140 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2141 print '''\
2142 parents %(parents)s\
2143''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2144 print '''\
2145}
2146'''
2147
2148
2149 for name,alias in hostgroup_details.iteritems():
2150 print '''\
2151define hostgroup {
2152 hostgroup_name %s
2153 alias %s
2154} ''' % (name, alias)
2155
2156
2157 elif sys.argv[1] == "full-export":
2158 hosts = {}
2159 for node in get_hostlist():
2160 datadump = get_yaml(node)
2161 hosts[datadump['nodename']] = datadump
2162 print yaml.dump(hosts)
2163
2164 elif sys.argv[1] == "dns":
2165 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2166 elif sys.argv[1] == "cleanup":
2167 # First generate all datadumps
2168 datadumps = dict()
2169 ssid_to_node = dict()
2170 for host in get_hostlist():
2171 logger.info("# Processing: %s", host)
2172 # Set some boring default values
2173 datadump = { 'board' : 'UNKNOWN' }
2174 datadump.update(get_yaml(host))
2175 datadumps[datadump['nodename']] = datadump
2176
2177 (poel, errors) = make_relations()
2178 print "\n".join(["# WARNING: %s" % x for x in errors])
2179
2180 for host,datadump in datadumps.iteritems():
2181 try:
2182 # Convert all yes and no to boolean values
2183 def fix_boolean(dump):
2184 for key in dump.keys():
2185 if type(dump[key]) == dict:
2186 dump[key] = fix_boolean(dump[key])
2187 elif str(dump[key]).lower() in ["yes", "true"]:
2188 dump[key] = True
2189 elif str(dump[key]).lower() in ["no", "false"]:
2190 # Compass richting no (Noord Oost) is valid input
2191 if key != "compass": dump[key] = False
2192 return dump
2193 datadump = fix_boolean(datadump)
2194
2195 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2196 if not 'latitude' in datadump and not 'longitude' in datadump:
2197 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2198 elif 'latitude' in datadump and 'longitude' in datadump:
2199 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2200 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2201 # TODO: Compare outcome of both coordinate systems and validate against each-other
2202
2203 if datadump['nodename'].startswith('Proxy'):
2204 datadump['nodename'] = datadump['nodename'].lower()
2205
2206 for iface_key in get_interface_keys(datadump):
2207 try:
2208 # All our normal wireless cards are normal APs now
2209 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2210 datadump[iface_key]['mode'] = 'ap'
2211 # Wireless Leiden SSID have an consistent lowercase/uppercase
2212 if datadump[iface_key].has_key('ssid'):
2213 ssid = datadump[iface_key]['ssid']
2214 prefix = 'ap-WirelessLeiden-'
2215 if ssid.lower().startswith(prefix.lower()):
2216 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2217 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2218 datadump[iface_key]['mode'] = 'autogen-FIXME'
2219 if not datadump[iface_key].has_key('comment'):
2220 datadump[iface_key]['comment'] = 'autogen-FIXME'
2221
2222 if datadump[iface_key].has_key('ns_mac'):
2223 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2224
2225 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2226 datadump[iface_key] = datadump[iface_key]['desc']
2227
2228 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2229 # due to better collision avoidance.
2230 if datadump[iface_key]['type'] == '11b':
2231 datadump[iface_key]['type'] = '11g'
2232
2233 # Setting 802.11g channels to de-facto standards, to avoid
2234 # un-detected sharing with other overlapping channels
2235 #
2236 # Technically we could also use channel 13 in NL, but this is not
2237 # recommended as foreign devices might not be able to select this
2238 # channel. Secondly using 1,5,9,13 instead is going to clash with
2239 # the de-facto usage of 1,6,11.
2240 #
2241 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2242 channels_at_2400Mhz = (1,6,11)
2243 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2244 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2245 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2246 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2247
2248 # Mandatory interface keys
2249 if not datadump[iface_key].has_key('status'):
2250 datadump[iface_key]['status'] = 'planned'
2251
2252 x = datadump[iface_key]['comment']
2253 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2254
2255 # Fixing bridge_type if none is found
2256 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2257 if not 'bridge_type' in datadump[iface_key]:
2258 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2259
2260 # Making sure description works
2261 if datadump[iface_key].has_key('desc'):
2262 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2263 del datadump[iface_key]['desc']
2264 else:
2265 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2266 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2267 if response:
2268 datadump[iface_key]['comment'] = response
2269 del datadump[iface_key]['desc']
2270
2271 # Check DHCP configuration
2272 dhcp_type(datadump[iface_key])
2273
2274 # Set the compass value based on the angle between the poels
2275 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2276 my_pool = poel[network(datadump[iface_key]['ip'])]
2277 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2278 if remote_hosts:
2279 compass_target = remote_hosts[0]
2280 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2281 # TODO: Compass wanted and actual direction might differ
2282
2283 # Monitoring Group default
2284 if not 'monitoring_group' in datadump:
2285 datadump['monitoring_group'] = 'wleiden'
2286
2287 except Exception:
2288 print "# Error while processing interface %s" % iface_key
2289 raise
2290 store_yaml(datadump)
2291 except Exception:
2292 print "# Error while processing %s" % host
2293 raise
2294 elif sys.argv[1] == "list":
2295 use_fqdn = False
2296 if len(sys.argv) < 4:
2297 usage()
2298 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2299 usage()
2300 if not sys.argv[3] in ["nodes","proxies","systems"]:
2301 usage()
2302
2303 if len(sys.argv) > 4:
2304 if sys.argv[4] == "fqdn":
2305 use_fqdn = True
2306 else:
2307 usage()
2308
2309 for system in get_hostlist():
2310 datadump = get_yaml(system)
2311 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2312 continue
2313
2314 output = datadump['autogen_fqdn'] if use_fqdn else system
2315 if sys.argv[2] == "all":
2316 print output
2317 elif datadump['status'] == sys.argv[2]:
2318 print output
2319 elif sys.argv[1] == "create":
2320 if sys.argv[2] == "network.kml":
2321 print make_network_kml.make_graph()
2322 elif sys.argv[2] == "host-ips.txt":
2323 for system in get_hostlist():
2324 datadump = get_yaml(system)
2325 ips = [datadump['masterip']]
2326 for ifkey in get_interface_keys(datadump):
2327 ips.append(datadump[ifkey]['ip'].split('/')[0])
2328 print system, ' '.join(ips)
2329 elif sys.argv[2] == "host-pos.txt":
2330 for system in get_hostlist():
2331 datadump = get_yaml(system)
2332 print system, datadump['rdnap_x'], datadump['rdnap_y']
2333 elif sys.argv[2] == 'ssh_config':
2334 print '''
2335Host *.wleiden.net
2336 User root
2337
2338Host 172.16.*.*
2339 User root
2340'''
2341 for system in get_hostlist():
2342 datadump = get_yaml(system)
2343 print '''\
2344Host %s
2345 User root
2346
2347Host %s
2348 User root
2349
2350Host %s
2351 User root
2352
2353Host %s
2354 User root
2355''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2356 else:
2357 usage()
2358 else:
2359 usage()
2360 else:
2361 # Do not enable debugging for config requests as it highly clutters the output
2362 if not is_text_request():
2363 cgitb.enable()
2364 response_headers, output = process_cgi_request()
2365 print_cgi_response(response_headers, output)
2366
2367def application(environ, start_response):
2368 status = '200 OK'
2369 response_headers, output = process_cgi_request(environ)
2370 start_response(status, response_headers)
2371
2372 # Debugging only
2373 # output = 'wsgi.multithread = %s' % repr(environ['wsgi.multithread'])
2374 # soutput += '\nwsgi.multiprocess = %s' % repr(environ['wsgi.multiprocess'])
2375 return [output]
2376
2377if __name__ == "__main__":
2378 main()
Note: See TracBrowser for help on using the repository browser.