source: genesis/tools/gformat.py@ 14341

Last change on this file since 14341 was 14335, checked in by rick, 6 years ago

Add helper PoC to help merge nodeplanner data

How to use:

  • Export new nodeplanner data:

$ ./gformat.py create nodeplanner.json > nodeplanner-genesis.json

  • Export nodeplanner website data:

$ curl -s ​https://maps.wirelessleiden.nl/nodeplanner/api/get/state > nodeplanner-usermod.json

  • Merge files:

$ ./merge-nodeplanner-json.py nodeplanner-genesis.json nodeplanner-usermod.json > nodeplanner-new.json

  • Use importer function to upload data:

​https://maps.wirelessleiden.nl/nodeplanner/

  • Check result
  • Use save function to save data:

​https://maps.wirelessleiden.nl/nodeplanner/
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 81.2 KB
Line 
1#!/usr/local/bin/python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
35SVNVERSION = filter(os.path.isfile, ('/usr/local/bin/svnversion', '/usr/bin/svnversion'))[0]
36
37import argparse
38import cgi
39import cgitb
40import copy
41import glob
42import make_network_kml
43import math
44import pyproj
45import random
46import re
47import socket
48import string
49import subprocess
50import textwrap
51import time
52import traceback
53import urlparse
54
55from pprint import pprint
56from collections import defaultdict, OrderedDict
57from sys import stderr
58try:
59 import yaml
60except ImportError, e:
61 print e
62 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
63 exit(1)
64
65try:
66 from yaml import CLoader as Loader
67 from yaml import CDumper as Dumper
68except ImportError:
69 from yaml import Loader, Dumper
70
71from jinja2 import Environment, Template
72def yesorno(value):
73 return "YES" if bool(value) else "NO"
74env = Environment()
75env.filters['yesorno'] = yesorno
76def render_template(datadump, template):
77 result = env.from_string(template).render(datadump)
78 # Make it look pretty to the naked eye, as jinja templates are not so
79 # friendly when it comes to whitespace formatting
80 ## Remove extra whitespace at end of line lstrip() style.
81 result = re.sub(r'\n[\ ]+','\n', result)
82 ## Include only a single newline between an definition and a comment
83 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
84 ## Remove extra newlines after single comment
85 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
86 return result
87
88import logging
89logging.basicConfig(format='# %(levelname)s: %(message)s' )
90logger = logging.getLogger()
91logger.setLevel(logging.DEBUG)
92
93
94if os.environ.has_key('CONFIGROOT'):
95 NODE_DIR = os.environ['CONFIGROOT']
96else:
97 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
98__version__ = '$Id: gformat.py 14335 2019-02-20 22:23:13Z rick $'
99
100CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
101
102files = [
103 'authorized_keys',
104 'dnsmasq.conf',
105 'dhcpd.conf',
106 'rc.conf.local',
107 'resolv.conf',
108 'motd',
109 'ntp.conf',
110 'pf.hybrid.conf.local',
111 'unbound.wleiden.conf',
112 'wleiden.yaml',
113 ]
114
115# Global variables uses
116OK = 10
117DOWN = 20
118UNKNOWN = 90
119
120
121ileiden_proxies = OrderedDict()
122normal_proxies = []
123datadump_cache = {}
124interface_list_cache = {}
125rc_conf_local_cache = {}
126nameservers_cache = []
127relations_cache = None
128
129NO_DHCP = 0
130DHCP_CLIENT = 10
131DHCP_SERVER = 20
132def dhcp_type(item):
133 if not item.has_key('dhcp'):
134 return NO_DHCP
135 elif not item['dhcp']:
136 return NO_DHCP
137 elif item['dhcp'].lower() == 'client':
138 return DHCP_CLIENT
139 else:
140 # Validation Checks
141 begin,end = map(int,item['dhcp'].split('-'))
142 if begin >= end:
143 raise ValueError("DHCP Start >= DHCP End")
144 return DHCP_SERVER
145
146def etrs2rd(lat, lon):
147 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
148 p2 = pyproj.Proj(init='EPSG:28992')
149 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
150 return (RDx, RDy)
151
152def rd2etrs(RDx, RDy):
153 p1 = pyproj.Proj(init='EPSG:28992')
154 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
155 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
156 return (lat, lon)
157
158def get_yaml(item,add_version_info=True):
159 try:
160 """ Get configuration yaml for 'item'"""
161 if datadump_cache.has_key(item):
162 return datadump_cache[item].copy()
163
164 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
165 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
166 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
167
168 # Default values
169 datadump = {
170 'autogen_revision' : 'NOTFOUND',
171 'autogen_gfile' : gfile,
172 'service_proxy_ileiden' : False,
173 'publicnat' : ['http', 'https'],
174 }
175 f = open(gfile, 'r')
176 datadump.update(yaml.load(f,Loader=Loader))
177 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
178 if datadump['nodetype'] == 'Hybrid':
179 # Some values are defined implicitly
180 if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
181 datadump['service_incoming_rdr'] = True
182 # Use some boring defaults
183 defaults = {
184 'service_proxy_normal' : False,
185 'service_accesspoint' : True,
186 'service_incoming_rdr' : False,
187 'service_concentrator' : False,
188 'monitoring_group' : 'wleiden',
189 }
190 for (key,value) in defaults.iteritems():
191 if not datadump.has_key(key):
192 datadump[key] = value
193 f.close()
194
195 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
196 if add_version_info:
197 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
198 lines = p.communicate()[0].split('\n')
199 if p.returncode == 0:
200 for line in lines:
201 if line:
202 (key, value) = line.split(': ')
203 datadump["autogen_" + key.lower().replace(' ','_')] = value
204
205 # Preformat certain needed variables for formatting and push those into special object
206 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
207
208 wlan_count=0
209 try:
210 for key in get_interface_keys(datadump, True):
211 datadump[key]['autogen_ifbase'] = key.split('_')[1]
212 datadump[key]['autogen_vlan'] = False
213 datadump[key]['autogen_vlan_alias'] = False
214
215 datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
216 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
217 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
218
219 if datadump[key].has_key('parent'):
220 if datadump[key].has_key('ip'):
221 raise ValueError("Interface bridge member cannot have IP assigned")
222 if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
223 raise ValueError("Interface bridge member cannot have DHCP set")
224
225 if datadump[key].has_key('ip'):
226 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
227
228 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
229 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
230 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
231 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
232 wlan_count += 1
233 else:
234 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
235 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
236 datadump[key]['autogen_if_dhcp'] = '.'.join(key.split('_')[1:3])
237 datadump[key]['autogen_vlan'] = key.split('_')[2]
238 datadump[key]['autogen_vlan_alias'] = '_alias' in key
239 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
240 else:
241 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
242 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
243
244 except Exception as exc:
245 exc.args = ("# Error while processing interface %s" % key,) + exc.args
246 raise
247
248 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
249 if dhcp_type(datadump[key]) == DHCP_SERVER]
250
251 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
252 datadump['autogen_item'] = item
253
254 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
255 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
256 datadump_cache[item] = datadump.copy()
257 except Exception as exc:
258 exc.args = ("# Error while processing %s" % item,) + exc.args
259 raise
260 return datadump
261
262
263def store_yaml(datadump, header=False):
264 """ Store configuration yaml for 'item'"""
265 item = datadump['autogen_item']
266 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
267
268 output = generate_wleiden_yaml(datadump, header)
269
270 f = open(gfile, 'w')
271 f.write(output)
272 f.close()
273
274
275def network(ip):
276 addr, mask = ip.split('/')
277 # Not parsing of these folks please
278 addr = parseaddr(addr)
279 mask = int(mask)
280 network = addr & ~((1 << (32 - mask)) - 1)
281 return network
282
283
284
285def make_relations():
286 """ Process _ALL_ yaml files to get connection relations """
287 global relations_cache
288
289 if relations_cache:
290 return relations_cache
291
292 errors = []
293 poel = defaultdict(list)
294
295 for host in get_hostlist():
296 datadump = get_yaml(host)
297 try:
298 for iface_key in get_interface_keys(datadump):
299 # Bridge members has no IP assigned
300 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
301 continue
302 net_addr = network(datadump[iface_key]['ip'])
303 poel[net_addr] += [(host,datadump[iface_key].copy())]
304 except (KeyError, ValueError) as e:
305 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
306 continue
307
308 relations_cache = (poel, errors)
309 return relations_cache
310
311
312
313def valid_addr(addr):
314 """ Show which address is valid in which are not """
315 return str(addr).startswith('172.')
316
317def get_hostlist():
318 """ Combined hosts and proxy list"""
319 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
320
321def angle_between_points(lat1,lat2,long1,long2):
322 """
323 Return Angle in radians between two GPS coordinates
324 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
325 """
326 dy = lat2 - lat1
327 dx = math.cos(lat1)*(long2 - long1)
328 angle = math.atan2(dy,dx)
329 return angle
330
331
332
333def angle_to_cd(angle):
334 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
335
336 # For easy conversion get positive degree
337 degrees = math.degrees(angle)
338 abs_degrees = 360 + degrees if degrees < 0 else degrees
339
340 # Numbers can be confusing calculate from the 4 main directions
341 p = 22.5
342 if abs_degrees < p:
343 cd = "n"
344 elif abs_degrees < (90 - p):
345 cd = "no"
346 elif abs_degrees < (90 + p):
347 cd = "o"
348 elif abs_degrees < (180 - p):
349 cd = "zo"
350 elif abs_degrees < (180 + p):
351 cd = "z"
352 elif abs_degrees < (270 - p):
353 cd = "zw"
354 elif abs_degrees < (270 + p):
355 cd = "w"
356 elif abs_degrees < (360 - p):
357 cd = "nw"
358 else:
359 cd = "n"
360 return cd
361
362
363
364def cd_between_hosts(hostA, hostB, datadumps):
365 # Using RDNAP coordinates
366 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
367 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
368 return angle_to_cd(math.atan2(dx,dy))
369
370 # GPS coordinates seems to fail somehow
371 #latA = float(datadumps[hostA]['latitude'])
372 #latB = float(datadumps[hostB]['latitude'])
373 #lonA = float(datadumps[hostA]['longitude'])
374 #lonB = float(datadumps[hostB]['longitude'])
375 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
376
377
378def generate_title(nodelist):
379 """ Main overview page """
380 items = { \
381 'root' : ".",
382 'version' : subprocess.Popen([SVNVERSION, "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
383 }
384 def fl(spaces, line):
385 return (' ' * spaces) + line + '\n'
386
387 output = """
388<html>
389 <head>
390 <title>Wireless leiden Configurator - GFormat</title>
391 <style type="text/css">
392 th {background-color: #999999}
393 tr:nth-child(odd) {background-color: #cccccc}
394 tr:nth-child(even) {background-color: #ffffff}
395 th, td {padding: 0.1em 1em}
396 </style>
397 </head>
398 <body>
399 <center>
400 <form type="GET" action="%(root)s">
401 <input type="hidden" name="action" value="update">
402 <input type="submit" value="Update Configuration Database (SVN)">
403 </form>
404 <table>
405 <caption><h3>Wireless Leiden Configurator - Revision %(version)s</h3></caption>
406 """ % items
407
408 for node in nodelist:
409 items['node'] = node
410 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
411 for config in files:
412 items['config'] = config
413 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
414 output += fl(5, "</tr>")
415 output += """
416 </table>
417 <hr />
418 <em>%s</em>
419 </center>
420 </body>
421</html>
422 """ % __version__
423
424 return output
425
426
427
428def generate_node(node):
429 """ Print overview of all files available for node """
430 return "\n".join(files)
431
432def generate_node_overview(host, datadump=False):
433 """ Print overview of all files available for node """
434 if not datadump:
435 datadump = get_yaml(host)
436 params = { 'host' : host }
437 output = "<em><a href='..'>Back to overview</a></em><hr />"
438 output += "<h2>Available files:</h2><ul>"
439 for cf in files:
440 params['cf'] = cf
441 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
442 output += "</ul>"
443
444 # Generate and connection listing
445 output += "<h2>Connected To:</h2><ul>"
446 (poel, errors) = make_relations()
447 for network, hosts in poel.iteritems():
448 if host in [x[0] for x in hosts]:
449 if len(hosts) == 1:
450 # Single not connected interface
451 continue
452 for remote,ifacedump in hosts:
453 if remote == host:
454 # This side of the interface
455 continue
456 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
457 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
458 output += "</ul>"
459 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
460
461 output += "<hr /><em><a href='..'>Back to overview</a></em>"
462 return output
463
464
465def generate_header(datadump, ctag="#"):
466 return """\
467%(ctag)s
468%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
469%(ctag)s
470""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
471
472
473
474def parseaddr(s):
475 """ Process IPv4 CIDR notation addr to a (binary) number """
476 f = s.split('.')
477 return (long(f[0]) << 24L) + \
478 (long(f[1]) << 16L) + \
479 (long(f[2]) << 8L) + \
480 long(f[3])
481
482
483
484def showaddr(a):
485 """ Display IPv4 addr in (dotted) CIDR notation """
486 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
487
488
489def is_member(ip, mask, canidate):
490 """ Return True if canidate is part of ip/mask block"""
491 ip_addr = parseaddr(ip)
492 ip_canidate = parseaddr(canidate)
493 mask = int(mask)
494 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
495 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
496 return ip_addr == ip_canidate
497
498
499
500def cidr2netmask(netmask):
501 """ Given a 'netmask' return corresponding CIDR """
502 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
503
504def get_network(addr, mask):
505 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
506
507
508def generate_dhcpd_conf(datadump):
509 """ Generate config file '/usr/local/etc/dhcpd.conf """
510 # Redundency support, in cause local DNS server is not running/responding.
511 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
512 output = generate_header(datadump)
513 output += Template("""\
514# option definitions common to all supported networks...
515option domain-name "dhcp.{{ autogen_fqdn }}";
516
517default-lease-time 600;
518max-lease-time 7200;
519
520# Use this to enble / disable dynamic dns updates globally.
521#ddns-update-style none;
522
523# If this DHCP server is the official DHCP server for the local
524# network, the authoritative directive should be uncommented.
525authoritative;
526
527# Use this to send dhcp log messages to a different log file (you also
528# have to hack syslog.conf to complete the redirection).
529log-facility local7;
530
531# UniFi Discovery Support
532option space ubnt;
533option ubnt.unifi-address code 1 = ip-address;
534#
535class "ubnt" {
536 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
537 option vendor-class-identifier "ubnt";
538 vendor-option-space ubnt;
539}
540
541#
542# Interface definitions
543#
544\n\n""").render(datadump)
545
546
547 # TODO: Use textwrap.fill instead
548 def indent(text, count):
549 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
550
551 # Process DHCP blocks
552 dhcp_out = defaultdict(list)
553 for iface_key in get_interface_keys(datadump):
554 ifname = datadump[iface_key]['autogen_ifbase']
555 groupif = datadump[iface_key]['autogen_if_dhcp']
556 if not datadump[iface_key].has_key('comment'):
557 datadump[iface_key]['comment'] = None
558
559 if not datadump[iface_key].has_key('ip'):
560 continue
561
562 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
563
564 (addr, mask) = datadump[iface_key]['ip'].split('/')
565 datadump[iface_key]['autogen_addr'] = addr
566 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
567 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
568
569 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
570 dhcp_out[groupif].append(textwrap.dedent("""\
571 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
572 ### not autoritive
573 }
574 """ % datadump[iface_key]))
575 continue
576
577 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
578 dhcp_part = ".".join(addr.split('.')[0:3])
579 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
580 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
581 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
582
583 # Assume the first 10 IPs could be used for static entries
584 if 'no_portal' in datadump:
585 fixed = 5
586 for mac in datadump['no_portal']:
587 dhcp_out[groupif].append(textwrap.dedent("""\
588 host fixed-%(ifname)s-%(fixed)s {
589 hardware ethernet %(mac)s;
590 fixed-address %(prefix)s.%(fixed)s;
591 }
592 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
593 fixed += 1
594
595 if 'dhcp_fixed' in datadump[iface_key]:
596 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
597 dhcp_out[groupif].append(textwrap.dedent("""\
598 host fixed-%(host)s {
599 hardware ethernet %(mac)s;
600 fixed-address %(addr)s;
601 }
602 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
603
604
605 dhcp_out[groupif].append(textwrap.dedent("""\
606 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
607 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
608 option routers %(autogen_addr)s;
609 option domain-name-servers %(autogen_dns_servers)s;
610 option ubnt.unifi-address 172.17.107.10;
611 }
612 """ % datadump[iface_key]))
613
614 # Output the blocks in groups
615 for ifname,value in sorted(dhcp_out.iteritems()):
616 output += ("shared-network \"%s\" {\n" % ifname) + indent(''.join(value), 2).rstrip() + '\n}\n\n'
617 return output
618
619
620
621def generate_dnsmasq_conf(datadump):
622 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
623 output = generate_header(datadump)
624 output += Template("""\
625# DHCP server options
626dhcp-authoritative
627dhcp-fqdn
628domain=dhcp.{{ autogen_fqdn }}
629domain-needed
630expand-hosts
631log-async=100
632
633# Low memory footprint
634cache-size=10000
635
636\n""").render(datadump)
637
638 for iface_key in get_interface_keys(datadump):
639 if not datadump[iface_key].has_key('comment'):
640 datadump[iface_key]['comment'] = None
641 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
642
643 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
644 output += "# not autoritive\n\n"
645 continue
646
647 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
648 (ip, cidr) = datadump[iface_key]['ip'].split('/')
649 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
650
651 dhcp_part = ".".join(ip.split('.')[0:3])
652 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
653 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
654 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
655
656 return output
657
658
659class AutoVivification(dict):
660 """Implementation of perl's autovivification feature."""
661 def __getitem__(self, item):
662 try:
663 return dict.__getitem__(self, item)
664 except KeyError:
665 value = self[item] = type(self)()
666 return value
667
668def make_interface_list(datadump):
669 if interface_list_cache.has_key(datadump['autogen_item']):
670 return (interface_list_cache[datadump['autogen_item']])
671 # lo0 configuration:
672 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
673 # - masterip is special as it needs to be assigned to at
674 # least one interface, so if not used assign to lo0
675 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
676 vlan_list = defaultdict(list)
677 bridge_list = defaultdict(list)
678 flags_if = AutoVivification()
679 dhclient_if = {'lo0' : False}
680
681 # XXX: Find some way of send this output nicely
682 output = ''
683
684 masterip_used = False
685 for iface_key in get_interface_keys(datadump):
686 if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
687 masterip_used = True
688 break
689 if not masterip_used:
690 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
691
692 if 'serviceid' in datadump:
693 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
694
695 for iface_key in get_interface_keys(datadump):
696 ifacedump = datadump[iface_key]
697 if ifacedump['autogen_bridge_alias']:
698 ifname = ifacedump['autogen_ifbase']
699 else:
700 ifname = ifacedump['autogen_ifname']
701
702 # If defined as vlan interface
703 if ifacedump['autogen_vlan']:
704 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
705
706 # If defined as bridge interface
707 if ifacedump['autogen_bridge_member']:
708 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
709
710
711 # Flag dhclient is possible
712 if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
713 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
714
715 # Ethernet address
716 if ifacedump.has_key('ether'):
717 flags_if[ifname]['ether'] = ifacedump['ether']
718
719 # Handle special interface states
720 flags_if[ifname]['status'] = ifacedump['status']
721
722 # Add interface IP to list
723 if ifacedump.has_key('ip'):
724 item = (ifacedump['ip'], ifacedump['comment'])
725 if addrs_list.has_key(ifname):
726 addrs_list[ifname].append(item)
727 else:
728 addrs_list[ifname] = [item]
729
730 # Alias only needs IP assignment for now, this might change if we
731 # are going to use virtual accesspoints
732 if "alias" in iface_key:
733 continue
734
735 # XXX: Might want to deduct type directly from interface name
736 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
737 # Default to station (client) mode
738 ifacedump['autogen_wlanmode'] = "sta"
739 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
740 ifacedump['autogen_wlanmode'] = "ap"
741
742 if not ifacedump.has_key('channel'):
743 if ifacedump['type'] == '11a':
744 ifacedump['channel'] = 36
745 else:
746 ifacedump['channel'] = 1
747
748 # Allow special hacks at the back like wds and stuff
749 if not ifacedump.has_key('extra'):
750 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
751 else:
752 ifacedump['autogen_extra'] = ifacedump['extra']
753
754 ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
755
756 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
757 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
758 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
759 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
760 output += "\n"
761
762 elif ifacedump['type'] in ['ethernet', 'eth']:
763 # No special config needed besides IP
764 pass
765 elif ifacedump['type'] in ['vlan']:
766 # VLAN member has no special configuration
767 pass
768 else:
769 assert False, "Unknown type " + ifacedump['type']
770
771 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
772 interface_list_cache[datadump['autogen_item']] = store
773 return(store)
774
775
776
777def create_proxies_list():
778 if not ileiden_proxies or not normal_proxies:
779 # Placeholder for to-be-installed proxies, this will avoid updating the all
780 # nodes to include this new machine, yet due to an unbound issue, this list
781 # has to be kept small.
782
783 for i in range(1,20):
784 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
785
786 for host in get_hostlist():
787 hostdump = get_yaml(host)
788 if hostdump['status'] == 'up':
789 if hostdump['service_proxy_ileiden']:
790 ileiden_proxies[hostdump['serviceid']] = hostdump
791 if hostdump['service_proxy_normal']:
792 normal_proxies.append(hostdump)
793
794
795
796def generate_rc_conf_local(datadump):
797 """ Generate configuration file '/etc/rc.conf.local' """
798 item = datadump['autogen_item']
799 if rc_conf_local_cache.has_key(item):
800 return rc_conf_local_cache[item]
801
802 if not datadump.has_key('ileiden'):
803 datadump['autogen_ileiden_enable'] = False
804 else:
805 datadump['autogen_ileiden_enable'] = datadump['ileiden']
806
807 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
808
809 create_proxies_list()
810 datadump['autogen_ileiden_proxies'] = ileiden_proxies
811 datadump['autogen_normal_proxies'] = normal_proxies
812 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
813 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
814 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
815 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
816
817 output = generate_header(datadump, "#");
818 output += render_template(datadump, """\
819hostname='{{ autogen_fqdn }}'
820location='{{ location }}'
821nodetype="{{ nodetype }}"
822
823#
824# Configured listings
825#
826captive_portal_whitelist=""
827{% if nodetype == "Proxy" %}
828#
829# Proxy Configuration
830#
831{% if gateway and service_proxy_ileiden -%}
832defaultrouter="{{ gateway }}"
833{% else -%}
834#defaultrouter="NOTSET"
835{% endif -%}
836internalif="{{ internalif }}"
837ileiden_enable="{{ autogen_ileiden_enable }}"
838gateway_enable="{{ autogen_ileiden_enable }}"
839pf_enable="yes"
840pf_rules="/etc/pf.conf"
841{% if autogen_ileiden_enable -%}
842pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={{ publicnat|join(',') }} -D ileiden_ports={{ publicnat|join(',') }}"
843lvrouted_enable="{{ autogen_ileiden_enable }}"
844lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
845{% else -%}
846pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
847{% endif -%}
848{% if internalroute -%}
849static_routes="wleiden"
850route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
851{% endif -%}
852
853{% elif nodetype == "Hybrid" %}
854 #
855 # Hybrid Configuration
856 #
857 list_ileiden_proxies="
858 {% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
859 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
860 {% endfor -%}
861 "
862 list_normal_proxies="
863 {% for item in autogen_normal_proxies -%}
864 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
865 {% endfor -%}
866 "
867
868 {% if autogen_dhcp_interfaces -%}
869 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
870 {% else %}
871 captive_portal_interfaces="dummy"
872 {% endif %}
873 externalif="{{ externalif|default('vr0', true) }}"
874 masterip="{{ masterip }}"
875
876 {% if gateway and service_proxy_ileiden %}
877 defaultrouter="{{ gateway }}"
878 {% else %}
879 #defaultrouter="NOTSET"
880 {% endif %}
881
882 #
883 # Defined services
884 #
885 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
886 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
887 service_accesspoint="{{ service_accesspoint|yesorno }}"
888 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
889 service_concentrator="{{ service_concentrator|yesorno }}"
890
891 {% if service_proxy_ileiden %}
892 pf_rules="/etc/pf.hybrid.conf"
893 {% if service_concentrator %}
894 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
895 {% else %}
896 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
897 {% endif %}
898 pf_flags="$pf_flags -D publicnat={{ publicnat|join(',') }}"
899 lvrouted_flags="$lvrouted_flags -g"
900 {% elif service_proxy_normal or service_incoming_rdr %}
901 pf_rules="/etc/pf.hybrid.conf"
902 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
903 pf_flags="$pf_flags -D publicnat=0"
904 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
905 named_setfib="1"
906 tinyproxy_setfib="1"
907 dnsmasq_setfib="1"
908 sshd_setfib="1"
909 {% else %}
910 named_auto_forward_only="YES"
911 pf_rules="/etc/pf.node.conf"
912 pf_flags="-D ileiden_ports={{ publicnat|join(',') }}"
913 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
914 {% endif %}
915 {% if service_concentrator %}
916 # Do mind installing certificates is NOT done automatically for security reasons
917 openvpn_enable="YES"
918 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
919 {% endif %}
920
921 {% if service_proxy_normal %}
922 tinyproxy_enable="yes"
923 {% else %}
924 pen_wrapper_enable="yes"
925 {% endif %}
926
927 {% if service_accesspoint %}
928 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
929 {% endif %}
930
931 {% if board == "ALIX2" or board == "net4801" %}
932 #
933 # ''Fat'' configuration, board has 256MB RAM
934 #
935 dnsmasq_enable="NO"
936 named_enable="YES"
937 unbound_enable="YES"
938 {% if autogen_dhcp_interfaces -%}
939 dhcpd_enable="YES"
940 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
941 {% endif -%}
942 {% elif board == "apu1d" %}
943 #
944 # ''Fat'' configuration, board has 1024MB RAM
945 #
946 dnsmasq_enable="NO"
947 unbound_enable="YES"
948 {% if autogen_dhcp_interfaces -%}
949 dhcpd_enable="YES"
950 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
951 {% endif -%}
952 {% endif -%}
953{% endif %}
954
955#
956# Script variables
957#
958attached_devices="{{ autogen_attached_devices|join(' ') }}"
959neighbours="{{ autogen_neighbours|join(' ') }}"
960
961
962#
963# Interface definitions
964#\n
965""")
966
967 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
968 for iface, vlans in sorted(vlan_list.items()):
969 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
970
971 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
972 for iface in sorted(vlan_list.keys()):
973 if not iface in addrs_list.keys():
974 output += "ifconfig_%s='up'\n" % iface
975
976 output += "\n"
977
978 # Bridge configuration:
979 if bridge_list.keys():
980 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
981
982 for iface in bridge_list.keys():
983 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
984
985 # Bridge member interfaces not containing a configuration should be marked active explcitly.
986 for _,members in bridge_list.items():
987 for iface in members:
988 if not iface in addrs_list.keys():
989 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
990
991 output += "\n"
992
993 # Details like SSID
994 if extra_ouput:
995 output += extra_ouput.strip() + "\n"
996
997 # Print IP address which needs to be assigned over here
998 output += "\n"
999 for iface,addrs in sorted(addrs_list.iteritems()):
1000 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1001 output += "# %s || %s || %s\n" % (iface, addr, comment)
1002
1003 prefix = ''
1004 if flags_if[iface]['status'] == 'broken':
1005 output += "# Interface %s disabled since status=broken\n" % iface
1006 prefix = '#'
1007
1008 # Write DHCLIENT entry
1009 if iface in dhclient_if and dhclient_if[iface]:
1010 output += prefix + "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1011 continue
1012
1013 # Make sure the external address is always first as this is needed in the
1014 # firewall setup
1015 addrs = sorted(
1016 [x for x in addrs if not '0.0.0.0' in x[0]],
1017 key=lambda x: x[0].split('.')[0],
1018 cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1019 )
1020
1021
1022 idx_offset = 0
1023 # Set MAC is required
1024 if flags_if[iface].has_key('ether'):
1025 output += prefix + "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1026 output += prefix + "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1027 idx_offset += 1
1028 elif iface in bridge_list:
1029 output += prefix + "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1030 idx_offset += 1
1031 else:
1032 output += prefix + "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1033
1034 for idx, addr in enumerate(addrs[1:]):
1035 output += prefix + "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1036
1037 output += "\n"
1038
1039 rc_conf_local_cache[datadump['autogen_item']] = output
1040 return output
1041
1042
1043
1044def get_all_configs():
1045 """ Get dict with key 'host' with all configs present """
1046 configs = dict()
1047 for host in get_hostlist():
1048 datadump = get_yaml(host)
1049 configs[host] = datadump
1050 return configs
1051
1052
1053def get_interface_keys(config, extra=False):
1054 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1055 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1056 if extra == False:
1057 return filter(lambda x: not "extra" in x, elems)
1058 else:
1059 return elems
1060
1061
1062def get_used_ips(configs):
1063 """ Return array of all IPs used in config files"""
1064 ip_list = []
1065 for config in configs:
1066 ip_list.append(config['masterip'])
1067 if 'serviceid' in config:
1068 ip_list.append(config['serviceid'])
1069 for iface_key in get_interface_keys(config, True):
1070 l = config[iface_key]['ip']
1071 addr, mask = l.split('/')
1072 # Special case do not process
1073 if valid_addr(addr):
1074 ip_list.append(addr)
1075 else:
1076 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1077 return sorted(ip_list)
1078
1079
1080
1081def get_nameservers(max_servers=None):
1082 if nameservers_cache:
1083 return nameservers_cache[0:max_servers]
1084
1085 for host in get_hostlist():
1086 hostdump = get_yaml(host)
1087 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1088 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1089
1090 return nameservers_cache[0:max_servers]
1091
1092
1093def get_neighbours(datadump):
1094 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1095
1096 (poel, errors) = make_relations()
1097 table = []
1098 for iface,addrs in sorted(addrs_list.iteritems()):
1099 if iface in ['lo0']:
1100 continue
1101
1102 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1103 if not addr.startswith('172.'):
1104 # Avoid listing internet connections as pool
1105 continue
1106 for neighbour in poel[network(addr)]:
1107 if neighbour[0] != datadump['autogen_item']:
1108 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1109 return table
1110
1111
1112def get_attached_devices(datadump, url=False):
1113 table = []
1114 for iface_key in get_interface_keys(datadump, True):
1115 ifacedump = datadump[iface_key]
1116
1117 if not ifacedump.has_key('ns_ip'):
1118 continue
1119
1120 x_ip = ifacedump['ns_ip'].split('/')[0]
1121
1122 if 'mode' in ifacedump:
1123 x_mode = ifacedump['mode']
1124 else:
1125 x_mode = 'unknown'
1126
1127 if 'bridge_type' in ifacedump:
1128 device_type = ifacedump['bridge_type']
1129 else:
1130 device_type = 'Unknown'
1131
1132 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1133 return table
1134
1135
1136def generate_resolv_conf(datadump):
1137 """ Generate configuration file '/etc/resolv.conf' """
1138 # XXX: This should properly going to be an datastructure soon
1139 datadump['autogen_header'] = generate_header(datadump, "#")
1140 datadump['autogen_edge_nameservers'] = ''
1141
1142
1143 for masterip,realname in get_nameservers():
1144 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1145
1146 return Template("""\
1147{{ autogen_header }}
1148search wleiden.net
1149
1150# Try local (cache) first
1151nameserver 127.0.0.1
1152
1153{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1154nameserver 8.8.8.8 # Google Public NameServer
1155nameserver 64.6.64.6 # Verisign Public NameServer
1156{% else -%}
1157# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1158{{ autogen_edge_nameservers }}
1159{% endif -%}
1160""").render(datadump)
1161
1162
1163
1164def generate_ntp_conf(datadump):
1165 """ Generate configuration file '/etc/ntp.conf' """
1166 # XXX: This should properly going to be an datastructure soon
1167
1168 datadump['autogen_header'] = generate_header(datadump, "#")
1169 datadump['autogen_ntp_servers'] = ''
1170 for host in get_hostlist():
1171 hostdump = get_yaml(host)
1172 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1173 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1174
1175 return Template("""\
1176{{ autogen_header }}
1177
1178{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1179# Machine hooked to internet.
1180server 0.nl.pool.ntp.org iburst maxpoll 9
1181server 1.nl.pool.ntp.org iburst maxpoll 9
1182server 2.nl.pool.ntp.org iburst maxpoll 9
1183server 3.nl.pool.ntp.org iburst maxpoll 9
1184{% else -%}
1185# Local Wireless Leiden NTP Servers.
1186server 0.pool.ntp.wleiden.net iburst maxpoll 9
1187server 1.pool.ntp.wleiden.net iburst maxpoll 9
1188server 2.pool.ntp.wleiden.net iburst maxpoll 9
1189server 3.pool.ntp.wleiden.net iburst maxpoll 9
1190
1191# All the configured NTP servers
1192{{ autogen_ntp_servers }}
1193{% endif %}
1194
1195# If a server loses sync with all upstream servers, NTP clients
1196# no longer follow that server. The local clock can be configured
1197# to provide a time source when this happens, but it should usually
1198# be configured on just one server on a network. For more details see
1199# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1200# The use of Orphan Mode may be preferable.
1201#
1202server 127.127.1.0
1203fudge 127.127.1.0 stratum 10
1204""").render(datadump)
1205
1206
1207def generate_pf_hybrid_conf_local(datadump):
1208 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1209 datadump['autogen_header'] = generate_header(datadump, "#")
1210 if datadump['service_incoming_rdr']:
1211 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1212 return Template("""\
1213{{ autogen_header }}
1214
1215# Redirect some internal facing services outside (7)
1216# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1217{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1218rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1219{% endfor -%}
1220# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1221{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1222rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1223{% endfor -%}
1224""").render(datadump)
1225
1226def generate_unbound_wleiden_conf(datadump):
1227 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1228 datadump['autogen_header'] = generate_header(datadump, "#")
1229
1230 autogen_ips = []
1231 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1232 for iface,addrs in sorted(addrs_list.iteritems()):
1233 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1234 if addr.startswith('172'):
1235 autogen_ips.append((addr.split('/')[0], comment))
1236 datadump['autogen_ips'] = autogen_ips
1237
1238 create_proxies_list()
1239 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1240 return Template("""\
1241{{ autogen_header }}
1242
1243server:
1244 ## Static definitions fail to start on systems with broken ue(4) interfaces
1245{%- for ip,comment in autogen_ips %}
1246 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1247{%- endfor %}
1248 ## Enabling wildcard matching as work-around
1249 interface: 0.0.0.0
1250 interface: ::0
1251
1252forward-zone:
1253 name: '.'
1254{%- if service_proxy_ileiden %}
1255 forward-addr: 8.8.8.8 # Google DNS A
1256 forward-addr: 8.8.4.4 # Google DNS B
1257 forward-addr: 208.67.222.222 # OpenDNS DNS A
1258 forward-addr: 208.67.220.220 # OpenDNS DNS B
1259{% else -%}
1260{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1261 {%- if loop.index <= 5 %}
1262 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1263 {%- endif %}
1264{%- endfor %}
1265{%- endif %}
1266""").render(datadump)
1267
1268def generate_motd(datadump):
1269 """ Generate configuration file '/etc/motd' """
1270 output = Template("""\
1271FreeBSD run ``service motd onestart'' to make me look normal
1272
1273 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1274 Loc: {{ location }}
1275
1276Services:
1277{% if board == "ALIX2" or board == "net4801" -%}
1278{{" -"}} Core Node ({{ board }})
1279{% else -%}
1280{{" -"}} Hulp Node ({{ board }})
1281{% endif -%}
1282{% if service_proxy_normal -%}
1283{{" -"}} Normal Proxy
1284{% endif -%}
1285{% if service_proxy_ileiden -%}
1286{{" -"}} iLeiden Proxy
1287{% endif -%}
1288{% if service_incoming_rdr -%}
1289{{" -"}} Incoming port redirects
1290{% endif %}
1291Interlinks:\n
1292""").render(datadump)
1293
1294
1295 def make_table(table):
1296 if not table:
1297 return " - none\n"
1298 else:
1299 lines = ""
1300 col_width = [max(len(x) for x in col) for col in zip(*table)]
1301 for row in table:
1302 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1303 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1304 return lines
1305
1306 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1307 table = []
1308 for iface,addrs in sorted(addrs_list.iteritems()):
1309 if iface in ['lo0']:
1310 continue
1311 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1312 table.append((iface, addr, comment))
1313
1314 output += make_table(table)
1315 output += '\n'
1316 output += """\
1317Attached devices:
1318"""
1319 output += make_table(get_attached_devices(datadump, url=True))
1320 output += '\n'
1321 output += """\
1322Available neighbours:
1323"""
1324 output += make_table(get_neighbours(datadump))
1325
1326 return output
1327
1328
1329def format_yaml_value(value):
1330 """ Get yaml value in right syntax for outputting """
1331 if isinstance(value,str):
1332 output = '"%s"' % value
1333 else:
1334 output = value
1335 return output
1336
1337
1338
1339def format_wleiden_yaml(datadump):
1340 """ Special formatting to ensure it is editable"""
1341 output = "# Genesis config yaml style\n"
1342 output += "# vim:ts=2:et:sw=2:ai\n"
1343 output += "#\n"
1344 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1345 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1346 if key == 'rdr_rules':
1347 output += '%-10s:\n' % 'rdr_rules'
1348 for rdr_rule in datadump[key]:
1349 output += '- %s\n' % rdr_rule
1350 else:
1351 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1352
1353 output += "\n\n"
1354
1355 # Format (key, required)
1356 key_order = (
1357 ('comment', True),
1358 ('parent', False),
1359 ('ip', False),
1360 ('ipv6', False),
1361 ('ether', False),
1362 ('desc', True),
1363 ('sdesc', True),
1364 ('mode', True),
1365 ('type', True),
1366 ('extra_type', False),
1367 ('channel', False),
1368 ('ssid', False),
1369 ('wlan_mac', False),
1370 ('dhcp', True),
1371 ('dhcp_fixed', False),
1372 ('compass', False),
1373 ('distance', False),
1374 ('ns_ip', False),
1375 ('repeater_ip', False),
1376 ('bullet2_ip', False),
1377 ('ns_mac', False),
1378 ('bullet2_mac', False),
1379 ('ns_type', False),
1380 ('bridge_type', False),
1381 ('encrypted', False),
1382 ('status', True),
1383 )
1384
1385 for iface_key in sorted(iface_keys):
1386 try:
1387 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1388 if remainder:
1389 raise KeyError("invalid keys: %s" % remainder)
1390
1391 output += "%s:\n" % iface_key
1392 for key,required in key_order:
1393 if datadump[iface_key].has_key(key):
1394 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1395 output += "\n\n"
1396 except Exception as exc:
1397 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
1398 raise
1399
1400 return output
1401
1402
1403
1404def generate_wleiden_yaml(datadump, header=True):
1405 """ Generate (petty) version of wleiden.yaml"""
1406 output = generate_header(datadump, "#") if header else ''
1407
1408 for key in datadump.keys():
1409 if key.startswith('autogen_'):
1410 del datadump[key]
1411 # Interface autogen cleanups
1412 elif type(datadump[key]) == dict:
1413 for key2 in datadump[key].keys():
1414 if key2.startswith('autogen_'):
1415 del datadump[key][key2]
1416
1417 output += format_wleiden_yaml(datadump)
1418 return output
1419
1420def generate_nanostation_config(datadump, iface, ns_type):
1421 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1422 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1423
1424 datadump.update(datadump['iface_%s' % iface])
1425
1426 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1427
1428def generate_yaml(datadump):
1429 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1430
1431
1432
1433def generate_config(node, config, datadump=None):
1434 """ Print configuration file 'config' of 'node' """
1435 output = ""
1436 try:
1437 # Load config file
1438 if datadump == None:
1439 datadump = get_yaml(node)
1440
1441 if config == 'wleiden.yaml':
1442 output += generate_wleiden_yaml(datadump)
1443 elif config == 'authorized_keys':
1444 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1445 output += f.read()
1446 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1447 # Fetch local keys if existing
1448 if os.path.exists(node_keys):
1449 output += open(node_keys, 'r').read()
1450 f.close()
1451 elif config == 'dnsmasq.conf':
1452 output += generate_dnsmasq_conf(datadump)
1453 elif config == 'dhcpd.conf':
1454 output += generate_dhcpd_conf(datadump)
1455 elif config == 'rc.conf.local':
1456 output += generate_rc_conf_local(datadump)
1457 elif config == 'resolv.conf':
1458 output += generate_resolv_conf(datadump)
1459 elif config == 'ntp.conf':
1460 output += generate_ntp_conf(datadump)
1461 elif config == 'motd':
1462 output += generate_motd(datadump)
1463 elif config == 'pf.hybrid.conf.local':
1464 output += generate_pf_hybrid_conf_local(datadump)
1465 elif config == 'unbound.wleiden.conf':
1466 output += generate_unbound_wleiden_conf(datadump)
1467 elif config.startswith('vr'):
1468 interface, ns_type = config.strip('.yaml').split('-')
1469 output += generate_nanostation_config(datadump, interface, ns_type)
1470 else:
1471 assert False, "Config not found!"
1472 except IOError, e:
1473 output += "[ERROR] Config file not found"
1474 return output
1475
1476
1477
1478def generate_static(output_dir, logging=True):
1479 items = {'output_dir' : output_dir}
1480 for node in get_hostlist():
1481 items['node'] = node
1482 items['wdir'] = "%(output_dir)s/%(node)s" % items
1483 if not os.path.isdir(items['wdir']):
1484 os.makedirs(items['wdir'])
1485 datadump = get_yaml(node)
1486 f = open("%(wdir)s/index.html" % items, "w")
1487 f.write(generate_node_overview(items['node'], datadump))
1488 f.close()
1489 for config in files:
1490 items['config'] = config
1491 if logging: logger.info("## Generating %(node)s %(config)s" % items)
1492 f = open("%(wdir)s/%(config)s" % items, "w")
1493 f.write(generate_config(node, config, datadump))
1494 f.close()
1495
1496
1497
1498def process_cgi_request(environ=os.environ):
1499 """ When calling from CGI """
1500 response_headers = []
1501 content_type = 'text/plain'
1502
1503 # Update repository if requested
1504 form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1505 if form and form.has_key("action") and "update" in form["action"]:
1506 refresh_rate = 5
1507 output = "[INFO] Updating subverion, please wait...\n"
1508 old_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1509 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1510 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1511 new_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1512 if old_version != new_version or (environ.has_key('QUERY_STRING') and 'force' in environ['QUERY_STRING']):
1513 try:
1514 generate_static(CACHE_DIR, False)
1515 except:
1516 output += traceback.format_exc()
1517 refresh_rate = 120
1518 pass
1519 output += "[INFO] All done, redirecting in %s seconds" % refresh_rate
1520 response_headers += [
1521 ('Refresh', '%s; url=.' % refresh_rate),
1522 ]
1523 else:
1524 # Bootstap cache directory if none exists
1525 if not glob.glob(os.path.join(CACHE_DIR,'*','index.html')):
1526 generate_static(CACHE_DIR, False)
1527
1528 # URL must be of format <prefix>/config/<path>
1529 base_uri = environ['REQUEST_URI']
1530 uri = filter(None, base_uri.split('/config/')[1].strip('/').split('/'))
1531
1532 output = "Template Holder"
1533 if base_uri.endswith('/create/network.kml'):
1534 content_type='application/vnd.google-earth.kml+xml'
1535 output = make_network_kml.make_graph()
1536 elif base_uri.endswith('/api/get/nodeplanner.json'):
1537 content_type='application/json'
1538 output = make_network_kml.make_nodeplanner_json()
1539 elif not uri:
1540 if is_text_request(environ):
1541 output = '\n'.join(get_hostlist())
1542 else:
1543 content_type = 'text/html'
1544 output = generate_title(get_hostlist())
1545 elif len(uri) == 1:
1546 if is_text_request(environ):
1547 output = generate_node(uri[0])
1548 else:
1549 content_type = 'text/html'
1550 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1551 elif len(uri) == 2:
1552 output = generate_config(uri[0], uri[1])
1553 else:
1554 assert False, "Invalid option"
1555
1556 # Return response
1557 response_headers += [
1558 ('Content-type', content_type),
1559 ('Content-Length', str(len(output))),
1560 ]
1561 return(response_headers, str(output))
1562
1563
1564def make_dns(output_dir = 'dns', external = False):
1565 items = dict()
1566
1567 # hostname is key, IP is value
1568 wleiden_zone = defaultdict(list)
1569 wleiden_cname = dict()
1570
1571 pool = dict()
1572 for node in get_hostlist():
1573 datadump = get_yaml(node)
1574
1575 fqdn = datadump['nodename']
1576
1577 if datadump.has_key('rdr_host'):
1578 remote_target = datadump['rdr_host']
1579 elif datadump.has_key('remote_access') and datadump['remote_access']:
1580 remote_target = datadump['remote_access'].split(':')[0]
1581 else:
1582 remote_target = None
1583
1584 if remote_target:
1585 try:
1586 parseaddr(remote_target)
1587 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1588 except (IndexError, ValueError):
1589 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1590
1591
1592 wleiden_zone[fqdn].append((datadump['masterip'], True))
1593
1594 # Hacking to get proper DHCP IPs and hostnames
1595 for iface_key in get_interface_keys(datadump):
1596 iface_name = iface_key.replace('_','-')
1597 if 'ip' in datadump[iface_key]:
1598 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1599 try:
1600 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1601 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1602 dhcp_part = ".".join(ip.split('.')[0:3])
1603 if ip != datadump['masterip']:
1604 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1605 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1606 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1607 except (AttributeError, ValueError, KeyError):
1608 # First push it into a pool, to indentify the counter-part later on
1609 addr = parseaddr(ip)
1610 cidr = int(cidr)
1611 addr = addr & ~((1 << (32 - cidr)) - 1)
1612 if pool.has_key(addr):
1613 pool[addr] += [(iface_name, fqdn, ip)]
1614 else:
1615 pool[addr] = [(iface_name, fqdn, ip)]
1616 continue
1617
1618
1619
1620 # WL uses an /29 to configure an interface. IP's are ordered like this:
1621 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1622
1623 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1624
1625 # Automatic naming convention of interlinks namely 2 + remote.lower()
1626 for (key,value) in pool.iteritems():
1627 # Make sure they are sorted from low-ip to high-ip
1628 value = sorted(value, key=lambda x: parseaddr(x[2]))
1629
1630 if len(value) == 1:
1631 (iface_name, fqdn, ip) = value[0]
1632 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1633
1634 # Device DNS names
1635 if 'cnode' in fqdn.lower():
1636 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1637 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1638
1639 elif len(value) == 2:
1640 (a_iface_name, a_fqdn, a_ip) = value[0]
1641 (b_iface_name, b_fqdn, b_ip) = value[1]
1642 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1643 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1644
1645 # Device DNS names
1646 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1647 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1648 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1649 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1650 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1651 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1652 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1653
1654 else:
1655 pool_members = [k[1] for k in value]
1656 for item in value:
1657 (iface_name, fqdn, ip) = item
1658 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1659
1660 # Include static DNS entries
1661 # XXX: Should they override the autogenerated results?
1662 # XXX: Convert input to yaml more useable.
1663 # Format:
1664 ##; this is a comment
1665 ## roomburgh=Roomburgh1
1666 ## apkerk1.Vosko=172.17.176.8 ;this as well
1667 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1668
1669 # Hack to allow special entries, for development
1670 wleiden_raw = {}
1671
1672 for line in dns_list:
1673 reverse = False
1674 k, items = line.items()[0]
1675 if type(items) == dict:
1676 if items.has_key('reverse'):
1677 reverse = items['reverse']
1678 items = items['a']
1679 else:
1680 items = items['cname']
1681 items = [items] if type(items) != list else items
1682 for item in items:
1683 if item.startswith('IN '):
1684 wleiden_raw[k] = item
1685 elif valid_addr(item):
1686 wleiden_zone[k].append((item, reverse))
1687 else:
1688 wleiden_cname[k] = item
1689
1690 # Hack to get dynamic pool listing
1691 def chunks(l, n):
1692 return [l[i:i+n] for i in range(0, len(l), n)]
1693
1694 ntp_servers = [x[0] for x in get_nameservers()]
1695 for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1696 for ntp_server in chunk:
1697 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1698
1699 details = dict()
1700 # 24 updates a day allowed
1701 details['serial'] = time.strftime('%Y%m%d%H')
1702
1703 if external:
1704 dns_masters = ['ns1.vanderzwet.net', 'ns1.anywi.com']
1705 else:
1706 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1707
1708 details['master'] = dns_masters[0]
1709 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1710
1711 dns_header = '''
1712$TTL 3h
1713%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 3h )
1714 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1715
1716%(ns_servers)s
1717 \n'''
1718
1719
1720 if not os.path.isdir(output_dir):
1721 os.makedirs(output_dir)
1722 details['zone'] = 'wleiden.net'
1723 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1724 f.write(dns_header % details)
1725
1726 for host,items in wleiden_zone.iteritems():
1727 for ip,reverse in items:
1728 if ip not in ['0.0.0.0']:
1729 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1730 for source,dest in wleiden_cname.iteritems():
1731 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1732 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1733 for source, dest in wleiden_raw.iteritems():
1734 f.write("%s.wleiden.net. %s\n" % (source, dest))
1735 f.close()
1736
1737 # Create whole bunch of specific sub arpa zones. To keep it compliant
1738 for s in range(16,32):
1739 details['zone'] = '%i.172.in-addr.arpa' % s
1740 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1741 f.write(dns_header % details)
1742
1743 #XXX: Not effient, fix to proper data structure and do checks at other
1744 # stages
1745 for host,items in wleiden_zone.iteritems():
1746 for ip,reverse in items:
1747 if not reverse:
1748 continue
1749 if valid_addr(ip):
1750 if valid_addr(ip):
1751 if int(ip.split('.')[1]) == s:
1752 rev_ip = '.'.join(reversed(ip.split('.')))
1753 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1754 f.close()
1755
1756
1757def usage():
1758 print """Usage: %(prog)s <argument>
1759Argument:
1760\tcleanup = Cleanup all YAML files to specified format
1761\tstandalone [port] = Run configurator webserver [8000]
1762\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1763\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1764\tfull-export = Generate yaml export script for heatmap.
1765\tstatic [outputdir] = Generate all config files and store on disk
1766\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1767\ttest <node> [<file>] = Receive output for certain node [all files].
1768\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1769\tlist <status> <items> = List systems which have certain status
1770\tcreate network.kml = Create Network KML file for use in Google Earth
1771
1772Arguments:
1773\t<node> = NodeName (example: HybridRick)
1774\t<file> = %(files)s
1775\t<status> = all|up|down|planned
1776\t<items> = systems|nodes|proxies
1777
1778NOTE FOR DEVELOPERS; you can test your changes like this:
1779 BEFORE any changes in this code:
1780 $ ./gformat.py static /tmp/pre
1781 AFTER the changes:
1782 $ ./gformat.py static /tmp/post
1783 VIEW differences and VERIFY all are OK:
1784 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1785""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1786 exit(0)
1787
1788
1789def is_text_request(environ=os.environ):
1790 """ Find out whether we are calling from the CLI or any text based CLI utility """
1791 if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1792 return True
1793
1794 if 'HTTP_USER_AGENT' in environ:
1795 return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1796 else:
1797 return False
1798
1799
1800def switchFormat(setting):
1801 if setting:
1802 return "YES"
1803 else:
1804 return "NO"
1805
1806def rlinput(prompt, prefill=''):
1807 import readline
1808 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1809 try:
1810 return raw_input(prompt)
1811 finally:
1812 readline.set_startup_hook()
1813
1814def fix_conflict(left, right, default='i'):
1815 while True:
1816 print "## %-30s | %-30s" % (left, right)
1817 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1818 if not c:
1819 c = default
1820
1821 if c in ['l','1']:
1822 return left
1823 elif c in ['r','2']:
1824 return right
1825 elif c in ['e', '3']:
1826 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1827 elif c in ['i', '4']:
1828 return None
1829 else:
1830 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1831
1832
1833
1834def print_cgi_response(response_headers, output):
1835 for header in response_headers:
1836 print "%s: %s" % header
1837 print
1838 print output
1839
1840
1841
1842def main():
1843 """Hard working sub"""
1844 # Allow easy hacking using the CLI
1845 if not os.environ.has_key('REQUEST_URI'):
1846 if len(sys.argv) < 2:
1847 usage()
1848
1849 if sys.argv[1] == "standalone":
1850 import SocketServer
1851 import CGIHTTPServer
1852 # Hop to the right working directory.
1853 os.chdir(os.path.dirname(__file__))
1854 try:
1855 PORT = int(sys.argv[2])
1856 except (IndexError,ValueError):
1857 PORT = 8000
1858
1859 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1860 """ Serve this CGI from the root of the webserver """
1861 def is_cgi(self):
1862 if "favicon" in self.path:
1863 return False
1864
1865 self.cgi_info = (os.path.basename(__file__), self.path)
1866 self.path = ''
1867 return True
1868 handler = MyCGIHTTPRequestHandler
1869 SocketServer.TCPServer.allow_reuse_address = True
1870 httpd = SocketServer.TCPServer(("", PORT), handler)
1871 httpd.server_name = 'localhost'
1872 httpd.server_port = PORT
1873
1874 logger.info("serving at port %s", PORT)
1875 try:
1876 httpd.serve_forever()
1877 except KeyboardInterrupt:
1878 httpd.shutdown()
1879 logger.info("All done goodbye")
1880 elif sys.argv[1] == "test":
1881 # Basic argument validation
1882 try:
1883 node = sys.argv[2]
1884 except IndexError:
1885 print "Invalid argument"
1886 exit(1)
1887 except IOError as e:
1888 print e
1889 exit(1)
1890
1891 datadump = get_yaml(node)
1892
1893
1894 # Get files to generate
1895 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1896
1897 # Actual config generation
1898 for config in gen_files:
1899 logger.info("## Generating %s %s", node, config)
1900 print generate_config(node, config, datadump)
1901 elif sys.argv[1] == "test-cgi":
1902 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1903 os.environ['SCRIPT_NAME'] = __file__
1904 response_headers, output = process_cgi_request()
1905 print_cgi_response(response_headers, output)
1906 elif sys.argv[1] == "static":
1907 generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
1908 elif sys.argv[1] == "wind-export":
1909 items = dict()
1910 for node in get_hostlist():
1911 datadump = get_yaml(node)
1912 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1913 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1914 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1915 VALUES (
1916 (SELECT id FROM users WHERE username = 'rvdzwet'),
1917 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1918 'Y');""" % datadump
1919 #for config in files:
1920 # items['config'] = config
1921 # print "## Generating %(node)s %(config)s" % items
1922 # f = open("%(wdir)s/%(config)s" % items, "w")
1923 # f.write(generate_config(node, config, datadump))
1924 # f.close()
1925 for node in get_hostlist():
1926 datadump = get_yaml(node)
1927 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1928 ifacedump = datadump[iface_key]
1929 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1930 ifacedump['nodename'] = datadump['nodename']
1931 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1932 ifacedump['channel'] = 0
1933 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1934 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1935 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1936 elif sys.argv[1] == "smokeping-export":
1937 for host in get_hostlist():
1938 datadump = get_yaml(host)
1939 if datadump.get('service_proxy_normal', False) or datadump.get('service_proxy_ileiden', False):
1940 print textwrap.dedent("""\
1941 ++ wleiden-gw-%(nodename)s
1942 menu = %(nodename)s.gw
1943 title = Wireless Leiden gateway %(nodename)s.gw.wleiden.net.
1944 host = %(nodename)s.gw.wleiden.net.
1945 """ % datadump)
1946 elif sys.argv[1] == "nagios-export":
1947 try:
1948 heavy_load = (sys.argv[2] == "--heavy-load")
1949 except IndexError:
1950 heavy_load = False
1951
1952 hostgroup_details = {
1953 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1954 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1955 'walphen' : 'Stichting Wireless Alphen',
1956 'westeinder' : 'Westeinder Plassen',
1957 }
1958
1959 # Convert IP to Host
1960 ip2host = {'root' : 'root'}
1961 for host in get_hostlist():
1962 datadump = get_yaml(host)
1963 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1964 for iface in get_interface_keys(datadump):
1965 if datadump[iface].has_key('autogen_gateway'):
1966 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1967
1968 # Find dependency tree based on output of lvrouted.mytree of nearest node
1969 parents = defaultdict(list)
1970 stack = ['root']
1971 prev_depth = 0
1972 if os.path.isfile('lvrouted.mytree'):
1973 for line in open('lvrouted.mytree').readlines():
1974 depth = line.count('\t')
1975 ip = line.strip().split()[0]
1976
1977 if prev_depth < depth:
1978 try:
1979 parents[ip2host[ip]].append(ip2host[stack[-1]])
1980 except KeyError as e:
1981 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1982 stack.append(ip)
1983 elif prev_depth > depth:
1984 stack = stack[:(depth - prev_depth)]
1985 elif prev_depth == depth:
1986 try:
1987 parents[ip2host[ip]].append(ip2host[stack[-1]])
1988 except KeyError as e:
1989 print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1990
1991
1992 prev_depth = depth
1993 # Observe that some nodes has themself as parent or multiple parents
1994 # for now take only the first parent, other behaviour is yet to be explained
1995
1996
1997
1998 params = {
1999 'check_interval' : 5 if heavy_load else 120,
2000 'retry_interval' : 1 if heavy_load else 10,
2001 'max_check_attempts' : 10 if heavy_load else 6,
2002 'notification_interval': 120 if heavy_load else 240,
2003 }
2004
2005 print '''\
2006define host {
2007 name wleiden-node ; Default Node Template
2008 use generic-host ; Use the standard template as initial starting point
2009 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2010 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2011 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2012 notification_interval %(notification_interval)s
2013 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2014 check_command check-host-alive ; Default command to check FreeBSD hosts
2015 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2016}
2017
2018define service {
2019 name wleiden-service ; Default Service Template
2020 use generic-service ; Use the standard template as initial starting point
2021 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2022 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2023 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2024 notification_interval %(notification_interval)s
2025 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2026 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2027}
2028
2029# Please make sure to install:
2030# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2031#
2032# Recompile net-mgmt/nagios-plugins to support check_snmp
2033# make -C /usr/ports/net-mgmt/nagios-plugins
2034#
2035# Install net/bind-tools to allow v2/check_dns_wl to work:
2036# pkg install bind-tools
2037#
2038define command{
2039 command_name check_snmp_disk
2040 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2041}
2042
2043define command{
2044 command_name check_netsnmp_load
2045 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2046}
2047
2048define command{
2049 command_name check_netsnmp_proc
2050 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2051}
2052
2053define command{
2054 command_name check_by_ssh
2055 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2056}
2057
2058define command{
2059 command_name check_dns_wl
2060 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2061}
2062
2063define command{
2064 command_name check_snmp_uptime
2065 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2066}
2067
2068
2069# TDB: dhcp leases
2070# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2071
2072# TDB: internet status
2073# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2074
2075# TDB: Advanced local passive checks
2076# /usr/local/libexec/nagios/check_by_ssh
2077''' % params
2078
2079 print '''\
2080# Service Group, not displayed by default
2081define hostgroup {
2082 hostgroup_name srv_hybrid
2083 alias All Hybrid Nodes
2084 register 0
2085}
2086
2087define service {
2088 use wleiden-service
2089 hostgroup_name srv_hybrid
2090 service_description SSH
2091 check_command check_ssh
2092}
2093
2094define service {
2095 use wleiden-service,service-pnp
2096 hostgroup_name srv_hybrid
2097 service_description HTTP
2098 check_command check_http
2099}
2100
2101define service {
2102 use wleiden-service
2103 hostgroup_name srv_hybrid
2104 service_description DNS
2105 check_command check_dns_wl!"www.wirelessleiden.nl"
2106}
2107'''
2108
2109 if heavy_load:
2110 print '''\
2111define service {
2112 use wleiden-service
2113 hostgroup_name srv_hybrid
2114 service_description UPTIME
2115 check_command check_snmp_uptime
2116}
2117
2118#define service {
2119# use wleiden-service
2120# hostgroup_name srv_hybrid
2121# service_description NTP
2122# check_command check_ntp_peer
2123#}
2124
2125define service {
2126 use wleiden-service
2127 hostgroup_name srv_hybrid
2128 service_description LOAD
2129 check_command check_netsnmp_load
2130}
2131
2132define service {
2133 use wleiden-service
2134 hostgroup_name srv_hybrid
2135 service_description PROC
2136 check_command check_netsnmp_proc
2137}
2138
2139define service {
2140 use wleiden-service
2141 hostgroup_name srv_hybrid
2142 service_description DISK
2143 check_command check_snmp_disk
2144}
2145'''
2146 for node in get_hostlist():
2147 datadump = get_yaml(node)
2148 if not datadump['status'] == 'up':
2149 continue
2150 if not hostgroup_details.has_key(datadump['monitoring_group']):
2151 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2152 print '''\
2153define host {
2154 use wleiden-node,host-pnp
2155 contact_groups admins
2156 host_name %(autogen_fqdn)s
2157 address %(masterip)s
2158 hostgroups srv_hybrid,%(monitoring_group)s\
2159''' % datadump
2160 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2161 print '''\
2162 parents %(parents)s\
2163''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2164 print '''\
2165}
2166'''
2167
2168
2169 for name,alias in hostgroup_details.iteritems():
2170 print '''\
2171define hostgroup {
2172 hostgroup_name %s
2173 alias %s
2174} ''' % (name, alias)
2175
2176
2177 elif sys.argv[1] == "full-export":
2178 hosts = {}
2179 for node in get_hostlist():
2180 datadump = get_yaml(node)
2181 hosts[datadump['nodename']] = datadump
2182 print yaml.dump(hosts)
2183
2184 elif sys.argv[1] == "dns":
2185 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2186 elif sys.argv[1] == "cleanup":
2187 # First generate all datadumps
2188 datadumps = dict()
2189 ssid_to_node = dict()
2190 for host in get_hostlist():
2191 logger.info("# Processing: %s", host)
2192 # Set some boring default values
2193 datadump = { 'board' : 'UNKNOWN' }
2194 datadump.update(get_yaml(host))
2195 datadumps[datadump['nodename']] = datadump
2196
2197 (poel, errors) = make_relations()
2198 print "\n".join(["# WARNING: %s" % x for x in errors])
2199
2200 for host,datadump in datadumps.iteritems():
2201 try:
2202 # Convert all yes and no to boolean values
2203 def fix_boolean(dump):
2204 for key in dump.keys():
2205 if type(dump[key]) == dict:
2206 dump[key] = fix_boolean(dump[key])
2207 elif str(dump[key]).lower() in ["yes", "true"]:
2208 dump[key] = True
2209 elif str(dump[key]).lower() in ["no", "false"]:
2210 # Compass richting no (Noord Oost) is valid input
2211 if key != "compass": dump[key] = False
2212 return dump
2213 datadump = fix_boolean(datadump)
2214
2215 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2216 if not 'latitude' in datadump and not 'longitude' in datadump:
2217 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2218 elif 'latitude' in datadump and 'longitude' in datadump:
2219 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2220 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2221 # TODO: Compare outcome of both coordinate systems and validate against each-other
2222
2223 if datadump['nodename'].startswith('Proxy'):
2224 datadump['nodename'] = datadump['nodename'].lower()
2225
2226 for iface_key in get_interface_keys(datadump):
2227 try:
2228 # All our normal wireless cards are normal APs now
2229 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2230 datadump[iface_key]['mode'] = 'ap'
2231 # Wireless Leiden SSID have an consistent lowercase/uppercase
2232 if datadump[iface_key].has_key('ssid'):
2233 ssid = datadump[iface_key]['ssid']
2234 prefix = 'ap-WirelessLeiden-'
2235 if ssid.lower().startswith(prefix.lower()):
2236 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2237 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2238 datadump[iface_key]['mode'] = 'autogen-FIXME'
2239 if not datadump[iface_key].has_key('comment'):
2240 datadump[iface_key]['comment'] = 'autogen-FIXME'
2241
2242 if datadump[iface_key].has_key('ns_mac'):
2243 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2244
2245 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2246 datadump[iface_key] = datadump[iface_key]['desc']
2247
2248 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2249 # due to better collision avoidance.
2250 if datadump[iface_key]['type'] == '11b':
2251 datadump[iface_key]['type'] = '11g'
2252
2253 # Setting 802.11g channels to de-facto standards, to avoid
2254 # un-detected sharing with other overlapping channels
2255 #
2256 # Technically we could also use channel 13 in NL, but this is not
2257 # recommended as foreign devices might not be able to select this
2258 # channel. Secondly using 1,5,9,13 instead is going to clash with
2259 # the de-facto usage of 1,6,11.
2260 #
2261 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2262 channels_at_2400Mhz = (1,6,11)
2263 if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2264 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2265 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2266 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2267
2268 # Mandatory interface keys
2269 if not datadump[iface_key].has_key('status'):
2270 datadump[iface_key]['status'] = 'planned'
2271
2272 x = datadump[iface_key]['comment']
2273 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2274
2275 # Fixing bridge_type if none is found
2276 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2277 if not 'bridge_type' in datadump[iface_key]:
2278 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2279
2280 # Making sure description works
2281 if datadump[iface_key].has_key('desc'):
2282 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2283 del datadump[iface_key]['desc']
2284 else:
2285 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2286 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2287 if response:
2288 datadump[iface_key]['comment'] = response
2289 del datadump[iface_key]['desc']
2290
2291 # Check DHCP configuration
2292 dhcp_type(datadump[iface_key])
2293
2294 # Set the compass value based on the angle between the poels
2295 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2296 my_pool = poel[network(datadump[iface_key]['ip'])]
2297 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2298 if remote_hosts:
2299 compass_target = remote_hosts[0]
2300 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2301 # TODO: Compass wanted and actual direction might differ
2302
2303 # Monitoring Group default
2304 if not 'monitoring_group' in datadump:
2305 datadump['monitoring_group'] = 'wleiden'
2306
2307 except Exception as exc:
2308 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
2309 raise
2310 store_yaml(datadump)
2311 except Exception as exc:
2312 exc.args = ("# Error while processing %s" % host,) + exc.args
2313 raise
2314 elif sys.argv[1] == "list":
2315 use_fqdn = False
2316 if len(sys.argv) < 4:
2317 usage()
2318 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2319 usage()
2320 if not sys.argv[3] in ["nodes","proxies","systems"]:
2321 usage()
2322
2323 if len(sys.argv) > 4:
2324 if sys.argv[4] == "fqdn":
2325 use_fqdn = True
2326 else:
2327 usage()
2328
2329 for system in get_hostlist():
2330 datadump = get_yaml(system)
2331 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2332 continue
2333
2334 output = datadump['autogen_fqdn'] if use_fqdn else system
2335 if sys.argv[2] == "all":
2336 print output
2337 elif datadump['status'] == sys.argv[2]:
2338 print output
2339 elif sys.argv[1] == "create":
2340 if sys.argv[2] == "network.kml":
2341 print make_network_kml.make_graph()
2342 elif sys.argv[2] == "host-ips.txt":
2343 for system in get_hostlist():
2344 datadump = get_yaml(system)
2345 ips = [datadump['masterip']]
2346 for ifkey in get_interface_keys(datadump):
2347 ips.append(datadump[ifkey]['ip'].split('/')[0])
2348 print system, ' '.join(ips)
2349 elif sys.argv[2] == "host-pos.txt":
2350 for system in get_hostlist():
2351 datadump = get_yaml(system)
2352 print system, datadump['rdnap_x'], datadump['rdnap_y']
2353 elif sys.argv[2] == "nodeplanner.json":
2354 print make_network_kml.make_nodeplanner_json()
2355 elif sys.argv[2] == 'ssh_config':
2356 print '''
2357Host *.wleiden.net
2358 User root
2359
2360Host 172.16.*.*
2361 User root
2362'''
2363 for system in get_hostlist():
2364 datadump = get_yaml(system)
2365 print '''\
2366Host %s
2367 User root
2368
2369Host %s
2370 User root
2371
2372Host %s
2373 User root
2374
2375Host %s
2376 User root
2377''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2378 else:
2379 usage()
2380 else:
2381 usage()
2382 else:
2383 # Do not enable debugging for config requests as it highly clutters the output
2384 if not is_text_request():
2385 cgitb.enable()
2386
2387 try:
2388 response_headers, output = process_cgi_request()
2389 except:
2390 print ''
2391 print ''
2392 raise
2393
2394 print_cgi_response(response_headers, output)
2395
2396if __name__ == "__main__":
2397 main()
Note: See TracBrowser for help on using the repository browser.