Context Navigation

gformat.py@ 14053

Last change on this file since 14053 was 14053, checked in by rick, 7 years ago

Fix alias VLAN dhcp declaration not mapped to shared-network

While here make all entries-shared networks, since it does not harm anyways:

Note that even when the shared-network declaration is absent, an empty
one is created by the server to contain the subnet (and any scoped
parameters included in the subnet). For practical purposes, this means
that "stateless" DHCP clients, which are not tied to addresses (and
therefore subnets) will receive the same configuration as stateful
ones.

Property svn:executable set to *
Property svn:keywords set to Id

File size: 79.7 KB

Line
1	#!/usr/local/bin/python
2	#
3	# vim:ts=2:et:sw=2:ai
4	# Wireless Leiden configuration generator, based on yaml files'
5	#
6	# XXX: This should be rewritten to make use of the ipaddr.py library.
7	#
8	# Sample apache configuration (mind the AcceptPathInfo!)
9	# Alias /config /usr/local/www/config
10	# <Directory /usr/local/www/config>
11	# AddHandler cgi-script .py
12	# Require all granted
13	#
14	# RewriteEngine on
15	# RewriteCond %{REQUEST_FILENAME} !-f
16	# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17	# Options +FollowSymlinks +ExecCGI
18	# </Directory>
19	#
20	# Package dependencies list:
21	# FreeBSD:
22	# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23	# Fedora:
24	# yum install python-yaml pyproj proj-epsg python-jinja2
25	#
26	# Rick van der Zwet <info@rickvanderzwet.nl>
27	#
28
29	# Hack to make the script directory is also threated as a module search path.
30	import sys
31	import os
32	sys.path.append(os.path.dirname(__file__))
33
34	SVN = filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn'))[0]
35	SVNVERSION = filter(os.path.isfile, ('/usr/local/bin/svnversion', '/usr/bin/svnversion'))[0]
36
37	import argparse
38	import cgi
39	import cgitb
40	import copy
41	import glob
42	import make_network_kml
43	import math
44	import pyproj
45	import random
46	import re
47	import socket
48	import string
49	import subprocess
50	import textwrap
51	import time
52	import traceback
53	import urlparse
54
55	from pprint import pprint
56	from collections import defaultdict, OrderedDict
57	from sys import stderr
58	try:
59	import yaml
60	except ImportError, e:
61	print e
62	print "[ERROR] Please install the python-yaml or devel/py-yaml package"
63	exit(1)
64
65	try:
66	from yaml import CLoader as Loader
67	from yaml import CDumper as Dumper
68	except ImportError:
69	from yaml import Loader, Dumper
70
71	from jinja2 import Environment, Template
72	def yesorno(value):
73	return "YES" if bool(value) else "NO"
74	env = Environment()
75	env.filters['yesorno'] = yesorno
76	def render_template(datadump, template):
77	result = env.from_string(template).render(datadump)
78	# Make it look pretty to the naked eye, as jinja templates are not so
79	# friendly when it comes to whitespace formatting
80	## Remove extra whitespace at end of line lstrip() style.
81	result = re.sub(r'\n[\ ]+','\n', result)
82	## Include only a single newline between an definition and a comment
83	result = re.sub(r'(["\'])\n+([a-z]\|\n#\n)',r'\1\n\2', result)
84	## Remove extra newlines after single comment
85	result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
86	return result
87
88	import logging
89	logging.basicConfig(format='# %(levelname)s: %(message)s' )
90	logger = logging.getLogger()
91	logger.setLevel(logging.DEBUG)
92
93
94	if os.environ.has_key('CONFIGROOT'):
95	NODE_DIR = os.environ['CONFIGROOT']
96	else:
97	NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
98	__version__ = '$Id: gformat.py 14053 2018-02-20 14:54:34Z rick $'
99
100	CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
101
102	files = [
103	'authorized_keys',
104	'dnsmasq.conf',
105	'dhcpd.conf',
106	'rc.conf.local',
107	'resolv.conf',
108	'motd',
109	'ntp.conf',
110	'pf.hybrid.conf.local',
111	'unbound.wleiden.conf',
112	'wleiden.yaml',
113	]
114
115	# Global variables uses
116	OK = 10
117	DOWN = 20
118	UNKNOWN = 90
119
120
121	ileiden_proxies = OrderedDict()
122	normal_proxies = []
123	datadump_cache = {}
124	interface_list_cache = {}
125	rc_conf_local_cache = {}
126	nameservers_cache = []
127	relations_cache = None
128
129	NO_DHCP = 0
130	DHCP_CLIENT = 10
131	DHCP_SERVER = 20
132	def dhcp_type(item):
133	if not item.has_key('dhcp'):
134	return NO_DHCP
135	elif not item['dhcp']:
136	return NO_DHCP
137	elif item['dhcp'].lower() == 'client':
138	return DHCP_CLIENT
139	else:
140	# Validation Checks
141	begin,end = map(int,item['dhcp'].split('-'))
142	if begin >= end:
143	raise ValueError("DHCP Start >= DHCP End")
144	return DHCP_SERVER
145
146	def etrs2rd(lat, lon):
147	p1 = pyproj.Proj(proj='latlon',datum='WGS84')
148	p2 = pyproj.Proj(init='EPSG:28992')
149	RDx, RDy = pyproj.transform(p1,p2,lon, lat)
150	return (RDx, RDy)
151
152	def rd2etrs(RDx, RDy):
153	p1 = pyproj.Proj(init='EPSG:28992')
154	p2 = pyproj.Proj(proj='latlon',datum='WGS84')
155	lon, lat = pyproj.transform(p1,p2, RDx, RDy)
156	return (lat, lon)
157
158	def get_yaml(item,add_version_info=True):
159	try:
160	""" Get configuration yaml for 'item'"""
161	if datadump_cache.has_key(item):
162	return datadump_cache[item].copy()
163
164	gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
165	global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
166	d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
167
168	# Default values
169	datadump = {
170	'autogen_revision' : 'NOTFOUND',
171	'autogen_gfile' : gfile,
172	'service_proxy_ileiden' : False,
173	}
174	f = open(gfile, 'r')
175	datadump.update(yaml.load(f,Loader=Loader))
176	datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
177	if datadump['nodetype'] == 'Hybrid':
178	# Some values are defined implicitly
179	if datadump.has_key('rdr_host') and datadump['rdr_host'] and not datadump.has_key('service_incoming_rdr'):
180	datadump['service_incoming_rdr'] = True
181	# Use some boring defaults
182	defaults = {
183	'service_proxy_normal' : False,
184	'service_accesspoint' : True,
185	'service_incoming_rdr' : False,
186	'service_concentrator' : False,
187	'monitoring_group' : 'wleiden',
188	}
189	for (key,value) in defaults.iteritems():
190	if not datadump.has_key(key):
191	datadump[key] = value
192	f.close()
193
194	# Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
195	if add_version_info:
196	p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
197	lines = p.communicate()[0].split('\n')
198	if p.returncode == 0:
199	for line in lines:
200	if line:
201	(key, value) = line.split(': ')
202	datadump["autogen_" + key.lower().replace(' ','_')] = value
203
204	# Preformat certain needed variables for formatting and push those into special object
205	datadump['autogen_iface_keys'] = get_interface_keys(datadump)
206
207	wlan_count=0
208	try:
209	for key in get_interface_keys(datadump, True):
210	datadump[key]['autogen_ifbase'] = key.split('_')[1]
211	datadump[key]['autogen_vlan'] = False
212
213	datadump[key]['autogen_bridge_member'] = datadump[key].has_key('parent')
214	datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
215	datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
216
217	if datadump[key].has_key('parent'):
218	if datadump[key].has_key('ip'):
219	raise ValueError("Interface bridge member cannot have IP assigned")
220	if datadump[key].has_key('dhcp') and datadump[key]['dhcp'] != False:
221	raise ValueError("Interface bridge member cannot have DHCP set")
222
223	if datadump[key].has_key('ip'):
224	datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
225
226	if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
227	datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
228	datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
229	datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
230	wlan_count += 1
231	else:
232	datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
233	if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
234	datadump[key]['autogen_vlan'] = key.split('_')[2]
235	datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
236	datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifname']
237	else:
238	datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
239	datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
240	print key,datadump[key]['autogen_iface'], datadump[key]['autogen_ifname'], datadump[key]['autogen_if_dhcp']
241
242	except Exception as exc:
243	exc.args = ("# Error while processing interface %s" % key,) + exc.args
244	raise
245
246	dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
247	if dhcp_type(datadump[key]) == DHCP_SERVER]
248
249	datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
250	datadump['autogen_item'] = item
251
252	datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
253	datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
254	datadump_cache[item] = datadump.copy()
255	except Exception as exc:
256	exc.args = ("# Error while processing %s" % item,) + exc.args
257	raise
258	return datadump
259
260
261	def store_yaml(datadump, header=False):
262	""" Store configuration yaml for 'item'"""
263	item = datadump['autogen_item']
264	gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
265
266	output = generate_wleiden_yaml(datadump, header)
267
268	f = open(gfile, 'w')
269	f.write(output)
270	f.close()
271
272
273	def network(ip):
274	addr, mask = ip.split('/')
275	# Not parsing of these folks please
276	addr = parseaddr(addr)
277	mask = int(mask)
278	network = addr & ~((1 << (32 - mask)) - 1)
279	return network
280
281
282
283	def make_relations():
284	""" Process _ALL_ yaml files to get connection relations """
285	global relations_cache
286
287	if relations_cache:
288	return relations_cache
289
290	errors = []
291	poel = defaultdict(list)
292
293	for host in get_hostlist():
294	datadump = get_yaml(host)
295	try:
296	for iface_key in get_interface_keys(datadump):
297	# Bridge members has no IP assigned
298	if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
299	continue
300	net_addr = network(datadump[iface_key]['ip'])
301	poel[net_addr] += [(host,datadump[iface_key].copy())]
302	except (KeyError, ValueError) as e:
303	errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
304	continue
305
306	relations_cache = (poel, errors)
307	return relations_cache
308
309
310
311	def valid_addr(addr):
312	""" Show which address is valid in which are not """
313	return str(addr).startswith('172.')
314
315	def get_hostlist():
316	""" Combined hosts and proxy list"""
317	return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
318
319	def angle_between_points(lat1,lat2,long1,long2):
320	"""
321	Return Angle in radians between two GPS coordinates
322	See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
323	"""
324	dy = lat2 - lat1
325	dx = math.cos(lat1)*(long2 - long1)
326	angle = math.atan2(dy,dx)
327	return angle
328
329
330
331	def angle_to_cd(angle):
332	""" Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
333
334	# For easy conversion get positive degree
335	degrees = math.degrees(angle)
336	abs_degrees = 360 + degrees if degrees < 0 else degrees
337
338	# Numbers can be confusing calculate from the 4 main directions
339	p = 22.5
340	if abs_degrees < p:
341	cd = "n"
342	elif abs_degrees < (90 - p):
343	cd = "no"
344	elif abs_degrees < (90 + p):
345	cd = "o"
346	elif abs_degrees < (180 - p):
347	cd = "zo"
348	elif abs_degrees < (180 + p):
349	cd = "z"
350	elif abs_degrees < (270 - p):
351	cd = "zw"
352	elif abs_degrees < (270 + p):
353	cd = "w"
354	elif abs_degrees < (360 - p):
355	cd = "nw"
356	else:
357	cd = "n"
358	return cd
359
360
361
362	def cd_between_hosts(hostA, hostB, datadumps):
363	# Using RDNAP coordinates
364	dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
365	dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
366	return angle_to_cd(math.atan2(dx,dy))
367
368	# GPS coordinates seems to fail somehow
369	#latA = float(datadumps[hostA]['latitude'])
370	#latB = float(datadumps[hostB]['latitude'])
371	#lonA = float(datadumps[hostA]['longitude'])
372	#lonB = float(datadumps[hostB]['longitude'])
373	#return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
374
375
376	def generate_title(nodelist):
377	""" Main overview page """
378	items = { \
379	'root' : ".",
380	'version' : subprocess.Popen([SVNVERSION, "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
381	}
382	def fl(spaces, line):
383	return (' ' * spaces) + line + '\n'
384
385	output = """
386	<html>
387	<head>
388	<title>Wireless leiden Configurator - GFormat</title>
389	<style type="text/css">
390	th {background-color: #999999}
391	tr:nth-child(odd) {background-color: #cccccc}
392	tr:nth-child(even) {background-color: #ffffff}
393	th, td {padding: 0.1em 1em}
394	</style>
395	</head>
396	<body>
397	<center>
398	<form type="GET" action="%(root)s">
399	<input type="hidden" name="action" value="update">
400	<input type="submit" value="Update Configuration Database (SVN)">
401	</form>
402	<table>
403	<caption><h3>Wireless Leiden Configurator - Revision %(version)s</h3></caption>
404	""" % items
405
406	for node in nodelist:
407	items['node'] = node
408	output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
409	for config in files:
410	items['config'] = config
411	output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
412	output += fl(5, "</tr>")
413	output += """
414	</table>
415	<hr />
416	<em>%s</em>
417	</center>
418	</body>
419	</html>
420	""" % __version__
421
422	return output
423
424
425
426	def generate_node(node):
427	""" Print overview of all files available for node """
428	return "\n".join(files)
429
430	def generate_node_overview(host, datadump=False):
431	""" Print overview of all files available for node """
432	if not datadump:
433	datadump = get_yaml(host)
434	params = { 'host' : host }
435	output = "<em><a href='..'>Back to overview</a></em><hr />"
436	output += "<h2>Available files:</h2><ul>"
437	for cf in files:
438	params['cf'] = cf
439	output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
440	output += "</ul>"
441
442	# Generate and connection listing
443	output += "<h2>Connected To:</h2><ul>"
444	(poel, errors) = make_relations()
445	for network, hosts in poel.iteritems():
446	if host in [x[0] for x in hosts]:
447	if len(hosts) == 1:
448	# Single not connected interface
449	continue
450	for remote,ifacedump in hosts:
451	if remote == host:
452	# This side of the interface
453	continue
454	params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
455	output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
456	output += "</ul>"
457	output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
458
459	output += "<hr /><em><a href='..'>Back to overview</a></em>"
460	return output
461
462
463	def generate_header(datadump, ctag="#"):
464	return """\
465	%(ctag)s
466	%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
467	%(ctag)s
468	""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
469
470
471
472	def parseaddr(s):
473	""" Process IPv4 CIDR notation addr to a (binary) number """
474	f = s.split('.')
475	return (long(f[0]) << 24L) + \
476	(long(f[1]) << 16L) + \
477	(long(f[2]) << 8L) + \
478	long(f[3])
479
480
481
482	def showaddr(a):
483	""" Display IPv4 addr in (dotted) CIDR notation """
484	return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
485
486
487	def is_member(ip, mask, canidate):
488	""" Return True if canidate is part of ip/mask block"""
489	ip_addr = parseaddr(ip)
490	ip_canidate = parseaddr(canidate)
491	mask = int(mask)
492	ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
493	ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
494	return ip_addr == ip_canidate
495
496
497
498	def cidr2netmask(netmask):
499	""" Given a 'netmask' return corresponding CIDR """
500	return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
501
502	def get_network(addr, mask):
503	return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
504
505
506	def generate_dhcpd_conf(datadump):
507	""" Generate config file '/usr/local/etc/dhcpd.conf """
508	# Redundency support, in cause local DNS server is not running/responding.
509	datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
510	output = generate_header(datadump)
511	output += Template("""\
512	# option definitions common to all supported networks...
513	option domain-name "dhcp.{{ autogen_fqdn }}";
514
515	default-lease-time 600;
516	max-lease-time 7200;
517
518	# Use this to enble / disable dynamic dns updates globally.
519	#ddns-update-style none;
520
521	# If this DHCP server is the official DHCP server for the local
522	# network, the authoritative directive should be uncommented.
523	authoritative;
524
525	# Use this to send dhcp log messages to a different log file (you also
526	# have to hack syslog.conf to complete the redirection).
527	log-facility local7;
528
529	# UniFi Discovery Support
530	option space ubnt;
531	option ubnt.unifi-address code 1 = ip-address;
532	#
533	class "ubnt" {
534	match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
535	option vendor-class-identifier "ubnt";
536	vendor-option-space ubnt;
537	}
538
539	#
540	# Interface definitions
541	#
542	\n\n""").render(datadump)
543
544
545	# TODO: Use textwrap.fill instead
546	def indent(text, count):
547	return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
548
549	# Process DHCP blocks
550	dhcp_out = defaultdict(list)
551	for iface_key in get_interface_keys(datadump):
552	ifname = datadump[iface_key]['autogen_ifbase']
553	groupif = datadump[iface_key]['autogen_if_dhcp']
554	if not datadump[iface_key].has_key('comment'):
555	datadump[iface_key]['comment'] = None
556
557	if not datadump[iface_key].has_key('ip'):
558	continue
559
560	dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
561
562	(addr, mask) = datadump[iface_key]['ip'].split('/')
563	datadump[iface_key]['autogen_addr'] = addr
564	datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
565	datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
566
567	if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
568	dhcp_out[groupif].append(textwrap.dedent("""\
569	subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
570	### not autoritive
571	}
572	""" % datadump[iface_key]))
573	continue
574
575	(dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
576	dhcp_part = ".".join(addr.split('.')[0:3])
577	datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
578	datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
579	datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
580
581	# Assume the first 10 IPs could be used for static entries
582	if 'no_portal' in datadump:
583	fixed = 5
584	for mac in datadump['no_portal']:
585	dhcp_out[groupif].append(textwrap.dedent("""\
586	host fixed-%(ifname)s-%(fixed)s {
587	hardware ethernet %(mac)s;
588	fixed-address %(prefix)s.%(fixed)s;
589	}
590	""" % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
591	fixed += 1
592
593	if 'dhcp_fixed' in datadump[iface_key]:
594	for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
595	dhcp_out[groupif].append(textwrap.dedent("""\
596	host fixed-%(host)s {
597	hardware ethernet %(mac)s;
598	fixed-address %(addr)s;
599	}
600	""" % { 'host' : host, 'mac' : mac, 'addr' : addr}))
601
602
603	dhcp_out[groupif].append(textwrap.dedent("""\
604	subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
605	range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
606	option routers %(autogen_addr)s;
607	option domain-name-servers %(autogen_dns_servers)s;
608	option ubnt.unifi-address 172.17.107.10;
609	}
610	""" % datadump[iface_key]))
611
612	# Output the blocks in groups
613	for ifname,value in sorted(dhcp_out.iteritems()):
614	output += ("shared-network %s {\n" % ifname) + indent(''.join(value), 2).rstrip() + '\n}\n\n'
615	return output
616
617
618
619	def generate_dnsmasq_conf(datadump):
620	""" Generate configuration file '/usr/local/etc/dnsmasq.conf' """
621	output = generate_header(datadump)
622	output += Template("""\
623	# DHCP server options
624	dhcp-authoritative
625	dhcp-fqdn
626	domain=dhcp.{{ autogen_fqdn }}
627	domain-needed
628	expand-hosts
629	log-async=100
630
631	# Low memory footprint
632	cache-size=10000
633
634	\n""").render(datadump)
635
636	for iface_key in get_interface_keys(datadump):
637	if not datadump[iface_key].has_key('comment'):
638	datadump[iface_key]['comment'] = None
639	output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
640
641	if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
642	output += "# not autoritive\n\n"
643	continue
644
645	(dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
646	(ip, cidr) = datadump[iface_key]['ip'].split('/')
647	datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
648
649	dhcp_part = ".".join(ip.split('.')[0:3])
650	datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
651	datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
652	output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
653
654	return output
655
656
657	class AutoVivification(dict):
658	"""Implementation of perl's autovivification feature."""
659	def __getitem__(self, item):
660	try:
661	return dict.__getitem__(self, item)
662	except KeyError:
663	value = self[item] = type(self)()
664	return value
665
666	def make_interface_list(datadump):
667	if interface_list_cache.has_key(datadump['autogen_item']):
668	return (interface_list_cache[datadump['autogen_item']])
669	# lo0 configuration:
670	# - 172.32.255.1/32 is the proxy.wleiden.net deflector
671	# - masterip is special as it needs to be assigned to at
672	# least one interface, so if not used assign to lo0
673	addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
674	vlan_list = defaultdict(list)
675	bridge_list = defaultdict(list)
676	flags_if = AutoVivification()
677	dhclient_if = {'lo0' : False}
678
679	# XXX: Find some way of send this output nicely
680	output = ''
681
682	masterip_used = False
683	for iface_key in get_interface_keys(datadump):
684	if datadump[iface_key].has_key('ip') and datadump[iface_key]['ip'].startswith(datadump['masterip']):
685	masterip_used = True
686	break
687	if not masterip_used:
688	addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
689
690	if 'serviceid' in datadump:
691	addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
692
693	for iface_key in get_interface_keys(datadump):
694	ifacedump = datadump[iface_key]
695	if ifacedump['autogen_bridge_alias']:
696	ifname = ifacedump['autogen_ifbase']
697	else:
698	ifname = ifacedump['autogen_ifname']
699
700	# If defined as vlan interface
701	if ifacedump['autogen_vlan']:
702	vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
703
704	# If defined as bridge interface
705	if ifacedump['autogen_bridge_member']:
706	bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
707
708
709	# Flag dhclient is possible
710	if not dhclient_if.has_key(ifname) or dhclient_if[ifname] == False:
711	dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
712
713	# Ethernet address
714	if ifacedump.has_key('ether'):
715	flags_if[ifname]['ether'] = ifacedump['ether']
716
717	# Add interface IP to list
718	if ifacedump.has_key('ip'):
719	item = (ifacedump['ip'], ifacedump['comment'])
720	if addrs_list.has_key(ifname):
721	addrs_list[ifname].append(item)
722	else:
723	addrs_list[ifname] = [item]
724
725	# Alias only needs IP assignment for now, this might change if we
726	# are going to use virtual accesspoints
727	if "alias" in iface_key:
728	continue
729
730	# XXX: Might want to deduct type directly from interface name
731	if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
732	# Default to station (client) mode
733	ifacedump['autogen_wlanmode'] = "sta"
734	if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
735	ifacedump['autogen_wlanmode'] = "ap"
736
737	if not ifacedump.has_key('channel'):
738	if ifacedump['type'] == '11a':
739	ifacedump['channel'] = 36
740	else:
741	ifacedump['channel'] = 1
742
743	# Allow special hacks at the back like wds and stuff
744	if not ifacedump.has_key('extra'):
745	ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
746	else:
747	ifacedump['autogen_extra'] = ifacedump['extra']
748
749	ifacedump['autogen_ssid_hex'] = '0x' + ''.join(x.encode('hex') for x in ifacedump['ssid'])
750
751	output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
752	output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
753	output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
754	"%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
755	output += "\n"
756
757	elif ifacedump['type'] in ['ethernet', 'eth']:
758	# No special config needed besides IP
759	pass
760	elif ifacedump['type'] in ['vlan']:
761	# VLAN member has no special configuration
762	pass
763	else:
764	assert False, "Unknown type " + ifacedump['type']
765
766	store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
767	interface_list_cache[datadump['autogen_item']] = store
768	return(store)
769
770
771
772	def create_proxies_list():
773	if not ileiden_proxies or not normal_proxies:
774	# Placeholder for to-be-installed proxies, this will avoid updating the all
775	# nodes to include this new machine, yet due to an unbound issue, this list
776	# has to be kept small.
777
778	for i in range(1,20):
779	ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
780
781	for host in get_hostlist():
782	hostdump = get_yaml(host)
783	if hostdump['status'] == 'up':
784	if hostdump['service_proxy_ileiden']:
785	ileiden_proxies[hostdump['serviceid']] = hostdump
786	if hostdump['service_proxy_normal']:
787	normal_proxies.append(hostdump)
788
789
790
791	def generate_rc_conf_local(datadump):
792	""" Generate configuration file '/etc/rc.conf.local' """
793	item = datadump['autogen_item']
794	if rc_conf_local_cache.has_key(item):
795	return rc_conf_local_cache[item]
796
797	if not datadump.has_key('ileiden'):
798	datadump['autogen_ileiden_enable'] = False
799	else:
800	datadump['autogen_ileiden_enable'] = datadump['ileiden']
801
802	datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
803
804	create_proxies_list()
805	datadump['autogen_ileiden_proxies'] = ileiden_proxies
806	datadump['autogen_normal_proxies'] = normal_proxies
807	datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
808	datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
809	datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
810	datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
811
812	output = generate_header(datadump, "#");
813	output += render_template(datadump, """\
814	hostname='{{ autogen_fqdn }}'
815	location='{{ location }}'
816	nodetype="{{ nodetype }}"
817
818	#
819	# Configured listings
820	#
821	captive_portal_whitelist=""
822	{% if nodetype == "Proxy" %}
823	#
824	# Proxy Configuration
825	#
826	{% if gateway and service_proxy_ileiden -%}
827	defaultrouter="{{ gateway }}"
828	{% else -%}
829	#defaultrouter="NOTSET"
830	{% endif -%}
831	internalif="{{ internalif }}"
832	ileiden_enable="{{ autogen_ileiden_enable }}"
833	gateway_enable="{{ autogen_ileiden_enable }}"
834	pf_enable="yes"
835	pf_rules="/etc/pf.conf"
836	{% if autogen_ileiden_enable -%}
837	pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
838	lvrouted_enable="{{ autogen_ileiden_enable }}"
839	lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
840	{% else -%}
841	pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
842	{% endif -%}
843	{% if internalroute -%}
844	static_routes="wleiden"
845	route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
846	{% endif -%}
847
848	{% elif nodetype == "Hybrid" %}
849	#
850	# Hybrid Configuration
851	#
852	list_ileiden_proxies="
853	{% for serviceid,item in autogen_ileiden_proxies.iteritems() -%}
854	{{ "%-16s"\|format(serviceid) }} # {{ item.nodename }}
855	{% endfor -%}
856	"
857	list_normal_proxies="
858	{% for item in autogen_normal_proxies -%}
859	{{ "%-16s"\|format(item.serviceid) }} # {{ item.nodename }}
860	{% endfor -%}
861	"
862
863	{% if autogen_dhcp_interfaces -%}
864	captive_portal_interfaces="{{ autogen_dhcp_interfaces\|join(',') }}"
865	{% else %}
866	captive_portal_interfaces="dummy"
867	{% endif %}
868	externalif="{{ externalif\|default('vr0', true) }}"
869	masterip="{{ masterip }}"
870
871	{% if gateway and service_proxy_ileiden %}
872	defaultrouter="{{ gateway }}"
873	{% else %}
874	#defaultrouter="NOTSET"
875	{% endif %}
876
877	#
878	# Defined services
879	#
880	service_proxy_ileiden="{{ service_proxy_ileiden\|yesorno }}"
881	service_proxy_normal="{{ service_proxy_normal\|yesorno }}"
882	service_accesspoint="{{ service_accesspoint\|yesorno }}"
883	service_incoming_rdr="{{ service_incoming_rdr\|yesorno }}"
884	service_concentrator="{{ service_concentrator\|yesorno }}"
885
886	{% if service_proxy_ileiden %}
887	pf_rules="/etc/pf.hybrid.conf"
888	{% if service_concentrator %}
889	pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
890	{% else %}
891	pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
892	{% endif %}
893	pf_flags="$pf_flags -D publicnat=80,443"
894	lvrouted_flags="$lvrouted_flags -g"
895	{% elif service_proxy_normal or service_incoming_rdr %}
896	pf_rules="/etc/pf.hybrid.conf"
897	pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
898	pf_flags="$pf_flags -D publicnat=0"
899	lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
900	named_setfib="1"
901	tinyproxy_setfib="1"
902	dnsmasq_setfib="1"
903	sshd_setfib="1"
904	{% else %}
905	named_auto_forward_only="YES"
906	pf_rules="/etc/pf.node.conf"
907	pf_flags=""
908	lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
909	{% endif %}
910	{% if service_concentrator %}
911	# Do mind installing certificates is NOT done automatically for security reasons
912	openvpn_enable="YES"
913	openvpn_configfile="/usr/local/etc/openvpn/client.conf"
914	{% endif %}
915
916	{% if service_proxy_normal %}
917	tinyproxy_enable="yes"
918	{% else %}
919	pen_wrapper_enable="yes"
920	{% endif %}
921
922	{% if service_accesspoint %}
923	pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
924	{% endif %}
925
926	{% if board == "ALIX2" or board == "net4801" %}
927	#
928	# ''Fat'' configuration, board has 256MB RAM
929	#
930	dnsmasq_enable="NO"
931	named_enable="YES"
932	unbound_enable="YES"
933	{% if autogen_dhcp_interfaces -%}
934	dhcpd_enable="YES"
935	dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces\|join(' ') }}"
936	{% endif -%}
937	{% elif board == "apu1d" %}
938	#
939	# ''Fat'' configuration, board has 1024MB RAM
940	#
941	dnsmasq_enable="NO"
942	unbound_enable="YES"
943	{% if autogen_dhcp_interfaces -%}
944	dhcpd_enable="YES"
945	dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces\|join(' ') }}"
946	{% endif -%}
947	{% endif -%}
948	{% endif %}
949
950	#
951	# Script variables
952	#
953	attached_devices="{{ autogen_attached_devices\|join(' ') }}"
954	neighbours="{{ autogen_neighbours\|join(' ') }}"
955
956
957	#
958	# Interface definitions
959	#\n
960	""")
961
962	(addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
963	for iface, vlans in sorted(vlan_list.items()):
964	output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
965
966	# VLAN Parent interfaces not containing a configuration should be marked active explcitly.
967	for iface in sorted(vlan_list.keys()):
968	if not iface in addrs_list.keys():
969	output += "ifconfig_%s='up'\n" % iface
970
971	output += "\n"
972
973	# Bridge configuration:
974	if bridge_list.keys():
975	output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
976
977	for iface in bridge_list.keys():
978	output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
979
980	# Bridge member interfaces not containing a configuration should be marked active explcitly.
981	for _,members in bridge_list.items():
982	for iface in members:
983	if not iface in addrs_list.keys():
984	output += "ifconfig_%s='up'\n" % iface.replace('.','_')
985
986	output += "\n"
987
988	# Details like SSID
989	if extra_ouput:
990	output += extra_ouput.strip() + "\n"
991
992	# Print IP address which needs to be assigned over here
993	output += "\n"
994	for iface,addrs in sorted(addrs_list.iteritems()):
995	for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
996	output += "# %s \|\| %s \|\| %s\n" % (iface, addr, comment)
997
998	# Write DHCLIENT entry
999	if iface in dhclient_if and dhclient_if[iface]:
1000	output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1001	continue
1002
1003	# Make sure the external address is always first as this is needed in the
1004	# firewall setup
1005	addrs = sorted(
1006	[x for x in addrs if not '0.0.0.0' in x[0]],
1007	key=lambda x: x[0].split('.')[0],
1008	cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0)
1009	)
1010
1011	idx_offset = 0
1012	# Set MAC is required
1013	if flags_if[iface].has_key('ether'):
1014	output += "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1015	output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1016	idx_offset += 1
1017	elif iface in bridge_list:
1018	output += "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1019	idx_offset += 1
1020	else:
1021	output += "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1022
1023	for idx, addr in enumerate(addrs[1:]):
1024	output += "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1025
1026	output += "\n"
1027
1028	rc_conf_local_cache[datadump['autogen_item']] = output
1029	return output
1030
1031
1032
1033	def get_all_configs():
1034	""" Get dict with key 'host' with all configs present """
1035	configs = dict()
1036	for host in get_hostlist():
1037	datadump = get_yaml(host)
1038	configs[host] = datadump
1039	return configs
1040
1041
1042	def get_interface_keys(config, extra=False):
1043	""" Quick hack to get all interface keys, later stage convert this to a iterator """
1044	elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1045	if extra == False:
1046	return filter(lambda x: not "extra" in x, elems)
1047	else:
1048	return elems
1049
1050
1051	def get_used_ips(configs):
1052	""" Return array of all IPs used in config files"""
1053	ip_list = []
1054	for config in configs:
1055	ip_list.append(config['masterip'])
1056	if 'serviceid' in config:
1057	ip_list.append(config['serviceid'])
1058	for iface_key in get_interface_keys(config, True):
1059	l = config[iface_key]['ip']
1060	addr, mask = l.split('/')
1061	# Special case do not process
1062	if valid_addr(addr):
1063	ip_list.append(addr)
1064	else:
1065	logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1066	return sorted(ip_list)
1067
1068
1069
1070	def get_nameservers(max_servers=None):
1071	if nameservers_cache:
1072	return nameservers_cache[0:max_servers]
1073
1074	for host in get_hostlist():
1075	hostdump = get_yaml(host)
1076	if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1077	nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1078
1079	return nameservers_cache[0:max_servers]
1080
1081
1082	def get_neighbours(datadump):
1083	(addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1084
1085	(poel, errors) = make_relations()
1086	table = []
1087	for iface,addrs in sorted(addrs_list.iteritems()):
1088	if iface in ['lo0']:
1089	continue
1090
1091	for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1092	if not addr.startswith('172.'):
1093	# Avoid listing internet connections as pool
1094	continue
1095	for neighbour in poel[network(addr)]:
1096	if neighbour[0] != datadump['autogen_item']:
1097	table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1098	return table
1099
1100
1101	def get_attached_devices(datadump, url=False):
1102	table = []
1103	for iface_key in get_interface_keys(datadump, True):
1104	ifacedump = datadump[iface_key]
1105
1106	if not ifacedump.has_key('ns_ip'):
1107	continue
1108
1109	x_ip = ifacedump['ns_ip'].split('/')[0]
1110
1111	if 'mode' in ifacedump:
1112	x_mode = ifacedump['mode']
1113	else:
1114	x_mode = 'unknown'
1115
1116	if 'bridge_type' in ifacedump:
1117	device_type = ifacedump['bridge_type']
1118	else:
1119	device_type = 'Unknown'
1120
1121	table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1122	return table
1123
1124
1125	def generate_resolv_conf(datadump):
1126	""" Generate configuration file '/etc/resolv.conf' """
1127	# XXX: This should properly going to be an datastructure soon
1128	datadump['autogen_header'] = generate_header(datadump, "#")
1129	datadump['autogen_edge_nameservers'] = ''
1130
1131
1132	for masterip,realname in get_nameservers():
1133	datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1134
1135	return Template("""\
1136	{{ autogen_header }}
1137	search wleiden.net
1138
1139	# Try local (cache) first
1140	nameserver 127.0.0.1
1141
1142	{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1143	nameserver 8.8.8.8 # Google Public NameServer
1144	nameserver 64.6.64.6 # Verisign Public NameServer
1145	{% else -%}
1146	# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1147	{{ autogen_edge_nameservers }}
1148	{% endif -%}
1149	""").render(datadump)
1150
1151
1152
1153	def generate_ntp_conf(datadump):
1154	""" Generate configuration file '/etc/ntp.conf' """
1155	# XXX: This should properly going to be an datastructure soon
1156
1157	datadump['autogen_header'] = generate_header(datadump, "#")
1158	datadump['autogen_ntp_servers'] = ''
1159	for host in get_hostlist():
1160	hostdump = get_yaml(host)
1161	if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1162	datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1163
1164	return Template("""\
1165	{{ autogen_header }}
1166
1167	{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1168	# Machine hooked to internet.
1169	server 0.nl.pool.ntp.org iburst maxpoll 9
1170	server 1.nl.pool.ntp.org iburst maxpoll 9
1171	server 2.nl.pool.ntp.org iburst maxpoll 9
1172	server 3.nl.pool.ntp.org iburst maxpoll 9
1173	{% else -%}
1174	# Local Wireless Leiden NTP Servers.
1175	server 0.pool.ntp.wleiden.net iburst maxpoll 9
1176	server 1.pool.ntp.wleiden.net iburst maxpoll 9
1177	server 2.pool.ntp.wleiden.net iburst maxpoll 9
1178	server 3.pool.ntp.wleiden.net iburst maxpoll 9
1179
1180	# All the configured NTP servers
1181	{{ autogen_ntp_servers }}
1182	{% endif %}
1183
1184	# If a server loses sync with all upstream servers, NTP clients
1185	# no longer follow that server. The local clock can be configured
1186	# to provide a time source when this happens, but it should usually
1187	# be configured on just one server on a network. For more details see
1188	# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1189	# The use of Orphan Mode may be preferable.
1190	#
1191	server 127.127.1.0
1192	fudge 127.127.1.0 stratum 10
1193	""").render(datadump)
1194
1195
1196	def generate_pf_hybrid_conf_local(datadump):
1197	""" Generate configuration file '/etc/pf.hybrid.conf.local' """
1198	datadump['autogen_header'] = generate_header(datadump, "#")
1199	if datadump['service_incoming_rdr']:
1200	datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1201	return Template("""\
1202	{{ autogen_header }}
1203
1204	# Redirect some internal facing services outside (7)
1205	# INFO: {{ global_rdr_rules\|count }} global_rdr_rules active on this node.
1206	{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1207	rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"\|format(dest_ip) }} port {{ "%4s"\|format(dest_port) }} # {{ comment }}
1208	{% endfor -%}
1209	# INFO: {{ rdr_rules\|count }} node specific rdr_rules defined.
1210	{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1211	rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"\|format(dest_ip) }} port {{ "%4s"\|format(dest_port) }} # {{ comment }}
1212	{% endfor -%}
1213	""").render(datadump)
1214
1215	def generate_unbound_wleiden_conf(datadump):
1216	""" Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1217	datadump['autogen_header'] = generate_header(datadump, "#")
1218
1219	autogen_ips = []
1220	(addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1221	for iface,addrs in sorted(addrs_list.iteritems()):
1222	for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1223	if addr.startswith('172'):
1224	autogen_ips.append((addr.split('/')[0], comment))
1225	datadump['autogen_ips'] = autogen_ips
1226
1227	create_proxies_list()
1228	datadump['autogen_ileiden_proxies'] = ileiden_proxies
1229	return Template("""\
1230	{{ autogen_header }}
1231
1232	server:
1233	## Static definitions fail to start on systems with broken ue(4) interfaces
1234	{%- for ip,comment in autogen_ips %}
1235	# interface: {{ "%-16s"\|format(ip) }} # {{ comment }}
1236	{%- endfor %}
1237	## Enabling wildcard matching as work-around
1238	interface: 0.0.0.0
1239	interface: ::0
1240
1241	forward-zone:
1242	name: '.'
1243	{%- if service_proxy_ileiden %}
1244	forward-addr: 8.8.8.8 # Google DNS A
1245	forward-addr: 8.8.4.4 # Google DNS B
1246	forward-addr: 208.67.222.222 # OpenDNS DNS A
1247	forward-addr: 208.67.220.220 # OpenDNS DNS B
1248	{% else -%}
1249	{%- for serviceid,item in autogen_ileiden_proxies.iteritems() %}
1250	{%- if loop.index <= 5 %}
1251	forward-addr: {{ "%-16s"\|format(serviceid) }} # {{ item.nodename }}
1252	{%- endif %}
1253	{%- endfor %}
1254	{%- endif %}
1255	""").render(datadump)
1256
1257	def generate_motd(datadump):
1258	""" Generate configuration file '/etc/motd' """
1259	output = Template("""\
1260	FreeBSD run ``service motd onestart'' to make me look normal
1261
1262	WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1263	Loc: {{ location }}
1264
1265	Services:
1266	{% if board == "ALIX2" or board == "net4801" -%}
1267	{{" -"}} Core Node ({{ board }})
1268	{% else -%}
1269	{{" -"}} Hulp Node ({{ board }})
1270	{% endif -%}
1271	{% if service_proxy_normal -%}
1272	{{" -"}} Normal Proxy
1273	{% endif -%}
1274	{% if service_proxy_ileiden -%}
1275	{{" -"}} iLeiden Proxy
1276	{% endif -%}
1277	{% if service_incoming_rdr -%}
1278	{{" -"}} Incoming port redirects
1279	{% endif %}
1280	Interlinks:\n
1281	""").render(datadump)
1282
1283
1284	def make_table(table):
1285	if not table:
1286	return " - none\n"
1287	else:
1288	lines = ""
1289	col_width = [max(len(x) for x in col) for col in zip(*table)]
1290	for row in table:
1291	# replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1292	lines += " - " + " \|\| ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1293	return lines
1294
1295	(addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1296	table = []
1297	for iface,addrs in sorted(addrs_list.iteritems()):
1298	if iface in ['lo0']:
1299	continue
1300	for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1301	table.append((iface, addr, comment))
1302
1303	output += make_table(table)
1304	output += '\n'
1305	output += """\
1306	Attached devices:
1307	"""
1308	output += make_table(get_attached_devices(datadump, url=True))
1309	output += '\n'
1310	output += """\
1311	Available neighbours:
1312	"""
1313	output += make_table(get_neighbours(datadump))
1314
1315	return output
1316
1317
1318	def format_yaml_value(value):
1319	""" Get yaml value in right syntax for outputting """
1320	if isinstance(value,str):
1321	output = '"%s"' % value
1322	else:
1323	output = value
1324	return output
1325
1326
1327
1328	def format_wleiden_yaml(datadump):
1329	""" Special formatting to ensure it is editable"""
1330	output = "# Genesis config yaml style\n"
1331	output += "# vim:ts=2:et:sw=2:ai\n"
1332	output += "#\n"
1333	iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1334	for key in sorted(set(datadump.keys()) - set(iface_keys)):
1335	if key == 'rdr_rules':
1336	output += '%-10s:\n' % 'rdr_rules'
1337	for rdr_rule in datadump[key]:
1338	output += '- %s\n' % rdr_rule
1339	else:
1340	output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1341
1342	output += "\n\n"
1343
1344	# Format (key, required)
1345	key_order = (
1346	('comment', True),
1347	('parent', False),
1348	('ip', False),
1349	('ipv6', False),
1350	('ether', False),
1351	('desc', True),
1352	('sdesc', True),
1353	('mode', True),
1354	('type', True),
1355	('extra_type', False),
1356	('channel', False),
1357	('ssid', False),
1358	('wlan_mac', False),
1359	('dhcp', True),
1360	('dhcp_fixed', False),
1361	('compass', False),
1362	('distance', False),
1363	('ns_ip', False),
1364	('repeater_ip', False),
1365	('bullet2_ip', False),
1366	('ns_mac', False),
1367	('bullet2_mac', False),
1368	('ns_type', False),
1369	('bridge_type', False),
1370	('encrypted', False),
1371	('status', True),
1372	)
1373
1374	for iface_key in sorted(iface_keys):
1375	try:
1376	remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1377	if remainder:
1378	raise KeyError("invalid keys: %s" % remainder)
1379
1380	output += "%s:\n" % iface_key
1381	for key,required in key_order:
1382	if datadump[iface_key].has_key(key):
1383	output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1384	output += "\n\n"
1385	except Exception as exc:
1386	exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
1387	raise
1388
1389	return output
1390
1391
1392
1393	def generate_wleiden_yaml(datadump, header=True):
1394	""" Generate (petty) version of wleiden.yaml"""
1395	output = generate_header(datadump, "#") if header else ''
1396
1397	for key in datadump.keys():
1398	if key.startswith('autogen_'):
1399	del datadump[key]
1400	# Interface autogen cleanups
1401	elif type(datadump[key]) == dict:
1402	for key2 in datadump[key].keys():
1403	if key2.startswith('autogen_'):
1404	del datadump[key][key2]
1405
1406	output += format_wleiden_yaml(datadump)
1407	return output
1408
1409	def generate_nanostation_config(datadump, iface, ns_type):
1410	#TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1411	datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1412
1413	datadump.update(datadump['iface_%s' % iface])
1414
1415	return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1416
1417	def generate_yaml(datadump):
1418	return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1419
1420
1421
1422	def generate_config(node, config, datadump=None):
1423	""" Print configuration file 'config' of 'node' """
1424	output = ""
1425	try:
1426	# Load config file
1427	if datadump == None:
1428	datadump = get_yaml(node)
1429
1430	if config == 'wleiden.yaml':
1431	output += generate_wleiden_yaml(datadump)
1432	elif config == 'authorized_keys':
1433	f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1434	output += f.read()
1435	node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1436	# Fetch local keys if existing
1437	if os.path.exists(node_keys):
1438	output += open(node_keys, 'r').read()
1439	f.close()
1440	elif config == 'dnsmasq.conf':
1441	output += generate_dnsmasq_conf(datadump)
1442	elif config == 'dhcpd.conf':
1443	output += generate_dhcpd_conf(datadump)
1444	elif config == 'rc.conf.local':
1445	output += generate_rc_conf_local(datadump)
1446	elif config == 'resolv.conf':
1447	output += generate_resolv_conf(datadump)
1448	elif config == 'ntp.conf':
1449	output += generate_ntp_conf(datadump)
1450	elif config == 'motd':
1451	output += generate_motd(datadump)
1452	elif config == 'pf.hybrid.conf.local':
1453	output += generate_pf_hybrid_conf_local(datadump)
1454	elif config == 'unbound.wleiden.conf':
1455	output += generate_unbound_wleiden_conf(datadump)
1456	elif config.startswith('vr'):
1457	interface, ns_type = config.strip('.yaml').split('-')
1458	output += generate_nanostation_config(datadump, interface, ns_type)
1459	else:
1460	assert False, "Config not found!"
1461	except IOError, e:
1462	output += "[ERROR] Config file not found"
1463	return output
1464
1465
1466
1467	def generate_static(output_dir, logging=True):
1468	items = {'output_dir' : output_dir}
1469	for node in get_hostlist():
1470	items['node'] = node
1471	items['wdir'] = "%(output_dir)s/%(node)s" % items
1472	if not os.path.isdir(items['wdir']):
1473	os.makedirs(items['wdir'])
1474	datadump = get_yaml(node)
1475	f = open("%(wdir)s/index.html" % items, "w")
1476	f.write(generate_node_overview(items['node'], datadump))
1477	f.close()
1478	for config in files:
1479	items['config'] = config
1480	if logging: logger.info("## Generating %(node)s %(config)s" % items)
1481	f = open("%(wdir)s/%(config)s" % items, "w")
1482	f.write(generate_config(node, config, datadump))
1483	f.close()
1484
1485
1486
1487	def process_cgi_request(environ=os.environ):
1488	""" When calling from CGI """
1489	response_headers = []
1490	content_type = 'text/plain'
1491
1492	# Update repository if requested
1493	form = urlparse.parse_qs(environ['QUERY_STRING']) if environ.has_key('QUERY_STRING') else None
1494	if form and form.has_key("action") and "update" in form["action"]:
1495	refresh_rate = 5
1496	output = "[INFO] Updating subverion, please wait...\n"
1497	old_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1498	output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1499	output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1500	new_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
1501	if old_version != new_version:
1502	try:
1503	generate_static(CACHE_DIR, False)
1504	except:
1505	output += traceback.format_exc()
1506	refresh_rate = 120
1507	pass
1508	output += "[INFO] All done, redirecting in %s seconds" % refresh_rate
1509	response_headers += [
1510	('Refresh', '%s; url=.' % refresh_rate),
1511	]
1512	else:
1513	base_uri = environ['REQUEST_URI']
1514	uri = base_uri.strip('/').split('/')[1:]
1515
1516	output = "Template Holder"
1517	if base_uri.endswith('/create/network.kml'):
1518	content_type='application/vnd.google-earth.kml+xml'
1519	output = make_network_kml.make_graph()
1520	elif base_uri.endswith('/api/get/nodeplanner.json'):
1521	content_type='application/json'
1522	output = make_network_kml.make_nodeplanner_json()
1523	elif not uri:
1524	if is_text_request(environ):
1525	output = '\n'.join(get_hostlist())
1526	else:
1527	content_type = 'text/html'
1528	output = generate_title(get_hostlist())
1529	elif len(uri) == 1:
1530	if is_text_request(environ):
1531	output = generate_node(uri[0])
1532	else:
1533	content_type = 'text/html'
1534	output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1535	elif len(uri) == 2:
1536	output = generate_config(uri[0], uri[1])
1537	else:
1538	assert False, "Invalid option"
1539
1540	# Return response
1541	response_headers += [
1542	('Content-type', content_type),
1543	('Content-Length', str(len(output))),
1544	]
1545	return(response_headers, str(output))
1546
1547
1548	def make_dns(output_dir = 'dns', external = False):
1549	items = dict()
1550
1551	# hostname is key, IP is value
1552	wleiden_zone = defaultdict(list)
1553	wleiden_cname = dict()
1554
1555	pool = dict()
1556	for node in get_hostlist():
1557	datadump = get_yaml(node)
1558
1559	fqdn = datadump['nodename']
1560
1561	if datadump.has_key('rdr_host'):
1562	remote_target = datadump['rdr_host']
1563	elif datadump.has_key('remote_access') and datadump['remote_access']:
1564	remote_target = datadump['remote_access'].split(':')[0]
1565	else:
1566	remote_target = None
1567
1568	if remote_target:
1569	try:
1570	parseaddr(remote_target)
1571	wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1572	except (IndexError, ValueError):
1573	wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1574
1575
1576	wleiden_zone[fqdn].append((datadump['masterip'], True))
1577
1578	# Hacking to get proper DHCP IPs and hostnames
1579	for iface_key in get_interface_keys(datadump):
1580	iface_name = iface_key.replace('_','-')
1581	if 'ip' in datadump[iface_key]:
1582	(ip, cidr) = datadump[iface_key]['ip'].split('/')
1583	try:
1584	(dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1585	datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1586	dhcp_part = ".".join(ip.split('.')[0:3])
1587	if ip != datadump['masterip']:
1588	wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1589	for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1590	wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1591	except (AttributeError, ValueError, KeyError):
1592	# First push it into a pool, to indentify the counter-part later on
1593	addr = parseaddr(ip)
1594	cidr = int(cidr)
1595	addr = addr & ~((1 << (32 - cidr)) - 1)
1596	if pool.has_key(addr):
1597	pool[addr] += [(iface_name, fqdn, ip)]
1598	else:
1599	pool[addr] = [(iface_name, fqdn, ip)]
1600	continue
1601
1602
1603
1604	# WL uses an /29 to configure an interface. IP's are ordered like this:
1605	# MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1606
1607	sn = lambda x: re.sub(r'(?i)^cnode','',x)
1608
1609	# Automatic naming convention of interlinks namely 2 + remote.lower()
1610	for (key,value) in pool.iteritems():
1611	# Make sure they are sorted from low-ip to high-ip
1612	value = sorted(value, key=lambda x: parseaddr(x[2]))
1613
1614	if len(value) == 1:
1615	(iface_name, fqdn, ip) = value[0]
1616	wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1617
1618	# Device DNS names
1619	if 'cnode' in fqdn.lower():
1620	wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1621	wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1622
1623	elif len(value) == 2:
1624	(a_iface_name, a_fqdn, a_ip) = value[0]
1625	(b_iface_name, b_fqdn, b_ip) = value[1]
1626	wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1627	wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1628
1629	# Device DNS names
1630	if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1631	wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1632	wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1633	wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1634	wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1635	wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1636	wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1637
1638	else:
1639	pool_members = [k[1] for k in value]
1640	for item in value:
1641	(iface_name, fqdn, ip) = item
1642	wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1643
1644	# Include static DNS entries
1645	# XXX: Should they override the autogenerated results?
1646	# XXX: Convert input to yaml more useable.
1647	# Format:
1648	##; this is a comment
1649	## roomburgh=Roomburgh1
1650	## apkerk1.Vosko=172.17.176.8 ;this as well
1651	dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1652
1653	# Hack to allow special entries, for development
1654	wleiden_raw = {}
1655
1656	for line in dns_list:
1657	reverse = False
1658	k, items = line.items()[0]
1659	if type(items) == dict:
1660	if items.has_key('reverse'):
1661	reverse = items['reverse']
1662	items = items['a']
1663	else:
1664	items = items['cname']
1665	items = [items] if type(items) != list else items
1666	for item in items:
1667	if item.startswith('IN '):
1668	wleiden_raw[k] = item
1669	elif valid_addr(item):
1670	wleiden_zone[k].append((item, reverse))
1671	else:
1672	wleiden_cname[k] = item
1673
1674	# Hack to get dynamic pool listing
1675	def chunks(l, n):
1676	return [l[i:i+n] for i in range(0, len(l), n)]
1677
1678	ntp_servers = [x[0] for x in get_nameservers()]
1679	for id, chunk in enumerate(chunks(ntp_servers,(len(ntp_servers)/4))):
1680	for ntp_server in chunk:
1681	wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1682
1683	details = dict()
1684	# 24 updates a day allowed
1685	details['serial'] = time.strftime('%Y%m%d%H')
1686
1687	if external:
1688	dns_masters = ['ns1.vanderzwet.net', 'ns1.transip.net', 'ns2.transip.eu']
1689	else:
1690	dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1691
1692	details['master'] = dns_masters[0]
1693	details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1694
1695	dns_header = '''
1696	$TTL 3h
1697	%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 60s )
1698	; Serial, Refresh, Retry, Expire, Neg. cache TTL
1699
1700	%(ns_servers)s
1701	\n'''
1702
1703
1704	if not os.path.isdir(output_dir):
1705	os.makedirs(output_dir)
1706	details['zone'] = 'wleiden.net'
1707	f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1708	f.write(dns_header % details)
1709
1710	for host,items in wleiden_zone.iteritems():
1711	for ip,reverse in items:
1712	if ip not in ['0.0.0.0']:
1713	f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1714	for source,dest in wleiden_cname.iteritems():
1715	dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1716	f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1717	for source, dest in wleiden_raw.iteritems():
1718	f.write("%s.wleiden.net. %s\n" % (source, dest))
1719	f.close()
1720
1721	# Create whole bunch of specific sub arpa zones. To keep it compliant
1722	for s in range(16,32):
1723	details['zone'] = '%i.172.in-addr.arpa' % s
1724	f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1725	f.write(dns_header % details)
1726
1727	#XXX: Not effient, fix to proper data structure and do checks at other
1728	# stages
1729	for host,items in wleiden_zone.iteritems():
1730	for ip,reverse in items:
1731	if not reverse:
1732	continue
1733	if valid_addr(ip):
1734	if valid_addr(ip):
1735	if int(ip.split('.')[1]) == s:
1736	rev_ip = '.'.join(reversed(ip.split('.')))
1737	f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1738	f.close()
1739
1740
1741	def usage():
1742	print """Usage: %(prog)s <argument>
1743	Argument:
1744	\tcleanup = Cleanup all YAML files to specified format
1745	\tstandalone [port] = Run configurator webserver [8000]
1746	\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1747	\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1748	\tfull-export = Generate yaml export script for heatmap.
1749	\tstatic [outputdir] = Generate all config files and store on disk
1750	\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1751	\ttest <node> [<file>] = Receive output for certain node [all files].
1752	\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1753	\tlist <status> <items> = List systems which have certain status
1754	\tcreate network.kml = Create Network KML file for use in Google Earth
1755
1756	Arguments:
1757	\t<node> = NodeName (example: HybridRick)
1758	\t<file> = %(files)s
1759	\t<status> = all\|up\|down\|planned
1760	\t<items> = systems\|nodes\|proxies
1761
1762	NOTE FOR DEVELOPERS; you can test your changes like this:
1763	BEFORE any changes in this code:
1764	$ ./gformat.py static /tmp/pre
1765	AFTER the changes:
1766	$ ./gformat.py static /tmp/post
1767	VIEW differences and VERIFY all are OK:
1768	$ diff -urI 'Generated' -r /tmp/pre /tmp/post
1769	""" % { 'prog' : sys.argv[0], 'files' : '\|'.join(files) }
1770	exit(0)
1771
1772
1773	def is_text_request(environ=os.environ):
1774	""" Find out whether we are calling from the CLI or any text based CLI utility """
1775	if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1776	return True
1777
1778	if 'HTTP_USER_AGENT' in environ:
1779	return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1780	else:
1781	return False
1782
1783
1784	def switchFormat(setting):
1785	if setting:
1786	return "YES"
1787	else:
1788	return "NO"
1789
1790	def rlinput(prompt, prefill=''):
1791	import readline
1792	readline.set_startup_hook(lambda: readline.insert_text(prefill))
1793	try:
1794	return raw_input(prompt)
1795	finally:
1796	readline.set_startup_hook()
1797
1798	def fix_conflict(left, right, default='i'):
1799	while True:
1800	print "## %-30s \| %-30s" % (left, right)
1801	c = raw_input("## Solve Conflict (h for help) <l\|r\|e\|i\|> [%s]: " % default)
1802	if not c:
1803	c = default
1804
1805	if c in ['l','1']:
1806	return left
1807	elif c in ['r','2']:
1808	return right
1809	elif c in ['e', '3']:
1810	return rlinput("Edit: ", "%30s \| %30s" % (left, right))
1811	elif c in ['i', '4']:
1812	return None
1813	else:
1814	print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1815
1816
1817
1818	def print_cgi_response(response_headers, output):
1819	for header in response_headers:
1820	print "%s: %s" % header
1821	print
1822	print output
1823
1824
1825
1826	def main():
1827	"""Hard working sub"""
1828	# Allow easy hacking using the CLI
1829	if not os.environ.has_key('REQUEST_URI'):
1830	if len(sys.argv) < 2:
1831	usage()
1832
1833	if sys.argv[1] == "standalone":
1834	import SocketServer
1835	import CGIHTTPServer
1836	# Hop to the right working directory.
1837	os.chdir(os.path.dirname(__file__))
1838	try:
1839	PORT = int(sys.argv[2])
1840	except (IndexError,ValueError):
1841	PORT = 8000
1842
1843	class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1844	""" Serve this CGI from the root of the webserver """
1845	def is_cgi(self):
1846	if "favicon" in self.path:
1847	return False
1848
1849	self.cgi_info = (os.path.basename(__file__), self.path)
1850	self.path = ''
1851	return True
1852	handler = MyCGIHTTPRequestHandler
1853	SocketServer.TCPServer.allow_reuse_address = True
1854	httpd = SocketServer.TCPServer(("", PORT), handler)
1855	httpd.server_name = 'localhost'
1856	httpd.server_port = PORT
1857
1858	logger.info("serving at port %s", PORT)
1859	try:
1860	httpd.serve_forever()
1861	except KeyboardInterrupt:
1862	httpd.shutdown()
1863	logger.info("All done goodbye")
1864	elif sys.argv[1] == "test":
1865	# Basic argument validation
1866	try:
1867	node = sys.argv[2]
1868	except IndexError:
1869	print "Invalid argument"
1870	exit(1)
1871	except IOError as e:
1872	print e
1873	exit(1)
1874
1875	datadump = get_yaml(node)
1876
1877
1878	# Get files to generate
1879	gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1880
1881	# Actual config generation
1882	for config in gen_files:
1883	logger.info("## Generating %s %s", node, config)
1884	print generate_config(node, config, datadump)
1885	elif sys.argv[1] == "test-cgi":
1886	os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1887	os.environ['SCRIPT_NAME'] = __file__
1888	response_headers, output = process_cgi_request()
1889	print_cgi_response(response_headers, output)
1890	elif sys.argv[1] == "static":
1891	generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
1892	elif sys.argv[1] == "wind-export":
1893	items = dict()
1894	for node in get_hostlist():
1895	datadump = get_yaml(node)
1896	sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1897	VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1898	sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1899	VALUES (
1900	(SELECT id FROM users WHERE username = 'rvdzwet'),
1901	(SELECT id FROM nodes WHERE name = '%(nodename)s'),
1902	'Y');""" % datadump
1903	#for config in files:
1904	# items['config'] = config
1905	# print "## Generating %(node)s %(config)s" % items
1906	# f = open("%(wdir)s/%(config)s" % items, "w")
1907	# f.write(generate_config(node, config, datadump))
1908	# f.close()
1909	for node in get_hostlist():
1910	datadump = get_yaml(node)
1911	for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1912	ifacedump = datadump[iface_key]
1913	if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1914	ifacedump['nodename'] = datadump['nodename']
1915	if not ifacedump.has_key('channel') or not ifacedump['channel']:
1916	ifacedump['channel'] = 0
1917	sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1918	VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1919	'%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1920	elif sys.argv[1] == "nagios-export":
1921	try:
1922	heavy_load = (sys.argv[2] == "--heavy-load")
1923	except IndexError:
1924	heavy_load = False
1925
1926	hostgroup_details = {
1927	'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1928	'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1929	'walphen' : 'Stichting Wireless Alphen',
1930	'westeinder' : 'Westeinder Plassen',
1931	}
1932
1933	# Convert IP to Host
1934	ip2host = {'root' : 'root'}
1935	for host in get_hostlist():
1936	datadump = get_yaml(host)
1937	ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1938	for iface in get_interface_keys(datadump):
1939	if datadump[iface].has_key('autogen_gateway'):
1940	ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1941
1942	# Find dependency tree based on output of lvrouted.mytree of nearest node
1943	parents = defaultdict(list)
1944	stack = ['root']
1945	prev_depth = 0
1946	for line in open('lvrouted.mytree').readlines():
1947	depth = line.count('\t')
1948	ip = line.strip().split()[0]
1949
1950	if prev_depth < depth:
1951	try:
1952	parents[ip2host[ip]].append(ip2host[stack[-1]])
1953	except KeyError as e:
1954	print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1955	stack.append(ip)
1956	elif prev_depth > depth:
1957	stack = stack[:(depth - prev_depth)]
1958	elif prev_depth == depth:
1959	try:
1960	parents[ip2host[ip]].append(ip2host[stack[-1]])
1961	except KeyError as e:
1962	print >> stderr, "# Unable to find %s in configuration files" % e.args[0]
1963
1964
1965	prev_depth = depth
1966	# Observe that some nodes has themself as parent or multiple parents
1967	# for now take only the first parent, other behaviour is yet to be explained
1968
1969
1970
1971	params = {
1972	'check_interval' : 5 if heavy_load else 120,
1973	'retry_interval' : 1 if heavy_load else 10,
1974	'max_check_attempts' : 10 if heavy_load else 6,
1975	'notification_interval': 120 if heavy_load else 240,
1976	}
1977
1978	print '''\
1979	define host {
1980	name wleiden-node ; Default Node Template
1981	use generic-host ; Use the standard template as initial starting point
1982	check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1983	check_interval %(check_interval)s ; Actively check the host every 5 minutes
1984	retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1985	notification_interval %(notification_interval)s
1986	max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1987	check_command check-host-alive ; Default command to check FreeBSD hosts
1988	register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
1989	}
1990
1991	define service {
1992	name wleiden-service ; Default Service Template
1993	use generic-service ; Use the standard template as initial starting point
1994	check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
1995	check_interval %(check_interval)s ; Actively check the host every 5 minutes
1996	retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
1997	notification_interval %(notification_interval)s
1998	max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
1999	register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2000	}
2001
2002	# Please make sure to install:
2003	# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2004	#
2005	# Recompile net-mgmt/nagios-plugins to support check_snmp
2006	# make -C /usr/ports/net-mgmt/nagios-plugins
2007	#
2008	# Install net/bind-tools to allow v2/check_dns_wl to work:
2009	# pkg install bind-tools
2010	#
2011	define command{
2012	command_name check_snmp_disk
2013	command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2014	}
2015
2016	define command{
2017	command_name check_netsnmp_load
2018	command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2019	}
2020
2021	define command{
2022	command_name check_netsnmp_proc
2023	command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2024	}
2025
2026	define command{
2027	command_name check_by_ssh
2028	command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2029	}
2030
2031	define command{
2032	command_name check_dns_wl
2033	command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2034	}
2035
2036	define command{
2037	command_name check_snmp_uptime
2038	command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2039	}
2040
2041
2042	# TDB: dhcp leases
2043	# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2044
2045	# TDB: internet status
2046	# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2047
2048	# TDB: Advanced local passive checks
2049	# /usr/local/libexec/nagios/check_by_ssh
2050	''' % params
2051
2052	print '''\
2053	# Service Group, not displayed by default
2054	define hostgroup {
2055	hostgroup_name srv_hybrid
2056	alias All Hybrid Nodes
2057	register 0
2058	}
2059
2060	define service {
2061	use wleiden-service
2062	hostgroup_name srv_hybrid
2063	service_description SSH
2064	check_command check_ssh
2065	}
2066
2067	define service {
2068	use wleiden-service,service-pnp
2069	hostgroup_name srv_hybrid
2070	service_description HTTP
2071	check_command check_http
2072	}
2073
2074	define service {
2075	use wleiden-service
2076	hostgroup_name srv_hybrid
2077	service_description DNS
2078	check_command check_dns_wl!"www.wirelessleiden.nl"
2079	}
2080	'''
2081
2082	if heavy_load:
2083	print '''\
2084	define service {
2085	use wleiden-service
2086	hostgroup_name srv_hybrid
2087	service_description UPTIME
2088	check_command check_snmp_uptime
2089	}
2090
2091	#define service {
2092	# use wleiden-service
2093	# hostgroup_name srv_hybrid
2094	# service_description NTP
2095	# check_command check_ntp_peer
2096	#}
2097
2098	define service {
2099	use wleiden-service
2100	hostgroup_name srv_hybrid
2101	service_description LOAD
2102	check_command check_netsnmp_load
2103	}
2104
2105	define service {
2106	use wleiden-service
2107	hostgroup_name srv_hybrid
2108	service_description PROC
2109	check_command check_netsnmp_proc
2110	}
2111
2112	define service {
2113	use wleiden-service
2114	hostgroup_name srv_hybrid
2115	service_description DISK
2116	check_command check_snmp_disk
2117	}
2118	'''
2119	for node in get_hostlist():
2120	datadump = get_yaml(node)
2121	if not datadump['status'] == 'up':
2122	continue
2123	if not hostgroup_details.has_key(datadump['monitoring_group']):
2124	hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2125	print '''\
2126	define host {
2127	use wleiden-node,host-pnp
2128	contact_groups admins
2129	host_name %(autogen_fqdn)s
2130	address %(masterip)s
2131	hostgroups srv_hybrid,%(monitoring_group)s\
2132	''' % datadump
2133	if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2134	print '''\
2135	parents %(parents)s\
2136	''' % { 'parents' : parents[datadump['autogen_fqdn']][0] }
2137	print '''\
2138	}
2139	'''
2140
2141
2142	for name,alias in hostgroup_details.iteritems():
2143	print '''\
2144	define hostgroup {
2145	hostgroup_name %s
2146	alias %s
2147	} ''' % (name, alias)
2148
2149
2150	elif sys.argv[1] == "full-export":
2151	hosts = {}
2152	for node in get_hostlist():
2153	datadump = get_yaml(node)
2154	hosts[datadump['nodename']] = datadump
2155	print yaml.dump(hosts)
2156
2157	elif sys.argv[1] == "dns":
2158	make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2159	elif sys.argv[1] == "cleanup":
2160	# First generate all datadumps
2161	datadumps = dict()
2162	ssid_to_node = dict()
2163	for host in get_hostlist():
2164	logger.info("# Processing: %s", host)
2165	# Set some boring default values
2166	datadump = { 'board' : 'UNKNOWN' }
2167	datadump.update(get_yaml(host))
2168	datadumps[datadump['nodename']] = datadump
2169
2170	(poel, errors) = make_relations()
2171	print "\n".join(["# WARNING: %s" % x for x in errors])
2172
2173	for host,datadump in datadumps.iteritems():
2174	try:
2175	# Convert all yes and no to boolean values
2176	def fix_boolean(dump):
2177	for key in dump.keys():
2178	if type(dump[key]) == dict:
2179	dump[key] = fix_boolean(dump[key])
2180	elif str(dump[key]).lower() in ["yes", "true"]:
2181	dump[key] = True
2182	elif str(dump[key]).lower() in ["no", "false"]:
2183	# Compass richting no (Noord Oost) is valid input
2184	if key != "compass": dump[key] = False
2185	return dump
2186	datadump = fix_boolean(datadump)
2187
2188	if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2189	if not 'latitude' in datadump and not 'longitude' in datadump:
2190	datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2191	elif 'latitude' in datadump and 'longitude' in datadump:
2192	if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2193	datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2194	# TODO: Compare outcome of both coordinate systems and validate against each-other
2195
2196	if datadump['nodename'].startswith('Proxy'):
2197	datadump['nodename'] = datadump['nodename'].lower()
2198
2199	for iface_key in get_interface_keys(datadump):
2200	try:
2201	# All our normal wireless cards are normal APs now
2202	if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2203	datadump[iface_key]['mode'] = 'ap'
2204	# Wireless Leiden SSID have an consistent lowercase/uppercase
2205	if datadump[iface_key].has_key('ssid'):
2206	ssid = datadump[iface_key]['ssid']
2207	prefix = 'ap-WirelessLeiden-'
2208	if ssid.lower().startswith(prefix.lower()):
2209	datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2210	if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
2211	datadump[iface_key]['mode'] = 'autogen-FIXME'
2212	if not datadump[iface_key].has_key('comment'):
2213	datadump[iface_key]['comment'] = 'autogen-FIXME'
2214
2215	if datadump[iface_key].has_key('ns_mac'):
2216	datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2217
2218	if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
2219	datadump[iface_key] = datadump[iface_key]['desc']
2220
2221	# We are not using 802.11b anymore. OFDM is preferred over DSSS
2222	# due to better collision avoidance.
2223	if datadump[iface_key]['type'] == '11b':
2224	datadump[iface_key]['type'] = '11g'
2225
2226	# Setting 802.11g channels to de-facto standards, to avoid
2227	# un-detected sharing with other overlapping channels
2228	#
2229	# Technically we could also use channel 13 in NL, but this is not
2230	# recommended as foreign devices might not be able to select this
2231	# channel. Secondly using 1,5,9,13 instead is going to clash with
2232	# the de-facto usage of 1,6,11.
2233	#
2234	# See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2235	channels_at_2400Mhz = (1,6,11)
2236	if datadump[iface_key]['type'] == '11g' and datadump[iface_key].has_key('channel'):
2237	datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2238	if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2239	datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2240
2241	# Mandatory interface keys
2242	if not datadump[iface_key].has_key('status'):
2243	datadump[iface_key]['status'] = 'planned'
2244
2245	x = datadump[iface_key]['comment']
2246	datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2247
2248	# Fixing bridge_type if none is found
2249	if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2250	if not 'bridge_type' in datadump[iface_key]:
2251	datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2252
2253	# Making sure description works
2254	if datadump[iface_key].has_key('desc'):
2255	if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2256	del datadump[iface_key]['desc']
2257	else:
2258	print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
2259	response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2260	if response:
2261	datadump[iface_key]['comment'] = response
2262	del datadump[iface_key]['desc']
2263
2264	# Check DHCP configuration
2265	dhcp_type(datadump[iface_key])
2266
2267	# Set the compass value based on the angle between the poels
2268	if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2269	my_pool = poel[network(datadump[iface_key]['ip'])]
2270	remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2271	if remote_hosts:
2272	compass_target = remote_hosts[0]
2273	datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2274	# TODO: Compass wanted and actual direction might differ
2275
2276	# Monitoring Group default
2277	if not 'monitoring_group' in datadump:
2278	datadump['monitoring_group'] = 'wleiden'
2279
2280	except Exception as exc:
2281	exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
2282	raise
2283	store_yaml(datadump)
2284	except Exception as exc:
2285	exc.args = ("# Error while processing %s" % host,) + exc.args
2286	raise
2287	elif sys.argv[1] == "list":
2288	use_fqdn = False
2289	if len(sys.argv) < 4:
2290	usage()
2291	if not sys.argv[2] in ["up", "down", "planned", "all"]:
2292	usage()
2293	if not sys.argv[3] in ["nodes","proxies","systems"]:
2294	usage()
2295
2296	if len(sys.argv) > 4:
2297	if sys.argv[4] == "fqdn":
2298	use_fqdn = True
2299	else:
2300	usage()
2301
2302	for system in get_hostlist():
2303	datadump = get_yaml(system)
2304	if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2305	continue
2306
2307	output = datadump['autogen_fqdn'] if use_fqdn else system
2308	if sys.argv[2] == "all":
2309	print output
2310	elif datadump['status'] == sys.argv[2]:
2311	print output
2312	elif sys.argv[1] == "create":
2313	if sys.argv[2] == "network.kml":
2314	print make_network_kml.make_graph()
2315	elif sys.argv[2] == "host-ips.txt":
2316	for system in get_hostlist():
2317	datadump = get_yaml(system)
2318	ips = [datadump['masterip']]
2319	for ifkey in get_interface_keys(datadump):
2320	ips.append(datadump[ifkey]['ip'].split('/')[0])
2321	print system, ' '.join(ips)
2322	elif sys.argv[2] == "host-pos.txt":
2323	for system in get_hostlist():
2324	datadump = get_yaml(system)
2325	print system, datadump['rdnap_x'], datadump['rdnap_y']
2326	elif sys.argv[2] == 'ssh_config':
2327	print '''
2328	Host *.wleiden.net
2329	User root
2330
2331	Host 172.16..
2332	User root
2333	'''
2334	for system in get_hostlist():
2335	datadump = get_yaml(system)
2336	print '''\
2337	Host %s
2338	User root
2339
2340	Host %s
2341	User root
2342
2343	Host %s
2344	User root
2345
2346	Host %s
2347	User root
2348	''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower())
2349	else:
2350	usage()
2351	else:
2352	usage()
2353	else:
2354	# Do not enable debugging for config requests as it highly clutters the output
2355	if not is_text_request():
2356	cgitb.enable()
2357	response_headers, output = process_cgi_request()
2358	print_cgi_response(response_headers, output)
2359
2360	if __name__ == "__main__":
2361	main()

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: genesis/tools/gformat.py@ 14053

Download in other formats: