Context Navigation

source: genesis/tools/gformat.py@ 10212

Last change on this file since 10212 was 10209, checked in by richardvm, 13 years ago
adding proxies anywat
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 32.6 KB

Rev	Line
[8242]	1	#!/usr/bin/env python
	2	#
	3	# vim:ts=2:et:sw=2:ai
	4	# Wireless Leiden configuration generator, based on yaml files'
[9957]	5	#
	6	# XXX: This should be rewritten to make use of the ipaddr.py library.
	7	#
[10058]	8	# Sample apache configuration (mind the AcceptPathInfo!)
	9	# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
	10	# <Directory /usr/local/www/genesis>
	11	# Allow from all
	12	# AcceptPathInfo On
	13	# </Directory>
	14	#
[8242]	15	# Rick van der Zwet <info@rickvanderzwet.nl>
[9957]	16	#
[8622]	17
	18	# Hack to make the script directory is also threated as a module search path.
	19	import sys
	20	import os
[9286]	21	import re
[8622]	22	sys.path.append(os.path.dirname(__file__))
	23
[8242]	24	import cgi
[8267]	25	import cgitb
	26	import copy
[8242]	27	import glob
	28	import socket
	29	import string
	30	import subprocess
	31	import time
[8622]	32	import rdnap
[8584]	33	from pprint import pprint
[8575]	34	try:
	35	import yaml
	36	except ImportError, e:
	37	print e
	38	print "[ERROR] Please install the python-yaml or devel/py-yaml package"
	39	exit(1)
[8588]	40
	41	try:
	42	from yaml import CLoader as Loader
	43	from yaml import CDumper as Dumper
	44	except ImportError:
	45	from yaml import Loader, Dumper
	46
[10110]	47	from jinja2 import Template
	48
[9697]	49	import logging
	50	logging.basicConfig(format='# %(levelname)s: %(message)s' )
	51	logger = logging.getLogger()
	52	logger.setLevel(logging.DEBUG)
[8242]	53
[9283]	54
[8948]	55	if os.environ.has_key('CONFIGROOT'):
	56	NODE_DIR = os.environ['CONFIGROOT']
	57	else:
[9283]	58	NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
[8242]	59	__version__ = '$Id: gformat.py 10209 2012-03-17 19:24:37Z richardvm $'
	60
[8267]	61
[9283]	62	files = [
[8242]	63	'authorized_keys',
	64	'dnsmasq.conf',
	65	'rc.conf.local',
	66	'resolv.conf',
[10069]	67	'motd',
[10054]	68	'wleiden.yaml',
[8242]	69	]
	70
[8319]	71	# Global variables uses
[8323]	72	OK = 10
	73	DOWN = 20
	74	UNKNOWN = 90
[8257]	75
	76
[8267]	77	def get_proxylist():
	78	"""Get all available proxies proxyX sorting based on X number"""
[10041]	79	proxylist = sorted([os.path.basename(x) for x in glob.glob("%s/proxy*" % NODE_DIR)],
[8267]	80	key=lambda name: int(''.join([c for c in name if c in string.digits])),
	81	cmp=lambda x,y: x - y)
	82	return proxylist
	83
[10192]	84	def get_hybridlist():
	85	"""Get all available proxies hybridX sorting based on X number"""
[10193]	86	hybridlist = sorted([os.path.basename(x) for x in glob.glob("%s/hybrid*" % NODE_DIR)],
[10192]	87	key=lambda name: int(''.join([c for c in name if c in string.digits])),
	88	cmp=lambda x,y: x - y)
	89	return hybridlist
[8267]	90
	91
[8321]	92	def valid_addr(addr):
	93	""" Show which address is valid in which are not """
	94	return str(addr).startswith('172.')
	95
	96
[8267]	97	def get_nodelist():
	98	""" Get all available nodes - sorted """
[10041]	99	nodelist = sorted([os.path.basename(x) for x in glob.glob("%s/CNode*" % NODE_DIR)])
[8267]	100	return nodelist
	101
[8296]	102	def get_hostlist():
	103	""" Combined hosts and proxy list"""
[10192]	104	return get_nodelist() + get_proxylist() + get_hybridlist()
[8267]	105
[8588]	106	def angle_between_points(lat1,lat2,long1,long2):
[9283]	107	"""
[8588]	108	Return Angle in radians between two GPS coordinates
	109	See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
	110	"""
	111	dy = lat2 - lat1
	112	dx = math.cos(math.pi/180lat1)(long2 - long1)
	113	angle = math.atan2(dy,dx)
	114	return angle
[8267]	115
[8588]	116	def angle_to_cd(angle):
	117	""" Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
	118
	119	# For easy conversion get positive degree
	120	degrees = math.degrees(angle)
	121	if degrees < 0:
	122	360 - abs(degrees)
	123
	124	# Numbers can be confusing calculate from the 4 main directions
	125	p = 22.5
	126	if degrees < p:
	127	return "n"
[9283]	128	elif degrees < (90 - p):
[8588]	129	return "no"
[9283]	130	elif degrees < (90 + p):
[8588]	131	return "o"
[9283]	132	elif degrees < (180 - p):
[8588]	133	return "zo"
[9283]	134	elif degrees < (180 + p):
[8588]	135	return "z"
[9283]	136	elif degrees < (270 - p):
[8588]	137	return "zw"
[9283]	138	elif degrees < (270 + p):
[8588]	139	return "w"
[9283]	140	elif degrees < (360 - p):
[8588]	141	return "nw"
	142	else:
	143	return "n"
	144
	145
[8267]	146	def generate_title(nodelist):
[8257]	147	""" Main overview page """
[9283]	148	items = {'root' : "." }
[8267]	149	output = """
[8257]	150	<html>
	151	<head>
	152	<title>Wireless leiden Configurator - GFormat</title>
	153	<style type="text/css">
	154	th {background-color: #999999}
	155	tr:nth-child(odd) {background-color: #cccccc}
	156	tr:nth-child(even) {background-color: #ffffff}
	157	th, td {padding: 0.1em 1em}
	158	</style>
	159	</head>
	160	<body>
	161	<center>
[8259]	162	<form type="GET" action="%(root)s">
[8257]	163	<input type="hidden" name="action" value="update">
	164	<input type="submit" value="Update Configuration Database (SVN)">
	165	</form>
	166	<table>
	167	<caption><h3>Wireless Leiden Configurator</h3></caption>
	168	""" % items
[8242]	169
[8296]	170	for node in nodelist:
[8257]	171	items['node'] = node
[8267]	172	output += '<tr><td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items
[8257]	173	for config in files:
	174	items['config'] = config
[8267]	175	output += '<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items
	176	output += "</tr>"
	177	output += """
[8257]	178	</table>
	179	<hr />
	180	<em>%s</em>
	181	</center>
	182	</body>
	183	</html>
	184	""" % __version__
[8242]	185
[8267]	186	return output
[8257]	187
	188
[8267]	189
	190	def generate_node(node):
[8257]	191	""" Print overview of all files available for node """
[8267]	192	return "\n".join(files)
[8242]	193
[8257]	194
	195
[8242]	196	def generate_header(ctag="#"):
	197	return """\
[9283]	198	%(ctag)s
[8242]	199	%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
	200	%(ctag)s Generated at %(date)s by %(host)s
[9283]	201	%(ctag)s
[8242]	202	""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname() }
	203
[8257]	204
	205
[8242]	206	def parseaddr(s):
[8257]	207	""" Process IPv4 CIDR notation addr to a (binary) number """
[8242]	208	f = s.split('.')
	209	return (long(f[0]) << 24L) + \
	210	(long(f[1]) << 16L) + \
	211	(long(f[2]) << 8L) + \
	212	long(f[3])
	213
[8257]	214
	215
[8242]	216	def showaddr(a):
[8257]	217	""" Display IPv4 addr in (dotted) CIDR notation """
[8242]	218	return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
	219
[8257]	220
[8584]	221	def is_member(ip, mask, canidate):
	222	""" Return True if canidate is part of ip/mask block"""
	223	ip_addr = gformat.parseaddr(ip)
	224	ip_canidate = gformat.parseaddr(canidate)
	225	mask = int(mask)
	226	ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
	227	ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
	228	return ip_addr == ip_canidate
[8257]	229
[8584]	230
	231
[9283]	232
[8242]	233	def netmask2subnet(netmask):
[8257]	234	""" Given a 'netmask' return corresponding CIDR """
[8242]	235	return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
	236
[8257]	237
	238
[8242]	239	def generate_dnsmasq_conf(datadump):
[8257]	240	""" Generate configuration file '/usr/local/etc/dnsmasq.conf' """
[8242]	241	output = generate_header()
	242	output += """\
[9283]	243	# DHCP server options
[8242]	244	dhcp-authoritative
	245	dhcp-fqdn
	246	domain=dhcp.%(nodename_lower)s.%(domain)s
	247	domain-needed
	248	expand-hosts
[10120]	249	log-async=100
[8242]	250
	251	# Low memory footprint
	252	cache-size=10000
	253	\n""" % datadump
	254
	255	for iface_key in datadump['iface_keys']:
[8262]	256	if not datadump[iface_key].has_key('comment'):
	257	datadump[iface_key]['comment'] = None
	258	output += "## %(interface)s - %(desc)s - %(comment)s\n" % datadump[iface_key]
[8242]	259
	260	try:
[8257]	261	(dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
[8242]	262	(ip, netmask) = datadump[iface_key]['ip'].split('/')
	263	datadump[iface_key]['subnet'] = netmask2subnet(netmask)
[8262]	264	except (AttributeError, ValueError):
[8242]	265	output += "# not autoritive\n\n"
	266	continue
	267
	268	dhcp_part = ".".join(ip.split('.')[0:3])
	269	datadump[iface_key]['dhcp_start'] = dhcp_part + "." + dhcp_start
	270	datadump[iface_key]['dhcp_stop'] = dhcp_part + "." + dhcp_stop
	271	output += "dhcp-range=%(interface)s,%(dhcp_start)s,%(dhcp_stop)s,%(subnet)s,24h\n\n" % datadump[iface_key]
[9283]	272
[8242]	273	return output
	274
[8257]	275
	276
[8242]	277	def generate_rc_conf_local(datadump):
[8257]	278	""" Generate configuration file '/etc/rc.conf.local' """
[10112]	279	datadump['autogen_ileiden_enable'] = 'yes' if datadump['ileiden'] else 'no'
[10110]	280
[10112]	281	ileiden_proxies = []
	282	for proxy in get_proxylist():
	283	proxydump = get_yaml(proxy)
	284	if proxydump['ileiden']:
	285	ileiden_proxies.append(proxydump)
	286	datadump['autogen_ileiden_proxies'] = ','.join([x['masterip'] for x in ileiden_proxies])
	287	datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
	288
[8242]	289	output = generate_header("#");
[10110]	290	output += Template("""\
	291	hostname='{{ autogen_fqdn }}.{{ domain }}'
	292	location='{{ location }}'
	293	nodetype="{{ nodetype }}"
[9283]	294
[10110]	295	{% if tproxy -%}
[8242]	296	tproxy_enable='YES'
[10110]	297	tproxy_range='{{ tproxy }}'
	298	{% else -%}
	299	tproxy_enable='NO'
	300	{% endif -%}
[9283]	301
[10190]	302	{% if nodetype == "Proxy" %}
[10054]	303	#
	304	# PROXY Configuration
	305	#
[10188]	306	{% if internalroute -%}
[10054]	307	static_routes="wleiden"
[10110]	308	route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
[10188]	309	{% endif -%}
	310
[10110]	311	{% if gateway -%}
	312	defaultrouter="{{ gateway }}"
	313	{% else -%}
	314	#defaultrouter="NOTSET"
	315	{% endif -%}
[10053]	316
[10110]	317	internalif="{{ internalif }}"
[10054]	318
	319	# PROXY iLeiden Configuration
[10112]	320	lvrouted_enable="{{ autogen_ileiden_enable }}"
	321	lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
	322	ileiden_enable="{{ autogen_ileiden_enable }}"
	323	gateway_enable="{{ autogen_ileiden_enable }}"
	324	firewall_enable="{{ autogen_ileiden_enable }}"
[10054]	325	firewall_script="/etc/ipfw.sh"
[10112]	326	firewall_nat_enable="{{ autogen_ileiden_enable }}"
[10110]	327	{% endif -%}
[10054]	328
[10110]	329	{% if nodetype == "CNode" %}
[10112]	330	#
[10054]	331	# NODE iLeiden Configuration
[10112]	332	#
	333	# iLeiden Proxies {{ autogen_ileiden_proxies_names }}
	334	lvrouted_flags="-u -s s00p3rs3kr3t -m 28 -z {{ autogen_ileiden_proxies }}"
	335
[10054]	336	captive_portal_whitelist=""
[10110]	337	captive_portal_interfaces="{{ autogen_dhcp_interfaces }}"
	338	{% endif %}
[10054]	339
[10183]	340	{% if nodetype == "Hybrid" %}
	341	#
	342	# Hybrid Node
	343	#
[10190]	344	{% if internalroute -%}
	345	static_routes="wleiden"
	346	route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
	347	{% endif -%}
	348
	349	{% if gateway -%}
	350	defaultrouter="{{ gateway }}"
	351	{% else -%}
	352	#defaultrouter="NOTSET"
	353	{% endif -%}
	354
	355	internalif="{{ internalif }}"
	356	{% if vpnif -%}
	357	vpnif="{{ vpnif }}"\n
	358	{% endif -%}
	359
	360	lvrouted_enable="{{ autogen_ileiden_enable }}"
[10183]	361	lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
	362
[10190]	363	gateway_enable="{{ autogen_ileiden_enable }}"
	364
	365	ileiden_enable="{{ autogen_ileiden_enable }}"
	366	firewall_enable="{{ autogen_ileiden_enable }}"
	367	firewall_script="/etc/ipfw.sh"
	368
[10183]	369	captive_portal_whitelist=""
	370	captive_portal_interfaces="{{ autogen_dhcp_interfaces }}"
	371	{% endif %}
	372
[10110]	373	""").render(datadump)
	374
[8242]	375	# lo0 configuration:
	376	# - 172.32.255.1/32 is the proxy.wleiden.net deflector
[9283]	377	# - masterip is special as it needs to be assigned to at
[8242]	378	# least one interface, so if not used assign to lo0
[9808]	379	addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
[9283]	380	iface_map = {'lo0' : 'lo0'}
[8242]	381
[8297]	382	masterip_used = False
	383	for iface_key in datadump['iface_keys']:
	384	if datadump[iface_key]['ip'].startswith(datadump['masterip']):
	385	masterip_used = True
	386	break
[9283]	387	if not masterip_used:
[10108]	388	addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
[8297]	389
[8242]	390	for iface_key in datadump['iface_keys']:
	391	ifacedump = datadump[iface_key]
[10162]	392	ifname = ifacedump['autogen_ifname']
[8242]	393
	394	# Add interface IP to list
[9808]	395	item = (ifacedump['ip'], ifacedump['desc'])
[10162]	396	if addrs_list.has_key(ifname):
	397	addrs_list[ifname].append(item)
[8242]	398	else:
[10162]	399	addrs_list[ifname] = [item]
[8242]	400
	401	# Alias only needs IP assignment for now, this might change if we
	402	# are going to use virtual accesspoints
	403	if "alias" in iface_key:
	404	continue
	405
	406	# XXX: Might want to deduct type directly from interface name
	407	if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
	408	# Default to station (client) mode
	409	ifacedump['wlanmode'] = "sta"
[10166]	410	if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
[8242]	411	ifacedump['wlanmode'] = "ap"
	412	# Default to 802.11b mode
	413	ifacedump['mode'] = '11b'
	414	if ifacedump['type'] in ['11a', '11b' '11g']:
[9283]	415	ifacedump['mode'] = ifacedump['type']
[8242]	416
	417	if not ifacedump.has_key('channel'):
	418	if ifacedump['type'] == '11a':
	419	ifacedump['channel'] = 36
	420	else:
	421	ifacedump['channel'] = 1
	422
	423	# Allow special hacks at the back like wds and stuff
	424	if not ifacedump.has_key('extra'):
	425	ifacedump['extra'] = 'regdomain ETSI country NL'
	426
[10054]	427	output += "wlans_%(interface)s='%(autogen_ifname)s'\n" % ifacedump
	428	output += ("create_args_%(autogen_ifname)s='wlanmode %(wlanmode)s mode " +\
[8274]	429	"%(mode)s ssid %(ssid)s %(extra)s channel %(channel)s'\n") % ifacedump
[9283]	430
[8242]	431	elif ifacedump['type'] in ['ethernet', 'eth']:
	432	# No special config needed besides IP
	433	pass
	434	else:
	435	assert False, "Unknown type " + ifacedump['type']
	436
[9283]	437	# Print IP address which needs to be assigned over here
[8242]	438	output += "\n"
	439	for iface,addrs in sorted(addrs_list.iteritems()):
[10079]	440	for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
[9808]	441	output += "# %s \|\| %s \|\| %s\n" % (iface, addr, comment)
[10162]	442	output += "ipv4_addrs_%s='%s'\n\n" % (iface, " ".join([x[0] for x in addrs]))
[8242]	443
	444	return output
	445
[8257]	446
	447
[8242]	448	def get_yaml(item):
[8257]	449	""" Get configuration yaml for 'item'"""
[9284]	450	gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
[8242]	451
	452	f = open(gfile, 'r')
[8588]	453	datadump = yaml.load(f,Loader=Loader)
[10052]	454	datadump['autogen_iface_keys'] = get_interface_keys(datadump)
[10054]	455
	456	wlan_count=0
	457	for key in datadump['autogen_iface_keys']:
	458	if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
	459	datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
	460	wlan_count += 1
	461	else:
[10162]	462	datadump[key]['autogen_ifname'] = datadump[key]['interface'].split(':')[0]
[10054]	463
	464
	465	dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] if datadump[key]['dhcp'] != 'no']
	466	datadump['autogen_dhcp_interfaces'] = ' '.join(dhcp_interfaces)
[10053]	467	datadump['autogen_item'] = item
[10054]	468	datadump['autogen_fqdn'] = get_fqdn(datadump)
[8242]	469	f.close()
	470
	471	return datadump
	472
[10074]	473	def store_yaml(datadump, header=False):
[8588]	474	""" Store configuration yaml for 'item'"""
[10053]	475	item = datadump['autogen_item']
[9284]	476	gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
[8257]	477
[8588]	478	f = open(gfile, 'w')
[10067]	479	f.write(generate_wleiden_yaml(datadump, header))
[8588]	480	f.close()
[8257]	481
[8588]	482
	483
[8317]	484	def get_all_configs():
	485	""" Get dict with key 'host' with all configs present """
	486	configs = dict()
	487	for host in get_hostlist():
	488	datadump = get_yaml(host)
	489	configs[host] = datadump
	490	return configs
	491
	492
[8319]	493	def get_interface_keys(config):
	494	""" Quick hack to get all interface keys, later stage convert this to a iterator """
[10054]	495	return sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
[8317]	496
[8319]	497
[8317]	498	def get_used_ips(configs):
	499	""" Return array of all IPs used in config files"""
	500	ip_list = []
[8319]	501	for config in configs:
[8317]	502	ip_list.append(config['masterip'])
[8319]	503	for iface_key in get_interface_keys(config):
[8317]	504	l = config[iface_key]['ip']
	505	addr, mask = l.split('/')
	506	# Special case do not process
[8332]	507	if valid_addr(addr):
	508	ip_list.append(addr)
	509	else:
[9728]	510	logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
[8317]	511	return sorted(ip_list)
	512
	513
	514
[8242]	515	def generate_resolv_conf(datadump):
[8257]	516	""" Generate configuration file '/etc/resolv.conf' """
[8242]	517	output = generate_header("#");
	518	output += """\
	519	search wleiden.net
[10053]	520	"""
[10197]	521	if datadump['nodetype'] == 'Proxy':
[10053]	522	output += """\
[10209]	523	# Try local (cache) first
	524	nameserver 127.0.0.1
[10053]	525	nameserver 8.8.8.8 # Google Public NameServer
	526	nameserver 8.8.4.4 # Google Public NameServer
	527	"""
[10197]	528	elif datadump['nodetype'] == 'Hybrid':
[10209]	529	for proxy in get_proxylist():
	530	proxy_ip = get_yaml(proxy)['masterip']
	531	output += "nameserver %-15s # %s\n" % (proxy_ip, proxy)
[10197]	532	output += """\
	533	nameserver 8.8.8.8 # Google Public NameServer
	534	nameserver 8.8.4.4 # Google Public NameServer
	535	"""
[10053]	536	else:
	537	output += """\
[10209]	538	# Try local (cache) first
	539	nameserver 127.0.0.1
	540
[9283]	541	# Proxies are recursive nameservers
[8242]	542	# needs to be in resolv.conf for dnsmasq as well
	543	""" % datadump
[10053]	544	for proxy in get_proxylist():
	545	proxy_ip = get_yaml(proxy)['masterip']
	546	output += "nameserver %-15s # %s\n" % (proxy_ip, proxy)
[9283]	547
[8242]	548	return output
	549
[10069]	550	def generate_motd(datadump):
	551	""" Generate configuration file '/etc/motd' """
	552	output = """\
	553	FreeBSD 9.0-RELEASE (kernel.wleiden) #0 r230587: Sun Jan 29 17:09:57 CET 2012
[8242]	554
[10069]	555	WWW: %(autogen_fqdn)s.wleiden.net - http://www.wirelessleiden.nl
[10113]	556	Loc: %(location)s
[8257]	557
[10069]	558	Interlinks:
	559	""" % datadump
	560
	561	# XXX: This is a hacky way to get the required data
	562	for line in generate_rc_conf_local(datadump).split('\n'):
	563	if '\|\|' in line and not line[1:].split()[0] in ['lo0', 'ath0'] :
	564	output += " - %s \n" % line[1:]
	565	output += """\
	566	Attached bridges:
	567	"""
	568	for iface_key in datadump['autogen_iface_keys']:
	569	ifacedump = datadump[iface_key]
	570	if ifacedump.has_key('ns_ip'):
	571	output += " - %(interface)s \|\| %(mode)s \|\| %(ns_ip)s\n" % ifacedump
	572
	573	return output
	574
	575
[8267]	576	def format_yaml_value(value):
	577	""" Get yaml value in right syntax for outputting """
	578	if isinstance(value,str):
[10049]	579	output = '"%s"' % value
[8267]	580	else:
	581	output = value
[9283]	582	return output
[8267]	583
	584
	585
	586	def format_wleiden_yaml(datadump):
[8242]	587	""" Special formatting to ensure it is editable"""
[9283]	588	output = "# Genesis config yaml style\n"
[8262]	589	output += "# vim:ts=2:et:sw=2:ai\n"
[8242]	590	output += "#\n"
	591	iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
	592	for key in sorted(set(datadump.keys()) - set(iface_keys)):
[8267]	593	output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
[9283]	594
[8242]	595	output += "\n\n"
[9283]	596
[8272]	597	key_order = [ 'comment', 'interface', 'ip', 'desc', 'sdesc', 'mode', 'type',
	598	'extra_type', 'channel', 'ssid', 'dhcp' ]
	599
[8242]	600	for iface_key in sorted(iface_keys):
	601	output += "%s:\n" % iface_key
[8272]	602	for key in key_order + list(sorted(set(datadump[iface_key].keys()) - set(key_order))):
	603	if datadump[iface_key].has_key(key):
[9283]	604	output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
[8242]	605	output += "\n\n"
	606
	607	return output
	608
	609
[8257]	610
[10067]	611	def generate_wleiden_yaml(datadump, header=True):
[8267]	612	""" Generate (petty) version of wleiden.yaml"""
[10053]	613	for key in datadump.keys():
	614	if key.startswith('autogen_'):
	615	del datadump[key]
[10054]	616	# Interface autogen cleanups
	617	elif type(datadump[key]) == dict:
	618	for key2 in datadump[key].keys():
	619	if key2.startswith('autogen_'):
	620	del datadump[key][key2]
	621
[10067]	622	output = generate_header("#") if header else ''
[8267]	623	output += format_wleiden_yaml(datadump)
	624	return output
	625
	626
[8588]	627	def generate_yaml(datadump):
	628	return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
[8267]	629
[8588]	630
[9283]	631
[8298]	632	def generate_config(node, config, datadump=None):
[8257]	633	""" Print configuration file 'config' of 'node' """
[8267]	634	output = ""
[8242]	635	try:
	636	# Load config file
[8298]	637	if datadump == None:
	638	datadump = get_yaml(node)
[9283]	639
[8267]	640	# Preformat certain needed variables for formatting and push those into special object
	641	datadump_extra = copy.deepcopy(datadump)
	642	if not datadump_extra.has_key('domain'):
	643	datadump_extra['domain'] = 'wleiden.net'
	644	datadump_extra['nodename_lower'] = datadump_extra['nodename'].lower()
	645	datadump_extra['iface_keys'] = sorted([elem for elem in datadump.keys() if elem.startswith('iface_')])
	646
[8242]	647	if config == 'wleiden.yaml':
[8267]	648	output += generate_wleiden_yaml(datadump)
	649	elif config == 'authorized_keys':
[10051]	650	f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
[8267]	651	output += f.read()
[8242]	652	f.close()
	653	elif config == 'dnsmasq.conf':
[8267]	654	output += generate_dnsmasq_conf(datadump_extra)
[8242]	655	elif config == 'rc.conf.local':
[8267]	656	output += generate_rc_conf_local(datadump_extra)
[8242]	657	elif config == 'resolv.conf':
[8267]	658	output += generate_resolv_conf(datadump_extra)
[10069]	659	elif config == 'motd':
	660	output += generate_motd(datadump_extra)
[8242]	661	else:
[9283]	662	assert False, "Config not found!"
[8242]	663	except IOError, e:
[8267]	664	output += "[ERROR] Config file not found"
	665	return output
[8242]	666
	667
[8257]	668
[8258]	669	def process_cgi_request():
	670	""" When calling from CGI """
	671	# Update repository if requested
	672	form = cgi.FieldStorage()
	673	if form.getvalue("action") == "update":
[8259]	674	print "Refresh: 5; url=."
[8258]	675	print "Content-type:text/plain\r\n\r\n",
	676	print "[INFO] Updating subverion, please wait..."
[10143]	677	print subprocess.Popen(['svn', 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0],
[10071]	678	print subprocess.Popen(['svn', 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0],
[8258]	679	print "[INFO] All done, redirecting in 5 seconds"
	680	sys.exit(0)
[9283]	681
	682
[8258]	683	uri = os.environ['PATH_INFO'].strip('/').split('/')
[8267]	684	output = ""
[8258]	685	if not uri[0]:
[10070]	686	if is_text_request():
[10060]	687	output += "Content-type:text/plain\r\n\r\n"
	688	output += '\n'.join(get_hostlist())
	689	else:
	690	output += "Content-type:text/html\r\n\r\n"
	691	output += generate_title(get_hostlist())
[8258]	692	elif len(uri) == 1:
[8267]	693	output += "Content-type:text/plain\r\n\r\n"
	694	output += generate_node(uri[0])
[8258]	695	elif len(uri) == 2:
[8267]	696	output += "Content-type:text/plain\r\n\r\n"
	697	output += generate_config(uri[0], uri[1])
[8258]	698	else:
	699	assert False, "Invalid option"
[8267]	700	print output
[8242]	701
[8588]	702	def get_fqdn(datadump):
	703	# Proxy naming convention is special
	704	if datadump['nodetype'] == 'Proxy':
	705	fqdn = datadump['nodename']
	706	else:
	707	# By default the full name is listed and also a shortname CNAME for easy use.
	708	fqdn = datadump['nodetype'] + datadump['nodename']
	709	return(fqdn)
[8259]	710
[9283]	711
	712
[9284]	713	def make_dns(output_dir = 'dns'):
[8588]	714	items = dict()
[8598]	715
[8588]	716	# hostname is key, IP is value
	717	wleiden_zone = dict()
	718	wleiden_cname = dict()
[8598]	719
[8588]	720	pool = dict()
	721	for node in get_hostlist():
[9697]	722	logger.info("Processing host %s", node)
[8588]	723	datadump = get_yaml(node)
[9283]	724
[8588]	725	# Proxy naming convention is special
	726	fqdn = get_fqdn(datadump)
	727	if datadump['nodetype'] == 'CNode':
	728	wleiden_cname[datadump['nodename']] = fqdn
	729
	730	wleiden_zone[fqdn] = datadump['masterip']
	731
[8598]	732	# Hacking to get proper DHCP IPs and hostnames
[8588]	733	for iface_key in get_interface_keys(datadump):
[8598]	734	iface_name = datadump[iface_key]['interface'].replace(':',"-alias-")
[8588]	735	(ip, netmask) = datadump[iface_key]['ip'].split('/')
	736	try:
	737	(dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
	738	datadump[iface_key]['subnet'] = netmask2subnet(netmask)
	739	dhcp_part = ".".join(ip.split('.')[0:3])
	740	if ip != datadump['masterip']:
	741	wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)] = ip
	742	for i in range(int(dhcp_start), int(dhcp_stop) + 1):
	743	wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)] = "%s.%s" % (dhcp_part, i)
	744	except (AttributeError, ValueError):
	745	# First push it into a pool, to indentify the counter-part later on
	746	addr = parseaddr(ip)
	747	netmask = int(netmask)
	748	addr = addr & ~((1 << (32 - netmask)) - 1)
[9283]	749	if pool.has_key(addr):
[8588]	750	pool[addr] += [(iface_name, fqdn, ip)]
[9283]	751	else:
[8588]	752	pool[addr] = [(iface_name, fqdn, ip)]
	753	continue
	754
[9286]	755
	756	def pool_to_name(node, pool_members):
	757	"""Convert the joined name to a usable pool name"""
	758
	759	# Get rid of the own entry
	760	pool_members = list(set(pool_members) - set([fqdn]))
	761
	762	target = oldname = ''
	763	for node in sorted(pool_members):
	764	(name, number) = re.match('^([A-Za-z]+)([0-9]*)$',node).group(1,2)
	765	target += "-" + number if name == oldname else "-" + node if target else node
	766	oldname = name
	767
	768	return target
	769
	770
[9957]	771	# WL uses an /29 to configure an interface. IP's are ordered like this:
[9958]	772	# MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
[9957]	773
	774	sn = lambda x: re.sub(r'(?i)^cnode','',x)
	775
[8598]	776	# Automatic naming convention of interlinks namely 2 + remote.lower()
[8588]	777	for (key,value) in pool.iteritems():
[9958]	778	# Make sure they are sorted from low-ip to high-ip
	779	value = sorted(value, key=lambda x: parseaddr(x[2]))
	780
[8588]	781	if len(value) == 1:
	782	(iface_name, fqdn, ip) = value[0]
	783	wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)] = ip
[9957]	784
	785	# Device DNS names
	786	if 'cnode' in fqdn.lower():
	787	wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)] = showaddr(parseaddr(ip) + 1)
	788	wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % (iface_name, fqdn)
	789
[8588]	790	elif len(value) == 2:
	791	(a_iface_name, a_fqdn, a_ip) = value[0]
	792	(b_iface_name, b_fqdn, b_ip) = value[1]
	793	wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)] = a_ip
	794	wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)] = b_ip
[9957]	795
	796	# Device DNS names
	797	if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
	798	wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)] = showaddr(parseaddr(a_ip) + 1)
[9958]	799	wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)] = showaddr(parseaddr(b_ip) - 1)
[9957]	800	wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
	801	wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
	802	wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
	803	wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
	804
[8588]	805	else:
	806	pool_members = [k[1] for k in value]
	807	for item in value:
[9283]	808	(iface_name, fqdn, ip) = item
[9286]	809	pool_name = "2pool-" + showaddr(key).replace('.','-') + "-" + pool_to_name(fqdn,pool_members)
[8588]	810	wleiden_zone["%s.%s" % (pool_name, fqdn)] = ip
[8598]	811
	812	# Include static DNS entries
	813	# XXX: Should they override the autogenerated results?
	814	# XXX: Convert input to yaml more useable.
	815	# Format:
	816	##; this is a comment
	817	## roomburgh=CNodeRoomburgh1
	818	## apkerk1.CNodeVosko=172.17.176.8 ;this as well
[9284]	819	dns = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
[9938]	820
	821	# Hack to allow special entries, for development
	822	wleiden_raw = dns['raw']
	823	del dns['raw']
	824
[8622]	825	for comment, block in dns.iteritems():
	826	for k,v in block.iteritems():
[8598]	827	if valid_addr(v):
	828	wleiden_zone[k] = v
	829	else:
	830	wleiden_cname[k] = v
[9283]	831
[8598]	832	details = dict()
	833	# 24 updates a day allowed
	834	details['serial'] = time.strftime('%Y%m%d%H')
	835
	836	dns_header = '''
	837	$TTL 3h
	838	%(zone)s. SOA sunny.wleiden.net. beheer.lijst.wirelessleiden.nl. ( %(serial)s 1d 12h 1w 3h )
	839	; Serial, Refresh, Retry, Expire, Neg. cache TTL
	840
	841	NS sunny.wleiden.net.
	842	\n'''
	843
[9283]	844
[8598]	845	if not os.path.isdir('dns'):
	846	os.makedirs('dns')
	847	details['zone'] = 'wleiden.net'
[9284]	848	f = open(os.path.join(output_dir,"db." + details['zone']), "w")
[8598]	849	f.write(dns_header % details)
	850
[8588]	851	for host,ip in wleiden_zone.iteritems():
[8598]	852	if valid_addr(ip):
[9283]	853	f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
[8588]	854	for source,dest in wleiden_cname.iteritems():
[8636]	855	f.write("%s.wleiden.net. IN CNAME %s.wleiden.net.\n" % (source.lower(), dest.lower()))
[9938]	856	for source, dest in wleiden_raw.iteritems():
	857	f.write("%s.wleiden.net. %s\n" % (source, dest))
[8588]	858	f.close()
[9283]	859
[8598]	860	# Create whole bunch of specific sub arpa zones. To keep it compliant
	861	for s in range(16,32):
	862	details['zone'] = '%i.172.in-addr.arpa' % s
[9284]	863	f = open(os.path.join(output_dir,"db." + details['zone']), "w")
[8598]	864	f.write(dns_header % details)
[8588]	865
[8598]	866	#XXX: Not effient, fix to proper data structure and do checks at other
	867	# stages
	868	for host,ip in wleiden_zone.iteritems():
	869	if valid_addr(ip):
	870	if int(ip.split('.')[1]) == s:
	871	rev_ip = '.'.join(reversed(ip.split('.')))
[9283]	872	f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
[8598]	873	f.close()
[8588]	874
[8598]	875
[8259]	876	def usage():
[8598]	877	print """Usage: %s <standalone [port] \|test [test arguments]\|static\|dns>
[8259]	878	Examples:
[9284]	879	\tdns [outputdir] = Generate BIND compliant zone files in dns.
[8259]	880	\tstandalone = Run configurator webserver [default port=8000]
[9589]	881	\twind-export = Generate SQL import scripts for WIND database
	882	\tfull-export = Generate yaml export script for heatmap.
[8296]	883	\tstatic = Generate all config files and store on disk
	884	\t with format ./static/%%NODE%%/%%FILE%%
[9283]	885	\ttest CNodeRick dnsmasq.conf = Receive output of CGI script
[8259]	886	\t for arguments CNodeRick/dnsmasq.conf
[9971]	887	\tlist <nodes\|proxies> = List systems which marked up.
[8259]	888	"""
	889	exit(0)
	890
	891
[10070]	892	def is_text_request():
[10107]	893	""" Find out whether we are calling from the CLI or any text based CLI utility """
	894	try:
	895	return os.environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
	896	except KeyError:
	897	return True
[8259]	898
[8267]	899	def main():
	900	"""Hard working sub"""
	901	# Allow easy hacking using the CLI
	902	if not os.environ.has_key('PATH_INFO'):
	903	if len(sys.argv) < 2:
	904	usage()
[9283]	905
[8267]	906	if sys.argv[1] == "standalone":
	907	import SocketServer
	908	import CGIHTTPServer
[10105]	909	# Hop to the right working directory.
	910	os.chdir(os.path.dirname(__file__))
[8267]	911	try:
	912	PORT = int(sys.argv[2])
	913	except (IndexError,ValueError):
	914	PORT = 8000
[9283]	915
[8267]	916	class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
	917	""" Serve this CGI from the root of the webserver """
	918	def is_cgi(self):
	919	if "favicon" in self.path:
	920	return False
[9283]	921
[8267]	922	self.cgi_info = (__file__, self.path)
	923	self.path = ''
	924	return True
	925	handler = MyCGIHTTPRequestHandler
[9807]	926	SocketServer.TCPServer.allow_reuse_address = True
[8267]	927	httpd = SocketServer.TCPServer(("", PORT), handler)
	928	httpd.server_name = 'localhost'
	929	httpd.server_port = PORT
[9283]	930
[9728]	931	logger.info("serving at port %s", PORT)
[8860]	932	try:
	933	httpd.serve_forever()
	934	except KeyboardInterrupt:
	935	httpd.shutdown()
[9728]	936	logger.info("All done goodbye")
[8267]	937	elif sys.argv[1] == "test":
	938	os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
	939	os.environ['SCRIPT_NAME'] = __file__
	940	process_cgi_request()
[10107]	941	elif sys.argv[1] == "unit-test":
	942	os.environ['SCRIPT_NAME'] = __file__
	943	for host in get_hostlist():
	944	for outfile in files:
	945	os.environ['PATH_INFO'] = "/".join([host,outfile])
	946	try:
	947	process_cgi_request()
	948	except Exception:
	949	print "# ERROR: %s" % os.environ['PATH_INFO']
	950	raise
	951
	952
[8296]	953	elif sys.argv[1] == "static":
	954	items = dict()
	955	for node in get_hostlist():
	956	items['node'] = node
	957	items['wdir'] = "./static/%(node)s" % items
	958	if not os.path.isdir(items['wdir']):
	959	os.makedirs(items['wdir'])
[8298]	960	datadump = get_yaml(node)
[8296]	961	for config in files:
	962	items['config'] = config
[9728]	963	logger.info("## Generating %(node)s %(config)s" % items)
[8296]	964	f = open("%(wdir)s/%(config)s" % items, "w")
[8298]	965	f.write(generate_config(node, config, datadump))
[8296]	966	f.close()
[9514]	967	elif sys.argv[1] == "wind-export":
	968	items = dict()
	969	for node in get_hostlist():
	970	datadump = get_yaml(node)
	971	sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
	972	VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
	973	sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
	974	VALUES (
	975	(SELECT id FROM users WHERE username = 'rvdzwet'),
	976	(SELECT id FROM nodes WHERE name = '%(nodename)s'),
	977	'Y');""" % datadump
	978	#for config in files:
	979	# items['config'] = config
	980	# print "## Generating %(node)s %(config)s" % items
	981	# f = open("%(wdir)s/%(config)s" % items, "w")
	982	# f.write(generate_config(node, config, datadump))
	983	# f.close()
	984	for node in get_hostlist():
	985	datadump = get_yaml(node)
	986	for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
	987	ifacedump = datadump[iface_key]
	988	if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
	989	ifacedump['nodename'] = datadump['nodename']
	990	if not ifacedump.has_key('channel') or not ifacedump['channel']:
	991	ifacedump['channel'] = 0
	992	sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
	993	VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
	994	'%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
[9589]	995	elif sys.argv[1] == "full-export":
	996	hosts = {}
	997	for node in get_hostlist():
	998	datadump = get_yaml(node)
	999	hosts[datadump['nodename']] = datadump
	1000	print yaml.dump(hosts)
	1001
[8584]	1002	elif sys.argv[1] == "dns":
[9285]	1003	make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns')
[9283]	1004	elif sys.argv[1] == "cleanup":
[8588]	1005	# First generate all datadumps
	1006	datadumps = dict()
	1007	for host in get_hostlist():
[9728]	1008	logger.info("# Processing: %s", host)
[8588]	1009	datadump = get_yaml(host)
	1010	datadumps[get_fqdn(datadump)] = datadump
[9283]	1011
[10156]	1012	for host,datadump in datadumps.iteritems():
[8622]	1013	datadump['latitude'], datadump['longitude'] = rdnap.rd2etrs(datadump['rdnap_x'], datadump['rdnap_y'])
[10156]	1014	for iface_key in datadump['autogen_iface_keys']:
	1015	# Wireless Leiden SSID have an consistent lowercase/uppercase
	1016	if datadump[iface_key].has_key('ssid'):
	1017	ssid = datadump[iface_key]['ssid']
	1018	prefix = 'ap-WirelessLeiden-'
	1019	if ssid.lower().startswith(prefix.lower()):
	1020	datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
[10162]	1021	if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
	1022	datadump[iface_key]['mode'] = 'autogen-FIXME'
	1023	if not datadump[iface_key].has_key('desc'):
	1024	datadump[iface_key]['desc'] = 'autogen-FIXME'
[10074]	1025	store_yaml(datadump)
[9971]	1026	elif sys.argv[1] == "list":
	1027	if sys.argv[2] == "nodes":
	1028	systems = get_nodelist()
	1029	elif sys.argv[2] == "proxies":
	1030	systems = get_proxylist()
	1031	else:
	1032	usage()
	1033	for system in systems:
	1034	datadump = get_yaml(system)
	1035	if datadump['status'] == "up":
	1036	print system
[9283]	1037	else:
	1038	usage()
	1039	else:
[10070]	1040	# Do not enable debugging for config requests as it highly clutters the output
	1041	if not is_text_request():
	1042	cgitb.enable()
[9283]	1043	process_cgi_request()
	1044
	1045
	1046	if __name__ == "__main__":
	1047	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: