source: genesis/tools/gformat.py@ 10461

Last change on this file since 10461 was 10461, checked in by rick, 13 years ago

Make sure Hybrid fall under totally new category and include items in proxies
lists for example. Also allow DNS entry to be 'core'

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 39.4 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# Rick van der Zwet <info@rickvanderzwet.nl>
16#
17
18# Hack to make the script directory is also threated as a module search path.
19import sys
20import os
21import re
22sys.path.append(os.path.dirname(__file__))
23
24import cgi
25import cgitb
26import copy
27import glob
28import socket
29import string
30import subprocess
31import time
32import rdnap
33import make_network_kml
34from pprint import pprint
35from collections import defaultdict
36try:
37 import yaml
38except ImportError, e:
39 print e
40 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
41 exit(1)
42
43try:
44 from yaml import CLoader as Loader
45 from yaml import CDumper as Dumper
46except ImportError:
47 from yaml import Loader, Dumper
48
49from jinja2 import Template
50
51import logging
52logging.basicConfig(format='# %(levelname)s: %(message)s' )
53logger = logging.getLogger()
54logger.setLevel(logging.DEBUG)
55
56
57if os.environ.has_key('CONFIGROOT'):
58 NODE_DIR = os.environ['CONFIGROOT']
59else:
60 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
61__version__ = '$Id: gformat.py 10461 2012-04-12 21:27:12Z rick $'
62
63
64files = [
65 'authorized_keys',
66 'dnsmasq.conf',
67 'dhcpd.conf',
68 'rc.conf.local',
69 'resolv.conf',
70 'motd',
71 'wleiden.yaml',
72 ]
73
74# Global variables uses
75OK = 10
76DOWN = 20
77UNKNOWN = 90
78
79def get_yaml(item):
80 """ Get configuration yaml for 'item'"""
81 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
82
83 # Use some boring defaults
84 datadump = { 'service_proxy' : False, 'service_ileiden' : False }
85 f = open(gfile, 'r')
86 datadump.update(yaml.load(f,Loader=Loader))
87 f.close()
88
89 # Preformat certain needed variables for formatting and push those into special object
90 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
91
92 wlan_count=0
93 for key in datadump['autogen_iface_keys']:
94 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
95 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
96 wlan_count += 1
97 else:
98 datadump[key]['autogen_ifname'] = datadump[key]['interface'].split(':')[0]
99
100 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] if datadump[key]['dhcp']]
101 datadump['autogen_dhcp_interfaces'] = ','.join(dhcp_interfaces)
102 datadump['autogen_item'] = item
103
104 datadump['autogen_realname'] = get_realname(datadump)
105 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
106 datadump['autogen_fqdn'] = datadump['autogen_realname'] + '.' + datadump['autogen_domain']
107 return datadump
108
109
110def store_yaml(datadump, header=False):
111 """ Store configuration yaml for 'item'"""
112 item = datadump['autogen_item']
113 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
114
115 f = open(gfile, 'w')
116 f.write(generate_wleiden_yaml(datadump, header))
117 f.close()
118
119
120
121def make_relations():
122 """ Process _ALL_ yaml files to get connection relations """
123 errors = ""
124 poel = defaultdict(list)
125 for host in get_hostlist():
126 try:
127 datadump = get_yaml(host)
128 for iface_key in datadump['autogen_iface_keys']:
129 l = datadump[iface_key]['ip']
130 addr, mask = l.split('/')
131
132 # Not parsing of these folks please
133 if not valid_addr(addr):
134 continue
135
136 addr = parseaddr(addr)
137 mask = int(mask)
138 network = addr & ~((1 << (32 - mask)) - 1)
139 poel[network] += [(host,datadump[iface_key])]
140 except (KeyError, ValueError), e:
141 errors += "[FOUT] in '%s' interface '%s'" % (host,iface_key)
142 errors += e
143 continue
144 return (poel, errors)
145
146
147def get_proxylist():
148 """Get all available proxies proxyX sorting based on X number"""
149 proxylist = sorted([os.path.basename(x) for x in glob.glob("%s/proxy*" % NODE_DIR)],
150 key=lambda name: int(''.join([c for c in name if c in string.digits])),
151 cmp=lambda x,y: x - y) + sorted([os.path.basename(x) for x in glob.glob("%s/Proxy*" % NODE_DIR)])
152 return proxylist
153
154def get_hybridlist():
155 """Get all available hybrid nodes/proxies"""
156 hybridlist = sorted([os.path.basename(x) for x in glob.glob("%s/Hybrid*" % NODE_DIR)])
157 return hybridlist
158
159
160def valid_addr(addr):
161 """ Show which address is valid in which are not """
162 return str(addr).startswith('172.')
163
164
165def get_nodelist():
166 """ Get all available nodes - sorted """
167 nodelist = sorted([os.path.basename(x) for x in glob.glob("%s/CNode*" % NODE_DIR)])
168 return nodelist
169
170def get_hostlist():
171 """ Combined hosts and proxy list"""
172 return get_nodelist() + get_proxylist() + get_hybridlist()
173
174def angle_between_points(lat1,lat2,long1,long2):
175 """
176 Return Angle in radians between two GPS coordinates
177 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
178 """
179 dy = lat2 - lat1
180 dx = math.cos(math.pi/180*lat1)*(long2 - long1)
181 angle = math.atan2(dy,dx)
182 return angle
183
184def angle_to_cd(angle):
185 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
186
187 # For easy conversion get positive degree
188 degrees = math.degrees(angle)
189 if degrees < 0:
190 360 - abs(degrees)
191
192 # Numbers can be confusing calculate from the 4 main directions
193 p = 22.5
194 if degrees < p:
195 return "n"
196 elif degrees < (90 - p):
197 return "no"
198 elif degrees < (90 + p):
199 return "o"
200 elif degrees < (180 - p):
201 return "zo"
202 elif degrees < (180 + p):
203 return "z"
204 elif degrees < (270 - p):
205 return "zw"
206 elif degrees < (270 + p):
207 return "w"
208 elif degrees < (360 - p):
209 return "nw"
210 else:
211 return "n"
212
213
214def generate_title(nodelist):
215 """ Main overview page """
216 items = {'root' : "." }
217 output = """
218<html>
219 <head>
220 <title>Wireless leiden Configurator - GFormat</title>
221 <style type="text/css">
222 th {background-color: #999999}
223 tr:nth-child(odd) {background-color: #cccccc}
224 tr:nth-child(even) {background-color: #ffffff}
225 th, td {padding: 0.1em 1em}
226 </style>
227 </head>
228 <body>
229 <center>
230 <form type="GET" action="%(root)s">
231 <input type="hidden" name="action" value="update">
232 <input type="submit" value="Update Configuration Database (SVN)">
233 </form>
234 <table>
235 <caption><h3>Wireless Leiden Configurator</h3></caption>
236 """ % items
237
238 for node in nodelist:
239 items['node'] = node
240 output += '<tr><td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items
241 for config in files:
242 items['config'] = config
243 output += '<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items
244 output += "</tr>"
245 output += """
246 </table>
247 <hr />
248 <em>%s</em>
249 </center>
250 </body>
251</html>
252 """ % __version__
253
254 return output
255
256
257
258def generate_node(node):
259 """ Print overview of all files available for node """
260 return "\n".join(files)
261
262def generate_node_overview(host):
263 """ Print overview of all files available for node """
264 datadump = get_yaml(host)
265 params = { 'host' : host }
266 output = "<em><a href='..'>Back to overview</a></em><hr />"
267 output += "<h2>Available files:</h2><ul>"
268 for cf in files:
269 params['cf'] = cf
270 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
271 output += "</ul>"
272
273 # Generate and connection listing
274 output += "<h2>Connected To:</h2><ul>"
275 (poel, errors) = make_relations()
276 for network, hosts in poel.iteritems():
277 if host in [x[0] for x in hosts]:
278 if len(hosts) == 1:
279 # Single not connected interface
280 continue
281 for remote,ifacedump in hosts:
282 if remote == host:
283 # This side of the interface
284 continue
285 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
286 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
287 output += "</ul>"
288 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
289
290 output += "<hr /><em><a href='..'>Back to overview</a></em>"
291 return output
292
293
294def generate_header(ctag="#"):
295 return """\
296%(ctag)s
297%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
298%(ctag)s Generated at %(date)s by %(host)s
299%(ctag)s
300""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname() }
301
302
303
304def parseaddr(s):
305 """ Process IPv4 CIDR notation addr to a (binary) number """
306 f = s.split('.')
307 return (long(f[0]) << 24L) + \
308 (long(f[1]) << 16L) + \
309 (long(f[2]) << 8L) + \
310 long(f[3])
311
312
313
314def showaddr(a):
315 """ Display IPv4 addr in (dotted) CIDR notation """
316 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
317
318
319def is_member(ip, mask, canidate):
320 """ Return True if canidate is part of ip/mask block"""
321 ip_addr = gformat.parseaddr(ip)
322 ip_canidate = gformat.parseaddr(canidate)
323 mask = int(mask)
324 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
325 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
326 return ip_addr == ip_canidate
327
328
329
330def cidr2netmask(netmask):
331 """ Given a 'netmask' return corresponding CIDR """
332 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
333
334def get_network(addr, mask):
335 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
336
337
338def generate_dhcpd_conf(datadump):
339 """ Generate config file '/usr/local/etc/dhcpd.conf """
340 output = generate_header()
341 output += Template("""\
342# option definitions common to all supported networks...
343option domain-name "dhcp.{{ autogen_fqdn }}";
344
345default-lease-time 600;
346max-lease-time 7200;
347
348# Use this to enble / disable dynamic dns updates globally.
349#ddns-update-style none;
350
351# If this DHCP server is the official DHCP server for the local
352# network, the authoritative directive should be uncommented.
353authoritative;
354
355# Use this to send dhcp log messages to a different log file (you also
356# have to hack syslog.conf to complete the redirection).
357log-facility local7;
358
359#
360# Interface definitions
361#
362\n""").render(datadump)
363
364 for iface_key in datadump['autogen_iface_keys']:
365 if not datadump[iface_key].has_key('comment'):
366 datadump[iface_key]['comment'] = None
367 output += "## %(interface)s - %(desc)s - %(comment)s\n" % datadump[iface_key]
368
369 (addr, mask) = datadump[iface_key]['ip'].split('/')
370 datadump[iface_key]['addr'] = addr
371 datadump[iface_key]['netmask'] = cidr2netmask(mask)
372 datadump[iface_key]['subnet'] = get_network(addr, mask)
373 try:
374 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
375 except (AttributeError, ValueError):
376 output += "subnet %(subnet)s netmask %(netmask)s {\n ### not autoritive\n}\n\n" % datadump[iface_key]
377 continue
378
379 dhcp_part = ".".join(addr.split('.')[0:3])
380 datadump[iface_key]['dhcp_start'] = dhcp_part + "." + dhcp_start
381 datadump[iface_key]['dhcp_stop'] = dhcp_part + "." + dhcp_stop
382 output += """\
383subnet %(subnet)s netmask %(netmask)s {
384 range %(dhcp_start)s %(dhcp_stop)s;
385 option routers %(addr)s;
386 option domain-name-servers %(addr)s;
387}
388\n""" % datadump[iface_key]
389
390 return output
391
392
393
394def generate_dnsmasq_conf(datadump):
395 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
396 output = generate_header()
397 output += Template("""\
398# DHCP server options
399dhcp-authoritative
400dhcp-fqdn
401domain=dhcp.{{ autogen_fqdn }}
402domain-needed
403expand-hosts
404log-async=100
405
406# Low memory footprint
407cache-size=10000
408
409\n""").render(datadump)
410
411 for iface_key in datadump['autogen_iface_keys']:
412 if not datadump[iface_key].has_key('comment'):
413 datadump[iface_key]['comment'] = None
414 output += "## %(interface)s - %(desc)s - %(comment)s\n" % datadump[iface_key]
415
416 try:
417 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
418 (ip, cidr) = datadump[iface_key]['ip'].split('/')
419 datadump[iface_key]['netmask'] = cidr2netmask(cidr)
420 except (AttributeError, ValueError):
421 output += "# not autoritive\n\n"
422 continue
423
424 dhcp_part = ".".join(ip.split('.')[0:3])
425 datadump[iface_key]['dhcp_start'] = dhcp_part + "." + dhcp_start
426 datadump[iface_key]['dhcp_stop'] = dhcp_part + "." + dhcp_stop
427 output += "dhcp-range=%(interface)s,%(dhcp_start)s,%(dhcp_stop)s,%(netmask)s,24h\n\n" % datadump[iface_key]
428
429 return output
430
431
432
433def generate_rc_conf_local(datadump):
434 """ Generate configuration file '/etc/rc.conf.local' """
435 if not datadump.has_key('ileiden'):
436 datadump['autogen_ileiden_enable'] = False
437 else:
438 datadump['autogen_ileiden_enable'] = datadump['ileiden']
439
440 ileiden_proxies = []
441 normal_proxies = []
442 for proxy in get_proxylist():
443 proxydump = get_yaml(proxy)
444 if proxydump['ileiden']:
445 ileiden_proxies.append(proxydump)
446 else:
447 normal_proxies.append(proxydump)
448 for host in get_hybridlist():
449 hostdump = get_yaml(host)
450 if hostdump['service_ileiden']:
451 ileiden_proxies.append(hostdump)
452 if hostdump['service_proxy']:
453 normal_proxies.append(hostdump)
454
455 datadump['autogen_ileiden_proxies'] = ','.join([x['masterip'] for x in ileiden_proxies])
456 datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
457 datadump['autogen_normal_proxies'] = ','.join([x['masterip'] for x in normal_proxies])
458 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
459
460 output = generate_header("#");
461 output += Template("""\
462hostname='{{ autogen_fqdn }}'
463location='{{ location }}'
464nodetype="{{ nodetype }}"
465
466#
467# Configured listings
468#
469captive_portal_whitelist=""
470captive_portal_interfaces="{{ autogen_dhcp_interfaces }}"
471# iLeiden Proxies {{ autogen_ileiden_proxies_names }}
472list_ileiden_proxies="{{ autogen_ileiden_proxies }}"
473# normal Proxies {{ autogen_normal_proxies_names }}
474list_normal_proxies="{{ autogen_normal_proxies }}"
475
476{% if nodetype == "Proxy" %}
477#
478# Proxy Configuration
479#
480{% if gateway -%}
481defaultrouter="{{ gateway }}"
482{% else -%}
483#defaultrouter="NOTSET"
484{% endif -%}
485internalif="{{ internalif }}"
486ileiden_enable="{{ autogen_ileiden_enable }}"
487gateway_enable="{{ autogen_ileiden_enable }}"
488pf_enable="yes"
489pf_rules="/etc/pf.conf"
490{% if autogen_ileiden_enable -%}
491pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
492lvrouted_enable="{{ autogen_ileiden_enable }}"
493lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
494{% else -%}
495pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
496{% endif -%}
497{% if internalroute -%}
498static_routes="wleiden"
499route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
500{% endif -%}
501{% endif -%}
502
503{% if nodetype == "Hybrid" %}
504#
505# Hybrid Configuration
506#
507captive_portal_interfaces="{{ autogen_dhcp_interfaces }}"
508
509pf_rules="/etc/pf.hybrid.conf"
510{% if service_ileiden -%}
511# Service iLeiden
512pf_flags="-D ext_if={{ externalif }} -D captive_portal_interfaces=${captive_portal_interfaces:-'none'} -D publicnat=80,443"
513{% else -%}
514pf_flags="-D ext_if={{ externalif }} -D captive_portal_interfaces=${captive_portal_interfaces:-'none'} -D publicnat=0"
515{% endif -%}
516
517{% if service_proxy %}
518# Service Proxy
519tinyproxy_enable="yes"
520{% endif -%}
521
522{% if board == "ALIX2" %}
523# ''Fat'' configuration, board has 256MB RAM
524dnsmasq_enable="NO"
525named_enable="YES"
526dhcpd_enable="YES"
527{% endif -%}
528
529{% if gateway -%}
530defaultrouter="{{ gateway }}"
531{% endif -%}
532#
533# END Hybrid Configuration
534#
535{% endif -%}
536
537
538{% if nodetype == "CNode" %}
539#
540# NODE iLeiden Configuration
541#
542
543{% if tproxy -%}
544tproxy_enable='YES'
545tproxy_range='{{ tproxy }}'
546{% else -%}
547tproxy_enable='NO'
548{% endif -%}
549
550lvrouted_flags="-u -s s00p3rs3kr3t -m 28 -z $list_ileiden_proxies"
551{% endif %}
552\n
553""").render(datadump)
554
555 # lo0 configuration:
556 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
557 # - masterip is special as it needs to be assigned to at
558 # least one interface, so if not used assign to lo0
559 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
560 iface_map = {'lo0' : 'lo0'}
561 dhclient_if = {'lo0' : False}
562
563 masterip_used = False
564 for iface_key in datadump['autogen_iface_keys']:
565 if datadump[iface_key]['ip'].startswith(datadump['masterip']):
566 masterip_used = True
567 break
568 if not masterip_used:
569 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
570
571 for iface_key in datadump['autogen_iface_keys']:
572 ifacedump = datadump[iface_key]
573 ifname = ifacedump['autogen_ifname']
574
575 # Flag dhclient is possible
576 dhclient_if[ifname] = ifacedump.has_key('dhcpclient') and ifacedump['dhcpclient']
577
578 # Add interface IP to list
579 item = (ifacedump['ip'], ifacedump['desc'])
580 if addrs_list.has_key(ifname):
581 addrs_list[ifname].append(item)
582 else:
583 addrs_list[ifname] = [item]
584
585 # Alias only needs IP assignment for now, this might change if we
586 # are going to use virtual accesspoints
587 if "alias" in iface_key:
588 continue
589
590 # XXX: Might want to deduct type directly from interface name
591 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
592 # Default to station (client) mode
593 ifacedump['wlanmode'] = "sta"
594 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
595 ifacedump['wlanmode'] = "ap"
596 # Default to 802.11b mode
597 ifacedump['mode'] = '11b'
598 if ifacedump['type'] in ['11a', '11b' '11g']:
599 ifacedump['mode'] = ifacedump['type']
600
601 if not ifacedump.has_key('channel'):
602 if ifacedump['type'] == '11a':
603 ifacedump['channel'] = 36
604 else:
605 ifacedump['channel'] = 1
606
607 # Allow special hacks at the back like wds and stuff
608 if not ifacedump.has_key('extra'):
609 ifacedump['extra'] = 'regdomain ETSI country NL'
610
611 output += "wlans_%(interface)s='%(autogen_ifname)s'\n" % ifacedump
612 output += ("create_args_%(autogen_ifname)s='wlanmode %(wlanmode)s mode " +\
613 "%(mode)s ssid %(ssid)s %(extra)s channel %(channel)s'\n") % ifacedump
614
615 elif ifacedump['type'] in ['ethernet', 'eth']:
616 # No special config needed besides IP
617 pass
618 else:
619 assert False, "Unknown type " + ifacedump['type']
620
621 # Print IP address which needs to be assigned over here
622 output += "\n"
623 for iface,addrs in sorted(addrs_list.iteritems()):
624 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
625 output += "# %s || %s || %s\n" % (iface, addr, comment)
626
627 # Write DHCLIENT entry
628 if dhclient_if[iface]:
629 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
630 else:
631 output += "ipv4_addrs_%s='%s'\n\n" % (iface, " ".join([x[0] for x in addrs]))
632
633 return output
634
635
636
637
638def get_all_configs():
639 """ Get dict with key 'host' with all configs present """
640 configs = dict()
641 for host in get_hostlist():
642 datadump = get_yaml(host)
643 configs[host] = datadump
644 return configs
645
646
647def get_interface_keys(config):
648 """ Quick hack to get all interface keys, later stage convert this to a iterator """
649 return sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
650
651
652def get_used_ips(configs):
653 """ Return array of all IPs used in config files"""
654 ip_list = []
655 for config in configs:
656 ip_list.append(config['masterip'])
657 for iface_key in get_interface_keys(config):
658 l = config[iface_key]['ip']
659 addr, mask = l.split('/')
660 # Special case do not process
661 if valid_addr(addr):
662 ip_list.append(addr)
663 else:
664 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
665 return sorted(ip_list)
666
667
668
669def generate_resolv_conf(datadump):
670 """ Generate configuration file '/etc/resolv.conf' """
671 output = generate_header("#");
672 output += """\
673search wleiden.net
674"""
675 if datadump['nodetype'] == 'Proxy':
676 output += """\
677# Try local (cache) first
678nameserver 127.0.0.1
679nameserver 8.8.8.8 # Google Public NameServer
680nameserver 8.8.4.4 # Google Public NameServer
681"""
682 elif datadump['nodetype'] == 'Hybrid':
683 for proxy in get_proxylist():
684 proxy_ip = get_yaml(proxy)['masterip']
685 output += "nameserver %-15s # %s\n" % (proxy_ip, proxy)
686 output += """\
687nameserver 8.8.8.8 # Google Public NameServer
688nameserver 8.8.4.4 # Google Public NameServer
689"""
690 else:
691 output += """\
692# Try local (cache) first
693nameserver 127.0.0.1
694
695# Proxies are recursive nameservers
696# needs to be in resolv.conf for dnsmasq as well
697""" % datadump
698 for proxy in get_proxylist():
699 proxy_ip = get_yaml(proxy)['masterip']
700 output += "nameserver %-15s # %s\n" % (proxy_ip, proxy)
701
702 return output
703
704def generate_motd(datadump):
705 """ Generate configuration file '/etc/motd' """
706 output = """\
707FreeBSD 9.0-RELEASE (kernel.wleiden) #0 r230587: Sun Jan 29 17:09:57 CET 2012
708
709 WWW: %(autogen_fqdn)s - http://www.wirelessleiden.nl
710 Loc: %(location)s
711
712Interlinks:
713""" % datadump
714
715 # XXX: This is a hacky way to get the required data
716 for line in generate_rc_conf_local(datadump).split('\n'):
717 if '||' in line and not line[1:].split()[0] in ['lo0', 'ath0'] :
718 output += " - %s \n" % line[1:]
719 output += """\
720Attached bridges:
721"""
722 for iface_key in datadump['autogen_iface_keys']:
723 ifacedump = datadump[iface_key]
724 if ifacedump.has_key('ns_ip'):
725 output += " - %(interface)s || %(mode)s || %(ns_ip)s\n" % ifacedump
726
727 return output
728
729
730def format_yaml_value(value):
731 """ Get yaml value in right syntax for outputting """
732 if isinstance(value,str):
733 output = '"%s"' % value
734 else:
735 output = value
736 return output
737
738
739
740def format_wleiden_yaml(datadump):
741 """ Special formatting to ensure it is editable"""
742 output = "# Genesis config yaml style\n"
743 output += "# vim:ts=2:et:sw=2:ai\n"
744 output += "#\n"
745 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
746 for key in sorted(set(datadump.keys()) - set(iface_keys)):
747 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
748
749 output += "\n\n"
750
751 key_order = [ 'comment', 'interface', 'ip', 'desc', 'sdesc', 'mode', 'type',
752 'extra_type', 'channel', 'ssid', 'dhcp' ]
753
754 for iface_key in sorted(iface_keys):
755 output += "%s:\n" % iface_key
756 for key in key_order + list(sorted(set(datadump[iface_key].keys()) - set(key_order))):
757 if datadump[iface_key].has_key(key):
758 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
759 output += "\n\n"
760
761 return output
762
763
764
765def generate_wleiden_yaml(datadump, header=True):
766 """ Generate (petty) version of wleiden.yaml"""
767 for key in datadump.keys():
768 if key.startswith('autogen_'):
769 del datadump[key]
770 # Interface autogen cleanups
771 elif type(datadump[key]) == dict:
772 for key2 in datadump[key].keys():
773 if key2.startswith('autogen_'):
774 del datadump[key][key2]
775
776 output = generate_header("#") if header else ''
777 output += format_wleiden_yaml(datadump)
778 return output
779
780
781def generate_yaml(datadump):
782 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
783
784
785
786def generate_config(node, config, datadump=None):
787 """ Print configuration file 'config' of 'node' """
788 output = ""
789 try:
790 # Load config file
791 if datadump == None:
792 datadump = get_yaml(node)
793
794 if config == 'wleiden.yaml':
795 output += generate_wleiden_yaml(datadump)
796 elif config == 'authorized_keys':
797 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
798 output += f.read()
799 f.close()
800 elif config == 'dnsmasq.conf':
801 output += generate_dnsmasq_conf(datadump)
802 elif config == 'dhcpd.conf':
803 output += generate_dhcpd_conf(datadump)
804 elif config == 'rc.conf.local':
805 output += generate_rc_conf_local(datadump)
806 elif config == 'resolv.conf':
807 output += generate_resolv_conf(datadump)
808 elif config == 'motd':
809 output += generate_motd(datadump)
810 else:
811 assert False, "Config not found!"
812 except IOError, e:
813 output += "[ERROR] Config file not found"
814 return output
815
816
817
818def process_cgi_request():
819 """ When calling from CGI """
820 # Update repository if requested
821 form = cgi.FieldStorage()
822 if form.getvalue("action") == "update":
823 print "Refresh: 5; url=."
824 print "Content-type:text/plain\r\n\r\n",
825 print "[INFO] Updating subverion, please wait..."
826 print subprocess.Popen(['svn', 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0],
827 print subprocess.Popen(['svn', 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0],
828 print "[INFO] All done, redirecting in 5 seconds"
829 sys.exit(0)
830
831
832 base_uri = os.environ['PATH_INFO']
833 uri = base_uri.strip('/').split('/')
834
835 output = ""
836 if base_uri.endswith('/create/network.kml'):
837 output += "Content-type:application/vnd.google-earth.kml+xml\r\n\r\n"
838 output += make_network_kml.make_graph()
839 elif not uri[0]:
840 if is_text_request():
841 output += "Content-type:text/plain\r\n\r\n"
842 output += '\n'.join(get_hostlist())
843 else:
844 output += "Content-type:text/html\r\n\r\n"
845 output += generate_title(get_hostlist())
846 elif len(uri) == 1:
847 if is_text_request():
848 output += "Content-type:text/plain\r\n\r\n"
849 output += generate_node(uri[0])
850 else:
851 output += "Content-type:text/html\r\n\r\n"
852 output += generate_node_overview(uri[0])
853 elif len(uri) == 2:
854 output += "Content-type:text/plain\r\n\r\n"
855 output += generate_config(uri[0], uri[1])
856 else:
857 assert False, "Invalid option"
858 print output
859
860def get_realname(datadump):
861 # Proxy naming convention is special, as the proxy name is also included in
862 # the nodename, when it comes to the numbered proxies.
863 if datadump['nodetype'] == 'Proxy':
864 realname = datadump['nodetype'] + datadump['nodename'].replace('proxy','')
865 else:
866 # By default the full name is listed and also a shortname CNAME for easy use.
867 realname = datadump['nodetype'] + datadump['nodename']
868 return(realname)
869
870
871
872def make_dns(output_dir = 'dns', external = False):
873 items = dict()
874
875 # hostname is key, IP is value
876 wleiden_zone = dict()
877 wleiden_cname = dict()
878
879 pool = dict()
880 for node in get_hostlist():
881 logger.info("Processing host %s", node)
882 datadump = get_yaml(node)
883
884 # Proxy naming convention is special
885 fqdn = datadump['autogen_realname']
886 if datadump['nodetype'] in ['CNode', 'Hybrid']:
887 wleiden_cname[datadump['nodename']] = fqdn
888
889 wleiden_zone[fqdn] = datadump['masterip']
890
891 # Hacking to get proper DHCP IPs and hostnames
892 for iface_key in get_interface_keys(datadump):
893 iface_name = datadump[iface_key]['interface'].replace(':',"-alias-")
894 (ip, cidr) = datadump[iface_key]['ip'].split('/')
895 try:
896 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
897 datadump[iface_key]['netmask'] = cidr2netmask(cidr)
898 dhcp_part = ".".join(ip.split('.')[0:3])
899 if ip != datadump['masterip']:
900 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)] = ip
901 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
902 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)] = "%s.%s" % (dhcp_part, i)
903 except (AttributeError, ValueError):
904 # First push it into a pool, to indentify the counter-part later on
905 addr = parseaddr(ip)
906 cidr = int(cidr)
907 addr = addr & ~((1 << (32 - cidr)) - 1)
908 if pool.has_key(addr):
909 pool[addr] += [(iface_name, fqdn, ip)]
910 else:
911 pool[addr] = [(iface_name, fqdn, ip)]
912 continue
913
914
915 def pool_to_name(node, pool_members):
916 """Convert the joined name to a usable pool name"""
917
918 # Get rid of the own entry
919 pool_members = list(set(pool_members) - set([fqdn]))
920
921 target = oldname = ''
922 for node in sorted(pool_members):
923 (name, number) = re.match('^([A-Za-z]+)([0-9]*)$',node).group(1,2)
924 target += "-" + number if name == oldname else "-" + node if target else node
925 oldname = name
926
927 return target
928
929
930 # WL uses an /29 to configure an interface. IP's are ordered like this:
931 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
932
933 sn = lambda x: re.sub(r'(?i)^cnode','',x)
934
935 # Automatic naming convention of interlinks namely 2 + remote.lower()
936 for (key,value) in pool.iteritems():
937 # Make sure they are sorted from low-ip to high-ip
938 value = sorted(value, key=lambda x: parseaddr(x[2]))
939
940 if len(value) == 1:
941 (iface_name, fqdn, ip) = value[0]
942 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)] = ip
943
944 # Device DNS names
945 if 'cnode' in fqdn.lower():
946 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)] = showaddr(parseaddr(ip) + 1)
947 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % (iface_name, fqdn)
948
949 elif len(value) == 2:
950 (a_iface_name, a_fqdn, a_ip) = value[0]
951 (b_iface_name, b_fqdn, b_ip) = value[1]
952 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)] = a_ip
953 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)] = b_ip
954
955 # Device DNS names
956 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
957 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)] = showaddr(parseaddr(a_ip) + 1)
958 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)] = showaddr(parseaddr(b_ip) - 1)
959 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
960 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
961 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
962 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
963
964 else:
965 pool_members = [k[1] for k in value]
966 for item in value:
967 (iface_name, fqdn, ip) = item
968 pool_name = "2pool-" + showaddr(key).replace('.','-') + "-" + pool_to_name(fqdn,pool_members)
969 wleiden_zone["%s.%s" % (pool_name, fqdn)] = ip
970
971 # Include static DNS entries
972 # XXX: Should they override the autogenerated results?
973 # XXX: Convert input to yaml more useable.
974 # Format:
975 ##; this is a comment
976 ## roomburgh=CNodeRoomburgh1
977 ## apkerk1.CNodeVosko=172.17.176.8 ;this as well
978 dns = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
979
980 # Hack to allow special entries, for development
981 wleiden_raw = dns['raw']
982 del dns['raw']
983
984 for comment, block in dns.iteritems():
985 for k,v in block.iteritems():
986 if valid_addr(v):
987 wleiden_zone[k] = v
988 else:
989 wleiden_cname[k] = v
990
991 details = dict()
992 # 24 updates a day allowed
993 details['serial'] = time.strftime('%Y%m%d%H')
994
995 if external:
996 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
997 else:
998 dns_masters = ['sunny.wleiden.net']
999
1000 details['master'] = dns_masters[0]
1001 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1002
1003 dns_header = '''
1004$TTL 3h
1005%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 1d 12h 1w 3h )
1006 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1007
1008%(ns_servers)s
1009 \n'''
1010
1011
1012 if not os.path.isdir(output_dir):
1013 os.makedirs(output_dir)
1014 details['zone'] = 'wleiden.net'
1015 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1016 f.write(dns_header % details)
1017
1018 for host,ip in wleiden_zone.iteritems():
1019 if valid_addr(ip):
1020 f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
1021 for source,dest in wleiden_cname.iteritems():
1022 f.write("%s.wleiden.net. IN CNAME %s.wleiden.net.\n" % (source.lower(), dest.lower()))
1023 for source, dest in wleiden_raw.iteritems():
1024 f.write("%s.wleiden.net. %s\n" % (source, dest))
1025 f.close()
1026
1027 # Create whole bunch of specific sub arpa zones. To keep it compliant
1028 for s in range(16,32):
1029 details['zone'] = '%i.172.in-addr.arpa' % s
1030 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1031 f.write(dns_header % details)
1032
1033 #XXX: Not effient, fix to proper data structure and do checks at other
1034 # stages
1035 for host,ip in wleiden_zone.iteritems():
1036 if valid_addr(ip):
1037 if int(ip.split('.')[1]) == s:
1038 rev_ip = '.'.join(reversed(ip.split('.')))
1039 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1040 f.close()
1041
1042
1043def usage():
1044 print """Usage: %s <standalone [port] |test [test arguments]|static|dns>
1045Examples:
1046\tdns [outputdir] = Generate BIND compliant zone files in dns.
1047\tstandalone = Run configurator webserver [default port=8000]
1048\twind-export = Generate SQL import scripts for WIND database
1049\tfull-export = Generate yaml export script for heatmap.
1050\tstatic = Generate all config files and store on disk
1051\t with format ./static/%%NODE%%/%%FILE%%
1052\ttest CNodeRick dnsmasq.conf = Receive output of CGI script
1053\t for arguments CNodeRick/dnsmasq.conf
1054\tlist <all|nodes|proxies> = List systems which marked up.
1055"""
1056 exit(0)
1057
1058
1059def is_text_request():
1060 """ Find out whether we are calling from the CLI or any text based CLI utility """
1061 try:
1062 return os.environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1063 except KeyError:
1064 return True
1065
1066def main():
1067 """Hard working sub"""
1068 # Allow easy hacking using the CLI
1069 if not os.environ.has_key('PATH_INFO'):
1070 if len(sys.argv) < 2:
1071 usage()
1072
1073 if sys.argv[1] == "standalone":
1074 import SocketServer
1075 import CGIHTTPServer
1076 # Hop to the right working directory.
1077 os.chdir(os.path.dirname(__file__))
1078 try:
1079 PORT = int(sys.argv[2])
1080 except (IndexError,ValueError):
1081 PORT = 8000
1082
1083 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1084 """ Serve this CGI from the root of the webserver """
1085 def is_cgi(self):
1086 if "favicon" in self.path:
1087 return False
1088
1089 self.cgi_info = (os.path.basename(__file__), self.path)
1090 self.path = ''
1091 return True
1092 handler = MyCGIHTTPRequestHandler
1093 SocketServer.TCPServer.allow_reuse_address = True
1094 httpd = SocketServer.TCPServer(("", PORT), handler)
1095 httpd.server_name = 'localhost'
1096 httpd.server_port = PORT
1097
1098 logger.info("serving at port %s", PORT)
1099 try:
1100 httpd.serve_forever()
1101 except KeyboardInterrupt:
1102 httpd.shutdown()
1103 logger.info("All done goodbye")
1104 elif sys.argv[1] == "test":
1105 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1106 os.environ['SCRIPT_NAME'] = __file__
1107 process_cgi_request()
1108 elif sys.argv[1] == "unit-test":
1109 os.environ['SCRIPT_NAME'] = __file__
1110 for host in get_hostlist():
1111 for outfile in files:
1112 os.environ['PATH_INFO'] = "/".join([host,outfile])
1113 try:
1114 process_cgi_request()
1115 except Exception:
1116 print "# ERROR: %s" % os.environ['PATH_INFO']
1117 raise
1118
1119
1120 elif sys.argv[1] == "static":
1121 items = dict()
1122 for node in get_hostlist():
1123 items['node'] = node
1124 items['wdir'] = "./static/%(node)s" % items
1125 if not os.path.isdir(items['wdir']):
1126 os.makedirs(items['wdir'])
1127 datadump = get_yaml(node)
1128 for config in files:
1129 items['config'] = config
1130 logger.info("## Generating %(node)s %(config)s" % items)
1131 f = open("%(wdir)s/%(config)s" % items, "w")
1132 f.write(generate_config(node, config, datadump))
1133 f.close()
1134 elif sys.argv[1] == "wind-export":
1135 items = dict()
1136 for node in get_hostlist():
1137 datadump = get_yaml(node)
1138 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1139 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1140 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1141 VALUES (
1142 (SELECT id FROM users WHERE username = 'rvdzwet'),
1143 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1144 'Y');""" % datadump
1145 #for config in files:
1146 # items['config'] = config
1147 # print "## Generating %(node)s %(config)s" % items
1148 # f = open("%(wdir)s/%(config)s" % items, "w")
1149 # f.write(generate_config(node, config, datadump))
1150 # f.close()
1151 for node in get_hostlist():
1152 datadump = get_yaml(node)
1153 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1154 ifacedump = datadump[iface_key]
1155 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1156 ifacedump['nodename'] = datadump['nodename']
1157 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1158 ifacedump['channel'] = 0
1159 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1160 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1161 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1162 elif sys.argv[1] == "full-export":
1163 hosts = {}
1164 for node in get_hostlist():
1165 datadump = get_yaml(node)
1166 hosts[datadump['nodename']] = datadump
1167 print yaml.dump(hosts)
1168
1169 elif sys.argv[1] == "dns":
1170 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
1171 elif sys.argv[1] == "cleanup":
1172 # First generate all datadumps
1173 datadumps = dict()
1174 for host in get_hostlist():
1175 logger.info("# Processing: %s", host)
1176 # Set some boring default values
1177 datadump = { 'board' : 'UNKNOWN' }
1178 datadump.update(get_yaml(host))
1179 datadumps[datadump['autogen_realname']] = datadump
1180
1181
1182 for host,datadump in datadumps.iteritems():
1183 # Convert all yes and no to boolean values
1184 def fix_boolean(dump):
1185 for key in dump.keys():
1186 if type(dump[key]) == dict:
1187 dump[key] = fix_boolean(dump[key])
1188 elif str(dump[key]).lower() in ["yes", "true"]:
1189 dump[key] = True
1190 elif str(dump[key]).lower() in ["no", "false"]:
1191 # Compass richting no (Noord Oost) is valid input
1192 if key != "compass": dump[key] = False
1193 return dump
1194 datadump = fix_boolean(datadump)
1195
1196 if datadump['rdnap_x'] and datadump['rdnap_y']:
1197 datadump['latitude'], datadump['longitude'] = rdnap.rd2etrs(datadump['rdnap_x'], datadump['rdnap_y'])
1198 elif datadump['latitude'] and datadump['longitude']:
1199 datadump['rdnap_x'], datadump['rdnap_y'] = rdnap.etrs2rd(datadump['latitude'], datadump['longitude'])
1200
1201 if datadump['nodename'].startswith('Proxy'):
1202 datadump['nodename'] = datadump['nodename'].lower()
1203
1204 for iface_key in datadump['autogen_iface_keys']:
1205 # Wireless Leiden SSID have an consistent lowercase/uppercase
1206 if datadump[iface_key].has_key('ssid'):
1207 ssid = datadump[iface_key]['ssid']
1208 prefix = 'ap-WirelessLeiden-'
1209 if ssid.lower().startswith(prefix.lower()):
1210 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
1211 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
1212 datadump[iface_key]['mode'] = 'autogen-FIXME'
1213 if not datadump[iface_key].has_key('desc'):
1214 datadump[iface_key]['desc'] = 'autogen-FIXME'
1215 store_yaml(datadump)
1216 elif sys.argv[1] == "list":
1217 if sys.argv[2] == "nodes":
1218 systems = get_nodelist()
1219 elif sys.argv[2] == "proxies":
1220 systems = get_proxylist()
1221 elif sys.argv[2] == "all":
1222 systems = get_hostlist()
1223 else:
1224 usage()
1225 for system in systems:
1226 datadump = get_yaml(system)
1227 if datadump['status'] == "up":
1228 print system
1229 elif sys.argv[1] == "create":
1230 if sys.argv[2] == "network.kml":
1231 print make_network_kml.make_graph()
1232 else:
1233 usage()
1234 usage()
1235 else:
1236 # Do not enable debugging for config requests as it highly clutters the output
1237 if not is_text_request():
1238 cgitb.enable()
1239 process_cgi_request()
1240
1241
1242if __name__ == "__main__":
1243 main()
Note: See TracBrowser for help on using the repository browser.