source: genesis/tools/gformat.py@ 10885

Last change on this file since 10885 was 10885, checked in by rick, 13 years ago

Handmatig(!) opruimen van alle conflict keys.

Related-To: nodefactory:ticket:218
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 52.3 KB
Line 
1#!/usr/bin/env python
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# ScriptAlias /wleiden/config /usr/local/www/genesis/tools/gformat.py
10# <Directory /usr/local/www/genesis>
11# Allow from all
12# AcceptPathInfo On
13# </Directory>
14#
15# Rick van der Zwet <info@rickvanderzwet.nl>
16#
17
18# Hack to make the script directory is also threated as a module search path.
19import sys
20import os
21import re
22sys.path.append(os.path.dirname(__file__))
23
24import cgi
25import cgitb
26import copy
27import glob
28import socket
29import string
30import subprocess
31import time
32import rdnap
33import math
34import make_network_kml
35from pprint import pprint
36from collections import defaultdict
37try:
38 import yaml
39except ImportError, e:
40 print e
41 print "[ERROR] Please install the python-yaml or devel/py-yaml package"
42 exit(1)
43
44try:
45 from yaml import CLoader as Loader
46 from yaml import CDumper as Dumper
47except ImportError:
48 from yaml import Loader, Dumper
49
50from jinja2 import Environment, Template
51def yesorno(value):
52 return "YES" if bool(value) else "NO"
53env = Environment()
54env.filters['yesorno'] = yesorno
55def render_template(datadump, template):
56 result = env.from_string(template).render(datadump)
57 # Make it look pretty to the naked eye, as jinja templates are not so
58 # friendly when it comes to whitespace formatting
59 ## Remove extra whitespace at end of line lstrip() style.
60 result = re.sub(r'\n[\ ]+','\n', result)
61 ## Include only a single newline between an definition and a comment
62 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
63 ## Remove extra newlines after single comment
64 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
65 return result
66
67import logging
68logging.basicConfig(format='# %(levelname)s: %(message)s' )
69logger = logging.getLogger()
70logger.setLevel(logging.DEBUG)
71
72
73if os.environ.has_key('CONFIGROOT'):
74 NODE_DIR = os.environ['CONFIGROOT']
75else:
76 NODE_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../nodes'
77__version__ = '$Id: gformat.py 10885 2012-05-16 20:56:46Z rick $'
78
79
80files = [
81 'authorized_keys',
82 'dnsmasq.conf',
83 'dhcpd.conf',
84 'rc.conf.local',
85 'resolv.conf',
86 'motd',
87 'ntp.conf',
88 'pf.hybrid.conf.local',
89 'wleiden.yaml',
90 ]
91
92# Global variables uses
93OK = 10
94DOWN = 20
95UNKNOWN = 90
96
97datadump_cache = {}
98
99def get_yaml(item):
100 try:
101 """ Get configuration yaml for 'item'"""
102 if datadump_cache.has_key(item):
103 return datadump_cache[item].copy()
104
105 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
106
107 datadump = {}
108 f = open(gfile, 'r')
109 datadump.update(yaml.load(f,Loader=Loader))
110 if datadump['nodetype'] == 'Hybrid':
111 # Some values are defined implicitly
112 if datadump.has_key('rdr_rules') and datadump['rdr_rules'] and not datadump.has_key('service_incoming_rdr'):
113 datadump['service_incoming_rdr'] = True
114 # Use some boring defaults
115 defaults = {
116 'service_proxy_normal' : False,
117 'service_proxy_ileiden' : False,
118 'service_accesspoint' : True,
119 'service_incoming_rdr' : False
120 }
121 for (key,value) in defaults.iteritems():
122 if not datadump.has_key(key):
123 datadump[key] = value
124 f.close()
125
126 # Preformat certain needed variables for formatting and push those into special object
127 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
128
129 wlan_count=0
130 try:
131 for key in datadump['autogen_iface_keys']:
132 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
133 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
134 wlan_count += 1
135 else:
136 datadump[key]['autogen_ifname'] = datadump[key]['interface'].split(':')[0]
137 except Exception as e:
138 print "# Error while processing interface %s" % key
139 raise
140
141 dhcp_interfaces = [datadump[key]['autogen_ifname'] for key in datadump['autogen_iface_keys'] if (datadump[key].has_key('dhcp') and datadump[key]['dhcp'])]
142 datadump['autogen_dhcp_interfaces'] = dhcp_interfaces
143 datadump['autogen_item'] = item
144
145 datadump['autogen_realname'] = get_realname(datadump)
146 datadump['autogen_domain'] = datadump['domain'] if datadump.has_key('domain') else 'wleiden.net.'
147 datadump['autogen_fqdn'] = datadump['autogen_realname'] + '.' + datadump['autogen_domain']
148 datadump_cache[item] = datadump.copy()
149 except Exception as e:
150 print "# Error while processing %s" % item
151 raise
152 return datadump
153
154
155def store_yaml(datadump, header=False):
156 """ Store configuration yaml for 'item'"""
157 item = datadump['autogen_item']
158 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
159
160 output = generate_wleiden_yaml(datadump, header)
161
162 f = open(gfile, 'w')
163 f.write(output)
164 f.close()
165
166
167def network(ip):
168 addr, mask = ip.split('/')
169 # Not parsing of these folks please
170 addr = parseaddr(addr)
171 mask = int(mask)
172 network = addr & ~((1 << (32 - mask)) - 1)
173 return network
174
175
176
177def make_relations(datadumps=None):
178 """ Process _ALL_ yaml files to get connection relations """
179 errors = []
180 poel = defaultdict(list)
181
182 if not datadumps:
183 for host in get_hostlist():
184 datadumps[host] = get_yaml(host)
185
186 for host, datadump in datadumps.iteritems():
187 try:
188 for iface_key in datadump['autogen_iface_keys']:
189 net_addr = network(datadump[iface_key]['ip'])
190 poel[net_addr] += [(host,datadump[iface_key])]
191 except (KeyError, ValueError), e:
192 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, e))
193 continue
194 return (poel, errors)
195
196
197
198def valid_addr(addr):
199 """ Show which address is valid in which are not """
200 return str(addr).startswith('172.')
201
202def get_system_list(prefix):
203 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/%s*/wleiden.yaml" % (NODE_DIR, prefix))])
204
205get_hybridlist = lambda: get_system_list("Hybrid")
206get_nodelist = lambda: get_system_list("CNode")
207get_proxylist = lambda: get_system_list("Proxy")
208
209def get_hostlist():
210 """ Combined hosts and proxy list"""
211 return get_nodelist() + get_proxylist() + get_hybridlist()
212
213def angle_between_points(lat1,lat2,long1,long2):
214 """
215 Return Angle in radians between two GPS coordinates
216 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
217 """
218 dy = lat2 - lat1
219 dx = math.cos(lat1)*(long2 - long1)
220 angle = math.atan2(dy,dx)
221 return angle
222
223
224
225def angle_to_cd(angle):
226 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
227
228 # For easy conversion get positive degree
229 degrees = math.degrees(angle)
230 abs_degrees = 360 + degrees if degrees < 0 else degrees
231
232 # Numbers can be confusing calculate from the 4 main directions
233 p = 22.5
234 if abs_degrees < p:
235 cd = "n"
236 elif abs_degrees < (90 - p):
237 cd = "no"
238 elif abs_degrees < (90 + p):
239 cd = "o"
240 elif abs_degrees < (180 - p):
241 cd = "zo"
242 elif abs_degrees < (180 + p):
243 cd = "z"
244 elif abs_degrees < (270 - p):
245 cd = "zw"
246 elif abs_degrees < (270 + p):
247 cd = "w"
248 elif abs_degrees < (360 - p):
249 cd = "nw"
250 else:
251 cd = "n"
252 return cd
253
254
255
256def cd_between_hosts(hostA, hostB, datadumps):
257 # Using RDNAP coordinates
258 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
259 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
260 return angle_to_cd(math.atan2(dx,dy))
261
262 # GPS coordinates seems to fail somehow
263 #latA = float(datadumps[hostA]['latitude'])
264 #latB = float(datadumps[hostB]['latitude'])
265 #lonA = float(datadumps[hostA]['longitude'])
266 #lonB = float(datadumps[hostB]['longitude'])
267 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
268
269
270def generate_title(nodelist):
271 """ Main overview page """
272 items = {'root' : "." }
273 def fl(spaces, line):
274 return (' ' * spaces) + line + '\n'
275
276 output = """
277<html>
278 <head>
279 <title>Wireless leiden Configurator - GFormat</title>
280 <style type="text/css">
281 th {background-color: #999999}
282 tr:nth-child(odd) {background-color: #cccccc}
283 tr:nth-child(even) {background-color: #ffffff}
284 th, td {padding: 0.1em 1em}
285 </style>
286 </head>
287 <body>
288 <center>
289 <form type="GET" action="%(root)s">
290 <input type="hidden" name="action" value="update">
291 <input type="submit" value="Update Configuration Database (SVN)">
292 </form>
293 <table>
294 <caption><h3>Wireless Leiden Configurator</h3></caption>
295 """ % items
296
297 for node in nodelist:
298 items['node'] = node
299 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
300 for config in files:
301 items['config'] = config
302 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
303 output += fl(5, "</tr>")
304 output += """
305 </table>
306 <hr />
307 <em>%s</em>
308 </center>
309 </body>
310</html>
311 """ % __version__
312
313 return output
314
315
316
317def generate_node(node):
318 """ Print overview of all files available for node """
319 return "\n".join(files)
320
321def generate_node_overview(host):
322 """ Print overview of all files available for node """
323 datadump = get_yaml(host)
324 params = { 'host' : host }
325 output = "<em><a href='..'>Back to overview</a></em><hr />"
326 output += "<h2>Available files:</h2><ul>"
327 for cf in files:
328 params['cf'] = cf
329 output += '<li><a href="%(host)s/%(cf)s">%(cf)s</a></li>\n' % params
330 output += "</ul>"
331
332 # Generate and connection listing
333 output += "<h2>Connected To:</h2><ul>"
334 (poel, errors) = make_relations()
335 for network, hosts in poel.iteritems():
336 if host in [x[0] for x in hosts]:
337 if len(hosts) == 1:
338 # Single not connected interface
339 continue
340 for remote,ifacedump in hosts:
341 if remote == host:
342 # This side of the interface
343 continue
344 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
345 output += '<li><a href="%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
346 output += "</ul>"
347 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
348
349 output += "<hr /><em><a href='..'>Back to overview</a></em>"
350 return output
351
352
353def generate_header(ctag="#"):
354 return """\
355%(ctag)s
356%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
357%(ctag)s Generated at %(date)s by %(host)s
358%(ctag)s
359""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname() }
360
361
362
363def parseaddr(s):
364 """ Process IPv4 CIDR notation addr to a (binary) number """
365 f = s.split('.')
366 return (long(f[0]) << 24L) + \
367 (long(f[1]) << 16L) + \
368 (long(f[2]) << 8L) + \
369 long(f[3])
370
371
372
373def showaddr(a):
374 """ Display IPv4 addr in (dotted) CIDR notation """
375 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
376
377
378def is_member(ip, mask, canidate):
379 """ Return True if canidate is part of ip/mask block"""
380 ip_addr = parseaddr(ip)
381 ip_canidate = parseaddr(canidate)
382 mask = int(mask)
383 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
384 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
385 return ip_addr == ip_canidate
386
387
388
389def cidr2netmask(netmask):
390 """ Given a 'netmask' return corresponding CIDR """
391 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
392
393def get_network(addr, mask):
394 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
395
396
397def generate_dhcpd_conf(datadump):
398 """ Generate config file '/usr/local/etc/dhcpd.conf """
399 output = generate_header()
400 output += Template("""\
401# option definitions common to all supported networks...
402option domain-name "dhcp.{{ autogen_fqdn }}";
403
404default-lease-time 600;
405max-lease-time 7200;
406
407# Use this to enble / disable dynamic dns updates globally.
408#ddns-update-style none;
409
410# If this DHCP server is the official DHCP server for the local
411# network, the authoritative directive should be uncommented.
412authoritative;
413
414# Use this to send dhcp log messages to a different log file (you also
415# have to hack syslog.conf to complete the redirection).
416log-facility local7;
417
418#
419# Interface definitions
420#
421\n""").render(datadump)
422
423 dhcp_out = defaultdict(list)
424 for iface_key in datadump['autogen_iface_keys']:
425 ifname = datadump[iface_key]['autogen_ifname']
426 if not datadump[iface_key].has_key('comment'):
427 datadump[iface_key]['comment'] = None
428 dhcp_out[ifname].append(" ## %(interface)s - %(comment)s\n" % datadump[iface_key])
429
430 (addr, mask) = datadump[iface_key]['ip'].split('/')
431 datadump[iface_key]['autogen_addr'] = addr
432 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
433 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
434 try:
435 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
436 except (AttributeError, ValueError, KeyError):
437 dhcp_out[ifname].append(" subnet %(autogen_subnet)s netmask %(autogen_netmask)s {\n ### not autoritive\n }\n" % datadump[iface_key])
438 continue
439
440 dhcp_part = ".".join(addr.split('.')[0:3])
441 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
442 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
443 dhcp_out[ifname].append("""\
444 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
445 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
446 option routers %(autogen_addr)s;
447 option domain-name-servers %(autogen_addr)s;
448 }
449""" % datadump[iface_key])
450
451 for ifname,value in dhcp_out.iteritems():
452 output += ("shared-network %s {\n" % ifname) + ''.join(value) + '}\n\n'
453 return output
454
455
456
457def generate_dnsmasq_conf(datadump):
458 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
459 output = generate_header()
460 output += Template("""\
461# DHCP server options
462dhcp-authoritative
463dhcp-fqdn
464domain=dhcp.{{ autogen_fqdn }}
465domain-needed
466expand-hosts
467log-async=100
468
469# Low memory footprint
470cache-size=10000
471
472\n""").render(datadump)
473
474 for iface_key in datadump['autogen_iface_keys']:
475 if not datadump[iface_key].has_key('comment'):
476 datadump[iface_key]['comment'] = None
477 output += "## %(interface)s - %(comment)s\n" % datadump[iface_key]
478
479 try:
480 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
481 (ip, cidr) = datadump[iface_key]['ip'].split('/')
482 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
483 except (AttributeError, ValueError, KeyError):
484 output += "# not autoritive\n\n"
485 continue
486
487 dhcp_part = ".".join(ip.split('.')[0:3])
488 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
489 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
490 output += "dhcp-range=%(interface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
491
492 return output
493
494
495ileiden_proxies = []
496normal_proxies = []
497rc_conf_local_cache = {}
498
499def generate_rc_conf_local(datadump):
500 """ Generate configuration file '/etc/rc.conf.local' """
501 item = datadump['autogen_item']
502 if rc_conf_local_cache.has_key(item):
503 return rc_conf_local_cache[item]
504
505 if not datadump.has_key('ileiden'):
506 datadump['autogen_ileiden_enable'] = False
507 else:
508 datadump['autogen_ileiden_enable'] = datadump['ileiden']
509
510 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
511
512 if not ileiden_proxies or not normal_proxies:
513 for proxy in get_proxylist():
514 proxydump = get_yaml(proxy)
515 if proxydump['ileiden']:
516 ileiden_proxies.append(proxydump)
517 else:
518 normal_proxies.append(proxydump)
519 for host in get_hybridlist():
520 hostdump = get_yaml(host)
521 if hostdump['service_proxy_ileiden']:
522 ileiden_proxies.append(hostdump)
523 if hostdump['service_proxy_normal']:
524 normal_proxies.append(hostdump)
525
526 datadump['autogen_ileiden_proxies'] = ileiden_proxies
527 datadump['autogen_normal_proxies'] = normal_proxies
528 datadump['autogen_ileiden_proxies_ips'] = ','.join([x['masterip'] for x in ileiden_proxies])
529 datadump['autogen_ileiden_proxies_names'] = ','.join([x['autogen_item'] for x in ileiden_proxies])
530 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
531 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
532
533 output = generate_header("#");
534 output += render_template(datadump, """\
535hostname='{{ autogen_fqdn }}'
536location='{{ location }}'
537nodetype="{{ nodetype }}"
538
539#
540# Configured listings
541#
542captive_portal_whitelist=""
543{% if nodetype == "Proxy" %}
544#
545# Proxy Configuration
546#
547{% if gateway -%}
548defaultrouter="{{ gateway }}"
549{% else -%}
550#defaultrouter="NOTSET"
551{% endif -%}
552internalif="{{ internalif }}"
553ileiden_enable="{{ autogen_ileiden_enable }}"
554gateway_enable="{{ autogen_ileiden_enable }}"
555pf_enable="yes"
556pf_rules="/etc/pf.conf"
557{% if autogen_ileiden_enable -%}
558pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={80,443}"
559lvrouted_enable="{{ autogen_ileiden_enable }}"
560lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
561{% else -%}
562pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
563{% endif -%}
564{% if internalroute -%}
565static_routes="wleiden"
566route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
567{% endif -%}
568
569{% elif nodetype == "Hybrid" %}
570 #
571 # Hybrid Configuration
572 #
573 list_ileiden_proxies="
574 {% for item in autogen_ileiden_proxies -%}
575 {{ "%-16s"|format(item.masterip) }} # {{ item.autogen_realname }}
576 {% endfor -%}
577 "
578 list_normal_proxies="
579 {% for item in autogen_normal_proxies -%}
580 {{ "%-16s"|format(item.masterip) }} # {{ item.autogen_realname }}
581 {% endfor -%}
582 "
583
584 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',')|default('none', true) }}"
585 externalif="{{ externalif|default('vr0', true) }}"
586 masterip="{{ masterip }}"
587
588 # Defined services
589 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
590 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
591 service_accesspoint="{{ service_accesspoint|yesorno }}"
592 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
593 #
594
595 {% if service_proxy_ileiden %}
596 pf_rules="/etc/pf.hybrid.conf"
597 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
598 pf_flags="$pf_flags -D publicnat=80,443"
599 {% elif service_proxy_normal or service_incoming_rdr %}
600 pf_rules="/etc/pf.hybrid.conf"
601 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
602 pf_flags="$pf_flags -D publicnat=0"
603 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
604 named_setfib="1"
605 tinyproxy_setfib="1"
606 dnsmasq_setfib="1"
607 sshd_setfib="1"
608 {% else %}
609 pf_rules="/etc/pf.node.conf"
610 pf_flags=""
611 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
612 {% endif %}
613
614 {% if service_proxy_normal %}
615 tinyproxy_enable="yes"
616 {% else %}
617 pen_wrapper_enable="yes"
618 {% endif %}
619
620 {% if service_accesspoint %}
621 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
622 {% endif %}
623
624 {% if board == "ALIX2" %}
625 #
626 # ''Fat'' configuration, board has 256MB RAM
627 #
628 dnsmasq_enable="NO"
629 named_enable="YES"
630 {% if autogen_dhcp_interfaces -%}
631 dhcpd_enable="YES"
632 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
633 {% endif -%}
634 {% endif -%}
635
636 {% if gateway %}
637 defaultrouter="{{ gateway }}"
638 {% endif %}
639{% elif nodetype == "CNode" %}
640#
641# NODE iLeiden Configuration
642#
643
644# iLeiden Proxies {{ autogen_ileiden_proxies_names }}
645list_ileiden_proxies="{{ autogen_ileiden_proxies_ips }}"
646# normal Proxies {{ autogen_normal_proxies_names }}
647list_normal_proxies="{{ autogen_normal_proxies_ips }}"
648
649captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
650
651lvrouted_flags="-u -s s00p3rs3kr3t -m 28 -z $list_ileiden_proxies"
652{% endif %}
653
654#
655# Interface definitions
656#\n
657""")
658
659 # lo0 configuration:
660 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
661 # - masterip is special as it needs to be assigned to at
662 # least one interface, so if not used assign to lo0
663 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
664 iface_map = {'lo0' : 'lo0'}
665 dhclient_if = {'lo0' : False}
666
667 masterip_used = False
668 for iface_key in datadump['autogen_iface_keys']:
669 if datadump[iface_key]['ip'].startswith(datadump['masterip']):
670 masterip_used = True
671 break
672 if not masterip_used:
673 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
674
675 for iface_key in datadump['autogen_iface_keys']:
676 ifacedump = datadump[iface_key]
677 ifname = ifacedump['autogen_ifname']
678
679 # Flag dhclient is possible
680 dhclient_if[ifname] = ifacedump.has_key('dhcpclient') and ifacedump['dhcpclient']
681
682 # Add interface IP to list
683 item = (ifacedump['ip'], ifacedump['comment'])
684 if addrs_list.has_key(ifname):
685 addrs_list[ifname].append(item)
686 else:
687 addrs_list[ifname] = [item]
688
689 # Alias only needs IP assignment for now, this might change if we
690 # are going to use virtual accesspoints
691 if "alias" in iface_key:
692 continue
693
694 # XXX: Might want to deduct type directly from interface name
695 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
696 # Default to station (client) mode
697 ifacedump['autogen_wlanmode'] = "sta"
698 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
699 ifacedump['autogen_wlanmode'] = "ap"
700 # Default to 802.11b mode
701 ifacedump['mode'] = '11b'
702 if ifacedump['type'] in ['11a', '11b' '11g']:
703 ifacedump['mode'] = ifacedump['type']
704
705 if not ifacedump.has_key('channel'):
706 if ifacedump['type'] == '11a':
707 ifacedump['channel'] = 36
708 else:
709 ifacedump['channel'] = 1
710
711 # Allow special hacks at the back like wds and stuff
712 if not ifacedump.has_key('extra'):
713 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
714 else:
715 ifacedump['autogen_extra'] = ifacedump['extra']
716
717
718 output += "wlans_%(interface)s='%(autogen_ifname)s'\n" % ifacedump
719 output += ("create_args_%(autogen_ifname)s='wlanmode %(autogen_wlanmode)s mode " +\
720 "%(mode)s ssid %(ssid)s %(autogen_extra)s channel %(channel)s'\n") % ifacedump
721
722 elif ifacedump['type'] in ['ethernet', 'eth']:
723 # No special config needed besides IP
724 pass
725 else:
726 assert False, "Unknown type " + ifacedump['type']
727
728 # Print IP address which needs to be assigned over here
729 output += "\n"
730 for iface,addrs in sorted(addrs_list.iteritems()):
731 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
732 output += "# %s || %s || %s\n" % (iface, addr, comment)
733
734 # Write DHCLIENT entry
735 if dhclient_if[iface]:
736 output += "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
737 else:
738 # Make sure the external address is always first as this is needed in the
739 # firewall setup
740 addrs = sorted(addrs,key=lambda x: x[0].split('.')[0], cmp=lambda x,y: cmp(1 if x == '172' else 0, 1 if y == '172' else 0))
741 output += "ipv4_addrs_%s='%s'\n\n" % (iface, " ".join([x[0] for x in addrs]))
742
743 rc_conf_local_cache[datadump['autogen_item']] = output
744 return output
745
746
747
748
749def get_all_configs():
750 """ Get dict with key 'host' with all configs present """
751 configs = dict()
752 for host in get_hostlist():
753 datadump = get_yaml(host)
754 configs[host] = datadump
755 return configs
756
757
758def get_interface_keys(config):
759 """ Quick hack to get all interface keys, later stage convert this to a iterator """
760 return sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
761
762
763def get_used_ips(configs):
764 """ Return array of all IPs used in config files"""
765 ip_list = []
766 for config in configs:
767 ip_list.append(config['masterip'])
768 for iface_key in get_interface_keys(config):
769 l = config[iface_key]['ip']
770 addr, mask = l.split('/')
771 # Special case do not process
772 if valid_addr(addr):
773 ip_list.append(addr)
774 else:
775 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
776 return sorted(ip_list)
777
778
779
780def generate_resolv_conf(datadump):
781 """ Generate configuration file '/etc/resolv.conf' """
782 # XXX: This should properly going to be an datastructure soon
783 datadump['autogen_header'] = generate_header("#")
784 datadump['autogen_edge_nameservers'] = ''
785 for host in get_proxylist():
786 hostdump = get_yaml(host)
787 datadump['autogen_edge_nameservers'] += "nameserver %(masterip)-15s # %(autogen_realname)s\n" % hostdump
788 for host in get_hybridlist():
789 hostdump = get_yaml(host)
790 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
791 datadump['autogen_edge_nameservers'] += "nameserver %(masterip)-15s # %(autogen_realname)s\n" % hostdump
792
793 return Template("""\
794{{ autogen_header }}
795search wleiden.net
796
797# Try local (cache) first
798nameserver 127.0.0.1
799
800{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
801nameserver 8.8.8.8 # Google Public NameServer
802nameserver 8.8.4.4 # Google Public NameServer
803{% else -%}
804# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
805{{ autogen_edge_nameservers }}
806{% endif -%}
807""").render(datadump)
808
809
810
811def generate_ntp_conf(datadump):
812 """ Generate configuration file '/etc/ntp.conf' """
813 # XXX: This should properly going to be an datastructure soon
814
815 datadump['autogen_header'] = generate_header("#")
816 datadump['autogen_ntp_servers'] = ''
817 for host in get_proxylist():
818 hostdump = get_yaml(host)
819 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(autogen_realname)s\n" % hostdump
820 for host in get_hybridlist():
821 hostdump = get_yaml(host)
822 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
823 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(autogen_realname)s\n" % hostdump
824
825 return Template("""\
826{{ autogen_header }}
827
828{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
829# Machine hooked to internet.
830server 0.nl.pool.ntp.org iburst maxpoll 9
831server 1.nl.pool.ntp.org iburst maxpoll 9
832server 2.nl.pool.ntp.org iburst maxpoll 9
833server 3.nl.pool.ntp.org iburst maxpoll 9
834{% else -%}
835# Local Wireless Leiden NTP Servers.
836server 0.pool.ntp.wleiden.net iburst maxpoll 9
837server 1.pool.ntp.wleiden.net iburst maxpoll 9
838server 2.pool.ntp.wleiden.net iburst maxpoll 9
839server 3.pool.ntp.wleiden.net iburst maxpoll 9
840
841# All the configured NTP servers
842{{ autogen_ntp_servers }}
843{% endif %}
844
845# If a server loses sync with all upstream servers, NTP clients
846# no longer follow that server. The local clock can be configured
847# to provide a time source when this happens, but it should usually
848# be configured on just one server on a network. For more details see
849# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
850# The use of Orphan Mode may be preferable.
851#
852server 127.127.1.0
853fudge 127.127.1.0 stratum 10
854""").render(datadump)
855
856
857def generate_pf_hybrid_conf_local(datadump):
858 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
859 datadump['autogen_header'] = generate_header("#")
860 return Template("""\
861{{ autogen_header }}
862
863# Redirect some internal facing services outside (7)
864# INFO: {{ rdr_rules|count }} rdr_rules (outside to internal redirect rules) defined.
865{% for protocol, src_port,dest_ip,dest_port in rdr_rules -%}
866rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ dest_ip }} port {{ dest_port }}
867{% endfor -%}
868""").render(datadump)
869
870def generate_motd(datadump):
871 """ Generate configuration file '/etc/motd' """
872 output = Template("""\
873FreeBSD run ``service motd onestart'' to make me look normal
874
875 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
876 Loc: {{ location }}
877
878Services:
879{% if board == "ALIX2" -%}
880 - Core Node ({{ board }})
881{% else -%}
882 - Hulp Node ({{ board }})
883{% endif -%}
884{% if service_proxy_normal -%}
885 - Normal Proxy
886{% endif -%}
887{% if service_proxy_ileiden -%}
888 - iLeiden Proxy
889{% endif -%}
890{% if service_incoming_rdr -%}
891 - Incoming port redirects
892{% endif %}
893Interlinks:\n
894""").render(datadump)
895
896 # XXX: This is a hacky way to get the required data
897 for line in generate_rc_conf_local(datadump).split('\n'):
898 if '||' in line and not line[1:].split()[0] in ['lo0', 'ath0'] :
899 output += " - %s \n" % line[1:]
900 output += """\
901Attached bridges:
902"""
903 for iface_key in datadump['autogen_iface_keys']:
904 ifacedump = datadump[iface_key]
905 if ifacedump.has_key('ns_ip'):
906 output += " - %(interface)s || %(mode)s || %(ns_ip)s\n" % ifacedump
907
908 return output
909
910
911def format_yaml_value(value):
912 """ Get yaml value in right syntax for outputting """
913 if isinstance(value,str):
914 output = '"%s"' % value
915 else:
916 output = value
917 return output
918
919
920
921def format_wleiden_yaml(datadump):
922 """ Special formatting to ensure it is editable"""
923 output = "# Genesis config yaml style\n"
924 output += "# vim:ts=2:et:sw=2:ai\n"
925 output += "#\n"
926 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
927 for key in sorted(set(datadump.keys()) - set(iface_keys)):
928 if key == 'rdr_rules':
929 output += '%-10s:\n' % 'rdr_rules'
930 for rdr_rule in datadump[key]:
931 output += '- %s\n' % rdr_rule
932 else:
933 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
934
935 output += "\n\n"
936
937 # Format (key, required)
938 key_order = (
939 ('comment', True),
940 ('interface', True),
941 ('ip', True),
942 ('desc', True),
943 ('sdesc', True),
944 ('mode', True),
945 ('type', True),
946 ('extra_type', False),
947 ('channel', False),
948 ('ssid', False),
949 ('dhcp', True),
950 ('dhcpclient', False),
951 ('compass', False),
952 ('distance', False),
953 ('ns_ip', False),
954 ('bullet2_ip', False),
955 ('ns_mac', False),
956 ('bullet2_mac', False),
957 ('ns_ssid', False),
958 ('ns_type', False),
959 ('status', True),
960 )
961
962 for iface_key in sorted(iface_keys):
963 try:
964 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
965 if remainder:
966 raise KeyError("invalid keys: %s" % remainder)
967
968 output += "%s:\n" % iface_key
969 for key,required in key_order:
970 if datadump[iface_key].has_key(key):
971 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
972 output += "\n\n"
973 except Exception as e:
974 print "# Error while processing interface %s" % iface_key
975 raise
976
977 return output
978
979
980
981def generate_wleiden_yaml(datadump, header=True):
982 """ Generate (petty) version of wleiden.yaml"""
983 for key in datadump.keys():
984 if key.startswith('autogen_'):
985 del datadump[key]
986 # Interface autogen cleanups
987 elif type(datadump[key]) == dict:
988 for key2 in datadump[key].keys():
989 if key2.startswith('autogen_'):
990 del datadump[key][key2]
991
992 output = generate_header("#") if header else ''
993 output += format_wleiden_yaml(datadump)
994 return output
995
996
997def generate_yaml(datadump):
998 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
999
1000
1001
1002def generate_config(node, config, datadump=None):
1003 """ Print configuration file 'config' of 'node' """
1004 output = ""
1005 try:
1006 # Load config file
1007 if datadump == None:
1008 datadump = get_yaml(node)
1009
1010 if config == 'wleiden.yaml':
1011 output += generate_wleiden_yaml(datadump)
1012 elif config == 'authorized_keys':
1013 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1014 output += f.read()
1015 f.close()
1016 elif config == 'dnsmasq.conf':
1017 output += generate_dnsmasq_conf(datadump)
1018 elif config == 'dhcpd.conf':
1019 output += generate_dhcpd_conf(datadump)
1020 elif config == 'rc.conf.local':
1021 output += generate_rc_conf_local(datadump)
1022 elif config == 'resolv.conf':
1023 output += generate_resolv_conf(datadump)
1024 elif config == 'ntp.conf':
1025 output += generate_ntp_conf(datadump)
1026 elif config == 'motd':
1027 output += generate_motd(datadump)
1028 elif config == 'pf.hybrid.conf.local':
1029 output += generate_pf_hybrid_conf_local(datadump)
1030 else:
1031 assert False, "Config not found!"
1032 except IOError, e:
1033 output += "[ERROR] Config file not found"
1034 return output
1035
1036
1037
1038def process_cgi_request():
1039 """ When calling from CGI """
1040 # Update repository if requested
1041 form = cgi.FieldStorage()
1042 if form.getvalue("action") == "update":
1043 print "Refresh: 5; url=."
1044 print "Content-type:text/plain\r\n\r\n",
1045 print "[INFO] Updating subverion, please wait..."
1046 print subprocess.Popen(['svn', 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0],
1047 print subprocess.Popen(['svn', 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0],
1048 print "[INFO] All done, redirecting in 5 seconds"
1049 sys.exit(0)
1050
1051
1052 base_uri = os.environ['PATH_INFO']
1053 uri = base_uri.strip('/').split('/')
1054
1055 output = "Template Holder"
1056 content_type='text/plain'
1057 if base_uri.endswith('/create/network.kml'):
1058 content_type='application/vnd.google-earth.kml+xml'
1059 output = make_network_kml.make_graph()
1060 elif not uri[0]:
1061 if is_text_request():
1062 content_type = 'text/plain'
1063 output = '\n'.join(get_hostlist())
1064 else:
1065 content_type = 'text/html'
1066 output = generate_title(get_hostlist())
1067 elif len(uri) == 1:
1068 if is_text_request():
1069 content_type = 'text/plain'
1070 output = generate_node(uri[0])
1071 else:
1072 content_type = 'text/html'
1073 output = generate_node_overview(uri[0])
1074 elif len(uri) == 2:
1075 content_type = 'text/plain'
1076 output = generate_config(uri[0], uri[1])
1077 else:
1078 assert False, "Invalid option"
1079
1080 print "Content-Type: %s" % content_type
1081 print "Content-Length: %s" % len(output)
1082 print ""
1083 print output
1084
1085def get_realname(datadump):
1086 # Proxy naming convention is special, as the proxy name is also included in
1087 # the nodename, when it comes to the numbered proxies.
1088 if datadump['nodetype'] == 'Proxy':
1089 realname = datadump['nodetype'] + datadump['nodename'].replace('proxy','')
1090 else:
1091 # By default the full name is listed and also a shortname CNAME for easy use.
1092 realname = datadump['nodetype'] + datadump['nodename']
1093 return(realname)
1094
1095
1096
1097def make_dns(output_dir = 'dns', external = False):
1098 items = dict()
1099
1100 # hostname is key, IP is value
1101 wleiden_zone = defaultdict(list)
1102 wleiden_cname = dict()
1103
1104 pool = dict()
1105 for node in get_hostlist():
1106 datadump = get_yaml(node)
1107
1108 # Proxy naming convention is special
1109 fqdn = datadump['autogen_realname']
1110 if datadump['nodetype'] in ['CNode', 'Hybrid']:
1111 wleiden_cname[datadump['nodename']] = fqdn
1112
1113 if datadump.has_key('rdr_host'):
1114 remote_target = datadump['rdr_host']
1115 elif datadump.has_key('remote_access') and datadump['remote_access']:
1116 remote_target = datadump['remote_access'].split(':')[0]
1117 else:
1118 remote_target = None
1119
1120 if remote_target:
1121 try:
1122 parseaddr(remote_target)
1123 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1124 except (IndexError, ValueError):
1125 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1126
1127
1128 wleiden_zone[fqdn].append((datadump['masterip'], True))
1129
1130 # Hacking to get proper DHCP IPs and hostnames
1131 for iface_key in get_interface_keys(datadump):
1132 iface_name = datadump[iface_key]['interface'].replace(':',"-alias-")
1133 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1134 try:
1135 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1136 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1137 dhcp_part = ".".join(ip.split('.')[0:3])
1138 if ip != datadump['masterip']:
1139 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, False))
1140 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1141 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1142 except (AttributeError, ValueError, KeyError):
1143 # First push it into a pool, to indentify the counter-part later on
1144 addr = parseaddr(ip)
1145 cidr = int(cidr)
1146 addr = addr & ~((1 << (32 - cidr)) - 1)
1147 if pool.has_key(addr):
1148 pool[addr] += [(iface_name, fqdn, ip)]
1149 else:
1150 pool[addr] = [(iface_name, fqdn, ip)]
1151 continue
1152
1153
1154 def pool_to_name(fqdn, pool_members):
1155 """Convert the joined name to a usable pool name"""
1156
1157 def isplit(item):
1158 (prefix, name, number) = re.match('^(cnode|hybrid|proxy)([a-z]+)([0-9]*)$',item.lower()).group(1,2,3)
1159 return (prefix, name, number)
1160
1161 my_name = isplit(fqdn.split('.')[0])[1]
1162
1163 short_names = defaultdict(list)
1164 for node in sorted(pool_members):
1165 (prefix, name, number) = isplit(node)
1166 short_names[name].append((prefix,number))
1167
1168 return '-'.join(sorted(short_names.keys()))
1169
1170
1171 # WL uses an /29 to configure an interface. IP's are ordered like this:
1172 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1173
1174 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1175
1176 # Automatic naming convention of interlinks namely 2 + remote.lower()
1177 for (key,value) in pool.iteritems():
1178 # Make sure they are sorted from low-ip to high-ip
1179 value = sorted(value, key=lambda x: parseaddr(x[2]))
1180
1181 if len(value) == 1:
1182 (iface_name, fqdn, ip) = value[0]
1183 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1184
1185 # Device DNS names
1186 if 'cnode' in fqdn.lower():
1187 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1188 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1189
1190 elif len(value) == 2:
1191 (a_iface_name, a_fqdn, a_ip) = value[0]
1192 (b_iface_name, b_fqdn, b_ip) = value[1]
1193 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1194 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1195
1196 # Device DNS names
1197 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1198 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1199 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1200 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1201 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1202 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1203 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1204
1205 else:
1206 pool_members = [k[1] for k in value]
1207 for item in value:
1208 (iface_name, fqdn, ip) = item
1209 pool_name = "2pool-" + pool_to_name(fqdn,pool_members)
1210 wleiden_zone["%s.%s" % (pool_name, fqdn)].append((ip, True))
1211
1212 # Include static DNS entries
1213 # XXX: Should they override the autogenerated results?
1214 # XXX: Convert input to yaml more useable.
1215 # Format:
1216 ##; this is a comment
1217 ## roomburgh=CNodeRoomburgh1
1218 ## apkerk1.CNodeVosko=172.17.176.8 ;this as well
1219 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'))
1220
1221 # Hack to allow special entries, for development
1222 wleiden_raw = {}
1223
1224 for line in dns_list:
1225 reverse = False
1226 k, items = line.items()[0]
1227 if type(items) == dict:
1228 if items.has_key('reverse'):
1229 reverse = items['reverse']
1230 items = items['a']
1231 else:
1232 items = items['cname']
1233 items = [items] if type(items) != list else items
1234 for item in items:
1235 if item.startswith('IN '):
1236 wleiden_raw[k] = item
1237 elif valid_addr(item):
1238 wleiden_zone[k].append((item, reverse))
1239 else:
1240 wleiden_cname[k] = item
1241
1242 details = dict()
1243 # 24 updates a day allowed
1244 details['serial'] = time.strftime('%Y%m%d%H')
1245
1246 if external:
1247 dns_masters = ['siteview.wirelessleiden.nl', 'ns1.vanderzwet.net']
1248 else:
1249 dns_masters = ['sunny.wleiden.net']
1250
1251 details['master'] = dns_masters[0]
1252 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1253
1254 dns_header = '''
1255$TTL 3h
1256%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 1d 12h 1w 60s )
1257 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1258
1259%(ns_servers)s
1260 \n'''
1261
1262
1263 if not os.path.isdir(output_dir):
1264 os.makedirs(output_dir)
1265 details['zone'] = 'wleiden.net'
1266 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1267 f.write(dns_header % details)
1268
1269 for host,items in wleiden_zone.iteritems():
1270 for ip,reverse in items:
1271 if ip not in ['0.0.0.0']:
1272 f.write("%s.wleiden.net. IN A %s \n" % (host.lower(), ip))
1273 for source,dest in wleiden_cname.iteritems():
1274 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1275 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1276 for source, dest in wleiden_raw.iteritems():
1277 f.write("%s.wleiden.net. %s\n" % (source, dest))
1278 f.close()
1279
1280 # Create whole bunch of specific sub arpa zones. To keep it compliant
1281 for s in range(16,32):
1282 details['zone'] = '%i.172.in-addr.arpa' % s
1283 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1284 f.write(dns_header % details)
1285
1286 #XXX: Not effient, fix to proper data structure and do checks at other
1287 # stages
1288 for host,items in wleiden_zone.iteritems():
1289 for ip,reverse in items:
1290 if not reverse:
1291 continue
1292 if valid_addr(ip):
1293 if valid_addr(ip):
1294 if int(ip.split('.')[1]) == s:
1295 rev_ip = '.'.join(reversed(ip.split('.')))
1296 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1297 f.close()
1298
1299
1300def usage():
1301 print """Usage: %(prog)s <argument>
1302Argument:
1303\tstandalone [port] = Run configurator webserver [8000]
1304\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1305\tfull-export = Generate yaml export script for heatmap.
1306\tstatic [outputdir] = Generate all config files and store on disk
1307\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1308\ttest <node> [<file>] = Receive output for certain node [all files].
1309\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1310\tlist <status> <items> = List systems which have certain status
1311
1312Arguments:
1313\t<node> = NodeName (example: HybridRick)
1314\t<file> = %(files)s
1315\t<status> = all|up|down|planned
1316\t<items> = systems|nodes|proxies
1317
1318NOTE FOR DEVELOPERS; you can test your changes like this:
1319 BEFORE any changes in this code:
1320 $ ./gformat.py static /tmp/pre
1321 AFTER the changes:
1322 $ ./gformat.py static /tmp/post
1323 VIEW differences and VERIFY all are OK:
1324 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1325""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) }
1326 exit(0)
1327
1328
1329def is_text_request():
1330 """ Find out whether we are calling from the CLI or any text based CLI utility """
1331 try:
1332 return os.environ['HTTP_USER_AGENT'].split()[0] in ['curl', 'fetch', 'wget']
1333 except KeyError:
1334 return True
1335
1336def switchFormat(setting):
1337 if setting:
1338 return "YES"
1339 else:
1340 return "NO"
1341
1342def rlinput(prompt, prefill=''):
1343 import readline
1344 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1345 try:
1346 return raw_input(prompt)
1347 finally:
1348 readline.set_startup_hook()
1349
1350def fix_conflict(left, right, default='i'):
1351 while True:
1352 print "## %-30s | %-30s" % (left, right)
1353 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1354 if not c:
1355 c = default
1356
1357 if c in ['l','1']:
1358 return left
1359 elif c in ['r','2']:
1360 return right
1361 elif c in ['e', '3']:
1362 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1363 elif c in ['i', '4']:
1364 return None
1365 else:
1366 print "#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c
1367
1368def main():
1369 """Hard working sub"""
1370 # Allow easy hacking using the CLI
1371 if not os.environ.has_key('PATH_INFO'):
1372 if len(sys.argv) < 2:
1373 usage()
1374
1375 if sys.argv[1] == "standalone":
1376 import SocketServer
1377 import CGIHTTPServer
1378 # Hop to the right working directory.
1379 os.chdir(os.path.dirname(__file__))
1380 try:
1381 PORT = int(sys.argv[2])
1382 except (IndexError,ValueError):
1383 PORT = 8000
1384
1385 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1386 """ Serve this CGI from the root of the webserver """
1387 def is_cgi(self):
1388 if "favicon" in self.path:
1389 return False
1390
1391 self.cgi_info = (os.path.basename(__file__), self.path)
1392 self.path = ''
1393 return True
1394 handler = MyCGIHTTPRequestHandler
1395 SocketServer.TCPServer.allow_reuse_address = True
1396 httpd = SocketServer.TCPServer(("", PORT), handler)
1397 httpd.server_name = 'localhost'
1398 httpd.server_port = PORT
1399
1400 logger.info("serving at port %s", PORT)
1401 try:
1402 httpd.serve_forever()
1403 except KeyboardInterrupt:
1404 httpd.shutdown()
1405 logger.info("All done goodbye")
1406 elif sys.argv[1] == "test":
1407 # Basic argument validation
1408 try:
1409 node = sys.argv[2]
1410 datadump = get_yaml(node)
1411 except IndexError:
1412 print "Invalid argument"
1413 exit(1)
1414 except IOError as e:
1415 print e
1416 exit(1)
1417
1418
1419 # Get files to generate
1420 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1421
1422 # Actual config generation
1423 for config in gen_files:
1424 logger.info("## Generating %s %s", node, config)
1425 print generate_config(node, config, datadump)
1426 elif sys.argv[1] == "test-cgi":
1427 os.environ['PATH_INFO'] = "/".join(sys.argv[2:])
1428 os.environ['SCRIPT_NAME'] = __file__
1429 process_cgi_request()
1430 elif sys.argv[1] == "static":
1431 items = dict()
1432 items['output_dir'] = sys.argv[2] if len(sys.argv) > 2 else "./static"
1433 for node in get_hostlist():
1434 items['node'] = node
1435 items['wdir'] = "%(output_dir)s/%(node)s" % items
1436 if not os.path.isdir(items['wdir']):
1437 os.makedirs(items['wdir'])
1438 datadump = get_yaml(node)
1439 for config in files:
1440 items['config'] = config
1441 logger.info("## Generating %(node)s %(config)s" % items)
1442 f = open("%(wdir)s/%(config)s" % items, "w")
1443 f.write(generate_config(node, config, datadump))
1444 f.close()
1445 elif sys.argv[1] == "wind-export":
1446 items = dict()
1447 for node in get_hostlist():
1448 datadump = get_yaml(node)
1449 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1450 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1451 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1452 VALUES (
1453 (SELECT id FROM users WHERE username = 'rvdzwet'),
1454 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1455 'Y');""" % datadump
1456 #for config in files:
1457 # items['config'] = config
1458 # print "## Generating %(node)s %(config)s" % items
1459 # f = open("%(wdir)s/%(config)s" % items, "w")
1460 # f.write(generate_config(node, config, datadump))
1461 # f.close()
1462 for node in get_hostlist():
1463 datadump = get_yaml(node)
1464 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1465 ifacedump = datadump[iface_key]
1466 if ifacedump.has_key('mode') and ifacedump['mode'] == 'ap-wds':
1467 ifacedump['nodename'] = datadump['nodename']
1468 if not ifacedump.has_key('channel') or not ifacedump['channel']:
1469 ifacedump['channel'] = 0
1470 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1471 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1472 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1473 elif sys.argv[1] == "full-export":
1474 hosts = {}
1475 for node in get_hostlist():
1476 datadump = get_yaml(node)
1477 hosts[datadump['nodename']] = datadump
1478 print yaml.dump(hosts)
1479
1480 elif sys.argv[1] == "dns":
1481 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
1482 elif sys.argv[1] == "cleanup":
1483 # First generate all datadumps
1484 datadumps = dict()
1485 ssid_to_node = dict()
1486 for host in get_hostlist():
1487 logger.info("# Processing: %s", host)
1488 # Set some boring default values
1489 datadump = { 'board' : 'UNKNOWN' }
1490 datadump.update(get_yaml(host))
1491 datadumps[datadump['autogen_realname']] = datadump
1492
1493 (poel, errors) = make_relations(datadumps)
1494 print "\n".join(["# WARNING: %s" % x for x in errors])
1495
1496 for host,datadump in datadumps.iteritems():
1497 try:
1498 # Convert all yes and no to boolean values
1499 def fix_boolean(dump):
1500 for key in dump.keys():
1501 if type(dump[key]) == dict:
1502 dump[key] = fix_boolean(dump[key])
1503 elif str(dump[key]).lower() in ["yes", "true"]:
1504 dump[key] = True
1505 elif str(dump[key]).lower() in ["no", "false"]:
1506 # Compass richting no (Noord Oost) is valid input
1507 if key != "compass": dump[key] = False
1508 return dump
1509 datadump = fix_boolean(datadump)
1510
1511 if datadump['rdnap_x'] and datadump['rdnap_y']:
1512 datadump['latitude'], datadump['longitude'] = rdnap.rd2etrs(datadump['rdnap_x'], datadump['rdnap_y'])
1513 elif datadump['latitude'] and datadump['longitude']:
1514 datadump['rdnap_x'], datadump['rdnap_y'] = rdnap.etrs2rd(datadump['latitude'], datadump['longitude'])
1515
1516 if datadump['nodename'].startswith('Proxy'):
1517 datadump['nodename'] = datadump['nodename'].lower()
1518
1519 for iface_key in datadump['autogen_iface_keys']:
1520 # All our normal wireless cards are normal APs now
1521 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
1522 datadump[iface_key]['mode'] = 'ap'
1523 # Wireless Leiden SSID have an consistent lowercase/uppercase
1524 if datadump[iface_key].has_key('ssid'):
1525 ssid = datadump[iface_key]['ssid']
1526 prefix = 'ap-WirelessLeiden-'
1527 if ssid.lower().startswith(prefix.lower()):
1528 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
1529 if datadump[iface_key].has_key('ns_ip') and not datadump[iface_key].has_key('mode'):
1530 datadump[iface_key]['mode'] = 'autogen-FIXME'
1531 if not datadump[iface_key].has_key('comment'):
1532 datadump[iface_key]['comment'] = 'autogen-FIXME'
1533
1534 if datadump[iface_key]['comment'].startswith('autogen-') and datadump[iface_key].has_key('comment'):
1535 datadump[iface_key] = datadump[iface_key]['desc']
1536
1537 x = datadump[iface_key]['comment']
1538 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
1539
1540
1541 if datadump[iface_key].has_key('desc'):
1542 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
1543 del datadump[iface_key]['desc']
1544 else:
1545 print "# ERROR: At %s - %s" % (datadump['nodename'], iface_key)
1546 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
1547 if response:
1548 datadump[iface_key]['comment'] = response
1549 del datadump[iface_key]['desc']
1550
1551 # Set the compass value based on the angle between the poels
1552 if datadump[iface_key].has_key('ns_ip'):
1553 my_pool = poel[network(datadump[iface_key]['ip'])]
1554 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
1555 if remote_hosts:
1556 compass_target = remote_hosts[0]
1557 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
1558
1559 store_yaml(datadump)
1560 except Exception as e:
1561 print "# Error while processing %s" % host
1562 raise
1563 elif sys.argv[1] == "list":
1564 use_fqdn = False
1565 if len(sys.argv) < 4 or not sys.argv[2] in ["up", "down", "planned", "all"]:
1566 usage()
1567 if sys.argv[3] == "nodes":
1568 systems = get_nodelist()
1569 elif sys.argv[3] == "proxies":
1570 systems = get_proxylist()
1571 elif sys.argv[3] == "systems":
1572 systems = get_hostlist()
1573 else:
1574 usage()
1575 if len(sys.argv) > 4:
1576 if sys.argv[4] == "fqdn":
1577 use_fqdn = True
1578 else:
1579 usage()
1580
1581 for system in systems:
1582 datadump = get_yaml(system)
1583
1584 output = datadump['autogen_fqdn'] if use_fqdn else system
1585 if sys.argv[2] == "all":
1586 print output
1587 elif datadump['status'] == sys.argv[2]:
1588 print output
1589 elif sys.argv[1] == "create":
1590 if sys.argv[2] == "network.kml":
1591 print make_network_kml.make_graph()
1592 else:
1593 usage()
1594 usage()
1595 else:
1596 # Do not enable debugging for config requests as it highly clutters the output
1597 if not is_text_request():
1598 cgitb.enable()
1599 process_cgi_request()
1600
1601
1602if __name__ == "__main__":
1603 main()
Note: See TracBrowser for help on using the repository browser.