source: genesis/tools/gformat.py@ 14389

Last change on this file since 14389 was 14384, checked in by rick, 5 years ago

Fix failing dns generation due to migration to python3
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 81.0 KB
Line 
1#!/usr/bin/env python3
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = next(filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn')))
35SVNVERSION = next(filter(os.path.isfile, ('/usr/local/bin/svnversion', '/usr/bin/svnversion')))
36
37import argparse
38import cgi
39import cgitb
40import codecs
41import copy
42import glob
43import make_network_kml
44import math
45import pyproj
46import random
47import re
48import socket
49import string
50import subprocess
51import textwrap
52import time
53import traceback
54import urllib
55
56from pprint import pprint
57from collections import defaultdict, OrderedDict
58from operator import itemgetter, attrgetter
59from sys import stderr
60try:
61 import yaml
62except ImportError as e:
63 print(e)
64 print("[ERROR] Please install the python-yaml or devel/py-yaml package")
65 exit(1)
66
67try:
68 from yaml import CLoader as Loader
69 from yaml import CDumper as Dumper
70except ImportError:
71 from yaml import Loader, Dumper
72
73from jinja2 import Environment, Template
74def yesorno(value):
75 return "YES" if bool(value) else "NO"
76env = Environment()
77env.filters['yesorno'] = yesorno
78def render_template(datadump, template):
79 result = env.from_string(template).render(datadump)
80 # Make it look pretty to the naked eye, as jinja templates are not so
81 # friendly when it comes to whitespace formatting
82 ## Remove extra whitespace at end of line lstrip() style.
83 result = re.sub(r'\n[\ ]+','\n', result)
84 ## Include only a single newline between an definition and a comment
85 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
86 ## Remove extra newlines after single comment
87 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
88 return result
89
90import logging
91logging.basicConfig(format='# %(levelname)s: %(message)s' )
92logger = logging.getLogger()
93logger.setLevel(logging.DEBUG)
94
95
96if os.environ.get('CONFIGROOT', None):
97 NODE_DIR = os.environ['CONFIGROOT']
98else:
99 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
100__version__ = '$Id: gformat.py 14384 2019-07-02 09:28:15Z rick $'
101
102CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
103
104files = [
105 'authorized_keys',
106 'dnsmasq.conf',
107 'dhcpd.conf',
108 'rc.conf.local',
109 'resolv.conf',
110 'motd',
111 'ntp.conf',
112 'pf.hybrid.conf.local',
113 'unbound.wleiden.conf',
114 'wleiden.yaml',
115 ]
116
117# Global variables uses
118OK = 10
119DOWN = 20
120UNKNOWN = 90
121
122
123ileiden_proxies = OrderedDict()
124normal_proxies = []
125datadump_cache = {}
126interface_list_cache = {}
127rc_conf_local_cache = {}
128nameservers_cache = []
129relations_cache = None
130
131NO_DHCP = 0
132DHCP_CLIENT = 10
133DHCP_SERVER = 20
134def dhcp_type(item):
135 if not 'dhcp' in item:
136 return NO_DHCP
137 elif not item['dhcp']:
138 return NO_DHCP
139 elif item['dhcp'].lower() == 'client':
140 return DHCP_CLIENT
141 else:
142 # Validation Checks
143 begin,end = map(int,item['dhcp'].split('-'))
144 if begin >= end:
145 raise ValueError("DHCP Start >= DHCP End")
146 return DHCP_SERVER
147
148def etrs2rd(lat, lon):
149 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
150 p2 = pyproj.Proj(init='EPSG:28992')
151 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
152 return (RDx, RDy)
153
154def rd2etrs(RDx, RDy):
155 p1 = pyproj.Proj(init='EPSG:28992')
156 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
157 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
158 return (lat, lon)
159
160def get_yaml(item,add_version_info=True):
161 try:
162 """ Get configuration yaml for 'item'"""
163 if item in datadump_cache:
164 return datadump_cache[item].copy()
165
166 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
167 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
168 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
169
170 # Default values
171 datadump = {
172 'autogen_revision' : 'NOTFOUND',
173 'autogen_gfile' : gfile,
174 'service_proxy_ileiden' : False,
175 'publicnat' : ['http', 'https'],
176 }
177 f = open(gfile, 'r')
178 datadump.update(yaml.load(f,Loader=Loader))
179 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
180 if datadump['nodetype'] == 'Hybrid':
181 # Some values are defined implicitly
182 if 'rdr_host' in datadump and datadump['rdr_host'] and not 'service_incoming_rdr' in datadump:
183 datadump['service_incoming_rdr'] = True
184 # Use some boring defaults
185 defaults = {
186 'service_proxy_normal' : False,
187 'service_accesspoint' : True,
188 'service_incoming_rdr' : False,
189 'service_concentrator' : False,
190 'monitoring_group' : 'wleiden',
191 }
192 for (key,value) in defaults.items():
193 if not key in datadump:
194 datadump[key] = value
195 f.close()
196
197 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
198 if add_version_info:
199 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True)
200 lines = p.communicate()[0].split('\n')
201 if p.returncode == 0:
202 for line in lines:
203 if line:
204 (key, value) = line.split(': ')
205 datadump["autogen_" + key.lower().replace(' ','_')] = value
206
207 # Preformat certain needed variables for formatting and push those into special object
208 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
209
210 wlan_count=0
211 try:
212 for key in get_interface_keys(datadump, True):
213 datadump[key]['autogen_ifbase'] = key.split('_')[1]
214 datadump[key]['autogen_vlan'] = False
215 datadump[key]['autogen_vlan_alias'] = False
216
217 datadump[key]['autogen_bridge_member'] = 'parent' in datadump[key]
218 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
219 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
220
221 if 'parent' in datadump[key]:
222 if 'ip' in datadump[key]:
223 raise ValueError("Interface bridge member cannot have IP assigned")
224 if 'dhcp' in datadump[key] and datadump[key]['dhcp'] != False:
225 raise ValueError("Interface bridge member cannot have DHCP set")
226
227 if 'ip' in datadump[key]:
228 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
229
230 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
231 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
232 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
233 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
234 wlan_count += 1
235 else:
236 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
237 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
238 datadump[key]['autogen_if_dhcp'] = '.'.join(key.split('_')[1:3])
239 datadump[key]['autogen_vlan'] = key.split('_')[2]
240 datadump[key]['autogen_vlan_alias'] = '_alias' in key
241 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
242 else:
243 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
244 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
245
246 except Exception as exc:
247 exc.args = ("# Error while processing interface %s" % key,) + exc.args
248 raise
249
250 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
251 if dhcp_type(datadump[key]) == DHCP_SERVER]
252
253 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
254 datadump['autogen_item'] = item
255
256 datadump['autogen_domain'] = datadump['domain'] if 'domain' in datadump else 'wleiden.net.'
257 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
258 datadump_cache[item] = datadump.copy()
259 except Exception as exc:
260 exc.args = ("# Error while processing %s" % item,) + exc.args
261 raise
262 return datadump
263
264
265def store_yaml(datadump, header=False):
266 """ Store configuration yaml for 'item'"""
267 item = datadump['autogen_item']
268 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
269
270 output = generate_wleiden_yaml(datadump, header)
271
272 f = open(gfile, 'w')
273 f.write(output)
274 f.close()
275
276
277def network(ip):
278 addr, mask = ip.split('/')
279 # Not parsing of these folks please
280 addr = parseaddr(addr)
281 mask = int(mask)
282 network = addr & ~((1 << (32 - mask)) - 1)
283 return network
284
285
286
287def make_relations():
288 """ Process _ALL_ yaml files to get connection relations """
289 global relations_cache
290
291 if relations_cache:
292 return relations_cache
293
294 errors = []
295 poel = defaultdict(list)
296
297 for host in get_hostlist():
298 datadump = get_yaml(host)
299 try:
300 for iface_key in get_interface_keys(datadump):
301 # Bridge members has no IP assigned
302 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
303 continue
304 net_addr = network(datadump[iface_key]['ip'])
305 poel[net_addr] += [(host,datadump[iface_key].copy())]
306 except (KeyError, ValueError) as e:
307 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
308 continue
309
310 relations_cache = (poel, errors)
311 return relations_cache
312
313
314
315def valid_addr(addr):
316 """ Show which address is valid in which are not """
317 return str(addr).startswith('172.')
318
319def get_hostlist():
320 """ Combined hosts and proxy list"""
321 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
322
323def angle_between_points(lat1,lat2,long1,long2):
324 """
325 Return Angle in radians between two GPS coordinates
326 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
327 """
328 dy = lat2 - lat1
329 dx = math.cos(lat1)*(long2 - long1)
330 angle = math.atan2(dy,dx)
331 return angle
332
333
334
335def angle_to_cd(angle):
336 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
337
338 # For easy conversion get positive degree
339 degrees = math.degrees(angle)
340 abs_degrees = 360 + degrees if degrees < 0 else degrees
341
342 # Numbers can be confusing calculate from the 4 main directions
343 p = 22.5
344 if abs_degrees < p:
345 cd = "n"
346 elif abs_degrees < (90 - p):
347 cd = "no"
348 elif abs_degrees < (90 + p):
349 cd = "o"
350 elif abs_degrees < (180 - p):
351 cd = "zo"
352 elif abs_degrees < (180 + p):
353 cd = "z"
354 elif abs_degrees < (270 - p):
355 cd = "zw"
356 elif abs_degrees < (270 + p):
357 cd = "w"
358 elif abs_degrees < (360 - p):
359 cd = "nw"
360 else:
361 cd = "n"
362 return cd
363
364
365
366def cd_between_hosts(hostA, hostB, datadumps):
367 # Using RDNAP coordinates
368 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
369 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
370 return angle_to_cd(math.atan2(dx,dy))
371
372 # GPS coordinates seems to fail somehow
373 #latA = float(datadumps[hostA]['latitude'])
374 #latB = float(datadumps[hostB]['latitude'])
375 #lonA = float(datadumps[hostA]['longitude'])
376 #lonB = float(datadumps[hostB]['longitude'])
377 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
378
379
380def generate_title(nodelist):
381 """ Main overview page """
382 items = { \
383 'root' : ".",
384 'version' : subprocess.Popen([SVNVERSION, "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
385 }
386 def fl(spaces, line):
387 return (' ' * spaces) + line + '\n'
388
389 output = """
390<html>
391 <head>
392 <title>Wireless leiden Configurator - GFormat</title>
393 <style type="text/css">
394 th {background-color: #999999}
395 tr:nth-child(odd) {background-color: #cccccc}
396 tr:nth-child(even) {background-color: #ffffff}
397 th, td {padding: 0.1em 1em}
398 </style>
399 </head>
400 <body>
401 <center>
402 <form type="GET" action="%(root)s">
403 <input type="hidden" name="action" value="update">
404 <input type="submit" value="Update Configuration Database (SVN)">
405 </form>
406 <table>
407 <caption><h3>Wireless Leiden Configurator - Revision %(version)s</h3></caption>
408 """ % items
409
410 for node in nodelist:
411 items['node'] = node
412 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
413 for config in files:
414 items['config'] = config
415 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
416 output += fl(5, "</tr>")
417 output += """
418 </table>
419 <hr />
420 <em>%s</em>
421 </center>
422 </body>
423</html>
424 """ % __version__
425
426 return output
427
428
429
430def generate_node(node):
431 """ Print overview of all files available for node """
432 return "\n".join(files)
433
434def generate_node_overview(host, datadump=False):
435 """ Print overview of all files available for node """
436 if not datadump:
437 datadump = get_yaml(host)
438 params = { 'host' : host }
439 output = "<em><a href='..'>Back to overview</a></em><hr />"
440 output += "<h2>Available files:</h2><ul>"
441 for cf in files:
442 params['cf'] = cf
443 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
444 output += "</ul>"
445
446 # Generate and connection listing
447 output += "<h2>Connected To:</h2><ul>"
448 (poel, errors) = make_relations()
449 for network, hosts in poel.items():
450 if host in [x[0] for x in hosts]:
451 if len(hosts) == 1:
452 # Single not connected interface
453 continue
454 for remote,ifacedump in hosts:
455 if remote == host:
456 # This side of the interface
457 continue
458 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
459 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
460 output += "</ul>"
461 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
462
463 output += "<hr /><em><a href='..'>Back to overview</a></em>"
464 return output
465
466
467def generate_header(datadump, ctag="#"):
468 return """\
469%(ctag)s
470%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
471%(ctag)s
472""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
473
474
475
476def parseaddr(s):
477 """ Process IPv4 CIDR notation addr to a (binary) number """
478 f = list(map(int, s.split('.')))
479 return ((f[0] << 24) + (f[1] << 16) + (f[2] << 8) + (f[3]))
480
481
482
483def showaddr(a):
484 """ Display IPv4 addr in (dotted) CIDR notation """
485 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
486
487
488def is_member(ip, mask, canidate):
489 """ Return True if canidate is part of ip/mask block"""
490 ip_addr = parseaddr(ip)
491 ip_canidate = parseaddr(canidate)
492 mask = int(mask)
493 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
494 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
495 return ip_addr == ip_canidate
496
497
498
499def cidr2netmask(netmask):
500 """ Given a 'netmask' return corresponding CIDR """
501 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
502
503def get_network(addr, mask):
504 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
505
506
507def generate_dhcpd_conf(datadump):
508 """ Generate config file '/usr/local/etc/dhcpd.conf """
509 # Redundency support, in cause local DNS server is not running/responding.
510 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
511 output = generate_header(datadump)
512 output += Template("""\
513# option definitions common to all supported networks...
514option domain-name "dhcp.{{ autogen_fqdn }}";
515
516default-lease-time 600;
517max-lease-time 7200;
518
519# Use this to enble / disable dynamic dns updates globally.
520#ddns-update-style none;
521
522# If this DHCP server is the official DHCP server for the local
523# network, the authoritative directive should be uncommented.
524authoritative;
525
526# Use this to send dhcp log messages to a different log file (you also
527# have to hack syslog.conf to complete the redirection).
528log-facility local7;
529
530# UniFi Discovery Support
531option space ubnt;
532option ubnt.unifi-address code 1 = ip-address;
533#
534class "ubnt" {
535 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
536 option vendor-class-identifier "ubnt";
537 vendor-option-space ubnt;
538}
539
540#
541# Interface definitions
542#
543\n\n""").render(datadump)
544
545
546 # TODO: Use textwrap.fill instead
547 def indent(text, count):
548 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
549
550 # Process DHCP blocks
551 dhcp_out = defaultdict(list)
552 for iface_key in get_interface_keys(datadump):
553 ifname = datadump[iface_key]['autogen_ifbase']
554 groupif = datadump[iface_key]['autogen_if_dhcp']
555 if not 'comment' in datadump[iface_key]:
556 datadump[iface_key]['comment'] = None
557
558 if not 'ip' in datadump[iface_key]:
559 continue
560
561 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
562
563 (addr, mask) = datadump[iface_key]['ip'].split('/')
564 datadump[iface_key]['autogen_addr'] = addr
565 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
566 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
567
568 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
569 dhcp_out[groupif].append(textwrap.dedent("""\
570 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
571 ### not autoritive
572 }
573 """ % datadump[iface_key]))
574 continue
575
576 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
577 dhcp_part = ".".join(addr.split('.')[0:3])
578 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
579 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
580 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
581
582 # Assume the first 10 IPs could be used for static entries
583 if 'no_portal' in datadump:
584 fixed = 5
585 for mac in datadump['no_portal']:
586 dhcp_out[groupif].append(textwrap.dedent("""\
587 host fixed-%(ifname)s-%(fixed)s {
588 hardware ethernet %(mac)s;
589 fixed-address %(prefix)s.%(fixed)s;
590 }
591 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
592 fixed += 1
593
594 if 'dhcp_fixed' in datadump[iface_key]:
595 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
596 dhcp_out[groupif].append(textwrap.dedent("""\
597 host fixed-%(host)s {
598 hardware ethernet %(mac)s;
599 fixed-address %(addr)s;
600 }
601 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
602
603
604 dhcp_out[groupif].append(textwrap.dedent("""\
605 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
606 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
607 option routers %(autogen_addr)s;
608 option domain-name-servers %(autogen_dns_servers)s;
609 option ubnt.unifi-address 172.17.107.10;
610 }
611 """ % datadump[iface_key]))
612
613 # Output the blocks in groups
614 for ifname,value in sorted(dhcp_out.items()):
615 output += ("shared-network \"%s\" {\n" % ifname) + indent(''.join(value), 2).rstrip() + '\n}\n\n'
616 return output
617
618
619
620def generate_dnsmasq_conf(datadump):
621 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
622 output = generate_header(datadump)
623 output += Template("""\
624# DHCP server options
625dhcp-authoritative
626dhcp-fqdn
627domain=dhcp.{{ autogen_fqdn }}
628domain-needed
629expand-hosts
630log-async=100
631
632# Low memory footprint
633cache-size=10000
634
635\n""").render(datadump)
636
637 for iface_key in get_interface_keys(datadump):
638 if not 'comment' in datadump[iface_key]:
639 datadump[iface_key]['comment'] = None
640 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
641
642 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
643 output += "# not autoritive\n\n"
644 continue
645
646 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
647 (ip, cidr) = datadump[iface_key]['ip'].split('/')
648 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
649
650 dhcp_part = ".".join(ip.split('.')[0:3])
651 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
652 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
653 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
654
655 return output
656
657
658class AutoVivification(dict):
659 """Implementation of perl's autovivification feature."""
660 def __getitem__(self, item):
661 try:
662 return dict.__getitem__(self, item)
663 except KeyError:
664 value = self[item] = type(self)()
665 return value
666
667def make_interface_list(datadump):
668 if datadump['autogen_item'] in interface_list_cache:
669 return (interface_list_cache[datadump['autogen_item']])
670 # lo0 configuration:
671 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
672 # - masterip is special as it needs to be assigned to at
673 # least one interface, so if not used assign to lo0
674 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
675 vlan_list = defaultdict(list)
676 bridge_list = defaultdict(list)
677 flags_if = AutoVivification()
678 dhclient_if = {'lo0' : False}
679
680 # XXX: Find some way of send this output nicely
681 output = ''
682
683 masterip_used = False
684 for iface_key in get_interface_keys(datadump):
685 if 'ip' in datadump[iface_key] and datadump[iface_key]['ip'].startswith(datadump['masterip']):
686 masterip_used = True
687 break
688 if not masterip_used:
689 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
690
691 if 'serviceid' in datadump:
692 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
693
694 for iface_key in get_interface_keys(datadump):
695 ifacedump = datadump[iface_key]
696 if ifacedump['autogen_bridge_alias']:
697 ifname = ifacedump['autogen_ifbase']
698 else:
699 ifname = ifacedump['autogen_ifname']
700
701 # If defined as vlan interface
702 if ifacedump['autogen_vlan']:
703 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
704
705 # If defined as bridge interface
706 if ifacedump['autogen_bridge_member']:
707 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
708
709
710 # Flag dhclient is possible
711 if not ifname in dhclient_if or dhclient_if[ifname] == False:
712 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
713
714 # Ethernet address
715 if 'ether' in ifacedump:
716 flags_if[ifname]['ether'] = ifacedump['ether']
717
718 # Handle special interface states
719 flags_if[ifname]['status'] = ifacedump['status']
720
721 # Add interface IP to list
722 if 'ip' in ifacedump:
723 item = (ifacedump['ip'], ifacedump['comment'])
724 if ifname in addrs_list:
725 addrs_list[ifname].append(item)
726 else:
727 addrs_list[ifname] = [item]
728
729 # Alias only needs IP assignment for now, this might change if we
730 # are going to use virtual accesspoints
731 if "alias" in iface_key:
732 continue
733
734 # XXX: Might want to deduct type directly from interface name
735 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
736 # Default to station (client) mode
737 ifacedump['autogen_wlanmode'] = "sta"
738 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
739 ifacedump['autogen_wlanmode'] = "ap"
740
741 if not 'channel' in ifacedump:
742 if ifacedump['type'] == '11a':
743 ifacedump['channel'] = 36
744 else:
745 ifacedump['channel'] = 1
746
747 # Allow special hacks at the back like wds and stuff
748 if not 'extra' in ifacedump:
749 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
750 else:
751 ifacedump['autogen_extra'] = ifacedump['extra']
752
753 ifacedump['autogen_ssid_hex'] = '0x' + codecs.encode(ifacedump['ssid'].encode('ascii'), 'hex').decode('ascii')
754
755 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
756 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
757 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
758 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
759 output += "\n"
760
761 elif ifacedump['type'] in ['ethernet', 'eth']:
762 # No special config needed besides IP
763 pass
764 elif ifacedump['type'] in ['vlan']:
765 # VLAN member has no special configuration
766 pass
767 else:
768 assert False, "Unknown type " + ifacedump['type']
769
770 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
771 interface_list_cache[datadump['autogen_item']] = store
772 return(store)
773
774
775
776def create_proxies_list():
777 if not ileiden_proxies or not normal_proxies:
778 # Placeholder for to-be-installed proxies, this will avoid updating the all
779 # nodes to include this new machine, yet due to an unbound issue, this list
780 # has to be kept small.
781
782 for i in range(1,20):
783 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
784
785 for host in get_hostlist():
786 hostdump = get_yaml(host)
787 if hostdump['status'] == 'up':
788 if hostdump['service_proxy_ileiden']:
789 ileiden_proxies[hostdump['serviceid']] = hostdump
790 if hostdump['service_proxy_normal']:
791 normal_proxies.append(hostdump)
792
793
794
795def generate_rc_conf_local(datadump):
796 """ Generate configuration file '/etc/rc.conf.local' """
797 item = datadump['autogen_item']
798 if item in rc_conf_local_cache:
799 return rc_conf_local_cache[item]
800
801 if not 'ileiden' in datadump:
802 datadump['autogen_ileiden_enable'] = False
803 else:
804 datadump['autogen_ileiden_enable'] = datadump['ileiden']
805
806 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
807
808 create_proxies_list()
809 datadump['autogen_ileiden_proxies'] = ileiden_proxies
810 datadump['autogen_normal_proxies'] = normal_proxies
811 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
812 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
813 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
814 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
815
816 output = generate_header(datadump, "#");
817 output += render_template(datadump, """\
818hostname='{{ autogen_fqdn }}'
819location='{{ location }}'
820nodetype="{{ nodetype }}"
821
822#
823# Configured listings
824#
825captive_portal_whitelist=""
826{% if nodetype == "Proxy" %}
827#
828# Proxy Configuration
829#
830{% if gateway and service_proxy_ileiden -%}
831defaultrouter="{{ gateway }}"
832{% else -%}
833#defaultrouter="NOTSET"
834{% endif -%}
835internalif="{{ internalif }}"
836ileiden_enable="{{ autogen_ileiden_enable }}"
837gateway_enable="{{ autogen_ileiden_enable }}"
838pf_enable="yes"
839pf_rules="/etc/pf.conf"
840{% if autogen_ileiden_enable -%}
841pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={{ publicnat|join(',') }} -D ileiden_ports={{ publicnat|join(',') }}"
842lvrouted_enable="{{ autogen_ileiden_enable }}"
843lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
844{% else -%}
845pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
846{% endif -%}
847{% if internalroute -%}
848static_routes="wleiden"
849route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
850{% endif -%}
851
852{% elif nodetype == "Hybrid" %}
853 #
854 # Hybrid Configuration
855 #
856 list_ileiden_proxies="
857 {% for serviceid,item in autogen_ileiden_proxies.items() -%}
858 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
859 {% endfor -%}
860 "
861 list_normal_proxies="
862 {% for item in autogen_normal_proxies -%}
863 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
864 {% endfor -%}
865 "
866
867 {% if autogen_dhcp_interfaces -%}
868 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
869 {% else %}
870 captive_portal_interfaces="dummy"
871 {% endif %}
872 externalif="{{ externalif|default('vr0', true) }}"
873 masterip="{{ masterip }}"
874
875 {% if gateway and service_proxy_ileiden %}
876 defaultrouter="{{ gateway }}"
877 {% else %}
878 #defaultrouter="NOTSET"
879 {% endif %}
880
881 #
882 # Defined services
883 #
884 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
885 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
886 service_accesspoint="{{ service_accesspoint|yesorno }}"
887 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
888 service_concentrator="{{ service_concentrator|yesorno }}"
889
890 {% if service_proxy_ileiden %}
891 pf_rules="/etc/pf.hybrid.conf"
892 {% if service_concentrator %}
893 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
894 {% else %}
895 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
896 {% endif %}
897 pf_flags="$pf_flags -D publicnat={{ publicnat|join(',') }}"
898 lvrouted_flags="$lvrouted_flags -g"
899 {% elif service_proxy_normal or service_incoming_rdr %}
900 pf_rules="/etc/pf.hybrid.conf"
901 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
902 pf_flags="$pf_flags -D publicnat=0"
903 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
904 named_setfib="1"
905 tinyproxy_setfib="1"
906 dnsmasq_setfib="1"
907 sshd_setfib="1"
908 {% else %}
909 named_auto_forward_only="YES"
910 pf_rules="/etc/pf.node.conf"
911 pf_flags="-D ileiden_ports={{ publicnat|join(',') }}"
912 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
913 {% endif %}
914 {% if service_concentrator %}
915 # Do mind installing certificates is NOT done automatically for security reasons
916 openvpn_enable="YES"
917 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
918 {% endif %}
919
920 {% if service_proxy_normal %}
921 tinyproxy_enable="yes"
922 {% else %}
923 pen_wrapper_enable="yes"
924 {% endif %}
925
926 {% if service_accesspoint %}
927 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
928 {% endif %}
929
930 {% if board == "ALIX2" or board == "net4801" %}
931 #
932 # ''Fat'' configuration, board has 256MB RAM
933 #
934 dnsmasq_enable="NO"
935 named_enable="YES"
936 unbound_enable="YES"
937 {% if autogen_dhcp_interfaces -%}
938 dhcpd_enable="YES"
939 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
940 {% endif -%}
941 {% elif board == "apu1d" %}
942 #
943 # ''Fat'' configuration, board has 1024MB RAM
944 #
945 dnsmasq_enable="NO"
946 unbound_enable="YES"
947 {% if autogen_dhcp_interfaces -%}
948 dhcpd_enable="YES"
949 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
950 {% endif -%}
951 {% endif -%}
952{% endif %}
953
954#
955# Script variables
956#
957attached_devices="{{ autogen_attached_devices|join(' ') }}"
958neighbours="{{ autogen_neighbours|join(' ') }}"
959
960
961#
962# Interface definitions
963#\n
964""")
965
966 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
967 for iface, vlans in sorted(vlan_list.items()):
968 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
969
970 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
971 for iface in sorted(vlan_list.keys()):
972 if not iface in addrs_list.keys():
973 output += "ifconfig_%s='up'\n" % iface
974
975 output += "\n"
976
977 # Bridge configuration:
978 if bridge_list.keys():
979 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
980
981 for iface in bridge_list.keys():
982 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
983
984 # Bridge member interfaces not containing a configuration should be marked active explcitly.
985 for _,members in bridge_list.items():
986 for iface in members:
987 if not iface in addrs_list.keys():
988 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
989
990 output += "\n"
991
992 # Details like SSID
993 if extra_ouput:
994 output += extra_ouput.strip() + "\n"
995
996 # Print IP address which needs to be assigned over here
997 output += "\n"
998 for iface,addrs in sorted(addrs_list.items()):
999 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1000 output += "# %s || %s || %s\n" % (iface, addr, comment)
1001
1002 prefix = ''
1003 if flags_if[iface]['status'] == 'broken':
1004 output += "# Interface %s disabled since status=broken\n" % iface
1005 prefix = '#'
1006
1007 # Write DHCLIENT entry
1008 if iface in dhclient_if and dhclient_if[iface]:
1009 output += prefix + "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1010 continue
1011
1012 # Make sure the external address is always first as this is needed in the
1013 # firewall setup
1014 addrs_tmp = []
1015 for addr in addrs:
1016 if addr[0].startswith('172'):
1017 addrs_tmp.insert(0, addr)
1018 else:
1019 addrs_tmp.append(addr)
1020 addrs = addrs_tmp
1021
1022 idx_offset = 0
1023 # Set MAC is required
1024 if 'ether' in flags_if[iface]:
1025 output += prefix + "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1026 output += prefix + "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1027 idx_offset += 1
1028 elif iface in bridge_list:
1029 output += prefix + "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1030 idx_offset += 1
1031 else:
1032 output += prefix + "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1033
1034 for idx, addr in enumerate(addrs[1:]):
1035 output += prefix + "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1036
1037 output += "\n"
1038
1039 rc_conf_local_cache[datadump['autogen_item']] = output
1040 return output
1041
1042
1043
1044def get_all_configs():
1045 """ Get dict with key 'host' with all configs present """
1046 configs = dict()
1047 for host in get_hostlist():
1048 datadump = get_yaml(host)
1049 configs[host] = datadump
1050 return configs
1051
1052
1053def get_interface_keys(config, extra=False):
1054 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1055 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1056 if extra == False:
1057 return list(filter(lambda x: not "extra" in x, elems))
1058 else:
1059 return elems
1060
1061
1062def get_used_ips(configs):
1063 """ Return array of all IPs used in config files"""
1064 ip_list = []
1065 for config in configs:
1066 ip_list.append(config['masterip'])
1067 if 'serviceid' in config:
1068 ip_list.append(config['serviceid'])
1069 for iface_key in get_interface_keys(config, True):
1070 l = config[iface_key]['ip']
1071 addr, mask = l.split('/')
1072 # Special case do not process
1073 if valid_addr(addr):
1074 ip_list.append(addr)
1075 else:
1076 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1077 return sorted(ip_list)
1078
1079
1080
1081def get_nameservers(max_servers=None):
1082 if nameservers_cache:
1083 return nameservers_cache[0:max_servers]
1084
1085 for host in get_hostlist():
1086 hostdump = get_yaml(host)
1087 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1088 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1089
1090 return nameservers_cache[0:max_servers]
1091
1092
1093def get_neighbours(datadump):
1094 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1095
1096 (poel, errors) = make_relations()
1097 table = []
1098 for iface,addrs in sorted(addrs_list.items()):
1099 if iface in ['lo0']:
1100 continue
1101
1102 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1103 if not addr.startswith('172.'):
1104 # Avoid listing internet connections as pool
1105 continue
1106 for neighbour in poel[network(addr)]:
1107 if neighbour[0] != datadump['autogen_item']:
1108 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1109 return table
1110
1111
1112def get_attached_devices(datadump, url=False):
1113 table = []
1114 for iface_key in get_interface_keys(datadump, True):
1115 ifacedump = datadump[iface_key]
1116
1117 if not 'ns_ip' in ifacedump:
1118 continue
1119
1120 x_ip = ifacedump['ns_ip'].split('/')[0]
1121
1122 if 'mode' in ifacedump:
1123 x_mode = ifacedump['mode']
1124 else:
1125 x_mode = 'unknown'
1126
1127 if 'bridge_type' in ifacedump:
1128 device_type = ifacedump['bridge_type']
1129 else:
1130 device_type = 'Unknown'
1131
1132 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, device_type))
1133 return table
1134
1135
1136def generate_resolv_conf(datadump):
1137 """ Generate configuration file '/etc/resolv.conf' """
1138 # XXX: This should properly going to be an datastructure soon
1139 datadump['autogen_header'] = generate_header(datadump, "#")
1140 datadump['autogen_edge_nameservers'] = ''
1141
1142
1143 for masterip,realname in get_nameservers():
1144 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1145
1146 return Template("""\
1147{{ autogen_header }}
1148search wleiden.net
1149
1150# Try local (cache) first
1151nameserver 127.0.0.1
1152
1153{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1154nameserver 8.8.8.8 # Google Public NameServer
1155nameserver 64.6.64.6 # Verisign Public NameServer
1156{% else -%}
1157# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1158{{ autogen_edge_nameservers }}
1159{% endif -%}
1160""").render(datadump)
1161
1162
1163
1164def generate_ntp_conf(datadump):
1165 """ Generate configuration file '/etc/ntp.conf' """
1166 # XXX: This should properly going to be an datastructure soon
1167
1168 datadump['autogen_header'] = generate_header(datadump, "#")
1169 datadump['autogen_ntp_servers'] = ''
1170 for host in get_hostlist():
1171 hostdump = get_yaml(host)
1172 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1173 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1174
1175 return Template("""\
1176{{ autogen_header }}
1177
1178{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1179# Machine hooked to internet.
1180server 0.nl.pool.ntp.org iburst maxpoll 9
1181server 1.nl.pool.ntp.org iburst maxpoll 9
1182server 2.nl.pool.ntp.org iburst maxpoll 9
1183server 3.nl.pool.ntp.org iburst maxpoll 9
1184{% else -%}
1185# Local Wireless Leiden NTP Servers.
1186server 0.pool.ntp.wleiden.net iburst maxpoll 9
1187server 1.pool.ntp.wleiden.net iburst maxpoll 9
1188server 2.pool.ntp.wleiden.net iburst maxpoll 9
1189server 3.pool.ntp.wleiden.net iburst maxpoll 9
1190
1191# All the configured NTP servers
1192{{ autogen_ntp_servers }}
1193{% endif %}
1194
1195# If a server loses sync with all upstream servers, NTP clients
1196# no longer follow that server. The local clock can be configured
1197# to provide a time source when this happens, but it should usually
1198# be configured on just one server on a network. For more details see
1199# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1200# The use of Orphan Mode may be preferable.
1201#
1202server 127.127.1.0
1203fudge 127.127.1.0 stratum 10
1204""").render(datadump)
1205
1206
1207def generate_pf_hybrid_conf_local(datadump):
1208 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1209 datadump['autogen_header'] = generate_header(datadump, "#")
1210 if datadump['service_incoming_rdr']:
1211 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1212 return Template("""\
1213{{ autogen_header }}
1214
1215# Redirect some internal facing services outside (7)
1216# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1217{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1218rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1219{% endfor -%}
1220# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1221{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1222rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1223{% endfor -%}
1224""").render(datadump)
1225
1226def generate_unbound_wleiden_conf(datadump):
1227 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1228 datadump['autogen_header'] = generate_header(datadump, "#")
1229
1230 autogen_ips = []
1231 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1232 for iface,addrs in sorted(addrs_list.items()):
1233 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1234 if addr.startswith('172'):
1235 autogen_ips.append((addr.split('/')[0], comment))
1236 datadump['autogen_ips'] = autogen_ips
1237
1238 create_proxies_list()
1239 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1240 return Template("""\
1241{{ autogen_header }}
1242
1243server:
1244 ## Static definitions fail to start on systems with broken ue(4) interfaces
1245{%- for ip,comment in autogen_ips %}
1246 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1247{%- endfor %}
1248 ## Enabling wildcard matching as work-around
1249 interface: 0.0.0.0
1250 interface: ::0
1251
1252forward-zone:
1253 name: '.'
1254{%- if service_proxy_ileiden %}
1255 forward-addr: 8.8.8.8 # Google DNS A
1256 forward-addr: 8.8.4.4 # Google DNS B
1257 forward-addr: 208.67.222.222 # OpenDNS DNS A
1258 forward-addr: 208.67.220.220 # OpenDNS DNS B
1259{% else -%}
1260{%- for serviceid,item in autogen_ileiden_proxies.items() %}
1261 {%- if loop.index <= 5 %}
1262 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1263 {%- endif %}
1264{%- endfor %}
1265{%- endif %}
1266""").render(datadump)
1267
1268def generate_motd(datadump):
1269 """ Generate configuration file '/etc/motd' """
1270 output = Template("""\
1271FreeBSD run ``service motd onestart'' to make me look normal
1272
1273 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1274 Loc: {{ location }}
1275
1276Services:
1277{% if board == "ALIX2" or board == "net4801" -%}
1278{{" -"}} Core Node ({{ board }})
1279{% else -%}
1280{{" -"}} Hulp Node ({{ board }})
1281{% endif -%}
1282{% if service_proxy_normal -%}
1283{{" -"}} Normal Proxy
1284{% endif -%}
1285{% if service_proxy_ileiden -%}
1286{{" -"}} iLeiden Proxy
1287{% endif -%}
1288{% if service_incoming_rdr -%}
1289{{" -"}} Incoming port redirects
1290{% endif %}
1291Interlinks:\n
1292""").render(datadump)
1293
1294
1295 def make_table(table):
1296 if not table:
1297 return " - none\n"
1298 else:
1299 lines = ""
1300 col_width = [max(len(x) for x in col) for col in zip(*table)]
1301 for row in table:
1302 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1303 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1304 return lines
1305
1306 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1307 table = []
1308 for iface,addrs in sorted(addrs_list.items()):
1309 if iface in ['lo0']:
1310 continue
1311 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1312 table.append((iface, addr, comment))
1313
1314 output += make_table(table)
1315 output += '\n'
1316 output += """\
1317Attached devices:
1318"""
1319 output += make_table(get_attached_devices(datadump, url=True))
1320 output += '\n'
1321 output += """\
1322Available neighbours:
1323"""
1324 output += make_table(get_neighbours(datadump))
1325
1326 return output
1327
1328
1329def format_yaml_value(value):
1330 """ Get yaml value in right syntax for outputting """
1331 if isinstance(value,str):
1332 output = '"%s"' % value
1333 else:
1334 output = value
1335 return output
1336
1337
1338
1339def format_wleiden_yaml(datadump):
1340 """ Special formatting to ensure it is editable"""
1341 output = "# Genesis config yaml style\n"
1342 output += "# vim:ts=2:et:sw=2:ai\n"
1343 output += "#\n"
1344 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1345 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1346 if key == 'rdr_rules':
1347 output += '%-10s:\n' % 'rdr_rules'
1348 for rdr_rule in datadump[key]:
1349 output += '- %s\n' % rdr_rule
1350 else:
1351 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1352
1353 output += "\n\n"
1354
1355 # Format (key, required)
1356 key_order = (
1357 ('comment', True),
1358 ('parent', False),
1359 ('ip', False),
1360 ('ipv6', False),
1361 ('ether', False),
1362 ('desc', True),
1363 ('sdesc', True),
1364 ('mode', True),
1365 ('type', True),
1366 ('extra_type', False),
1367 ('channel', False),
1368 ('ssid', False),
1369 ('wlan_mac', False),
1370 ('dhcp', True),
1371 ('dhcp_fixed', False),
1372 ('compass', False),
1373 ('distance', False),
1374 ('ns_ip', False),
1375 ('repeater_ip', False),
1376 ('bullet2_ip', False),
1377 ('ns_mac', False),
1378 ('bullet2_mac', False),
1379 ('ns_type', False),
1380 ('bridge_type', False),
1381 ('encrypted', False),
1382 ('status', True),
1383 )
1384
1385 for iface_key in sorted(iface_keys):
1386 try:
1387 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1388 if remainder:
1389 raise KeyError("invalid keys: %s" % remainder)
1390
1391 output += "%s:\n" % iface_key
1392 for key,required in key_order:
1393 if key in datadump[iface_key]:
1394 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1395 output += "\n\n"
1396 except Exception as exc:
1397 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
1398 raise
1399
1400 return output
1401
1402
1403
1404def generate_wleiden_yaml(datadump, header=True):
1405 """ Generate (petty) version of wleiden.yaml"""
1406 output = generate_header(datadump, "#") if header else ''
1407
1408 for key in list(datadump.keys()):
1409 if key.startswith('autogen_'):
1410 del datadump[key]
1411 # Interface autogen cleanups
1412 elif type(datadump[key]) == dict:
1413 for key2 in list(datadump[key].keys()):
1414 if key2.startswith('autogen_'):
1415 del datadump[key][key2]
1416
1417 output += format_wleiden_yaml(datadump)
1418 return output
1419
1420def generate_nanostation_config(datadump, iface, ns_type):
1421 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1422 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1423
1424 datadump.update(datadump['iface_%s' % iface])
1425
1426 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1427
1428def generate_yaml(datadump):
1429 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1430
1431
1432
1433def generate_config(node, config, datadump=None):
1434 """ Print configuration file 'config' of 'node' """
1435 output = ""
1436 try:
1437 # Load config file
1438 if datadump == None:
1439 datadump = get_yaml(node)
1440
1441 if config == 'wleiden.yaml':
1442 output += generate_wleiden_yaml(datadump)
1443 elif config == 'authorized_keys':
1444 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1445 output += f.read()
1446 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1447 # Fetch local keys if existing
1448 if os.path.exists(node_keys):
1449 output += open(node_keys, 'r').read()
1450 f.close()
1451 elif config == 'dnsmasq.conf':
1452 output += generate_dnsmasq_conf(datadump)
1453 elif config == 'dhcpd.conf':
1454 output += generate_dhcpd_conf(datadump)
1455 elif config == 'rc.conf.local':
1456 output += generate_rc_conf_local(datadump)
1457 elif config == 'resolv.conf':
1458 output += generate_resolv_conf(datadump)
1459 elif config == 'ntp.conf':
1460 output += generate_ntp_conf(datadump)
1461 elif config == 'motd':
1462 output += generate_motd(datadump)
1463 elif config == 'pf.hybrid.conf.local':
1464 output += generate_pf_hybrid_conf_local(datadump)
1465 elif config == 'unbound.wleiden.conf':
1466 output += generate_unbound_wleiden_conf(datadump)
1467 elif config.startswith('vr'):
1468 interface, ns_type = config.strip('.yaml').split('-')
1469 output += generate_nanostation_config(datadump, interface, ns_type)
1470 else:
1471 assert False, "Config not found!"
1472 except IOError as e:
1473 output += "[ERROR] Config file not found"
1474 return output
1475
1476
1477
1478def generate_static(output_dir, logging=True):
1479 items = {'output_dir' : output_dir}
1480 for node in get_hostlist():
1481 items['node'] = node
1482 items['wdir'] = "%(output_dir)s/%(node)s" % items
1483 if not os.path.isdir(items['wdir']):
1484 os.makedirs(items['wdir'])
1485 datadump = get_yaml(node)
1486 f = open("%(wdir)s/index.html" % items, "w")
1487 f.write(generate_node_overview(items['node'], datadump))
1488 f.close()
1489 for config in files:
1490 items['config'] = config
1491 if logging: logger.info("## Generating %(node)s %(config)s" % items)
1492 f = open("%(wdir)s/%(config)s" % items, "w")
1493 f.write(generate_config(node, config, datadump))
1494 f.close()
1495
1496
1497
1498def process_cgi_request(environ=os.environ):
1499 """ When calling from CGI """
1500 response_headers = []
1501 content_type = 'text/plain'
1502
1503 # Update repository if requested
1504 form = urllib.parse.parse_qs(environ.get('QUERY_STRING'))
1505 if form and 'action' in form and 'update' in form['action']:
1506 refresh_rate = 5
1507 output = "[INFO] Updating subverion, please wait...\n"
1508 old_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
1509 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
1510 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
1511 new_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
1512 if old_version != new_version or 'force' in environ.get('QUERY_STRING'):
1513 try:
1514 generate_static(CACHE_DIR, False)
1515 except:
1516 output += traceback.format_exc()
1517 refresh_rate = 120
1518 pass
1519 output += "[INFO] All done, redirecting in %s seconds" % refresh_rate
1520 response_headers += [
1521 ('Refresh', '%s; url=.' % refresh_rate),
1522 ]
1523 else:
1524 # Bootstap cache directory if none exists
1525 if not glob.glob(os.path.join(CACHE_DIR,'*','index.html')):
1526 generate_static(CACHE_DIR, False)
1527
1528 # URL must be of format <prefix>/config/<path>
1529 base_uri = environ['REQUEST_URI']
1530 uri = list(filter(None, base_uri.split('/config/')[1].strip('/').split('/')))
1531
1532 output = "Template Holder"
1533 if base_uri.endswith('/create/network.kml'):
1534 content_type='application/vnd.google-earth.kml+xml'
1535 output = make_network_kml.make_graph()
1536 elif base_uri.endswith('/api/get/nodeplanner.json'):
1537 content_type='application/json'
1538 output = make_network_kml.make_nodeplanner_json()
1539 elif not uri:
1540 if is_text_request(environ):
1541 output = '\n'.join(get_hostlist())
1542 else:
1543 content_type = 'text/html'
1544 output = generate_title(get_hostlist())
1545 elif len(uri) == 1:
1546 if is_text_request(environ):
1547 output = generate_node(uri[0])
1548 else:
1549 content_type = 'text/html'
1550 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1551 elif len(uri) == 2:
1552 output = generate_config(uri[0], uri[1])
1553 else:
1554 assert False, "Invalid option"
1555
1556 # Return response
1557 response_headers += [
1558 ('Content-type', content_type),
1559 ('Content-Length', str(len(output))),
1560 ]
1561 return(response_headers, str(output))
1562
1563
1564def make_dns(output_dir = 'dns', external = False):
1565 items = dict()
1566
1567 # hostname is key, IP is value
1568 wleiden_zone = defaultdict(list)
1569 wleiden_cname = dict()
1570
1571 pool = dict()
1572 for node in get_hostlist():
1573 datadump = get_yaml(node)
1574
1575 fqdn = datadump['nodename']
1576
1577 if 'rdr_host' in datadump:
1578 remote_target = datadump['rdr_host']
1579 elif 'remote_access' in datadump and datadump['remote_access']:
1580 remote_target = datadump['remote_access'].split(':')[0]
1581 else:
1582 remote_target = None
1583
1584 if remote_target:
1585 try:
1586 parseaddr(remote_target)
1587 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1588 except (IndexError, ValueError):
1589 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1590
1591
1592 wleiden_zone[fqdn].append((datadump['masterip'], True))
1593
1594 # Hacking to get proper DHCP IPs and hostnames
1595 for iface_key in get_interface_keys(datadump):
1596 iface_name = iface_key.replace('_','-')
1597 if 'ip' in datadump[iface_key]:
1598 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1599 try:
1600 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1601 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1602 dhcp_part = ".".join(ip.split('.')[0:3])
1603 if ip != datadump['masterip']:
1604 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1605 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1606 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1607 except (AttributeError, ValueError, KeyError):
1608 # First push it into a pool, to indentify the counter-part later on
1609 addr = parseaddr(ip)
1610 cidr = int(cidr)
1611 addr = addr & ~((1 << (32 - cidr)) - 1)
1612 if addr in pool:
1613 pool[addr] += [(iface_name, fqdn, ip)]
1614 else:
1615 pool[addr] = [(iface_name, fqdn, ip)]
1616 continue
1617
1618
1619
1620 # WL uses an /29 to configure an interface. IP's are ordered like this:
1621 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1622
1623 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1624
1625 # Automatic naming convention of interlinks namely 2 + remote.lower()
1626 for (key,value) in pool.items():
1627 # Make sure they are sorted from low-ip to high-ip
1628 value = sorted(value, key=lambda x: parseaddr(x[2]))
1629
1630 if len(value) == 1:
1631 (iface_name, fqdn, ip) = value[0]
1632 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1633
1634 # Device DNS names
1635 if 'cnode' in fqdn.lower():
1636 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1637 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1638
1639 elif len(value) == 2:
1640 (a_iface_name, a_fqdn, a_ip) = value[0]
1641 (b_iface_name, b_fqdn, b_ip) = value[1]
1642 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1643 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1644
1645 # Device DNS names
1646 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1647 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1648 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1649 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1650 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1651 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1652 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1653
1654 else:
1655 pool_members = [k[1] for k in value]
1656 for item in value:
1657 (iface_name, fqdn, ip) = item
1658 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1659
1660 # Include static DNS entries
1661 # XXX: Should they override the autogenerated results?
1662 # XXX: Convert input to yaml more useable.
1663 # Format:
1664 ##; this is a comment
1665 ## roomburgh=Roomburgh1
1666 ## apkerk1.Vosko=172.17.176.8 ;this as well
1667 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'), Loader=Loader)
1668
1669 # Hack to allow special entries, for development
1670 wleiden_raw = {}
1671
1672 for line in dns_list:
1673 reverse = False
1674 k, items = list(line.items())[0]
1675 if type(items) == dict:
1676 if 'reverse' in items:
1677 reverse = items['reverse']
1678 items = items['a']
1679 else:
1680 items = items['cname']
1681 items = [items] if type(items) != list else items
1682 for item in items:
1683 if item.startswith('IN '):
1684 wleiden_raw[k] = item
1685 elif valid_addr(item):
1686 wleiden_zone[k].append((item, reverse))
1687 else:
1688 wleiden_cname[k] = item
1689
1690 # Hack to get dynamic pool listing
1691 def chunks(l, n):
1692 return [l[i:i+n] for i in range(0, len(l), n)]
1693
1694 ntp_servers = [x[0] for x in get_nameservers()]
1695 for id, chunk in enumerate(chunks(ntp_servers,(math.floor(len(ntp_servers)/4)))):
1696 for ntp_server in chunk:
1697 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1698
1699 details = dict()
1700 # 24 updates a day allowed
1701 details['serial'] = time.strftime('%Y%m%d%H')
1702
1703 if external:
1704 dns_masters = ['ns1.vanderzwet.net', 'ns1.anywi.com']
1705 else:
1706 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1707
1708 details['master'] = dns_masters[0]
1709 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1710
1711 dns_header = '''
1712$TTL 3h
1713%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 3h )
1714 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1715
1716%(ns_servers)s
1717 \n'''
1718
1719
1720 if not os.path.isdir(output_dir):
1721 os.makedirs(output_dir)
1722 details['zone'] = 'wleiden.net'
1723 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1724 f.write(dns_header % details)
1725
1726 for host,items in wleiden_zone.items():
1727 for ip,reverse in items:
1728 if ip not in ['0.0.0.0']:
1729 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1730 for source,dest in wleiden_cname.items():
1731 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1732 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1733 for source, dest in wleiden_raw.items():
1734 f.write("%s.wleiden.net. %s\n" % (source, dest))
1735 f.close()
1736
1737 # Create whole bunch of specific sub arpa zones. To keep it compliant
1738 for s in range(16,32):
1739 details['zone'] = '%i.172.in-addr.arpa' % s
1740 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1741 f.write(dns_header % details)
1742
1743 #XXX: Not effient, fix to proper data structure and do checks at other
1744 # stages
1745 for host,items in wleiden_zone.items():
1746 for ip,reverse in items:
1747 if not reverse:
1748 continue
1749 if valid_addr(ip):
1750 if valid_addr(ip):
1751 if int(ip.split('.')[1]) == s:
1752 rev_ip = '.'.join(reversed(ip.split('.')))
1753 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1754 f.close()
1755
1756
1757def usage():
1758 print("""Usage: %(prog)s <argument>
1759Argument:
1760\tcleanup = Cleanup all YAML files to specified format
1761\tstandalone [port] = Run configurator webserver [8000]
1762\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1763\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1764\tfull-export = Generate yaml export script for heatmap.
1765\tstatic [outputdir] = Generate all config files and store on disk
1766\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1767\ttest <node> [<file>] = Receive output for certain node [all files].
1768\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1769\tlist <status> <items> = List systems which have certain status
1770\tcreate network.kml = Create Network KML file for use in Google Earth
1771
1772Arguments:
1773\t<node> = NodeName (example: HybridRick)
1774\t<file> = %(files)s
1775\t<status> = all|up|down|planned
1776\t<items> = systems|nodes|proxies
1777
1778NOTE FOR DEVELOPERS; you can test your changes like this:
1779 BEFORE any changes in this code:
1780 $ ./gformat.py static /tmp/pre
1781 AFTER the changes:
1782 $ ./gformat.py static /tmp/post
1783 VIEW differences and VERIFY all are OK:
1784 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1785""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) })
1786 exit(0)
1787
1788
1789def is_text_request(environ=os.environ):
1790 """ Find out whether we are calling from the CLI or any text based CLI utility """
1791 if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1792 return True
1793
1794 if 'HTTP_USER_AGENT' in environ:
1795 return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1796 else:
1797 return False
1798
1799
1800def switchFormat(setting):
1801 if setting:
1802 return "YES"
1803 else:
1804 return "NO"
1805
1806def rlinput(prompt, prefill=''):
1807 import readline
1808 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1809 try:
1810 return raw_input(prompt)
1811 finally:
1812 readline.set_startup_hook()
1813
1814def fix_conflict(left, right, default='i'):
1815 while True:
1816 print("## %-30s | %-30s" % (left, right))
1817 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1818 if not c:
1819 c = default
1820
1821 if c in ['l','1']:
1822 return left
1823 elif c in ['r','2']:
1824 return right
1825 elif c in ['e', '3']:
1826 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1827 elif c in ['i', '4']:
1828 return None
1829 else:
1830 print("#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c)
1831
1832
1833
1834def print_cgi_response(response_headers, output):
1835 for header in response_headers:
1836 print("%s: %s" % header)
1837 print("")
1838 print(output)
1839
1840
1841
1842def main():
1843 """Hard working sub"""
1844 # Allow easy hacking using the CLI
1845 if not os.environ.get('REQUEST_URI', None):
1846 if len(sys.argv) < 2:
1847 usage()
1848
1849 if sys.argv[1] == "standalone":
1850 import SocketServer
1851 import CGIHTTPServer
1852 # Hop to the right working directory.
1853 os.chdir(os.path.dirname(__file__))
1854 try:
1855 PORT = int(sys.argv[2])
1856 except (IndexError,ValueError):
1857 PORT = 8000
1858
1859 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1860 """ Serve this CGI from the root of the webserver """
1861 def is_cgi(self):
1862 if "favicon" in self.path:
1863 return False
1864
1865 self.cgi_info = (os.path.basename(__file__), self.path)
1866 self.path = ''
1867 return True
1868 handler = MyCGIHTTPRequestHandler
1869 SocketServer.TCPServer.allow_reuse_address = True
1870 httpd = SocketServer.TCPServer(("", PORT), handler)
1871 httpd.server_name = 'localhost'
1872 httpd.server_port = PORT
1873
1874 logger.info("serving at port %s", PORT)
1875 try:
1876 httpd.serve_forever()
1877 except KeyboardInterrupt:
1878 httpd.shutdown()
1879 logger.info("All done goodbye")
1880 elif sys.argv[1] == "test":
1881 # Basic argument validation
1882 try:
1883 node = sys.argv[2]
1884 except IndexError:
1885 print("Invalid argument")
1886 exit(1)
1887 except IOError as e:
1888 print(e)
1889 exit(1)
1890
1891 datadump = get_yaml(node)
1892
1893
1894 # Get files to generate
1895 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1896
1897 # Actual config generation
1898 for config in gen_files:
1899 logger.info("## Generating %s %s", node, config)
1900 print(generate_config(node, config, datadump))
1901 elif sys.argv[1] == "test-cgi":
1902 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1903 os.environ['SCRIPT_NAME'] = __file__
1904 response_headers, output = process_cgi_request()
1905 print_cgi_response(response_headers, output)
1906 elif sys.argv[1] == "static":
1907 generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
1908 elif sys.argv[1] == "wind-export":
1909 items = dict()
1910 for node in get_hostlist():
1911 datadump = get_yaml(node)
1912 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1913 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1914 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1915 VALUES (
1916 (SELECT id FROM users WHERE username = 'rvdzwet'),
1917 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1918 'Y');""" % datadump
1919 #for config in files:
1920 # items['config'] = config
1921 # print "## Generating %(node)s %(config)s" % items
1922 # f = open("%(wdir)s/%(config)s" % items, "w")
1923 # f.write(generate_config(node, config, datadump))
1924 # f.close()
1925 for node in get_hostlist():
1926 datadump = get_yaml(node)
1927 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1928 ifacedump = datadump[iface_key]
1929 if 'mode' in ifacedump and ifacedump['mode'] == 'ap-wds':
1930 ifacedump['nodename'] = datadump['nodename']
1931 if not 'channel' in ifacedump or not ifacedump['channel']:
1932 ifacedump['channel'] = 0
1933 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1934 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1935 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1936 elif sys.argv[1] == "smokeping-export":
1937 for host in get_hostlist():
1938 datadump = get_yaml(host)
1939 if datadump.get('service_proxy_normal', False) or datadump.get('service_proxy_ileiden', False):
1940 print(textwrap.dedent("""\
1941 ++ wleiden-gw-%(nodename)s
1942 menu = %(nodename)s.gw
1943 title = Wireless Leiden gateway %(nodename)s.gw.wleiden.net.
1944 host = %(nodename)s.gw.wleiden.net.
1945 """ % datadump))
1946 elif sys.argv[1] == "nagios-export":
1947 try:
1948 heavy_load = (sys.argv[2] == "--heavy-load")
1949 except IndexError:
1950 heavy_load = False
1951
1952 hostgroup_details = {
1953 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1954 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1955 'walphen' : 'Stichting Wireless Alphen',
1956 'westeinder' : 'Westeinder Plassen',
1957 }
1958
1959 # Convert IP to Host
1960 ip2host = {'root' : 'root'}
1961 for host in get_hostlist():
1962 datadump = get_yaml(host)
1963 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1964 for iface in get_interface_keys(datadump):
1965 if 'autogen_gateway' in datadump[iface]:
1966 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1967
1968 # Find dependency tree based on output of lvrouted.mytree of nearest node
1969 parents = defaultdict(list)
1970 stack = ['root']
1971 prev_depth = 0
1972 if os.path.isfile('lvrouted.mytree'):
1973 for line in open('lvrouted.mytree').readlines():
1974 depth = line.count('\t')
1975 ip = line.strip().split()[0]
1976
1977 if prev_depth < depth:
1978 try:
1979 parents[ip2host[ip]].append(ip2host[stack[-1]])
1980 except KeyError as e:
1981 print("# Unable to find %s in configuration files" % e.args[0])
1982 stack.append(ip)
1983 elif prev_depth > depth:
1984 stack = stack[:(depth - prev_depth)]
1985 elif prev_depth == depth:
1986 try:
1987 parents[ip2host[ip]].append(ip2host[stack[-1]])
1988 except KeyError as e:
1989 print("# Unable to find %s in configuration files" % e.args[0])
1990
1991
1992 prev_depth = depth
1993 # Observe that some nodes has themself as parent or multiple parents
1994 # for now take only the first parent, other behaviour is yet to be explained
1995
1996
1997
1998 params = {
1999 'check_interval' : 5 if heavy_load else 120,
2000 'retry_interval' : 1 if heavy_load else 10,
2001 'max_check_attempts' : 10 if heavy_load else 6,
2002 'notification_interval': 120 if heavy_load else 240,
2003 }
2004
2005 print('''\
2006define host {
2007 name wleiden-node ; Default Node Template
2008 use generic-host ; Use the standard template as initial starting point
2009 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2010 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2011 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2012 notification_interval %(notification_interval)s
2013 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2014 check_command check-host-alive ; Default command to check FreeBSD hosts
2015 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2016}
2017
2018define service {
2019 name wleiden-service ; Default Service Template
2020 use generic-service ; Use the standard template as initial starting point
2021 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2022 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2023 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2024 notification_interval %(notification_interval)s
2025 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2026 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2027}
2028
2029# Please make sure to install:
2030# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2031#
2032# Recompile net-mgmt/nagios-plugins to support check_snmp
2033# make -C /usr/ports/net-mgmt/nagios-plugins
2034#
2035# Install net/bind-tools to allow v2/check_dns_wl to work:
2036# pkg install bind-tools
2037#
2038define command{
2039 command_name check_snmp_disk
2040 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2041}
2042
2043define command{
2044 command_name check_netsnmp_load
2045 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2046}
2047
2048define command{
2049 command_name check_netsnmp_proc
2050 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2051}
2052
2053define command{
2054 command_name check_by_ssh
2055 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2056}
2057
2058define command{
2059 command_name check_dns_wl
2060 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2061}
2062
2063define command{
2064 command_name check_snmp_uptime
2065 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2066}
2067
2068
2069# TDB: dhcp leases
2070# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2071
2072# TDB: internet status
2073# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2074
2075# TDB: Advanced local passive checks
2076# /usr/local/libexec/nagios/check_by_ssh
2077''' % params)
2078
2079 print('''\
2080# Service Group, not displayed by default
2081define hostgroup {
2082 hostgroup_name srv_hybrid
2083 alias All Hybrid Nodes
2084 register 0
2085}
2086
2087define service {
2088 use wleiden-service
2089 hostgroup_name srv_hybrid
2090 service_description SSH
2091 check_command check_ssh
2092}
2093
2094define service {
2095 use wleiden-service,service-pnp
2096 hostgroup_name srv_hybrid
2097 service_description HTTP
2098 check_command check_http
2099}
2100
2101define service {
2102 use wleiden-service
2103 hostgroup_name srv_hybrid
2104 service_description DNS
2105 check_command check_dns_wl!"www.wirelessleiden.nl"
2106}
2107''')
2108
2109 if heavy_load:
2110 print('''\
2111define service {
2112 use wleiden-service
2113 hostgroup_name srv_hybrid
2114 service_description UPTIME
2115 check_command check_snmp_uptime
2116}
2117
2118#define service {
2119# use wleiden-service
2120# hostgroup_name srv_hybrid
2121# service_description NTP
2122# check_command check_ntp_peer
2123#}
2124
2125define service {
2126 use wleiden-service
2127 hostgroup_name srv_hybrid
2128 service_description LOAD
2129 check_command check_netsnmp_load
2130}
2131
2132define service {
2133 use wleiden-service
2134 hostgroup_name srv_hybrid
2135 service_description PROC
2136 check_command check_netsnmp_proc
2137}
2138
2139define service {
2140 use wleiden-service
2141 hostgroup_name srv_hybrid
2142 service_description DISK
2143 check_command check_snmp_disk
2144}
2145''')
2146 for node in get_hostlist():
2147 datadump = get_yaml(node)
2148 if not datadump['status'] == 'up':
2149 continue
2150 if not datadump['monitoring_group'] in hostgroup_details:
2151 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2152 print('''\
2153define host {
2154 use wleiden-node,host-pnp
2155 contact_groups admins
2156 host_name %(autogen_fqdn)s
2157 address %(masterip)s
2158 hostgroups srv_hybrid,%(monitoring_group)s\
2159''' % datadump)
2160 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2161 print('''\
2162 parents %(parents)s\
2163''' % { 'parents' : parents[datadump['autogen_fqdn']][0] })
2164 print('''\
2165}
2166''')
2167
2168
2169 for name,alias in hostgroup_details.items():
2170 print('''\
2171define hostgroup {
2172 hostgroup_name %s
2173 alias %s
2174} ''' % (name, alias))
2175
2176
2177 elif sys.argv[1] == "full-export":
2178 hosts = {}
2179 for node in get_hostlist():
2180 datadump = get_yaml(node)
2181 hosts[datadump['nodename']] = datadump
2182 print(yaml.dump(hosts))
2183
2184 elif sys.argv[1] == "dns":
2185 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2186 elif sys.argv[1] == "cleanup":
2187 # First generate all datadumps
2188 datadumps = dict()
2189 ssid_to_node = dict()
2190 for host in get_hostlist():
2191 logger.info("# Processing: %s", host)
2192 # Set some boring default values
2193 datadump = { 'board' : 'UNKNOWN' }
2194 datadump.update(get_yaml(host))
2195 datadumps[datadump['nodename']] = datadump
2196
2197 (poel, errors) = make_relations()
2198 print("\n".join(["# WARNING: %s" % x for x in errors]))
2199
2200 for host,datadump in datadumps.items():
2201 try:
2202 # Convert all yes and no to boolean values
2203 def fix_boolean(dump):
2204 for key in dump.keys():
2205 if type(dump[key]) == dict:
2206 dump[key] = fix_boolean(dump[key])
2207 elif str(dump[key]).lower() in ["yes", "true"]:
2208 dump[key] = True
2209 elif str(dump[key]).lower() in ["no", "false"]:
2210 # Compass richting no (Noord Oost) is valid input
2211 if key != "compass": dump[key] = False
2212 return dump
2213 datadump = fix_boolean(datadump)
2214
2215 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2216 if not 'latitude' in datadump and not 'longitude' in datadump:
2217 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2218 elif 'latitude' in datadump and 'longitude' in datadump:
2219 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2220 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2221 # TODO: Compare outcome of both coordinate systems and validate against each-other
2222
2223 if datadump['nodename'].startswith('Proxy'):
2224 datadump['nodename'] = datadump['nodename'].lower()
2225
2226 for iface_key in get_interface_keys(datadump):
2227 try:
2228 # All our normal wireless cards are normal APs now
2229 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2230 datadump[iface_key]['mode'] = 'ap'
2231 # Wireless Leiden SSID have an consistent lowercase/uppercase
2232 if 'ssid' in datadump[iface_key]:
2233 ssid = datadump[iface_key]['ssid']
2234 prefix = 'ap-WirelessLeiden-'
2235 if ssid.lower().startswith(prefix.lower()):
2236 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2237 if 'ns_ip' in datadump[iface_key] and not 'mode' in datadump[iface_key]:
2238 datadump[iface_key]['mode'] = 'autogen-FIXME'
2239 if not 'comment' in datadump[iface_key]:
2240 datadump[iface_key]['comment'] = 'autogen-FIXME'
2241
2242 if 'ns_mac' in datadump[iface_key]:
2243 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2244
2245 if 'comment' in datadump[iface_key] and datadump[iface_key]['comment'].startswith('autogen-'):
2246 datadump[iface_key] = datadump[iface_key]['desc']
2247
2248 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2249 # due to better collision avoidance.
2250 if datadump[iface_key]['type'] == '11b':
2251 datadump[iface_key]['type'] = '11g'
2252
2253 # Setting 802.11g channels to de-facto standards, to avoid
2254 # un-detected sharing with other overlapping channels
2255 #
2256 # Technically we could also use channel 13 in NL, but this is not
2257 # recommended as foreign devices might not be able to select this
2258 # channel. Secondly using 1,5,9,13 instead is going to clash with
2259 # the de-facto usage of 1,6,11.
2260 #
2261 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2262 channels_at_2400Mhz = (1,6,11)
2263 if datadump[iface_key]['type'] == '11g' and 'channel' in datadump[iface_key]:
2264 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2265 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2266 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2267
2268 # Mandatory interface keys
2269 if not 'status' in datadump[iface_key]:
2270 datadump[iface_key]['status'] = 'planned'
2271
2272 x = datadump[iface_key]['comment']
2273 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2274
2275 # Fixing bridge_type if none is found
2276 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2277 if not 'bridge_type' in datadump[iface_key]:
2278 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2279
2280 # Making sure description works
2281 if 'desc' in datadump[iface_key]:
2282 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2283 del datadump[iface_key]['desc']
2284 else:
2285 print("# ERROR: At %s - %s" % (datadump['nodename'], iface_key))
2286 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2287 if response:
2288 datadump[iface_key]['comment'] = response
2289 del datadump[iface_key]['desc']
2290
2291 # Check DHCP configuration
2292 dhcp_type(datadump[iface_key])
2293
2294 # Set the compass value based on the angle between the poels
2295 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2296 my_pool = poel[network(datadump[iface_key]['ip'])]
2297 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2298 if remote_hosts:
2299 compass_target = remote_hosts[0]
2300 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2301 # TODO: Compass wanted and actual direction might differ
2302
2303 # Monitoring Group default
2304 if not 'monitoring_group' in datadump:
2305 datadump['monitoring_group'] = 'wleiden'
2306
2307 except Exception as exc:
2308 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
2309 raise
2310 store_yaml(datadump)
2311 except Exception as exc:
2312 exc.args = ("# Error while processing %s" % host,) + exc.args
2313 raise
2314 elif sys.argv[1] == "list":
2315 use_fqdn = False
2316 if len(sys.argv) < 4:
2317 usage()
2318 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2319 usage()
2320 if not sys.argv[3] in ["nodes","proxies","systems"]:
2321 usage()
2322
2323 if len(sys.argv) > 4:
2324 if sys.argv[4] == "fqdn":
2325 use_fqdn = True
2326 else:
2327 usage()
2328
2329 for system in get_hostlist():
2330 datadump = get_yaml(system)
2331 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2332 continue
2333
2334 output = datadump['autogen_fqdn'] if use_fqdn else system
2335 if sys.argv[2] == "all":
2336 print(output)
2337 elif datadump['status'] == sys.argv[2]:
2338 print(output)
2339 elif sys.argv[1] == "create":
2340 if sys.argv[2] == "network.kml":
2341 print(make_network_kml.make_graph())
2342 elif sys.argv[2] == "host-ips.txt":
2343 for system in get_hostlist():
2344 datadump = get_yaml(system)
2345 ips = [datadump['masterip']]
2346 for ifkey in get_interface_keys(datadump):
2347 ips.append(datadump[ifkey]['ip'].split('/')[0])
2348 print(system, ' '.join(ips))
2349 elif sys.argv[2] == "host-pos.txt":
2350 for system in get_hostlist():
2351 datadump = get_yaml(system)
2352 print(system, datadump['rdnap_x'], datadump['rdnap_y'])
2353 elif sys.argv[2] == "nodeplanner.json":
2354 print(make_network_kml.make_nodeplanner_json())
2355 elif sys.argv[2] == 'ssh_config':
2356 print('''
2357Host *.wleiden.net
2358 User root
2359
2360Host 172.16.*.*
2361 User root
2362''')
2363 for system in get_hostlist():
2364 datadump = get_yaml(system)
2365 print('''\
2366Host %s
2367 User root
2368
2369Host %s
2370 User root
2371
2372Host %s
2373 User root
2374
2375Host %s
2376 User root
2377''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower()))
2378 else:
2379 usage()
2380 else:
2381 usage()
2382 else:
2383 # Do not enable debugging for config requests as it highly clutters the output
2384 if not is_text_request():
2385 cgitb.enable()
2386
2387 try:
2388 response_headers, output = process_cgi_request()
2389 except:
2390 print('')
2391 print('')
2392 raise
2393
2394 print_cgi_response(response_headers, output)
2395
2396if __name__ == "__main__":
2397 main()
Note: See TracBrowser for help on using the repository browser.