source: genesis/tools/gformat.py@ 14428

Last change on this file since 14428 was 14408, checked in by rick, 5 years ago

Add extra devices in motd listing for quick access
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 81.1 KB
Line 
1#!/usr/bin/env python3
2#
3# vim:ts=2:et:sw=2:ai
4# Wireless Leiden configuration generator, based on yaml files'
5#
6# XXX: This should be rewritten to make use of the ipaddr.py library.
7#
8# Sample apache configuration (mind the AcceptPathInfo!)
9# Alias /config /usr/local/www/config
10# <Directory /usr/local/www/config>
11# AddHandler cgi-script .py
12# Require all granted
13#
14# RewriteEngine on
15# RewriteCond %{REQUEST_FILENAME} !-f
16# RewriteRule ^(.*)$ gformat.py/$1 [L,QSA]
17# Options +FollowSymlinks +ExecCGI
18# </Directory>
19#
20# Package dependencies list:
21# FreeBSD:
22# pkg install devel/py-Jinja2 graphics/py-pyproj devel/py-yaml lang/python
23# Fedora:
24# yum install python-yaml pyproj proj-epsg python-jinja2
25#
26# Rick van der Zwet <info@rickvanderzwet.nl>
27#
28
29# Hack to make the script directory is also threated as a module search path.
30import sys
31import os
32sys.path.append(os.path.dirname(__file__))
33
34SVN = next(filter(os.path.isfile, ('/usr/local/bin/svn', '/usr/bin/svn')))
35SVNVERSION = next(filter(os.path.isfile, ('/usr/local/bin/svnversion', '/usr/bin/svnversion')))
36
37import argparse
38import cgi
39import cgitb
40import codecs
41import copy
42import glob
43import make_network_kml
44import math
45import pyproj
46import random
47import re
48import socket
49import string
50import subprocess
51import textwrap
52import time
53import traceback
54import urllib
55
56from pprint import pprint
57from collections import defaultdict, OrderedDict
58from operator import itemgetter, attrgetter
59from sys import stderr
60try:
61 import yaml
62except ImportError as e:
63 print(e)
64 print("[ERROR] Please install the python-yaml or devel/py-yaml package")
65 exit(1)
66
67try:
68 from yaml import CLoader as Loader
69 from yaml import CDumper as Dumper
70except ImportError:
71 from yaml import Loader, Dumper
72
73from jinja2 import Environment, Template
74def yesorno(value):
75 return "YES" if bool(value) else "NO"
76env = Environment()
77env.filters['yesorno'] = yesorno
78def render_template(datadump, template):
79 result = env.from_string(template).render(datadump)
80 # Make it look pretty to the naked eye, as jinja templates are not so
81 # friendly when it comes to whitespace formatting
82 ## Remove extra whitespace at end of line lstrip() style.
83 result = re.sub(r'\n[\ ]+','\n', result)
84 ## Include only a single newline between an definition and a comment
85 result = re.sub(r'(["\'])\n+([a-z]|\n#\n)',r'\1\n\2', result)
86 ## Remove extra newlines after single comment
87 result = re.sub(r'(#\n)\n+([a-z])',r'\1\2', result)
88 return result
89
90import logging
91logging.basicConfig(format='# %(levelname)s: %(message)s' )
92logger = logging.getLogger()
93logger.setLevel(logging.DEBUG)
94
95
96if os.environ.get('CONFIGROOT', None):
97 NODE_DIR = os.environ['CONFIGROOT']
98else:
99 NODE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/../nodes'
100__version__ = '$Id: gformat.py 14408 2019-12-16 20:49:59Z rick $'
101
102CACHE_DIR = os.path.abspath(os.path.dirname(__file__))
103
104files = [
105 'authorized_keys',
106 'dnsmasq.conf',
107 'dhcpd.conf',
108 'rc.conf.local',
109 'resolv.conf',
110 'motd',
111 'ntp.conf',
112 'pf.hybrid.conf.local',
113 'unbound.wleiden.conf',
114 'wleiden.yaml',
115 ]
116
117# Global variables uses
118OK = 10
119DOWN = 20
120UNKNOWN = 90
121
122
123ileiden_proxies = OrderedDict()
124normal_proxies = []
125datadump_cache = {}
126interface_list_cache = {}
127rc_conf_local_cache = {}
128nameservers_cache = []
129relations_cache = None
130
131NO_DHCP = 0
132DHCP_CLIENT = 10
133DHCP_SERVER = 20
134def dhcp_type(item):
135 if not 'dhcp' in item:
136 return NO_DHCP
137 elif not item['dhcp']:
138 return NO_DHCP
139 elif item['dhcp'].lower() == 'client':
140 return DHCP_CLIENT
141 else:
142 # Validation Checks
143 begin,end = map(int,item['dhcp'].split('-'))
144 if begin >= end:
145 raise ValueError("DHCP Start >= DHCP End")
146 return DHCP_SERVER
147
148def etrs2rd(lat, lon):
149 p1 = pyproj.Proj(proj='latlon',datum='WGS84')
150 p2 = pyproj.Proj(init='EPSG:28992')
151 RDx, RDy = pyproj.transform(p1,p2,lon, lat)
152 return (RDx, RDy)
153
154def rd2etrs(RDx, RDy):
155 p1 = pyproj.Proj(init='EPSG:28992')
156 p2 = pyproj.Proj(proj='latlon',datum='WGS84')
157 lon, lat = pyproj.transform(p1,p2, RDx, RDy)
158 return (lat, lon)
159
160def get_yaml(item,add_version_info=True):
161 try:
162 """ Get configuration yaml for 'item'"""
163 if item in datadump_cache:
164 return datadump_cache[item].copy()
165
166 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
167 global_rdr_file = os.path.join(NODE_DIR,'global_rdr_rules.yaml')
168 d = yaml.load(open(global_rdr_file, 'r'), Loader=Loader)
169
170 # Default values
171 datadump = {
172 'autogen_revision' : 'NOTFOUND',
173 'autogen_gfile' : gfile,
174 'service_proxy_ileiden' : False,
175 'publicnat' : ['http', 'https'],
176 }
177 f = open(gfile, 'r')
178 datadump.update(yaml.load(f,Loader=Loader))
179 datadump['autogen_global_rdr_rules'] = d['global_rdr_rules']
180 if datadump['nodetype'] == 'Hybrid':
181 # Some values are defined implicitly
182 if 'rdr_host' in datadump and datadump['rdr_host'] and not 'service_incoming_rdr' in datadump:
183 datadump['service_incoming_rdr'] = True
184 # Use some boring defaults
185 defaults = {
186 'service_proxy_normal' : False,
187 'service_accesspoint' : True,
188 'service_incoming_rdr' : False,
189 'service_concentrator' : False,
190 'monitoring_group' : 'wleiden',
191 }
192 for (key,value) in defaults.items():
193 if not key in datadump:
194 datadump[key] = value
195 f.close()
196
197 # Sometimes getting version information is useless or harmfull, like in the pre-commit hooks
198 if add_version_info:
199 p = subprocess.Popen([SVN, 'info', datadump['autogen_gfile']], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True)
200 lines = p.communicate()[0].split('\n')
201 if p.returncode == 0:
202 for line in lines:
203 if line:
204 (key, value) = line.split(': ')
205 datadump["autogen_" + key.lower().replace(' ','_')] = value
206
207 # Preformat certain needed variables for formatting and push those into special object
208 datadump['autogen_iface_keys'] = get_interface_keys(datadump)
209
210 wlan_count=0
211 try:
212 for key in get_interface_keys(datadump, True):
213 datadump[key]['autogen_ifbase'] = key.split('_')[1]
214 datadump[key]['autogen_vlan'] = False
215 datadump[key]['autogen_vlan_alias'] = False
216
217 datadump[key]['autogen_bridge_member'] = 'parent' in datadump[key]
218 datadump[key]['autogen_bridge'] = datadump[key]['autogen_ifbase'].startswith('bridge')
219 datadump[key]['autogen_bridge_alias'] = datadump[key]['autogen_ifbase'].startswith('bridge') and '_alias' in key
220
221 if 'parent' in datadump[key]:
222 if 'ip' in datadump[key]:
223 raise ValueError("Interface bridge member cannot have IP assigned")
224 if 'dhcp' in datadump[key] and datadump[key]['dhcp'] != False:
225 raise ValueError("Interface bridge member cannot have DHCP set")
226
227 if 'ip' in datadump[key]:
228 datadump[key]['autogen_gateway'] = datadump[key]['ip'].split('/')[0]
229
230 if datadump[key]['type'] in ['11a', '11b', '11g', 'wireless']:
231 datadump[key]['autogen_ifname'] = 'wlan%i' % wlan_count
232 datadump[key]['autogen_iface'] = 'wlan%i' % wlan_count
233 datadump[key]['autogen_if_dhcp'] = 'wlan%i' % wlan_count
234 wlan_count += 1
235 else:
236 datadump[key]['autogen_ifname'] = '_'.join(key.split('_')[1:])
237 if len(key.split('_')) > 2 and key.split('_')[2].isdigit():
238 datadump[key]['autogen_if_dhcp'] = '.'.join(key.split('_')[1:3])
239 datadump[key]['autogen_vlan'] = key.split('_')[2]
240 datadump[key]['autogen_vlan_alias'] = '_alias' in key
241 datadump[key]['autogen_iface'] = '.'.join(key.split('_')[1:])
242 else:
243 datadump[key]['autogen_if_dhcp'] = datadump[key]['autogen_ifbase']
244 datadump[key]['autogen_iface'] = '_'.join(key.split('_')[1:])
245
246 except Exception as exc:
247 exc.args = ("# Error while processing interface %s" % key,) + exc.args
248 raise
249
250 dhcp_interfaces = [datadump[key]['autogen_if_dhcp'] for key in datadump['autogen_iface_keys'] \
251 if dhcp_type(datadump[key]) == DHCP_SERVER]
252
253 datadump['autogen_dhcp_interfaces'] = [x.replace('_','.') for x in set(dhcp_interfaces)]
254 datadump['autogen_item'] = item
255
256 datadump['autogen_domain'] = datadump['domain'] if 'domain' in datadump else 'wleiden.net.'
257 datadump['autogen_fqdn'] = datadump['nodename'] + '.' + datadump['autogen_domain']
258 datadump_cache[item] = datadump.copy()
259 except Exception as exc:
260 exc.args = ("# Error while processing %s" % item,) + exc.args
261 raise
262 return datadump
263
264
265def store_yaml(datadump, header=False):
266 """ Store configuration yaml for 'item'"""
267 item = datadump['autogen_item']
268 gfile = os.path.join(NODE_DIR,item,'wleiden.yaml')
269
270 output = generate_wleiden_yaml(datadump, header)
271
272 f = open(gfile, 'w')
273 f.write(output)
274 f.close()
275
276
277def network(ip):
278 addr, mask = ip.split('/')
279 # Not parsing of these folks please
280 addr = parseaddr(addr)
281 mask = int(mask)
282 network = addr & ~((1 << (32 - mask)) - 1)
283 return network
284
285
286
287def make_relations():
288 """ Process _ALL_ yaml files to get connection relations """
289 global relations_cache
290
291 if relations_cache:
292 return relations_cache
293
294 errors = []
295 poel = defaultdict(list)
296
297 for host in get_hostlist():
298 datadump = get_yaml(host)
299 try:
300 for iface_key in get_interface_keys(datadump):
301 # Bridge members has no IP assigned
302 if 'parent' in datadump[iface_key] and not 'ip' in datadump[iface_key]:
303 continue
304 net_addr = network(datadump[iface_key]['ip'])
305 poel[net_addr] += [(host,datadump[iface_key].copy())]
306 except (KeyError, ValueError) as e:
307 errors.append("[FOUT] in '%s' interface '%s' (%s)" % (host,iface_key, type(e).__name__ + ': ' + str(e)))
308 continue
309
310 relations_cache = (poel, errors)
311 return relations_cache
312
313
314
315def valid_addr(addr):
316 """ Show which address is valid in which are not """
317 return str(addr).startswith('172.')
318
319def get_hostlist():
320 """ Combined hosts and proxy list"""
321 return sorted([os.path.basename(os.path.dirname(x)) for x in glob.glob("%s/*/wleiden.yaml" % (NODE_DIR))])
322
323def angle_between_points(lat1,lat2,long1,long2):
324 """
325 Return Angle in radians between two GPS coordinates
326 See: http://stackoverflow.com/questions/3809179/angle-between-2-gps-coordinates
327 """
328 dy = lat2 - lat1
329 dx = math.cos(lat1)*(long2 - long1)
330 angle = math.atan2(dy,dx)
331 return angle
332
333
334
335def angle_to_cd(angle):
336 """ Return Dutch Cardinal Direction estimation in 'one digit' of radian angle """
337
338 # For easy conversion get positive degree
339 degrees = math.degrees(angle)
340 abs_degrees = 360 + degrees if degrees < 0 else degrees
341
342 # Numbers can be confusing calculate from the 4 main directions
343 p = 22.5
344 if abs_degrees < p:
345 cd = "n"
346 elif abs_degrees < (90 - p):
347 cd = "no"
348 elif abs_degrees < (90 + p):
349 cd = "o"
350 elif abs_degrees < (180 - p):
351 cd = "zo"
352 elif abs_degrees < (180 + p):
353 cd = "z"
354 elif abs_degrees < (270 - p):
355 cd = "zw"
356 elif abs_degrees < (270 + p):
357 cd = "w"
358 elif abs_degrees < (360 - p):
359 cd = "nw"
360 else:
361 cd = "n"
362 return cd
363
364
365
366def cd_between_hosts(hostA, hostB, datadumps):
367 # Using RDNAP coordinates
368 dx = float(int(datadumps[hostA]['rdnap_x']) - int(datadumps[hostB]['rdnap_x'])) * -1
369 dy = float(int(datadumps[hostA]['rdnap_y']) - int(datadumps[hostB]['rdnap_y'])) * -1
370 return angle_to_cd(math.atan2(dx,dy))
371
372 # GPS coordinates seems to fail somehow
373 #latA = float(datadumps[hostA]['latitude'])
374 #latB = float(datadumps[hostB]['latitude'])
375 #lonA = float(datadumps[hostA]['longitude'])
376 #lonB = float(datadumps[hostB]['longitude'])
377 #return angle_to_cd(angle_between_points(latA, latB, lonA, lonB))
378
379
380def generate_title(nodelist):
381 """ Main overview page """
382 items = { \
383 'root' : ".",
384 'version' : subprocess.Popen([SVNVERSION, "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0]
385 }
386 def fl(spaces, line):
387 return (' ' * spaces) + line + '\n'
388
389 output = """
390<html>
391 <head>
392 <title>Wireless leiden Configurator - GFormat</title>
393 <style type="text/css">
394 th {background-color: #999999}
395 tr:nth-child(odd) {background-color: #cccccc}
396 tr:nth-child(even) {background-color: #ffffff}
397 th, td {padding: 0.1em 1em}
398 </style>
399 </head>
400 <body>
401 <center>
402 <form type="GET" action="%(root)s">
403 <input type="hidden" name="action" value="update">
404 <input type="submit" value="Update Configuration Database (SVN)">
405 </form>
406 <table>
407 <caption><h3>Wireless Leiden Configurator - Revision %(version)s</h3></caption>
408 """ % items
409
410 for node in nodelist:
411 items['node'] = node
412 output += fl(5, '<tr>') + fl(7,'<td><a href="%(root)s/%(node)s">%(node)s</a></td>' % items)
413 for config in files:
414 items['config'] = config
415 output += fl(7,'<td><a href="%(root)s/%(node)s/%(config)s">%(config)s</a></td>' % items)
416 output += fl(5, "</tr>")
417 output += """
418 </table>
419 <hr />
420 <em>%s</em>
421 </center>
422 </body>
423</html>
424 """ % __version__
425
426 return output
427
428
429
430def generate_node(node):
431 """ Print overview of all files available for node """
432 return "\n".join(files)
433
434def generate_node_overview(host, datadump=False):
435 """ Print overview of all files available for node """
436 if not datadump:
437 datadump = get_yaml(host)
438 params = { 'host' : host }
439 output = "<em><a href='..'>Back to overview</a></em><hr />"
440 output += "<h2>Available files:</h2><ul>"
441 for cf in files:
442 params['cf'] = cf
443 output += '<li><a href="%(cf)s">%(cf)s</a></li>\n' % params
444 output += "</ul>"
445
446 # Generate and connection listing
447 output += "<h2>Connected To:</h2><ul>"
448 (poel, errors) = make_relations()
449 for network, hosts in poel.items():
450 if host in [x[0] for x in hosts]:
451 if len(hosts) == 1:
452 # Single not connected interface
453 continue
454 for remote,ifacedump in hosts:
455 if remote == host:
456 # This side of the interface
457 continue
458 params = { 'remote': remote, 'remote_ip' : ifacedump['ip'] }
459 output += '<li><a href="../%(remote)s">%(remote)s</a> -- %(remote_ip)s</li>\n' % params
460 output += "</ul>"
461 output += "<h2>MOTD details:</h2><pre>" + generate_motd(datadump) + "</pre>"
462
463 output += "<hr /><em><a href='..'>Back to overview</a></em>"
464 return output
465
466
467def generate_header(datadump, ctag="#"):
468 return """\
469%(ctag)s
470%(ctag)s DO NOT EDIT - Automatically generated by 'gformat'
471%(ctag)s
472""" % { 'ctag' : ctag, 'date' : time.ctime(), 'host' : socket.gethostname(), 'revision' : datadump['autogen_revision'] }
473
474
475
476def parseaddr(s):
477 """ Process IPv4 CIDR notation addr to a (binary) number """
478 f = list(map(int, s.split('.')))
479 return ((f[0] << 24) + (f[1] << 16) + (f[2] << 8) + (f[3]))
480
481
482
483def showaddr(a):
484 """ Display IPv4 addr in (dotted) CIDR notation """
485 return "%d.%d.%d.%d" % ((a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff)
486
487
488def is_member(ip, mask, canidate):
489 """ Return True if canidate is part of ip/mask block"""
490 ip_addr = parseaddr(ip)
491 ip_canidate = parseaddr(canidate)
492 mask = int(mask)
493 ip_addr = ip_addr & ~((1 << (32 - mask)) - 1)
494 ip_canidate = ip_canidate & ~((1 << (32 - mask)) - 1)
495 return ip_addr == ip_canidate
496
497
498
499def cidr2netmask(netmask):
500 """ Given a 'netmask' return corresponding CIDR """
501 return showaddr(0xffffffff & (0xffffffff << (32 - int(netmask))))
502
503def get_network(addr, mask):
504 return showaddr(parseaddr(addr) & ~((1 << (32 - int(mask))) - 1))
505
506
507def generate_dhcpd_conf(datadump):
508 """ Generate config file '/usr/local/etc/dhcpd.conf """
509 # Redundency support, in cause local DNS server is not running/responding.
510 datadump['autogen_backup_dns_servers'] = [x[1] for x in get_neighbours(datadump)]
511 output = generate_header(datadump)
512 output += Template("""\
513# option definitions common to all supported networks...
514option domain-name "dhcp.{{ autogen_fqdn }}";
515
516default-lease-time 600;
517max-lease-time 7200;
518
519# Use this to enble / disable dynamic dns updates globally.
520#ddns-update-style none;
521
522# If this DHCP server is the official DHCP server for the local
523# network, the authoritative directive should be uncommented.
524authoritative;
525
526# Use this to send dhcp log messages to a different log file (you also
527# have to hack syslog.conf to complete the redirection).
528log-facility local7;
529
530# UniFi Discovery Support
531option space ubnt;
532option ubnt.unifi-address code 1 = ip-address;
533#
534class "ubnt" {
535 match if substring (option vendor-class-identifier, 0, 4) = "ubnt";
536 option vendor-class-identifier "ubnt";
537 vendor-option-space ubnt;
538}
539
540#
541# Interface definitions
542#
543\n\n""").render(datadump)
544
545
546 # TODO: Use textwrap.fill instead
547 def indent(text, count):
548 return '\n'.join(map(lambda x: ' ' * count + x, text.split('\n')))
549
550 # Process DHCP blocks
551 dhcp_out = defaultdict(list)
552 for iface_key in get_interface_keys(datadump):
553 ifname = datadump[iface_key]['autogen_ifbase']
554 groupif = datadump[iface_key]['autogen_if_dhcp']
555 if not 'comment' in datadump[iface_key]:
556 datadump[iface_key]['comment'] = None
557
558 if not 'ip' in datadump[iface_key]:
559 continue
560
561 dhcp_out[groupif].append("## %(autogen_iface)s - %(comment)s\n" % datadump[iface_key])
562
563 (addr, mask) = datadump[iface_key]['ip'].split('/')
564 datadump[iface_key]['autogen_addr'] = addr
565 datadump[iface_key]['autogen_netmask'] = cidr2netmask(mask)
566 datadump[iface_key]['autogen_subnet'] = get_network(addr, mask)
567
568 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
569 dhcp_out[groupif].append(textwrap.dedent("""\
570 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
571 ### not autoritive
572 }
573 """ % datadump[iface_key]))
574 continue
575
576 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
577 dhcp_part = ".".join(addr.split('.')[0:3])
578 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
579 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
580 datadump[iface_key]['autogen_dns_servers'] = ','.join([datadump[iface_key]['autogen_addr']] + datadump['autogen_backup_dns_servers'])
581
582 # Assume the first 10 IPs could be used for static entries
583 if 'no_portal' in datadump:
584 fixed = 5
585 for mac in datadump['no_portal']:
586 dhcp_out[groupif].append(textwrap.dedent("""\
587 host fixed-%(ifname)s-%(fixed)s {
588 hardware ethernet %(mac)s;
589 fixed-address %(prefix)s.%(fixed)s;
590 }
591 """ % { 'ifname' : ifname, 'mac' : mac, 'prefix': dhcp_part, 'fixed' : fixed }))
592 fixed += 1
593
594 if 'dhcp_fixed' in datadump[iface_key]:
595 for (mac,addr,host) in datadump[iface_key]['dhcp_fixed']:
596 dhcp_out[groupif].append(textwrap.dedent("""\
597 host fixed-%(host)s {
598 hardware ethernet %(mac)s;
599 fixed-address %(addr)s;
600 }
601 """ % { 'host' : host, 'mac' : mac, 'addr' : addr}))
602
603
604 dhcp_out[groupif].append(textwrap.dedent("""\
605 subnet %(autogen_subnet)s netmask %(autogen_netmask)s {
606 range %(autogen_dhcp_start)s %(autogen_dhcp_stop)s;
607 option routers %(autogen_addr)s;
608 option domain-name-servers %(autogen_dns_servers)s;
609 option ubnt.unifi-address 172.17.107.10;
610 }
611 """ % datadump[iface_key]))
612
613 # Output the blocks in groups
614 for ifname,value in sorted(dhcp_out.items()):
615 output += ("shared-network \"%s\" {\n" % ifname) + indent(''.join(value), 2).rstrip() + '\n}\n\n'
616 return output
617
618
619
620def generate_dnsmasq_conf(datadump):
621 """ Generate configuration file '/usr/local/etc/dnsmasq.conf' """
622 output = generate_header(datadump)
623 output += Template("""\
624# DHCP server options
625dhcp-authoritative
626dhcp-fqdn
627domain=dhcp.{{ autogen_fqdn }}
628domain-needed
629expand-hosts
630log-async=100
631
632# Low memory footprint
633cache-size=10000
634
635\n""").render(datadump)
636
637 for iface_key in get_interface_keys(datadump):
638 if not 'comment' in datadump[iface_key]:
639 datadump[iface_key]['comment'] = None
640 output += "## %(autogen_ifname)s - %(comment)s\n" % datadump[iface_key]
641
642 if dhcp_type(datadump[iface_key]) != DHCP_SERVER:
643 output += "# not autoritive\n\n"
644 continue
645
646 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
647 (ip, cidr) = datadump[iface_key]['ip'].split('/')
648 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
649
650 dhcp_part = ".".join(ip.split('.')[0:3])
651 datadump[iface_key]['autogen_dhcp_start'] = dhcp_part + "." + dhcp_start
652 datadump[iface_key]['autogen_dhcp_stop'] = dhcp_part + "." + dhcp_stop
653 output += "dhcp-range=%(autogen_iface)s,%(autogen_dhcp_start)s,%(autogen_dhcp_stop)s,%(autogen_netmask)s,24h\n\n" % datadump[iface_key]
654
655 return output
656
657
658class AutoVivification(dict):
659 """Implementation of perl's autovivification feature."""
660 def __getitem__(self, item):
661 try:
662 return dict.__getitem__(self, item)
663 except KeyError:
664 value = self[item] = type(self)()
665 return value
666
667def make_interface_list(datadump):
668 if datadump['autogen_item'] in interface_list_cache:
669 return (interface_list_cache[datadump['autogen_item']])
670 # lo0 configuration:
671 # - 172.32.255.1/32 is the proxy.wleiden.net deflector
672 # - masterip is special as it needs to be assigned to at
673 # least one interface, so if not used assign to lo0
674 addrs_list = { 'lo0' : [("127.0.0.1/8", "LocalHost"), ("172.31.255.1/32","Proxy IP")] }
675 vlan_list = defaultdict(list)
676 bridge_list = defaultdict(list)
677 flags_if = AutoVivification()
678 dhclient_if = {'lo0' : False}
679
680 # XXX: Find some way of send this output nicely
681 output = ''
682
683 masterip_used = False
684 for iface_key in get_interface_keys(datadump):
685 if 'ip' in datadump[iface_key] and datadump[iface_key]['ip'].startswith(datadump['masterip']):
686 masterip_used = True
687 break
688 if not masterip_used:
689 addrs_list['lo0'].append((datadump['masterip'] + "/32", 'Master IP Not used in interface'))
690
691 if 'serviceid' in datadump:
692 addrs_list['lo0'].append((datadump['serviceid'] + "/32", 'Lvrouted GW IP'))
693
694 for iface_key in get_interface_keys(datadump):
695 ifacedump = datadump[iface_key]
696 if ifacedump['autogen_bridge_alias']:
697 ifname = ifacedump['autogen_ifbase']
698 else:
699 ifname = ifacedump['autogen_ifname']
700
701 # If defined as vlan interface
702 if ifacedump['autogen_vlan']:
703 vlan_list[ifacedump['autogen_ifbase']].append(ifacedump['autogen_vlan'])
704
705 # If defined as bridge interface
706 if ifacedump['autogen_bridge_member']:
707 bridge_list[ifacedump['parent']].append(ifacedump['autogen_iface'])
708
709
710 # Flag dhclient is possible
711 if not ifname in dhclient_if or dhclient_if[ifname] == False:
712 dhclient_if[ifname] = dhcp_type(ifacedump) == DHCP_CLIENT
713
714 # Ethernet address
715 if 'ether' in ifacedump:
716 flags_if[ifname]['ether'] = ifacedump['ether']
717
718 # Handle special interface states
719 flags_if[ifname]['status'] = ifacedump['status']
720
721 # Add interface IP to list
722 if 'ip' in ifacedump:
723 item = (ifacedump['ip'], ifacedump['comment'])
724 if ifname in addrs_list:
725 addrs_list[ifname].append(item)
726 else:
727 addrs_list[ifname] = [item]
728
729 # Alias only needs IP assignment for now, this might change if we
730 # are going to use virtual accesspoints
731 if "alias" in iface_key:
732 continue
733
734 # XXX: Might want to deduct type directly from interface name
735 if ifacedump['type'] in ['11a', '11b', '11g', 'wireless']:
736 # Default to station (client) mode
737 ifacedump['autogen_wlanmode'] = "sta"
738 if ifacedump['mode'] in ['master', 'master-wds', 'ap', 'ap-wds']:
739 ifacedump['autogen_wlanmode'] = "ap"
740
741 if not 'channel' in ifacedump:
742 if ifacedump['type'] == '11a':
743 ifacedump['channel'] = 36
744 else:
745 ifacedump['channel'] = 1
746
747 # Allow special hacks at the back like wds and stuff
748 if not 'extra' in ifacedump:
749 ifacedump['autogen_extra'] = 'regdomain ETSI country NL'
750 else:
751 ifacedump['autogen_extra'] = ifacedump['extra']
752
753 ifacedump['autogen_ssid_hex'] = '0x' + codecs.encode(ifacedump['ssid'].encode('ascii'), 'hex').decode('ascii')
754
755 output += "wlans_%(autogen_ifbase)s='%(autogen_ifname)s'\n" % ifacedump
756 output += "# SSID is encoded in Hexadecimal to support spaces, plain text value is '%(ssid)s'\n" % ifacedump
757 output += ("create_args_%(autogen_ifname)s=\"wlanmode %(autogen_wlanmode)s mode " +\
758 "%(type)s ssid %(autogen_ssid_hex)s %(autogen_extra)s channel %(channel)s\"\n") % ifacedump
759 output += "\n"
760
761 elif ifacedump['type'] in ['ethernet', 'eth']:
762 # No special config needed besides IP
763 pass
764 elif ifacedump['type'] in ['vlan']:
765 # VLAN member has no special configuration
766 pass
767 else:
768 assert False, "Unknown type " + ifacedump['type']
769
770 store = (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, output)
771 interface_list_cache[datadump['autogen_item']] = store
772 return(store)
773
774
775
776def create_proxies_list():
777 if not ileiden_proxies or not normal_proxies:
778 # Placeholder for to-be-installed proxies, this will avoid updating the all
779 # nodes to include this new machine, yet due to an unbound issue, this list
780 # has to be kept small.
781
782 for i in range(1,20):
783 ileiden_proxies['172.31.254.%i' % i] = {'nodename' : 'unused'}
784
785 for host in get_hostlist():
786 hostdump = get_yaml(host)
787 if hostdump['status'] == 'up':
788 if hostdump['service_proxy_ileiden']:
789 ileiden_proxies[hostdump['serviceid']] = hostdump
790 if hostdump['service_proxy_normal']:
791 normal_proxies.append(hostdump)
792
793
794
795def generate_rc_conf_local(datadump):
796 """ Generate configuration file '/etc/rc.conf.local' """
797 item = datadump['autogen_item']
798 if item in rc_conf_local_cache:
799 return rc_conf_local_cache[item]
800
801 if not 'ileiden' in datadump:
802 datadump['autogen_ileiden_enable'] = False
803 else:
804 datadump['autogen_ileiden_enable'] = datadump['ileiden']
805
806 datadump['autogen_ileiden_enable'] = switchFormat(datadump['autogen_ileiden_enable'])
807
808 create_proxies_list()
809 datadump['autogen_ileiden_proxies'] = ileiden_proxies
810 datadump['autogen_normal_proxies'] = normal_proxies
811 datadump['autogen_normal_proxies_ips'] = ','.join([x['masterip'] for x in normal_proxies])
812 datadump['autogen_normal_proxies_names'] = ','.join([x['autogen_item'] for x in normal_proxies])
813 datadump['autogen_attached_devices'] = [x[2] for x in get_attached_devices(datadump)]
814 datadump['autogen_neighbours'] = [x[1] for x in get_neighbours(datadump)]
815
816 output = generate_header(datadump, "#");
817 output += render_template(datadump, """\
818hostname='{{ autogen_fqdn }}'
819location='{{ location }}'
820nodetype="{{ nodetype }}"
821
822#
823# Configured listings
824#
825captive_portal_whitelist=""
826{% if nodetype == "Proxy" %}
827#
828# Proxy Configuration
829#
830{% if gateway and service_proxy_ileiden -%}
831defaultrouter="{{ gateway }}"
832{% else -%}
833#defaultrouter="NOTSET"
834{% endif -%}
835internalif="{{ internalif }}"
836ileiden_enable="{{ autogen_ileiden_enable }}"
837gateway_enable="{{ autogen_ileiden_enable }}"
838pf_enable="yes"
839pf_rules="/etc/pf.conf"
840{% if autogen_ileiden_enable -%}
841pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={{ publicnat|join(',') }} -D ileiden_ports={{ publicnat|join(',') }}"
842lvrouted_enable="{{ autogen_ileiden_enable }}"
843lvrouted_flags="-u -s s00p3rs3kr3t -m 28"
844{% else -%}
845pf_flags="-D ext_if={{ externalif }} -D int_if={{ internalif }} -D publicnat={0}"
846{% endif -%}
847{% if internalroute -%}
848static_routes="wleiden"
849route_wleiden="-net 172.16.0.0/12 {{ internalroute }}"
850{% endif -%}
851
852{% elif nodetype == "Hybrid" %}
853 #
854 # Hybrid Configuration
855 #
856 list_ileiden_proxies="
857 {% for serviceid,item in autogen_ileiden_proxies.items() -%}
858 {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
859 {% endfor -%}
860 "
861 list_normal_proxies="
862 {% for item in autogen_normal_proxies -%}
863 {{ "%-16s"|format(item.serviceid) }} # {{ item.nodename }}
864 {% endfor -%}
865 "
866
867 {% if autogen_dhcp_interfaces -%}
868 captive_portal_interfaces="{{ autogen_dhcp_interfaces|join(',') }}"
869 {% else %}
870 captive_portal_interfaces="dummy"
871 {% endif %}
872 externalif="{{ externalif|default('vr0', true) }}"
873 masterip="{{ masterip }}"
874
875 {% if gateway and service_proxy_ileiden %}
876 defaultrouter="{{ gateway }}"
877 {% else %}
878 #defaultrouter="NOTSET"
879 {% endif %}
880
881 #
882 # Defined services
883 #
884 service_proxy_ileiden="{{ service_proxy_ileiden|yesorno }}"
885 service_proxy_normal="{{ service_proxy_normal|yesorno }}"
886 service_accesspoint="{{ service_accesspoint|yesorno }}"
887 service_incoming_rdr="{{ service_incoming_rdr|yesorno }}"
888 service_concentrator="{{ service_concentrator|yesorno }}"
889
890 {% if service_proxy_ileiden %}
891 pf_rules="/etc/pf.hybrid.conf"
892 {% if service_concentrator %}
893 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=tun0 -D inet_ip='(tun0)' -D masterip=$masterip"
894 {% else %}
895 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D inet_if=$externalif -D inet_ip='($externalif:0)' -D masterip=$masterip"
896 {% endif %}
897 pf_flags="$pf_flags -D publicnat={{ publicnat|join(',') }}"
898 lvrouted_flags="$lvrouted_flags -g"
899 {% elif service_proxy_normal or service_incoming_rdr %}
900 pf_rules="/etc/pf.hybrid.conf"
901 pf_flags="-D ext_if=$externalif -D ext_if_net=$externalif:network -D masterip=$masterip"
902 pf_flags="$pf_flags -D publicnat=0"
903 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
904 named_setfib="1"
905 tinyproxy_setfib="1"
906 dnsmasq_setfib="1"
907 sshd_setfib="1"
908 {% else %}
909 named_auto_forward_only="YES"
910 pf_rules="/etc/pf.node.conf"
911 pf_flags="-D ileiden_ports={{ publicnat|join(',') }}"
912 lvrouted_flags="$lvrouted_flags -z `make_list "$list_ileiden_proxies" ","`"
913 {% endif %}
914 {% if service_concentrator %}
915 # Do mind installing certificates is NOT done automatically for security reasons
916 openvpn_enable="YES"
917 openvpn_configfile="/usr/local/etc/openvpn/client.conf"
918 {% endif %}
919
920 {% if service_proxy_normal %}
921 tinyproxy_enable="yes"
922 {% else %}
923 pen_wrapper_enable="yes"
924 {% endif %}
925
926 {% if service_accesspoint %}
927 pf_flags="$pf_flags -D captive_portal_interfaces=$captive_portal_interfaces"
928 {% endif %}
929
930 {% if board == "ALIX2" or board == "net4801" %}
931 #
932 # ''Fat'' configuration, board has 256MB RAM
933 #
934 dnsmasq_enable="NO"
935 named_enable="YES"
936 unbound_enable="YES"
937 {% if autogen_dhcp_interfaces -%}
938 dhcpd_enable="YES"
939 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
940 {% endif -%}
941 {% elif board == "apu1d" %}
942 #
943 # ''Fat'' configuration, board has 1024MB RAM
944 #
945 dnsmasq_enable="NO"
946 unbound_enable="YES"
947 {% if autogen_dhcp_interfaces -%}
948 dhcpd_enable="YES"
949 dhcpd_flags="$dhcpd_flags {{ autogen_dhcp_interfaces|join(' ') }}"
950 {% endif -%}
951 {% endif -%}
952{% endif %}
953
954#
955# Script variables
956#
957attached_devices="{{ autogen_attached_devices|join(' ') }}"
958neighbours="{{ autogen_neighbours|join(' ') }}"
959
960
961#
962# Interface definitions
963#\n
964""")
965
966 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
967 for iface, vlans in sorted(vlan_list.items()):
968 output += 'vlans_%s="%s"\n' % (iface, ' '.join(sorted(set(vlans))))
969
970 # VLAN Parent interfaces not containing a configuration should be marked active explcitly.
971 for iface in sorted(vlan_list.keys()):
972 if not iface in addrs_list.keys():
973 output += "ifconfig_%s='up'\n" % iface
974
975 output += "\n"
976
977 # Bridge configuration:
978 if bridge_list.keys():
979 output += "cloned_interfaces='%s'\n" % ' '.join(bridge_list.keys())
980
981 for iface in bridge_list.keys():
982 output += "ifconfig_%s='%s up'\n" % (iface, ' '.join(['addm %(iface)s private %(iface)s' % {'iface': x} for x in bridge_list[iface]]))
983
984 # Bridge member interfaces not containing a configuration should be marked active explcitly.
985 for _,members in bridge_list.items():
986 for iface in members:
987 if not iface in addrs_list.keys():
988 output += "ifconfig_%s='up'\n" % iface.replace('.','_')
989
990 output += "\n"
991
992 # Details like SSID
993 if extra_ouput:
994 output += extra_ouput.strip() + "\n"
995
996 # Print IP address which needs to be assigned over here
997 output += "\n"
998 for iface,addrs in sorted(addrs_list.items()):
999 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1000 output += "# %s || %s || %s\n" % (iface, addr, comment)
1001
1002 prefix = ''
1003 if flags_if[iface]['status'] == 'broken':
1004 output += "# Interface %s disabled since status=broken\n" % iface
1005 prefix = '#'
1006
1007 # Write DHCLIENT entry
1008 if iface in dhclient_if and dhclient_if[iface]:
1009 output += prefix + "ifconfig_%s='SYNCDHCP'\n\n" % (iface)
1010 continue
1011
1012 # Make sure the external address is always first as this is needed in the
1013 # firewall setup
1014 addrs_tmp = []
1015 for addr in addrs:
1016 if addr[0].startswith('172'):
1017 addrs_tmp.insert(0, addr)
1018 else:
1019 addrs_tmp.append(addr)
1020 addrs = addrs_tmp
1021
1022 idx_offset = 0
1023 # Set MAC is required
1024 if 'ether' in flags_if[iface]:
1025 output += prefix + "ifconfig_%s='link %s'\n" % (iface, flags_if[iface]['ether'])
1026 output += prefix + "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1027 idx_offset += 1
1028 elif iface in bridge_list:
1029 output += prefix + "ifconfig_%s_alias0='inet %s'\n" % (iface, addrs[0][0])
1030 idx_offset += 1
1031 else:
1032 output += prefix + "ifconfig_%s='inet %s'\n" % (iface, addrs[0][0])
1033
1034 for idx, addr in enumerate(addrs[1:]):
1035 output += prefix + "ifconfig_%s_alias%s='inet %s'\n" % (iface, idx + idx_offset, addr[0])
1036
1037 output += "\n"
1038
1039 rc_conf_local_cache[datadump['autogen_item']] = output
1040 return output
1041
1042
1043
1044def get_all_configs():
1045 """ Get dict with key 'host' with all configs present """
1046 configs = dict()
1047 for host in get_hostlist():
1048 datadump = get_yaml(host)
1049 configs[host] = datadump
1050 return configs
1051
1052
1053def get_interface_keys(config, extra=False):
1054 """ Quick hack to get all interface keys, later stage convert this to a iterator """
1055 elems = sorted([elem for elem in config.keys() if (elem.startswith('iface_') and not "lo0" in elem)])
1056 if extra == False:
1057 return list(filter(lambda x: not "extra" in x, elems))
1058 else:
1059 return elems
1060
1061
1062def get_used_ips(configs):
1063 """ Return array of all IPs used in config files"""
1064 ip_list = []
1065 for config in configs:
1066 ip_list.append(config['masterip'])
1067 if 'serviceid' in config:
1068 ip_list.append(config['serviceid'])
1069 for iface_key in get_interface_keys(config, True):
1070 l = config[iface_key]['ip']
1071 addr, mask = l.split('/')
1072 # Special case do not process
1073 if valid_addr(addr):
1074 ip_list.append(addr)
1075 else:
1076 logger.error("## IP '%s' in '%s' not valid" % (addr, config['nodename']))
1077 return sorted(ip_list)
1078
1079
1080
1081def get_nameservers(max_servers=None):
1082 if nameservers_cache:
1083 return nameservers_cache[0:max_servers]
1084
1085 for host in get_hostlist():
1086 hostdump = get_yaml(host)
1087 if hostdump['status'] == 'up' and (hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']):
1088 nameservers_cache.append((hostdump['serviceid'], hostdump['nodename']))
1089
1090 return nameservers_cache[0:max_servers]
1091
1092
1093def get_neighbours(datadump):
1094 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1095
1096 (poel, errors) = make_relations()
1097 table = []
1098 for iface,addrs in sorted(addrs_list.items()):
1099 if iface in ['lo0']:
1100 continue
1101
1102 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1103 if not addr.startswith('172.'):
1104 # Avoid listing internet connections as pool
1105 continue
1106 for neighbour in poel[network(addr)]:
1107 if neighbour[0] != datadump['autogen_item']:
1108 table.append((iface, neighbour[1]['ip'].split('/')[0], neighbour[0] + " (" + neighbour[1]['autogen_iface'] + ")", neighbour[1]['comment']))
1109 return table
1110
1111
1112def get_attached_devices(datadump, url=False):
1113 table = []
1114 for iface_key in get_interface_keys(datadump, True):
1115 ifacedump = datadump[iface_key]
1116
1117 if 'mac' in ifacedump:
1118 x_ip = ifacedump['ip'].split('/')[0]
1119 x_mode = 'active' if ifacedump.get('status') == 'up' else 'disabled'
1120 elif 'ns_ip' in ifacedump:
1121 x_ip = ifacedump['ns_ip'].split('/')[0]
1122 x_mode = ifacedump.get('mode') or 'unknown'
1123 else:
1124 continue
1125
1126 device_type = ifacedump.get('bridge_type') or 'Unknown'
1127 table.append((ifacedump['autogen_iface'], x_mode, 'http://%s' % x_ip if url else x_ip, "%s (%s)" % (device_type, ifacedump.get('comment',''))))
1128
1129 return table
1130
1131
1132def generate_resolv_conf(datadump):
1133 """ Generate configuration file '/etc/resolv.conf' """
1134 # XXX: This should properly going to be an datastructure soon
1135 datadump['autogen_header'] = generate_header(datadump, "#")
1136 datadump['autogen_edge_nameservers'] = ''
1137
1138
1139 for masterip,realname in get_nameservers():
1140 datadump['autogen_edge_nameservers'] += "nameserver %-15s # %s\n" % (masterip, realname)
1141
1142 return Template("""\
1143{{ autogen_header }}
1144search wleiden.net
1145
1146# Try local (cache) first
1147nameserver 127.0.0.1
1148
1149{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1150nameserver 8.8.8.8 # Google Public NameServer
1151nameserver 64.6.64.6 # Verisign Public NameServer
1152{% else -%}
1153# START DYNAMIC LIST - updated by /tools/nameserver-shuffle
1154{{ autogen_edge_nameservers }}
1155{% endif -%}
1156""").render(datadump)
1157
1158
1159
1160def generate_ntp_conf(datadump):
1161 """ Generate configuration file '/etc/ntp.conf' """
1162 # XXX: This should properly going to be an datastructure soon
1163
1164 datadump['autogen_header'] = generate_header(datadump, "#")
1165 datadump['autogen_ntp_servers'] = ''
1166 for host in get_hostlist():
1167 hostdump = get_yaml(host)
1168 if hostdump['service_proxy_ileiden'] or hostdump['service_proxy_normal']:
1169 datadump['autogen_ntp_servers'] += "server %(masterip)-15s iburst maxpoll 9 # %(nodename)s\n" % hostdump
1170
1171 return Template("""\
1172{{ autogen_header }}
1173
1174{% if service_proxy_normal or service_proxy_ileiden or nodetype == 'Proxy' -%}
1175# Machine hooked to internet.
1176server 0.nl.pool.ntp.org iburst maxpoll 9
1177server 1.nl.pool.ntp.org iburst maxpoll 9
1178server 2.nl.pool.ntp.org iburst maxpoll 9
1179server 3.nl.pool.ntp.org iburst maxpoll 9
1180{% else -%}
1181# Local Wireless Leiden NTP Servers.
1182server 0.pool.ntp.wleiden.net iburst maxpoll 9
1183server 1.pool.ntp.wleiden.net iburst maxpoll 9
1184server 2.pool.ntp.wleiden.net iburst maxpoll 9
1185server 3.pool.ntp.wleiden.net iburst maxpoll 9
1186
1187# All the configured NTP servers
1188{{ autogen_ntp_servers }}
1189{% endif %}
1190
1191# If a server loses sync with all upstream servers, NTP clients
1192# no longer follow that server. The local clock can be configured
1193# to provide a time source when this happens, but it should usually
1194# be configured on just one server on a network. For more details see
1195# http://support.ntp.org/bin/view/Support/UndisciplinedLocalClock
1196# The use of Orphan Mode may be preferable.
1197#
1198server 127.127.1.0
1199fudge 127.127.1.0 stratum 10
1200""").render(datadump)
1201
1202
1203def generate_pf_hybrid_conf_local(datadump):
1204 """ Generate configuration file '/etc/pf.hybrid.conf.local' """
1205 datadump['autogen_header'] = generate_header(datadump, "#")
1206 if datadump['service_incoming_rdr']:
1207 datadump['global_rdr_rules'] = datadump['autogen_global_rdr_rules']
1208 return Template("""\
1209{{ autogen_header }}
1210
1211# Redirect some internal facing services outside (7)
1212# INFO: {{ global_rdr_rules|count }} global_rdr_rules active on this node.
1213{% for protocol, src_port,dest_ip,dest_port,comment in global_rdr_rules -%}
1214rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1215{% endfor -%}
1216# INFO: {{ rdr_rules|count }} node specific rdr_rules defined.
1217{% for protocol, src_port,dest_ip,dest_port,comment in rdr_rules -%}
1218rdr on $ext_if inet proto {{ protocol }} from any to $ext_if port {{ src_port }} tag SRV -> {{ "%-14s"|format(dest_ip) }} port {{ "%4s"|format(dest_port) }} # {{ comment }}
1219{% endfor -%}
1220""").render(datadump)
1221
1222def generate_unbound_wleiden_conf(datadump):
1223 """ Generate configuration file '/usr/local/etc/unbound.wleiden.conf' """
1224 datadump['autogen_header'] = generate_header(datadump, "#")
1225
1226 autogen_ips = []
1227 (addrs_list, _, _, dhclient_if, _, extra_ouput) = make_interface_list(datadump)
1228 for iface,addrs in sorted(addrs_list.items()):
1229 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1230 if addr.startswith('172'):
1231 autogen_ips.append((addr.split('/')[0], comment))
1232 datadump['autogen_ips'] = autogen_ips
1233
1234 create_proxies_list()
1235 datadump['autogen_ileiden_proxies'] = ileiden_proxies
1236 return Template("""\
1237{{ autogen_header }}
1238
1239server:
1240 ## Static definitions fail to start on systems with broken ue(4) interfaces
1241{%- for ip,comment in autogen_ips %}
1242 # interface: {{ "%-16s"|format(ip) }} # {{ comment }}
1243{%- endfor %}
1244 ## Enabling wildcard matching as work-around
1245 interface: 0.0.0.0
1246 interface: ::0
1247
1248forward-zone:
1249 name: '.'
1250{%- if service_proxy_ileiden %}
1251 forward-addr: 8.8.8.8 # Google DNS A
1252 forward-addr: 8.8.4.4 # Google DNS B
1253 forward-addr: 208.67.222.222 # OpenDNS DNS A
1254 forward-addr: 208.67.220.220 # OpenDNS DNS B
1255{% else -%}
1256{%- for serviceid,item in autogen_ileiden_proxies.items() %}
1257 {%- if loop.index <= 5 %}
1258 forward-addr: {{ "%-16s"|format(serviceid) }} # {{ item.nodename }}
1259 {%- endif %}
1260{%- endfor %}
1261{%- endif %}
1262""").render(datadump)
1263
1264def generate_motd(datadump):
1265 """ Generate configuration file '/etc/motd' """
1266 output = Template("""\
1267FreeBSD run ``service motd onestart'' to make me look normal
1268
1269 WWW: {{ autogen_fqdn }} - http://www.wirelessleiden.nl
1270 Loc: {{ location }}
1271
1272Services:
1273{% if board == "ALIX2" or board == "net4801" -%}
1274{{" -"}} Core Node ({{ board }})
1275{% else -%}
1276{{" -"}} Hulp Node ({{ board }})
1277{% endif -%}
1278{% if service_proxy_normal -%}
1279{{" -"}} Normal Proxy
1280{% endif -%}
1281{% if service_proxy_ileiden -%}
1282{{" -"}} iLeiden Proxy
1283{% endif -%}
1284{% if service_incoming_rdr -%}
1285{{" -"}} Incoming port redirects
1286{% endif %}
1287Interlinks:\n
1288""").render(datadump)
1289
1290
1291 def make_table(table):
1292 if not table:
1293 return " - none\n"
1294 else:
1295 lines = ""
1296 col_width = [max(len(x) for x in col) for col in zip(*table)]
1297 for row in table:
1298 # replace('_','.') is a hack to convert vlan interfaces to proper named interfaces
1299 lines += " - " + " || ".join("{:{}}".format(x.replace('_','.'), col_width[i]) for i, x in enumerate(row)) + "\n"
1300 return lines
1301
1302 (addrs_list, vlan_list, bridge_list, dhclient_if, flags_if, extra_ouput) = make_interface_list(datadump)
1303 table = []
1304 for iface,addrs in sorted(addrs_list.items()):
1305 if iface in ['lo0']:
1306 continue
1307 for addr, comment in sorted(addrs,key=lambda x: parseaddr(x[0].split('/')[0])):
1308 table.append((iface, addr, comment))
1309
1310 output += make_table(table)
1311 output += '\n'
1312 output += """\
1313Attached devices:
1314"""
1315 output += make_table(get_attached_devices(datadump, url=True))
1316 output += '\n'
1317 output += """\
1318Available neighbours:
1319"""
1320 output += make_table(get_neighbours(datadump))
1321
1322 return output
1323
1324
1325def format_yaml_value(value):
1326 """ Get yaml value in right syntax for outputting """
1327 if isinstance(value,str):
1328 output = '"%s"' % value
1329 else:
1330 output = value
1331 return output
1332
1333
1334
1335def format_wleiden_yaml(datadump):
1336 """ Special formatting to ensure it is editable"""
1337 output = "# Genesis config yaml style\n"
1338 output += "# vim:ts=2:et:sw=2:ai\n"
1339 output += "#\n"
1340 iface_keys = [elem for elem in datadump.keys() if elem.startswith('iface_')]
1341 for key in sorted(set(datadump.keys()) - set(iface_keys)):
1342 if key == 'rdr_rules':
1343 output += '%-10s:\n' % 'rdr_rules'
1344 for rdr_rule in datadump[key]:
1345 output += '- %s\n' % rdr_rule
1346 else:
1347 output += "%-10s: %s\n" % (key, format_yaml_value(datadump[key]))
1348
1349 output += "\n\n"
1350
1351 # Format (key, required)
1352 key_order = (
1353 ('comment', True),
1354 ('parent', False),
1355 ('ip', False),
1356 ('ipv6', False),
1357 ('ether', False),
1358 ('desc', True),
1359 ('sdesc', True),
1360 ('mode', True),
1361 ('type', True),
1362 ('extra_type', False),
1363 ('channel', False),
1364 ('ssid', False),
1365 ('wlan_mac', False),
1366 ('dhcp', True),
1367 ('dhcp_fixed', False),
1368 ('compass', False),
1369 ('distance', False),
1370 ('ns_ip', False),
1371 ('repeater_ip', False),
1372 ('bullet2_ip', False),
1373 ('ns_mac', False),
1374 ('bullet2_mac', False),
1375 ('mac', False),
1376 ('ns_type', False),
1377 ('bridge_type', False),
1378 ('encrypted', False),
1379 ('status', True),
1380 )
1381
1382 for iface_key in sorted(iface_keys):
1383 try:
1384 remainder = set(datadump[iface_key].keys()) - set([x[0] for x in key_order])
1385 if remainder:
1386 raise KeyError("invalid keys: %s" % remainder)
1387
1388 output += "%s:\n" % iface_key
1389 for key,required in key_order:
1390 if key in datadump[iface_key]:
1391 output += " %-11s: %s\n" % (key, format_yaml_value(datadump[iface_key][key]))
1392 output += "\n\n"
1393 except Exception as exc:
1394 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
1395 raise
1396
1397 return output
1398
1399
1400
1401def generate_wleiden_yaml(datadump, header=True):
1402 """ Generate (petty) version of wleiden.yaml"""
1403 output = generate_header(datadump, "#") if header else ''
1404
1405 for key in list(datadump.keys()):
1406 if key.startswith('autogen_'):
1407 del datadump[key]
1408 # Interface autogen cleanups
1409 elif type(datadump[key]) == dict:
1410 for key2 in list(datadump[key].keys()):
1411 if key2.startswith('autogen_'):
1412 del datadump[key][key2]
1413
1414 output += format_wleiden_yaml(datadump)
1415 return output
1416
1417def generate_nanostation_config(datadump, iface, ns_type):
1418 #TODO(rvdz): Make sure the proper nanostation IP and subnet is set
1419 datadump['iface_%s' % iface]['ns_ip'] = datadump['iface_%s' % iface]['ns_ip'].split('/')[0]
1420
1421 datadump.update(datadump['iface_%s' % iface])
1422
1423 return open(os.path.join(os.path.dirname(__file__), 'ns5m.cfg.tmpl'),'r').read() % datadump
1424
1425def generate_yaml(datadump):
1426 return generate_config(datadump['nodename'], "wleiden.yaml", datadump)
1427
1428
1429
1430def generate_config(node, config, datadump=None):
1431 """ Print configuration file 'config' of 'node' """
1432 output = ""
1433 try:
1434 # Load config file
1435 if datadump == None:
1436 datadump = get_yaml(node)
1437
1438 if config == 'wleiden.yaml':
1439 output += generate_wleiden_yaml(datadump)
1440 elif config == 'authorized_keys':
1441 f = open(os.path.join(NODE_DIR,"global_keys"), 'r')
1442 output += f.read()
1443 node_keys = os.path.join(NODE_DIR,node,'authorized_keys')
1444 # Fetch local keys if existing
1445 if os.path.exists(node_keys):
1446 output += open(node_keys, 'r').read()
1447 f.close()
1448 elif config == 'dnsmasq.conf':
1449 output += generate_dnsmasq_conf(datadump)
1450 elif config == 'dhcpd.conf':
1451 output += generate_dhcpd_conf(datadump)
1452 elif config == 'rc.conf.local':
1453 output += generate_rc_conf_local(datadump)
1454 elif config == 'resolv.conf':
1455 output += generate_resolv_conf(datadump)
1456 elif config == 'ntp.conf':
1457 output += generate_ntp_conf(datadump)
1458 elif config == 'motd':
1459 output += generate_motd(datadump)
1460 elif config == 'pf.hybrid.conf.local':
1461 output += generate_pf_hybrid_conf_local(datadump)
1462 elif config == 'unbound.wleiden.conf':
1463 output += generate_unbound_wleiden_conf(datadump)
1464 elif config.startswith('vr'):
1465 interface, ns_type = config.strip('.yaml').split('-')
1466 output += generate_nanostation_config(datadump, interface, ns_type)
1467 else:
1468 assert False, "Config not found!"
1469 except IOError as e:
1470 output += "[ERROR] Config file not found"
1471 return output
1472
1473
1474
1475def generate_static(output_dir, logging=True):
1476 items = {'output_dir' : output_dir}
1477 for node in get_hostlist():
1478 items['node'] = node
1479 items['wdir'] = "%(output_dir)s/%(node)s" % items
1480 if not os.path.isdir(items['wdir']):
1481 os.makedirs(items['wdir'])
1482 datadump = get_yaml(node)
1483 f = open("%(wdir)s/index.html" % items, "w")
1484 f.write(generate_node_overview(items['node'], datadump))
1485 f.close()
1486 for config in files:
1487 items['config'] = config
1488 if logging: logger.info("## Generating %(node)s %(config)s" % items)
1489 f = open("%(wdir)s/%(config)s" % items, "w")
1490 f.write(generate_config(node, config, datadump))
1491 f.close()
1492
1493
1494
1495def process_cgi_request(environ=os.environ):
1496 """ When calling from CGI """
1497 response_headers = []
1498 content_type = 'text/plain'
1499
1500 # Update repository if requested
1501 form = urllib.parse.parse_qs(environ.get('QUERY_STRING'))
1502 if form and 'action' in form and 'update' in form['action']:
1503 refresh_rate = 5
1504 output = "[INFO] Updating subverion, please wait...\n"
1505 old_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
1506 output += subprocess.Popen([SVN, 'cleanup', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
1507 output += subprocess.Popen([SVN, 'up', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
1508 new_version = subprocess.Popen([SVNVERSION, '-c', "%s/.." % NODE_DIR], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
1509 if old_version != new_version or 'force' in environ.get('QUERY_STRING'):
1510 try:
1511 generate_static(CACHE_DIR, False)
1512 except:
1513 output += traceback.format_exc()
1514 refresh_rate = 120
1515 pass
1516 output += "[INFO] All done, redirecting in %s seconds" % refresh_rate
1517 response_headers += [
1518 ('Refresh', '%s; url=.' % refresh_rate),
1519 ]
1520 else:
1521 # Bootstap cache directory if none exists
1522 if not glob.glob(os.path.join(CACHE_DIR,'*','index.html')):
1523 generate_static(CACHE_DIR, False)
1524
1525 # URL must be of format <prefix>/config/<path>
1526 base_uri = environ['REQUEST_URI']
1527 uri = list(filter(None, base_uri.split('/config/')[1].strip('/').split('/')))
1528
1529 output = "Template Holder"
1530 if base_uri.endswith('/create/network.kml'):
1531 content_type='application/vnd.google-earth.kml+xml'
1532 output = make_network_kml.make_graph()
1533 elif base_uri.endswith('/api/get/nodeplanner.json'):
1534 content_type='application/json'
1535 output = make_network_kml.make_nodeplanner_json()
1536 elif not uri:
1537 if is_text_request(environ):
1538 output = '\n'.join(get_hostlist())
1539 else:
1540 content_type = 'text/html'
1541 output = generate_title(get_hostlist())
1542 elif len(uri) == 1:
1543 if is_text_request(environ):
1544 output = generate_node(uri[0])
1545 else:
1546 content_type = 'text/html'
1547 output = open(os.path.join(CACHE_DIR, uri[0], 'index.html'), 'r').read()
1548 elif len(uri) == 2:
1549 output = generate_config(uri[0], uri[1])
1550 else:
1551 assert False, "Invalid option"
1552
1553 # Return response
1554 response_headers += [
1555 ('Content-type', content_type),
1556 ('Content-Length', str(len(output))),
1557 ]
1558 return(response_headers, str(output))
1559
1560
1561def make_dns(output_dir = 'dns', external = False):
1562 items = dict()
1563
1564 # hostname is key, IP is value
1565 wleiden_zone = defaultdict(list)
1566 wleiden_cname = dict()
1567
1568 pool = dict()
1569 for node in get_hostlist():
1570 datadump = get_yaml(node)
1571
1572 fqdn = datadump['nodename']
1573
1574 if 'rdr_host' in datadump:
1575 remote_target = datadump['rdr_host']
1576 elif 'remote_access' in datadump and datadump['remote_access']:
1577 remote_target = datadump['remote_access'].split(':')[0]
1578 else:
1579 remote_target = None
1580
1581 if remote_target:
1582 try:
1583 parseaddr(remote_target)
1584 wleiden_zone[datadump['nodename'] + '.gw'].append((remote_target, False))
1585 except (IndexError, ValueError):
1586 wleiden_cname[datadump['nodename'] + '.gw'] = remote_target + '.'
1587
1588
1589 wleiden_zone[fqdn].append((datadump['masterip'], True))
1590
1591 # Hacking to get proper DHCP IPs and hostnames
1592 for iface_key in get_interface_keys(datadump):
1593 iface_name = iface_key.replace('_','-')
1594 if 'ip' in datadump[iface_key]:
1595 (ip, cidr) = datadump[iface_key]['ip'].split('/')
1596 try:
1597 (dhcp_start, dhcp_stop) = datadump[iface_key]['dhcp'].split('-')
1598 datadump[iface_key]['autogen_netmask'] = cidr2netmask(cidr)
1599 dhcp_part = ".".join(ip.split('.')[0:3])
1600 if ip != datadump['masterip']:
1601 wleiden_zone["dhcp-gateway-%s.%s" % (iface_name, fqdn)].append((ip, True))
1602 for i in range(int(dhcp_start), int(dhcp_stop) + 1):
1603 wleiden_zone["dhcp-%s-%s.%s" % (i, iface_name, fqdn)].append(("%s.%s" % (dhcp_part, i), True))
1604 except (AttributeError, ValueError, KeyError):
1605 # First push it into a pool, to indentify the counter-part later on
1606 addr = parseaddr(ip)
1607 cidr = int(cidr)
1608 addr = addr & ~((1 << (32 - cidr)) - 1)
1609 if addr in pool:
1610 pool[addr] += [(iface_name, fqdn, ip)]
1611 else:
1612 pool[addr] = [(iface_name, fqdn, ip)]
1613 continue
1614
1615
1616
1617 # WL uses an /29 to configure an interface. IP's are ordered like this:
1618 # MasterA (.1) -- DeviceA (.2) <<>> DeviceB (.3) --- SlaveB (.4)
1619
1620 sn = lambda x: re.sub(r'(?i)^cnode','',x)
1621
1622 # Automatic naming convention of interlinks namely 2 + remote.lower()
1623 for (key,value) in pool.items():
1624 # Make sure they are sorted from low-ip to high-ip
1625 value = sorted(value, key=lambda x: parseaddr(x[2]))
1626
1627 if len(value) == 1:
1628 (iface_name, fqdn, ip) = value[0]
1629 wleiden_zone["2unused-%s.%s" % (iface_name, fqdn)].append((ip, True))
1630
1631 # Device DNS names
1632 if 'cnode' in fqdn.lower():
1633 wleiden_zone["d-at-%s.%s" % (iface_name, fqdn)].append((showaddr(parseaddr(ip) + 1), False))
1634 wleiden_cname["d-at-%s.%s" % (iface_name,sn(fqdn))] = "d-at-%s.%s" % ((iface_name, fqdn))
1635
1636 elif len(value) == 2:
1637 (a_iface_name, a_fqdn, a_ip) = value[0]
1638 (b_iface_name, b_fqdn, b_ip) = value[1]
1639 wleiden_zone["2%s.%s" % (b_fqdn,a_fqdn)].append((a_ip, True))
1640 wleiden_zone["2%s.%s" % (a_fqdn,b_fqdn)].append((b_ip, True))
1641
1642 # Device DNS names
1643 if 'cnode' in a_fqdn.lower() and 'cnode' in b_fqdn.lower():
1644 wleiden_zone["d-at-%s.%s" % (a_iface_name, a_fqdn)].append((showaddr(parseaddr(a_ip) + 1), False))
1645 wleiden_zone["d-at-%s.%s" % (b_iface_name, b_fqdn)].append((showaddr(parseaddr(b_ip) - 1), False))
1646 wleiden_cname["d-at-%s.%s" % (a_iface_name,sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1647 wleiden_cname["d-at-%s.%s" % (b_iface_name,sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1648 wleiden_cname["d2%s.%s" % (sn(b_fqdn),sn(a_fqdn))] = "d-at-%s.%s" % (a_iface_name, a_fqdn)
1649 wleiden_cname["d2%s.%s" % (sn(a_fqdn),sn(b_fqdn))] = "d-at-%s.%s" % (b_iface_name, b_fqdn)
1650
1651 else:
1652 pool_members = [k[1] for k in value]
1653 for item in value:
1654 (iface_name, fqdn, ip) = item
1655 wleiden_zone["2ring.%s" % (fqdn)].append((ip, True))
1656
1657 # Include static DNS entries
1658 # XXX: Should they override the autogenerated results?
1659 # XXX: Convert input to yaml more useable.
1660 # Format:
1661 ##; this is a comment
1662 ## roomburgh=Roomburgh1
1663 ## apkerk1.Vosko=172.17.176.8 ;this as well
1664 dns_list = yaml.load(open(os.path.join(NODE_DIR,'../dns/staticDNS.yaml'),'r'), Loader=Loader)
1665
1666 # Hack to allow special entries, for development
1667 wleiden_raw = {}
1668
1669 for line in dns_list:
1670 reverse = False
1671 k, items = list(line.items())[0]
1672 if type(items) == dict:
1673 if 'reverse' in items:
1674 reverse = items['reverse']
1675 items = items['a']
1676 else:
1677 items = items['cname']
1678 items = [items] if type(items) != list else items
1679 for item in items:
1680 if item.startswith('IN '):
1681 wleiden_raw[k] = item
1682 elif valid_addr(item):
1683 wleiden_zone[k].append((item, reverse))
1684 else:
1685 wleiden_cname[k] = item
1686
1687 # Hack to get dynamic pool listing
1688 def chunks(l, n):
1689 return [l[i:i+n] for i in range(0, len(l), n)]
1690
1691 ntp_servers = [x[0] for x in get_nameservers()]
1692 for id, chunk in enumerate(chunks(ntp_servers,(math.floor(len(ntp_servers)/4)))):
1693 for ntp_server in chunk:
1694 wleiden_zone['%i.pool.ntp' % id].append((ntp_server, False))
1695
1696 details = dict()
1697 # 24 updates a day allowed
1698 details['serial'] = time.strftime('%Y%m%d%H')
1699
1700 if external:
1701 dns_masters = ['ns1.vanderzwet.net', 'ns1.anywi.com']
1702 else:
1703 dns_masters = ['druif.wleiden.net'] + ["%s.wleiden.net" % x[1] for x in get_nameservers(max_servers=3)]
1704
1705 details['master'] = dns_masters[0]
1706 details['ns_servers'] = '\n'.join(['\tNS\t%s.' % x for x in dns_masters])
1707
1708 dns_header = '''
1709$TTL 3h
1710%(zone)s. SOA %(master)s. beheer.lijst.wirelessleiden.nl. ( %(serial)s 15m 15m 1w 3h )
1711 ; Serial, Refresh, Retry, Expire, Neg. cache TTL
1712
1713%(ns_servers)s
1714 \n'''
1715
1716
1717 if not os.path.isdir(output_dir):
1718 os.makedirs(output_dir)
1719 details['zone'] = 'wleiden.net'
1720 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1721 f.write(dns_header % details)
1722
1723 for host,items in wleiden_zone.items():
1724 for ip,reverse in items:
1725 if ip not in ['0.0.0.0']:
1726 f.write("%s.wleiden.net. IN A %s\n" % (host.lower(), ip))
1727 for source,dest in wleiden_cname.items():
1728 dest = dest if dest.endswith('.') else dest + ".wleiden.net."
1729 f.write("%s.wleiden.net. IN CNAME %s\n" % (source.lower(), dest.lower()))
1730 for source, dest in wleiden_raw.items():
1731 f.write("%s.wleiden.net. %s\n" % (source, dest))
1732 f.close()
1733
1734 # Create whole bunch of specific sub arpa zones. To keep it compliant
1735 for s in range(16,32):
1736 details['zone'] = '%i.172.in-addr.arpa' % s
1737 f = open(os.path.join(output_dir,"db." + details['zone']), "w")
1738 f.write(dns_header % details)
1739
1740 #XXX: Not effient, fix to proper data structure and do checks at other
1741 # stages
1742 for host,items in wleiden_zone.items():
1743 for ip,reverse in items:
1744 if not reverse:
1745 continue
1746 if valid_addr(ip):
1747 if valid_addr(ip):
1748 if int(ip.split('.')[1]) == s:
1749 rev_ip = '.'.join(reversed(ip.split('.')))
1750 f.write("%s.in-addr.arpa. IN PTR %s.wleiden.net.\n" % (rev_ip.lower(), host.lower()))
1751 f.close()
1752
1753
1754def usage():
1755 print("""Usage: %(prog)s <argument>
1756Argument:
1757\tcleanup = Cleanup all YAML files to specified format
1758\tstandalone [port] = Run configurator webserver [8000]
1759\tdns [outputdir] = Generate BIND compliant zone files in dns [./dns]
1760\tnagios-export [--heavy-load] = Generate basic nagios configuration file.
1761\tfull-export = Generate yaml export script for heatmap.
1762\tstatic [outputdir] = Generate all config files and store on disk
1763\t with format ./<outputdir>/%%NODE%%/%%FILE%% [./static]
1764\ttest <node> [<file>] = Receive output for certain node [all files].
1765\ttest-cgi <node> <file> = Receive output of CGI script [all files].
1766\tlist <status> <items> = List systems which have certain status
1767\tcreate network.kml = Create Network KML file for use in Google Earth
1768
1769Arguments:
1770\t<node> = NodeName (example: HybridRick)
1771\t<file> = %(files)s
1772\t<status> = all|up|down|planned
1773\t<items> = systems|nodes|proxies
1774
1775NOTE FOR DEVELOPERS; you can test your changes like this:
1776 BEFORE any changes in this code:
1777 $ ./gformat.py static /tmp/pre
1778 AFTER the changes:
1779 $ ./gformat.py static /tmp/post
1780 VIEW differences and VERIFY all are OK:
1781 $ diff -urI 'Generated' -r /tmp/pre /tmp/post
1782""" % { 'prog' : sys.argv[0], 'files' : '|'.join(files) })
1783 exit(0)
1784
1785
1786def is_text_request(environ=os.environ):
1787 """ Find out whether we are calling from the CLI or any text based CLI utility """
1788 if 'CONTENT_TYPE' in os.environ and os.environ['CONTENT_TYPE'] == 'text/plain':
1789 return True
1790
1791 if 'HTTP_USER_AGENT' in environ:
1792 return any([os.environ['HTTP_USER_AGENT'].lower().startswith(x) for x in ['curl', 'fetch', 'wget']])
1793 else:
1794 return False
1795
1796
1797def switchFormat(setting):
1798 if setting:
1799 return "YES"
1800 else:
1801 return "NO"
1802
1803def rlinput(prompt, prefill=''):
1804 import readline
1805 readline.set_startup_hook(lambda: readline.insert_text(prefill))
1806 try:
1807 return raw_input(prompt)
1808 finally:
1809 readline.set_startup_hook()
1810
1811def fix_conflict(left, right, default='i'):
1812 while True:
1813 print("## %-30s | %-30s" % (left, right))
1814 c = raw_input("## Solve Conflict (h for help) <l|r|e|i|> [%s]: " % default)
1815 if not c:
1816 c = default
1817
1818 if c in ['l','1']:
1819 return left
1820 elif c in ['r','2']:
1821 return right
1822 elif c in ['e', '3']:
1823 return rlinput("Edit: ", "%30s | %30s" % (left, right))
1824 elif c in ['i', '4']:
1825 return None
1826 else:
1827 print("#ERROR: '%s' is invalid input (left, right, edit or ignore)!" % c)
1828
1829
1830
1831def print_cgi_response(response_headers, output):
1832 for header in response_headers:
1833 print("%s: %s" % header)
1834 print("")
1835 print(output)
1836
1837
1838
1839def main():
1840 """Hard working sub"""
1841 # Allow easy hacking using the CLI
1842 if not os.environ.get('REQUEST_URI', None):
1843 if len(sys.argv) < 2:
1844 usage()
1845
1846 if sys.argv[1] == "standalone":
1847 import SocketServer
1848 import CGIHTTPServer
1849 # Hop to the right working directory.
1850 os.chdir(os.path.dirname(__file__))
1851 try:
1852 PORT = int(sys.argv[2])
1853 except (IndexError,ValueError):
1854 PORT = 8000
1855
1856 class MyCGIHTTPRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler):
1857 """ Serve this CGI from the root of the webserver """
1858 def is_cgi(self):
1859 if "favicon" in self.path:
1860 return False
1861
1862 self.cgi_info = (os.path.basename(__file__), self.path)
1863 self.path = ''
1864 return True
1865 handler = MyCGIHTTPRequestHandler
1866 SocketServer.TCPServer.allow_reuse_address = True
1867 httpd = SocketServer.TCPServer(("", PORT), handler)
1868 httpd.server_name = 'localhost'
1869 httpd.server_port = PORT
1870
1871 logger.info("serving at port %s", PORT)
1872 try:
1873 httpd.serve_forever()
1874 except KeyboardInterrupt:
1875 httpd.shutdown()
1876 logger.info("All done goodbye")
1877 elif sys.argv[1] == "test":
1878 # Basic argument validation
1879 try:
1880 node = sys.argv[2]
1881 except IndexError:
1882 print("Invalid argument")
1883 exit(1)
1884 except IOError as e:
1885 print(e)
1886 exit(1)
1887
1888 datadump = get_yaml(node)
1889
1890
1891 # Get files to generate
1892 gen_files = sys.argv[3:] if len(sys.argv) > 3 else files
1893
1894 # Actual config generation
1895 for config in gen_files:
1896 logger.info("## Generating %s %s", node, config)
1897 print(generate_config(node, config, datadump))
1898 elif sys.argv[1] == "test-cgi":
1899 os.environ['REQUEST_URI'] = "/".join(['config'] + sys.argv[2:])
1900 os.environ['SCRIPT_NAME'] = __file__
1901 response_headers, output = process_cgi_request()
1902 print_cgi_response(response_headers, output)
1903 elif sys.argv[1] == "static":
1904 generate_static(sys.argv[2] if len(sys.argv) > 2 else "./static")
1905 elif sys.argv[1] == "wind-export":
1906 items = dict()
1907 for node in get_hostlist():
1908 datadump = get_yaml(node)
1909 sql = """INSERT IGNORE INTO nodes (name, name_ns, longitude, latitude)
1910 VALUES ('%(nodename)s', '%(nodename)s', %(latitude)s, %(longitude)s);""" % datadump;
1911 sql = """INSERT IGNORE INTO users_nodes (user_id, node_id, owner)
1912 VALUES (
1913 (SELECT id FROM users WHERE username = 'rvdzwet'),
1914 (SELECT id FROM nodes WHERE name = '%(nodename)s'),
1915 'Y');""" % datadump
1916 #for config in files:
1917 # items['config'] = config
1918 # print "## Generating %(node)s %(config)s" % items
1919 # f = open("%(wdir)s/%(config)s" % items, "w")
1920 # f.write(generate_config(node, config, datadump))
1921 # f.close()
1922 for node in get_hostlist():
1923 datadump = get_yaml(node)
1924 for iface_key in sorted([elem for elem in datadump.keys() if elem.startswith('iface_')]):
1925 ifacedump = datadump[iface_key]
1926 if 'mode' in ifacedump and ifacedump['mode'] == 'ap-wds':
1927 ifacedump['nodename'] = datadump['nodename']
1928 if not 'channel' in ifacedump or not ifacedump['channel']:
1929 ifacedump['channel'] = 0
1930 sql = """INSERT INTO links (node_id, type, ssid, protocol, channel, status)
1931 VALUES ((SELECT id FROM nodes WHERE name = '%(nodename)s'), 'ap',
1932 '%(ssid)s', 'IEEE 802.11b', %(channel)s, 'active');""" % ifacedump
1933 elif sys.argv[1] == "smokeping-export":
1934 for host in get_hostlist():
1935 datadump = get_yaml(host)
1936 if datadump.get('service_proxy_normal', False) or datadump.get('service_proxy_ileiden', False):
1937 print(textwrap.dedent("""\
1938 ++ wleiden-gw-%(nodename)s
1939 menu = %(nodename)s.gw
1940 title = Wireless Leiden gateway %(nodename)s.gw.wleiden.net.
1941 host = %(nodename)s.gw.wleiden.net.
1942 """ % datadump))
1943 elif sys.argv[1] == "nagios-export":
1944 try:
1945 heavy_load = (sys.argv[2] == "--heavy-load")
1946 except IndexError:
1947 heavy_load = False
1948
1949 hostgroup_details = {
1950 'wleiden' : 'Stichting Wireless Leiden - FreeBSD Nodes',
1951 'wzoeterwoude' : 'Stichting Wireless Leiden - Afdeling Zoeterwoude - Free-WiFi Project',
1952 'walphen' : 'Stichting Wireless Alphen',
1953 'westeinder' : 'Westeinder Plassen',
1954 }
1955
1956 # Convert IP to Host
1957 ip2host = {'root' : 'root'}
1958 for host in get_hostlist():
1959 datadump = get_yaml(host)
1960 ip2host[datadump['masterip']] = datadump['autogen_fqdn']
1961 for iface in get_interface_keys(datadump):
1962 if 'autogen_gateway' in datadump[iface]:
1963 ip2host[datadump[iface]['autogen_gateway']] = datadump['autogen_fqdn']
1964
1965 # Find dependency tree based on output of lvrouted.mytree of nearest node
1966 parents = defaultdict(list)
1967 stack = ['root']
1968 prev_depth = 0
1969 if os.path.isfile('lvrouted.mytree'):
1970 for line in open('lvrouted.mytree').readlines():
1971 depth = line.count('\t')
1972 ip = line.strip().split()[0]
1973
1974 if prev_depth < depth:
1975 try:
1976 parents[ip2host[ip]].append(ip2host[stack[-1]])
1977 except KeyError as e:
1978 print("# Unable to find %s in configuration files" % e.args[0])
1979 stack.append(ip)
1980 elif prev_depth > depth:
1981 stack = stack[:(depth - prev_depth)]
1982 elif prev_depth == depth:
1983 try:
1984 parents[ip2host[ip]].append(ip2host[stack[-1]])
1985 except KeyError as e:
1986 print("# Unable to find %s in configuration files" % e.args[0])
1987
1988
1989 prev_depth = depth
1990 # Observe that some nodes has themself as parent or multiple parents
1991 # for now take only the first parent, other behaviour is yet to be explained
1992
1993
1994
1995 params = {
1996 'check_interval' : 5 if heavy_load else 120,
1997 'retry_interval' : 1 if heavy_load else 10,
1998 'max_check_attempts' : 10 if heavy_load else 6,
1999 'notification_interval': 120 if heavy_load else 240,
2000 }
2001
2002 print('''\
2003define host {
2004 name wleiden-node ; Default Node Template
2005 use generic-host ; Use the standard template as initial starting point
2006 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2007 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2008 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2009 notification_interval %(notification_interval)s
2010 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2011 check_command check-host-alive ; Default command to check FreeBSD hosts
2012 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2013}
2014
2015define service {
2016 name wleiden-service ; Default Service Template
2017 use generic-service ; Use the standard template as initial starting point
2018 check_period 24x7 ; By default, FreeBSD hosts are checked round the clock
2019 check_interval %(check_interval)s ; Actively check the host every 5 minutes
2020 retry_interval %(retry_interval)s ; Schedule host check retries at 1 minute intervals
2021 notification_interval %(notification_interval)s
2022 max_check_attempts %(max_check_attempts)s ; Check each FreeBSD host 10 times (max)
2023 register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
2024}
2025
2026# Please make sure to install:
2027# make -C /usr/ports/net-mgmt/nagios-check_netsnmp install clean
2028#
2029# Recompile net-mgmt/nagios-plugins to support check_snmp
2030# make -C /usr/ports/net-mgmt/nagios-plugins
2031#
2032# Install net/bind-tools to allow v2/check_dns_wl to work:
2033# pkg install bind-tools
2034#
2035define command{
2036 command_name check_snmp_disk
2037 command_line $USER1$/check_snmp_disk -H $HOSTADDRESS$ -C public
2038}
2039
2040define command{
2041 command_name check_netsnmp_load
2042 command_line $USER1$/check_snmp_load.pl -H $HOSTADDRESS$ -C public -w 80 -c 90
2043}
2044
2045define command{
2046 command_name check_netsnmp_proc
2047 command_line $USER1$/check_snmp_proc -H $HOSTADDRESS$ -C public
2048}
2049
2050define command{
2051 command_name check_by_ssh
2052 command_line $USER1$/check_by_ssh -H $HOSTADDRESS$ -p $ARG1$ -C "$ARG2$ $ARG3$ $ARG4$ $ARG5$ $ARG6$"
2053}
2054
2055define command{
2056 command_name check_dns_wl
2057 command_line $USER1$/v2/check_dns_wl $HOSTADDRESS$ $ARG1$
2058}
2059
2060define command{
2061 command_name check_snmp_uptime
2062 command_line $USER1$/check_snmp -H $HOSTADDRESS$ -C public -o .1.3.6.1.2.1.1.3.0
2063}
2064
2065
2066# TDB: dhcp leases
2067# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 exec
2068
2069# TDB: internet status
2070# /usr/local/libexec/nagios/check_netsnmp -H 192.168.178.47 --oid 1 file
2071
2072# TDB: Advanced local passive checks
2073# /usr/local/libexec/nagios/check_by_ssh
2074''' % params)
2075
2076 print('''\
2077# Service Group, not displayed by default
2078define hostgroup {
2079 hostgroup_name srv_hybrid
2080 alias All Hybrid Nodes
2081 register 0
2082}
2083
2084define service {
2085 use wleiden-service
2086 hostgroup_name srv_hybrid
2087 service_description SSH
2088 check_command check_ssh
2089}
2090
2091define service {
2092 use wleiden-service,service-pnp
2093 hostgroup_name srv_hybrid
2094 service_description HTTP
2095 check_command check_http
2096}
2097
2098define service {
2099 use wleiden-service
2100 hostgroup_name srv_hybrid
2101 service_description DNS
2102 check_command check_dns_wl!"www.wirelessleiden.nl"
2103}
2104''')
2105
2106 if heavy_load:
2107 print('''\
2108define service {
2109 use wleiden-service
2110 hostgroup_name srv_hybrid
2111 service_description UPTIME
2112 check_command check_snmp_uptime
2113}
2114
2115#define service {
2116# use wleiden-service
2117# hostgroup_name srv_hybrid
2118# service_description NTP
2119# check_command check_ntp_peer
2120#}
2121
2122define service {
2123 use wleiden-service
2124 hostgroup_name srv_hybrid
2125 service_description LOAD
2126 check_command check_netsnmp_load
2127}
2128
2129define service {
2130 use wleiden-service
2131 hostgroup_name srv_hybrid
2132 service_description PROC
2133 check_command check_netsnmp_proc
2134}
2135
2136define service {
2137 use wleiden-service
2138 hostgroup_name srv_hybrid
2139 service_description DISK
2140 check_command check_snmp_disk
2141}
2142''')
2143 for node in get_hostlist():
2144 datadump = get_yaml(node)
2145 if not datadump['status'] == 'up':
2146 continue
2147 if not datadump['monitoring_group'] in hostgroup_details:
2148 hostgroup_details[datadump['monitoring_group']] = datadump['monitoring_group']
2149 print('''\
2150define host {
2151 use wleiden-node,host-pnp
2152 contact_groups admins
2153 host_name %(autogen_fqdn)s
2154 address %(masterip)s
2155 hostgroups srv_hybrid,%(monitoring_group)s\
2156''' % datadump)
2157 if (len(parents[datadump['autogen_fqdn']]) > 0) and parents[datadump['autogen_fqdn']][0] != 'root':
2158 print('''\
2159 parents %(parents)s\
2160''' % { 'parents' : parents[datadump['autogen_fqdn']][0] })
2161 print('''\
2162}
2163''')
2164
2165
2166 for name,alias in hostgroup_details.items():
2167 print('''\
2168define hostgroup {
2169 hostgroup_name %s
2170 alias %s
2171} ''' % (name, alias))
2172
2173
2174 elif sys.argv[1] == "full-export":
2175 hosts = {}
2176 for node in get_hostlist():
2177 datadump = get_yaml(node)
2178 hosts[datadump['nodename']] = datadump
2179 print(yaml.dump(hosts))
2180
2181 elif sys.argv[1] == "dns":
2182 make_dns(sys.argv[2] if len(sys.argv) > 2 else 'dns', 'external' in sys.argv)
2183 elif sys.argv[1] == "cleanup":
2184 # First generate all datadumps
2185 datadumps = dict()
2186 ssid_to_node = dict()
2187 for host in get_hostlist():
2188 logger.info("# Processing: %s", host)
2189 # Set some boring default values
2190 datadump = { 'board' : 'UNKNOWN' }
2191 datadump.update(get_yaml(host))
2192 datadumps[datadump['nodename']] = datadump
2193
2194 (poel, errors) = make_relations()
2195 print("\n".join(["# WARNING: %s" % x for x in errors]))
2196
2197 for host,datadump in datadumps.items():
2198 try:
2199 # Convert all yes and no to boolean values
2200 def fix_boolean(dump):
2201 for key in dump.keys():
2202 if type(dump[key]) == dict:
2203 dump[key] = fix_boolean(dump[key])
2204 elif str(dump[key]).lower() in ["yes", "true"]:
2205 dump[key] = True
2206 elif str(dump[key]).lower() in ["no", "false"]:
2207 # Compass richting no (Noord Oost) is valid input
2208 if key != "compass": dump[key] = False
2209 return dump
2210 datadump = fix_boolean(datadump)
2211
2212 if 'rdnap_x' in datadump and 'rdnap_y' in datadump:
2213 if not 'latitude' in datadump and not 'longitude' in datadump:
2214 datadump['latitude'], datadump['longitude'] = map(lambda x: "%.5f" % x, rd2etrs(datadump['rdnap_x'], datadump['rdnap_y']))
2215 elif 'latitude' in datadump and 'longitude' in datadump:
2216 if not 'rdnap_x' in datadump and not 'rdnap_y' in datadump:
2217 datadump['rdnap_x'], datadump['rdnap_y'] = etrs2rd(datadump['latitude'], datadump['longitude'])
2218 # TODO: Compare outcome of both coordinate systems and validate against each-other
2219
2220 if datadump['nodename'].startswith('Proxy'):
2221 datadump['nodename'] = datadump['nodename'].lower()
2222
2223 for iface_key in get_interface_keys(datadump):
2224 try:
2225 # All our normal wireless cards are normal APs now
2226 if datadump[iface_key]['type'] in ['11a', '11b', '11g', 'wireless']:
2227 datadump[iface_key]['mode'] = 'ap'
2228 # Wireless Leiden SSID have an consistent lowercase/uppercase
2229 if 'ssid' in datadump[iface_key]:
2230 ssid = datadump[iface_key]['ssid']
2231 prefix = 'ap-WirelessLeiden-'
2232 if ssid.lower().startswith(prefix.lower()):
2233 datadump[iface_key]['ssid'] = prefix + ssid[len(prefix)].upper() + ssid[len(prefix) + 1:]
2234 if 'ns_ip' in datadump[iface_key] and not 'mode' in datadump[iface_key]:
2235 datadump[iface_key]['mode'] = 'autogen-FIXME'
2236 if not 'comment' in datadump[iface_key]:
2237 datadump[iface_key]['comment'] = 'autogen-FIXME'
2238
2239 if 'ns_mac' in datadump[iface_key]:
2240 datadump[iface_key]['ns_mac'] = datadump[iface_key]['ns_mac'].lower()
2241
2242 if 'comment' in datadump[iface_key] and datadump[iface_key]['comment'].startswith('autogen-'):
2243 datadump[iface_key] = datadump[iface_key]['desc']
2244
2245 # We are not using 802.11b anymore. OFDM is preferred over DSSS
2246 # due to better collision avoidance.
2247 if datadump[iface_key]['type'] == '11b':
2248 datadump[iface_key]['type'] = '11g'
2249
2250 # Setting 802.11g channels to de-facto standards, to avoid
2251 # un-detected sharing with other overlapping channels
2252 #
2253 # Technically we could also use channel 13 in NL, but this is not
2254 # recommended as foreign devices might not be able to select this
2255 # channel. Secondly using 1,5,9,13 instead is going to clash with
2256 # the de-facto usage of 1,6,11.
2257 #
2258 # See: https://en.wikipedia.org/wiki/List_of_WLAN_channels
2259 channels_at_2400Mhz = (1,6,11)
2260 if datadump[iface_key]['type'] == '11g' and 'channel' in datadump[iface_key]:
2261 datadump[iface_key]['channel'] = int(datadump[iface_key]['channel'])
2262 if datadump[iface_key]['channel'] not in channels_at_2400Mhz:
2263 datadump[iface_key]['channel'] = random.choice(channels_at_2400Mhz)
2264
2265 # Mandatory interface keys
2266 if not 'status' in datadump[iface_key]:
2267 datadump[iface_key]['status'] = 'planned'
2268
2269 x = datadump[iface_key]['comment']
2270 datadump[iface_key]['comment'] = x[0].upper() + x[1:]
2271
2272 # Fixing bridge_type if none is found
2273 if datadump[iface_key].get('extra_type', '') == 'eth2wifibridge':
2274 if not 'bridge_type' in datadump[iface_key]:
2275 datadump[iface_key]['bridge_type'] = 'NanoStation M5'
2276
2277 # Making sure description works
2278 if 'desc' in datadump[iface_key]:
2279 if datadump[iface_key]['comment'].lower() == datadump[iface_key]['desc'].lower():
2280 del datadump[iface_key]['desc']
2281 else:
2282 print("# ERROR: At %s - %s" % (datadump['nodename'], iface_key))
2283 response = fix_conflict(datadump[iface_key]['comment'], datadump[iface_key]['desc'])
2284 if response:
2285 datadump[iface_key]['comment'] = response
2286 del datadump[iface_key]['desc']
2287
2288 # Check DHCP configuration
2289 dhcp_type(datadump[iface_key])
2290
2291 # Set the compass value based on the angle between the poels
2292 if 'ns_ip' in datadump[iface_key] and 'ip' in datadump[iface_key] and not 'compass' in datadump[iface_key]:
2293 my_pool = poel[network(datadump[iface_key]['ip'])]
2294 remote_hosts = list(set([x[0] for x in my_pool]) - set([host]))
2295 if remote_hosts:
2296 compass_target = remote_hosts[0]
2297 datadump[iface_key]['compass'] = cd_between_hosts(host, compass_target, datadumps)
2298 # TODO: Compass wanted and actual direction might differ
2299
2300 # Monitoring Group default
2301 if not 'monitoring_group' in datadump:
2302 datadump['monitoring_group'] = 'wleiden'
2303
2304 except Exception as exc:
2305 exc.args = ("# Error while processing interface %s" % iface_key,) + exc.args
2306 raise
2307 store_yaml(datadump)
2308 except Exception as exc:
2309 exc.args = ("# Error while processing %s" % host,) + exc.args
2310 raise
2311 elif sys.argv[1] == "list":
2312 use_fqdn = False
2313 if len(sys.argv) < 4:
2314 usage()
2315 if not sys.argv[2] in ["up", "down", "planned", "all"]:
2316 usage()
2317 if not sys.argv[3] in ["nodes","proxies","systems"]:
2318 usage()
2319
2320 if len(sys.argv) > 4:
2321 if sys.argv[4] == "fqdn":
2322 use_fqdn = True
2323 else:
2324 usage()
2325
2326 for system in get_hostlist():
2327 datadump = get_yaml(system)
2328 if sys.argv[3] == 'proxies' and not datadump['service_proxy_ileiden']:
2329 continue
2330
2331 output = datadump['autogen_fqdn'] if use_fqdn else system
2332 if sys.argv[2] == "all":
2333 print(output)
2334 elif datadump['status'] == sys.argv[2]:
2335 print(output)
2336 elif sys.argv[1] == "create":
2337 if sys.argv[2] == "network.kml":
2338 print(make_network_kml.make_graph())
2339 elif sys.argv[2] == "host-ips.txt":
2340 for system in get_hostlist():
2341 datadump = get_yaml(system)
2342 ips = [datadump['masterip']]
2343 for ifkey in get_interface_keys(datadump):
2344 ips.append(datadump[ifkey]['ip'].split('/')[0])
2345 print(system, ' '.join(ips))
2346 elif sys.argv[2] == "host-pos.txt":
2347 for system in get_hostlist():
2348 datadump = get_yaml(system)
2349 print(system, datadump['rdnap_x'], datadump['rdnap_y'])
2350 elif sys.argv[2] == "nodeplanner.json":
2351 print(make_network_kml.make_nodeplanner_json())
2352 elif sys.argv[2] == 'ssh_config':
2353 print('''
2354Host *.wleiden.net
2355 User root
2356
2357Host 172.16.*.*
2358 User root
2359''')
2360 for system in get_hostlist():
2361 datadump = get_yaml(system)
2362 print('''\
2363Host %s
2364 User root
2365
2366Host %s
2367 User root
2368
2369Host %s
2370 User root
2371
2372Host %s
2373 User root
2374''' % (system, system.lower(), datadump['nodename'], datadump['nodename'].lower()))
2375 else:
2376 usage()
2377 else:
2378 usage()
2379 else:
2380 # Do not enable debugging for config requests as it highly clutters the output
2381 if not is_text_request():
2382 cgitb.enable()
2383
2384 try:
2385 response_headers, output = process_cgi_request()
2386 except:
2387 print('')
2388 print('')
2389 raise
2390
2391 print_cgi_response(response_headers, output)
2392
2393if __name__ == "__main__":
2394 main()
Note: See TracBrowser for help on using the repository browser.