source: genesis/nodes/get-network-status.py@ 8584

Last change on this file since 8584 was 8517, checked in by rick, 14 years ago

Exception handling to a yet unknown error.

  • Property svn:executable set to *
File size: 14.6 KB
Line 
1#!/usr/bin/env python
2# vim:ts=2:et:sw=2:ai
3#
4# Scan Wireless Leiden Network and report status of links and nodes
5#
6# Rick van der Zwet <info@rickvanderzwet.nl>
7
8from pprint import pprint
9from xml.dom.minidom import parse, parseString
10import gformat
11import os.path
12import re
13import subprocess
14import sys
15import time
16import yaml
17from datetime import datetime
18
19# When force is used as argument, use this range
20DEFAULT_SCAN_RANGE= ['172.16.0.0/12']
21
22# Default node status output
23nodemap_status_file = '/tmp/nodemap_status.yaml'
24
25#
26# BEGIN nmap XML parser
27# XXX: Should properly go to seperate class/module
28def get_attribute(node,attr):
29 return node.attributes[attr].value
30
31def attribute_from_node(parent,node,attr):
32 return parent.getElementsByTagName(node)[0].attributes[attr].value
33
34def parse_port(node):
35 item = dict()
36 item['protocol'] = get_attribute(node,'protocol')
37 item['portid'] = get_attribute(node,'portid')
38 item['state'] = attribute_from_node(node,'state','state')
39 item['reason'] = attribute_from_node(node,'state','reason')
40 return item
41
42def parse_ports(node):
43 item = dict()
44 for port in node.getElementsByTagName('port'):
45 port_item = parse_port(port)
46 item[port_item['portid']] = port_item
47 return item
48
49def parse_host(node):
50 # Host status
51 item = dict()
52 item['state'] = attribute_from_node(node,'status','state')
53 item['reason'] = attribute_from_node(node,'status','reason')
54 item['addr'] = attribute_from_node(node,'address','addr')
55 item['addrtype'] = attribute_from_node(node,'address','addrtype')
56
57 # Service status
58 ports = node.getElementsByTagName('ports')
59 if ports:
60 item['port'] = parse_ports(ports[0])
61 return item
62
63def parse_nmap(root):
64 status = dict()
65 for node in root.childNodes[2].getElementsByTagName('host'):
66 scan = parse_host(node)
67 if not status.has_key(scan['addr']):
68 status[scan['addr']] = scan
69 return status
70#
71# END nmap parser
72#
73
74
75
76def _do_nmap_scan(command, iphosts):
77 """ Run/Read nmap XML with various choices"""
78 command = "nmap -n -iL - -oX - %s" %(command)
79 print "# New run '%s', can take a while to complete" % (command)
80 p = subprocess.Popen(command.split(),
81 stdout=subprocess.PIPE,
82 stderr=subprocess.PIPE,
83 stdin=subprocess.PIPE, bufsize=-1)
84
85 (stdoutdata, stderrdata) = p.communicate("\n".join(iphosts))
86 if p.returncode != 0:
87 print "# [ERROR] nmap failed to complete '%s'" % stderrdata
88 sys.exit(1)
89
90 dom = parseString(stdoutdata)
91 return (parse_nmap(dom),stdoutdata)
92
93
94
95def do_nmap_scan(command, iphosts, result_file=None, forced_scan=False):
96 """ Wrapper around _run_nmap to get listing of all hosts, the default nmap
97 does not return results for failed hosts"""
98 # Get all hosts to be processed
99 (init_status, stdoutdata) = _do_nmap_scan(" -sL",iphosts)
100
101 # Return stored file if exists
102 if not forced_scan and result_file and os.path.exists(result_file) \
103 and os.path.getsize(result_file) > 0:
104 print "# Reading stored NMAP results from '%s'" % (result_file)
105 status = parse_nmap(parse(result_file))
106 else:
107 # New scan
108 (status, stdoutdata) = _do_nmap_scan(command, iphosts)
109
110 # Store result if requested
111 if result_file:
112 print "# Saving results in %s" % (result_file)
113 f = file(result_file,'w')
114 f.write(stdoutdata)
115 f.close()
116
117 init_status.update(status)
118 return init_status
119
120
121
122def do_snmpwalk(host, oid):
123 """ Do snmpwalk, returns (p, stdout, stderr)"""
124 # GLobal SNMP walk options
125 snmpwalk = ('snmpwalk -r 0 -t 1 -OX -c public -v 2c %s' % host).split()
126 p = subprocess.Popen(snmpwalk + [oid],
127 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
128 (stdoutdata, stderrdata) = p.communicate()
129 stdoutdata = stdoutdata.split('\n')[:-1]
130 stderrdata = stderrdata.split('\n')[:-1]
131 return (p, stdoutdata, stderrdata)
132
133
134
135
136def do_snmp_scan(iphosts, status, stored_status=dict(), forced_scan=False):
137 """ SNMP scanning, based on results fould in NMAP scan"""
138 mac_to_host = dict()
139 host_processed = dict()
140
141 #
142 # Gather SNMP data from hosts
143 for host in iphosts:
144 # Status might be containing old hosts as well and visa-versa
145 if not status.has_key(host):
146 print "## [ERROR] No nmap result found"
147 continue
148
149 scan = status[host]
150 if scan['state'] != "up":
151 continue
152
153 print '# Processing host %s' % host
154 # IP -> Mac addresses found in host ARP table, with key IP
155 status[host]['arpmac'] = dict()
156 # MAC -> iface addresses, with key MAC
157 status[host]['mac'] = dict()
158 # Mirrored: iface -> MAC addresses, with key interface name
159 status[host]['iface'] = dict()
160 try:
161 if not forced_scan and stored_status[host]['snmp_retval'] != 0:
162 print "## SNMP Connect failed last time, ignoring"
163 continue
164 except:
165 pass
166
167 stored_status[host] = dict()
168 if not "open" in scan['port']['161']['state']:
169 print "## [ERROR] SNMP port not opened"
170 continue
171
172 (p, output, stderrdata) = do_snmpwalk(host, 'SNMPv2-MIB::sysDescr')
173 stored_status[host]['snmp_retval'] = p.returncode
174 # Assume host remain reachable troughout all the SNMP queries
175 if p.returncode != 0:
176 print "## [ERROR] SNMP failed '%s'" % ",".join(stderrdata)
177 continue
178
179 # Get some host details
180 # SNMPv2-MIB::sysDescr.0 = STRING: FreeBSD CNodeSOM1.wLeiden.NET
181 # 8.0-RELEASE-p2 FreeBSD 8.0-RELEASE-p2 #2: Fri Feb 19 18:24:23 CET 2010
182 # root@80fab2:/usr/obj/nanobsd.wleiden/usr/src/sys/kernel.wleiden i386
183 status[host]['sys_desc'] = output[0]
184 hostname = output[0].split(' ')[4]
185 release = output[0].split(' ')[5]
186 stored_status[host]['hostname'] = status[host]['hostname'] = hostname
187 stored_status[host]['release'] = status[host]['release'] = release
188 print "## %(hostname)s - %(release)s" % stored_status[host]
189
190 # Check if the host is already done processing
191 # Note: the host is marked done processing at the end
192 if host_processed.has_key(hostname):
193 print "## Host already processed this run"
194 continue
195
196 # Interface list with key the index number
197 iface_descr = dict()
198 # IF-MIB::ifDescr.1 = STRING: ath0
199 r = re.compile('^IF-MIB::ifDescr\[([0-9]+)\] = STRING: ([a-z0-9]+)$')
200 (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifDescr')
201 for line in output:
202 m = r.match(line)
203 iface_descr[m.group(1)] = m.group(2)
204
205 # IF-MIB::ifPhysAddress[1] = STRING: 0:80:48:54:bb:52
206 r = re.compile('^IF-MIB::ifPhysAddress\[([0-9]+)\] = STRING: ([0-9a-f:]*)$')
207 (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifPhysAddress')
208 for line in output:
209 m = r.match(line)
210 # Ignore lines which has no MAC address
211 if not m.group(2): continue
212 index = m.group(1)
213 # Convert to proper MAC
214 mac = ":".join(["%02X" % int(x,16) for x in m.group(2).split(':')])
215 if not iface_descr.has_key(index):
216 print "## Index cannot be mapped to a key, available:"
217 for index, value in iface_descr.iteritems():
218 print "## - %s [%s]" % (value, index)
219 else:
220 print "## Local MAC %s [index:%s] -> %s" % (iface_descr[index], index, mac)
221 status[host]['mac'][mac] = iface_descr[index]
222 status[host]['iface'][iface_descr[index]] = mac
223 mac_to_host[mac] = hostname
224
225 # Process host SNMP status
226 (p, output, stderrdata) = do_snmpwalk(host, 'RFC1213-MIB::atPhysAddress')
227 # RFC1213-MIB::atPhysAddress[8][1.172.21.160.34] = Hex-STRING: 00 17 C4 CC 5B F2
228 r = re.compile('^RFC1213-MIB::atPhysAddress\[[0-9]+\]\[1\.([0-9\.]+)\] = Hex-STRING: ([0-9A-F\ ]+)$')
229 for line in output:
230 m = r.match(line)
231 if not m:
232 print "## ERROR Unable to parse '%s'" % line
233 continue
234 ip = m.group(1)
235 # Replace spaces in MAC with :
236 mac = ":".join(m.group(2).split(' ')[:-1])
237 status[host]['arpmac'][ip] = mac
238
239 local = '[remote]'
240 if mac in status[host]['mac'].keys():
241 local = '[local]'
242 print "## Arp table MAC %s -> %s %s" % (ip, mac, local)
243
244 # Make sure we keep a record of the processed host which ip entry to check
245 host_processed[hostname] = host
246
247 stored_status['host_processed'] = host_processed
248 stored_status['mac_to_host'] = mac_to_host
249 stored_status['nmap_status'] = status
250 return stored_status
251
252
253
254
255def generate_status(configs, stored_status):
256 """ Generate result file from stored_status """
257 host_processed = stored_status['host_processed']
258 mac_to_host = stored_status['mac_to_host']
259 status = stored_status['nmap_status']
260
261 # Data store format used for nodemap generation
262 nodemap = { 'node' : {}, 'link' : {}}
263
264 # XXX: Pushed back till we actually store the MAC in the config files automatically
265 #configmac_to_host = dict()
266 #for host,config in configs.iteritems():
267 # for iface_key in gformat.get_interface_keys(config):
268 # configmac_to_host[config[iface_key]['mac']] = host
269
270 # List of hosts which has some kind of problem
271 for host in configs.keys():
272 fqdn = host + ".wLeiden.NET"
273 if fqdn in host_processed.keys():
274 continue
275 config = configs[host]
276 print "# Problems in host '%s'" % host
277 host_down = True
278 for ip in gformat.get_used_ips([config]):
279 if not gformat.valid_addr(ip):
280 continue
281 if status[ip]['state'] == "up":
282 host_down = False
283 print "## - ", ip, status[ip]['state']
284 if host_down:
285 print "## HOST is DOWN!"
286 nodemap['node'][fqdn] = gformat.DOWN
287 else:
288 print "## SNMP problems (not reachable, deamon not running, etc)"
289 nodemap['node'][fqdn] = gformat.UNKNOWN
290
291
292
293 # Correlation mapping
294 for fqdn, ip in host_processed.iteritems():
295 details = status[ip]
296 nodemap['node'][fqdn] = gformat.OK
297 print "# Working on %s" % fqdn
298 for ip, arpmac in details['arpmac'].iteritems():
299 if arpmac in details['mac'].keys():
300 # Local MAC address
301 continue
302 if not mac_to_host.has_key(arpmac):
303 print "## [WARN] No parent host for MAC %s (%s) found" % (arpmac, ip)
304 else:
305 print "## Interlink %s - %s" % (fqdn, mac_to_host[arpmac])
306 nodemap['link'][(fqdn,mac_to_host[arpmac])] = gformat.OK
307
308 stream = file(nodemap_status_file,'w')
309 yaml.dump(nodemap, stream, default_flow_style=False)
310 print "# Wrote nodemap status to '%s'" % nodemap_status_file
311
312
313def do_merge(files):
314 """ Merge all external statuses in our own nodestatus, using optimistic approch """
315 try:
316 stream = file(nodemap_status_file,'r')
317 status = yaml.load(stream)
318 except IOError, e:
319 # Data store format used for nodemap generation
320 status = { 'node' : {}, 'link' : {}}
321
322 for cfile in files:
323 try:
324 print "# Merging '%s'" % cfile
325 stream = file(cfile,'r')
326 new_status = yaml.load(stream)
327 for item in ['node', 'link']:
328 for key, value in new_status[item].iteritems():
329 if not status[item].has_key(key):
330 # New items always welcome
331 status[item][key] = value
332 print "## [%s][%s] is new (%s)" % (item, key, value)
333 elif value < status[item][key]:
334 # Better values always welcome
335 status[item][key] = value
336 print "## [%s][%s] is better (%s)" % (item, key, value)
337 except IOError, e:
338 print "## ERROR '%s'" % e
339
340 # Save results back to file
341 stream = file(nodemap_status_file,'w')
342 yaml.dump(status, stream, default_flow_style=False)
343
344def usage():
345 print "Usage: %s <arguments>"
346 print "Arguments:"
347 print "\tall = scan all known ips, using cached nmap"
348 print "\tnmap-only = scan all known ips, using nmap only"
349 print "\tsnmp-only = scan all known ips, using snmp only"
350 print "\tforce = scan all known ips, no cache used"
351 print "\tforced-snmp = scan all known ips, no snmp cache"
352 print "\tstored = generate status file using stored entries"
353 print "\thost <HOST1> [HOST2 ...] = generate status file using stored entries"
354 print "\tmerge <FILE1> [FILE2 ...] = merge status file with other status files"
355 sys.exit(0)
356
357
358def main():
359 start_time = datetime.now()
360 stored_status_file = '/tmp/stored_status.yaml'
361 nmap_result_file = '/tmp/test.xml'
362
363 stored_status = dict()
364 nmap_status = dict()
365 snmp_status = dict()
366
367 opt_nmap_scan = True
368 opt_store_scan = True
369 opt_snmp_scan = True
370 opt_force_snmp = False
371 opt_force_scan = False
372 opt_force_range = False
373 if len(sys.argv) == 1:
374 usage()
375
376 if sys.argv[1] == "all":
377 pass
378 elif sys.argv[1] == "nmap-only":
379 opt_snmp_scan = False
380 elif sys.argv[1] == "snmp-only":
381 opt_nmap_scan = False
382 elif sys.argv[1] == "force":
383 opt_force_scan = True
384 elif sys.argv[1] == "forced-snmp":
385 opt_nmap_scan = False
386 opt_force_snmp = True
387 elif sys.argv[1] == "host":
388 opt_force_range = True
389 opt_force_scan = True
390 elif sys.argv[1] == "stored":
391 opt_snmp_scan = False
392 opt_nmap_scan = False
393 opt_store_scan = False
394 elif sys.argv[1] == "merge":
395 do_merge(sys.argv[2:])
396 sys.exit(0)
397 else:
398 usage()
399
400 # By default get all IPs defined in config, else own range
401 if not opt_force_range:
402 configs = gformat.get_all_configs()
403 iplist = gformat.get_used_ips(configs.values())
404 else:
405 iplist = sys.argv[1:]
406
407 # Load data hints from previous run if exists
408 if not opt_force_scan and os.path.exists(stored_status_file) and os.path.getsize(stored_status_file) > 0:
409 print "## Loading stored data hints from '%s'" % stored_status_file
410 stream = file(stored_status_file,'r')
411 stored_status = yaml.load(stream)
412 else:
413 print "[ERROR] '%s' does not exists" % stored_status_file
414
415 # Do a NMAP discovery
416 if opt_nmap_scan:
417 if not opt_store_scan:
418 nmap_result_file = None
419 nmap_status = do_nmap_scan(
420 "-p T:ssh,U:domain,T:80,T:ntp,U:snmp,T:8080 -sU -sT ",
421 iplist,nmap_result_file, opt_force_scan)
422
423 else:
424 nmap_status = stored_status['nmap_status']
425
426 # Save the MAC -> HOST mappings, by default as it helps indentifing the
427 # 'unknown links'
428 mac_to_host = {}
429 if stored_status:
430 mac_to_host = stored_status['mac_to_host']
431
432 # Do SNMP discovery
433 if opt_snmp_scan:
434 snmp_status = do_snmp_scan(iplist, nmap_status, stored_status, opt_force_snmp)
435 else:
436 snmp_status = stored_status
437
438 # Include our saved MAC -> HOST mappings
439 mac_to_host.update(snmp_status['mac_to_host'])
440 snmp_status['mac_to_host'] = mac_to_host
441
442 # Store changed data to disk
443 if opt_store_scan:
444 stream = file(stored_status_file,'w')
445 yaml.dump(snmp_status, stream, default_flow_style=False)
446 print "## Stored data hints to '%s'" % stored_status_file
447
448 # Finally generated status
449 generate_status(configs, snmp_status)
450 print "# Took %s seconds to complete" % (datetime.now() - start_time).seconds
451
452
453
454if __name__ == "__main__":
455 main()
Note: See TracBrowser for help on using the repository browser.