source: genesis/nodes/get-network-status.py@ 8347

Last change on this file since 8347 was 8342, checked in by rick, 14 years ago
  • genesis-to-yaml.pl wildcard hack
  • merge does not need to have initial file
  • Property svn:executable set to *
File size: 14.5 KB
Line 
1#!/usr/bin/env python
2# vim:ts=2:et:sw=2:ai
3#
4# Scan Wireless Leiden Network and report status of links and nodes
5#
6# Rick van der Zwet <info@rickvanderzwet.nl>
7
8from pprint import pprint
9from xml.dom.minidom import parse, parseString
10import gformat
11import os.path
12import re
13import subprocess
14import sys
15import time
16import yaml
17from datetime import datetime
18
19# When force is used as argument, use this range
20DEFAULT_SCAN_RANGE= ['172.16.0.0/21']
21
22# Default node status output
23nodemap_status_file = '/tmp/nodemap_status.yaml'
24
25#
26# BEGIN nmap XML parser
27# XXX: Should properly go to seperate class/module
28def get_attribute(node,attr):
29 return node.attributes[attr].value
30
31def attribute_from_node(parent,node,attr):
32 return parent.getElementsByTagName(node)[0].attributes[attr].value
33
34def parse_port(node):
35 item = dict()
36 item['protocol'] = get_attribute(node,'protocol')
37 item['portid'] = get_attribute(node,'portid')
38 item['state'] = attribute_from_node(node,'state','state')
39 item['reason'] = attribute_from_node(node,'state','reason')
40 return item
41
42def parse_ports(node):
43 item = dict()
44 for port in node.getElementsByTagName('port'):
45 port_item = parse_port(port)
46 item[port_item['portid']] = port_item
47 return item
48
49def parse_host(node):
50 # Host status
51 item = dict()
52 item['state'] = attribute_from_node(node,'status','state')
53 item['reason'] = attribute_from_node(node,'status','reason')
54 item['addr'] = attribute_from_node(node,'address','addr')
55 item['addrtype'] = attribute_from_node(node,'address','addrtype')
56
57 # Service status
58 ports = node.getElementsByTagName('ports')
59 if ports:
60 item['port'] = parse_ports(ports[0])
61 return item
62
63def parse_nmap(root):
64 status = dict()
65 for node in root.childNodes[2].getElementsByTagName('host'):
66 scan = parse_host(node)
67 if not status.has_key(scan['addr']):
68 status[scan['addr']] = scan
69 return status
70#
71# END nmap parser
72#
73
74
75
76def _do_nmap_scan(command, iphosts):
77 """ Run/Read nmap XML with various choices"""
78 command = "nmap -n -iL - -oX - %s" %(command)
79 print "# New run '%s', can take a while to complete" % (command)
80 p = subprocess.Popen(command.split(),
81 stdout=subprocess.PIPE,
82 stderr=subprocess.PIPE,
83 stdin=subprocess.PIPE, bufsize=-1)
84
85 (stdoutdata, stderrdata) = p.communicate("\n".join(iphosts))
86 if p.returncode != 0:
87 print "# [ERROR] nmap failed to complete '%s'" % stderrdata
88 sys.exit(1)
89
90 dom = parseString(stdoutdata)
91 return (parse_nmap(dom),stdoutdata)
92
93
94
95def do_nmap_scan(command, iphosts, result_file=None, forced_scan=False):
96 """ Wrapper around _run_nmap to get listing of all hosts, the default nmap
97 does not return results for failed hosts"""
98 # Get all hosts to be processed
99 (init_status, stdoutdata) = _do_nmap_scan(" -sL",iphosts)
100
101 # Return stored file if exists
102 if not forced_scan and result_file and os.path.exists(result_file) \
103 and os.path.getsize(result_file) > 0:
104 print "# Reading stored NMAP results from '%s'" % (result_file)
105 status = parse_nmap(parse(result_file))
106 else:
107 # New scan
108 (status, stdoutdata) = _do_nmap_scan(command, iphosts)
109
110 # Store result if requested
111 if result_file:
112 print "# Saving results in %s" % (result_file)
113 f = file(result_file,'w')
114 f.write(stdoutdata)
115 f.close()
116
117 init_status.update(status)
118 return init_status
119
120
121
122def do_snmpwalk(host, oid):
123 """ Do snmpwalk, returns (p, stdout, stderr)"""
124 # GLobal SNMP walk options
125 snmpwalk = ('snmpwalk -r 0 -t 1 -OX -c public -v 2c %s' % host).split()
126 p = subprocess.Popen(snmpwalk + [oid],
127 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
128 (stdoutdata, stderrdata) = p.communicate()
129 stdoutdata = stdoutdata.split('\n')[:-1]
130 stderrdata = stderrdata.split('\n')[:-1]
131 return (p, stdoutdata, stderrdata)
132
133
134
135
136def do_snmp_scan(iphosts, status, stored_status=dict(), forced_scan=False):
137 """ SNMP scanning, based on results fould in NMAP scan"""
138 mac_to_host = dict()
139 host_processed = dict()
140
141 #
142 # Gather SNMP data from hosts
143 for host in iphosts:
144 # Status might be containing old hosts as well and visa-versa
145 if not status.has_key(host):
146 print "## [ERROR] No nmap result found"
147 continue
148
149 scan = status[host]
150 if scan['state'] != "up":
151 continue
152
153 print '# Processing host %s' % host
154 # IP -> Mac addresses found in host ARP table, with key IP
155 status[host]['arpmac'] = dict()
156 # MAC -> iface addresses, with key MAC
157 status[host]['mac'] = dict()
158 # Mirrored: iface -> MAC addresses, with key interface name
159 status[host]['iface'] = dict()
160 try:
161 if not forced_scan and stored_status[host]['snmp_retval'] != 0:
162 print "## SNMP Connect failed last time, ignoring"
163 continue
164 except:
165 pass
166
167 stored_status[host] = dict()
168 if not "open" in scan['port']['161']['state']:
169 print "## [ERROR] SNMP port not opened"
170 continue
171
172 (p, output, stderrdata) = do_snmpwalk(host, 'SNMPv2-MIB::sysDescr')
173 stored_status[host]['snmp_retval'] = p.returncode
174 # Assume host remain reachable troughout all the SNMP queries
175 if p.returncode != 0:
176 print "## [ERROR] SNMP failed '%s'" % ",".join(stderrdata)
177 continue
178
179 # Get some host details
180 # SNMPv2-MIB::sysDescr.0 = STRING: FreeBSD CNodeSOM1.wLeiden.NET
181 # 8.0-RELEASE-p2 FreeBSD 8.0-RELEASE-p2 #2: Fri Feb 19 18:24:23 CET 2010
182 # root@80fab2:/usr/obj/nanobsd.wleiden/usr/src/sys/kernel.wleiden i386
183 status[host]['sys_desc'] = output[0]
184 hostname = output[0].split(' ')[4]
185 release = output[0].split(' ')[5]
186 stored_status[host]['hostname'] = status[host]['hostname'] = hostname
187 stored_status[host]['release'] = status[host]['release'] = release
188 print "## %(hostname)s - %(release)s" % stored_status[host]
189
190 # Check if the host is already done processing
191 # Note: the host is marked done processing at the end
192 if host_processed.has_key(hostname):
193 print "## Host already processed this run"
194 continue
195
196 # Interface list with key the index number
197 iface_descr = dict()
198 # IF-MIB::ifDescr.1 = STRING: ath0
199 r = re.compile('^IF-MIB::ifDescr\[([0-9]+)\] = STRING: ([a-z0-9]+)$')
200 (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifDescr')
201 for line in output:
202 m = r.match(line)
203 iface_descr[m.group(1)] = m.group(2)
204
205 # IF-MIB::ifPhysAddress[1] = STRING: 0:80:48:54:bb:52
206 r = re.compile('^IF-MIB::ifPhysAddress\[([0-9]+)\] = STRING: ([0-9a-f:]*)$')
207 (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifPhysAddress')
208 for line in output:
209 m = r.match(line)
210 # Ignore lines which has no MAC address
211 if not m.group(2): continue
212 index = m.group(1)
213 # Convert to proper MAC
214 mac = ":".join(["%02X" % int(x,16) for x in m.group(2).split(':')])
215 if not iface_descr.has_key(index):
216 print "## Index cannot be mapped to a key, available:"
217 for index, value in iface_descr.iteritems():
218 print "## - %s [%s]" % (value, index)
219 else:
220 print "## Local MAC %s [index:%s] -> %s" % (iface_descr[index], index, mac)
221 status[host]['mac'][mac] = iface_descr[index]
222 status[host]['iface'][iface_descr[index]] = mac
223 mac_to_host[mac] = hostname
224
225 # Process host SNMP status
226 (p, output, stderrdata) = do_snmpwalk(host, 'RFC1213-MIB::atPhysAddress')
227 # RFC1213-MIB::atPhysAddress[8][1.172.21.160.34] = Hex-STRING: 00 17 C4 CC 5B F2
228 r = re.compile('^RFC1213-MIB::atPhysAddress\[[0-9]+\]\[1\.([0-9\.]+)\] = Hex-STRING: ([0-9A-F\ ]+)$')
229 for line in output:
230 m = r.match(line)
231 ip = m.group(1)
232 # Replace spaces in MAC with :
233 mac = ":".join(m.group(2).split(' ')[:-1])
234 status[host]['arpmac'][ip] = mac
235
236 local = '[remote]'
237 if mac in status[host]['mac'].keys():
238 local = '[local]'
239 print "## Arp table MAC %s -> %s %s" % (ip, mac, local)
240
241 # Make sure we keep a record of the processed host which ip entry to check
242 host_processed[hostname] = host
243
244 stored_status['host_processed'] = host_processed
245 stored_status['mac_to_host'] = mac_to_host
246 stored_status['nmap_status'] = status
247 return stored_status
248
249
250
251
252def generate_status(configs, stored_status):
253 """ Generate result file from stored_status """
254 host_processed = stored_status['host_processed']
255 mac_to_host = stored_status['mac_to_host']
256 status = stored_status['nmap_status']
257
258 # Data store format used for nodemap generation
259 nodemap = { 'node' : {}, 'link' : {}}
260
261 # XXX: Pushed back till we actually store the MAC in the config files automatically
262 #configmac_to_host = dict()
263 #for host,config in configs.iteritems():
264 # for iface_key in gformat.get_interface_keys(config):
265 # configmac_to_host[config[iface_key]['mac']] = host
266
267 # List of hosts which has some kind of problem
268 for host in configs.keys():
269 fqdn = host + ".wLeiden.NET"
270 if fqdn in host_processed.keys():
271 continue
272 config = configs[host]
273 print "# Problems in host '%s'" % host
274 host_down = True
275 for ip in gformat.get_used_ips([config]):
276 if not gformat.valid_addr(ip):
277 continue
278 if status[ip]['state'] == "up":
279 host_down = False
280 print "## - ", ip, status[ip]['state']
281 if host_down:
282 print "## HOST is DOWN!"
283 nodemap['node'][fqdn] = gformat.DOWN
284 else:
285 print "## SNMP problems (not reachable, deamon not running, etc)"
286 nodemap['node'][fqdn] = gformat.UNKNOWN
287
288
289
290 # Correlation mapping
291 for fqdn, ip in host_processed.iteritems():
292 details = status[ip]
293 nodemap['node'][fqdn] = gformat.OK
294 print "# Working on %s" % fqdn
295 for ip, arpmac in details['arpmac'].iteritems():
296 if arpmac in details['mac'].keys():
297 # Local MAC address
298 continue
299 if not mac_to_host.has_key(arpmac):
300 print "## [WARN] No parent host for MAC %s (%s) found" % (arpmac, ip)
301 else:
302 print "## Interlink %s - %s" % (fqdn, mac_to_host[arpmac])
303 nodemap['link'][(fqdn,mac_to_host[arpmac])] = gformat.OK
304
305 stream = file(nodemap_status_file,'w')
306 yaml.dump(nodemap, stream, default_flow_style=False)
307 print "# Wrote nodemap status to '%s'" % nodemap_status_file
308
309
310def do_merge(files):
311 """ Merge all external statuses in our own nodestatus, using optimistic approch """
312 try:
313 stream = file(nodemap_status_file,'r')
314 status = yaml.load(stream)
315 except IOError, e:
316 # Data store format used for nodemap generation
317 status = { 'node' : {}, 'link' : {}}
318
319 for cfile in files:
320 try:
321 print "# Merging '%s'" % cfile
322 stream = file(cfile,'r')
323 new_status = yaml.load(stream)
324 for item in ['node', 'link']:
325 for key, value in new_status[item].iteritems():
326 if not status[item].has_key(key):
327 # New items always welcome
328 status[item][key] = value
329 print "## [%s][%s] is new (%s)" % (item, key, value)
330 elif value < status[item][key]:
331 # Better values always welcome
332 status[item][key] = value
333 print "## [%s][%s] is better (%s)" % (item, key, value)
334 except IOError, e:
335 print "## ERROR '%s'" % e
336
337 # Save results back to file
338 stream = file(nodemap_status_file,'w')
339 yaml.dump(status, stream, default_flow_style=False)
340
341def usage():
342 print "Usage: %s <arguments>"
343 print "Arguments:"
344 print "\tall = scan all known ips, using cached nmap"
345 print "\tnmap-only = scan all known ips, using nmap only"
346 print "\tsnmp-only = scan all known ips, using snmp only"
347 print "\tforce = scan all known ips, no cache used"
348 print "\tforced-snmp = scan all known ips, no snmp cache"
349 print "\tstored = generate status file using stored entries"
350 print "\thost <HOST1> [HOST2 ...] = generate status file using stored entries"
351 print "\tmerge <FILE1> [FILE2 ...] = merge status file with other status files"
352 sys.exit(0)
353
354
355def main():
356 start_time = datetime.now()
357 stored_status_file = '/tmp/stored_status.yaml'
358 nmap_result_file = '/tmp/test.xml'
359
360 stored_status = dict()
361 nmap_status = dict()
362 snmp_status = dict()
363
364 opt_nmap_scan = True
365 opt_store_scan = True
366 opt_snmp_scan = True
367 opt_force_snmp = False
368 opt_force_scan = False
369 opt_force_range = False
370 if len(sys.argv) == 1:
371 usage()
372
373 if sys.argv[1] == "all":
374 pass
375 elif sys.argv[1] == "nmap-only":
376 opt_snmp_scan = False
377 elif sys.argv[1] == "snmp-only":
378 opt_nmap_scan = False
379 elif sys.argv[1] == "force":
380 opt_force_scan = True
381 elif sys.argv[1] == "forced-snmp":
382 opt_nmap_scan = False
383 opt_force_snmp = True
384 elif sys.argv[1] == "host":
385 opt_force_range = True
386 opt_force_scan = True
387 elif sys.argv[1] == "stored":
388 opt_snmp_scan = False
389 opt_nmap_scan = False
390 opt_store_scan = False
391 elif sys.argv[1] == "merge":
392 do_merge(sys.argv[2:])
393 sys.exit(0)
394 else:
395 usage()
396
397 # By default get all IPs defined in config, else own range
398 if not opt_force_range:
399 configs = gformat.get_all_configs()
400 iplist = gformat.get_used_ips(configs.values())
401 else:
402 iplist = sys.argv[1:]
403
404 # Load data hints from previous run if exists
405 if not opt_force_scan and os.path.exists(stored_status_file) and os.path.getsize(stored_status_file) > 0:
406 print "## Loading stored data hints from '%s'" % stored_status_file
407 stream = file(stored_status_file,'r')
408 stored_status = yaml.load(stream)
409 else:
410 print "[ERROR] '%s' does not exists" % stored_status_file
411
412 # Do a NMAP discovery
413 if opt_nmap_scan:
414 if not opt_store_scan:
415 nmap_result_file = None
416 nmap_status = do_nmap_scan(
417 "-p T:ssh,U:domain,T:80,T:ntp,U:snmp,T:8080 -sU -sT ",
418 iplist,nmap_result_file, opt_force_scan)
419
420 else:
421 nmap_status = stored_status['nmap_status']
422
423 # Save the MAC -> HOST mappings, by default as it helps indentifing the
424 # 'unknown links'
425 mac_to_host = {}
426 if stored_status:
427 mac_to_host = stored_status['mac_to_host']
428
429 # Do SNMP discovery
430 if opt_snmp_scan:
431 snmp_status = do_snmp_scan(iplist, nmap_status, stored_status, opt_force_snmp)
432 else:
433 snmp_status = stored_status
434
435 # Include our saved MAC -> HOST mappings
436 mac_to_host.update(snmp_status['mac_to_host'])
437 snmp_status['mac_to_host'] = mac_to_host
438
439 # Store changed data to disk
440 if opt_store_scan:
441 stream = file(stored_status_file,'w')
442 yaml.dump(snmp_status, stream, default_flow_style=False)
443 print "## Stored data hints to '%s'" % stored_status_file
444
445 # Finally generated status
446 generate_status(configs, snmp_status)
447 print "# Took %s seconds to complete" % (datetime.now() - start_time).seconds
448
449
450
451if __name__ == "__main__":
452 main()
Note: See TracBrowser for help on using the repository browser.