source: genesis/nodes/get-network-status.py@ 8337

Last change on this file since 8337 was 8333, checked in by rick, 15 years ago
  • Darn keys are sometimes not around. Seems like a fetch or SNMP fail
  • Property svn:executable set to *
File size: 13.3 KB
Line 
1#!/usr/bin/env python
2# vim:ts=2:et:sw=2:ai
3#
4# Scan Wireless Leiden Network and report status of links and nodes
5#
6# Rick van der Zwet <info@rickvanderzwet.nl>
7
8from pprint import pprint
9from xml.dom.minidom import parse, parseString
10import gformat
11import os.path
12import re
13import subprocess
14import sys
15import time
16import yaml
17from datetime import datetime
18
19# When force is used as argument, use this range
20DEFAULT_SCAN_RANGE= ['172.16.0.0/21']
21
22#
23# BEGIN nmap XML parser
24# XXX: Should properly go to seperate class/module
25def get_attribute(node,attr):
26 return node.attributes[attr].value
27
28def attribute_from_node(parent,node,attr):
29 return parent.getElementsByTagName(node)[0].attributes[attr].value
30
31def parse_port(node):
32 item = dict()
33 item['protocol'] = get_attribute(node,'protocol')
34 item['portid'] = get_attribute(node,'portid')
35 item['state'] = attribute_from_node(node,'state','state')
36 item['reason'] = attribute_from_node(node,'state','reason')
37 return item
38
39def parse_ports(node):
40 item = dict()
41 for port in node.getElementsByTagName('port'):
42 port_item = parse_port(port)
43 item[port_item['portid']] = port_item
44 return item
45
46def parse_host(node):
47 # Host status
48 item = dict()
49 item['state'] = attribute_from_node(node,'status','state')
50 item['reason'] = attribute_from_node(node,'status','reason')
51 item['addr'] = attribute_from_node(node,'address','addr')
52 item['addrtype'] = attribute_from_node(node,'address','addrtype')
53
54 # Service status
55 ports = node.getElementsByTagName('ports')
56 if ports:
57 item['port'] = parse_ports(ports[0])
58 return item
59
60def parse_nmap(root):
61 status = dict()
62 for node in root.childNodes[2].getElementsByTagName('host'):
63 scan = parse_host(node)
64 if not status.has_key(scan['addr']):
65 status[scan['addr']] = scan
66 return status
67#
68# END nmap parser
69#
70
71
72
73def _do_nmap_scan(command, iphosts):
74 """ Run/Read nmap XML with various choices"""
75 command = "nmap -n -iL - -oX - %s" %(command)
76 print "# New run '%s', can take a while to complete" % (command)
77 p = subprocess.Popen(command.split(),
78 stdout=subprocess.PIPE,
79 stderr=subprocess.PIPE,
80 stdin=subprocess.PIPE, bufsize=-1)
81
82 (stdoutdata, stderrdata) = p.communicate("\n".join(iphosts))
83 if p.returncode != 0:
84 print "# [ERROR] nmap failed to complete '%s'" % stderrdata
85 sys.exit(1)
86
87 dom = parseString(stdoutdata)
88 return (parse_nmap(dom),stdoutdata)
89
90
91
92def do_nmap_scan(command, iphosts, result_file=None, forced_scan=False):
93 """ Wrapper around _run_nmap to get listing of all hosts, the default nmap
94 does not return results for failed hosts"""
95 # Get all hosts to be processed
96 (init_status, stdoutdata) = _do_nmap_scan(" -sL",iphosts)
97
98 # Return stored file if exists
99 if not forced_scan and result_file and os.path.exists(result_file) \
100 and os.path.getsize(result_file) > 0:
101 print "# Reading stored NMAP results from '%s'" % (result_file)
102 status = parse_nmap(parse(result_file))
103 else:
104 # New scan
105 (status, stdoutdata) = _do_nmap_scan(command, iphosts)
106
107 # Store result if requested
108 if result_file:
109 print "# Saving results in %s" % (result_file)
110 f = file(result_file,'w')
111 f.write(stdoutdata)
112 f.close()
113
114 init_status.update(status)
115 return init_status
116
117
118
119def do_snmpwalk(host, oid):
120 """ Do snmpwalk, returns (p, stdout, stderr)"""
121 # GLobal SNMP walk options
122 snmpwalk = ('snmpwalk -r 0 -t 1 -OX -c public -v 2c %s' % host).split()
123 p = subprocess.Popen(snmpwalk + [oid],
124 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
125 (stdoutdata, stderrdata) = p.communicate()
126 stdoutdata = stdoutdata.split('\n')[:-1]
127 stderrdata = stderrdata.split('\n')[:-1]
128 return (p, stdoutdata, stderrdata)
129
130
131
132
133def do_snmp_scan(iphosts, status, stored_status=dict(), forced_scan=False):
134 """ SNMP scanning, based on results fould in NMAP scan"""
135 mac_to_host = dict()
136 host_processed = dict()
137
138 #
139 # Gather SNMP data from hosts
140 for host in iphosts:
141 # Status might be containing old hosts as well and visa-versa
142 if not status.has_key(host):
143 print "## [ERROR] No nmap result found"
144 continue
145
146 scan = status[host]
147 if scan['state'] != "up":
148 continue
149
150 print '# Processing host %s' % host
151 # IP -> Mac addresses found in host ARP table, with key IP
152 status[host]['arpmac'] = dict()
153 # MAC -> iface addresses, with key MAC
154 status[host]['mac'] = dict()
155 # Mirrored: iface -> MAC addresses, with key interface name
156 status[host]['iface'] = dict()
157 try:
158 if not forced_scan and stored_status[host]['snmp_retval'] != 0:
159 print "## SNMP Connect failed last time, ignoring"
160 continue
161 except:
162 pass
163
164 stored_status[host] = dict()
165 if not "open" in scan['port']['161']['state']:
166 print "## [ERROR] SNMP port not opened"
167 continue
168
169 (p, output, stderrdata) = do_snmpwalk(host, 'SNMPv2-MIB::sysDescr')
170 stored_status[host]['snmp_retval'] = p.returncode
171 # Assume host remain reachable troughout all the SNMP queries
172 if p.returncode != 0:
173 print "## [ERROR] SNMP failed '%s'" % ",".join(stderrdata)
174 continue
175
176 # Get some host details
177 # SNMPv2-MIB::sysDescr.0 = STRING: FreeBSD CNodeSOM1.wLeiden.NET
178 # 8.0-RELEASE-p2 FreeBSD 8.0-RELEASE-p2 #2: Fri Feb 19 18:24:23 CET 2010
179 # root@80fab2:/usr/obj/nanobsd.wleiden/usr/src/sys/kernel.wleiden i386
180 status[host]['sys_desc'] = output[0]
181 hostname = output[0].split(' ')[4]
182 release = output[0].split(' ')[5]
183 stored_status[host]['hostname'] = status[host]['hostname'] = hostname
184 stored_status[host]['release'] = status[host]['release'] = release
185 print "## %(hostname)s - %(release)s" % stored_status[host]
186
187 # Check if the host is already done processing
188 # Note: the host is marked done processing at the end
189 if host_processed.has_key(hostname):
190 print "## Host already processed this run"
191 continue
192
193 # Interface list with key the index number
194 iface_descr = dict()
195 # IF-MIB::ifDescr.1 = STRING: ath0
196 r = re.compile('^IF-MIB::ifDescr\[([0-9]+)\] = STRING: ([a-z0-9]+)$')
197 (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifDescr')
198 for line in output:
199 m = r.match(line)
200 iface_descr[m.group(1)] = m.group(2)
201
202 # IF-MIB::ifPhysAddress[1] = STRING: 0:80:48:54:bb:52
203 r = re.compile('^IF-MIB::ifPhysAddress\[([0-9]+)\] = STRING: ([0-9a-f:]*)$')
204 (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifPhysAddress')
205 for line in output:
206 m = r.match(line)
207 # Ignore lines which has no MAC address
208 if not m.group(2): continue
209 index = m.group(1)
210 # Convert to proper MAC
211 mac = ":".join(["%02X" % int(x,16) for x in m.group(2).split(':')])
212 if not iface_descr.has_key(index):
213 print "## Index cannot be mapped to a key, available:"
214 for index, value in iface_descr.iteritems():
215 print "## - %s [%s]" % (value, index)
216 else:
217 print "## Local MAC %s [index:%s] -> %s" % (iface_descr[index], index, mac)
218 status[host]['mac'][mac] = iface_descr[index]
219 status[host]['iface'][iface_descr[index]] = mac
220 mac_to_host[mac] = hostname
221
222 # Process host SNMP status
223 (p, output, stderrdata) = do_snmpwalk(host, 'RFC1213-MIB::atPhysAddress')
224 # RFC1213-MIB::atPhysAddress[8][1.172.21.160.34] = Hex-STRING: 00 17 C4 CC 5B F2
225 r = re.compile('^RFC1213-MIB::atPhysAddress\[[0-9]+\]\[1\.([0-9\.]+)\] = Hex-STRING: ([0-9A-F\ ]+)$')
226 for line in output:
227 m = r.match(line)
228 ip = m.group(1)
229 # Replace spaces in MAC with :
230 mac = ":".join(m.group(2).split(' ')[:-1])
231 status[host]['arpmac'][ip] = mac
232
233 local = '[remote]'
234 if mac in status[host]['mac'].keys():
235 local = '[local]'
236 print "## Arp table MAC %s -> %s %s" % (ip, mac, local)
237
238 # Make sure we keep a record of the processed host which ip entry to check
239 host_processed[hostname] = host
240
241 stored_status['host_processed'] = host_processed
242 stored_status['mac_to_host'] = mac_to_host
243 stored_status['nmap_status'] = status
244 return stored_status
245
246
247
248
249def generate_status(configs, stored_status):
250 """ Generate result file from stored_status """
251 host_processed = stored_status['host_processed']
252 mac_to_host = stored_status['mac_to_host']
253 status = stored_status['nmap_status']
254
255 # Data store format used for nodemap generation
256 nodemap_status_file = '/tmp/nodemap_status.yaml'
257 nodemap = { 'node' : {}, 'link' : {}}
258
259 # XXX: Pushed back till we actually store the MAC in the config files automatically
260 #configmac_to_host = dict()
261 #for host,config in configs.iteritems():
262 # for iface_key in gformat.get_interface_keys(config):
263 # configmac_to_host[config[iface_key]['mac']] = host
264
265 # List of hosts which has some kind of problem
266 for host in configs.keys():
267 fqdn = host + ".wLeiden.NET"
268 if fqdn in host_processed.keys():
269 continue
270 config = configs[host]
271 print "# Problems in host '%s'" % host
272 host_down = True
273 for ip in gformat.get_used_ips([config]):
274 if not gformat.valid_addr(ip):
275 continue
276 if status[ip]['state'] == "up":
277 host_down = False
278 print "## - ", ip, status[ip]['state']
279 if host_down:
280 print "## HOST is DOWN!"
281 nodemap['node'][fqdn] = gformat.DOWN
282 else:
283 print "## SNMP problems (not reachable, deamon not running, etc)"
284 nodemap['node'][fqdn] = gformat.UNKNOWN
285
286
287
288 # Correlation mapping
289 for fqdn, ip in host_processed.iteritems():
290 details = status[ip]
291 nodemap['node'][fqdn] = gformat.OK
292 print "# Working on %s" % fqdn
293 for ip, arpmac in details['arpmac'].iteritems():
294 if arpmac in details['mac'].keys():
295 # Local MAC address
296 continue
297 if not mac_to_host.has_key(arpmac):
298 print "## [WARN] No parent host for MAC %s (%s) found" % (arpmac, ip)
299 else:
300 print "## Interlink %s - %s" % (fqdn, mac_to_host[arpmac])
301 nodemap['link'][(fqdn,mac_to_host[arpmac])] = gformat.OK
302
303 stream = file(nodemap_status_file,'w')
304 yaml.dump(nodemap, stream, default_flow_style=False)
305 print "# Wrote nodemap status to '%s'" % nodemap_status_file
306
307def usage():
308 print "Usage: %s <arguments>"
309 print "Arguments:"
310 print "\tall = scan all known ips, using cached nmap"
311 print "\tnmap-only = scan all known ips, using nmap only"
312 print "\tsnmp-only = scan all known ips, using snmp only"
313 print "\tforce = scan all known ips, no cache used"
314 print "\tforced-snmp = scan all known ips, no snmp cache"
315 print "\tstored = generate status file using stored entries"
316 print "\thost <HOST1> [HOST2 ...] = generate status file using stored entries"
317 print "\tmerge <FILE1> [FILE2 ...] = merge status file with other status files"
318 sys.exit(0)
319
320
321def main():
322 start_time = datetime.now()
323 stored_status_file = '/tmp/stored_status.yaml'
324 nmap_result_file = '/tmp/test.xml'
325
326 stored_status = dict()
327 nmap_status = dict()
328 snmp_status = dict()
329
330 opt_nmap_scan = True
331 opt_store_scan = True
332 opt_snmp_scan = True
333 opt_force_snmp = False
334 opt_force_scan = False
335 opt_force_range = False
336 if len(sys.argv) == 1:
337 usage()
338
339 if sys.argv[1] == "all":
340 pass
341 elif sys.argv[1] == "nmap-only":
342 opt_snmp_scan = False
343 elif sys.argv[1] == "snmp-only":
344 opt_nmap_scan = False
345 elif sys.argv[1] == "force":
346 opt_force_scan = True
347 elif sys.argv[1] == "forced-snmp":
348 opt_nmap_scan = False
349 opt_force_snmp = True
350 elif sys.argv[1] == "host":
351 opt_force_range = True
352 opt_force_scan = True
353 elif sys.argv[1] == "stored":
354 opt_snmp_scan = False
355 opt_nmap_scan = False
356 opt_store_scan = False
357 else:
358 usage()
359
360 # By default get all IPs defined in config, else own range
361 if not opt_force_range:
362 configs = gformat.get_all_configs()
363 iplist = gformat.get_used_ips(configs.values())
364 else:
365 iplist = sys.argv[1:]
366
367 # Load data hints from previous run if exists
368 if not opt_force_scan and os.path.exists(stored_status_file) and os.path.getsize(stored_status_file) > 0:
369 print "## Loading stored data hints from '%s'" % stored_status_file
370 stream = file(stored_status_file,'r')
371 stored_status = yaml.load(stream)
372 else:
373 print "[ERROR] '%s' does not exists" % stored_status_file
374
375 # Do a NMAP discovery
376 if opt_nmap_scan:
377 if not opt_store_scan:
378 nmap_result_file = None
379 nmap_status = do_nmap_scan(
380 "-p T:ssh,U:domain,T:80,T:ntp,U:snmp,T:8080 -sU -sT ",
381 iplist,nmap_result_file, opt_force_scan)
382
383 else:
384 nmap_status = stored_status['nmap_status']
385
386 # Save the MAC -> HOST mappings, by default as it helps indentifing the
387 # 'unknown links'
388 mac_to_host = {}
389 if stored_status:
390 mac_to_host = stored_status['mac_to_host']
391
392 # Do SNMP discovery
393 if opt_snmp_scan:
394 snmp_status = do_snmp_scan(iplist, nmap_status, stored_status, opt_force_snmp)
395 else:
396 snmp_status = stored_status
397
398 # Include our saved MAC -> HOST mappings
399 mac_to_host.update(snmp_status['mac_to_host'])
400 snmp_status['mac_to_host'] = mac_to_host
401
402 # Store changed data to disk
403 if opt_store_scan:
404 stream = file(stored_status_file,'w')
405 yaml.dump(snmp_status, stream, default_flow_style=False)
406 print "## Stored data hints to '%s'" % stored_status_file
407
408 # Finally generated status
409 generate_status(configs, snmp_status)
410 print "# Took %s seconds to complete" % (datetime.now() - start_time).seconds
411
412
413
414if __name__ == "__main__":
415 main()
Note: See TracBrowser for help on using the repository browser.