1 | #!/usr/bin/env python
|
---|
2 | # vim:ts=2:et:sw=2:ai
|
---|
3 | #
|
---|
4 | # Scan Wireless Leiden Network and report status of links and nodes
|
---|
5 | #
|
---|
6 | # Rick van der Zwet <info@rickvanderzwet.nl>
|
---|
7 |
|
---|
8 | from pprint import pprint
|
---|
9 | from xml.dom.minidom import parse, parseString
|
---|
10 | import gformat
|
---|
11 | import os.path
|
---|
12 | import re
|
---|
13 | import subprocess
|
---|
14 | import sys
|
---|
15 | import time
|
---|
16 | import yaml
|
---|
17 | from datetime import datetime
|
---|
18 |
|
---|
19 | # When force is used as argument, use this range
|
---|
20 | DEFAULT_SCAN_RANGE= ['172.16.0.0/12']
|
---|
21 |
|
---|
22 | # Default node status output
|
---|
23 | nodemap_status_file = '/tmp/nodemap_status.yaml'
|
---|
24 |
|
---|
25 | #
|
---|
26 | # BEGIN nmap XML parser
|
---|
27 | # XXX: Should properly go to seperate class/module
|
---|
28 | def get_attribute(node,attr):
|
---|
29 | return node.attributes[attr].value
|
---|
30 |
|
---|
31 | def attribute_from_node(parent,node,attr):
|
---|
32 | return parent.getElementsByTagName(node)[0].attributes[attr].value
|
---|
33 |
|
---|
34 | def parse_port(node):
|
---|
35 | item = dict()
|
---|
36 | item['protocol'] = get_attribute(node,'protocol')
|
---|
37 | item['portid'] = get_attribute(node,'portid')
|
---|
38 | item['state'] = attribute_from_node(node,'state','state')
|
---|
39 | item['reason'] = attribute_from_node(node,'state','reason')
|
---|
40 | return item
|
---|
41 |
|
---|
42 | def parse_ports(node):
|
---|
43 | item = dict()
|
---|
44 | for port in node.getElementsByTagName('port'):
|
---|
45 | port_item = parse_port(port)
|
---|
46 | item[port_item['portid']] = port_item
|
---|
47 | return item
|
---|
48 |
|
---|
49 | def parse_host(node):
|
---|
50 | # Host status
|
---|
51 | item = dict()
|
---|
52 | item['state'] = attribute_from_node(node,'status','state')
|
---|
53 | item['reason'] = attribute_from_node(node,'status','reason')
|
---|
54 | item['addr'] = attribute_from_node(node,'address','addr')
|
---|
55 | item['addrtype'] = attribute_from_node(node,'address','addrtype')
|
---|
56 |
|
---|
57 | # Service status
|
---|
58 | ports = node.getElementsByTagName('ports')
|
---|
59 | if ports:
|
---|
60 | item['port'] = parse_ports(ports[0])
|
---|
61 | return item
|
---|
62 |
|
---|
63 | def parse_nmap(root):
|
---|
64 | status = dict()
|
---|
65 | for node in root.childNodes[2].getElementsByTagName('host'):
|
---|
66 | scan = parse_host(node)
|
---|
67 | if not status.has_key(scan['addr']):
|
---|
68 | status[scan['addr']] = scan
|
---|
69 | return status
|
---|
70 | #
|
---|
71 | # END nmap parser
|
---|
72 | #
|
---|
73 |
|
---|
74 |
|
---|
75 |
|
---|
76 | def _do_nmap_scan(command, iphosts):
|
---|
77 | """ Run/Read nmap XML with various choices"""
|
---|
78 | command = "nmap -n -iL - -oX - %s" %(command)
|
---|
79 | print "# New run '%s', can take a while to complete" % (command)
|
---|
80 | p = subprocess.Popen(command.split(),
|
---|
81 | stdout=subprocess.PIPE,
|
---|
82 | stderr=subprocess.PIPE,
|
---|
83 | stdin=subprocess.PIPE, bufsize=-1)
|
---|
84 |
|
---|
85 | (stdoutdata, stderrdata) = p.communicate("\n".join(iphosts))
|
---|
86 | if p.returncode != 0:
|
---|
87 | print "# [ERROR] nmap failed to complete '%s'" % stderrdata
|
---|
88 | sys.exit(1)
|
---|
89 |
|
---|
90 | dom = parseString(stdoutdata)
|
---|
91 | return (parse_nmap(dom),stdoutdata)
|
---|
92 |
|
---|
93 |
|
---|
94 |
|
---|
95 | def do_nmap_scan(command, iphosts, result_file=None, forced_scan=False):
|
---|
96 | """ Wrapper around _run_nmap to get listing of all hosts, the default nmap
|
---|
97 | does not return results for failed hosts"""
|
---|
98 | # Get all hosts to be processed
|
---|
99 | (init_status, stdoutdata) = _do_nmap_scan(" -sL",iphosts)
|
---|
100 |
|
---|
101 | # Return stored file if exists
|
---|
102 | if not forced_scan and result_file and os.path.exists(result_file) \
|
---|
103 | and os.path.getsize(result_file) > 0:
|
---|
104 | print "# Reading stored NMAP results from '%s'" % (result_file)
|
---|
105 | status = parse_nmap(parse(result_file))
|
---|
106 | else:
|
---|
107 | # New scan
|
---|
108 | (status, stdoutdata) = _do_nmap_scan(command, iphosts)
|
---|
109 |
|
---|
110 | # Store result if requested
|
---|
111 | if result_file:
|
---|
112 | print "# Saving results in %s" % (result_file)
|
---|
113 | f = file(result_file,'w')
|
---|
114 | f.write(stdoutdata)
|
---|
115 | f.close()
|
---|
116 |
|
---|
117 | init_status.update(status)
|
---|
118 | return init_status
|
---|
119 |
|
---|
120 |
|
---|
121 |
|
---|
122 | def do_snmpwalk(host, oid):
|
---|
123 | """ Do snmpwalk, returns (p, stdout, stderr)"""
|
---|
124 | # GLobal SNMP walk options
|
---|
125 | snmpwalk = ('snmpwalk -r 0 -t 1 -OX -c public -v 2c %s' % host).split()
|
---|
126 | p = subprocess.Popen(snmpwalk + [oid],
|
---|
127 | stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
---|
128 | (stdoutdata, stderrdata) = p.communicate()
|
---|
129 | stdoutdata = stdoutdata.split('\n')[:-1]
|
---|
130 | stderrdata = stderrdata.split('\n')[:-1]
|
---|
131 | return (p, stdoutdata, stderrdata)
|
---|
132 |
|
---|
133 |
|
---|
134 |
|
---|
135 |
|
---|
136 | def do_snmp_scan(iphosts, status, stored_status=dict(), forced_scan=False):
|
---|
137 | """ SNMP scanning, based on results fould in NMAP scan"""
|
---|
138 | mac_to_host = dict()
|
---|
139 | host_processed = dict()
|
---|
140 |
|
---|
141 | #
|
---|
142 | # Gather SNMP data from hosts
|
---|
143 | for host in iphosts:
|
---|
144 | # Status might be containing old hosts as well and visa-versa
|
---|
145 | if not status.has_key(host):
|
---|
146 | print "## [ERROR] No nmap result found"
|
---|
147 | continue
|
---|
148 |
|
---|
149 | scan = status[host]
|
---|
150 | if scan['state'] != "up":
|
---|
151 | continue
|
---|
152 |
|
---|
153 | print '# Processing host %s' % host
|
---|
154 | # IP -> Mac addresses found in host ARP table, with key IP
|
---|
155 | status[host]['arpmac'] = dict()
|
---|
156 | # MAC -> iface addresses, with key MAC
|
---|
157 | status[host]['mac'] = dict()
|
---|
158 | # Mirrored: iface -> MAC addresses, with key interface name
|
---|
159 | status[host]['iface'] = dict()
|
---|
160 | try:
|
---|
161 | if not forced_scan and stored_status[host]['snmp_retval'] != 0:
|
---|
162 | print "## SNMP Connect failed last time, ignoring"
|
---|
163 | continue
|
---|
164 | except:
|
---|
165 | pass
|
---|
166 |
|
---|
167 | stored_status[host] = dict()
|
---|
168 | if not "open" in scan['port']['161']['state']:
|
---|
169 | print "## [ERROR] SNMP port not opened"
|
---|
170 | continue
|
---|
171 |
|
---|
172 | (p, output, stderrdata) = do_snmpwalk(host, 'SNMPv2-MIB::sysDescr')
|
---|
173 | stored_status[host]['snmp_retval'] = p.returncode
|
---|
174 | # Assume host remain reachable troughout all the SNMP queries
|
---|
175 | if p.returncode != 0:
|
---|
176 | print "## [ERROR] SNMP failed '%s'" % ",".join(stderrdata)
|
---|
177 | continue
|
---|
178 |
|
---|
179 | # Get some host details
|
---|
180 | # SNMPv2-MIB::sysDescr.0 = STRING: FreeBSD CNodeSOM1.wLeiden.NET
|
---|
181 | # 8.0-RELEASE-p2 FreeBSD 8.0-RELEASE-p2 #2: Fri Feb 19 18:24:23 CET 2010
|
---|
182 | # root@80fab2:/usr/obj/nanobsd.wleiden/usr/src/sys/kernel.wleiden i386
|
---|
183 | status[host]['sys_desc'] = output[0]
|
---|
184 | hostname = output[0].split(' ')[4]
|
---|
185 | release = output[0].split(' ')[5]
|
---|
186 | stored_status[host]['hostname'] = status[host]['hostname'] = hostname
|
---|
187 | stored_status[host]['release'] = status[host]['release'] = release
|
---|
188 | print "## %(hostname)s - %(release)s" % stored_status[host]
|
---|
189 |
|
---|
190 | # Check if the host is already done processing
|
---|
191 | # Note: the host is marked done processing at the end
|
---|
192 | if host_processed.has_key(hostname):
|
---|
193 | print "## Host already processed this run"
|
---|
194 | continue
|
---|
195 |
|
---|
196 | # Interface list with key the index number
|
---|
197 | iface_descr = dict()
|
---|
198 | # IF-MIB::ifDescr.1 = STRING: ath0
|
---|
199 | r = re.compile('^IF-MIB::ifDescr\[([0-9]+)\] = STRING: ([a-z0-9]+)$')
|
---|
200 | (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifDescr')
|
---|
201 | for line in output:
|
---|
202 | m = r.match(line)
|
---|
203 | iface_descr[m.group(1)] = m.group(2)
|
---|
204 |
|
---|
205 | # IF-MIB::ifPhysAddress[1] = STRING: 0:80:48:54:bb:52
|
---|
206 | r = re.compile('^IF-MIB::ifPhysAddress\[([0-9]+)\] = STRING: ([0-9a-f:]*)$')
|
---|
207 | (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifPhysAddress')
|
---|
208 | for line in output:
|
---|
209 | m = r.match(line)
|
---|
210 | # Ignore lines which has no MAC address
|
---|
211 | if not m.group(2): continue
|
---|
212 | index = m.group(1)
|
---|
213 | # Convert to proper MAC
|
---|
214 | mac = ":".join(["%02X" % int(x,16) for x in m.group(2).split(':')])
|
---|
215 | if not iface_descr.has_key(index):
|
---|
216 | print "## Index cannot be mapped to a key, available:"
|
---|
217 | for index, value in iface_descr.iteritems():
|
---|
218 | print "## - %s [%s]" % (value, index)
|
---|
219 | else:
|
---|
220 | print "## Local MAC %s [index:%s] -> %s" % (iface_descr[index], index, mac)
|
---|
221 | status[host]['mac'][mac] = iface_descr[index]
|
---|
222 | status[host]['iface'][iface_descr[index]] = mac
|
---|
223 | mac_to_host[mac] = hostname
|
---|
224 |
|
---|
225 | # Process host SNMP status
|
---|
226 | (p, output, stderrdata) = do_snmpwalk(host, 'RFC1213-MIB::atPhysAddress')
|
---|
227 | # RFC1213-MIB::atPhysAddress[8][1.172.21.160.34] = Hex-STRING: 00 17 C4 CC 5B F2
|
---|
228 | r = re.compile('^RFC1213-MIB::atPhysAddress\[[0-9]+\]\[1\.([0-9\.]+)\] = Hex-STRING: ([0-9A-F\ ]+)$')
|
---|
229 | for line in output:
|
---|
230 | m = r.match(line)
|
---|
231 | if not m:
|
---|
232 | print "## ERROR Unable to parse '%s'" % line
|
---|
233 | continue
|
---|
234 | ip = m.group(1)
|
---|
235 | # Replace spaces in MAC with :
|
---|
236 | mac = ":".join(m.group(2).split(' ')[:-1])
|
---|
237 | status[host]['arpmac'][ip] = mac
|
---|
238 |
|
---|
239 | local = '[remote]'
|
---|
240 | if mac in status[host]['mac'].keys():
|
---|
241 | local = '[local]'
|
---|
242 | print "## Arp table MAC %s -> %s %s" % (ip, mac, local)
|
---|
243 |
|
---|
244 | # Make sure we keep a record of the processed host which ip entry to check
|
---|
245 | host_processed[hostname] = host
|
---|
246 |
|
---|
247 | stored_status['host_processed'] = host_processed
|
---|
248 | stored_status['mac_to_host'] = mac_to_host
|
---|
249 | stored_status['nmap_status'] = status
|
---|
250 | return stored_status
|
---|
251 |
|
---|
252 |
|
---|
253 |
|
---|
254 |
|
---|
255 | def generate_status(configs, stored_status):
|
---|
256 | """ Generate result file from stored_status """
|
---|
257 | host_processed = stored_status['host_processed']
|
---|
258 | mac_to_host = stored_status['mac_to_host']
|
---|
259 | status = stored_status['nmap_status']
|
---|
260 |
|
---|
261 | # Data store format used for nodemap generation
|
---|
262 | nodemap = { 'node' : {}, 'link' : {}}
|
---|
263 |
|
---|
264 | # XXX: Pushed back till we actually store the MAC in the config files automatically
|
---|
265 | #configmac_to_host = dict()
|
---|
266 | #for host,config in configs.iteritems():
|
---|
267 | # for iface_key in gformat.get_interface_keys(config):
|
---|
268 | # configmac_to_host[config[iface_key]['mac']] = host
|
---|
269 |
|
---|
270 | # List of hosts which has some kind of problem
|
---|
271 | for host in configs.keys():
|
---|
272 | fqdn = host + ".wLeiden.NET"
|
---|
273 | if fqdn in host_processed.keys():
|
---|
274 | continue
|
---|
275 | config = configs[host]
|
---|
276 | print "# Problems in host '%s'" % host
|
---|
277 | host_down = True
|
---|
278 | for ip in gformat.get_used_ips([config]):
|
---|
279 | if not gformat.valid_addr(ip):
|
---|
280 | continue
|
---|
281 | if status[ip]['state'] == "up":
|
---|
282 | host_down = False
|
---|
283 | print "## - ", ip, status[ip]['state']
|
---|
284 | if host_down:
|
---|
285 | print "## HOST is DOWN!"
|
---|
286 | nodemap['node'][fqdn] = gformat.DOWN
|
---|
287 | else:
|
---|
288 | print "## SNMP problems (not reachable, deamon not running, etc)"
|
---|
289 | nodemap['node'][fqdn] = gformat.UNKNOWN
|
---|
290 |
|
---|
291 |
|
---|
292 |
|
---|
293 | # Correlation mapping
|
---|
294 | for fqdn, ip in host_processed.iteritems():
|
---|
295 | details = status[ip]
|
---|
296 | nodemap['node'][fqdn] = gformat.OK
|
---|
297 | print "# Working on %s" % fqdn
|
---|
298 | for ip, arpmac in details['arpmac'].iteritems():
|
---|
299 | if arpmac in details['mac'].keys():
|
---|
300 | # Local MAC address
|
---|
301 | continue
|
---|
302 | if not mac_to_host.has_key(arpmac):
|
---|
303 | print "## [WARN] No parent host for MAC %s (%s) found" % (arpmac, ip)
|
---|
304 | else:
|
---|
305 | print "## Interlink %s - %s" % (fqdn, mac_to_host[arpmac])
|
---|
306 | nodemap['link'][(fqdn,mac_to_host[arpmac])] = gformat.OK
|
---|
307 |
|
---|
308 | stream = file(nodemap_status_file,'w')
|
---|
309 | yaml.dump(nodemap, stream, default_flow_style=False)
|
---|
310 | print "# Wrote nodemap status to '%s'" % nodemap_status_file
|
---|
311 |
|
---|
312 |
|
---|
313 | def do_merge(files):
|
---|
314 | """ Merge all external statuses in our own nodestatus, using optimistic approch """
|
---|
315 | try:
|
---|
316 | stream = file(nodemap_status_file,'r')
|
---|
317 | status = yaml.load(stream)
|
---|
318 | except IOError, e:
|
---|
319 | # Data store format used for nodemap generation
|
---|
320 | status = { 'node' : {}, 'link' : {}}
|
---|
321 |
|
---|
322 | for cfile in files:
|
---|
323 | try:
|
---|
324 | print "# Merging '%s'" % cfile
|
---|
325 | stream = file(cfile,'r')
|
---|
326 | new_status = yaml.load(stream)
|
---|
327 | for item in ['node', 'link']:
|
---|
328 | for key, value in new_status[item].iteritems():
|
---|
329 | if not status[item].has_key(key):
|
---|
330 | # New items always welcome
|
---|
331 | status[item][key] = value
|
---|
332 | print "## [%s][%s] is new (%s)" % (item, key, value)
|
---|
333 | elif value < status[item][key]:
|
---|
334 | # Better values always welcome
|
---|
335 | status[item][key] = value
|
---|
336 | print "## [%s][%s] is better (%s)" % (item, key, value)
|
---|
337 | except IOError, e:
|
---|
338 | print "## ERROR '%s'" % e
|
---|
339 |
|
---|
340 | # Save results back to file
|
---|
341 | stream = file(nodemap_status_file,'w')
|
---|
342 | yaml.dump(status, stream, default_flow_style=False)
|
---|
343 |
|
---|
344 | def usage():
|
---|
345 | print "Usage: %s <arguments>"
|
---|
346 | print "Arguments:"
|
---|
347 | print "\tall = scan all known ips, using cached nmap"
|
---|
348 | print "\tnmap-only = scan all known ips, using nmap only"
|
---|
349 | print "\tsnmp-only = scan all known ips, using snmp only"
|
---|
350 | print "\tforce = scan all known ips, no cache used"
|
---|
351 | print "\tforced-snmp = scan all known ips, no snmp cache"
|
---|
352 | print "\tstored = generate status file using stored entries"
|
---|
353 | print "\thost <HOST1> [HOST2 ...] = generate status file using stored entries"
|
---|
354 | print "\tmerge <FILE1> [FILE2 ...] = merge status file with other status files"
|
---|
355 | sys.exit(0)
|
---|
356 |
|
---|
357 |
|
---|
358 | def main():
|
---|
359 | start_time = datetime.now()
|
---|
360 | stored_status_file = '/tmp/stored_status.yaml'
|
---|
361 | nmap_result_file = '/tmp/test.xml'
|
---|
362 |
|
---|
363 | stored_status = dict()
|
---|
364 | nmap_status = dict()
|
---|
365 | snmp_status = dict()
|
---|
366 |
|
---|
367 | opt_nmap_scan = True
|
---|
368 | opt_store_scan = True
|
---|
369 | opt_snmp_scan = True
|
---|
370 | opt_force_snmp = False
|
---|
371 | opt_force_scan = False
|
---|
372 | opt_force_range = False
|
---|
373 | if len(sys.argv) == 1:
|
---|
374 | usage()
|
---|
375 |
|
---|
376 | if sys.argv[1] == "all":
|
---|
377 | pass
|
---|
378 | elif sys.argv[1] == "nmap-only":
|
---|
379 | opt_snmp_scan = False
|
---|
380 | elif sys.argv[1] == "snmp-only":
|
---|
381 | opt_nmap_scan = False
|
---|
382 | elif sys.argv[1] == "force":
|
---|
383 | opt_force_scan = True
|
---|
384 | elif sys.argv[1] == "forced-snmp":
|
---|
385 | opt_nmap_scan = False
|
---|
386 | opt_force_snmp = True
|
---|
387 | elif sys.argv[1] == "host":
|
---|
388 | opt_force_range = True
|
---|
389 | opt_force_scan = True
|
---|
390 | elif sys.argv[1] == "stored":
|
---|
391 | opt_snmp_scan = False
|
---|
392 | opt_nmap_scan = False
|
---|
393 | opt_store_scan = False
|
---|
394 | elif sys.argv[1] == "merge":
|
---|
395 | do_merge(sys.argv[2:])
|
---|
396 | sys.exit(0)
|
---|
397 | else:
|
---|
398 | usage()
|
---|
399 |
|
---|
400 | # By default get all IPs defined in config, else own range
|
---|
401 | if not opt_force_range:
|
---|
402 | configs = gformat.get_all_configs()
|
---|
403 | iplist = gformat.get_used_ips(configs.values())
|
---|
404 | else:
|
---|
405 | iplist = sys.argv[1:]
|
---|
406 |
|
---|
407 | # Load data hints from previous run if exists
|
---|
408 | if not opt_force_scan and os.path.exists(stored_status_file) and os.path.getsize(stored_status_file) > 0:
|
---|
409 | print "## Loading stored data hints from '%s'" % stored_status_file
|
---|
410 | stream = file(stored_status_file,'r')
|
---|
411 | stored_status = yaml.load(stream)
|
---|
412 | else:
|
---|
413 | print "[ERROR] '%s' does not exists" % stored_status_file
|
---|
414 |
|
---|
415 | # Do a NMAP discovery
|
---|
416 | if opt_nmap_scan:
|
---|
417 | if not opt_store_scan:
|
---|
418 | nmap_result_file = None
|
---|
419 | nmap_status = do_nmap_scan(
|
---|
420 | "-p T:ssh,U:domain,T:80,T:ntp,U:snmp,T:8080 -sU -sT ",
|
---|
421 | iplist,nmap_result_file, opt_force_scan)
|
---|
422 |
|
---|
423 | else:
|
---|
424 | nmap_status = stored_status['nmap_status']
|
---|
425 |
|
---|
426 | # Save the MAC -> HOST mappings, by default as it helps indentifing the
|
---|
427 | # 'unknown links'
|
---|
428 | mac_to_host = {}
|
---|
429 | if stored_status:
|
---|
430 | mac_to_host = stored_status['mac_to_host']
|
---|
431 |
|
---|
432 | # Do SNMP discovery
|
---|
433 | if opt_snmp_scan:
|
---|
434 | snmp_status = do_snmp_scan(iplist, nmap_status, stored_status, opt_force_snmp)
|
---|
435 | else:
|
---|
436 | snmp_status = stored_status
|
---|
437 |
|
---|
438 | # Include our saved MAC -> HOST mappings
|
---|
439 | mac_to_host.update(snmp_status['mac_to_host'])
|
---|
440 | snmp_status['mac_to_host'] = mac_to_host
|
---|
441 |
|
---|
442 | # Store changed data to disk
|
---|
443 | if opt_store_scan:
|
---|
444 | stream = file(stored_status_file,'w')
|
---|
445 | yaml.dump(snmp_status, stream, default_flow_style=False)
|
---|
446 | print "## Stored data hints to '%s'" % stored_status_file
|
---|
447 |
|
---|
448 | # Finally generated status
|
---|
449 | generate_status(configs, snmp_status)
|
---|
450 | print "# Took %s seconds to complete" % (datetime.now() - start_time).seconds
|
---|
451 |
|
---|
452 |
|
---|
453 |
|
---|
454 | if __name__ == "__main__":
|
---|
455 | main()
|
---|