| 1 | #!/usr/bin/env python
|
|---|
| 2 | # vim:ts=2:et:sw=2:ai
|
|---|
| 3 | #
|
|---|
| 4 | # Scan Wireless Leiden Network and report status of links and nodes
|
|---|
| 5 | #
|
|---|
| 6 | # Rick van der Zwet <info@rickvanderzwet.nl>
|
|---|
| 7 |
|
|---|
| 8 | from pprint import pprint
|
|---|
| 9 | from xml.dom.minidom import parse, parseString
|
|---|
| 10 | import gformat
|
|---|
| 11 | import os.path
|
|---|
| 12 | import re
|
|---|
| 13 | import subprocess
|
|---|
| 14 | import sys
|
|---|
| 15 | import time
|
|---|
| 16 | import yaml
|
|---|
| 17 | from datetime import datetime
|
|---|
| 18 |
|
|---|
| 19 | # When force is used as argument, use this range
|
|---|
| 20 | DEFAULT_SCAN_RANGE= ['172.16.0.0/21']
|
|---|
| 21 |
|
|---|
| 22 | #
|
|---|
| 23 | # BEGIN nmap XML parser
|
|---|
| 24 | # XXX: Should properly go to seperate class/module
|
|---|
| 25 | def get_attribute(node,attr):
|
|---|
| 26 | return node.attributes[attr].value
|
|---|
| 27 |
|
|---|
| 28 | def attribute_from_node(parent,node,attr):
|
|---|
| 29 | return parent.getElementsByTagName(node)[0].attributes[attr].value
|
|---|
| 30 |
|
|---|
| 31 | def parse_port(node):
|
|---|
| 32 | item = dict()
|
|---|
| 33 | item['protocol'] = get_attribute(node,'protocol')
|
|---|
| 34 | item['portid'] = get_attribute(node,'portid')
|
|---|
| 35 | item['state'] = attribute_from_node(node,'state','state')
|
|---|
| 36 | item['reason'] = attribute_from_node(node,'state','reason')
|
|---|
| 37 | return item
|
|---|
| 38 |
|
|---|
| 39 | def parse_ports(node):
|
|---|
| 40 | item = dict()
|
|---|
| 41 | for port in node.getElementsByTagName('port'):
|
|---|
| 42 | port_item = parse_port(port)
|
|---|
| 43 | item[port_item['portid']] = port_item
|
|---|
| 44 | return item
|
|---|
| 45 |
|
|---|
| 46 | def parse_host(node):
|
|---|
| 47 | # Host status
|
|---|
| 48 | item = dict()
|
|---|
| 49 | item['state'] = attribute_from_node(node,'status','state')
|
|---|
| 50 | item['reason'] = attribute_from_node(node,'status','reason')
|
|---|
| 51 | item['addr'] = attribute_from_node(node,'address','addr')
|
|---|
| 52 | item['addrtype'] = attribute_from_node(node,'address','addrtype')
|
|---|
| 53 |
|
|---|
| 54 | # Service status
|
|---|
| 55 | ports = node.getElementsByTagName('ports')
|
|---|
| 56 | if ports:
|
|---|
| 57 | item['port'] = parse_ports(ports[0])
|
|---|
| 58 | return item
|
|---|
| 59 |
|
|---|
| 60 | def parse_nmap(root):
|
|---|
| 61 | status = dict()
|
|---|
| 62 | for node in root.childNodes[2].getElementsByTagName('host'):
|
|---|
| 63 | scan = parse_host(node)
|
|---|
| 64 | if not status.has_key(scan['addr']):
|
|---|
| 65 | status[scan['addr']] = scan
|
|---|
| 66 | return status
|
|---|
| 67 | #
|
|---|
| 68 | # END nmap parser
|
|---|
| 69 | #
|
|---|
| 70 |
|
|---|
| 71 |
|
|---|
| 72 |
|
|---|
| 73 | def _do_nmap_scan(command, iphosts):
|
|---|
| 74 | """ Run/Read nmap XML with various choices"""
|
|---|
| 75 | command = "nmap -n -iL - -oX - %s" %(command)
|
|---|
| 76 | print "# New run '%s', can take a while to complete" % (command)
|
|---|
| 77 | p = subprocess.Popen(command.split(),
|
|---|
| 78 | stdout=subprocess.PIPE,
|
|---|
| 79 | stderr=subprocess.PIPE,
|
|---|
| 80 | stdin=subprocess.PIPE, bufsize=-1)
|
|---|
| 81 |
|
|---|
| 82 | (stdoutdata, stderrdata) = p.communicate("\n".join(iphosts))
|
|---|
| 83 | if p.returncode != 0:
|
|---|
| 84 | print "# [ERROR] nmap failed to complete '%s'" % stderrdata
|
|---|
| 85 | sys.exit(1)
|
|---|
| 86 |
|
|---|
| 87 | dom = parseString(stdoutdata)
|
|---|
| 88 | return (parse_nmap(dom),stdoutdata)
|
|---|
| 89 |
|
|---|
| 90 |
|
|---|
| 91 |
|
|---|
| 92 | def do_nmap_scan(command, iphosts, result_file=None):
|
|---|
| 93 | """ Wrapper around _run_nmap to get listing of all hosts, the default nmap
|
|---|
| 94 | does not return results for failed hosts"""
|
|---|
| 95 | # Get all hosts to be processed
|
|---|
| 96 | (init_status, stdoutdata) = _do_nmap_scan(" -sL",iphosts)
|
|---|
| 97 |
|
|---|
| 98 | # Return stored file if exists
|
|---|
| 99 | if result_file and os.path.exists(result_file) and os.path.getsize(result_file) > 0:
|
|---|
| 100 | print "# Reading stored NMAP results from '%s'" % (result_file)
|
|---|
| 101 | status = parse_nmap(parse(result_file))
|
|---|
| 102 | else:
|
|---|
| 103 | # New scan
|
|---|
| 104 | (status, stdoutdata) = _do_nmap_scan(command, iphosts)
|
|---|
| 105 |
|
|---|
| 106 | # Store result if requested
|
|---|
| 107 | if result_file:
|
|---|
| 108 | print "# Saving results in %s" % (result_file)
|
|---|
| 109 | f = file(result_file,'w')
|
|---|
| 110 | f.write(stdoutdata)
|
|---|
| 111 | f.close()
|
|---|
| 112 |
|
|---|
| 113 | init_status.update(status)
|
|---|
| 114 | return init_status
|
|---|
| 115 |
|
|---|
| 116 |
|
|---|
| 117 |
|
|---|
| 118 | def do_snmpwalk(host, oid):
|
|---|
| 119 | """ Do snmpwalk, returns (p, stdout, stderr)"""
|
|---|
| 120 | # GLobal SNMP walk options
|
|---|
| 121 | snmpwalk = ('snmpwalk -r 0 -t 1 -OX -c public -v 2c %s' % host).split()
|
|---|
| 122 | p = subprocess.Popen(snmpwalk + [oid],
|
|---|
| 123 | stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|---|
| 124 | (stdoutdata, stderrdata) = p.communicate()
|
|---|
| 125 | stdoutdata = stdoutdata.split('\n')[:-1]
|
|---|
| 126 | stderrdata = stderrdata.split('\n')[:-1]
|
|---|
| 127 | return (p, stdoutdata, stderrdata)
|
|---|
| 128 |
|
|---|
| 129 |
|
|---|
| 130 |
|
|---|
| 131 |
|
|---|
| 132 | def do_snmp_scan(iphosts, status, stored_status=dict()):
|
|---|
| 133 | """ SNMP scanning, based on results fould in NMAP scan"""
|
|---|
| 134 | mac_to_host = dict()
|
|---|
| 135 | host_processed = dict()
|
|---|
| 136 |
|
|---|
| 137 | #
|
|---|
| 138 | # Gather SNMP data from hosts
|
|---|
| 139 | for host, scan in status.iteritems():
|
|---|
| 140 | if scan['state'] != "up":
|
|---|
| 141 | continue
|
|---|
| 142 |
|
|---|
| 143 | # Filter set? use it
|
|---|
| 144 | if iphosts and not host in iphosts:
|
|---|
| 145 | print "## IP '%s' not in specified filter" % host
|
|---|
| 146 | continue
|
|---|
| 147 |
|
|---|
| 148 | print '# Processing host %s' % host
|
|---|
| 149 | # IP -> Mac addresses found in host ARP table, with key IP
|
|---|
| 150 | status[host]['arpmac'] = dict()
|
|---|
| 151 | # MAC -> iface addresses, with key MAC
|
|---|
| 152 | status[host]['mac'] = dict()
|
|---|
| 153 | # Mirrored: iface -> MAC addresses, with key interface name
|
|---|
| 154 | status[host]['iface'] = dict()
|
|---|
| 155 | try:
|
|---|
| 156 | if stored_status[host]['snmp_retval'] != 0:
|
|---|
| 157 | print "## SNMP Connect failed last time, ignoring"
|
|---|
| 158 | continue
|
|---|
| 159 | except:
|
|---|
| 160 | pass
|
|---|
| 161 |
|
|---|
| 162 | stored_status[host] = dict()
|
|---|
| 163 | if not "open" in scan['port']['161']['state']:
|
|---|
| 164 | print "## [ERROR] SNMP port not opened"
|
|---|
| 165 | continue
|
|---|
| 166 |
|
|---|
| 167 | (p, output, stderrdata) = do_snmpwalk(host, 'SNMPv2-MIB::sysDescr')
|
|---|
| 168 | stored_status[host]['snmp_retval'] = p.returncode
|
|---|
| 169 | # Assume host remain reachable troughout all the SNMP queries
|
|---|
| 170 | if p.returncode != 0:
|
|---|
| 171 | print "## [ERROR] SNMP failed '%s'" % ",".join(stderrdata)
|
|---|
| 172 | continue
|
|---|
| 173 |
|
|---|
| 174 | # Get some host details
|
|---|
| 175 | # SNMPv2-MIB::sysDescr.0 = STRING: FreeBSD CNodeSOM1.wLeiden.NET
|
|---|
| 176 | # 8.0-RELEASE-p2 FreeBSD 8.0-RELEASE-p2 #2: Fri Feb 19 18:24:23 CET 2010
|
|---|
| 177 | # root@80fab2:/usr/obj/nanobsd.wleiden/usr/src/sys/kernel.wleiden i386
|
|---|
| 178 | status[host]['sys_desc'] = output[0]
|
|---|
| 179 | hostname = output[0].split(' ')[4]
|
|---|
| 180 | release = output[0].split(' ')[5]
|
|---|
| 181 | stored_status[host]['hostname'] = status[host]['hostname'] = hostname
|
|---|
| 182 | stored_status[host]['release'] = status[host]['release'] = release
|
|---|
| 183 | print "## %(hostname)s - %(release)s" % stored_status[host]
|
|---|
| 184 |
|
|---|
| 185 | # Check if the host is already done processing
|
|---|
| 186 | # Note: the host is marked done processing at the end
|
|---|
| 187 | if host_processed.has_key(hostname):
|
|---|
| 188 | print "## Host already processed this run"
|
|---|
| 189 | continue
|
|---|
| 190 |
|
|---|
| 191 | # Interface list with key the index number
|
|---|
| 192 | iface_descr = dict()
|
|---|
| 193 | # IF-MIB::ifDescr.1 = STRING: ath0
|
|---|
| 194 | r = re.compile('^IF-MIB::ifDescr\[([0-9]+)\] = STRING: ([a-z0-9]+)$')
|
|---|
| 195 | (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifDescr')
|
|---|
| 196 | for line in output:
|
|---|
| 197 | m = r.match(line)
|
|---|
| 198 | iface_descr[m.group(1)] = m.group(2)
|
|---|
| 199 |
|
|---|
| 200 | # IF-MIB::ifPhysAddress[1] = STRING: 0:80:48:54:bb:52
|
|---|
| 201 | r = re.compile('^IF-MIB::ifPhysAddress\[([0-9]+)\] = STRING: ([0-9a-f:]*)$')
|
|---|
| 202 | (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifPhysAddress')
|
|---|
| 203 | for line in output:
|
|---|
| 204 | m = r.match(line)
|
|---|
| 205 | # Ignore lines which has no MAC address
|
|---|
| 206 | if not m.group(2): continue
|
|---|
| 207 | index = m.group(1)
|
|---|
| 208 | # Convert to proper MAC
|
|---|
| 209 | mac = ":".join(["%02X" % int(x,16) for x in m.group(2).split(':')])
|
|---|
| 210 | print "## Local MAC %s [index:%s] -> %s" % (iface_descr[index], index, mac)
|
|---|
| 211 | status[host]['mac'][mac] = iface_descr[index]
|
|---|
| 212 | status[host]['iface'][iface_descr[index]] = mac
|
|---|
| 213 | mac_to_host[mac] = hostname
|
|---|
| 214 |
|
|---|
| 215 | # Process host SNMP status
|
|---|
| 216 | (p, output, stderrdata) = do_snmpwalk(host, 'RFC1213-MIB::atPhysAddress')
|
|---|
| 217 | # RFC1213-MIB::atPhysAddress[8][1.172.21.160.34] = Hex-STRING: 00 17 C4 CC 5B F2
|
|---|
| 218 | r = re.compile('^RFC1213-MIB::atPhysAddress\[[0-9]+\]\[1\.([0-9\.]+)\] = Hex-STRING: ([0-9A-F\ ]+)$')
|
|---|
| 219 | for line in output:
|
|---|
| 220 | m = r.match(line)
|
|---|
| 221 | ip = m.group(1)
|
|---|
| 222 | # Replace spaces in MAC with :
|
|---|
| 223 | mac = ":".join(m.group(2).split(' ')[:-1])
|
|---|
| 224 | status[host]['arpmac'][ip] = mac
|
|---|
| 225 |
|
|---|
| 226 | local = '[remote]'
|
|---|
| 227 | if mac in status[host]['mac'].keys():
|
|---|
| 228 | local = '[local]'
|
|---|
| 229 | print "## Arp table MAC %s -> %s %s" % (ip, mac, local)
|
|---|
| 230 |
|
|---|
| 231 | # Make sure we keep a record of the processed host
|
|---|
| 232 | host_processed[hostname] = status[host]
|
|---|
| 233 |
|
|---|
| 234 | stored_status['host_processed'] = host_processed
|
|---|
| 235 | stored_status['mac_to_host'] = mac_to_host
|
|---|
| 236 | stored_status['nmap_status'] = status
|
|---|
| 237 | return stored_status
|
|---|
| 238 |
|
|---|
| 239 |
|
|---|
| 240 |
|
|---|
| 241 |
|
|---|
| 242 | def generate_status(configs, stored_status):
|
|---|
| 243 | """ Generate result file from stored_status """
|
|---|
| 244 | host_processed = stored_status['host_processed']
|
|---|
| 245 | mac_to_host = stored_status['mac_to_host']
|
|---|
| 246 | status = stored_status['nmap_status']
|
|---|
| 247 |
|
|---|
| 248 | # Data store format used for nodemap generation
|
|---|
| 249 | nodemap_status_file = '/tmp/nodemap_status.yaml'
|
|---|
| 250 | nodemap = { 'node' : {}, 'link' : {}}
|
|---|
| 251 |
|
|---|
| 252 | # XXX: Pushed back till we actually store the MAC in the config files automatically
|
|---|
| 253 | #configmac_to_host = dict()
|
|---|
| 254 | #for host,config in configs.iteritems():
|
|---|
| 255 | # for iface_key in gformat.get_interface_keys(config):
|
|---|
| 256 | # configmac_to_host[config[iface_key]['mac']] = host
|
|---|
| 257 |
|
|---|
| 258 | # List of hosts which has some kind of problem
|
|---|
| 259 | print host_processed.keys()
|
|---|
| 260 | print configs.keys()
|
|---|
| 261 | for host in configs.keys():
|
|---|
| 262 | fqdn = host + ".wLeiden.NET"
|
|---|
| 263 | if fqdn in host_processed.keys():
|
|---|
| 264 | continue
|
|---|
| 265 | config = configs[host]
|
|---|
| 266 | print "# Problems in host '%s'" % host
|
|---|
| 267 | host_down = True
|
|---|
| 268 | for ip in gformat.get_used_ips([config]):
|
|---|
| 269 | if not gformat.valid_addr(ip):
|
|---|
| 270 | continue
|
|---|
| 271 | if status[ip]['state'] == "up":
|
|---|
| 272 | host_down = False
|
|---|
| 273 | print "## - ", ip, status[ip]['state']
|
|---|
| 274 | if host_down:
|
|---|
| 275 | print "## HOST is DOWN!"
|
|---|
| 276 | nodemap['node'][fqdn] = gformat.DOWN
|
|---|
| 277 | else:
|
|---|
| 278 | print "## SNMP problems (not reachable, deamon not running, etc)"
|
|---|
| 279 | nodemap['node'][fqdn] = gformat.UNKNOWN
|
|---|
| 280 |
|
|---|
| 281 |
|
|---|
| 282 |
|
|---|
| 283 | # Correlation mapping
|
|---|
| 284 | for fqdn, details in host_processed.iteritems():
|
|---|
| 285 | nodemap['node'][fqdn] = gformat.OK
|
|---|
| 286 | print "# Working on %s" % fqdn
|
|---|
| 287 | for ip, arpmac in details['arpmac'].iteritems():
|
|---|
| 288 | if arpmac in details['mac'].keys():
|
|---|
| 289 | # Local MAC address
|
|---|
| 290 | continue
|
|---|
| 291 | if not mac_to_host.has_key(arpmac):
|
|---|
| 292 | print "## [WARN] No parent host for MAC %s (%s) found" % (arpmac, ip)
|
|---|
| 293 | else:
|
|---|
| 294 | print "## Interlink %s - %s" % (fqdn, mac_to_host[arpmac])
|
|---|
| 295 | nodemap['link'][(fqdn,mac_to_host[arpmac])] = gformat.OK
|
|---|
| 296 |
|
|---|
| 297 | stream = file(nodemap_status_file,'w')
|
|---|
| 298 | yaml.dump(nodemap, stream, default_flow_style=False)
|
|---|
| 299 | print "# Wrote nodemap status to '%s'" % nodemap_status_file
|
|---|
| 300 |
|
|---|
| 301 | def usage():
|
|---|
| 302 | print "Usage: %s <all|force|stored|host HOST1 [HOST2 ...]>" % sys.argv[0]
|
|---|
| 303 | sys.exit(0)
|
|---|
| 304 |
|
|---|
| 305 |
|
|---|
| 306 | def main():
|
|---|
| 307 | start_time = datetime.now()
|
|---|
| 308 | stored_status_file = '/tmp/stored_status.yaml'
|
|---|
| 309 | nmap_result_file = '/tmp/test.xml'
|
|---|
| 310 |
|
|---|
| 311 | stored_status = dict()
|
|---|
| 312 | nmap_status = dict()
|
|---|
| 313 | snmp_status = dict()
|
|---|
| 314 |
|
|---|
| 315 | opt_nmap_scan = True
|
|---|
| 316 | opt_store_scan = True
|
|---|
| 317 | opt_snmp_scan = True
|
|---|
| 318 | opt_force_snmp = False
|
|---|
| 319 | opt_force_scan = False
|
|---|
| 320 | opt_force_range = False
|
|---|
| 321 | if len(sys.argv) == 1:
|
|---|
| 322 | usage()
|
|---|
| 323 |
|
|---|
| 324 | if sys.argv[1] == "all":
|
|---|
| 325 | pass
|
|---|
| 326 | elif sys.argv[1] == "nmap-only":
|
|---|
| 327 | opt_snmp_scan = False
|
|---|
| 328 | elif sys.argv[1] == "snmp-only":
|
|---|
| 329 | opt_nmap_scan = False
|
|---|
| 330 | elif sys.argv[1] == "force":
|
|---|
| 331 | opt_force_scan = True
|
|---|
| 332 | elif sys.argv[1] == "forced-snmp":
|
|---|
| 333 | opt_nmap_scan = False
|
|---|
| 334 | opt_force_snmp = True
|
|---|
| 335 | elif sys.argv[1] == "host":
|
|---|
| 336 | opt_force_range = True
|
|---|
| 337 | opt_force_scan = True
|
|---|
| 338 | elif sys.argv[1] == "stored":
|
|---|
| 339 | opt_snmp_scan = False
|
|---|
| 340 | opt_nmap_scan = False
|
|---|
| 341 | opt_store_scan = False
|
|---|
| 342 | else:
|
|---|
| 343 | usage()
|
|---|
| 344 |
|
|---|
| 345 | # By default get all IPs defined in config, else own range
|
|---|
| 346 | if not opt_force_range:
|
|---|
| 347 | configs = gformat.get_all_configs()
|
|---|
| 348 | iplist = gformat.get_used_ips(configs.values())
|
|---|
| 349 | else:
|
|---|
| 350 | iplist = sys.argv[1:]
|
|---|
| 351 |
|
|---|
| 352 | # Load data hints from previous run if exists
|
|---|
| 353 | if not opt_force_scan and os.path.exists(stored_status_file) and os.path.getsize(stored_status_file) > 0:
|
|---|
| 354 | print "## Loading stored data hints from '%s'" % stored_status_file
|
|---|
| 355 | stream = file(stored_status_file,'r')
|
|---|
| 356 | stored_status = yaml.load(stream)
|
|---|
| 357 | else:
|
|---|
| 358 | print "[ERROR] '%s' does not exists" % stored_status_file
|
|---|
| 359 |
|
|---|
| 360 | # Do a NMAP discovery
|
|---|
| 361 | if opt_nmap_scan:
|
|---|
| 362 | if not opt_store_scan:
|
|---|
| 363 | nmap_result_file = None
|
|---|
| 364 | nmap_status = do_nmap_scan("-p T:ssh,U:domain,T:80,T:ntp,U:snmp,T:8080 -sU -sT ",iplist, nmap_result_file)
|
|---|
| 365 | else:
|
|---|
| 366 | nmap_status = stored_status['nmap_status']
|
|---|
| 367 |
|
|---|
| 368 | # Save the MAC -> HOST mappings, by default as it helps indentifing the
|
|---|
| 369 | # 'unknown links'
|
|---|
| 370 | mac_to_host = {}
|
|---|
| 371 | if stored_status:
|
|---|
| 372 | mac_to_host = stored_status['mac_to_host']
|
|---|
| 373 |
|
|---|
| 374 | # Do SNMP discovery
|
|---|
| 375 | if opt_snmp_scan:
|
|---|
| 376 | snmp_status = do_snmp_scan(iplist, nmap_status, stored_status)
|
|---|
| 377 | else:
|
|---|
| 378 | snmp_status = stored_status
|
|---|
| 379 |
|
|---|
| 380 | # Include our saved MAC -> HOST mappings
|
|---|
| 381 | mac_to_host.update(snmp_status['mac_to_host'])
|
|---|
| 382 | snmp_status['mac_to_host'] = mac_to_host
|
|---|
| 383 |
|
|---|
| 384 | # Store changed data to disk
|
|---|
| 385 | if opt_store_scan:
|
|---|
| 386 | stream = file(stored_status_file,'w')
|
|---|
| 387 | yaml.dump(snmp_status, stream, default_flow_style=False)
|
|---|
| 388 | print "## Stored data hints to '%s'" % stored_status_file
|
|---|
| 389 |
|
|---|
| 390 | # Finally generated status
|
|---|
| 391 | generate_status(configs, snmp_status)
|
|---|
| 392 | print "# Took %s seconds to complete" % (datetime.now() - start_time).seconds
|
|---|
| 393 |
|
|---|
| 394 |
|
|---|
| 395 |
|
|---|
| 396 | if __name__ == "__main__":
|
|---|
| 397 | main()
|
|---|
