| 1 | #!/usr/bin/env python
|
|---|
| 2 | # vim:ts=2:et:sw=2:ai
|
|---|
| 3 | #
|
|---|
| 4 | # Scan Wireless Leiden Network and report status of links and nodes
|
|---|
| 5 | #
|
|---|
| 6 | # Rick van der Zwet <info@rickvanderzwet.nl>
|
|---|
| 7 |
|
|---|
| 8 | from pprint import pprint
|
|---|
| 9 | from xml.dom.minidom import parse, parseString
|
|---|
| 10 | import gformat
|
|---|
| 11 | import os.path
|
|---|
| 12 | import re
|
|---|
| 13 | import subprocess
|
|---|
| 14 | import sys
|
|---|
| 15 | import time
|
|---|
| 16 | import yaml
|
|---|
| 17 | from datetime import datetime
|
|---|
| 18 |
|
|---|
| 19 | # When force is used as argument, use this range
|
|---|
| 20 | DEFAULT_SCAN_RANGE= ['172.16.0.0/21']
|
|---|
| 21 |
|
|---|
| 22 | #
|
|---|
| 23 | # BEGIN nmap XML parser
|
|---|
| 24 | # XXX: Should properly go to seperate class/module
|
|---|
| 25 | def get_attribute(node,attr):
|
|---|
| 26 | return node.attributes[attr].value
|
|---|
| 27 |
|
|---|
| 28 | def attribute_from_node(parent,node,attr):
|
|---|
| 29 | return parent.getElementsByTagName(node)[0].attributes[attr].value
|
|---|
| 30 |
|
|---|
| 31 | def parse_port(node):
|
|---|
| 32 | item = dict()
|
|---|
| 33 | item['protocol'] = get_attribute(node,'protocol')
|
|---|
| 34 | item['portid'] = get_attribute(node,'portid')
|
|---|
| 35 | item['state'] = attribute_from_node(node,'state','state')
|
|---|
| 36 | item['reason'] = attribute_from_node(node,'state','reason')
|
|---|
| 37 | return item
|
|---|
| 38 |
|
|---|
| 39 | def parse_ports(node):
|
|---|
| 40 | item = dict()
|
|---|
| 41 | for port in node.getElementsByTagName('port'):
|
|---|
| 42 | port_item = parse_port(port)
|
|---|
| 43 | item[port_item['portid']] = port_item
|
|---|
| 44 | return item
|
|---|
| 45 |
|
|---|
| 46 | def parse_host(node):
|
|---|
| 47 | # Host status
|
|---|
| 48 | item = dict()
|
|---|
| 49 | item['state'] = attribute_from_node(node,'status','state')
|
|---|
| 50 | item['reason'] = attribute_from_node(node,'status','reason')
|
|---|
| 51 | item['addr'] = attribute_from_node(node,'address','addr')
|
|---|
| 52 | item['addrtype'] = attribute_from_node(node,'address','addrtype')
|
|---|
| 53 |
|
|---|
| 54 | # Service status
|
|---|
| 55 | ports = node.getElementsByTagName('ports')
|
|---|
| 56 | if ports:
|
|---|
| 57 | item['port'] = parse_ports(ports[0])
|
|---|
| 58 | return item
|
|---|
| 59 |
|
|---|
| 60 | def parse_nmap(root):
|
|---|
| 61 | status = dict()
|
|---|
| 62 | for node in root.childNodes[2].getElementsByTagName('host'):
|
|---|
| 63 | scan = parse_host(node)
|
|---|
| 64 | if not status.has_key(scan['addr']):
|
|---|
| 65 | status[scan['addr']] = scan
|
|---|
| 66 | return status
|
|---|
| 67 | #
|
|---|
| 68 | # END nmap parser
|
|---|
| 69 | #
|
|---|
| 70 |
|
|---|
| 71 |
|
|---|
| 72 |
|
|---|
| 73 | def _do_nmap_scan(command, iphosts):
|
|---|
| 74 | """ Run/Read nmap XML with various choices"""
|
|---|
| 75 | command = "nmap -n -iL - -oX - %s" %(command)
|
|---|
| 76 | print "# New run '%s', can take a while to complete" % (command)
|
|---|
| 77 | p = subprocess.Popen(command.split(),
|
|---|
| 78 | stdout=subprocess.PIPE,
|
|---|
| 79 | stderr=subprocess.PIPE,
|
|---|
| 80 | stdin=subprocess.PIPE, bufsize=-1)
|
|---|
| 81 |
|
|---|
| 82 | (stdoutdata, stderrdata) = p.communicate("\n".join(iphosts))
|
|---|
| 83 | if p.returncode != 0:
|
|---|
| 84 | print "# [ERROR] nmap failed to complete '%s'" % stderrdata
|
|---|
| 85 | sys.exit(1)
|
|---|
| 86 |
|
|---|
| 87 | dom = parseString(stdoutdata)
|
|---|
| 88 | return (parse_nmap(dom),stdoutdata)
|
|---|
| 89 |
|
|---|
| 90 |
|
|---|
| 91 |
|
|---|
| 92 | def do_nmap_scan(command, iphosts, result_file=None, forced_scan=False):
|
|---|
| 93 | """ Wrapper around _run_nmap to get listing of all hosts, the default nmap
|
|---|
| 94 | does not return results for failed hosts"""
|
|---|
| 95 | # Get all hosts to be processed
|
|---|
| 96 | (init_status, stdoutdata) = _do_nmap_scan(" -sL",iphosts)
|
|---|
| 97 |
|
|---|
| 98 | # Return stored file if exists
|
|---|
| 99 | if not forced_scan and result_file and os.path.exists(result_file) \
|
|---|
| 100 | and os.path.getsize(result_file) > 0:
|
|---|
| 101 | print "# Reading stored NMAP results from '%s'" % (result_file)
|
|---|
| 102 | status = parse_nmap(parse(result_file))
|
|---|
| 103 | else:
|
|---|
| 104 | # New scan
|
|---|
| 105 | (status, stdoutdata) = _do_nmap_scan(command, iphosts)
|
|---|
| 106 |
|
|---|
| 107 | # Store result if requested
|
|---|
| 108 | if result_file:
|
|---|
| 109 | print "# Saving results in %s" % (result_file)
|
|---|
| 110 | f = file(result_file,'w')
|
|---|
| 111 | f.write(stdoutdata)
|
|---|
| 112 | f.close()
|
|---|
| 113 |
|
|---|
| 114 | init_status.update(status)
|
|---|
| 115 | return init_status
|
|---|
| 116 |
|
|---|
| 117 |
|
|---|
| 118 |
|
|---|
| 119 | def do_snmpwalk(host, oid):
|
|---|
| 120 | """ Do snmpwalk, returns (p, stdout, stderr)"""
|
|---|
| 121 | # GLobal SNMP walk options
|
|---|
| 122 | snmpwalk = ('snmpwalk -r 0 -t 1 -OX -c public -v 2c %s' % host).split()
|
|---|
| 123 | p = subprocess.Popen(snmpwalk + [oid],
|
|---|
| 124 | stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|---|
| 125 | (stdoutdata, stderrdata) = p.communicate()
|
|---|
| 126 | stdoutdata = stdoutdata.split('\n')[:-1]
|
|---|
| 127 | stderrdata = stderrdata.split('\n')[:-1]
|
|---|
| 128 | return (p, stdoutdata, stderrdata)
|
|---|
| 129 |
|
|---|
| 130 |
|
|---|
| 131 |
|
|---|
| 132 |
|
|---|
| 133 | def do_snmp_scan(iphosts, status, stored_status=dict(), forced_scan=False):
|
|---|
| 134 | """ SNMP scanning, based on results fould in NMAP scan"""
|
|---|
| 135 | mac_to_host = dict()
|
|---|
| 136 | host_processed = dict()
|
|---|
| 137 |
|
|---|
| 138 | #
|
|---|
| 139 | # Gather SNMP data from hosts
|
|---|
| 140 | for host in iphosts:
|
|---|
| 141 | # Status might be containing old hosts as well and visa-versa
|
|---|
| 142 | if not status.has_key(host):
|
|---|
| 143 | print "## [ERROR] No nmap result found"
|
|---|
| 144 | continue
|
|---|
| 145 |
|
|---|
| 146 | scan = status[host]
|
|---|
| 147 | if scan['state'] != "up":
|
|---|
| 148 | continue
|
|---|
| 149 |
|
|---|
| 150 | print '# Processing host %s' % host
|
|---|
| 151 | # IP -> Mac addresses found in host ARP table, with key IP
|
|---|
| 152 | status[host]['arpmac'] = dict()
|
|---|
| 153 | # MAC -> iface addresses, with key MAC
|
|---|
| 154 | status[host]['mac'] = dict()
|
|---|
| 155 | # Mirrored: iface -> MAC addresses, with key interface name
|
|---|
| 156 | status[host]['iface'] = dict()
|
|---|
| 157 | try:
|
|---|
| 158 | if not forced_scan and stored_status[host]['snmp_retval'] != 0:
|
|---|
| 159 | print "## SNMP Connect failed last time, ignoring"
|
|---|
| 160 | continue
|
|---|
| 161 | except:
|
|---|
| 162 | pass
|
|---|
| 163 |
|
|---|
| 164 | stored_status[host] = dict()
|
|---|
| 165 | if not "open" in scan['port']['161']['state']:
|
|---|
| 166 | print "## [ERROR] SNMP port not opened"
|
|---|
| 167 | continue
|
|---|
| 168 |
|
|---|
| 169 | (p, output, stderrdata) = do_snmpwalk(host, 'SNMPv2-MIB::sysDescr')
|
|---|
| 170 | stored_status[host]['snmp_retval'] = p.returncode
|
|---|
| 171 | # Assume host remain reachable troughout all the SNMP queries
|
|---|
| 172 | if p.returncode != 0:
|
|---|
| 173 | print "## [ERROR] SNMP failed '%s'" % ",".join(stderrdata)
|
|---|
| 174 | continue
|
|---|
| 175 |
|
|---|
| 176 | # Get some host details
|
|---|
| 177 | # SNMPv2-MIB::sysDescr.0 = STRING: FreeBSD CNodeSOM1.wLeiden.NET
|
|---|
| 178 | # 8.0-RELEASE-p2 FreeBSD 8.0-RELEASE-p2 #2: Fri Feb 19 18:24:23 CET 2010
|
|---|
| 179 | # root@80fab2:/usr/obj/nanobsd.wleiden/usr/src/sys/kernel.wleiden i386
|
|---|
| 180 | status[host]['sys_desc'] = output[0]
|
|---|
| 181 | hostname = output[0].split(' ')[4]
|
|---|
| 182 | release = output[0].split(' ')[5]
|
|---|
| 183 | stored_status[host]['hostname'] = status[host]['hostname'] = hostname
|
|---|
| 184 | stored_status[host]['release'] = status[host]['release'] = release
|
|---|
| 185 | print "## %(hostname)s - %(release)s" % stored_status[host]
|
|---|
| 186 |
|
|---|
| 187 | # Check if the host is already done processing
|
|---|
| 188 | # Note: the host is marked done processing at the end
|
|---|
| 189 | if host_processed.has_key(hostname):
|
|---|
| 190 | print "## Host already processed this run"
|
|---|
| 191 | continue
|
|---|
| 192 |
|
|---|
| 193 | # Interface list with key the index number
|
|---|
| 194 | iface_descr = dict()
|
|---|
| 195 | # IF-MIB::ifDescr.1 = STRING: ath0
|
|---|
| 196 | r = re.compile('^IF-MIB::ifDescr\[([0-9]+)\] = STRING: ([a-z0-9]+)$')
|
|---|
| 197 | (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifDescr')
|
|---|
| 198 | for line in output:
|
|---|
| 199 | m = r.match(line)
|
|---|
| 200 | iface_descr[m.group(1)] = m.group(2)
|
|---|
| 201 |
|
|---|
| 202 | # IF-MIB::ifPhysAddress[1] = STRING: 0:80:48:54:bb:52
|
|---|
| 203 | r = re.compile('^IF-MIB::ifPhysAddress\[([0-9]+)\] = STRING: ([0-9a-f:]*)$')
|
|---|
| 204 | (p, output, stderrdata) = do_snmpwalk(host, 'IF-MIB::ifPhysAddress')
|
|---|
| 205 | for line in output:
|
|---|
| 206 | m = r.match(line)
|
|---|
| 207 | # Ignore lines which has no MAC address
|
|---|
| 208 | if not m.group(2): continue
|
|---|
| 209 | index = m.group(1)
|
|---|
| 210 | # Convert to proper MAC
|
|---|
| 211 | mac = ":".join(["%02X" % int(x,16) for x in m.group(2).split(':')])
|
|---|
| 212 | print "## Local MAC %s [index:%s] -> %s" % (iface_descr[index], index, mac)
|
|---|
| 213 | status[host]['mac'][mac] = iface_descr[index]
|
|---|
| 214 | status[host]['iface'][iface_descr[index]] = mac
|
|---|
| 215 | mac_to_host[mac] = hostname
|
|---|
| 216 |
|
|---|
| 217 | # Process host SNMP status
|
|---|
| 218 | (p, output, stderrdata) = do_snmpwalk(host, 'RFC1213-MIB::atPhysAddress')
|
|---|
| 219 | # RFC1213-MIB::atPhysAddress[8][1.172.21.160.34] = Hex-STRING: 00 17 C4 CC 5B F2
|
|---|
| 220 | r = re.compile('^RFC1213-MIB::atPhysAddress\[[0-9]+\]\[1\.([0-9\.]+)\] = Hex-STRING: ([0-9A-F\ ]+)$')
|
|---|
| 221 | for line in output:
|
|---|
| 222 | m = r.match(line)
|
|---|
| 223 | ip = m.group(1)
|
|---|
| 224 | # Replace spaces in MAC with :
|
|---|
| 225 | mac = ":".join(m.group(2).split(' ')[:-1])
|
|---|
| 226 | status[host]['arpmac'][ip] = mac
|
|---|
| 227 |
|
|---|
| 228 | local = '[remote]'
|
|---|
| 229 | if mac in status[host]['mac'].keys():
|
|---|
| 230 | local = '[local]'
|
|---|
| 231 | print "## Arp table MAC %s -> %s %s" % (ip, mac, local)
|
|---|
| 232 |
|
|---|
| 233 | # Make sure we keep a record of the processed host which ip entry to check
|
|---|
| 234 | host_processed[hostname] = host
|
|---|
| 235 |
|
|---|
| 236 | stored_status['host_processed'] = host_processed
|
|---|
| 237 | stored_status['mac_to_host'] = mac_to_host
|
|---|
| 238 | stored_status['nmap_status'] = status
|
|---|
| 239 | return stored_status
|
|---|
| 240 |
|
|---|
| 241 |
|
|---|
| 242 |
|
|---|
| 243 |
|
|---|
| 244 | def generate_status(configs, stored_status):
|
|---|
| 245 | """ Generate result file from stored_status """
|
|---|
| 246 | host_processed = stored_status['host_processed']
|
|---|
| 247 | mac_to_host = stored_status['mac_to_host']
|
|---|
| 248 | status = stored_status['nmap_status']
|
|---|
| 249 |
|
|---|
| 250 | # Data store format used for nodemap generation
|
|---|
| 251 | nodemap_status_file = '/tmp/nodemap_status.yaml'
|
|---|
| 252 | nodemap = { 'node' : {}, 'link' : {}}
|
|---|
| 253 |
|
|---|
| 254 | # XXX: Pushed back till we actually store the MAC in the config files automatically
|
|---|
| 255 | #configmac_to_host = dict()
|
|---|
| 256 | #for host,config in configs.iteritems():
|
|---|
| 257 | # for iface_key in gformat.get_interface_keys(config):
|
|---|
| 258 | # configmac_to_host[config[iface_key]['mac']] = host
|
|---|
| 259 |
|
|---|
| 260 | # List of hosts which has some kind of problem
|
|---|
| 261 | for host in configs.keys():
|
|---|
| 262 | fqdn = host + ".wLeiden.NET"
|
|---|
| 263 | if fqdn in host_processed.keys():
|
|---|
| 264 | continue
|
|---|
| 265 | config = configs[host]
|
|---|
| 266 | print "# Problems in host '%s'" % host
|
|---|
| 267 | host_down = True
|
|---|
| 268 | for ip in gformat.get_used_ips([config]):
|
|---|
| 269 | if not gformat.valid_addr(ip):
|
|---|
| 270 | continue
|
|---|
| 271 | if status[ip]['state'] == "up":
|
|---|
| 272 | host_down = False
|
|---|
| 273 | print "## - ", ip, status[ip]['state']
|
|---|
| 274 | if host_down:
|
|---|
| 275 | print "## HOST is DOWN!"
|
|---|
| 276 | nodemap['node'][fqdn] = gformat.DOWN
|
|---|
| 277 | else:
|
|---|
| 278 | print "## SNMP problems (not reachable, deamon not running, etc)"
|
|---|
| 279 | nodemap['node'][fqdn] = gformat.UNKNOWN
|
|---|
| 280 |
|
|---|
| 281 |
|
|---|
| 282 |
|
|---|
| 283 | # Correlation mapping
|
|---|
| 284 | for fqdn, ip in host_processed.iteritems():
|
|---|
| 285 | details = status[ip]
|
|---|
| 286 | nodemap['node'][fqdn] = gformat.OK
|
|---|
| 287 | print "# Working on %s" % fqdn
|
|---|
| 288 | for ip, arpmac in details['arpmac'].iteritems():
|
|---|
| 289 | if arpmac in details['mac'].keys():
|
|---|
| 290 | # Local MAC address
|
|---|
| 291 | continue
|
|---|
| 292 | if not mac_to_host.has_key(arpmac):
|
|---|
| 293 | print "## [WARN] No parent host for MAC %s (%s) found" % (arpmac, ip)
|
|---|
| 294 | else:
|
|---|
| 295 | print "## Interlink %s - %s" % (fqdn, mac_to_host[arpmac])
|
|---|
| 296 | nodemap['link'][(fqdn,mac_to_host[arpmac])] = gformat.OK
|
|---|
| 297 |
|
|---|
| 298 | stream = file(nodemap_status_file,'w')
|
|---|
| 299 | yaml.dump(nodemap, stream, default_flow_style=False)
|
|---|
| 300 | print "# Wrote nodemap status to '%s'" % nodemap_status_file
|
|---|
| 301 |
|
|---|
| 302 | def usage():
|
|---|
| 303 | print "Usage: %s <arguments>"
|
|---|
| 304 | print "Arguments:"
|
|---|
| 305 | print "\tall = scan all known ips, using cached nmap"
|
|---|
| 306 | print "\tnmap-only = scan all known ips, using nmap only"
|
|---|
| 307 | print "\tsnmp-only = scan all known ips, using snmp only"
|
|---|
| 308 | print "\tforce = scan all known ips, no cache used"
|
|---|
| 309 | print "\tforced-snmp = scan all known ips, no snmp cache"
|
|---|
| 310 | print "\tstored = generate status file using stored entries"
|
|---|
| 311 | print "\thost <HOST1> [HOST2 ...] = generate status file using stored entries"
|
|---|
| 312 | print "\tmerge <FILE1> [FILE2 ...] = merge status file with other status files"
|
|---|
| 313 | sys.exit(0)
|
|---|
| 314 |
|
|---|
| 315 |
|
|---|
| 316 | def main():
|
|---|
| 317 | start_time = datetime.now()
|
|---|
| 318 | stored_status_file = '/tmp/stored_status.yaml'
|
|---|
| 319 | nmap_result_file = '/tmp/test.xml'
|
|---|
| 320 |
|
|---|
| 321 | stored_status = dict()
|
|---|
| 322 | nmap_status = dict()
|
|---|
| 323 | snmp_status = dict()
|
|---|
| 324 |
|
|---|
| 325 | opt_nmap_scan = True
|
|---|
| 326 | opt_store_scan = True
|
|---|
| 327 | opt_snmp_scan = True
|
|---|
| 328 | opt_force_snmp = False
|
|---|
| 329 | opt_force_scan = False
|
|---|
| 330 | opt_force_range = False
|
|---|
| 331 | if len(sys.argv) == 1:
|
|---|
| 332 | usage()
|
|---|
| 333 |
|
|---|
| 334 | if sys.argv[1] == "all":
|
|---|
| 335 | pass
|
|---|
| 336 | elif sys.argv[1] == "nmap-only":
|
|---|
| 337 | opt_snmp_scan = False
|
|---|
| 338 | elif sys.argv[1] == "snmp-only":
|
|---|
| 339 | opt_nmap_scan = False
|
|---|
| 340 | elif sys.argv[1] == "force":
|
|---|
| 341 | opt_force_scan = True
|
|---|
| 342 | elif sys.argv[1] == "forced-snmp":
|
|---|
| 343 | opt_nmap_scan = False
|
|---|
| 344 | opt_force_snmp = True
|
|---|
| 345 | elif sys.argv[1] == "host":
|
|---|
| 346 | opt_force_range = True
|
|---|
| 347 | opt_force_scan = True
|
|---|
| 348 | elif sys.argv[1] == "stored":
|
|---|
| 349 | opt_snmp_scan = False
|
|---|
| 350 | opt_nmap_scan = False
|
|---|
| 351 | opt_store_scan = False
|
|---|
| 352 | else:
|
|---|
| 353 | usage()
|
|---|
| 354 |
|
|---|
| 355 | # By default get all IPs defined in config, else own range
|
|---|
| 356 | if not opt_force_range:
|
|---|
| 357 | configs = gformat.get_all_configs()
|
|---|
| 358 | iplist = gformat.get_used_ips(configs.values())
|
|---|
| 359 | else:
|
|---|
| 360 | iplist = sys.argv[1:]
|
|---|
| 361 |
|
|---|
| 362 | # Load data hints from previous run if exists
|
|---|
| 363 | if not opt_force_scan and os.path.exists(stored_status_file) and os.path.getsize(stored_status_file) > 0:
|
|---|
| 364 | print "## Loading stored data hints from '%s'" % stored_status_file
|
|---|
| 365 | stream = file(stored_status_file,'r')
|
|---|
| 366 | stored_status = yaml.load(stream)
|
|---|
| 367 | else:
|
|---|
| 368 | print "[ERROR] '%s' does not exists" % stored_status_file
|
|---|
| 369 |
|
|---|
| 370 | # Do a NMAP discovery
|
|---|
| 371 | if opt_nmap_scan:
|
|---|
| 372 | if not opt_store_scan:
|
|---|
| 373 | nmap_result_file = None
|
|---|
| 374 | nmap_status = do_nmap_scan(
|
|---|
| 375 | "-p T:ssh,U:domain,T:80,T:ntp,U:snmp,T:8080 -sU -sT ",
|
|---|
| 376 | iplist,nmap_result_file, opt_force_scan)
|
|---|
| 377 |
|
|---|
| 378 | else:
|
|---|
| 379 | nmap_status = stored_status['nmap_status']
|
|---|
| 380 |
|
|---|
| 381 | # Save the MAC -> HOST mappings, by default as it helps indentifing the
|
|---|
| 382 | # 'unknown links'
|
|---|
| 383 | mac_to_host = {}
|
|---|
| 384 | if stored_status:
|
|---|
| 385 | mac_to_host = stored_status['mac_to_host']
|
|---|
| 386 |
|
|---|
| 387 | # Do SNMP discovery
|
|---|
| 388 | if opt_snmp_scan:
|
|---|
| 389 | snmp_status = do_snmp_scan(iplist, nmap_status, stored_status, opt_force_snmp)
|
|---|
| 390 | else:
|
|---|
| 391 | snmp_status = stored_status
|
|---|
| 392 |
|
|---|
| 393 | # Include our saved MAC -> HOST mappings
|
|---|
| 394 | mac_to_host.update(snmp_status['mac_to_host'])
|
|---|
| 395 | snmp_status['mac_to_host'] = mac_to_host
|
|---|
| 396 |
|
|---|
| 397 | # Store changed data to disk
|
|---|
| 398 | if opt_store_scan:
|
|---|
| 399 | stream = file(stored_status_file,'w')
|
|---|
| 400 | yaml.dump(snmp_status, stream, default_flow_style=False)
|
|---|
| 401 | print "## Stored data hints to '%s'" % stored_status_file
|
|---|
| 402 |
|
|---|
| 403 | # Finally generated status
|
|---|
| 404 | generate_status(configs, snmp_status)
|
|---|
| 405 | print "# Took %s seconds to complete" % (datetime.now() - start_time).seconds
|
|---|
| 406 |
|
|---|
| 407 |
|
|---|
| 408 |
|
|---|
| 409 | if __name__ == "__main__":
|
|---|
| 410 | main()
|
|---|
