Changeset 9619 for src

Timestamp:

Aug 30, 2011, 2:14:03 PM (13 years ago)

Author:

rick

Message:

Importer method for netstumbler, prepared for merging, al import_droidstumbler, import_kismet, import_netstumber share quite some commom code.

File:

• src/django_gheat/gheat/management/commands/import_netstumbler.py (copied) (copied from src/django_gheat/gheat/management/commands/import_kismet.py ) (7 diffs)

src/django_gheat/gheat/management/commands/import_netstumbler.py

@@ -2 +2 @@
 # -*- coding: utf-8 -*-
 #
-# Script for importing .gpsxml and .netxml files (Kismet output)
+# Script for importing .ns1 files (Netstumber output)
 #
 # Rick van der Zwet <info@rickvanderzwet.nl>
@@ -19 +19 @@
 from collections import defaultdict
 
+from netstumbler import parse_netstumbler
 from import_droidstumbler import bulk_sql,get_organization_id_by_ssid
 
@@ -33 +34 @@
 
 
-def import_kismet_netxml(netxml_file):
-  netxml_doc = etree.parse(open_file(netxml_file))
-
-  counters = { 'ap_added' : 0, 'ap_total' : 0, 'ap_failed' : 0, 'ap_ignored' : 0,
-               'client_added' : 0, 'client_total' : 0, 'client_failed' : 0, 'client_ignored' : 0}
-
-  # Prepare new accespoints and measurements
-  wnetworks = netxml_doc.findall('wireless-network')
-
-  # Temponary holders
-  ap_pool = {}
-  client_pool = {}
-
-  # Create all accesspoints and for caching validation purposes store them
-  # locally as well
-  for wnetwork in wnetworks:
-    bssid = wnetwork.find('BSSID').text
-    ap_type = wnetwork.attrib['type']
-    # Only access points and clients (for ignore listings)
-    if ap_type in ['infrastructure', 'data']:
-      counters['ap_total'] += 1
-      encryption = (wnetwork.find('SSID/encryption') != None)
-      ssid_node = wnetwork.find('SSID/essid[@cloaked="false"]')
-      ssid = ssid_node.text if ssid_node != None else 'hidden'
-
-      ap_pool[bssid] = (ssid, encryption)
-    elif ap_type in ['probe', 'ad-hoc']:
-      counters['client_total'] += 1
-      client_pool[bssid] = True
-    else:
-      logger.error('Unknown type %s - %s',bssid, wnetwork.attrib['type'])
-
-
+def import_accespoints(ap_pool, counters):
   # Determine which Accespoints to add
-  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).values_list('mac', flat=True)
+  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).\
+    values_list('mac', flat=True)
   bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
 
@@ -76 +46 @@
       ssid, encryption = ap_pool[bssid]
       # Special trick in SSID ts avoid escaping in later stage
-      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,get_organization_id_by_ssid(ssid)))
+      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,
+        get_organization_id_by_ssid(ssid)))
       sql_values.append(item)
-    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`, `encryptie`, `organization_id`)',sql_values)
-
-  # Determine which Wireless Clients to add
-  bssid_list_present = WirelessClient.objects.filter(mac__in=client_pool.keys()).values_list('mac', flat=True)
-  bssid_list_insert = set(client_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      sql_values.append("('%s')" % bssid.upper())
-    counters['client_added'] = bulk_sql('gheat_wirelessclient (`mac`)',sql_values)
-
+    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`,\
+      `encryptie`, `organization_id`)',sql_values)
   return counters
 
 
 
-def import_kismet_gpsxml(gpsxml_file, meetrondje):
-  gpsxml_doc = etree.parse(open_file(gpsxml_file))
-
-  #Various statistics
-  counters = {'meting_added' : 0, 'meting_total' : 0, 'meting_failed' : 0, 'meting_ignored' :0}
-
+def import_metingen(meetrondje, meting_pool, counters):
+  # Temponary holders
   bssid_failed = defaultdict(int)
-
-  # Prepare new accespoints and measurements
-  points = gpsxml_doc.findall('gps-point')
-
-  # Temponary holders
-  meting_pool = defaultdict(list)
-
-  for point in points:
-    counters['meting_total'] += 1
-    #XXX: This needs to be either the 'bssid' or the 'source', 
-    #XXX: accesspoint from or too data.
-    bssid = point.attrib['bssid']
-    # XXX: Filter this in the beginning with XPath, but etree does not support
-    # that (yet).
-    if bssid in ['GP:SD:TR:AC:KL:OG','00:00:00:00:00:00']:
-      counters['meting_ignored'] =+ 1
-      continue
-    # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
-    try:
-      level = point.attrib['signal_dbm']
-    except KeyError:
-      logger.debug("Point '%s' does not have signal strengh" % point)
-      counters['meting_failed'] += 1
-      continue
-    # We store all values found, avg or max will be done later on
-    key = (bssid, point.attrib['lat'], point.attrib['lon'])
-    signaal=100 + int(level)
-    meting_pool[key].append(signaal)
 
   bssid_list = [x[0] for x in meting_pool.keys()]
   # Build mapping for meting import
   mac2id = {}
-  for mac,id in Accespoint.objects.filter(mac__in=bssid_list).values_list('mac','id'):
+  for mac,id in Accespoint.objects.filter(mac__in=bssid_list).\
+    values_list('mac','id'):
     mac2id[mac] = int(id)
 
   clients = {}
-  for mac in WirelessClient.objects.filter(mac__in=bssid_list).values_list('mac',flat=True):
+  for mac in WirelessClient.objects.filter(mac__in=bssid_list).\
+    values_list('mac',flat=True):
     clients[mac] = True
 
@@ -149 +79 @@
       bssid_failed[bssid] += len(signals)
     else:
-      item = str((int(meetrondje.id),mac2id[bssid],float(lat),float(lon),max(signals)))
+      item = str((int(meetrondje.id),mac2id[bssid],float(lat),\
+        float(lon),max(signals)))
       sql_values.append(item)
 
@@ -157 +88 @@
 
   if sql_values:
-    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`, `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
+    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`,\
+      `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
   return counters
 
 
+
+def process_netstumber(filename):
+  data = parse_netstumbler(open_file(filename))
+
+  counters = {
+    'ap_added' : 0, 'ap_total' : 0,
+    'ap_failed' : 0, 'ap_ignored' : 0,
+    'client_added' : 0, 'client_total' : 0,
+    'client_failed' : 0, 'client_ignored' : 0,
+    'meting_added' : 0, 'meting_total' : 0,
+    'meting_failed' : 0, 'meting_ignored' : 0
+    }
+
+  # Temponary holders
+  meting_pool = defaultdict(list)
+  ap_pool = {}
+
+  for ap in data['aps']:
+    # XXX: How is encryption coded?
+    encryption = False
+    ap_pool[ap['BSSID']]= (ap['SSID'], encryption)
+    for point in ap["measurements"]:
+     counters['meting_total'] += 1
+     if point['LocationSource'] == 0:
+       logger.debug("No GPS Coordinates found for BSSID %s @ %s", 
+         ap['BSSID'], point['Time'])
+       counters['meting_ignored'] += 1
+       continue
+     # We store all values found, avg or max will be done later on
+     key = (ap['BSSID'], point["Latitude"], point["Longitude"])
+
+     # Known measurement error
+     if point['Signal'] == -32767: continue
+
+     # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
+     signaal= 100 + point['Signal']
+     if signaal > 100 or signaal < 0:
+       logger.warning("Signal %s is not valid entry for BSSID %s @ %s",
+         point['Signal'], ap['BSSID'], point['Time'])
+       continue
+     meting_pool[key].append(signaal)
+
+  return (counters, ap_pool, meting_pool)
+
+
+
+
 class Command(BaseCommand):
-  args = '<gpsxml|netxml>[.gz] [gpsxml2[.gz]  gpsxml3[.gz] ...]'
+  args = '<netstumber.ns1>[.gz] [netstumber2.ns1[.gz]  netstumber3.ns1[.gz] ...]'
   option_list = BaseCommand.option_list + (
-    make_option('-k', '--kaart', dest='kaart', default='onbekend', help="Kaart gebruikt"),
+    make_option('-k', '--kaart', dest='kaart', default='onbekend', 
+      help="Kaart gebruikt"),
     make_option('-m', '--meetrondje', dest='meetrondje', default=None),
-    make_option('-g', '--gebruiker', dest='gebruiker', default='username',help='Naam van de persoon die de meting uitgevoerd heeft'),
-    make_option('-e', '--email', dest='email', default='foo@bar.org',help='Email van de persoon die de meting uitgevoerd heeft'),
-    make_option('-d', '--datum', dest='datum', default=None, help="Provide date  \
-      in following format: '%Y%m%d-%H-%M-%S-1', by default it will be generated from \
-      the filename"),
+    make_option('-g', '--gebruiker', dest='gebruiker', default='username',
+      help='Naam van de persoon die de meting uitgevoerd heeft'),
+    make_option('-e', '--email', dest='email', default='foo@bar.org',
+      help='Email van de persoon die de meting uitgevoerd heeft'),
+    make_option('-d', '--datum', dest='datum', default=None,
+      help="Provide date in following format: '%Y%m%d-%H-%M-%S-1', by \
+      default it will be generated from the filename"),
   )
 
   def handle(self, *args, **options):
+    if options['verbosity'] > 1:
+      logger.setLevel(logging.DEBUG)
     if len(args) == 0:
       self.print_help(sys.argv[0],sys.argv[1])
@@ -179 +163 @@
 
     # Please first the netxml and then the gpsxml files
-    sorted_args = [x for x in args if "netxml" in x] + [x for x in args if "gpsxml" in x]
+    sorted_args = [x for x in args if "ns1" in x]
     remainder = list(set(args) - set(sorted_args))
     args = sorted_args + remainder
     logger.debug("Parsing files in the following order: %s", args)
 
-    for xml_file in args:
-      if not os.path.isfile(xml_file):
-        raise CommandError("xml file '%s' does not exists" % xml_file)
-
-    for xml_file in args:
-      logger.info("Processing '%s'" % xml_file)
-      if 'netxml' in xml_file:
-        counters = import_kismet_netxml(xml_file)
-        logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
-        logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
-      elif 'gpsxml' in xml_file:
+    for filename in args:
+      if not os.path.isfile(filename):
+        raise CommandError("file '%s' does not exists" % filename)
+
+    def process_date(datestr):
+      try:
+         # Kismet-20110805-15-37-30-1
+         return datetime.datetime.strptime(datestr,'%Y%m%d-%H-%M-%S-1')
+      except ValueError:
+        raise CommandError("Invalid date '%s'" % options['datum'])
+
+    for filename in args:
+      logger.info("Processing '%s'" % filename)
+      if 'ns1' in filename:
         if options['datum'] == None:
-           datum = os.path.basename(xml_file).lstrip('Kismet-').rstrip('.gz').rstrip('.gpsxml').rstrip('.netxml')
+           datestr = os.path.basename(filename).lstrip('Kismet-').\
+             rstrip('.gz').rstrip('.gpsxml').rstrip('.netxml').rstrip('.ns1')
+           datum = process_date(datestr)
+        elif options['datum'] == 'now':
+           datum = datetime.datetime.now()
         else:
-           datum = options['datum']
-        try:
-           # Kismet-20110805-15-37-30-1
-           datum = datetime.datetime.strptime(datum,'%Y%m%d-%H-%M-%S-1')
-        except ValueError:
-          raise CommandError("Invalid date '%s'" % options['datum'])
+           datum = process_date(options['datum'])
 
         # Meetrondje from filename if needed
         if options['meetrondje'] == None:
-          meetrondje = os.path.basename(xml_file).rstrip('.gz').rstrip('.gpsxml')
+          meetrondje = os.path.basename(filename).rstrip('.gz').rstrip('.ns1')
         else:
           meetrondje = options['meetrondje']
 
         # Create meetrondje object
-        g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'] , email=options['email'])
+        g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'],
+          email=options['email'])
         a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
-        mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
+        mr, created = MeetRondje.objects.get_or_create(datum=datum,
+          naam=meetrondje, gebruiker=g, apparatuur=a)
         logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
         if not created:
           logger.error("Meetrondje '%s' already imported" % mr)
           continue
-        counters = import_kismet_gpsxml(xml_file, mr)
-        logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
+        (counters, ap_pool, meting_pool) = process_netstumber(filename)
+        counters = import_accespoints(ap_pool, counters)
+        counters = import_metingen(mr, meting_pool, counters)
+
+        logger.info(("summary metingen   : total:%(meting_total)-6s" +
+          "added:%(meting_added)-6s failed:%(meting_failed)-6s" +
+          "ignored:%(meting_ignored)-6s") % counters)
       else:
-        raise CommandError("xml file '%s' format not recognized" % xml_file)
+        raise CommandError("file '%s' format not recognized" % filename)