Changeset 9561 for src/django_gheat/gheat

Timestamp:

Aug 25, 2011, 12:31:50 PM (13 years ago)

Author:

rick

Message:

gwSplit netxml and gpsxml into to different calls, as you first like to import
_all_ netxml before importing _all_ gpsxml, as the gps probes might reference a
unknown accesspoint.

File:

• src/django_gheat/gheat/management/commands/import_kismet.py (modified) (6 diffs)

src/django_gheat/gheat/management/commands/import_kismet.py

@@ -24 +24 @@
 logger.setLevel(logging.INFO)
 
-def import_kismet(gpsxml_file, netxml_file, meetrondje):
+# Open files for reading
+def open_file(file):
+ if file.endswith('.gz'):
+   return gzip.open(file,'rb')
+ else:
+  return open(file,'rb')
 
-  # Open files for reading
-  def open_file(file):
-   if file.endswith('.gz'):
-     return gzip.open(file,'rb')
-   else:
-    return open(file,'rb')
-  gpsxml_doc = etree.parse(open_file(gpsxml_file))
+
+
+def import_kismet_netxml(netxml_file):
   netxml_doc = etree.parse(open_file(netxml_file))
 
+  counters = { 'ap_added' : 0, 'ap_total' : 0, 'ap_failed' : 0, 'ap_ignored' : 0}
+
+  # Prepare new accespoints and measurements
+  wnetworks = netxml_doc.findall('wireless-network')
+
+  # Temponary holders
+  ap_pool = {}
+
+  # Create all accesspoints and for caching validation purposes store them
+  # locally as well
+  for wnetwork in wnetworks:
+    counters['ap_total'] += 1
+    bssid = wnetwork.find('BSSID').text
+    # Only store access points
+    ap_type = wnetwork.attrib['type']
+    if ap_type in ['infrastructure', 'data']:
+      encryption = (wnetwork.find('SSID/encryption') != None)
+      ssid_node = wnetwork.find('SSID/essid[@cloaked="false"]')
+      ssid = ssid_node.text if ssid_node != None else 'hidden'
+
+      ap_pool[bssid] = (ssid, encryption)
+    elif ap_type in ['probe', 'ad-hoc']:
+      counters['ap_ignored'] += 1
+      continue
+    else:
+      logger.error('Unknown type %s - %s',bssid, wnetwork.attrib['type'])
+
+
+  # Determine which entries we need to add
+  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).values_list('mac', flat=True)
+  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
+
+  # Create a bulk import list and import
+  if bssid_list_insert:
+    sql_values = []
+    for bssid in bssid_list_insert:
+      ssid, encryption = ap_pool[bssid]
+      # Special trick in SSID ts avoid escaping in later stage
+      item = str((bssid,ssid.replace('%','%%'),encryption))
+      sql_values.append(item)
+    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`, `encryptie`)',sql_values)
+
+  return counters
+
+
+
+def import_kismet_gpsxml(gpsxml_file, meetrondje):
+  gpsxml_doc = etree.parse(open_file(gpsxml_file))
+
   #Various statistics
-  counters = {'meting_added' : 0, 'meting_total' : 0, 'meting_failed' : 0, 
-              'ap_added' : 0, 'ap_total' : 0, 'ap_failed' : 0}
+  counters = {'meting_added' : 0, 'meting_total' : 0, 'meting_failed' : 0, 'meting_ignored' :0}
 
   bssid_failed = defaultdict(int)
 
   # Prepare new accespoints and measurements
-  wnetworks = netxml_doc.findall('wireless-network')
   points = gpsxml_doc.findall('gps-point')
 
   # Temponary holders
   meting_pool = defaultdict(list)
-  ap_pool = {}
-
-  # Create all accesspoints and for caching validation purposes store them
-  # locally as well
-  ap_ignore = []
-  for wnetwork in wnetworks:
-    bssid = wnetwork.find('BSSID').text
-    # Only store access points
-    if wnetwork.attrib['type'] != "infrastructure":
-      ap_ignore.append(bssid)
-      continue
-
-    encryption = (wnetwork.find('SSID/encryption') != None)
-    ssid_node = wnetwork.find('SSID/essid[@cloaked="false"]')
-    ssid = ssid_node.text if ssid_node != None else 'hidden'
-
-    counters['meting_total'] += 1
-    ap_pool[bssid] = (ssid, encryption)
-
 
   for point in points:
+    counters['meting_total'] += 1
     #XXX: This needs to be either the 'bssid' or the 'source', 
     #XXX: accesspoint from or too data.
@@ -74 +104 @@
     # that (yet).
     if bssid in ['GP:SD:TR:AC:KL:OG','00:00:00:00:00:00']:
-      continue
-    elif bssid in ap_ignore:
+      counters['meting_ignored'] =+ 1
       continue
     # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
@@ -88 +117 @@
     signaal=100 + int(level)
     meting_pool[key].append(signaal)
-
-
-  # Determine which entries we need to add
-  counters['ap_total'] = len(ap_pool)
-  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).values_list('mac', flat=True)
-  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      ssid, encryption = ap_pool[bssid]
-      # Special trick in SSID ts avoid escaping in later stage
-      item = str((bssid,ssid.replace('%','%%'),encryption))
-      sql_values.append(item)
-    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`, `encryptie`)',sql_values)
 
   # Build mapping for meting import
@@ -123 +136 @@
     logger.debug("Missing BSSID %s found %3s times", bssid, count)
 
-
   if sql_values:
     counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`, `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
@@ -130 +142 @@
 
 class Command(BaseCommand):
-  args = '<gpsxml>[.gz] [gpsxml2[.gz]  gpsxml3[.gz] ...]'
+  args = '<gpsxml|netxml>[.gz] [gpsxml2[.gz]  gpsxml3[.gz] ...]'
   option_list = BaseCommand.option_list + (
     make_option('-k', '--kaart', dest='kaart', default='onbekend', help="Kaart gebruikt"),
@@ -146 +158 @@
       raise CommandError("Not all arguments are provided")
 
-    for gpsxml_file in args:
-      if not os.path.isfile(gpsxml_file):
-        raise CommandError("gpsxml file '%s' does not exists" % gpsxml_file)
+    for xml_file in args:
+      if not os.path.isfile(xml_file):
+        raise CommandError("xml file '%s' does not exists" % xml_file)
 
-      netxml_file = gpsxml_file.replace('.gpsxml','.netxml')
-      if not os.path.isfile(netxml_file):
-        raise CommandError("correlated netxml file '%s' does not exists" % netxml_file)
 
-      logger.info("Processing '%s'" % gpsxml_file)
-      if options['datum'] == None:
-         datum = os.path.basename(gpsxml_file).lstrip('Kismet-').rstrip('.gpsxml.gz')
+    for xml_file in args:
+      logger.info("Processing '%s'" % xml_file)
+      if 'netxml' in xml_file:
+        counters = import_kismet_netxml(xml_file)
+        logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
+      elif 'gpsxml' in xml_file:
+        if options['datum'] == None:
+           datum = os.path.basename(xml_file).lstrip('Kismet-').rstrip('.gz').rstrip('.gpsxml').rstrip('.netxml')
+        else:
+           datum = options['datum']
+        try:
+           # Kismet-20110805-15-37-30-1
+           datum = datetime.datetime.strptime(datum,'%Y%m%d-%H-%M-%S-1')
+        except ValueError:
+          raise CommandError("Invalid date '%s'" % options['datum'])
+
+        # Meetrondje from filename if needed
+        if options['meetrondje'] == None:
+          meetrondje = os.path.basename(xml_file).rstrip('.gz').rstrip('.gpsxml')
+        else:
+          meetrondje = options['meetrondje']
+
+        # Create meetrondje object
+        g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'] , email=options['email'])
+        a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
+        mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
+        logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
+        if not created:
+          logger.error("Meetrondje '%s' already imported" % mr)
+          sys.exit(1)
+        counters = import_kismet_gpsxml(xml_file, mr)
+        logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
       else:
-         datum = options['datum']
-      try:
-         # Kismet-20110805-15-37-30-1
-         datum = datetime.datetime.strptime(datum,'%Y%m%d-%H-%M-%S-1')
-      except ValueError:
-        raise CommandError("Invalid date '%s'" % options['datum'])
-
-      # Meetrondje from filename if needed
-      if options['meetrondje'] == None:
-        meetrondje = os.path.basename(gpsxml_file).rstrip('.gz').rstrip('.gpsxml')
-      else:
-        meetrondje = options['meetrondje']
-
-      # Create meetrondje object
-      g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'] , email=options['email'])
-      a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
-      mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
-      logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
-      if not created:
-        logger.error("Meetrondje '%s' already imported" % mr)
-        sys.exit(1)
-      counters = import_kismet(gpsxml_file, netxml_file, mr)
-      logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s" % counters)
-      logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s" % counters)
+        raise CommandError("xml file '%s' format not recognized" % xml_file)