Changeset 9158 for src

Timestamp:

May 7, 2011, 8:06:03 PM (14 years ago)

Author:

rick

Message:

New netstumber import, based on gpsxml and netxml input files.

File:

• src/django_gheat/gheat/management/commands/import_kismet.py (copied) (copied from src/django_gheat/gheat/management/commands/dataimport.py ) (1 diff)

src/django_gheat/gheat/management/commands/import_kismet.py

@@ -1 +1 @@
 #!/usr/bin/env python
 #
-###########################################
+# Script for importing .gpsxml and .netxml files (Kismet output)
 #
-# Script for importing .netxml files (Kismet output)
-#
-# In theory, only the -f option is needed, but for overview's sake, please use the others aswell.
-# -f = location of the .netxml, e.g. '/home/test.netxml'
-# -m = name of the dataset, e.g. 'Walk in park' or 'Trip with boat'
-# -g = your name
-# -e = your email address
-#
-# (Run from project root)
-# ./manage.py dataimport -f <file location> -m <dataset name> -g <username> -e <email>
-#
-# Make sure your file has atleast the following elements:
-#
-# <wireless-network>
-#   <SSID>
-#     <encryption> </encryption>
-#     <essid cloaked=""> </essid>
-#   </SSID>
-#   <BSSID> </BSSID>
-#   <gps-info>
-#     <min-lat> </min-lat>
-#     <min-lon> </min-lon>
-#   </gps-info>
-# </wireless-network>
-#
-#
-# Dennis Wagenaar
-# d.wagenaar@gmail.com
-#
-###########################################
 
-from django.core.management import setup_environ
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand,CommandError
 from optparse import OptionParser, make_option
-import settings
-setup_environ(settings)
 from gheat.models import *
 from lxml import etree
 import datetime
+import os
+import sys
 
-def import_file(location, meetrondje, gebruiker, email):
+def import_file(gpsxml_file, netxml_file, meetrondje, gebruiker, email):
+  gpsxml_doc = etree.parse(gpsxml_file)
+  netxml_doc = etree.parse(netxml_file)
 
-  datasource = etree.parse(location)
-  doc = datasource.findall('wireless-network')
-  kaart = datasource.find('card-source/card-source')
-  kaart = kaart.text
-  g, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
-  a, created = Apparatuur.objects.get_or_create(antenne='test' , kaart=kaart)
-  mr = MeetRondje.objects.create(datum=datetime.datetime.now() , naam=meetrondje , gebruiker=g , apparatuur=a)
+  points = gpsxml_doc.findall('gps-point')
+  wnetworks = netxml_doc.findall('wireless-network')
 
-  for wnetwork in doc:
-    enc = wnetwork.find('SSID/encryption')
-    if enc != None: enc = True
-    else: enc = False
-    ssid = wnetwork.find('SSID/essid')
-    if ssid != None: ssid = ssid.text
-    else: ssid = 'hidden'
-    bssid = wnetwork.find('BSSID')
-    if bssid != None: bssid = bssid.text
-    min_sig = wnetwork.find('snr-info/min_signal_dbm')
-    if min_sig != None: min_sig = int(min_sig.text)
-    else: min_sig = (-99)
-    max_sig = wnetwork.find('snr-info/max_signal_dbm')
-    if max_sig != None: max_sig = int(max_sig.text)
-    else: max_sig= (-99)
-    lat = wnetwork.find('gps-info/min-lat')
-    if lat != None: lat = lat.text
-    lon = wnetwork.find('gps-info/min-lon')
-    if lon != None: lon = lon.text
-    if ssid is None: continue
-    
-    avg_sig = (min_sig+max_sig)/2
-    sig = (1.67*avg_sig)+166.67
-    if sig > 100: sig = 100
-    if sig < 0: sig = 0
+  # TODO: Source source is variable entitity, based on mesurement
+  kaart = 'deadcode'
+  gebruiker, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
+  apparatuur, created = Apparatuur.objects.get_or_create(antenne='test' , kaart=kaart)
+  # TODO: Date is set to import date, but should pick the date from the netxml file
+  mr = MeetRondje.objects.create(datum=datetime.datetime.now(),
+    naam=meetrondje , gebruiker=gebruiker , apparatuur=apparatuur)
 
-    print enc, ssid, bssid, lat, lon, sig
+  # Create all accesspoints and for caching validation purposes store them
+  # locally as well
+  ap_cache = {}
+  ap_ignore = []
+  print "#INFO: Going to import %s accesspoints" % len(wnetworks)
+  for wnetwork in wnetworks:
+    bssid = wnetwork.find('BSSID').text
+    # Only store access points
+    if wnetwork.attrib['type'] != "infrastructure":
+      ap_ignore.append(bssid)
+      continue
+
+    enc = (wnetwork.find('SSID/encryption') != None)
+    ssid_node = wnetwork.find('SSID/essid[@cloaked="false"]')
+    ssid = ssid_node.text if ssid_node != None else 'hidden'
 
     ap, created = Accespoint.objects.get_or_create(mac=bssid, ssid=ssid, encryptie=enc)
-    m = Meting.objects.create(meetrondje=mr, accespoint=ap, latitude=lat, longitude=lon, signaal=sig)
+    ap_cache[bssid] = ap
 
+  count = 0
+  #XXX: This is not effient at all, try to wrap it into a a bulk insert would
+  # be much more effient as for example: http://djangosnippets.org/snippets/2362/
+  print "#INFO: Going to import %s points" % len(points)
+  for point in points:
+    # XXX: Filter this in the beginning with XPath, but etree does not support that (yet).
+    if point.attrib['bssid'] in ['GP:SD:TR:AC:KL:OG','00:00:00:00:00:00']:
+      continue
+    if point.attrib['bssid'] in ap_ignore:
+      continue
+    ap = ap_cache[point.attrib['bssid']]
+
+    # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
+    signaal = 100 + int(point.attrib['signal_dbm'])
+
+    meting,created = Meting.objects.get_or_create(meetrondje=mr, accespoint=ap,
+      latitude=point.attrib['lat'], longitude=point.attrib['lon'],
+      signaal=signaal)
+    # Give some feedback to the user
+    if created:
+      count += 1
+      if (count % 1000) == 0:
+        print count,
+      elif (count % 100) == 0:
+        print ".",
 
 class Command(BaseCommand):
+  args = '<gpsxml> <netxml>'
   option_list = BaseCommand.option_list + (
-    make_option('-f', '--location', dest='location', default='location'),
-    make_option('-m', '--meetrondje', dest='meetrondje', default='rondje'),
-    make_option('-g', '--gebruiker', dest='gebruiker', default='username'),
-    make_option('-e', '--email', dest='email', default='foo@bar.org'),
+    make_option('-m', '--meetrondje', dest='meetrondje', default='rondje',help='Naam van het meetrondje'),
+    make_option('-g', '--gebruiker', dest='gebruiker', default='username',help='Naam van de persoon die de meting uitgevoerd heeft'),
+    make_option('-e', '--email', dest='email', default='foo@bar.org',help='Email van de persoon die de meting uitgevoerd heeft'),
     )
 
+  # TODO: Condider netxml file to be the gpsxml file with diffent suffix.
   def handle(self, *args, **options):
-    import_file(options['location'],options['meetrondje'],options['gebruiker'],options['email'])
+    try:
+      (gpsxml_file, netxml_file) = args
+    except ValueError:
+      self.print_help(sys.argv[0],sys.argv[1])
+      raise CommandError("Not all arguments are provided")
+    if not os.path.isfile(gpsxml_file):
+      raise CommandError("gpsxml file '%s' does not exists" % gpsxml_file)
+    if not os.path.isfile(netxml_file):
+      raise CommandError("netxml file '%s' does not exists" % netxml_file)
+
+    import_file(gpsxml_file, netxml_file ,options['meetrondje'],options['gebruiker'],options['email'])