Index: src/django_gheat/gheat/management/commands/droidstumbler.py
===================================================================
--- src/django_gheat/gheat/management/commands/droidstumbler.py	(revision 9623)
+++ src/django_gheat/gheat/management/commands/droidstumbler.py	(revision 9623)
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Rick van der Zwet <info@rickvanderzwet.nl>
+#
+import csv
+import logging
+
+from collections import defaultdict
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+def process_csv(fh, counters):
+  """ Import all points, return tuple with summary"""
+
+  # Temponary holders
+  meting_pool = defaultdict(list)
+  ap_pool = {}
+
+  csvfile = csv.reader(fh, delimiter=',')
+  # Process file, preparing new access points and measurements
+  for row in csvfile:
+    try:
+      epoch, msg_type, lat, lon, accuracy, ssid, bssid, level, frequency, capabilities = row
+      bssid = bssid.upper()
+    except ValueError:
+      # Known error, please ignore
+      if row[1] == 'gps' and len(row) == 12: continue
+      logger.error("Unable to parse line:%i '%s'" % (csvfile.line_num, row))
+      continue
+    if msg_type == "data" and lat and lon:
+      counters['meting_total'] += 1
+      if not ap_pool.has_key(bssid):
+        encryption = 'WPA' in capabilities or 'WEP' in capabilities
+        ap_pool[bssid] = (ssid, encryption)
+
+      # We store the best value found
+      key = (bssid, lat, lon)
+      signaal=(100 + int(level))
+      meting_pool[key].append(signaal)
+
+  return (counters, ap_pool, None, meting_pool)
Index: src/django_gheat/gheat/management/commands/import_csv.py
===================================================================
--- src/django_gheat/gheat/management/commands/import_csv.py	(revision 9619)
+++ 	(revision )
@@ -1,94 +1,0 @@
-#!/usr/bin/env python
-#
-###########################################
-#
-# Script for importing .csv files
-#
-# In theory, only the -f option is needed, but for overview's sake, please use the others aswell.
-# -f = location of the .csv, e.g. '/home/test.csv'
-# -m = name of the dataset, e.g. 'Walk in park' or 'Trip with boat'
-# -g = your name
-# -e = your email address
-# -a = antenna
-# -k = network card
-#
-# (Run from project root)
-# ./manage.py import_csv -f <file location> -m <dataset name> -g <username> -e <email> - a <antenna> -k <network card>
-#
-# Make sure the variables in this script match the column numbers in your file e.g.;
-# Lat is read from the first column [0], if the lat in your file is in the 4th column, change
-# 'lat = row[0]' to 'lat = row[3]'.
-# Also, take note of the 'replace()' and 'strip()' functions. These should probably be edited aswell.
-#
-# Dennis Wagenaar
-# d.wagenaar@gmail.com
-#
-###########################################
-
-from django.core.management import setup_environ
-from django.core.management.base import BaseCommand
-from optparse import OptionParser, make_option
-import settings
-setup_environ(settings)
-from gheat.models import *
-import datetime
-import csv
-
-# Function that imports a csv and processes it.
-def import_file(location, meetrondje, gebruiker, email, antenne, kaart):
-
-  # Creating some objects that have to be created only once. Uses 'get_or_create' to rule out duplicates.
-  # Creating user object. Depends on basecommand options.
-  g, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
-  # Creating equipment object. Depends on basecomand options.
-  a, created = Apparatuur.objects.get_or_create(antenne=antenne , kaart=kaart)
-  # Creating dataset object. Depends on baseommand options.
-  mr = MeetRondje.objects.create(datum=datetime.datetime.now() , naam=meetrondje , gebruiker=g , apparatuur=a)
-
-  # Open the csv.
-  csvfile = csv.reader(open(location, 'rb'), delimiter='\t')
-  # Read every row individually and extract the required data.
-  for row in csvfile:
-    # '.replace' & '.strip' are for replacing and stripping some unwished characters. Edit where necessary.
-    lat = row[0].replace('N ', '')
-    lon = row[1].replace('E ', '')
-    ssid = row[2].strip('( )')
-    bssid = row[4].strip('( )')
-    enc = row[8]
-    if enc[2] == '1': enc = True
-    else: enc = False
-    sig = 100 # Get's fix soon
-    print lat, lon, ssid, bssid, enc, sig
-
-    # Simple check for bad values
-    if lat.startswith('0.'):
-      print 'Bad lat'
-      continue
-    elif lon.startswith('0.'):
-      print 'Bad lon'
-      continue
-    elif sig not in range(1,101):
-      print 'Bad sig'
-      continue
-    else:
-      print 'No bad values'
-
-    # Creating accespoint objects. Avoiding duplicates.
-    ap, created = Accespoint.objects.get_or_create(mac=bssid, ssid=ssid, encryptie=enc)
-    # Creating the measurement objects.
-    m = Meting.objects.create(meetrondje=mr, accespoint=ap, latitude=lat, longitude=lon, signaal=sig)
-
-# Basecommand with options. This gives the user some control over the execution of this script.
-class Command(BaseCommand):
-  option_list = BaseCommand.option_list + (
-    make_option('-f', '--location', dest='location', default='location'),
-    make_option('-m', '--meetrondje', dest='meetrondje', default='rondje'),
-    make_option('-g', '--gebruiker', dest='gebruiker', default='username'),
-    make_option('-e', '--email', dest='email', default='foo@bar.org'),
-    make_option('-a', '--antenne', dest='antenne', default='geen'),
-    make_option('-k', '--kaart', dest='kaart', default='interne kaart'),
-    )
-  
-  # The function 'import_file' will be executed with the default option values, or the values specified by the user.
-  def handle(self, *args, **options):
-    import_file(options['location'],options['meetrondje'],options['gebruiker'],options['email'],options['antenne'],options['kaart'])
Index: src/django_gheat/gheat/management/commands/import_datafile.py
===================================================================
--- src/django_gheat/gheat/management/commands/import_datafile.py	(revision 9623)
+++ src/django_gheat/gheat/management/commands/import_datafile.py	(revision 9623)
@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Script for importing various stumble files in a modular fasion:
+# - .ns1 (Netstumber)
+# - .gpsxml .netxml (Kismet)
+# - DroidStumbler-*.csv (DroidStumber)
+#
+# Rick van der Zwet <info@rickvanderzwet.nl>
+#
+from django.core.management.base import BaseCommand,CommandError
+from django.db.utils import IntegrityError
+from optparse import OptionParser, make_option
+from gheat.models import *
+from lxml import etree
+import datetime
+import gzip
+import os
+import sys
+import logging
+
+from collections import defaultdict
+
+import netstumbler
+import kismet
+import droidstumbler
+from import_droidstumbler import bulk_sql,get_organization_id_by_ssid
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+# Open files for reading
+def open_file(file):
+ if file.endswith('.gz'):
+   return gzip.open(file,'rb')
+ else:
+  return open(file,'rb')
+
+# Prefix/Suffix removal
+valid_prefix = ['DroidStumbler-', 'Kismet-']
+def strip_prefix(filename):
+  for suffix in valid_suffixes:
+    filename = filename.rstrip(suffix)
+  return filename
+valid_suffixes = ['.gz', '.gpsxml', '.netxml', '.csv', '.ns1']
+def strip_suffix(filename):
+  for suffix in valid_suffixes:
+    filename = filename.rstrip(suffix)
+  return filename
+def strip_file(filename):
+  return strip_suffix(strip_prefix(filename))
+
+
+def import_accespoints(ap_pool, counters):
+  # Determine which Accespoints to add
+  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).\
+    values_list('mac', flat=True)
+  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
+
+  # Create a bulk import list and import
+  if bssid_list_insert:
+    sql_values = []
+    for bssid in bssid_list_insert:
+      ssid, encryption = ap_pool[bssid]
+      # Special trick in SSID ts avoid escaping in later stage
+      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,
+        get_organization_id_by_ssid(ssid)))
+      sql_values.append(item)
+    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`,\
+      `encryptie`, `organization_id`)',sql_values)
+  return counters
+
+
+
+def import_metingen(meetrondje, meting_pool, counters):
+  # Temponary holders
+  bssid_failed = defaultdict(int)
+
+  bssid_list = [x[0] for x in meting_pool.keys()]
+  # Build mapping for meting import
+  mac2id = {}
+  for mac,id in Accespoint.objects.filter(mac__in=bssid_list).\
+    values_list('mac','id'):
+    mac2id[mac] = int(id)
+
+  clients = {}
+  for mac in WirelessClient.objects.filter(mac__in=bssid_list).\
+    values_list('mac',flat=True):
+    clients[mac] = True
+
+  sql_values = []
+  for (bssid,lat,lon),signals in meting_pool.iteritems():
+    if clients.has_key(bssid):
+      counters['meting_ignored'] += len(signals)
+    elif not mac2id.has_key(bssid):
+      counters['meting_failed'] += len(signals)
+      bssid_failed[bssid] += len(signals)
+    else:
+      item = str((int(meetrondje.id),mac2id[bssid],float(lat),\
+        float(lon),max(signals)))
+      sql_values.append(item)
+
+  for bssid,count in sorted(bssid_failed.items(),
+      key=lambda item: item[1], reverse=True):
+    logger.debug("Missing BSSID %s found %3s times", bssid, count)
+
+  if sql_values:
+    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`,\
+      `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
+  return counters
+
+
+def import_clients(client_pool, counters):
+  # Determine which Wireless Clients to add
+  bssid_list_present = WirelessClient.objects.filter(mac__in=client_pool.keys()).values_list('mac', flat=True)
+  bssid_list_insert = set(client_pool.keys()) - set(bssid_list_present)
+
+  # Create a bulk import list and import
+  if bssid_list_insert:
+    sql_values = []
+    for bssid in bssid_list_insert:
+      sql_values.append("('%s')" % bssid.upper())
+    counters['client_added'] = bulk_sql('gheat_wirelessclient (`mac`)',sql_values)
+
+  return counters
+
+
+
+
+
+class Command(BaseCommand):
+  args = '<netstumber.ns1>[.gz] [netstumber2.ns1[.gz]  netstumber3.ns1[.gz] ...]'
+  option_list = BaseCommand.option_list + (
+    make_option('-k', '--kaart', dest='kaart', default='onbekend', 
+      help="Kaart gebruikt"),
+    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
+    make_option('-g', '--gebruiker', dest='gebruiker', default='username',
+      help='Naam van de persoon die de meting uitgevoerd heeft'),
+    make_option('-e', '--email', dest='email', default='foo@bar.org',
+      help='Email van de persoon die de meting uitgevoerd heeft'),
+    make_option('-d', '--datum', dest='datum', default=None,
+      help="Provide date in following format: '%Y%m%d-%H-%M-%S-1', by \
+      default it will be generated from the filename"),
+  )
+
+  def handle(self, *args, **options):
+    if options['verbosity'] > 1:
+      logger.setLevel(logging.DEBUG)
+    if len(args) == 0:
+      self.print_help(sys.argv[0],sys.argv[1])
+      raise CommandError("Not all arguments are provided")
+
+    # Please first the netxml and the gpsxml files and the rest
+    sorted_args = [x for x in args if "netxml" in x] +\
+     [x for x in args if "gpsxml" in x] +\
+     [x for x in args if "ns1" in x]
+    remainder = list(set(args) - set(sorted_args))
+    args = sorted_args + remainder
+    logger.debug("Parsing files in the following order: %s", args)
+
+    # Make sure the all exists at first
+    for filename in args:
+      if not os.path.isfile(filename):
+        raise CommandError("file '%s' does not exists" % filename)
+
+
+    def get_date(filename):
+      def process_date(datestr):
+        try:
+           # Kismet-20110805-15-37-30-1
+           return datetime.datetime.strptime(datestr,'%Y%m%d-%H-%M-%S-1')
+        except ValueError:
+          raise CommandError("Invalid date '%s'" % options['datum'])
+      if options['datum'] == None:
+         datestr = strip_file(os.path.basename(filename))
+         datum = process_date(datestr)
+      elif options['datum'] == 'now':
+         datum = datetime.datetime.now()
+      else:
+         datum = process_date(options['datum'])
+      return datum
+
+    def get_meetrondje(meetrondje):
+      # Meetrondje from filename if needed
+      if options['meetrondje'] == None:
+        meetrondje = strip_suffix(os.path.basename(filename))
+      else:
+        meetrondje = options['meetrondje']
+      return meetrondje
+
+    # Get Gheat Objects, pre-req
+    g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'],
+      email=options['email'])
+    a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
+
+    # Check if all files are valid
+    for filename in args:
+      logger.info("Processing '%s'" % filename)
+      mr, created = MeetRondje.objects.get_or_create(
+        datum=get_date(filename), naam=get_meetrondje(filename),
+        gebruiker=g, apparatuur=a)
+      if not created:
+        logger.error("Meetrondje '%s' already imported",  mr)
+        continue
+
+      counters = {
+        'ap_added' : 0, 'ap_total' : 0,
+        'ap_failed' : 0, 'ap_ignored' : 0,
+        'client_added' : 0, 'client_total' : 0,
+        'client_failed' : 0, 'client_ignored' : 0,
+        'meting_added' : 0, 'meting_total' : 0,
+        'meting_failed' : 0, 'meting_ignored' : 0
+        }
+      logger.info('Meetrondje: %s', mr)
+      fh = open_file(filename)
+      if 'ns1' in filename:
+        (counters, ap_pool, client_pool, meting_pool) = netstumbler.process_ns1(fh, counters)
+      elif 'gpsxml' in filename:
+        (counters, ap_pool, client_pool, meting_pool) = kismet.process_gpsxml(fh, counters)
+      elif 'netxml' in filename:
+        (counters, ap_pool, client_pool, meting_pool) = kismet.process_netxml(fh, counters)
+      elif 'ScanResult' in filename:
+        (counters, ap_pool, client_pool, meting_pool) = droidstumbler.process_csv(fh, counters)
+      else:
+        raise CommandError("file '%s' format not recognized" % filename)
+
+      if ap_pool:
+        counters = import_accespoints(ap_pool, counters)
+      if client_pool:
+        counters = import_clients(client_pool, counters)
+      if meting_pool:
+        counters = import_metingen(mr, meting_pool, counters)
+
+      logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
+      logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
+      logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
Index: src/django_gheat/gheat/management/commands/import_droidstumbler.py
===================================================================
--- src/django_gheat/gheat/management/commands/import_droidstumbler.py	(revision 9619)
+++ 	(revision )
@@ -1,174 +1,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Script for importing DroidStumbler .csv files, which takes the best value of
-# each measurement point.
-#
-# Original by Dennis Wagenaar <d.wagenaar@gmail.com>
-#
-# Rick van der Zwet <info@rickvanderzwet.nl>
-#
-from django.core.management import setup_environ
-from django.core.management.base import BaseCommand, CommandError
-from django.db import connection, transaction
-from django.db.utils import IntegrityError
-from _mysql_exceptions import OperationalError
-from optparse import OptionParser, make_option
-from gheat.models import *
-import csv
-import datetime
-import gzip
-import logging
-import os
-import sys
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-def bulk_sql(sql_table, sql_values):
-  if len(sql_values) == 0:
-    raise ValueError, "No data to import"
-
-  cursor = connection.cursor()
-  try:
-    # Make sure the special NULL is preserved
-    sql = "INSERT INTO %s VALUES %s" % (sql_table, ','.join(sql_values).replace("'NULL'",'NULL'))
-    count = cursor.execute(sql)
-    transaction.commit_unless_managed()
-  except OperationalError, e:
-    logger.error("%s - %s ", sql_table, sql_values[0])
-    raise
-  except IntegrityError, e:
-    logger.error("Unable to import - %s" %  e)
-    raise
-  return count
-
-organizations = dict(Organization.objects.all().values_list('name','id'))
-def get_organization_id_by_ssid(ssid):
-  name = Organization.get_name_by_ssid(ssid)
-  if not name:
-    return 'NULL'
-  else:
-    return int(organizations[name])
-
-def import_droidstumbler(filename, meetrondje):
-  """ Import all points, return tuple with summary"""
-
-  # Open file for reading
-  if filename.endswith('.gz'):
-    fh = gzip.open(filename,'rb')
-  else:
-    fh = open(filename,'rb')
-  csvfile = csv.reader(fh, delimiter=',')
-
-  #Various statistics
-  counters = {'meting_added' : 0, 'meting_total' : 0, 'ap_added' : 0, 'ap_total' : 0}
-
-  # Temponary holders
-  meting_pool = {}
-  ap_pool = {}
-  # Process file, preparing new access points and measurements
-  for row in csvfile:
-    try:
-      epoch, msg_type, lat, lon, accuracy, ssid, bssid, level, frequency, capabilities = row
-      bssid = bssid.upper()
-    except ValueError:
-      # Known error, please ignore
-      if row[1] == 'gps' and len(row) == 12: continue
-      logger.error("Unable to parse line:%i '%s'" % (csvfile.line_num, row))
-      continue
-    if msg_type == "data" and lat and lon:
-      counters['meting_total'] += 1
-      if not ap_pool.has_key(bssid):
-        encryption = 'WPA' in capabilities or 'WEP' in capabilities
-        ap_pool[bssid] = (ssid, encryption)
-
-      # We store the best value found
-      key = (bssid, lat, lon)
-      signaal=(100 + int(level))
-      if meting_pool.has_key(key):
-        meting_pool[key] = max(meting_pool[key], signaal)
-      else:
-        meting_pool[key] = signaal
-
-
-  # Determine which entries we need to add
-  counters['ap_total'] = len(ap_pool)
-  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).values_list('mac', flat=True)
-  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      ssid, encryption = ap_pool[bssid]
-      # Special trick in SSID ts avoid escaping in later stage
-      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,get_organization_id_by_ssid(ssid)))
-      sql_values.append(item)
-    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`, `encryptie`, `organization_id`)',sql_values)
-
-  # Build mapping for meting import
-  mac2id = {}
-  for mac,id in Accespoint.objects.filter(mac__in=ap_pool.keys()).values_list('mac','id'):
-    mac2id[mac] = int(id)
-
-  sql_values = []
-  for (bssid,lat,lon),signal in meting_pool.iteritems():
-    item = str((int(meetrondje.id),mac2id[bssid],float(lat),float(lon),int(signaal)))
-    sql_values.append(item)
-
-  # Bulk Import data if possible
-  if sql_values:
-    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`, `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
-  return counters
-
-
-class Command(BaseCommand):
-  args = '<csvfile>[.gz] [csvfile2[.gz] [csvfile3[.gz] ...] '
-  option_list = BaseCommand.option_list + (
-    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
-    make_option('-g', '--gebruiker', dest='gebruiker', default=os.environ['USER']),
-    make_option('-e', '--email', dest='email', default=os.environ['USER'] + '@example.org'),
-    make_option('-k', '--kaart', dest='kaart', default='onbekend', help="Kaart gebruikt"),
-    make_option('-d', '--datum', dest='datum', default=None, help="Provide date  \
-      in following format: %Y-%m-%d-%H%M%S, by default it will be generated from \
-      the filename"),
-  )
-
-  def handle(self, *args, **options):
-    if len(args) == 0:
-      self.print_help(sys.argv[0],sys.argv[1])
-      raise CommandError("Not all arguments are provided")
-
-    for csv_file in args:
-      # Make sure to check files before we going to do importing at all
-      if not os.path.isfile(csv_file):
-        raise CommandError("csv file '%s' does not exists" % csv_file)
-      logger.info("Processing '%s'" % csv_file)
-
-      # Meetrondje from filename if needed
-      if options['meetrondje'] == None:
-        meetrondje = os.path.basename(csv_file).rstrip('.gz').rstrip('.csv') 
-      else:
-        meetrondje = options['meetrondje']
-      # Date from filename if needed
-      if options['datum'] == None:
-         datum = os.path.basename(csv_file).lstrip('ScanResult-').rstrip('.csv.gz')
-      else:
-         datum = options['datum']
-      try:
-         datum = datetime.datetime.strptime(datum,'%Y-%m-%d-%H%M%S')
-      except ValueError:
-        raise CommandError("Invalid date '%s'" % options['datum'])
-
-      # Create meetrondje object
-      g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'] , email=options['email'])
-      a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
-      mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
-      logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
-      if not created:
-        logger.error("Meetrondje '%s' already imported" % mr)
-        sys.exit(1)
-      counters = import_droidstumbler(csv_file,mr)
-      logger.info("summary accespoints: added:%(ap_added)-6s processed:%(ap_total)-6s" % counters)
-      logger.info("summary metingen   : added:%(meting_added)-6s processed:%(meting_total)-6s" % counters)
Index: src/django_gheat/gheat/management/commands/import_kismet.py
===================================================================
--- src/django_gheat/gheat/management/commands/import_kismet.py	(revision 9619)
+++ 	(revision )
@@ -1,224 +1,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Script for importing .gpsxml and .netxml files (Kismet output)
-#
-# Rick van der Zwet <info@rickvanderzwet.nl>
-#
-from django.core.management.base import BaseCommand,CommandError
-from django.db.utils import IntegrityError
-from optparse import OptionParser, make_option
-from gheat.models import *
-from lxml import etree
-import datetime
-import gzip
-import os
-import sys
-import logging
-
-from collections import defaultdict
-
-from import_droidstumbler import bulk_sql,get_organization_id_by_ssid
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
-# Open files for reading
-def open_file(file):
- if file.endswith('.gz'):
-   return gzip.open(file,'rb')
- else:
-  return open(file,'rb')
-
-
-
-def import_kismet_netxml(netxml_file):
-  netxml_doc = etree.parse(open_file(netxml_file))
-
-  counters = { 'ap_added' : 0, 'ap_total' : 0, 'ap_failed' : 0, 'ap_ignored' : 0,
-               'client_added' : 0, 'client_total' : 0, 'client_failed' : 0, 'client_ignored' : 0}
-
-  # Prepare new accespoints and measurements
-  wnetworks = netxml_doc.findall('wireless-network')
-
-  # Temponary holders
-  ap_pool = {}
-  client_pool = {}
-
-  # Create all accesspoints and for caching validation purposes store them
-  # locally as well
-  for wnetwork in wnetworks:
-    bssid = wnetwork.find('BSSID').text
-    ap_type = wnetwork.attrib['type']
-    # Only access points and clients (for ignore listings)
-    if ap_type in ['infrastructure', 'data']:
-      counters['ap_total'] += 1
-      encryption = (wnetwork.find('SSID/encryption') != None)
-      ssid_node = wnetwork.find('SSID/essid[@cloaked="false"]')
-      ssid = ssid_node.text if ssid_node != None else 'hidden'
-
-      ap_pool[bssid] = (ssid, encryption)
-    elif ap_type in ['probe', 'ad-hoc']:
-      counters['client_total'] += 1
-      client_pool[bssid] = True
-    else:
-      logger.error('Unknown type %s - %s',bssid, wnetwork.attrib['type'])
-
-
-  # Determine which Accespoints to add
-  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).values_list('mac', flat=True)
-  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      ssid, encryption = ap_pool[bssid]
-      # Special trick in SSID ts avoid escaping in later stage
-      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,get_organization_id_by_ssid(ssid)))
-      sql_values.append(item)
-    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`, `encryptie`, `organization_id`)',sql_values)
-
-  # Determine which Wireless Clients to add
-  bssid_list_present = WirelessClient.objects.filter(mac__in=client_pool.keys()).values_list('mac', flat=True)
-  bssid_list_insert = set(client_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      sql_values.append("('%s')" % bssid.upper())
-    counters['client_added'] = bulk_sql('gheat_wirelessclient (`mac`)',sql_values)
-
-  return counters
-
-
-
-def import_kismet_gpsxml(gpsxml_file, meetrondje):
-  gpsxml_doc = etree.parse(open_file(gpsxml_file))
-
-  #Various statistics
-  counters = {'meting_added' : 0, 'meting_total' : 0, 'meting_failed' : 0, 'meting_ignored' :0}
-
-  bssid_failed = defaultdict(int)
-
-  # Prepare new accespoints and measurements
-  points = gpsxml_doc.findall('gps-point')
-
-  # Temponary holders
-  meting_pool = defaultdict(list)
-
-  for point in points:
-    counters['meting_total'] += 1
-    #XXX: This needs to be either the 'bssid' or the 'source', 
-    #XXX: accesspoint from or too data.
-    bssid = point.attrib['bssid']
-    # XXX: Filter this in the beginning with XPath, but etree does not support
-    # that (yet).
-    if bssid in ['GP:SD:TR:AC:KL:OG','00:00:00:00:00:00']:
-      counters['meting_ignored'] =+ 1
-      continue
-    # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
-    try:
-      level = point.attrib['signal_dbm']
-    except KeyError:
-      logger.debug("Point '%s' does not have signal strengh" % point)
-      counters['meting_failed'] += 1
-      continue
-    # We store all values found, avg or max will be done later on
-    key = (bssid, point.attrib['lat'], point.attrib['lon'])
-    signaal=100 + int(level)
-    meting_pool[key].append(signaal)
-
-  bssid_list = [x[0] for x in meting_pool.keys()]
-  # Build mapping for meting import
-  mac2id = {}
-  for mac,id in Accespoint.objects.filter(mac__in=bssid_list).values_list('mac','id'):
-    mac2id[mac] = int(id)
-
-  clients = {}
-  for mac in WirelessClient.objects.filter(mac__in=bssid_list).values_list('mac',flat=True):
-    clients[mac] = True
-
-  sql_values = []
-  for (bssid,lat,lon),signals in meting_pool.iteritems():
-    if clients.has_key(bssid):
-      counters['meting_ignored'] += len(signals)
-    elif not mac2id.has_key(bssid):
-      counters['meting_failed'] += len(signals)
-      bssid_failed[bssid] += len(signals)
-    else:
-      item = str((int(meetrondje.id),mac2id[bssid],float(lat),float(lon),max(signals)))
-      sql_values.append(item)
-
-  for bssid,count in sorted(bssid_failed.items(),
-      key=lambda item: item[1], reverse=True):
-    logger.debug("Missing BSSID %s found %3s times", bssid, count)
-
-  if sql_values:
-    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`, `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
-  return counters
-
-
-class Command(BaseCommand):
-  args = '<gpsxml|netxml>[.gz] [gpsxml2[.gz]  gpsxml3[.gz] ...]'
-  option_list = BaseCommand.option_list + (
-    make_option('-k', '--kaart', dest='kaart', default='onbekend', help="Kaart gebruikt"),
-    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
-    make_option('-g', '--gebruiker', dest='gebruiker', default='username',help='Naam van de persoon die de meting uitgevoerd heeft'),
-    make_option('-e', '--email', dest='email', default='foo@bar.org',help='Email van de persoon die de meting uitgevoerd heeft'),
-    make_option('-d', '--datum', dest='datum', default=None, help="Provide date  \
-      in following format: '%Y%m%d-%H-%M-%S-1', by default it will be generated from \
-      the filename"),
-  )
-
-  def handle(self, *args, **options):
-    if len(args) == 0:
-      self.print_help(sys.argv[0],sys.argv[1])
-      raise CommandError("Not all arguments are provided")
-
-    # Please first the netxml and then the gpsxml files
-    sorted_args = [x for x in args if "netxml" in x] + [x for x in args if "gpsxml" in x]
-    remainder = list(set(args) - set(sorted_args))
-    args = sorted_args + remainder
-    logger.debug("Parsing files in the following order: %s", args)
-
-    for xml_file in args:
-      if not os.path.isfile(xml_file):
-        raise CommandError("xml file '%s' does not exists" % xml_file)
-
-    for xml_file in args:
-      logger.info("Processing '%s'" % xml_file)
-      if 'netxml' in xml_file:
-        counters = import_kismet_netxml(xml_file)
-        logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
-        logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
-      elif 'gpsxml' in xml_file:
-        if options['datum'] == None:
-           datum = os.path.basename(xml_file).lstrip('Kismet-').rstrip('.gz').rstrip('.gpsxml').rstrip('.netxml')
-        else:
-           datum = options['datum']
-        try:
-           # Kismet-20110805-15-37-30-1
-           datum = datetime.datetime.strptime(datum,'%Y%m%d-%H-%M-%S-1')
-        except ValueError:
-          raise CommandError("Invalid date '%s'" % options['datum'])
-
-        # Meetrondje from filename if needed
-        if options['meetrondje'] == None:
-          meetrondje = os.path.basename(xml_file).rstrip('.gz').rstrip('.gpsxml')
-        else:
-          meetrondje = options['meetrondje']
-
-        # Create meetrondje object
-        g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'] , email=options['email'])
-        a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
-        mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
-        logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
-        if not created:
-          logger.error("Meetrondje '%s' already imported" % mr)
-          continue
-        counters = import_kismet_gpsxml(xml_file, mr)
-        logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
-      else:
-        raise CommandError("xml file '%s' format not recognized" % xml_file)
Index: src/django_gheat/gheat/management/commands/import_netstumbler.py
===================================================================
--- src/django_gheat/gheat/management/commands/import_netstumbler.py	(revision 9619)
+++ src/django_gheat/gheat/management/commands/import_netstumbler.py	(revision 9623)
@@ -94,5 +94,5 @@
 
 
-def process_netstumber(filename):
+def process_ns1(filename):
   data = parse_netstumbler(open_file(filename))
 
Index: src/django_gheat/gheat/management/commands/kismet.py
===================================================================
--- src/django_gheat/gheat/management/commands/kismet.py	(revision 9623)
+++ src/django_gheat/gheat/management/commands/kismet.py	(revision 9623)
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Rick van der Zwet <info@rickvanderzwet.nl>
+#
+from lxml import etree
+import logging
+
+from collections import defaultdict
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+def process_netxml(fh,counters):
+  netxml_doc = etree.parse(fh)
+
+  # Prepare new accespoints and measurements
+  wnetworks = netxml_doc.findall('wireless-network')
+
+  # Temponary holders
+  ap_pool = {}
+  client_pool = {}
+
+  # Create all accesspoints and for caching validation purposes store them
+  # locally as well
+  for wnetwork in wnetworks:
+    bssid = wnetwork.find('BSSID').text
+    ap_type = wnetwork.attrib['type']
+    # Only access points and clients (for ignore listings)
+    if ap_type in ['infrastructure', 'data']:
+      counters['ap_total'] += 1
+      encryption = (wnetwork.find('SSID/encryption') != None)
+      ssid_node = wnetwork.find('SSID/essid[@cloaked="false"]')
+      ssid = ssid_node.text if ssid_node != None else 'hidden'
+
+      ap_pool[bssid] = (ssid, encryption)
+    elif ap_type in ['probe', 'ad-hoc']:
+      counters['client_total'] += 1
+      client_pool[bssid] = True
+    else:
+      logger.error('Unknown type %s - %s',bssid, wnetwork.attrib['type'])
+
+  return (counters, ap_pool, None, None)
+
+
+
+def process_gpsxml(fh,counters):
+  gpsxml_doc = etree.parse(fh)
+
+  bssid_failed = defaultdict(int)
+
+  # Prepare new accespoints and measurements
+  points = gpsxml_doc.findall('gps-point')
+
+  # Temponary holders
+  meting_pool = defaultdict(list)
+
+  for point in points:
+    counters['meting_total'] += 1
+    #XXX: This needs to be either the 'bssid' or the 'source', 
+    #XXX: accesspoint from or too data.
+    bssid = point.attrib['bssid']
+    # XXX: Filter this in the beginning with XPath, but etree does not support
+    # that (yet).
+    if bssid in ['GP:SD:TR:AC:KL:OG','00:00:00:00:00:00']:
+      counters['meting_ignored'] =+ 1
+      continue
+    # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
+    try:
+      level = point.attrib['signal_dbm']
+    except KeyError:
+      logger.debug("Point '%s' does not have signal strengh" % point)
+      counters['meting_failed'] += 1
+      continue
+    # We store all values found, avg or max will be done later on
+    key = (bssid, point.attrib['lat'], point.attrib['lon'])
+    signaal=100 + int(level)
+    meting_pool[key].append(signaal)
+  return (counters, None, None, meting_pool)
Index: src/django_gheat/gheat/management/commands/netstumbler.py
===================================================================
--- src/django_gheat/gheat/management/commands/netstumbler.py	(revision 9619)
+++ src/django_gheat/gheat/management/commands/netstumbler.py	(revision 9623)
@@ -8,5 +8,11 @@
 from struct import unpack
 
-def parse_netstumbler(fh):
+from collections import defaultdict
+import logging
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+def parse_ns1(fh):
   def get_int32(size=1):
     v = unpack('<' + 'i'*size,fh.read(4*size))
@@ -103,7 +109,41 @@
   return data
 
+def process_ns1(fh, counters):
+  data = parse_ns1(fh)
+
+
+  # Temponary holders
+  meting_pool = defaultdict(list)
+  ap_pool = {}
+
+  for ap in data['aps']:
+    # XXX: How is encryption coded?
+    encryption = False
+    ap_pool[ap['BSSID']]= (ap['SSID'], encryption)
+    for point in ap["measurements"]:
+     counters['meting_total'] += 1
+     if point['LocationSource'] == 0:
+       logger.debug("No GPS Coordinates found for BSSID %s @ %s", 
+         ap['BSSID'], point['Time'])
+       counters['meting_ignored'] += 1
+       continue
+     # We store all values found, avg or max will be done later on
+     key = (ap['BSSID'], point["Latitude"], point["Longitude"])
+
+     # Known measurement error
+     if point['Signal'] == -32767: continue
+
+     # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
+     signaal= 100 + point['Signal']
+     if signaal > 100 or signaal < 0:
+       logger.warning("Signal %s is not valid entry for BSSID %s @ %s",
+         point['Signal'], ap['BSSID'], point['Time'])
+       continue
+     meting_pool[key].append(signaal)
+
+  return (counters, ap_pool, None, meting_pool)
 if __name__ == '__main__':
   import sys
   import pprint
   pp = pprint.PrettyPrinter(indent=2)
-  pp.pprint(parse_netstumbler(open(sys.argv[1],'r')))
+  pp.pprint(parse_ns1(open(sys.argv[1],'r')))
Index: src/django_gheat/import_scan_data.sh
===================================================================
--- src/django_gheat/import_scan_data.sh	(revision 9619)
+++ src/django_gheat/import_scan_data.sh	(revision 9623)
@@ -22,5 +22,5 @@
 #Kismet netxml Imports
 FILES="`find $DATAROOT -name '*netxml*' | sort`"
-[ -n "$FILES" ] && $MANAGE import_kismet $ARGS $FILES
+[ -n "$FILES" ] && $MANAGE import_common $ARGS $FILES
 
 # Import all measure data
@@ -34,9 +34,13 @@
     #Kismet gpsxml Imports
     FILES="`find $DEVICEDIR -name '*gpsxml*'`"
-    [ -n "$FILES" ] && $MANAGE import_kismet $ARGS $FILES
+    [ -n "$FILES" ] && $MANAGE import_datafile $ARGS $FILES
 
     # DroidStumber imports
     FILES=`find $DEVICEDIR -name 'ScanResult-*'`
-    [ -n "$FILES" ] && $MANAGE import_droidstumbler $ARGS $FILES
+    [ -n "$FILES" ] && $MANAGE import_datafile $ARGS $FILES
+
+    # Kismet imports
+    FILES=`find $DEVICEDIR -name '*.ns1'`
+    [ -n "$FILES" ] && $MANAGE import_datafile $ARGS $FILES
   done
 done