Index: src/django_gheat/gheat/management/commands/import_netstumbler.py
===================================================================
--- src/django_gheat/gheat/management/commands/import_netstumbler.py	(revision 9619)
+++ src/django_gheat/gheat/management/commands/import_netstumbler.py	(revision 9619)
@@ -0,0 +1,217 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Script for importing .ns1 files (Netstumber output)
+#
+# Rick van der Zwet <info@rickvanderzwet.nl>
+#
+from django.core.management.base import BaseCommand,CommandError
+from django.db.utils import IntegrityError
+from optparse import OptionParser, make_option
+from gheat.models import *
+from lxml import etree
+import datetime
+import gzip
+import os
+import sys
+import logging
+
+from collections import defaultdict
+
+from netstumbler import parse_netstumbler
+from import_droidstumbler import bulk_sql,get_organization_id_by_ssid
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+# Open files for reading
+def open_file(file):
+ if file.endswith('.gz'):
+   return gzip.open(file,'rb')
+ else:
+  return open(file,'rb')
+
+
+
+def import_accespoints(ap_pool, counters):
+  # Determine which Accespoints to add
+  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).\
+    values_list('mac', flat=True)
+  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
+
+  # Create a bulk import list and import
+  if bssid_list_insert:
+    sql_values = []
+    for bssid in bssid_list_insert:
+      ssid, encryption = ap_pool[bssid]
+      # Special trick in SSID ts avoid escaping in later stage
+      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,
+        get_organization_id_by_ssid(ssid)))
+      sql_values.append(item)
+    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`,\
+      `encryptie`, `organization_id`)',sql_values)
+  return counters
+
+
+
+def import_metingen(meetrondje, meting_pool, counters):
+  # Temponary holders
+  bssid_failed = defaultdict(int)
+
+  bssid_list = [x[0] for x in meting_pool.keys()]
+  # Build mapping for meting import
+  mac2id = {}
+  for mac,id in Accespoint.objects.filter(mac__in=bssid_list).\
+    values_list('mac','id'):
+    mac2id[mac] = int(id)
+
+  clients = {}
+  for mac in WirelessClient.objects.filter(mac__in=bssid_list).\
+    values_list('mac',flat=True):
+    clients[mac] = True
+
+  sql_values = []
+  for (bssid,lat,lon),signals in meting_pool.iteritems():
+    if clients.has_key(bssid):
+      counters['meting_ignored'] += len(signals)
+    elif not mac2id.has_key(bssid):
+      counters['meting_failed'] += len(signals)
+      bssid_failed[bssid] += len(signals)
+    else:
+      item = str((int(meetrondje.id),mac2id[bssid],float(lat),\
+        float(lon),max(signals)))
+      sql_values.append(item)
+
+  for bssid,count in sorted(bssid_failed.items(),
+      key=lambda item: item[1], reverse=True):
+    logger.debug("Missing BSSID %s found %3s times", bssid, count)
+
+  if sql_values:
+    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`,\
+      `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
+  return counters
+
+
+
+def process_netstumber(filename):
+  data = parse_netstumbler(open_file(filename))
+
+  counters = {
+    'ap_added' : 0, 'ap_total' : 0,
+    'ap_failed' : 0, 'ap_ignored' : 0,
+    'client_added' : 0, 'client_total' : 0,
+    'client_failed' : 0, 'client_ignored' : 0,
+    'meting_added' : 0, 'meting_total' : 0,
+    'meting_failed' : 0, 'meting_ignored' : 0
+    }
+
+  # Temponary holders
+  meting_pool = defaultdict(list)
+  ap_pool = {}
+
+  for ap in data['aps']:
+    # XXX: How is encryption coded?
+    encryption = False
+    ap_pool[ap['BSSID']]= (ap['SSID'], encryption)
+    for point in ap["measurements"]:
+     counters['meting_total'] += 1
+     if point['LocationSource'] == 0:
+       logger.debug("No GPS Coordinates found for BSSID %s @ %s", 
+         ap['BSSID'], point['Time'])
+       counters['meting_ignored'] += 1
+       continue
+     # We store all values found, avg or max will be done later on
+     key = (ap['BSSID'], point["Latitude"], point["Longitude"])
+
+     # Known measurement error
+     if point['Signal'] == -32767: continue
+
+     # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
+     signaal= 100 + point['Signal']
+     if signaal > 100 or signaal < 0:
+       logger.warning("Signal %s is not valid entry for BSSID %s @ %s",
+         point['Signal'], ap['BSSID'], point['Time'])
+       continue
+     meting_pool[key].append(signaal)
+
+  return (counters, ap_pool, meting_pool)
+
+
+
+
+class Command(BaseCommand):
+  args = '<netstumber.ns1>[.gz] [netstumber2.ns1[.gz]  netstumber3.ns1[.gz] ...]'
+  option_list = BaseCommand.option_list + (
+    make_option('-k', '--kaart', dest='kaart', default='onbekend', 
+      help="Kaart gebruikt"),
+    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
+    make_option('-g', '--gebruiker', dest='gebruiker', default='username',
+      help='Naam van de persoon die de meting uitgevoerd heeft'),
+    make_option('-e', '--email', dest='email', default='foo@bar.org',
+      help='Email van de persoon die de meting uitgevoerd heeft'),
+    make_option('-d', '--datum', dest='datum', default=None,
+      help="Provide date in following format: '%Y%m%d-%H-%M-%S-1', by \
+      default it will be generated from the filename"),
+  )
+
+  def handle(self, *args, **options):
+    if options['verbosity'] > 1:
+      logger.setLevel(logging.DEBUG)
+    if len(args) == 0:
+      self.print_help(sys.argv[0],sys.argv[1])
+      raise CommandError("Not all arguments are provided")
+
+    # Please first the netxml and then the gpsxml files
+    sorted_args = [x for x in args if "ns1" in x]
+    remainder = list(set(args) - set(sorted_args))
+    args = sorted_args + remainder
+    logger.debug("Parsing files in the following order: %s", args)
+
+    for filename in args:
+      if not os.path.isfile(filename):
+        raise CommandError("file '%s' does not exists" % filename)
+
+    def process_date(datestr):
+      try:
+         # Kismet-20110805-15-37-30-1
+         return datetime.datetime.strptime(datestr,'%Y%m%d-%H-%M-%S-1')
+      except ValueError:
+        raise CommandError("Invalid date '%s'" % options['datum'])
+
+    for filename in args:
+      logger.info("Processing '%s'" % filename)
+      if 'ns1' in filename:
+        if options['datum'] == None:
+           datestr = os.path.basename(filename).lstrip('Kismet-').\
+             rstrip('.gz').rstrip('.gpsxml').rstrip('.netxml').rstrip('.ns1')
+           datum = process_date(datestr)
+        elif options['datum'] == 'now':
+           datum = datetime.datetime.now()
+        else:
+           datum = process_date(options['datum'])
+
+        # Meetrondje from filename if needed
+        if options['meetrondje'] == None:
+          meetrondje = os.path.basename(filename).rstrip('.gz').rstrip('.ns1')
+        else:
+          meetrondje = options['meetrondje']
+
+        # Create meetrondje object
+        g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'],
+          email=options['email'])
+        a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
+        mr, created = MeetRondje.objects.get_or_create(datum=datum,
+          naam=meetrondje, gebruiker=g, apparatuur=a)
+        logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
+        if not created:
+          logger.error("Meetrondje '%s' already imported" % mr)
+          continue
+        (counters, ap_pool, meting_pool) = process_netstumber(filename)
+        counters = import_accespoints(ap_pool, counters)
+        counters = import_metingen(mr, meting_pool, counters)
+
+        logger.info(("summary metingen   : total:%(meting_total)-6s" +
+          "added:%(meting_added)-6s failed:%(meting_failed)-6s" +
+          "ignored:%(meting_ignored)-6s") % counters)
+      else:
+        raise CommandError("file '%s' format not recognized" % filename)