Index: src/django_gheat/gheat/management/commands/dataimport.py
===================================================================
--- src/django_gheat/gheat/management/commands/dataimport.py	(revision 9623)
+++ 	(revision )
@@ -1,96 +1,0 @@
-#!/usr/bin/env python
-#
-###########################################
-#
-# Script for importing .netxml files (Kismet output)
-#
-# In theory, only the -f option is needed, but for overview's sake, please use the others aswell.
-# -f = location of the .netxml, e.g. '/home/test.netxml'
-# -m = name of the dataset, e.g. 'Walk in park' or 'Trip with boat'
-# -g = your name
-# -e = your email address
-#
-# (Run from project root)
-# ./manage.py dataimport -f <file location> -m <dataset name> -g <username> -e <email>
-#
-# Make sure your file has atleast the following elements:
-#
-# <wireless-network>
-#   <SSID>
-#     <encryption> </encryption>
-#     <essid cloaked=""> </essid>
-#   </SSID>
-#   <BSSID> </BSSID>
-#   <gps-info>
-#     <min-lat> </min-lat>
-#     <min-lon> </min-lon>
-#   </gps-info>
-# </wireless-network>
-#
-#
-# Dennis Wagenaar
-# d.wagenaar@gmail.com
-#
-###########################################
-
-from django.core.management import setup_environ
-from django.core.management.base import BaseCommand
-from optparse import OptionParser, make_option
-import settings
-setup_environ(settings)
-from gheat.models import *
-from lxml import etree
-import datetime
-
-def import_file(location, meetrondje, gebruiker, email):
-
-  datasource = etree.parse(location)
-  doc = datasource.findall('wireless-network')
-  kaart = datasource.find('card-source/card-source')
-  kaart = kaart.text
-  g, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
-  a, created = Apparatuur.objects.get_or_create(antenne='test' , kaart=kaart)
-  mr = MeetRondje.objects.create(datum=datetime.datetime.now() , naam=meetrondje , gebruiker=g , apparatuur=a)
-
-  for wnetwork in doc:
-    enc = wnetwork.find('SSID/encryption')
-    if enc != None: enc = True
-    else: enc = False
-    ssid = wnetwork.find('SSID/essid')
-    if ssid != None: ssid = ssid.text
-    else: ssid = 'hidden'
-    bssid = wnetwork.find('BSSID')
-    if bssid != None: bssid = bssid.text
-    min_sig = wnetwork.find('snr-info/min_signal_dbm')
-    if min_sig != None: min_sig = int(min_sig.text)
-    else: min_sig = (-99)
-    max_sig = wnetwork.find('snr-info/max_signal_dbm')
-    if max_sig != None: max_sig = int(max_sig.text)
-    else: max_sig= (-99)
-    lat = wnetwork.find('gps-info/min-lat')
-    if lat != None: lat = lat.text
-    lon = wnetwork.find('gps-info/min-lon')
-    if lon != None: lon = lon.text
-    if ssid is None: continue
-    
-    avg_sig = (min_sig+max_sig)/2
-    sig = (1.67*avg_sig)+166.67
-    if sig > 100: sig = 100
-    if sig < 0: sig = 0
-
-    print enc, ssid, bssid, lat, lon, sig
-
-    ap, created = Accespoint.objects.get_or_create(mac=bssid, ssid=ssid, encryptie=enc)
-    m = Meting.objects.create(meetrondje=mr, accespoint=ap, latitude=lat, longitude=lon, signaal=sig)
-
-
-class Command(BaseCommand):
-  option_list = BaseCommand.option_list + (
-    make_option('-f', '--location', dest='location', default='location'),
-    make_option('-m', '--meetrondje', dest='meetrondje', default='rondje'),
-    make_option('-g', '--gebruiker', dest='gebruiker', default='username'),
-    make_option('-e', '--email', dest='email', default='foo@bar.org'),
-    )
-
-  def handle(self, *args, **options):
-    import_file(options['location'],options['meetrondje'],options['gebruiker'],options['email'])
Index: src/django_gheat/gheat/management/commands/import_netstumbler.py
===================================================================
--- src/django_gheat/gheat/management/commands/import_netstumbler.py	(revision 9623)
+++ 	(revision )
@@ -1,217 +1,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Script for importing .ns1 files (Netstumber output)
-#
-# Rick van der Zwet <info@rickvanderzwet.nl>
-#
-from django.core.management.base import BaseCommand,CommandError
-from django.db.utils import IntegrityError
-from optparse import OptionParser, make_option
-from gheat.models import *
-from lxml import etree
-import datetime
-import gzip
-import os
-import sys
-import logging
-
-from collections import defaultdict
-
-from netstumbler import parse_netstumbler
-from import_droidstumbler import bulk_sql,get_organization_id_by_ssid
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
-# Open files for reading
-def open_file(file):
- if file.endswith('.gz'):
-   return gzip.open(file,'rb')
- else:
-  return open(file,'rb')
-
-
-
-def import_accespoints(ap_pool, counters):
-  # Determine which Accespoints to add
-  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).\
-    values_list('mac', flat=True)
-  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      ssid, encryption = ap_pool[bssid]
-      # Special trick in SSID ts avoid escaping in later stage
-      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,
-        get_organization_id_by_ssid(ssid)))
-      sql_values.append(item)
-    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`,\
-      `encryptie`, `organization_id`)',sql_values)
-  return counters
-
-
-
-def import_metingen(meetrondje, meting_pool, counters):
-  # Temponary holders
-  bssid_failed = defaultdict(int)
-
-  bssid_list = [x[0] for x in meting_pool.keys()]
-  # Build mapping for meting import
-  mac2id = {}
-  for mac,id in Accespoint.objects.filter(mac__in=bssid_list).\
-    values_list('mac','id'):
-    mac2id[mac] = int(id)
-
-  clients = {}
-  for mac in WirelessClient.objects.filter(mac__in=bssid_list).\
-    values_list('mac',flat=True):
-    clients[mac] = True
-
-  sql_values = []
-  for (bssid,lat,lon),signals in meting_pool.iteritems():
-    if clients.has_key(bssid):
-      counters['meting_ignored'] += len(signals)
-    elif not mac2id.has_key(bssid):
-      counters['meting_failed'] += len(signals)
-      bssid_failed[bssid] += len(signals)
-    else:
-      item = str((int(meetrondje.id),mac2id[bssid],float(lat),\
-        float(lon),max(signals)))
-      sql_values.append(item)
-
-  for bssid,count in sorted(bssid_failed.items(),
-      key=lambda item: item[1], reverse=True):
-    logger.debug("Missing BSSID %s found %3s times", bssid, count)
-
-  if sql_values:
-    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`,\
-      `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
-  return counters
-
-
-
-def process_ns1(filename):
-  data = parse_netstumbler(open_file(filename))
-
-  counters = {
-    'ap_added' : 0, 'ap_total' : 0,
-    'ap_failed' : 0, 'ap_ignored' : 0,
-    'client_added' : 0, 'client_total' : 0,
-    'client_failed' : 0, 'client_ignored' : 0,
-    'meting_added' : 0, 'meting_total' : 0,
-    'meting_failed' : 0, 'meting_ignored' : 0
-    }
-
-  # Temponary holders
-  meting_pool = defaultdict(list)
-  ap_pool = {}
-
-  for ap in data['aps']:
-    # XXX: How is encryption coded?
-    encryption = False
-    ap_pool[ap['BSSID']]= (ap['SSID'], encryption)
-    for point in ap["measurements"]:
-     counters['meting_total'] += 1
-     if point['LocationSource'] == 0:
-       logger.debug("No GPS Coordinates found for BSSID %s @ %s", 
-         ap['BSSID'], point['Time'])
-       counters['meting_ignored'] += 1
-       continue
-     # We store all values found, avg or max will be done later on
-     key = (ap['BSSID'], point["Latitude"], point["Longitude"])
-
-     # Known measurement error
-     if point['Signal'] == -32767: continue
-
-     # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
-     signaal= 100 + point['Signal']
-     if signaal > 100 or signaal < 0:
-       logger.warning("Signal %s is not valid entry for BSSID %s @ %s",
-         point['Signal'], ap['BSSID'], point['Time'])
-       continue
-     meting_pool[key].append(signaal)
-
-  return (counters, ap_pool, meting_pool)
-
-
-
-
-class Command(BaseCommand):
-  args = '<netstumber.ns1>[.gz] [netstumber2.ns1[.gz]  netstumber3.ns1[.gz] ...]'
-  option_list = BaseCommand.option_list + (
-    make_option('-k', '--kaart', dest='kaart', default='onbekend', 
-      help="Kaart gebruikt"),
-    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
-    make_option('-g', '--gebruiker', dest='gebruiker', default='username',
-      help='Naam van de persoon die de meting uitgevoerd heeft'),
-    make_option('-e', '--email', dest='email', default='foo@bar.org',
-      help='Email van de persoon die de meting uitgevoerd heeft'),
-    make_option('-d', '--datum', dest='datum', default=None,
-      help="Provide date in following format: '%Y%m%d-%H-%M-%S-1', by \
-      default it will be generated from the filename"),
-  )
-
-  def handle(self, *args, **options):
-    if options['verbosity'] > 1:
-      logger.setLevel(logging.DEBUG)
-    if len(args) == 0:
-      self.print_help(sys.argv[0],sys.argv[1])
-      raise CommandError("Not all arguments are provided")
-
-    # Please first the netxml and then the gpsxml files
-    sorted_args = [x for x in args if "ns1" in x]
-    remainder = list(set(args) - set(sorted_args))
-    args = sorted_args + remainder
-    logger.debug("Parsing files in the following order: %s", args)
-
-    for filename in args:
-      if not os.path.isfile(filename):
-        raise CommandError("file '%s' does not exists" % filename)
-
-    def process_date(datestr):
-      try:
-         # Kismet-20110805-15-37-30-1
-         return datetime.datetime.strptime(datestr,'%Y%m%d-%H-%M-%S-1')
-      except ValueError:
-        raise CommandError("Invalid date '%s'" % options['datum'])
-
-    for filename in args:
-      logger.info("Processing '%s'" % filename)
-      if 'ns1' in filename:
-        if options['datum'] == None:
-           datestr = os.path.basename(filename).lstrip('Kismet-').\
-             rstrip('.gz').rstrip('.gpsxml').rstrip('.netxml').rstrip('.ns1')
-           datum = process_date(datestr)
-        elif options['datum'] == 'now':
-           datum = datetime.datetime.now()
-        else:
-           datum = process_date(options['datum'])
-
-        # Meetrondje from filename if needed
-        if options['meetrondje'] == None:
-          meetrondje = os.path.basename(filename).rstrip('.gz').rstrip('.ns1')
-        else:
-          meetrondje = options['meetrondje']
-
-        # Create meetrondje object
-        g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'],
-          email=options['email'])
-        a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
-        mr, created = MeetRondje.objects.get_or_create(datum=datum,
-          naam=meetrondje, gebruiker=g, apparatuur=a)
-        logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
-        if not created:
-          logger.error("Meetrondje '%s' already imported" % mr)
-          continue
-        (counters, ap_pool, meting_pool) = process_netstumber(filename)
-        counters = import_accespoints(ap_pool, counters)
-        counters = import_metingen(mr, meting_pool, counters)
-
-        logger.info(("summary metingen   : total:%(meting_total)-6s" +
-          "added:%(meting_added)-6s failed:%(meting_failed)-6s" +
-          "ignored:%(meting_ignored)-6s") % counters)
-      else:
-        raise CommandError("file '%s' format not recognized" % filename)