Index: src/django_gheat/gheat/management/commands/droidstumbler.py
===================================================================
--- src/django_gheat/gheat/management/commands/droidstumbler.py	(revision 9639)
+++ 	(revision )
@@ -1,42 +1,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Rick van der Zwet <info@rickvanderzwet.nl>
-#
-import csv
-import logging
-
-from collections import defaultdict
-
-logger = logging.getLogger(__name__)
-
-def process_csv(fh, counters):
-  """ Import all points, return tuple with summary"""
-
-  # Temponary holders
-  meting_pool = defaultdict(list)
-  ap_pool = {}
-
-  csvfile = csv.reader(fh, delimiter=',')
-  # Process file, preparing new access points and measurements
-  for row in csvfile:
-    try:
-      epoch, msg_type, lat, lon, accuracy, ssid, bssid, level, frequency, capabilities = row
-      bssid = bssid.upper()
-    except ValueError:
-      # Known error, please ignore
-      if row[1] == 'gps' and len(row) == 12: continue
-      logger.error("Unable to parse line:%i '%s'" % (csvfile.line_num, row))
-      continue
-    if msg_type == "data" and lat and lon:
-      counters['meting_total'] += 1
-      if not ap_pool.has_key(bssid):
-        encryption = 'WPA' in capabilities or 'WEP' in capabilities
-        ap_pool[bssid] = (ssid, encryption)
-
-      # We store the best value found
-      key = (bssid, lat, lon)
-      signaal=(100 + int(level))
-      meting_pool[key].append(signaal)
-
-  return (counters, ap_pool, None, meting_pool)
Index: src/django_gheat/gheat/management/commands/import_datafile.py
===================================================================
--- src/django_gheat/gheat/management/commands/import_datafile.py	(revision 9639)
+++ src/django_gheat/gheat/management/commands/import_datafile.py	(revision 9640)
@@ -2,12 +2,8 @@
 # -*- coding: utf-8 -*-
 #
-# Script for importing various stumble files in a modular fasion:
-# - .ns1 (Netstumber)
-# - .gpsxml .netxml (Kismet)
-# - DroidStumbler-*.csv (DroidStumber)
+# Script for importing various stumble files.
 #
 # Rick van der Zwet <info@rickvanderzwet.nl>
 #
-from _mysql_exceptions import OperationalError
 from django.core.files import File
 from django.core.management.base import BaseCommand,CommandError
@@ -24,17 +20,8 @@
 from collections import defaultdict
 
-import netstumbler
-import kismet
-import droidstumbler
+from gheat.dataimport import import_file
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
-
-def open_file(file):
-  """ Open files for reading, unzip if needed """
-  if file.endswith('.gz'):
-    return gzip.open(file,'rb')
-  else:
-   return open(file,'rb')
 
 
@@ -73,152 +60,4 @@
   logger.error("Invalid date '%s', options: %s, using: now()", datestr, strptime_choices)
   return datetime.datetime.now()
-
-
-def bulk_sql(sql_table, sql_values):
-  """ Awefull hack to ensure we can do mass imports into the DJANGO databases"""
-  if len(sql_values) == 0:
-    raise ValueError, "No data to import"
-
-  cursor = connection.cursor()
-  try:
-    # Make sure the special NULL is preserved
-    sql = "INSERT INTO %s VALUES %s" % (sql_table, ','.join(sql_values).replace("'NULL'",'NULL'))
-    count = cursor.execute(sql)
-    transaction.commit_unless_managed()
-  except OperationalError, e:
-    logger.error("%s - %s ", sql_table, sql_values[0])
-    raise
-  except IntegrityError, e:
-    logger.error("Unable to import - %s" %  e)
-    raise
-  return count
-
-
-organizations = None
-def get_organization_id_by_ssid(ssid):
-  """ Wrapper to return Organization ID of a certain SSID Type
-  XXX: This should technically be casted into the Organization properly, but
-  XXX: that properly does not cache properly.
-  """
-  global organizations
-  if not organizations:
-   organizations = dict(Organization.objects.all().values_list('name','id'))
-  
-  name = Organization.get_name_by_ssid(ssid)
-  if not name:
-    return 'NULL'
-  else:
-    return int(organizations[name])
-
-
-
-def import_accespoints(ap_pool, counters):
-  # Determine which Accespoints to add
-  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).\
-    values_list('mac', flat=True)
-  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      ssid, encryption = ap_pool[bssid]
-      # Special trick in SSID ts avoid escaping in later stage
-      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,
-        get_organization_id_by_ssid(ssid)))
-      sql_values.append(item)
-    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`,\
-      `encryptie`, `organization_id`)',sql_values)
-  return counters
-
-
-
-def import_metingen(meetrondje, meting_pool, counters):
-  # Temponary holders
-  bssid_failed = defaultdict(int)
-
-  bssid_list = [x[0] for x in meting_pool.keys()]
-  # Build mapping for meting import
-  mac2id = {}
-  for mac,id in Accespoint.objects.filter(mac__in=bssid_list).\
-    values_list('mac','id'):
-    mac2id[mac] = int(id)
-
-  clients = {}
-  for mac in WirelessClient.objects.filter(mac__in=bssid_list).\
-    values_list('mac',flat=True):
-    clients[mac] = True
-
-  sql_values = []
-  for (bssid,lat,lon),signals in meting_pool.iteritems():
-    if clients.has_key(bssid):
-      counters['meting_ignored'] += len(signals)
-    elif not mac2id.has_key(bssid):
-      counters['meting_failed'] += len(signals)
-      bssid_failed[bssid] += len(signals)
-    else:
-      item = str((int(meetrondje.id),mac2id[bssid],float(lat),\
-        float(lon),max(signals)))
-      sql_values.append(item)
-
-  for bssid,count in sorted(bssid_failed.items(),
-      key=lambda item: item[1], reverse=True):
-    logger.debug("Missing BSSID %s found %3s times", bssid, count)
-
-  if sql_values:
-    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`,\
-      `accespoint_id`, `latitude`, `longitude`, `signaal`)',sql_values)
-  return counters
-
-
-def import_clients(client_pool, counters):
-  # Determine which Wireless Clients to add
-  bssid_list_present = WirelessClient.objects.filter(mac__in=client_pool.keys()).values_list('mac', flat=True)
-  bssid_list_insert = set(client_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      sql_values.append("('%s')" % bssid.upper())
-    counters['client_added'] = bulk_sql('gheat_wirelessclient (`mac`)',sql_values)
-
-  return counters
-
-
-
-def import_file(filename,meetrondje):
-  counters = {
-    'ap_added' : 0, 'ap_total' : 0,
-    'ap_failed' : 0, 'ap_ignored' : 0,
-    'client_added' : 0, 'client_total' : 0,
-    'client_failed' : 0, 'client_ignored' : 0,
-    'meting_added' : 0, 'meting_total' : 0,
-    'meting_failed' : 0, 'meting_ignored' : 0
-    }
-
-  fh = open_file(filename)
-  if 'ns1' in filename:
-    (counters, ap_pool, client_pool, meting_pool) = netstumbler.process_ns1(fh, counters)
-  elif 'gpsxml' in filename:
-    (counters, ap_pool, client_pool, meting_pool) = kismet.process_gpsxml(fh, counters)
-  elif 'netxml' in filename:
-    (counters, ap_pool, client_pool, meting_pool) = kismet.process_netxml(fh, counters)
-  elif 'ScanResult' in filename:
-    (counters, ap_pool, client_pool, meting_pool) = droidstumbler.process_csv(fh, counters)
-  else:
-    raise CommandError("file '%s' format not recognized" % filename)
-
-  if ap_pool:
-    counters = import_accespoints(ap_pool, counters)
-  if client_pool:
-    counters = import_clients(client_pool, counters)
-  if meting_pool:
-    counters = import_metingen(meetrondje, meting_pool, counters)
-
-  logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
-  logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
-  logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
-
 
 
@@ -297,4 +136,10 @@
       meetbestand.save()
 
-      import_file(filename,meetrondje)
+      counters = import_file(filename,meetrondje)
+      logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
+      logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
+      logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
 
+
+
+
Index: src/django_gheat/gheat/management/commands/kismet.py
===================================================================
--- src/django_gheat/gheat/management/commands/kismet.py	(revision 9639)
+++ 	(revision )
@@ -1,78 +1,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Rick van der Zwet <info@rickvanderzwet.nl>
-#
-from lxml import etree
-import logging
-
-from collections import defaultdict
-
-logger = logging.getLogger(__name__)
-
-def process_netxml(fh,counters):
-  netxml_doc = etree.parse(fh)
-
-  # Prepare new accespoints and measurements
-  wnetworks = netxml_doc.findall('wireless-network')
-
-  # Temponary holders
-  ap_pool = {}
-  client_pool = {}
-
-  # Create all accesspoints and for caching validation purposes store them
-  # locally as well
-  for wnetwork in wnetworks:
-    bssid = wnetwork.find('BSSID').text
-    ap_type = wnetwork.attrib['type']
-    # Only access points and clients (for ignore listings)
-    if ap_type in ['infrastructure', 'data']:
-      counters['ap_total'] += 1
-      encryption = (wnetwork.find('SSID/encryption') != None)
-      ssid_node = wnetwork.find('SSID/essid[@cloaked="false"]')
-      ssid = ssid_node.text if ssid_node != None else 'hidden'
-
-      ap_pool[bssid] = (ssid, encryption)
-    elif ap_type in ['probe', 'ad-hoc']:
-      counters['client_total'] += 1
-      client_pool[bssid] = True
-    else:
-      logger.error('Unknown type %s - %s',bssid, wnetwork.attrib['type'])
-
-  return (counters, ap_pool, None, None)
-
-
-
-def process_gpsxml(fh,counters):
-  gpsxml_doc = etree.parse(fh)
-
-  bssid_failed = defaultdict(int)
-
-  # Prepare new accespoints and measurements
-  points = gpsxml_doc.findall('gps-point')
-
-  # Temponary holders
-  meting_pool = defaultdict(list)
-
-  for point in points:
-    counters['meting_total'] += 1
-    #XXX: This needs to be either the 'bssid' or the 'source', 
-    #XXX: accesspoint from or too data.
-    bssid = point.attrib['bssid']
-    # XXX: Filter this in the beginning with XPath, but etree does not support
-    # that (yet).
-    if bssid in ['GP:SD:TR:AC:KL:OG','00:00:00:00:00:00']:
-      counters['meting_ignored'] =+ 1
-      continue
-    # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
-    try:
-      level = point.attrib['signal_dbm']
-    except KeyError:
-      logger.debug("Point '%s' does not have signal strengh" % point)
-      counters['meting_failed'] += 1
-      continue
-    # We store all values found, avg or max will be done later on
-    key = (bssid, point.attrib['lat'], point.attrib['lon'])
-    signaal=100 + int(level)
-    meting_pool[key].append(signaal)
-  return (counters, None, None, meting_pool)
Index: src/django_gheat/gheat/management/commands/netstumbler.py
===================================================================
--- src/django_gheat/gheat/management/commands/netstumbler.py	(revision 9639)
+++ 	(revision )
@@ -1,148 +1,0 @@
-#!/usr/bin/env python
-#
-# Reading NS1 files - http://www.stumbler.net/ns1files.html
-#
-# Rick van der Zwet <info@rickvanderzwet.nl>
-#
-import datetime
-from struct import unpack
-
-from collections import defaultdict
-import logging
-
-logger = logging.getLogger(__name__)
-
-def parse_ns1(fh):
-  def get_int32(size=1):
-    v = unpack('<' + 'i'*size,fh.read(4*size))
-    return v[0] if size == 1 else v
-  
-  def get_uint32(size=1):
-    v = unpack('<' + 'I'*size,fh.read(4*size))
-    return v[0] if size == 1 else v
-  
-  def get_uint64(size=1):
-    v = unpack('<' + 'Q'*size,fh.read(8*size))
-    return v[0] if size == 1 else v
-  
-  def get_uint8(size=1):
-    v = unpack('<' + 'B'*size,fh.read(1*size))
-    return v[0] if size == 1 else v
-  
-  def get_filetime():
-    _FILETIME_null_date = datetime.datetime(1601, 1, 1, 0, 0, 0)
-    ns = unpack('<Q',fh.read(8))[0] * 10
-    sec = ns / 10**8
-    d = datetime.timedelta(seconds=sec)
-    return _FILETIME_null_date + d
-  
-  def get_char(size):
-    return fh.read(size)
-  
-  def get_double(size=1):
-    v = unpack('<' + 'd'*size,fh.read(8*size))
-    return v[0] if size == 1 else v
-  
-  def get_mac():
-    return ':'.join(["%02X" % x for x in unpack('BBBBBB',fh.read(6))])
-
-  data = {}
-  
-  data["dwSignature"] = get_char(4)
-  data["dwFileVerunpack"] = get_uint32()
-  ApCount = get_uint32()
-  data["ApCount"] = ApCount
-  
-  data["aps"] = []
-  for a in range(0,ApCount):
-    ap = {}
-    SSIDLength = get_uint8()
-    ap["SSIDLength"] = SSIDLength
-    ap["SSID"] = get_char(SSIDLength)
-    ap["BSSID"] = get_mac()
-    ap["MaxSignal"] = get_int32()
-    ap["MinNoise"] = get_int32()
-    ap["MaxSNR"] = get_int32()
-    ap["Flags"] =  get_uint32()
-    ap["BeaconInterval"] = get_uint32()
-    ap["FirstSeen"] = get_filetime()
-    ap["LastSeen"] = get_filetime()
-    ap["BestLat"] = get_double()
-    ap["BestLong"] = get_double()
-    DataCount = get_uint32()
-    ap["DataCount"] = DataCount
-    ap["measurements"] = []
-    for c in range(0,DataCount):
-      ms = {}
-      ms["Time"] = get_filetime()
-      ms["Signal"] = get_int32()
-      ms["Noice"] = get_int32()
-      LocationSource = get_int32()
-      ms["LocationSource"] = LocationSource
-      if LocationSource == 1:
-        ms["Latitude"] = get_double()
-        ms["Longitude"] = get_double()
-        ms["Altitude"] = get_double()
-        ms["NumStats"] = get_uint32()
-        ms["Speed"] = get_double()
-        ms["Track"] = get_double()
-        ms["MagVariation"] = get_double()
-        ms["Hdop"] = get_double()
-      ap["measurements"].append(ms)
-    NameLength = get_uint8()
-    ap["NameLength"] = NameLength
-    ap["Name"] = get_char(NameLength)
-    ap["Channels"] = get_uint64()
-    ap["LastChannel"] = get_uint32()
-    ap["IPAddress"] = get_uint32()
-    ap["MinSignal"] = get_int32()
-    ap["MaxSignal"] = get_int32()
-    ap["DataRate"] = get_uint32()
-    ap["IPSubnet"] = get_uint32()
-    ap["IPMask"] = get_uint32()
-    ap["ApFlags"] = get_uint32()
-    IELength = get_uint32()
-    ap["IELength"] = IELength
-    ap["InformationElements"] = get_uint8(IELength)
-    data["aps"].append(ap)
-  return data
-
-def process_ns1(fh, counters):
-  data = parse_ns1(fh)
-
-
-  # Temponary holders
-  meting_pool = defaultdict(list)
-  ap_pool = {}
-
-  for ap in data['aps']:
-    # XXX: How is encryption coded?
-    encryption = False
-    ap_pool[ap['BSSID']]= (ap['SSID'], encryption)
-    for point in ap["measurements"]:
-     counters['meting_total'] += 1
-     if point['LocationSource'] == 0:
-       logger.debug("No GPS Coordinates found for BSSID %s @ %s", 
-         ap['BSSID'], point['Time'])
-       counters['meting_ignored'] += 1
-       continue
-     # We store all values found, avg or max will be done later on
-     key = (ap['BSSID'], point["Latitude"], point["Longitude"])
-
-     # Known measurement error
-     if point['Signal'] == -32767: continue
-
-     # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
-     signaal= 100 + point['Signal']
-     if signaal > 100 or signaal < 0:
-       logger.warning("Signal %s is not valid entry for BSSID %s @ %s",
-         point['Signal'], ap['BSSID'], point['Time'])
-       continue
-     meting_pool[key].append(signaal)
-
-  return (counters, ap_pool, None, meting_pool)
-if __name__ == '__main__':
-  import sys
-  import pprint
-  pp = pprint.PrettyPrinter(indent=2)
-  pp.pprint(parse_ns1(open(sys.argv[1],'r')))
