Changeset 9640 for src/django_gheat/gheat/management/commands/import_datafile.py

Timestamp:

Aug 30, 2011, 8:36:25 PM (13 years ago)

Author:

rick

Message:

Cast the import logic to a special place, cause we are going to need it soon
when we want to auto-import files from the WEB (admin interface or form).

File:

• src/django_gheat/gheat/management/commands/import_datafile.py (modified) (4 diffs)

src/django_gheat/gheat/management/commands/import_datafile.py

@@ -2 +2 @@
 # -*- coding: utf-8 -*-
 #
-# Script for importing various stumble files in a modular fasion:
-# - .ns1 (Netstumber)
-# - .gpsxml .netxml (Kismet)
-# - DroidStumbler-*.csv (DroidStumber)
+# Script for importing various stumble files.
 #
 # Rick van der Zwet <info@rickvanderzwet.nl>
 #
-from _mysql_exceptions import OperationalError
 from django.core.files import File
 from django.core.management.base import BaseCommand,CommandError
@@ -24 +20 @@
 from collections import defaultdict
 
-import netstumbler
-import kismet
-import droidstumbler
+from gheat.dataimport import import_file
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
-
-def open_file(file):
-  """ Open files for reading, unzip if needed """
-  if file.endswith('.gz'):
-    return gzip.open(file,'rb')
-  else:
-   return open(file,'rb')
 
 
@@ -73 +60 @@
   logger.error("Invalid date '%s', options: %s, using: now()", datestr, strptime_choices)
   return datetime.datetime.now()
-
-
-def bulk_sql(sql_table, sql_values):
-  """ Awefull hack to ensure we can do mass imports into the DJANGO databases"""
-  if len(sql_values) == 0:
-    raise ValueError, "No data to import"
-
-  cursor = connection.cursor()
-  try:
-    # Make sure the special NULL is preserved
-    sql = "INSERT INTO %s VALUES %s" % (sql_table, ','.join(sql_values).replace("'NULL'",'NULL'))
-    count = cursor.execute(sql)
-    transaction.commit_unless_managed()
-  except OperationalError, e:
-    logger.error("%s - %s ", sql_table, sql_values[0])
-    raise
-  except IntegrityError, e:
-    logger.error("Unable to import - %s" %  e)
-    raise
-  return count
-
-
-organizations = None
-def get_organization_id_by_ssid(ssid):
-  """ Wrapper to return Organization ID of a certain SSID Type
-  XXX: This should technically be casted into the Organization properly, but
-  XXX: that properly does not cache properly.
-  """
-  global organizations
-  if not organizations:
-   organizations = dict(Organization.objects.all().values_list('name','id'))
-  
-  name = Organization.get_name_by_ssid(ssid)
-  if not name:
-    return 'NULL'
-  else:
-    return int(organizations[name])
-
-
-
-def import_accespoints(ap_pool, counters):
-  # Determine which Accespoints to add
-  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).\
-    values_list('mac', flat=True)
-  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      ssid, encryption = ap_pool[bssid]
-      # Special trick in SSID ts avoid escaping in later stage
-      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,
-        get_organization_id_by_ssid(ssid)))
-      sql_values.append(item)
-    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`,\
-      `encryptie`, `organization_id`)',sql_values)
-  return counters
-
-
-
-def import_metingen(meetrondje, meting_pool, counters):
-  # Temponary holders
-  bssid_failed = defaultdict(int)
-
-  bssid_list = [x[0] for x in meting_pool.keys()]
-  # Build mapping for meting import
-  mac2id = {}
-  for mac,id in Accespoint.objects.filter(mac__in=bssid_list).\
-    values_list('mac','id'):
-    mac2id[mac] = int(id)
-
-  clients = {}
-  for mac in WirelessClient.objects.filter(mac__in=bssid_list).\
-    values_list('mac',flat=True):
-    clients[mac] = True
-
-  sql_values = []
-  for (bssid,lat,lon),signals in meting_pool.iteritems():
-    if clients.has_key(bssid):
-      counters['meting_ignored'] += len(signals)
-    elif not mac2id.has_key(bssid):
-      counters['meting_failed'] += len(signals)
-      bssid_failed[bssid] += len(signals)
-    else:
-      item = str((int(meetrondje.id),mac2id[bssid],float(lat),\
-        float(lon),max(signals)))
-      sql_values.append(item)
-
-  for bssid,count in sorted(bssid_failed.items(),
-      key=lambda item: item[1], reverse=True):
-    logger.debug("Missing BSSID %s found %3s times", bssid, count)
-
-  if sql_values:
-    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`,\
-      `accespoint_id`, `latitude`, `longitude`, `signaal`)',sql_values)
-  return counters
-
-
-def import_clients(client_pool, counters):
-  # Determine which Wireless Clients to add
-  bssid_list_present = WirelessClient.objects.filter(mac__in=client_pool.keys()).values_list('mac', flat=True)
-  bssid_list_insert = set(client_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      sql_values.append("('%s')" % bssid.upper())
-    counters['client_added'] = bulk_sql('gheat_wirelessclient (`mac`)',sql_values)
-
-  return counters
-
-
-
-def import_file(filename,meetrondje):
-  counters = {
-    'ap_added' : 0, 'ap_total' : 0,
-    'ap_failed' : 0, 'ap_ignored' : 0,
-    'client_added' : 0, 'client_total' : 0,
-    'client_failed' : 0, 'client_ignored' : 0,
-    'meting_added' : 0, 'meting_total' : 0,
-    'meting_failed' : 0, 'meting_ignored' : 0
-    }
-
-  fh = open_file(filename)
-  if 'ns1' in filename:
-    (counters, ap_pool, client_pool, meting_pool) = netstumbler.process_ns1(fh, counters)
-  elif 'gpsxml' in filename:
-    (counters, ap_pool, client_pool, meting_pool) = kismet.process_gpsxml(fh, counters)
-  elif 'netxml' in filename:
-    (counters, ap_pool, client_pool, meting_pool) = kismet.process_netxml(fh, counters)
-  elif 'ScanResult' in filename:
-    (counters, ap_pool, client_pool, meting_pool) = droidstumbler.process_csv(fh, counters)
-  else:
-    raise CommandError("file '%s' format not recognized" % filename)
-
-  if ap_pool:
-    counters = import_accespoints(ap_pool, counters)
-  if client_pool:
-    counters = import_clients(client_pool, counters)
-  if meting_pool:
-    counters = import_metingen(meetrondje, meting_pool, counters)
-
-  logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
-  logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
-  logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
-
 
 
@@ -297 +136 @@
       meetbestand.save()
 
-      import_file(filename,meetrondje)
+      counters = import_file(filename,meetrondje)
+      logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
+      logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
+      logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
 
+
+
+