Changeset 9623 for src/django_gheat/gheat/management/commands/kismet.py

Timestamp:

Aug 30, 2011, 6:13:57 PM (13 years ago)

Author:

rick

Message:

Merge and migrate all files into common files to get rid of all duplicate codes.

File:

• src/django_gheat/gheat/management/commands/kismet.py (moved) (moved from src/django_gheat/gheat/management/commands/import_kismet.py ) (3 diffs)

src/django_gheat/gheat/management/commands/kismet.py

@@ -2 +2 @@
 # -*- coding: utf-8 -*-
 #
-# Script for importing .gpsxml and .netxml files (Kismet output)
-#
 # Rick van der Zwet <info@rickvanderzwet.nl>
 #
-from django.core.management.base import BaseCommand,CommandError
-from django.db.utils import IntegrityError
-from optparse import OptionParser, make_option
-from gheat.models import *
 from lxml import etree
-import datetime
-import gzip
-import os
-import sys
 import logging
 
 from collections import defaultdict
 
-from import_droidstumbler import bulk_sql,get_organization_id_by_ssid
-
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
-# Open files for reading
-def open_file(file):
- if file.endswith('.gz'):
-   return gzip.open(file,'rb')
- else:
-  return open(file,'rb')
-
-
-
-def import_kismet_netxml(netxml_file):
-  netxml_doc = etree.parse(open_file(netxml_file))
-
-  counters = { 'ap_added' : 0, 'ap_total' : 0, 'ap_failed' : 0, 'ap_ignored' : 0,
-               'client_added' : 0, 'client_total' : 0, 'client_failed' : 0, 'client_ignored' : 0}
+def process_netxml(fh,counters):
+  netxml_doc = etree.parse(fh)
 
   # Prepare new accespoints and measurements
@@ -65 +41 @@
       logger.error('Unknown type %s - %s',bssid, wnetwork.attrib['type'])
 
-
-  # Determine which Accespoints to add
-  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).values_list('mac', flat=True)
-  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      ssid, encryption = ap_pool[bssid]
-      # Special trick in SSID ts avoid escaping in later stage
-      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,get_organization_id_by_ssid(ssid)))
-      sql_values.append(item)
-    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`, `encryptie`, `organization_id`)',sql_values)
-
-  # Determine which Wireless Clients to add
-  bssid_list_present = WirelessClient.objects.filter(mac__in=client_pool.keys()).values_list('mac', flat=True)
-  bssid_list_insert = set(client_pool.keys()) - set(bssid_list_present)
-
-  # Create a bulk import list and import
-  if bssid_list_insert:
-    sql_values = []
-    for bssid in bssid_list_insert:
-      sql_values.append("('%s')" % bssid.upper())
-    counters['client_added'] = bulk_sql('gheat_wirelessclient (`mac`)',sql_values)
-
-  return counters
+  return (counters, ap_pool, None, None)
 
 
 
-def import_kismet_gpsxml(gpsxml_file, meetrondje):
-  gpsxml_doc = etree.parse(open_file(gpsxml_file))
-
-  #Various statistics
-  counters = {'meting_added' : 0, 'meting_total' : 0, 'meting_failed' : 0, 'meting_ignored' :0}
+def process_gpsxml(fh,counters):
+  gpsxml_doc = etree.parse(fh)
 
   bssid_failed = defaultdict(int)
@@ -130 +77 @@
     signaal=100 + int(level)
     meting_pool[key].append(signaal)
-
-  bssid_list = [x[0] for x in meting_pool.keys()]
-  # Build mapping for meting import
-  mac2id = {}
-  for mac,id in Accespoint.objects.filter(mac__in=bssid_list).values_list('mac','id'):
-    mac2id[mac] = int(id)
-
-  clients = {}
-  for mac in WirelessClient.objects.filter(mac__in=bssid_list).values_list('mac',flat=True):
-    clients[mac] = True
-
-  sql_values = []
-  for (bssid,lat,lon),signals in meting_pool.iteritems():
-    if clients.has_key(bssid):
-      counters['meting_ignored'] += len(signals)
-    elif not mac2id.has_key(bssid):
-      counters['meting_failed'] += len(signals)
-      bssid_failed[bssid] += len(signals)
-    else:
-      item = str((int(meetrondje.id),mac2id[bssid],float(lat),float(lon),max(signals)))
-      sql_values.append(item)
-
-  for bssid,count in sorted(bssid_failed.items(),
-      key=lambda item: item[1], reverse=True):
-    logger.debug("Missing BSSID %s found %3s times", bssid, count)
-
-  if sql_values:
-    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`, `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
-  return counters
-
-
-class Command(BaseCommand):
-  args = '<gpsxml|netxml>[.gz] [gpsxml2[.gz]  gpsxml3[.gz] ...]'
-  option_list = BaseCommand.option_list + (
-    make_option('-k', '--kaart', dest='kaart', default='onbekend', help="Kaart gebruikt"),
-    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
-    make_option('-g', '--gebruiker', dest='gebruiker', default='username',help='Naam van de persoon die de meting uitgevoerd heeft'),
-    make_option('-e', '--email', dest='email', default='foo@bar.org',help='Email van de persoon die de meting uitgevoerd heeft'),
-    make_option('-d', '--datum', dest='datum', default=None, help="Provide date  \
-      in following format: '%Y%m%d-%H-%M-%S-1', by default it will be generated from \
-      the filename"),
-  )
-
-  def handle(self, *args, **options):
-    if len(args) == 0:
-      self.print_help(sys.argv[0],sys.argv[1])
-      raise CommandError("Not all arguments are provided")
-
-    # Please first the netxml and then the gpsxml files
-    sorted_args = [x for x in args if "netxml" in x] + [x for x in args if "gpsxml" in x]
-    remainder = list(set(args) - set(sorted_args))
-    args = sorted_args + remainder
-    logger.debug("Parsing files in the following order: %s", args)
-
-    for xml_file in args:
-      if not os.path.isfile(xml_file):
-        raise CommandError("xml file '%s' does not exists" % xml_file)
-
-    for xml_file in args:
-      logger.info("Processing '%s'" % xml_file)
-      if 'netxml' in xml_file:
-        counters = import_kismet_netxml(xml_file)
-        logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
-        logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
-      elif 'gpsxml' in xml_file:
-        if options['datum'] == None:
-           datum = os.path.basename(xml_file).lstrip('Kismet-').rstrip('.gz').rstrip('.gpsxml').rstrip('.netxml')
-        else:
-           datum = options['datum']
-        try:
-           # Kismet-20110805-15-37-30-1
-           datum = datetime.datetime.strptime(datum,'%Y%m%d-%H-%M-%S-1')
-        except ValueError:
-          raise CommandError("Invalid date '%s'" % options['datum'])
-
-        # Meetrondje from filename if needed
-        if options['meetrondje'] == None:
-          meetrondje = os.path.basename(xml_file).rstrip('.gz').rstrip('.gpsxml')
-        else:
-          meetrondje = options['meetrondje']
-
-        # Create meetrondje object
-        g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'] , email=options['email'])
-        a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
-        mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
-        logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
-        if not created:
-          logger.error("Meetrondje '%s' already imported" % mr)
-          continue
-        counters = import_kismet_gpsxml(xml_file, mr)
-        logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
-      else:
-        raise CommandError("xml file '%s' format not recognized" % xml_file)
+  return (counters, None, None, meting_pool)