source: src/django_gheat/gheat/management/commands/import_kismet.py@ 9557

#!/usr/bin/env python
#
# Script for importing .gpsxml and .netxml files (Kismet output)
#

from django.core.management.base import BaseCommand,CommandError
from django.db.utils import IntegrityError
from optparse import OptionParser, make_option
from gheat.models import *
from lxml import etree
import datetime
import gzip
import os
import sys
import logging

def import_file(gpsxml_file, netxml_file, meetrondje, kaart, gebruiker, email):
  # TODO: Source source is variable entitity, based on mesurement
  kaart = 'deadcode'
  gebruiker, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
  apparatuur, created = Apparatuur.objects.get_or_create(kaart=kaart)
  # TODO: Date is set to import date, but should pick the date from the netxml file
  mr = MeetRondje.objects.create(datum=None,
    naam=meetrondje , gebruiker=gebruiker , apparatuur=apparatuur)
  if not created:
    logging.error("Meetrondje '%s' already imported" % mr)
    sys.exit(1)

  open_file = lambda file: gzip.open(file,'rb') if file.endswith('.gz') else open(file,'rb')
  gpsxml_doc = etree.parse(open_file(gpsxml_file))
  netxml_doc = etree.parse(open_file(netxml_file))

  points = gpsxml_doc.findall('gps-point')
  wnetworks = netxml_doc.findall('wireless-network')

  # Create all accesspoints and for caching validation purposes store them
  # locally as well
  ap_cache = {}
  ap_ignore = []
  print "#INFO: Going to import %s accesspoints" % len(wnetworks)
  for wnetwork in wnetworks:
    bssid = wnetwork.find('BSSID').text
    # Only store access points
    if wnetwork.attrib['type'] != "infrastructure":
      ap_ignore.append(bssid)
      continue

    enc = (wnetwork.find('SSID/encryption') != None)
    ssid_node = wnetwork.find('SSID/essid[@cloaked="false"]')
    ssid = ssid_node.text if ssid_node != None else 'hidden'

    ap, created = Accespoint.objects.get_or_create(mac=bssid, ssid=ssid, encryptie=enc)
    ap_cache[bssid] = ap

  count = 0
  #XXX: This is not effient at all, try to wrap it into a a bulk insert would
  # be much more effient as for example: http://djangosnippets.org/snippets/2362/
  print "#INFO: Going to import %s points" % len(points)
  for point in points:
    #XXX: This needs to be either the 'bssid' or the 'source', accesspoint from or too data.
    bssid = point.attrib['bssid']
    # XXX: Filter this in the beginning with XPath, but etree does not support that (yet).
    if bssid in ['GP:SD:TR:AC:KL:OG','00:00:00:00:00:00']:
      continue
    elif bssid in ap_ignore:
      continue
    elif not ap_cache.has_key(bssid):
      try:
        ap = Accespoint.objects.get(mac=bssid)
        ap_cache[bssid] = ap
      except Accespoint.DoesNotExist:
        print "#ERROR: Cannot found SSID for BSSID '%s'" % bssid
        continue

    # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
    try:
      signaal = 100 + int(point.attrib['signal_dbm'])
    except KeyError:
      print "#ERROR: Point '%s' does not have signal strengh" % point

    # TODO: This also saves semi-duplicates; multiple entries with the same values, except
    # the signal strength is different. Should get an AVG or something.
    try:
      meting= Meting.objects.create(meetrondje=mr, accespoint=ap_cache[bssid],
        latitude=point.attrib['lat'], longitude=point.attrib['lon'],
        signaal=signaal)
    except IntegrityError, e:
      continue
    # Give some feedback to the user
    count += 1
    if (count % 1000) == 0:
      sys.stdout.write(str(count))
    elif (count % 100) == 0:
      sys.stdout.write(".")
    sys.stdout.flush()

  sys.stdout.write("%s\n" % count)
  print "#INFO: All done, goodbye"


class Command(BaseCommand):
  args = '<gpsxml>[.gz] [<netxml>[.gz]]'
  option_list = BaseCommand.option_list + (
    make_option('-m', '--meetrondje', dest='meetrondje', default='rondje',help='Naam van het meetrondje'),
    make_option('-k', '--kaart', dest='kaart', default='onbekend', help="Kaart gebruikt"),
    make_option('-g', '--gebruiker', dest='gebruiker', default='username',help='Naam van de persoon die de meting uitgevoerd heeft'),
    make_option('-e', '--email', dest='email', default='foo@bar.org',help='Email van de persoon die de meting uitgevoerd heeft'),
    )

  def handle(self, *args, **options):
    try:
      if len(args) == 2:
        (gpsxml_file, netxml_file) = args
      elif len(args) == 1:
        (gpsxml_file,) = args
        netxml_file = gpsxml_file.replace('.gpsxml','.netxml')
      else:
        raise ValueError
    except ValueError:
      self.print_help(sys.argv[0],sys.argv[1])
      raise CommandError("Not all arguments are provided")
    if not os.path.isfile(gpsxml_file):
      raise CommandError("gpsxml file '%s' does not exists" % gpsxml_file)
    if not os.path.isfile(netxml_file):
      raise CommandError("netxml file '%s' does not exists" % netxml_file)

    import_file(gpsxml_file, netxml_file ,options['meetrondje'], options['kaart'],options['gebruiker'],options['email'])