source: src/django_gheat/gheat/management/commands/import_netstumbler.py@ 9623

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Script for importing .ns1 files (Netstumber output)
#
# Rick van der Zwet <info@rickvanderzwet.nl>
#
from django.core.management.base import BaseCommand,CommandError
from django.db.utils import IntegrityError
from optparse import OptionParser, make_option
from gheat.models import *
from lxml import etree
import datetime
import gzip
import os
import sys
import logging

from collections import defaultdict

from netstumbler import parse_netstumbler
from import_droidstumbler import bulk_sql,get_organization_id_by_ssid

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

# Open files for reading
def open_file(file):
 if file.endswith('.gz'):
   return gzip.open(file,'rb')
 else:
  return open(file,'rb')



def import_accespoints(ap_pool, counters):
  # Determine which Accespoints to add
  bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).\
    values_list('mac', flat=True)
  bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)

  # Create a bulk import list and import
  if bssid_list_insert:
    sql_values = []
    for bssid in bssid_list_insert:
      ssid, encryption = ap_pool[bssid]
      # Special trick in SSID ts avoid escaping in later stage
      item = str((bssid.upper(),ssid.replace('%','%%'),encryption,
        get_organization_id_by_ssid(ssid)))
      sql_values.append(item)
    counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`,\
      `encryptie`, `organization_id`)',sql_values)
  return counters



def import_metingen(meetrondje, meting_pool, counters):
  # Temponary holders
  bssid_failed = defaultdict(int)

  bssid_list = [x[0] for x in meting_pool.keys()]
  # Build mapping for meting import
  mac2id = {}
  for mac,id in Accespoint.objects.filter(mac__in=bssid_list).\
    values_list('mac','id'):
    mac2id[mac] = int(id)

  clients = {}
  for mac in WirelessClient.objects.filter(mac__in=bssid_list).\
    values_list('mac',flat=True):
    clients[mac] = True

  sql_values = []
  for (bssid,lat,lon),signals in meting_pool.iteritems():
    if clients.has_key(bssid):
      counters['meting_ignored'] += len(signals)
    elif not mac2id.has_key(bssid):
      counters['meting_failed'] += len(signals)
      bssid_failed[bssid] += len(signals)
    else:
      item = str((int(meetrondje.id),mac2id[bssid],float(lat),\
        float(lon),max(signals)))
      sql_values.append(item)

  for bssid,count in sorted(bssid_failed.items(),
      key=lambda item: item[1], reverse=True):
    logger.debug("Missing BSSID %s found %3s times", bssid, count)

  if sql_values:
    counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`,\
      `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
  return counters



def process_ns1(filename):
  data = parse_netstumbler(open_file(filename))

  counters = {
    'ap_added' : 0, 'ap_total' : 0,
    'ap_failed' : 0, 'ap_ignored' : 0,
    'client_added' : 0, 'client_total' : 0,
    'client_failed' : 0, 'client_ignored' : 0,
    'meting_added' : 0, 'meting_total' : 0,
    'meting_failed' : 0, 'meting_ignored' : 0
    }

  # Temponary holders
  meting_pool = defaultdict(list)
  ap_pool = {}

  for ap in data['aps']:
    # XXX: How is encryption coded?
    encryption = False
    ap_pool[ap['BSSID']]= (ap['SSID'], encryption)
    for point in ap["measurements"]:
     counters['meting_total'] += 1
     if point['LocationSource'] == 0:
       logger.debug("No GPS Coordinates found for BSSID %s @ %s", 
         ap['BSSID'], point['Time'])
       counters['meting_ignored'] += 1
       continue
     # We store all values found, avg or max will be done later on
     key = (ap['BSSID'], point["Latitude"], point["Longitude"])

     # Known measurement error
     if point['Signal'] == -32767: continue

     # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
     signaal= 100 + point['Signal']
     if signaal > 100 or signaal < 0:
       logger.warning("Signal %s is not valid entry for BSSID %s @ %s",
         point['Signal'], ap['BSSID'], point['Time'])
       continue
     meting_pool[key].append(signaal)

  return (counters, ap_pool, meting_pool)




class Command(BaseCommand):
  args = '<netstumber.ns1>[.gz] [netstumber2.ns1[.gz]  netstumber3.ns1[.gz] ...]'
  option_list = BaseCommand.option_list + (
    make_option('-k', '--kaart', dest='kaart', default='onbekend', 
      help="Kaart gebruikt"),
    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
    make_option('-g', '--gebruiker', dest='gebruiker', default='username',
      help='Naam van de persoon die de meting uitgevoerd heeft'),
    make_option('-e', '--email', dest='email', default='foo@bar.org',
      help='Email van de persoon die de meting uitgevoerd heeft'),
    make_option('-d', '--datum', dest='datum', default=None,
      help="Provide date in following format: '%Y%m%d-%H-%M-%S-1', by \
      default it will be generated from the filename"),
  )

  def handle(self, *args, **options):
    if options['verbosity'] > 1:
      logger.setLevel(logging.DEBUG)
    if len(args) == 0:
      self.print_help(sys.argv[0],sys.argv[1])
      raise CommandError("Not all arguments are provided")

    # Please first the netxml and then the gpsxml files
    sorted_args = [x for x in args if "ns1" in x]
    remainder = list(set(args) - set(sorted_args))
    args = sorted_args + remainder
    logger.debug("Parsing files in the following order: %s", args)

    for filename in args:
      if not os.path.isfile(filename):
        raise CommandError("file '%s' does not exists" % filename)

    def process_date(datestr):
      try:
         # Kismet-20110805-15-37-30-1
         return datetime.datetime.strptime(datestr,'%Y%m%d-%H-%M-%S-1')
      except ValueError:
        raise CommandError("Invalid date '%s'" % options['datum'])

    for filename in args:
      logger.info("Processing '%s'" % filename)
      if 'ns1' in filename:
        if options['datum'] == None:
           datestr = os.path.basename(filename).lstrip('Kismet-').\
             rstrip('.gz').rstrip('.gpsxml').rstrip('.netxml').rstrip('.ns1')
           datum = process_date(datestr)
        elif options['datum'] == 'now':
           datum = datetime.datetime.now()
        else:
           datum = process_date(options['datum'])

        # Meetrondje from filename if needed
        if options['meetrondje'] == None:
          meetrondje = os.path.basename(filename).rstrip('.gz').rstrip('.ns1')
        else:
          meetrondje = options['meetrondje']

        # Create meetrondje object
        g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'],
          email=options['email'])
        a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
        mr, created = MeetRondje.objects.get_or_create(datum=datum,
          naam=meetrondje, gebruiker=g, apparatuur=a)
        logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
        if not created:
          logger.error("Meetrondje '%s' already imported" % mr)
          continue
        (counters, ap_pool, meting_pool) = process_netstumber(filename)
        counters = import_accespoints(ap_pool, counters)
        counters = import_metingen(mr, meting_pool, counters)

        logger.info(("summary metingen   : total:%(meting_total)-6s" +
          "added:%(meting_added)-6s failed:%(meting_failed)-6s" +
          "ignored:%(meting_ignored)-6s") % counters)
      else:
        raise CommandError("file '%s' format not recognized" % filename)