source: src/django_gheat/gheat/management/commands/import_droidstumbler.py@ 9179

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Script for importing DroidStumbler .csv files, which takes the best value of
# each measurement point.
#
# Original by Dennis Wagenaar <d.wagenaar@gmail.com>
#
# Rick van der Zwet <info@rickvanderzwet.nl>
#
from django.core.management import setup_environ
from django.core.management.base import BaseCommand, CommandError
from django.db.utils import IntegrityError
from optparse import OptionParser, make_option
from gheat.models import *
import csv
import datetime
import gzip
import logging
import os
import sys

def user_feedback(count, dot_step, summary_report):
  if (count % summary_report) == 0:
    sys.stdout.write(str(count))
  elif (count % dot_step) == 0:
    sys.stdout.write(".")
  sys.stdout.flush()
  return count + 1


def import_droidstumbler(location, meetrondje, gebruiker, email, datum,show_progres=False):

  g, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
  a, created = Apparatuur.objects.get_or_create(antenne='buildin' , kaart='mobilePhone')
  mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)

  ap_cache = {}
  meting_pool = {}

  # Process file
  if location.endswith('.gz'):
    fh = gzip.open(location,'rb')
  else:
    fh = open(location,'rb')
  csvfile = csv.reader(fh, delimiter=',')
  for row in csvfile:
    try:
      epoch, msg_type, lat, lon, accuracy, ssid, bssid, level, frequency, capabilities = row
    except ValueError:
      logging.error("Unable to parse line:%i '%s'" % (csvfile.line_num, row))
      continue
    if msg_type == "data" and lat and lon:
      if not ap_cache.has_key(bssid):
        ap_cache[bssid], created = Accespoint.objects.get_or_create(mac=bssid, ssid=ssid, encryptie=capabilities)

      # We store the best value found
      key = (ap_cache[bssid], lat, lon)
      signaal=(100 + int(level))
      if meting_pool.has_key(key):
        meting_pool[key] = max(meting_pool[key], signaal)
      else:
        meting_pool[key] = signaal


  # Import the data into the database
  count = 0
  for (ap,lat,lon),signal in meting_pool.iteritems():
    try:
      m = Meting.objects.create(meetrondje=mr, accespoint=ap, latitude=lat, longitude=lon, signaal=signaal)
    except IntegrityError, e:
      logging.error("Unable to import - %s" %  e)
      continue

    # Give some feedback to the user
    if show_progres:
      count = user_feedback(count, 100, 1000)

  if show_progres:
    sys.stdout.write("%s\n" % count)


class Command(BaseCommand):
  args = '<csvfile>[.gz]'
  option_list = BaseCommand.option_list + (
    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
    make_option('-g', '--gebruiker', dest='gebruiker', default=os.environ['USER']),
    make_option('-e', '--email', dest='email', default=os.environ['USER'] + '@example.org'),
    make_option('-d', '--datum', dest='datum', default=None, help="Provide date  \
      in following format: %Y-%m-%d-%H%M%S, by default it will be generated from \
      the filename"),)

  def handle(self, *args, **options):
    try:
      (csv_file,) = args
    except ValueError:
      self.print_help(sys.argv[0],sys.argv[1])
      raise CommandError("Not all arguments are provided")
    if not os.path.isfile(csv_file):
      raise CommandError("csv file '%s' does not exists" % csv_file)

    # Meetrondje from filename if needed
    if options['meetrondje'] == None:
      meetrondje = os.path.basename(csv_file).rstrip('.gz').rstrip('.csv') 
    else:
      meetrondje = options['meetrondje']
    # Date from filename if needed
    if options['datum'] == None:
       datum = os.path.basename(csv_file).lstrip('ScanResult-').rstrip('.csv.gz')
    else:
       datum = options['datum']
    try:
       datum = datetime.datetime.strptime(datum,'%Y-%m-%d-%H%M%S')
    except ValueError:
      logging.error("Invalid date '%s'" % options['datum'])
      sys.exit(1)

    self.stdout.write('Meetrondje: %s @ %s\n' % (meetrondje, datum))
    self.stdout.write("Going to import '%s' for gebruiker '%s <%s>'\n" % (os.path.basename(csv_file), options['gebruiker'], options['email']))
    import_droidstumbler(csv_file,meetrondje,options['gebruiker'],options['email'], datum, True)
    self.stdout.write("All done, goodbye")