source: src/django_gheat/gheat/management/commands/import_droidstumbler.py@ 9369

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Script for importing DroidStumbler .csv files, which takes the best value of
# each measurement point.
#
# Original by Dennis Wagenaar <d.wagenaar@gmail.com>
#
# Rick van der Zwet <info@rickvanderzwet.nl>
#
from django.core.management import setup_environ
from django.core.management.base import BaseCommand, CommandError
from django.db import connection, transaction
from django.db.utils import IntegrityError
from optparse import OptionParser, make_option
from gheat.models import *
import csv
import datetime
import gzip
import logging
import os
import sys

def user_feedback(output, count, dot_step, summary_report):
  if (count % summary_report) == 0:
    output.write(str(count))
  elif (count % dot_step) == 0:
    output.write(".")
  output.flush()
  return count + 1


def import_droidstumbler(location, meetrondje, gebruiker, email, datum,show_progres=False,output=sys.stdout,bulk_import=True):
  """ Import all points, return tuple with summary"""
  g, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
  a, created = Apparatuur.objects.get_or_create(antenne='buildin' , kaart='mobilePhone')
  mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)

  meting_count = 0
  new_ap_count = 0
  ap_cache = {}
  meting_pool = {}

  if show_progres: output.write('#INFO: Pre-Processing file: ')
  if location.endswith('.gz'):
    fh = gzip.open(location,'rb')
  else:
    fh = open(location,'rb')
  csvfile = csv.reader(fh, delimiter=',')
  count = 0
  for row in csvfile:
    if show_progres: count = user_feedback(output, count, 100, 1000)
    try:
      epoch, msg_type, lat, lon, accuracy, ssid, bssid, level, frequency, capabilities = row
    except ValueError:
      logging.error("Unable to parse line:%i '%s'" % (csvfile.line_num, row))
      continue
    if msg_type == "data" and lat and lon:
      if not ap_cache.has_key(bssid):
        ap_cache[bssid], created = Accespoint.objects.get_or_create(mac=bssid, ssid=ssid, encryptie=capabilities)
        if created: new_ap_count += 1

      # We store the best value found
      key = (ap_cache[bssid], lat, lon)
      signaal=(100 + int(level))
      if meting_pool.has_key(key):
        meting_pool[key] = max(meting_pool[key], signaal)
      else:
        meting_pool[key] = signaal
  if show_progres: output.write("%s\n" % count)


  if show_progres: output.write('#INFO: Importing the data into the database: ')
  meting_count = 0
  cursor = connection.cursor()
  sql_values = []
  for (ap,lat,lon),signal in meting_pool.iteritems():
    try:
      if bulk_import:
        value_str = str((int(mr.id),int(ap.id),float(lat),float(lon),int(signaal)))
        sql_values.append(value_str)
      else:
        m = Meting.objects.create(meetrondje=mr, accespoint=ap, latitude=lat, longitude=lon, signaal=signaal)
    except IntegrityError, e:
      logging.error("Unable to import - %s" %  e)
      continue

    # Give some feedback to the user
    if show_progres: meting_count = user_feedback(output, meting_count, 100, 1000)

  # Make sure to include closing newline
  if show_progres: output.write("%s\n" % meting_count)

  # Bulk Import data if possible
  if bulk_import:
    if len(sql_values) == 0:
      logging.warn("No data to import")
    else:
      try:
        sql = "INSERT INTO gheat_meting (`meetrondje_id`, `accespoint_id`, `lat`, `lng`, `signaal`) VALUES %s" % ','.join(sql_values)
        cursor.execute(sql)
        transaction.commit_unless_managed()
      except IntegrityError, e:
        logging.error("Unable to import - %s" %  e)
        pass

  return (len(ap_cache), new_ap_count, meting_count, len(meting_pool) - meting_count)


class Command(BaseCommand):
  args = '<csvfile>[.gz] [csvfile2[.gz] [csvfile3[.gz] ...] '
  option_list = BaseCommand.option_list + (
    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
    make_option('-g', '--gebruiker', dest='gebruiker', default=os.environ['USER']),
    make_option('-e', '--email', dest='email', default=os.environ['USER'] + '@example.org'),
    make_option('-d', '--datum', dest='datum', default=None, help="Provide date  \
      in following format: %Y-%m-%d-%H%M%S, by default it will be generated from \
      the filename"),)

  def handle(self, *args, **options):
    if len(args) == 0:
      self.print_help(sys.argv[0],sys.argv[1])
      raise CommandError("Not all arguments are provided")

    for csv_file in args:
      # Make sure to check files before we going to do importing at all
      if not os.path.isfile(csv_file):
        raise CommandError("csv file '%s' does not exists" % csv_file)
      self.stdout.write("#INFO: Processing '%s'\n" % csv_file)

      # Meetrondje from filename if needed
      if options['meetrondje'] == None:
        meetrondje = os.path.basename(csv_file).rstrip('.gz').rstrip('.csv') 
      else:
        meetrondje = options['meetrondje']
      # Date from filename if needed
      if options['datum'] == None:
         datum = os.path.basename(csv_file).lstrip('ScanResult-').rstrip('.csv.gz')
      else:
         datum = options['datum']
      try:
         datum = datetime.datetime.strptime(datum,'%Y-%m-%d-%H%M%S')
      except ValueError:
        raise CommandError("Invalid date '%s'\n" % options['datum'])

      self.stdout.write('#INFO: Meetrondje: %s @ %s\n' % (meetrondje, datum))
      self.stdout.write("#INFO: Going to import '%s' for gebruiker '%s <%s>'\n" % (os.path.basename(csv_file), options['gebruiker'], options['email']))
      (ap_count, new_ap_count, meting_count, meting_error_count) = import_droidstumbler(csv_file,meetrondje,options['gebruiker'],options['email'], datum, True)
      self.stdout.write("#INFO: Import summary accespoints: added:%s processed:%s\n" % (new_ap_count, ap_count))
      self.stdout.write("#INFO: Import summary metingen: added:%s error:%s\n" % (meting_count, meting_error_count))