| 1 | #!/usr/bin/env python
|
|---|
| 2 | # -*- coding: utf-8 -*-
|
|---|
| 3 | #
|
|---|
| 4 | # Script for importing DroidStumbler .csv files, which takes the best value of
|
|---|
| 5 | # each measurement point.
|
|---|
| 6 | #
|
|---|
| 7 | # Original by Dennis Wagenaar <d.wagenaar@gmail.com>
|
|---|
| 8 | #
|
|---|
| 9 | # Rick van der Zwet <info@rickvanderzwet.nl>
|
|---|
| 10 | #
|
|---|
| 11 | from django.core.management import setup_environ
|
|---|
| 12 | from django.core.management.base import BaseCommand, CommandError
|
|---|
| 13 | from django.db.utils import IntegrityError
|
|---|
| 14 | from optparse import OptionParser, make_option
|
|---|
| 15 | from gheat.models import *
|
|---|
| 16 | import csv
|
|---|
| 17 | import datetime
|
|---|
| 18 | import gzip
|
|---|
| 19 | import logging
|
|---|
| 20 | import os
|
|---|
| 21 | import sys
|
|---|
| 22 |
|
|---|
| 23 | def user_feedback(count, dot_step, summary_report):
|
|---|
| 24 | if (count % summary_report) == 0:
|
|---|
| 25 | sys.stdout.write(str(count))
|
|---|
| 26 | elif (count % dot_step) == 0:
|
|---|
| 27 | sys.stdout.write(".")
|
|---|
| 28 | sys.stdout.flush()
|
|---|
| 29 | return count + 1
|
|---|
| 30 |
|
|---|
| 31 |
|
|---|
| 32 | def import_droidstumbler(location, meetrondje, gebruiker, email, datum,show_progres=False):
|
|---|
| 33 | """ Import all points, return tuple with summary"""
|
|---|
| 34 | g, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
|
|---|
| 35 | a, created = Apparatuur.objects.get_or_create(antenne='buildin' , kaart='mobilePhone')
|
|---|
| 36 | mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
|
|---|
| 37 |
|
|---|
| 38 | meting_count = 0
|
|---|
| 39 | new_ap_count = 0
|
|---|
| 40 | ap_cache = {}
|
|---|
| 41 | meting_pool = {}
|
|---|
| 42 |
|
|---|
| 43 | # Process file
|
|---|
| 44 | if location.endswith('.gz'):
|
|---|
| 45 | fh = gzip.open(location,'rb')
|
|---|
| 46 | else:
|
|---|
| 47 | fh = open(location,'rb')
|
|---|
| 48 | csvfile = csv.reader(fh, delimiter=',')
|
|---|
| 49 | for row in csvfile:
|
|---|
| 50 | try:
|
|---|
| 51 | epoch, msg_type, lat, lon, accuracy, ssid, bssid, level, frequency, capabilities = row
|
|---|
| 52 | except ValueError:
|
|---|
| 53 | logging.error("Unable to parse line:%i '%s'" % (csvfile.line_num, row))
|
|---|
| 54 | continue
|
|---|
| 55 | if msg_type == "data" and lat and lon:
|
|---|
| 56 | if not ap_cache.has_key(bssid):
|
|---|
| 57 | ap_cache[bssid], created = Accespoint.objects.get_or_create(mac=bssid, ssid=ssid, encryptie=capabilities)
|
|---|
| 58 | if created: new_ap_count += 1
|
|---|
| 59 |
|
|---|
| 60 | # We store the best value found
|
|---|
| 61 | key = (ap_cache[bssid], lat, lon)
|
|---|
| 62 | signaal=(100 + int(level))
|
|---|
| 63 | if meting_pool.has_key(key):
|
|---|
| 64 | meting_pool[key] = max(meting_pool[key], signaal)
|
|---|
| 65 | else:
|
|---|
| 66 | meting_pool[key] = signaal
|
|---|
| 67 |
|
|---|
| 68 |
|
|---|
| 69 | # Import the data into the database
|
|---|
| 70 | meting_count = 0
|
|---|
| 71 | for (ap,lat,lon),signal in meting_pool.iteritems():
|
|---|
| 72 | try:
|
|---|
| 73 | m = Meting.objects.create(meetrondje=mr, accespoint=ap, latitude=lat, longitude=lon, signaal=signaal)
|
|---|
| 74 | except IntegrityError, e:
|
|---|
| 75 | logging.error("Unable to import - %s" % e)
|
|---|
| 76 | continue
|
|---|
| 77 |
|
|---|
| 78 | # Give some feedback to the user
|
|---|
| 79 | if show_progres:
|
|---|
| 80 | meting_count = user_feedback(meting_count, 100, 1000)
|
|---|
| 81 |
|
|---|
| 82 | if show_progres:
|
|---|
| 83 | sys.stdout.write("%s\n" % meting_count)
|
|---|
| 84 |
|
|---|
| 85 | return (len(ap_cache), new_ap_count, meting_count, len(meting_pool) - meting_count)
|
|---|
| 86 |
|
|---|
| 87 |
|
|---|
| 88 | class Command(BaseCommand):
|
|---|
| 89 | args = '<csvfile>[.gz]'
|
|---|
| 90 | option_list = BaseCommand.option_list + (
|
|---|
| 91 | make_option('-m', '--meetrondje', dest='meetrondje', default=None),
|
|---|
| 92 | make_option('-g', '--gebruiker', dest='gebruiker', default=os.environ['USER']),
|
|---|
| 93 | make_option('-e', '--email', dest='email', default=os.environ['USER'] + '@example.org'),
|
|---|
| 94 | make_option('-d', '--datum', dest='datum', default=None, help="Provide date \
|
|---|
| 95 | in following format: %Y-%m-%d-%H%M%S, by default it will be generated from \
|
|---|
| 96 | the filename"),)
|
|---|
| 97 |
|
|---|
| 98 | def handle(self, *args, **options):
|
|---|
| 99 | try:
|
|---|
| 100 | (csv_file,) = args
|
|---|
| 101 | except ValueError:
|
|---|
| 102 | self.print_help(sys.argv[0],sys.argv[1])
|
|---|
| 103 | raise CommandError("Not all arguments are provided")
|
|---|
| 104 | if not os.path.isfile(csv_file):
|
|---|
| 105 | raise CommandError("csv file '%s' does not exists" % csv_file)
|
|---|
| 106 |
|
|---|
| 107 | # Meetrondje from filename if needed
|
|---|
| 108 | if options['meetrondje'] == None:
|
|---|
| 109 | meetrondje = os.path.basename(csv_file).rstrip('.gz').rstrip('.csv')
|
|---|
| 110 | else:
|
|---|
| 111 | meetrondje = options['meetrondje']
|
|---|
| 112 | # Date from filename if needed
|
|---|
| 113 | if options['datum'] == None:
|
|---|
| 114 | datum = os.path.basename(csv_file).lstrip('ScanResult-').rstrip('.csv.gz')
|
|---|
| 115 | else:
|
|---|
| 116 | datum = options['datum']
|
|---|
| 117 | try:
|
|---|
| 118 | datum = datetime.datetime.strptime(datum,'%Y-%m-%d-%H%M%S')
|
|---|
| 119 | except ValueError:
|
|---|
| 120 | raise CommandError("Invalid date '%s'\n" % options['datum'])
|
|---|
| 121 |
|
|---|
| 122 | self.stdout.write('#INFO: Meetrondje: %s @ %s\n' % (meetrondje, datum))
|
|---|
| 123 | self.stdout.write("#INFO: Going to import '%s' for gebruiker '%s <%s>'\n" % (os.path.basename(csv_file), options['gebruiker'], options['email']))
|
|---|
| 124 | (ap_count, new_ap_count, meting_count, meting_error_count) = import_droidstumbler(csv_file,meetrondje,options['gebruiker'],options['email'], datum, True)
|
|---|
| 125 | self.stdout.write("#INFO: Import summary accespoints: added:%s processed:%s\n" % (new_ap_count, ap_count))
|
|---|
| 126 | self.stdout.write("#INFO: Import summary metingen: added:%s error:%s\n" % (meting_count, meting_error_count))
|
|---|
