Changeset 9179

Timestamp:

May 12, 2011, 7:34:26 AM (14 years ago)

Author:

rick

Message:

Make sure points are unique (max value uses) to avoids duplicates all over the
database. Re-init database to make sure settings are into effect.

Do pre-processing on import_droidstumbler to make sure input values _are_ unique.

Location:

src/django_gheat/gheat

Files:

• management/commands/import_droidstumbler.py (modified) (6 diffs)
• models.py (modified) (1 diff)

src/django_gheat/gheat/management/commands/import_droidstumbler.py

@@ -2 +2 @@
 # -*- coding: utf-8 -*-
 #
-# Script for importing DroidStumbler .csv files
+# Script for importing DroidStumbler .csv files, which takes the best value of
+# each measurement point.
 #
 # Original by Dennis Wagenaar <d.wagenaar@gmail.com>
@@ -10 +11 @@
 from django.core.management import setup_environ
 from django.core.management.base import BaseCommand, CommandError
+from django.db.utils import IntegrityError
 from optparse import OptionParser, make_option
 from gheat.models import *
@@ -19 +21 @@
 import sys
 
+def user_feedback(count, dot_step, summary_report):
+  if (count % summary_report) == 0:
+    sys.stdout.write(str(count))
+  elif (count % dot_step) == 0:
+    sys.stdout.write(".")
+  sys.stdout.flush()
+  return count + 1
+
+
 def import_droidstumbler(location, meetrondje, gebruiker, email, datum,show_progres=False):
 
   g, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
   a, created = Apparatuur.objects.get_or_create(antenne='buildin' , kaart='mobilePhone')
-  mr = MeetRondje.objects.create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
+  mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
 
+  ap_cache = {}
+  meting_pool = {}
+
+  # Process file
   if location.endswith('.gz'):
     fh = gzip.open(location,'rb')
@@ -30 +45 @@
     fh = open(location,'rb')
   csvfile = csv.reader(fh, delimiter=',')
-  count = 0
-  ap_cache = {}
   for row in csvfile:
     try:
@@ -42 +55 @@
         ap_cache[bssid], created = Accespoint.objects.get_or_create(mac=bssid, ssid=ssid, encryptie=capabilities)
 
-      m = Meting.objects.create(meetrondje=mr, accespoint=ap_cache[bssid], latitude=lat, longitude=lon, signaal=(100 + int(level)))
-      # Give some feedback to the user
-      count += 1
-      if show_progres:
-        if (count % 1000) == 0:
-          sys.stdout.write(str(count))
-        elif (count % 100) == 0:
-          sys.stdout.write(".")
-        sys.stdout.flush()
+      # We store the best value found
+      key = (ap_cache[bssid], lat, lon)
+      signaal=(100 + int(level))
+      if meting_pool.has_key(key):
+        meting_pool[key] = max(meting_pool[key], signaal)
+      else:
+        meting_pool[key] = signaal
+
+
+  # Import the data into the database
+  count = 0
+  for (ap,lat,lon),signal in meting_pool.iteritems():
+    try:
+      m = Meting.objects.create(meetrondje=mr, accespoint=ap, latitude=lat, longitude=lon, signaal=signaal)
+    except IntegrityError, e:
+      logging.error("Unable to import - %s" %  e)
+      continue
+
+    # Give some feedback to the user
+    if show_progres:
+      count = user_feedback(count, 100, 1000)
 
   if show_progres:
@@ -91 +116 @@
       sys.exit(1)
 
+    self.stdout.write('Meetrondje: %s @ %s\n' % (meetrondje, datum))
     self.stdout.write("Going to import '%s' for gebruiker '%s <%s>'\n" % (os.path.basename(csv_file), options['gebruiker'], options['email']))
     import_droidstumbler(csv_file,meetrondje,options['gebruiker'],options['email'], datum, True)
-    self.stdout.writeln("All done, goodbye")
+    self.stdout.write("All done, goodbye")

src/django_gheat/gheat/models.py

@@ -39 +39 @@
   def __unicode__(self):
     return "%s @ %.5f,%.5f : %s" % (self.accespoint.ssid, float(self.latitude), float(self.longitude), self.signaal)
+  class Meta:
+    # This implies that you cannot have multiple messurements on the same
+    # location for a single 'run', thus the data needs cleaned before to make
+    # this properly hold.
+    unique_together = ('meetrondje', 'accespoint', 'latitude', 'longitude'),
 
 class Node(models.Model):