Changeset 9181 for src/django_gheat/gheat/management/commands/import_droidstumbler.py

Timestamp:

May 12, 2011, 8:20:33 AM (14 years ago)

Author:

rick

Message:

Example for bulk importing data.

File:

• src/django_gheat/gheat/management/commands/import_droidstumbler.py (modified) (6 diffs)

src/django_gheat/gheat/management/commands/import_droidstumbler.py

@@ -11 +11 @@
 from django.core.management import setup_environ
 from django.core.management.base import BaseCommand, CommandError
+from django.db import connection, transaction
 from django.db.utils import IntegrityError
 from optparse import OptionParser, make_option
@@ -21 +22 @@
 import sys
 
-def user_feedback(count, dot_step, summary_report):
+def user_feedback(output, count, dot_step, summary_report):
   if (count % summary_report) == 0:
-    sys.stdout.write(str(count))
+    output.write(str(count))
   elif (count % dot_step) == 0:
-    sys.stdout.write(".")
-  sys.stdout.flush()
+    output.write(".")
+  output.flush()
   return count + 1
 
 
-def import_droidstumbler(location, meetrondje, gebruiker, email, datum,show_progres=False):
+def import_droidstumbler(location, meetrondje, gebruiker, email, datum,show_progres=False,output=sys.stdout,bulk_import=True):
   """ Import all points, return tuple with summary"""
   g, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
@@ -41 +42 @@
   meting_pool = {}
 
-  # Process file
+  if show_progres: output.write('#INFO: Pre-Processing file: ')
   if location.endswith('.gz'):
     fh = gzip.open(location,'rb')
@@ -47 +48 @@
     fh = open(location,'rb')
   csvfile = csv.reader(fh, delimiter=',')
+  count = 0
   for row in csvfile:
+    if show_progres: count = user_feedback(output, count, 100, 1000)
     try:
       epoch, msg_type, lat, lon, accuracy, ssid, bssid, level, frequency, capabilities = row
@@ -65 +68 @@
       else:
         meting_pool[key] = signaal
+  if show_progres: output.write("%s\n" % count)
 
 
-  # Import the data into the database
+  if show_progres: output.write('#INFO: Importing the data into the database: ')
   meting_count = 0
+  cursor = connection.cursor()
+  sql_values = []
   for (ap,lat,lon),signal in meting_pool.iteritems():
     try:
-      m = Meting.objects.create(meetrondje=mr, accespoint=ap, latitude=lat, longitude=lon, signaal=signaal)
+      if bulk_import:
+        value_str = str((int(mr.id),int(ap.id),float(lat),float(lon),int(signaal)))
+        sql_values.append(value_str)
+      else:
+        m = Meting.objects.create(meetrondje=mr, accespoint=ap, latitude=lat, longitude=lon, signaal=signaal)
     except IntegrityError, e:
       logging.error("Unable to import - %s" %  e)
@@ -77 +87 @@
 
     # Give some feedback to the user
-    if show_progres:
-      meting_count = user_feedback(meting_count, 100, 1000)
+    if show_progres: meting_count = user_feedback(output, meting_count, 100, 1000)
 
-  if show_progres:
-    sys.stdout.write("%s\n" % meting_count)
+  # Make sure to include closing newline
+  if show_progres: output.write("%s\n" % meting_count)
+  if bulk_import:
+    sql = "INSERT INTO gheat_meting (`meetrondje_id`, `accespoint_id`, `lat`, `lng`, `signaal`) VALUES %s" % ','.join(sql_values)
+    cursor.execute(sql)
+    transaction.commit_unless_managed()
 
   return (len(ap_cache), new_ap_count, meting_count, len(meting_pool) - meting_count)