Changeset 9187 for src/django_gheat/gheat

Timestamp:

May 12, 2011, 1:03:33 PM (14 years ago)

Author:

rick

Message:

... and some way to clean the cache files if needed.

File:

• src/django_gheat/gheat/management/commands/clear_tile_cache.py (copied) (copied from src/django_gheat/gheat/management/commands/import_droidstumbler.py ) (1 diff)

src/django_gheat/gheat/management/commands/clear_tile_cache.py

@@ -2 +2 @@
 # -*- coding: utf-8 -*-
 #
-# Script for importing DroidStumbler .csv files, which takes the best value of
-# each measurement point.
-#
-# Original by Dennis Wagenaar <d.wagenaar@gmail.com>
+# Clean/Clear the cached tiles
 #
 # Rick van der Zwet <info@rickvanderzwet.nl>
 #
-from django.core.management import setup_environ
 from django.core.management.base import BaseCommand, CommandError
-from django.db import connection, transaction
-from django.db.utils import IntegrityError
 from optparse import OptionParser, make_option
-from gheat.models import *
-import csv
+from gheat.models import TileCache
 import datetime
-import gzip
-import logging
-import os
 import sys
 
-def user_feedback(output, count, dot_step, summary_report):
-  if (count % summary_report) == 0:
-    output.write(str(count))
-  elif (count % dot_step) == 0:
-    output.write(".")
-  output.flush()
-  return count + 1
-
-
-def import_droidstumbler(location, meetrondje, gebruiker, email, datum,show_progres=False,output=sys.stdout,bulk_import=True):
-  """ Import all points, return tuple with summary"""
-  g, created = Gebruiker.objects.get_or_create(naam=gebruiker , email=email)
-  a, created = Apparatuur.objects.get_or_create(antenne='buildin' , kaart='mobilePhone')
-  mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
-
-  meting_count = 0
-  new_ap_count = 0
-  ap_cache = {}
-  meting_pool = {}
-
-  if show_progres: output.write('#INFO: Pre-Processing file: ')
-  if location.endswith('.gz'):
-    fh = gzip.open(location,'rb')
-  else:
-    fh = open(location,'rb')
-  csvfile = csv.reader(fh, delimiter=',')
-  count = 0
-  for row in csvfile:
-    if show_progres: count = user_feedback(output, count, 100, 1000)
-    try:
-      epoch, msg_type, lat, lon, accuracy, ssid, bssid, level, frequency, capabilities = row
-    except ValueError:
-      logging.error("Unable to parse line:%i '%s'" % (csvfile.line_num, row))
-      continue
-    if msg_type == "data" and lat and lon:
-      if not ap_cache.has_key(bssid):
-        ap_cache[bssid], created = Accespoint.objects.get_or_create(mac=bssid, ssid=ssid, encryptie=capabilities)
-        if created: new_ap_count += 1
-
-      # We store the best value found
-      key = (ap_cache[bssid], lat, lon)
-      signaal=(100 + int(level))
-      if meting_pool.has_key(key):
-        meting_pool[key] = max(meting_pool[key], signaal)
-      else:
-        meting_pool[key] = signaal
-  if show_progres: output.write("%s\n" % count)
-
-
-  if show_progres: output.write('#INFO: Importing the data into the database: ')
-  meting_count = 0
-  cursor = connection.cursor()
-  sql_values = []
-  for (ap,lat,lon),signal in meting_pool.iteritems():
-    try:
-      if bulk_import:
-        value_str = str((int(mr.id),int(ap.id),float(lat),float(lon),int(signaal)))
-        sql_values.append(value_str)
-      else:
-        m = Meting.objects.create(meetrondje=mr, accespoint=ap, latitude=lat, longitude=lon, signaal=signaal)
-    except IntegrityError, e:
-      logging.error("Unable to import - %s" %  e)
-      continue
-
-    # Give some feedback to the user
-    if show_progres: meting_count = user_feedback(output, meting_count, 100, 1000)
-
-  # Make sure to include closing newline
-  if show_progres: output.write("%s\n" % meting_count)
-
-  # Bulk Import data if possible
-  if bulk_import:
-    if len(sql_values) == 0:
-      logging.warn("No data to import")
-    else:
-      try:
-        sql = "INSERT INTO gheat_meting (`meetrondje_id`, `accespoint_id`, `lat`, `lng`, `signaal`) VALUES %s" % ','.join(sql_values)
-        cursor.execute(sql)
-        transaction.commit_unless_managed()
-      except IntegrityError, e:
-        logging.error("Unable to import - %s" %  e)
-        pass
-
-  return (len(ap_cache), new_ap_count, meting_count, len(meting_pool) - meting_count)
-
-
 class Command(BaseCommand):
-  args = '<csvfile>[.gz] [csvfile2[.gz] [csvfile3[.gz] ...] '
   option_list = BaseCommand.option_list + (
-    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
-    make_option('-g', '--gebruiker', dest='gebruiker', default=os.environ['USER']),
-    make_option('-e', '--email', dest='email', default=os.environ['USER'] + '@example.org'),
-    make_option('-d', '--datum', dest='datum', default=None, help="Provide date  \
-      in following format: %Y-%m-%d-%H%M%S, by default it will be generated from \
-      the filename"),)
+    make_option('-d', '--datum', dest='datum', default=None, help="""Provide date
+      in following format: %Y-%m-%d[-%H%M], to delete all time generated before
+      this time the filename"""),)
 
   def handle(self, *args, **options):
-    if len(args) == 0:
-      self.print_help(sys.argv[0],sys.argv[1])
-      raise CommandError("Not all arguments are provided")
-
-    for csv_file in args:
-      # Make sure to check files before we going to do importing at all
-      if not os.path.isfile(csv_file):
-        raise CommandError("csv file '%s' does not exists" % csv_file)
-      self.stdout.write("#INFO: Processing '%s'\n" % csv_file)
-
-      # Meetrondje from filename if needed
-      if options['meetrondje'] == None:
-        meetrondje = os.path.basename(csv_file).rstrip('.gz').rstrip('.csv') 
-      else:
-        meetrondje = options['meetrondje']
-      # Date from filename if needed
-      if options['datum'] == None:
-         datum = os.path.basename(csv_file).lstrip('ScanResult-').rstrip('.csv.gz')
-      else:
-         datum = options['datum']
-      try:
-         datum = datetime.datetime.strptime(datum,'%Y-%m-%d-%H%M%S')
-      except ValueError:
+    if options['datum'] == None:
+       datum = datetime.datetime.now()
+    else:
+      # Try parsing different formats
+      for fmt in ['%Y-%m-%d-%H%M', '%Y-%m-%d']:
+        try:
+          datum = datetime.datetime.strptime(options['datum'],fmt)
+          break
+        except ValueError:
+          datum = None
+      if not datum:
         raise CommandError("Invalid date '%s'\n" % options['datum'])
 
-      self.stdout.write('#INFO: Meetrondje: %s @ %s\n' % (meetrondje, datum))
-      self.stdout.write("#INFO: Going to import '%s' for gebruiker '%s <%s>'\n" % (os.path.basename(csv_file), options['gebruiker'], options['email']))
-      (ap_count, new_ap_count, meting_count, meting_error_count) = import_droidstumbler(csv_file,meetrondje,options['gebruiker'],options['email'], datum, True)
-      self.stdout.write("#INFO: Import summary accespoints: added:%s processed:%s\n" % (new_ap_count, ap_count))
-      self.stdout.write("#INFO: Import summary metingen: added:%s error:%s\n" % (meting_count, meting_error_count))
+    obj = TileCache.objects.filter(creation_time__range=(datetime.datetime.fromtimestamp(0), datum))
+    print "#INFO: Deleting '%s' objects" % obj.count()
+    obj.delete()
+