Changeset 9640 for src/django_gheat/gheat/dataimport

Timestamp:

Aug 30, 2011, 8:36:25 PM (13 years ago)

Author:

rick

Message:

Cast the import logic to a special place, cause we are going to need it soon
when we want to auto-import files from the WEB (admin interface or form).

Location:

src/django_gheat/gheat/dataimport

Files:

• . (added)
• __init__.py (copied) (copied from src/django_gheat/gheat/management/commands/import_datafile.py ) (6 diffs)
• droidstumbler.py (moved) (moved from src/django_gheat/gheat/management/commands/droidstumbler.py )
• kismet.py (moved) (moved from src/django_gheat/gheat/management/commands/kismet.py )
• netstumbler.py (moved) (moved from src/django_gheat/gheat/management/commands/netstumbler.py )

src/django_gheat/gheat/dataimport/__init__.py

@@ -10 +10 @@
 #
 from _mysql_exceptions import OperationalError
-from django.core.files import File
-from django.core.management.base import BaseCommand,CommandError
 from django.db import connection, transaction
 from django.db.utils import IntegrityError
 from gheat.models import *
-from optparse import OptionParser, make_option
-import datetime
 import gzip
 import os
@@ -29 +25 @@
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 def open_file(file):
@@ -39 +34 @@
 
 
-valid_prefixes = ['DroidStumbler-', 'Kismet-','ScanResult-']
-def strip_prefix(filename):
-  """ Prefix removal """
-  for prefix in valid_prefixes:
-    if filename.startswith(prefix):
-      filename = filename[len(prefix):]
-  return filename
-
-
-valid_suffixes = ['.gz', '.gpsxml', '.netxml', '.csv', '.ns1']
-def strip_suffix(filename):
-  """ Suffix removal """
-  for suffix in valid_suffixes:
-    if filename.endswith(suffix):
-      filename = filename[:-len(suffix)]
-  return filename
-
-
-def strip_file(filename):
-  """ Prefix and suffix removal """
-  return strip_suffix(strip_prefix(filename))
-
-
-#Kismet-20110805-15-37-30-1
-#ScanResult-2011-05-09-201117
-strptime_choices = ['%Y%m%d-%H-%M-%S-1', '%Y-%m-%d-%H%M%S']
-def process_date(datestr):
-  for strptime in strptime_choices:
-    try:
-      return datetime.datetime.strptime(datestr,strptime)
-    except ValueError:
-      pass
-  logger.error("Invalid date '%s', options: %s, using: now()", datestr, strptime_choices)
-  return datetime.datetime.now()
 
 
@@ -187 +148 @@
 
 
-
 def import_file(filename,meetrondje):
+  """ Import a file (on disk) """
   counters = {
     'ap_added' : 0, 'ap_total' : 0,
@@ -208 +169 @@
     (counters, ap_pool, client_pool, meting_pool) = droidstumbler.process_csv(fh, counters)
   else:
-    raise CommandError("file '%s' format not recognized" % filename)
+    (ap_pool, client_pool, meting_pool) = (None, None, None)
+    logger.error("file '%s' format not recognized")
 
   if ap_pool:
@@ -217 +179 @@
     counters = import_metingen(meetrondje, meting_pool, counters)
 
-  logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
-  logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
-  logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
+  logger.debug("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
+  logger.debug("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
+  logger.debug("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
 
-
-
-class Command(BaseCommand):
-  args = '<netstumber.ns1>[.gz] [netstumber2.ns1[.gz]  netstumber3.ns1[.gz] ...]'
-  option_list = BaseCommand.option_list + (
-    make_option('-k', '--kaart', dest='kaart', default='onbekend', 
-      help="Kaart gebruikt"),
-    make_option('-m', '--meetrondje', dest='meetrondje', default=None),
-    make_option('-g', '--gebruiker', dest='gebruiker', default='username',
-      help='Naam van de persoon die de meting uitgevoerd heeft'),
-    make_option('-e', '--email', dest='email', default='foo@bar.org',
-      help='Email van de persoon die de meting uitgevoerd heeft'),
-    make_option('-d', '--datum', dest='datum', default=None,
-      help="Provide date in following format: '%Y%m%d-%H-%M-%S-1', by \
-      default it will be generated from the filename"),
-  )
-
-  def handle(self, *args, **options):
-    if options['verbosity'] == 2:
-      logger.setLevel(logging.DEBUG)
-    if len(args) == 0:
-      self.print_help(sys.argv[0],sys.argv[1])
-      raise CommandError("Not all arguments are provided")
-
-    # Please first the netxml and the gpsxml files and the rest
-    sorted_args = [x for x in args if "netxml" in x] +\
-     [x for x in args if "gpsxml" in x] +\
-     [x for x in args if "ns1" in x]
-    remainder = list(set(args) - set(sorted_args))
-    args = sorted_args + remainder
-    logger.debug("Parsing files in the following order: %s", args)
-
-    # Make sure the all exists at first
-    for filename in args:
-      if not os.path.isfile(filename):
-        raise CommandError("file '%s' does not exists" % filename)
-
-
-    def get_date(filename):
-      if options['datum'] == None:
-         datestr = strip_file(os.path.basename(filename))
-         datum = process_date(datestr)
-      elif options['datum'] == 'now':
-         datum = datetime.datetime.now()
-      else:
-         datum = process_date(options['datum'])
-      return datum
-
-    def get_meetrondje(meetrondje):
-      # Meetrondje from filename if needed
-      if options['meetrondje'] == None:
-        meetrondje = strip_suffix(os.path.basename(filename))
-      else:
-        meetrondje = options['meetrondje']
-      return meetrondje
-
-    # Get Gheat Objects, pre-req
-    gebruiker, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'],
-      email=options['email'])
-    apparatuur, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
-
-    # Check if all files are valid
-    for filename in args:
-      logger.info("Processing '%s'" % filename)
-      meetrondje, created = MeetRondje.objects.get_or_create(
-        datum=get_date(filename), naam=get_meetrondje(filename),
-        gebruiker=gebruiker, apparatuur=apparatuur)
-      if not created:
-        logger.error("Meetrondje '%s' already imported",  meetrondje)
-        continue
-
-      logger.info('Meetrondje: %s', meetrondje)
-      meetbestand = MeetBestand(meetrondje=meetrondje,is_imported=True)
-      meetbestand.bestand.save(os.path.basename(filename),File(open(filename)))
-      meetbestand.save()
-
-      import_file(filename,meetrondje)
-
+  return counters