Changeset 9623 for src/django_gheat


Ignore:
Timestamp:
Aug 30, 2011, 6:13:57 PM (13 years ago)
Author:
rick
Message:

Merge and migrate all files into common files to get rid of all duplicate codes.

Location:
src/django_gheat
Files:
1 deleted
3 edited
1 copied
2 moved

Legend:

Unmodified
Added
Removed

  • src/django_gheat/gheat/management/commands/droidstumbler.py

    r9598 r9623  
    22# -*- coding: utf-8 -*-
    33#
    4 # Script for importing DroidStumbler .csv files, which takes the best value of
    5 # each measurement point.
    6 #
    7 # Original by Dennis Wagenaar <d.wagenaar@gmail.com>
    8 #
    94# Rick van der Zwet <info@rickvanderzwet.nl>
    105#
    11 from django.core.management import setup_environ
    12 from django.core.management.base import BaseCommand, CommandError
    13 from django.db import connection, transaction
    14 from django.db.utils import IntegrityError
    15 from _mysql_exceptions import OperationalError
    16 from optparse import OptionParser, make_option
    17 from gheat.models import *
    186import csv
    19 import datetime
    20 import gzip
    217import logging
    22 import os
    23 import sys
     8
     9from collections import defaultdict
    2410
    2511logger = logging.getLogger(__name__)
    2612logger.setLevel(logging.DEBUG)
    2713
    28 def bulk_sql(sql_table, sql_values):
    29   if len(sql_values) == 0:
    30     raise ValueError, "No data to import"
    31 
    32   cursor = connection.cursor()
    33   try:
    34     # Make sure the special NULL is preserved
    35     sql = "INSERT INTO %s VALUES %s" % (sql_table, ','.join(sql_values).replace("'NULL'",'NULL'))
    36     count = cursor.execute(sql)
    37     transaction.commit_unless_managed()
    38   except OperationalError, e:
    39     logger.error("%s - %s ", sql_table, sql_values[0])
    40     raise
    41   except IntegrityError, e:
    42     logger.error("Unable to import - %s" %  e)
    43     raise
    44   return count
    45 
    46 organizations = dict(Organization.objects.all().values_list('name','id'))
    47 def get_organization_id_by_ssid(ssid):
    48   name = Organization.get_name_by_ssid(ssid)
    49   if not name:
    50     return 'NULL'
    51   else:
    52     return int(organizations[name])
    53 
    54 def import_droidstumbler(filename, meetrondje):
     14def process_csv(fh, counters):
    5515  """ Import all points, return tuple with summary"""
    5616
    57   # Open file for reading
    58   if filename.endswith('.gz'):
    59     fh = gzip.open(filename,'rb')
    60   else:
    61     fh = open(filename,'rb')
     17  # Temponary holders
     18  meting_pool = defaultdict(list)
     19  ap_pool = {}
     20
    6221  csvfile = csv.reader(fh, delimiter=',')
    63 
    64   #Various statistics
    65   counters = {'meting_added' : 0, 'meting_total' : 0, 'ap_added' : 0, 'ap_total' : 0}
    66 
    67   # Temponary holders
    68   meting_pool = {}
    69   ap_pool = {}
    7022  # Process file, preparing new access points and measurements
    7123  for row in csvfile:
    … …  
    8739      key = (bssid, lat, lon)
    8840      signaal=(100 + int(level))
    89       if meting_pool.has_key(key):
    90         meting_pool[key] = max(meting_pool[key], signaal)
    91       else:
    92         meting_pool[key] = signaal
     41      meting_pool[key].append(signaal)
    9342
    94 
    95   # Determine which entries we need to add
    96   counters['ap_total'] = len(ap_pool)
    97   bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).values_list('mac', flat=True)
    98   bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
    99 
    100   # Create a bulk import list and import
    101   if bssid_list_insert:
    102     sql_values = []
    103     for bssid in bssid_list_insert:
    104       ssid, encryption = ap_pool[bssid]
    105       # Special trick in SSID ts avoid escaping in later stage
    106       item = str((bssid.upper(),ssid.replace('%','%%'),encryption,get_organization_id_by_ssid(ssid)))
    107       sql_values.append(item)
    108     counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`, `encryptie`, `organization_id`)',sql_values)
    109 
    110   # Build mapping for meting import
    111   mac2id = {}
    112   for mac,id in Accespoint.objects.filter(mac__in=ap_pool.keys()).values_list('mac','id'):
    113     mac2id[mac] = int(id)
    114 
    115   sql_values = []
    116   for (bssid,lat,lon),signal in meting_pool.iteritems():
    117     item = str((int(meetrondje.id),mac2id[bssid],float(lat),float(lon),int(signaal)))
    118     sql_values.append(item)
    119 
    120   # Bulk Import data if possible
    121   if sql_values:
    122     counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`, `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
    123   return counters
    124 
    125 
    126 class Command(BaseCommand):
    127   args = '<csvfile>[.gz] [csvfile2[.gz] [csvfile3[.gz] ...] '
    128   option_list = BaseCommand.option_list + (
    129     make_option('-m', '--meetrondje', dest='meetrondje', default=None),
    130     make_option('-g', '--gebruiker', dest='gebruiker', default=os.environ['USER']),
    131     make_option('-e', '--email', dest='email', default=os.environ['USER'] + '@example.org'),
    132     make_option('-k', '--kaart', dest='kaart', default='onbekend', help="Kaart gebruikt"),
    133     make_option('-d', '--datum', dest='datum', default=None, help="Provide date  \
    134       in following format: %Y-%m-%d-%H%M%S, by default it will be generated from \
    135       the filename"),
    136   )
    137 
    138   def handle(self, *args, **options):
    139     if len(args) == 0:
    140       self.print_help(sys.argv[0],sys.argv[1])
    141       raise CommandError("Not all arguments are provided")
    142 
    143     for csv_file in args:
    144       # Make sure to check files before we going to do importing at all
    145       if not os.path.isfile(csv_file):
    146         raise CommandError("csv file '%s' does not exists" % csv_file)
    147       logger.info("Processing '%s'" % csv_file)
    148 
    149       # Meetrondje from filename if needed
    150       if options['meetrondje'] == None:
    151         meetrondje = os.path.basename(csv_file).rstrip('.gz').rstrip('.csv')
    152       else:
    153         meetrondje = options['meetrondje']
    154       # Date from filename if needed
    155       if options['datum'] == None:
    156          datum = os.path.basename(csv_file).lstrip('ScanResult-').rstrip('.csv.gz')
    157       else:
    158          datum = options['datum']
    159       try:
    160          datum = datetime.datetime.strptime(datum,'%Y-%m-%d-%H%M%S')
    161       except ValueError:
    162         raise CommandError("Invalid date '%s'" % options['datum'])
    163 
    164       # Create meetrondje object
    165       g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'] , email=options['email'])
    166       a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
    167       mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
    168       logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
    169       if not created:
    170         logger.error("Meetrondje '%s' already imported" % mr)
    171         sys.exit(1)
    172       counters = import_droidstumbler(csv_file,mr)
    173       logger.info("summary accespoints: added:%(ap_added)-6s processed:%(ap_total)-6s" % counters)
    174       logger.info("summary metingen   : added:%(meting_added)-6s processed:%(meting_total)-6s" % counters)
     43  return (counters, ap_pool, None, meting_pool)

  • src/django_gheat/gheat/management/commands/import_datafile.py

    r9619 r9623  
    22# -*- coding: utf-8 -*-
    33#
    4 # Script for importing .ns1 files (Netstumber output)
     4# Script for importing various stumble files in a modular fasion:
     5# - .ns1 (Netstumber)
     6# - .gpsxml .netxml (Kismet)
     7# - DroidStumbler-*.csv (DroidStumber)
    58#
    69# Rick van der Zwet <info@rickvanderzwet.nl>
    … …  
    1922from collections import defaultdict
    2023
    21 from netstumbler import parse_netstumbler
     24import netstumbler
     25import kismet
     26import droidstumbler
    2227from import_droidstumbler import bulk_sql,get_organization_id_by_ssid
    2328
    … …  
    3237  return open(file,'rb')
    3338
     39# Prefix/Suffix removal
     40valid_prefix = ['DroidStumbler-', 'Kismet-']
     41def strip_prefix(filename):
     42  for suffix in valid_suffixes:
     43    filename = filename.rstrip(suffix)
     44  return filename
     45valid_suffixes = ['.gz', '.gpsxml', '.netxml', '.csv', '.ns1']
     46def strip_suffix(filename):
     47  for suffix in valid_suffixes:
     48    filename = filename.rstrip(suffix)
     49  return filename
     50def strip_file(filename):
     51  return strip_suffix(strip_prefix(filename))
    3452
    3553
    … …  
    93111
    94112
    95 
    96 def process_netstumber(filename):
    97   data = parse_netstumbler(open_file(filename))
    98 
    99   counters = {
    100     'ap_added' : 0, 'ap_total' : 0,
    101     'ap_failed' : 0, 'ap_ignored' : 0,
    102     'client_added' : 0, 'client_total' : 0,
    103     'client_failed' : 0, 'client_ignored' : 0,
    104     'meting_added' : 0, 'meting_total' : 0,
    105     'meting_failed' : 0, 'meting_ignored' : 0
    106     }
    107 
    108   # Temponary holders
    109   meting_pool = defaultdict(list)
    110   ap_pool = {}
    111 
    112   for ap in data['aps']:
    113     # XXX: How is encryption coded?
    114     encryption = False
    115     ap_pool[ap['BSSID']]= (ap['SSID'], encryption)
    116     for point in ap["measurements"]:
    117      counters['meting_total'] += 1
    118      if point['LocationSource'] == 0:
    119        logger.debug("No GPS Coordinates found for BSSID %s @ %s",
    120          ap['BSSID'], point['Time'])
    121        counters['meting_ignored'] += 1
    122        continue
    123      # We store all values found, avg or max will be done later on
    124      key = (ap['BSSID'], point["Latitude"], point["Longitude"])
    125 
    126      # Known measurement error
    127      if point['Signal'] == -32767: continue
    128 
    129      # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
    130      signaal= 100 + point['Signal']
    131      if signaal > 100 or signaal < 0:
    132        logger.warning("Signal %s is not valid entry for BSSID %s @ %s",
    133          point['Signal'], ap['BSSID'], point['Time'])
    134        continue
    135      meting_pool[key].append(signaal)
    136 
    137   return (counters, ap_pool, meting_pool)
     113def import_clients(client_pool, counters):
     114  # Determine which Wireless Clients to add
     115  bssid_list_present = WirelessClient.objects.filter(mac__in=client_pool.keys()).values_list('mac', flat=True)
     116  bssid_list_insert = set(client_pool.keys()) - set(bssid_list_present)
     117
     118  # Create a bulk import list and import
     119  if bssid_list_insert:
     120    sql_values = []
     121    for bssid in bssid_list_insert:
     122      sql_values.append("('%s')" % bssid.upper())
     123    counters['client_added'] = bulk_sql('gheat_wirelessclient (`mac`)',sql_values)
     124
     125  return counters
     126
    138127
    139128
    … …  
    162151      raise CommandError("Not all arguments are provided")
    163152
    164     # Please first the netxml and then the gpsxml files
    165     sorted_args = [x for x in args if "ns1" in x]
     153    # Please first the netxml and the gpsxml files and the rest
     154    sorted_args = [x for x in args if "netxml" in x] +\
     155     [x for x in args if "gpsxml" in x] +\
     156     [x for x in args if "ns1" in x]
    166157    remainder = list(set(args) - set(sorted_args))
    167158    args = sorted_args + remainder
    168159    logger.debug("Parsing files in the following order: %s", args)
    169160
     161    # Make sure the all exists at first
    170162    for filename in args:
    171163      if not os.path.isfile(filename):
    172164        raise CommandError("file '%s' does not exists" % filename)
    173165
    174     def process_date(datestr):
    175       try:
    176          # Kismet-20110805-15-37-30-1
    177          return datetime.datetime.strptime(datestr,'%Y%m%d-%H-%M-%S-1')
    178       except ValueError:
    179         raise CommandError("Invalid date '%s'" % options['datum'])
    180 
     166
     167    def get_date(filename):
     168      def process_date(datestr):
     169        try:
     170           # Kismet-20110805-15-37-30-1
     171           return datetime.datetime.strptime(datestr,'%Y%m%d-%H-%M-%S-1')
     172        except ValueError:
     173          raise CommandError("Invalid date '%s'" % options['datum'])
     174      if options['datum'] == None:
     175         datestr = strip_file(os.path.basename(filename))
     176         datum = process_date(datestr)
     177      elif options['datum'] == 'now':
     178         datum = datetime.datetime.now()
     179      else:
     180         datum = process_date(options['datum'])
     181      return datum
     182
     183    def get_meetrondje(meetrondje):
     184      # Meetrondje from filename if needed
     185      if options['meetrondje'] == None:
     186        meetrondje = strip_suffix(os.path.basename(filename))
     187      else:
     188        meetrondje = options['meetrondje']
     189      return meetrondje
     190
     191    # Get Gheat Objects, pre-req
     192    g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'],
     193      email=options['email'])
     194    a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
     195
     196    # Check if all files are valid
    181197    for filename in args:
    182198      logger.info("Processing '%s'" % filename)
     199      mr, created = MeetRondje.objects.get_or_create(
     200        datum=get_date(filename), naam=get_meetrondje(filename),
     201        gebruiker=g, apparatuur=a)
     202      if not created:
     203        logger.error("Meetrondje '%s' already imported",  mr)
     204        continue
     205
     206      counters = {
     207        'ap_added' : 0, 'ap_total' : 0,
     208        'ap_failed' : 0, 'ap_ignored' : 0,
     209        'client_added' : 0, 'client_total' : 0,
     210        'client_failed' : 0, 'client_ignored' : 0,
     211        'meting_added' : 0, 'meting_total' : 0,
     212        'meting_failed' : 0, 'meting_ignored' : 0
     213        }
     214      logger.info('Meetrondje: %s', mr)
     215      fh = open_file(filename)
    183216      if 'ns1' in filename:
    184         if options['datum'] == None:
    185            datestr = os.path.basename(filename).lstrip('Kismet-').\
    186              rstrip('.gz').rstrip('.gpsxml').rstrip('.netxml').rstrip('.ns1')
    187            datum = process_date(datestr)
    188         elif options['datum'] == 'now':
    189            datum = datetime.datetime.now()
    190         else:
    191            datum = process_date(options['datum'])
    192 
    193         # Meetrondje from filename if needed
    194         if options['meetrondje'] == None:
    195           meetrondje = os.path.basename(filename).rstrip('.gz').rstrip('.ns1')
    196         else:
    197           meetrondje = options['meetrondje']
    198 
    199         # Create meetrondje object
    200         g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'],
    201           email=options['email'])
    202         a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
    203         mr, created = MeetRondje.objects.get_or_create(datum=datum,
    204           naam=meetrondje, gebruiker=g, apparatuur=a)
    205         logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
    206         if not created:
    207           logger.error("Meetrondje '%s' already imported" % mr)
    208           continue
    209         (counters, ap_pool, meting_pool) = process_netstumber(filename)
    210         counters = import_accespoints(ap_pool, counters)
    211         counters = import_metingen(mr, meting_pool, counters)
    212 
    213         logger.info(("summary metingen   : total:%(meting_total)-6s" +
    214           "added:%(meting_added)-6s failed:%(meting_failed)-6s" +
    215           "ignored:%(meting_ignored)-6s") % counters)
     217        (counters, ap_pool, client_pool, meting_pool) = netstumbler.process_ns1(fh, counters)
     218      elif 'gpsxml' in filename:
     219        (counters, ap_pool, client_pool, meting_pool) = kismet.process_gpsxml(fh, counters)
     220      elif 'netxml' in filename:
     221        (counters, ap_pool, client_pool, meting_pool) = kismet.process_netxml(fh, counters)
     222      elif 'ScanResult' in filename:
     223        (counters, ap_pool, client_pool, meting_pool) = droidstumbler.process_csv(fh, counters)
    216224      else:
    217225        raise CommandError("file '%s' format not recognized" % filename)
     226
     227      if ap_pool:
     228        counters = import_accespoints(ap_pool, counters)
     229      if client_pool:
     230        counters = import_clients(client_pool, counters)
     231      if meting_pool:
     232        counters = import_metingen(mr, meting_pool, counters)
     233
     234      logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
     235      logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
     236      logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)

  • src/django_gheat/gheat/management/commands/import_netstumbler.py

    r9619 r9623  
    9494
    9595
    96 def process_netstumber(filename):
     96def process_ns1(filename):
    9797  data = parse_netstumbler(open_file(filename))
    9898

  • src/django_gheat/gheat/management/commands/kismet.py

    r9592 r9623  
    22# -*- coding: utf-8 -*-
    33#
    4 # Script for importing .gpsxml and .netxml files (Kismet output)
    5 #
    64# Rick van der Zwet <info@rickvanderzwet.nl>
    75#
    8 from django.core.management.base import BaseCommand,CommandError
    9 from django.db.utils import IntegrityError
    10 from optparse import OptionParser, make_option
    11 from gheat.models import *
    126from lxml import etree
    13 import datetime
    14 import gzip
    15 import os
    16 import sys
    177import logging
    188
    199from collections import defaultdict
    2010
    21 from import_droidstumbler import bulk_sql,get_organization_id_by_ssid
    22 
    2311logger = logging.getLogger(__name__)
    2412logger.setLevel(logging.INFO)
    2513
    26 # Open files for reading
    27 def open_file(file):
    28  if file.endswith('.gz'):
    29    return gzip.open(file,'rb')
    30  else:
    31   return open(file,'rb')
    32 
    33 
    34 
    35 def import_kismet_netxml(netxml_file):
    36   netxml_doc = etree.parse(open_file(netxml_file))
    37 
    38   counters = { 'ap_added' : 0, 'ap_total' : 0, 'ap_failed' : 0, 'ap_ignored' : 0,
    39                'client_added' : 0, 'client_total' : 0, 'client_failed' : 0, 'client_ignored' : 0}
     14def process_netxml(fh,counters):
     15  netxml_doc = etree.parse(fh)
    4016
    4117  # Prepare new accespoints and measurements
    … …  
    6541      logger.error('Unknown type %s - %s',bssid, wnetwork.attrib['type'])
    6642
    67 
    68   # Determine which Accespoints to add
    69   bssid_list_present = Accespoint.objects.filter(mac__in=ap_pool.keys()).values_list('mac', flat=True)
    70   bssid_list_insert = set(ap_pool.keys()) - set(bssid_list_present)
    71 
    72   # Create a bulk import list and import
    73   if bssid_list_insert:
    74     sql_values = []
    75     for bssid in bssid_list_insert:
    76       ssid, encryption = ap_pool[bssid]
    77       # Special trick in SSID ts avoid escaping in later stage
    78       item = str((bssid.upper(),ssid.replace('%','%%'),encryption,get_organization_id_by_ssid(ssid)))
    79       sql_values.append(item)
    80     counters['ap_added'] = bulk_sql('gheat_accespoint (`mac`, `ssid`, `encryptie`, `organization_id`)',sql_values)
    81 
    82   # Determine which Wireless Clients to add
    83   bssid_list_present = WirelessClient.objects.filter(mac__in=client_pool.keys()).values_list('mac', flat=True)
    84   bssid_list_insert = set(client_pool.keys()) - set(bssid_list_present)
    85 
    86   # Create a bulk import list and import
    87   if bssid_list_insert:
    88     sql_values = []
    89     for bssid in bssid_list_insert:
    90       sql_values.append("('%s')" % bssid.upper())
    91     counters['client_added'] = bulk_sql('gheat_wirelessclient (`mac`)',sql_values)
    92 
    93   return counters
     43  return (counters, ap_pool, None, None)
    9444
    9545
    9646
    97 def import_kismet_gpsxml(gpsxml_file, meetrondje):
    98   gpsxml_doc = etree.parse(open_file(gpsxml_file))
    99 
    100   #Various statistics
    101   counters = {'meting_added' : 0, 'meting_total' : 0, 'meting_failed' : 0, 'meting_ignored' :0}
     47def process_gpsxml(fh,counters):
     48  gpsxml_doc = etree.parse(fh)
    10249
    10350  bssid_failed = defaultdict(int)
    … …  
    13077    signaal=100 + int(level)
    13178    meting_pool[key].append(signaal)
    132 
    133   bssid_list = [x[0] for x in meting_pool.keys()]
    134   # Build mapping for meting import
    135   mac2id = {}
    136   for mac,id in Accespoint.objects.filter(mac__in=bssid_list).values_list('mac','id'):
    137     mac2id[mac] = int(id)
    138 
    139   clients = {}
    140   for mac in WirelessClient.objects.filter(mac__in=bssid_list).values_list('mac',flat=True):
    141     clients[mac] = True
    142 
    143   sql_values = []
    144   for (bssid,lat,lon),signals in meting_pool.iteritems():
    145     if clients.has_key(bssid):
    146       counters['meting_ignored'] += len(signals)
    147     elif not mac2id.has_key(bssid):
    148       counters['meting_failed'] += len(signals)
    149       bssid_failed[bssid] += len(signals)
    150     else:
    151       item = str((int(meetrondje.id),mac2id[bssid],float(lat),float(lon),max(signals)))
    152       sql_values.append(item)
    153 
    154   for bssid,count in sorted(bssid_failed.items(),
    155       key=lambda item: item[1], reverse=True):
    156     logger.debug("Missing BSSID %s found %3s times", bssid, count)
    157 
    158   if sql_values:
    159     counters['meting_added'] = bulk_sql('gheat_meting (`meetrondje_id`, `accespoint_id`, `lat`, `lng`, `signaal`)',sql_values)
    160   return counters
    161 
    162 
    163 class Command(BaseCommand):
    164   args = '<gpsxml|netxml>[.gz] [gpsxml2[.gz]  gpsxml3[.gz] ...]'
    165   option_list = BaseCommand.option_list + (
    166     make_option('-k', '--kaart', dest='kaart', default='onbekend', help="Kaart gebruikt"),
    167     make_option('-m', '--meetrondje', dest='meetrondje', default=None),
    168     make_option('-g', '--gebruiker', dest='gebruiker', default='username',help='Naam van de persoon die de meting uitgevoerd heeft'),
    169     make_option('-e', '--email', dest='email', default='foo@bar.org',help='Email van de persoon die de meting uitgevoerd heeft'),
    170     make_option('-d', '--datum', dest='datum', default=None, help="Provide date  \
    171       in following format: '%Y%m%d-%H-%M-%S-1', by default it will be generated from \
    172       the filename"),
    173   )
    174 
    175   def handle(self, *args, **options):
    176     if len(args) == 0:
    177       self.print_help(sys.argv[0],sys.argv[1])
    178       raise CommandError("Not all arguments are provided")
    179 
    180     # Please first the netxml and then the gpsxml files
    181     sorted_args = [x for x in args if "netxml" in x] + [x for x in args if "gpsxml" in x]
    182     remainder = list(set(args) - set(sorted_args))
    183     args = sorted_args + remainder
    184     logger.debug("Parsing files in the following order: %s", args)
    185 
    186     for xml_file in args:
    187       if not os.path.isfile(xml_file):
    188         raise CommandError("xml file '%s' does not exists" % xml_file)
    189 
    190     for xml_file in args:
    191       logger.info("Processing '%s'" % xml_file)
    192       if 'netxml' in xml_file:
    193         counters = import_kismet_netxml(xml_file)
    194         logger.info("summary accespoints: total:%(ap_total)-6s added:%(ap_added)-6s failed:%(ap_failed)-6s ignored:%(ap_ignored)-6s" % counters)
    195         logger.info("summary client     : total:%(client_total)-6s added:%(client_added)-6s failed:%(client_failed)-6s ignored:%(client_ignored)-6s" % counters)
    196       elif 'gpsxml' in xml_file:
    197         if options['datum'] == None:
    198            datum = os.path.basename(xml_file).lstrip('Kismet-').rstrip('.gz').rstrip('.gpsxml').rstrip('.netxml')
    199         else:
    200            datum = options['datum']
    201         try:
    202            # Kismet-20110805-15-37-30-1
    203            datum = datetime.datetime.strptime(datum,'%Y%m%d-%H-%M-%S-1')
    204         except ValueError:
    205           raise CommandError("Invalid date '%s'" % options['datum'])
    206 
    207         # Meetrondje from filename if needed
    208         if options['meetrondje'] == None:
    209           meetrondje = os.path.basename(xml_file).rstrip('.gz').rstrip('.gpsxml')
    210         else:
    211           meetrondje = options['meetrondje']
    212 
    213         # Create meetrondje object
    214         g, created = Gebruiker.objects.get_or_create(naam=options['gebruiker'] , email=options['email'])
    215         a, created = Apparatuur.objects.get_or_create(kaart=options['kaart'])
    216         mr, created = MeetRondje.objects.get_or_create(datum=datum , naam=meetrondje , gebruiker=g , apparatuur=a)
    217         logger.info('Meetrondje: %s @ %s' % (meetrondje, datum))
    218         if not created:
    219           logger.error("Meetrondje '%s' already imported" % mr)
    220           continue
    221         counters = import_kismet_gpsxml(xml_file, mr)
    222         logger.info("summary metingen   : total:%(meting_total)-6s added:%(meting_added)-6s failed:%(meting_failed)-6s ignored:%(meting_ignored)-6s" % counters)
    223       else:
    224         raise CommandError("xml file '%s' format not recognized" % xml_file)
     79  return (counters, None, None, meting_pool)

  • src/django_gheat/gheat/management/commands/netstumbler.py

    r9618 r9623  
    88from struct import unpack
    99
    10 def parse_netstumbler(fh):
     10from collections import defaultdict
     11import logging
     12
     13logger = logging.getLogger(__name__)
     14logger.setLevel(logging.INFO)
     15
     16def parse_ns1(fh):
    1117  def get_int32(size=1):
    1218    v = unpack('<' + 'i'*size,fh.read(4*size))
    … …  
    103109  return data
    104110
     111def process_ns1(fh, counters):
     112  data = parse_ns1(fh)
     113
     114
     115  # Temponary holders
     116  meting_pool = defaultdict(list)
     117  ap_pool = {}
     118
     119  for ap in data['aps']:
     120    # XXX: How is encryption coded?
     121    encryption = False
     122    ap_pool[ap['BSSID']]= (ap['SSID'], encryption)
     123    for point in ap["measurements"]:
     124     counters['meting_total'] += 1
     125     if point['LocationSource'] == 0:
     126       logger.debug("No GPS Coordinates found for BSSID %s @ %s",
     127         ap['BSSID'], point['Time'])
     128       counters['meting_ignored'] += 1
     129       continue
     130     # We store all values found, avg or max will be done later on
     131     key = (ap['BSSID'], point["Latitude"], point["Longitude"])
     132
     133     # Known measurement error
     134     if point['Signal'] == -32767: continue
     135
     136     # XXX: Signal need properly be a relation of signal_dbm and noice_dbm
     137     signaal= 100 + point['Signal']
     138     if signaal > 100 or signaal < 0:
     139       logger.warning("Signal %s is not valid entry for BSSID %s @ %s",
     140         point['Signal'], ap['BSSID'], point['Time'])
     141       continue
     142     meting_pool[key].append(signaal)
     143
     144  return (counters, ap_pool, None, meting_pool)
    105145if __name__ == '__main__':
    106146  import sys
    107147  import pprint
    108148  pp = pprint.PrettyPrinter(indent=2)
    109   pp.pprint(parse_netstumbler(open(sys.argv[1],'r')))
     149  pp.pprint(parse_ns1(open(sys.argv[1],'r')))

  • src/django_gheat/import_scan_data.sh

    r9569 r9623  
    2222#Kismet netxml Imports
    2323FILES="`find $DATAROOT -name '*netxml*' | sort`"
    24 [ -n "$FILES" ] && $MANAGE import_kismet $ARGS $FILES
     24[ -n "$FILES" ] && $MANAGE import_common $ARGS $FILES
    2525
    2626# Import all measure data
    … …  
    3434    #Kismet gpsxml Imports
    3535    FILES="`find $DEVICEDIR -name '*gpsxml*'`"
    36     [ -n "$FILES" ] && $MANAGE import_kismet $ARGS $FILES
     36    [ -n "$FILES" ] && $MANAGE import_datafile $ARGS $FILES
    3737
    3838    # DroidStumber imports
    3939    FILES=`find $DEVICEDIR -name 'ScanResult-*'`
    40     [ -n "$FILES" ] && $MANAGE import_droidstumbler $ARGS $FILES
     40    [ -n "$FILES" ] && $MANAGE import_datafile $ARGS $FILES
     41
     42    # Kismet imports
     43    FILES=`find $DEVICEDIR -name '*.ns1'`
     44    [ -n "$FILES" ] && $MANAGE import_datafile $ARGS $FILES
    4145  done
    4246done
Note: See TracChangeset for help on using the changeset viewer.