source: src/django_gheat/import_scan_data.sh@ 11407

#!/bin/sh
#
# Quick and dirty hack to import all data
#
# Rick van der Zwet <rick@wirelessleiden.nl>

BASEROOT=`cd $(dirname $0); pwd -P`
DATAROOT="$BASEROOT/scan-data"
SSH_USER="backup"
MAP_CACHE=${MAP_CACHE:-"/usr/local/var/httpd/cache/maps"}
MANAGE="`dirname $0`/manage.py"
SSH_KEY="`dirname $0`/id_rsa"
export PYTHON_EGG_CACHE=/tmp/

DEBUG=${DEBUG:-"false"}
force=false

new_imports=false

if [ "$1" = "-f" ]; then
  force=true
fi

debug() {
  $DEBUG && echo $*
}

debug "# DATAROOT: $DATAROOT"
debug "# SSH_USER: $SSH_USER"

if [ ! -r "$SSH_KEY" ]; then
  echo "# ERROR: automatic ssh key $SSH_KEY not readable"
  exit 1
fi

debug "# Rsync webfolder"
changes_files=`rsync -O --out-format="%n%L" -az -e "ssh -i $SSH_KEY -l $SSH_USER -o StrictHostKeyChecking=no" tyan.wirelessleiden.nl:/usr/local/www/webfolder/scan-data/ $DATAROOT`

if ! $force && [ -z "$changes_files" ]; then
  exit 0
fi

debug "# unzip compressed .tar.gz files"
for FILE in `find $DATAROOT -name '*.tar.gz' | sort`; do
  TARDIR=`dirname $FILE`/`basename $FILE .tar.gz`
  if [ ! -d $TARDIR ]; then
    mkdir $TARDIR
    tar -C $TARDIR -xvzf $FILE
  fi
done

debug "# unzip compressed .zip files"
for FILE in `find $DATAROOT -name '*.zip' | sort`; do
  ARCHIVEDIR=`dirname $FILE`/`basename $FILE .zip`
  if [ ! -d $ARCHIVEDIR ]; then
    mkdir $ARCHIVEDIR
    (cd $ARCHIVEDIR; unzip -qn $FILE )
  fi
done

do_import() {
  $MANAGE import_datafile $* 1>/dev/null 2>/dev/null && new_imports=true
}

# Import all measure data
for USERDIR in `find $DATAROOT -mindepth 1 -maxdepth 1 -type d | sort`; do
  for DEVICEDIR in `find $USERDIR -mindepth 1 -maxdepth 1 -type d | sort`; do
    USERNAME=`basename $USERDIR`
    DEVICE=`basename $DEVICEDIR`
    ARGS="-b -g $USERNAME -e $USERNAME@wirelessleiden.nl -k $DEVICE"

    echo "# INFO: Processing $USERNAME/$DEVICE"
    # DroidStumber imports
    FILES=`find $DEVICEDIR -name 'ScanResult-*'`
    if [ -n "$FILES" ]; then
      do_import $ARGS $FILES
    fi

    #Kismet Imports
    FILES="`find $DEVICEDIR -name '*gpsxml*'`"
    for GPSXML in $FILES; do
      NETXML=`echo $GPSXML | sed 's/gpsxml/netxml/g' `
      do_import $ARGS $NETXML $GPSXML
    done

    # NetStumbler imports
    FILES=`find $DEVICEDIR -name '*.ns1'`
    if [ -n "$FILES" ]; then
      do_import $ARGS $FILES
    fi
  done
done

if $new_imports && [ -r $MAP_CACHE ]; then
  echo "# INFO: New imports, clearing cache of $MAP_CACHE"
  htcacheclean -v -l10K -p$MAP_CACHE 
fi