source: src/django_gheat/import_scan_data.sh@ 10615

#!/bin/sh
#
# Quick and dirty hack to import all data

BASEROOT=`cd $(dirname $0); pwd -P`
DATAROOT=${1:-"$BASEROOT/scan-data"}
MAP_CACHE=${MAP_CACHE:-"/usr/local/var/httpd/cache/maps"}
MANAGE="`dirname $0`/manage.py"
SSH_KEY="`dirname $0`/id_rsa"
export PYTHON_EGG_CACHE=/tmp/

DATAROOT_CHOICES="$DATAROOT /usr/local/scan-data"

DEBUG=${DEBUG:-false}

debug() {
  $DEBUG && echo $*
}

if [ "$1" ]; then
  DATAROOT=$1
else
  for D in $DATAROOT_CHOICES; do
    if [ -d $D ]; then
      DATAROOT=$D
    fi
  done
fi
debug "# DATAROOT: $DATAROOT"
debug "# SSH_USER: $SSH_USER"

echo "# rsync webfolder" 1>&2
if [ -r "$SSH_KEY" ]; then
  SSH_KEY_OPT="-i $SSH_KEY"
  SSH_USER=backup
else
  echo "# warning automatic ssh key $SSH_KEY not readable"
  SSH_KEY_OPT=''
  SSH_USER=${SSH_USER:-${SUDO_USER:-$USER}}
fi
rsync -O --out-format="%n%L" -az -e "ssh $SSH_KEY_OPT -l $SSH_USER -o StrictHostKeyChecking=no" tyan.wirelessleiden.nl:/usr/local/www/webfolder/scan-data/ $DATAROOT

debug "# unzip compressed .tar.gz files"
for FILE in `find $DATAROOT -name '*.tar.gz' | sort`; do
  TARDIR=`dirname $FILE`/`basename $FILE .tar.gz`
  if [ ! -d $TARDIR ]; then
    mkdir $TARDIR
    tar -C $TARDIR -xvzf $FILE
  fi
done

debug "# unzip compressed .zip files"
for FILE in `find $DATAROOT -name '*.zip' | sort`; do
  ARCHIVEDIR=`dirname $FILE`/`basename $FILE .zip`
  if [ ! -d $ARCHIVEDIR ]; then
    mkdir $ARCHIVEDIR
    (cd $ARCHIVEDIR; unzip -qn $FILE )
  fi
done

# Import all measure data
for USERDIR in `find $DATAROOT -mindepth 1 -maxdepth 1 -type d | sort`; do
  for DEVICEDIR in `find $USERDIR -mindepth 1 -maxdepth 1 -type d | sort`; do
    USERNAME=`basename $USERDIR`
    DEVICE=`basename $DEVICEDIR`
    ARGS="-g $USERNAME -e $USERNAME@wirelessleiden.nl -k $DEVICE"

    echo "# INFO: Processing $USERNAME/$DEVICE"
    # DroidStumber imports
    FILES=`find $DEVICEDIR -name 'ScanResult-*'`
    if [ -n "$FILES" ]; then
      $MANAGE import_datafile $ARGS $FILES
    fi

    #Kismet Imports
    FILES="`find $DEVICEDIR -name '*gpsxml*'`"
    for GPSXML in $FILES; do
      NETXML=`echo $GPSXML | sed 's/gpsxml/netxml/g' `
      $MANAGE import_datafile $ARGS $NETXML $GPSXML
    done

    # NetStumbler imports
    FILES=`find $DEVICEDIR -name '*.ns1'`
    if [ -n "$FILES" ]; then
      $MANAGE import_datafile $ARGS $FILES
    fi
  done
done

if [ -r $MAP_CACHE ]; then
  debug "# Clear cache $MAP_CACHE"
  htcacheclean -v -l10K -p$MAP_CACHE 
fi