Changeset 10817 for src/django_gheat/import_scan_data.sh

Timestamp:

May 13, 2012, 11:12:05 AM (13 years ago)

Author:

rick

Message:

Only run the heavy lifting is we actually do have new files.

While here remove the lets-fix-all-use-cases-in-one-script-and-make-it-unreadable. stuff.

File:

• src/django_gheat/import_scan_data.sh (modified) (8 diffs)

src/django_gheat/import_scan_data.sh

@@ -2 +2 @@
 #
 # Quick and dirty hack to import all data
+#
+# Rick van der Zwet <rick@wirelessleiden.nl>
 
 BASEROOT=`cd $(dirname $0); pwd -P`
 DATAROOT=${1:-"$BASEROOT/scan-data"}
+SSH_USER=${2:-"backup"}
 MAP_CACHE=${MAP_CACHE:-"/usr/local/var/httpd/cache/maps"}
 MANAGE="`dirname $0`/manage.py"
@@ -10 +13 @@
 export PYTHON_EGG_CACHE=/tmp/
 
-DATAROOT_CHOICES="$DATAROOT /usr/local/scan-data"
+DEBUG=${DEBUG:-"false"}
 
-DEBUG=${DEBUG:-false}
+new_imports=false
 
 debug() {
@@ -18 +21 @@
 }
 
-if [ "$1" ]; then
-  DATAROOT=$1
-else
-  for D in $DATAROOT_CHOICES; do
-    if [ -d $D ]; then
-      DATAROOT=$D
-    fi
-  done
-fi
 debug "# DATAROOT: $DATAROOT"
 debug "# SSH_USER: $SSH_USER"
 
 echo "# rsync webfolder" 1>&2
-if [ -r "$SSH_KEY" ]; then
-  SSH_KEY_OPT="-i $SSH_KEY"
-  SSH_USER=backup
-else
-  echo "# warning automatic ssh key $SSH_KEY not readable"
-  SSH_KEY_OPT=''
-  SSH_USER=${SSH_USER:-${SUDO_USER:-$USER}}
+if [ ! -r "$SSH_KEY" ]; then
+  echo "# ERROR: automatic ssh key $SSH_KEY not readable"
+  exit 1
 fi
-rsync -O --out-format="%n%L" -az -e "ssh $SSH_KEY_OPT -l $SSH_USER -o StrictHostKeyChecking=no" tyan.wirelessleiden.nl:/usr/local/www/webfolder/scan-data/ $DATAROOT
+
+changes_files=`rsync -O --out-format="%n%L" -az -e "ssh -i $SSH_KEY -l $SSH_USER -o StrictHostKeyChecking=no" tyan.wirelessleiden.nl:/usr/local/www/webfolder/scan-data/ $DATAROOT`
+
+if [ -z "$changes_files" ]; then
+  exit 0
+fi
 
 debug "# unzip compressed .tar.gz files"
@@ -59 +54 @@
 done
 
+do_import() {
+  $MANAGE import_datafile $* 1>/dev/null 2>/dev/null && new_imports=true
+}
+
 # Import all measure data
 for USERDIR in `find $DATAROOT -mindepth 1 -maxdepth 1 -type d | sort`; do
@@ -64 +63 @@
     USERNAME=`basename $USERDIR`
     DEVICE=`basename $DEVICEDIR`
-    ARGS="-g $USERNAME -e $USERNAME@wirelessleiden.nl -k $DEVICE"
+    ARGS="-b -g $USERNAME -e $USERNAME@wirelessleiden.nl -k $DEVICE"
 
     echo "# INFO: Processing $USERNAME/$DEVICE"
@@ -70 +69 @@
     FILES=`find $DEVICEDIR -name 'ScanResult-*'`
     if [ -n "$FILES" ]; then
-      $MANAGE import_datafile $ARGS $FILES
+      do_import $ARGS $FILES
     fi
 
@@ -77 +76 @@
     for GPSXML in $FILES; do
       NETXML=`echo $GPSXML | sed 's/gpsxml/netxml/g' `
-      $MANAGE import_datafile $ARGS $NETXML $GPSXML
+      do_import $ARGS $NETXML $GPSXML
     done
 
@@ -83 +82 @@
     FILES=`find $DEVICEDIR -name '*.ns1'`
     if [ -n "$FILES" ]; then
-      $MANAGE import_datafile $ARGS $FILES
+      do_import $ARGS $FILES
     fi
   done
 done
 
-if [ -r $MAP_CACHE ]; then
-  debug "# Clear cache $MAP_CACHE"
+if $new_imports && [ -r $MAP_CACHE ]; then
+  echo "# INFO: New imports, clearing cache of $MAP_CACHE"
   htcacheclean -v -l10K -p$MAP_CACHE 
 fi