source: src/django_gheat/import_scan_data.sh

Last change on this file was 11312, checked in by rick, 12 years ago

Be less verbose to make cron running happy.

  • Property svn:executable set to *
File size: 2.4 KB
Line 
1#!/bin/sh
2#
3# Quick and dirty hack to import all data
4#
5# Rick van der Zwet <rick@wirelessleiden.nl>
6
7BASEROOT=`cd $(dirname $0); pwd -P`
8DATAROOT="$BASEROOT/scan-data"
9SSH_USER="backup"
10MAP_CACHE=${MAP_CACHE:-"/usr/local/var/httpd/cache/maps"}
11MANAGE="`dirname $0`/manage.py"
12SSH_KEY="`dirname $0`/id_rsa"
13export PYTHON_EGG_CACHE=/tmp/
14
15DEBUG=${DEBUG:-"false"}
16force=false
17
18new_imports=false
19
20if [ "$1" = "-f" ]; then
21 force=true
22fi
23
24debug() {
25 $DEBUG && echo $*
26}
27
28debug "# DATAROOT: $DATAROOT"
29debug "# SSH_USER: $SSH_USER"
30
31if [ ! -r "$SSH_KEY" ]; then
32 echo "# ERROR: automatic ssh key $SSH_KEY not readable"
33 exit 1
34fi
35
36debug "# Rsync webfolder"
37changes_files=`rsync -O --out-format="%n%L" -az -e "ssh -i $SSH_KEY -l $SSH_USER -o StrictHostKeyChecking=no" tyan.wirelessleiden.nl:/usr/local/www/webfolder/scan-data/ $DATAROOT`
38
39if ! $force && [ -z "$changes_files" ]; then
40 exit 0
41fi
42
43debug "# unzip compressed .tar.gz files"
44for FILE in `find $DATAROOT -name '*.tar.gz' | sort`; do
45 TARDIR=`dirname $FILE`/`basename $FILE .tar.gz`
46 if [ ! -d $TARDIR ]; then
47 mkdir $TARDIR
48 tar -C $TARDIR -xvzf $FILE
49 fi
50done
51
52debug "# unzip compressed .zip files"
53for FILE in `find $DATAROOT -name '*.zip' | sort`; do
54 ARCHIVEDIR=`dirname $FILE`/`basename $FILE .zip`
55 if [ ! -d $ARCHIVEDIR ]; then
56 mkdir $ARCHIVEDIR
57 (cd $ARCHIVEDIR; unzip -qn $FILE )
58 fi
59done
60
61do_import() {
62 $MANAGE import_datafile $* 1>/dev/null 2>/dev/null && new_imports=true
63}
64
65# Import all measure data
66for USERDIR in `find $DATAROOT -mindepth 1 -maxdepth 1 -type d | sort`; do
67 for DEVICEDIR in `find $USERDIR -mindepth 1 -maxdepth 1 -type d | sort`; do
68 USERNAME=`basename $USERDIR`
69 DEVICE=`basename $DEVICEDIR`
70 ARGS="-b -g $USERNAME -e $USERNAME@wirelessleiden.nl -k $DEVICE"
71
72 echo "# INFO: Processing $USERNAME/$DEVICE"
73 # DroidStumber imports
74 FILES=`find $DEVICEDIR -name 'ScanResult-*'`
75 if [ -n "$FILES" ]; then
76 do_import $ARGS $FILES
77 fi
78
79 #Kismet Imports
80 FILES="`find $DEVICEDIR -name '*gpsxml*'`"
81 for GPSXML in $FILES; do
82 NETXML=`echo $GPSXML | sed 's/gpsxml/netxml/g' `
83 do_import $ARGS $NETXML $GPSXML
84 done
85
86 # NetStumbler imports
87 FILES=`find $DEVICEDIR -name '*.ns1'`
88 if [ -n "$FILES" ]; then
89 do_import $ARGS $FILES
90 fi
91 done
92done
93
94if $new_imports && [ -r $MAP_CACHE ]; then
95 echo "# INFO: New imports, clearing cache of $MAP_CACHE"
96 htcacheclean -v -l10K -p$MAP_CACHE
97fi
98
Note: See TracBrowser for help on using the repository browser.