dte=$2
#rm -rf "$HOME/temp/files/raw/$dte"
rawDir="$HOME/temp/files/raw/"
mkdir -p "$rawDir"
#s3cmd sync s3://bucketName/import/nokia/store/downloads_completed/"$dte" "$rawDir"
cd "$rawDir$dte"
for a in *.gz; do gunzip $a; done
cd "$HOME/temp/backfillTemp/"
a=`find "$rawDir$dte" -type f | wc -l`
#echo "$a"
n=$1
b=$(((a+n-1)/n))
#echo "$b"
for ((i=1;i<$n;i++));
do
#echo "$i"
targetDir="$HOME/temp/files/part$i"
#echo "$targetDir"
mkdir -p "$targetDir"
finalCmd="find $rawDir$dte -type f | head -n $b | xargs -r sh -c 'mv \"\$0\" \"\$@\" $targetDir'"
#echo "$finalCmd"
eval $finalCmd
realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' >/dev/null 2>&1 &"
#realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' > $i.txt &"
#echo "$realCmd"
eval $realCmd
pidArray[$i]=$!
done
### moving rest files
#echo "$i"
targetDir="$HOME/temp/files/part$i"
#echo "$targetDir"
mkdir -p "$targetDir"
finalCmd="find $rawDir$dte -type f | xargs -r sh -c 'mv \"\$0\" \"\$@\" $targetDir'"
#echo "$finalCmd"
eval $finalCmd
realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' >/dev/null 2>&1 &"
#realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' > $i.txt &"
#echo "$realCmd"
eval $realCmd
pidArray[$i]=$!
for ((i=1;i<$n;i++));
do
wait ${pidArray[$i]}
done
Tuesday, May 20, 2014
shell script to move files under one directory to multiple other directory.
Subscribe to:
Post Comments (Atom)
No comments:
Post a Comment