iDatamining.org

I am looking for projects to work on
Please contact with me at yiyu.jia@BostonInfoPro.com!

Tuesday, May 20, 2014

shell script to move files under one directory to multiple other directory.

dte=$2
#rm -rf "$HOME/temp/files/raw/$dte"
rawDir="$HOME/temp/files/raw/"
mkdir -p "$rawDir"
#s3cmd sync s3://bucketName/import/nokia/store/downloads_completed/"$dte" "$rawDir"
cd "$rawDir$dte"
for a in *.gz; do gunzip $a; done
cd "$HOME/temp/backfillTemp/"
a=`find "$rawDir$dte" -type f | wc -l`
#echo "$a"
n=$1
b=$(((a+n-1)/n)) 
#echo "$b"
for ((i=1;i<$n;i++));
do 
  #echo "$i"
  targetDir="$HOME/temp/files/part$i"
  #echo "$targetDir"  
  mkdir -p "$targetDir"  
  finalCmd="find $rawDir$dte -type f | head -n $b | xargs  -r sh -c 'mv \"\$0\" \"\$@\" $targetDir'"
  #echo "$finalCmd"
  eval $finalCmd
  realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' >/dev/null 2>&1 &"
  #realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' > $i.txt &"
  #echo "$realCmd"
  eval $realCmd
  pidArray[$i]=$!
done
### moving rest files
#echo "$i"
targetDir="$HOME/temp/files/part$i"
  #echo "$targetDir"
  mkdir -p "$targetDir"  
  finalCmd="find $rawDir$dte -type f | xargs  -r sh -c 'mv \"\$0\" \"\$@\" $targetDir'"
  #echo "$finalCmd"
  eval $finalCmd
  realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' >/dev/null 2>&1 &"
  #realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' > $i.txt &"
  #echo "$realCmd"
  eval $realCmd
  pidArray[$i]=$!
for ((i=1;i<$n;i++));
do
 wait ${pidArray[$i]} 
done

No comments:

Post a Comment