dte=$2 #rm -rf "$HOME/temp/files/raw/$dte" rawDir="$HOME/temp/files/raw/" mkdir -p "$rawDir" #s3cmd sync s3://bucketName/import/nokia/store/downloads_completed/"$dte" "$rawDir" cd "$rawDir$dte" for a in *.gz; do gunzip $a; done cd "$HOME/temp/backfillTemp/" a=`find "$rawDir$dte" -type f | wc -l` #echo "$a" n=$1 b=$(((a+n-1)/n)) #echo "$b" for ((i=1;i<$n;i++)); do #echo "$i" targetDir="$HOME/temp/files/part$i" #echo "$targetDir" mkdir -p "$targetDir" finalCmd="find $rawDir$dte -type f | head -n $b | xargs -r sh -c 'mv \"\$0\" \"\$@\" $targetDir'" #echo "$finalCmd" eval $finalCmd realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' >/dev/null 2>&1 &" #realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' > $i.txt &" #echo "$realCmd" eval $realCmd pidArray[$i]=$! done ### moving rest files #echo "$i" targetDir="$HOME/temp/files/part$i" #echo "$targetDir" mkdir -p "$targetDir" finalCmd="find $rawDir$dte -type f | xargs -r sh -c 'mv \"\$0\" \"\$@\" $targetDir'" #echo "$finalCmd" eval $finalCmd realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' >/dev/null 2>&1 &" #realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' > $i.txt &" #echo "$realCmd" eval $realCmd pidArray[$i]=$! for ((i=1;i<$n;i++)); do wait ${pidArray[$i]} done
Tuesday, May 20, 2014
shell script to move files under one directory to multiple other directory.
Subscribe to:
Post Comments (Atom)
No comments:
Post a Comment