dte=$2 #rm -rf "$HOME/temp/files/raw/$dte" rawDir="$HOME/temp/files/raw/" mkdir -p "$rawDir" #s3cmd sync s3://bucketName/import/nokia/store/downloads_completed/"$dte" "$rawDir" cd "$rawDir$dte" for a in *.gz; do gunzip $a; done cd "$HOME/temp/backfillTemp/" a=`find "$rawDir$dte" -type f | wc -l` #echo "$a" n=$1 b=$(((a+n-1)/n)) #echo "$b" for ((i=1;i<$n;i++)); do #echo "$i" targetDir="$HOME/temp/files/part$i" #echo "$targetDir" mkdir -p "$targetDir" finalCmd="find $rawDir$dte -type f | head -n $b | xargs -r sh -c 'mv \"\$0\" \"\$@\" $targetDir'" #echo "$finalCmd" eval $finalCmd realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' >/dev/null 2>&1 &" #realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' > $i.txt &" #echo "$realCmd" eval $realCmd pidArray[$i]=$! done ### moving rest files #echo "$i" targetDir="$HOME/temp/files/part$i" #echo "$targetDir" mkdir -p "$targetDir" finalCmd="find $rawDir$dte -type f | xargs -r sh -c 'mv \"\$0\" \"\$@\" $targetDir'" #echo "$finalCmd" eval $finalCmd realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' >/dev/null 2>&1 &" #realCmd="nohup $HOME/etl/transformScripts/ETLBackfill/doit.sh '$i' '$dte' > $i.txt &" #echo "$realCmd" eval $realCmd pidArray[$i]=$! for ((i=1;i<$n;i++)); do wait ${pidArray[$i]} done
Tuesday, May 20, 2014
shell script to move files under one directory to multiple other directory.
Thursday, May 15, 2014
setup scala version manager on CentOS
[root@new-host yiyujia]# yum install coreutils [root@new-host yiyujia]# wget https://raw.githubusercontent.com/yuroyoro/svm/master/svm [root@new-host yiyujia]# vi /usr/local/bin/svm [root@new-host yiyujia]# chmod +x /usr/local/bin/svm [root@new-host yiyujia]# chmod 755 /usr/local/bin/svm [root@new-host yiyujia]# vi ~/.bash_profile export SCALA_HOME=~/.svm/current/rt export PATH=$SCALA_HOME/bin:$PATH source ~/.bash_profile 01:26 yiyujia@new-host:~ $ svm install 2.11.2
http://scala.masanorihashimoto.com/2014/04/how-to-install-the-scala-version-manager-on-your-mac/
Subscribe to:
Posts (Atom)