-------------------------------------------------------------------------------- Notes -------------------------------------------------------------------------------- # Tar/SSH/Gzip tar -cf - /source/dir/ | gzip -c --fast | ssh user@host "gunzip -c | tar -xf - -C /dest/dir" tar -zcf - /source/dir/ ssh user@host "tar -zxf - -C /dest/dir" # Tar/SSH/Pigz tar -cf - /source/dir/ | pigz -c | ssh user@host "unpigz -c | tar -xf - -C /dest/dir" # Tar/NC/Gzip tar -cf - /source/dir/ | gzip -c | nc --> nc -l -p | gunzip -c | tar -xf - -C /dest/dir # Tar/NC/Pigz tar -cf - /source/dir/ | pigz -c | nc --> nc -l -p | unpigz -c | tar -xf - -C /dest/dir # LFTP (parallel mirror) lftp -u username,password sftp://ip-address -e 'mirror --only-newer --no-dereference --parallel=5 /remote/path/ /destination/;quit' # Rsync with compression (or ssh compression?) rsync -az /source/dir/ -e "ssh -C" user@host:/dest/dir/ -------------------------------------------------------------------------------- References -------------------------------------------------------------------------------- http://copy_millions_tiny_files.onlinephpfunctions.com/ http://moo.nac.uci.edu/~hjm/HOWTO_move_data.html http://moo.nac.uci.edu/~hjm/parsync/ http://intermediatesql.com/linux/scrap-the-scp-how-to-copy-data-fast-using-pigz-and-nc/ http://varunbpatil.github.io/2013/08/05/ncp/#.VwTJUyYoDtR -- http://pastebin.com/raw.php?i=VyHeasH0 http://www.gnu.org/software/parallel/man.html#EXAMPLE:-Parallelizing-rsync http://www.yourownlinux.com/2015/04/speed-up-file-transfers-using-rsync-with-gnu-parallel.html https://gist.github.com/rcoup/5358786 http://www.jveweb.net/en/archives/2011/01/running-rsync-as-a-daemon.html -------------------------------------------------------------------------------- Results -------------------------------------------------------------------------------- 5.8GB (500,000x 10k files) ## Parallel SFTP using LFTP (20 threads) [root@thewolf sda3]# time (lftp -e 'cd /mnt/HDA2/; mirror -R -P=20 FileTransferTest' -u root, sftp://192.168.1.241) cd ok, cwd=/mnt/HDA2 Total: 101 directories, 500103 files, 0 symlinks New: 500103 files, 0 symlinks 5210555944 bytes transferred in 2758 seconds (1.80M/s) lftp root@192.168.1.241:/mnt/HDA2> exit real 56m10.012s user 7m39.829s sys 1m41.887s ## Tar With Parallel Compression (pigz) Piped Over Netcat [root@thewolf sda3]# time (tar -cf - FileTransferTest/ | pigz -c --fast | nc 192.168.1.241 1234) real 55m39.562s user 1m32.268s sys 0m45.686s ## Tar With Parallel Compression (pigz) Piped Over SSH [root@thewolf sda3]# time (tar -cf - FileTransferTest/ | pigz -c --fast | ssh root@192.168.1.241 "tar -zxf - -C /mnt/HDA2/") real 55m5.405s user 1m44.976s sys 0m42.399s ## Tar Without Compression Piped Over SSH (weak encryption) [root@thewolf sda3]# time (tar -cf - FileTransferTest/ | ssh -c arcfour root@192.168.1.241 "tar -xf - -C /mnt/HDA2/") real 52m28.850s user 0m55.745s sys 0m49.625s ## Rsync (Default) Over SSH (Defualt) [root@nas4free /mnt/HDA2]# time (rsync -a root@192.168.1.99:/media/sda3/FileTransferTest /mnt/HDA2/); rm -rf FileTransferTest/ real 26m25.237s user 6m11.919s sys 14m34.115s ## Rsync (Default) Over SSH (Weak Encryption) [root@thewolf sda3]# time (rsync -a -e 'ssh -c arcfour' FileTransferTest root@192.168.1.241:/mnt/HDA2/) real 24m51.993s user 1m10.504s sys 0m53.578s ## Rsync Connecting to Rsync Daemon [root@thewolf sda3]# time (rsync -a FileTransferTest rsyncclient@192.168.1.241::share); real 21m12.252s user 0m21.331s sys 0m34.698s ## Rsync (Default) Over SSH (Compression and Weak Encryption) [root@thewolf sda3]# time (rsync -a -e 'ssh -C -c arcfour' FileTransferTest root@192.168.1.241:/mnt/HDA2/) real 20m26.390s user 2m43.820s sys 0m58.049s ## Rsync (Compression) Over SSH (Default) [root@thewolf sda3]# time (rsync -az FileTransferTest root@192.168.1.241:/mnt/HDA2/) real 20m24.405s user 6m11.842s sys 0m43.705s ## Rsync (Compression) Over SSH (Weak Encryption) [root@thewolf sda3]# time (rsync -az -e 'ssh -c arcfour' FileTransferTest root@192.168.1.241:/mnt/HDA2/) real 19m41.940s user 5m40.416s sys 0m47.681s ## Rsync (Max Compression) over SSH (Weak Cipher) [root@nas4free /mnt/HDA2]# time (rsync -az --compress-level=9 -e 'ssh -c arcfour' root@192.168.1.99:/media/sda3/FileTransferTest /mnt/HDA2/); real 19m39.693s user 3m56.371s sys 12m31.090s ## Rsync Connecting to Rsync Daemon (Compression) [root@thewolf sda3]# time (rsync -az FileTransferTest rsyncclient@192.168.1.241::share); real 19m0.474s user 5m22.454s sys 0m35.871s ## Rsync Connecting to Rsync Daemon (Max Compression) [root@thewolf sda3]# time (rsync -az --compress-level=9 FileTransferTest rsyncclient@192.168.1.241::share); real 18m30.249s user 5m18.394s sys 0m35.214s ## Rsync (Max Compression) over SSH (Compression and Weak Cipher) [root@nas4free /mnt/HDA2]# time (rsync -az --compress-level=9 -e 'ssh -C -c arcfour' root@192.168.1.99:/media/sda3/FileTransferTest /mnt/HDA2/); rm -rf FileTransferTest/ real 18m27.117s user 3m35.014s sys 12m34.970s ## Rsync (Compression) Over SSH (Compression and Weak Encryption) [root@thewolf sda3]# time (rsync -az -e 'ssh -C -c arcfour' FileTransferTest root@192.168.1.241:/mnt/HDA2/) real 18m19.447s user 6m8.785s sys 0m40.854s -------------------------------------------------------------------------------- Replication (Pulling from SRC to DEST) -------------------------------------------------------------------------------- [root@nas4free /mnt/HDA2]# for x in {1..5}; do echo "rsync -az -e 'ssh -C -c arcfour' :: $x"; time (rsync -az -e 'ssh -C -c arcfour' root@192.168.1.99:/media/sda3/FileTransferTest /mnt/HDA2/); rm -rf FileTransferTest/; done ## Rsync (Compression) Over SSH (Compression and Weak Encryption) rsync -az -e 'ssh -C -c arcfour' :: 1 real 18m21.124s user 3m35.029s sys 12m12.764s rsync -az -e 'ssh -C -c arcfour' :: 2 real 18m26.651s user 3m31.820s sys 12m39.276s rsync -az -e 'ssh -C -c arcfour' :: 3 real 18m22.436s user 3m31.505s sys 12m38.711s rsync -az -e 'ssh -C -c arcfour' :: 4 real 18m8.670s user 3m34.751s sys 12m12.833s rsync -az -e 'ssh -C -c arcfour' :: 5 real 18m14.150s user 3m34.955s sys 12m24.599s