Java大文件磁盘IO性能

Java大文件磁盘IO性能

我尝试了三种不同的方法来比较两个相同的3,8 gb文件，缓冲区大小在8 kb和1 MB之间。第一种方法仅使用两个缓冲的输入流

第二种方法使用一个线程池，该线程池读取两个不同的线程并在第三个线程中进行比较。这会以较高的cpu利用率为代价获得更高的吞吐量。那些短期运行的任务的线程池管理需要大量的开销。

第三种方法使用nio，如laginimaineb发布

如您所见，一般的方法相差不大。更重要的是正确的缓冲区大小。

奇怪的是我使用线程少读了1个字节。我无法发现错误很难。

comparing just with two streamsI was equal, even after 3684070360 bytes and reading for 704813 ms (4,98MB/sec * 2) with a buffer size of 8 kBI was equal, even after 3684070360 bytes and reading for 578563 ms (6,07MB/sec * 2) with a buffer size of 16 kBI was equal, even after 3684070360 bytes and reading for 515422 ms (6,82MB/sec * 2) with a buffer size of 32 kBI was equal, even after 3684070360 bytes and reading for 534532 ms (6,57MB/sec * 2) with a buffer size of 64 kBI was equal, even after 3684070360 bytes and reading for 422953 ms (8,31MB/sec * 2) with a buffer size of 128 kBI was equal, even after 3684070360 bytes and reading for 793359 ms (4,43MB/sec * 2) with a buffer size of 256 kBI was equal, even after 3684070360 bytes and reading for 746344 ms (4,71MB/sec * 2) with a buffer size of 512 kBI was equal, even after 3684070360 bytes and reading for 669969 ms (5,24MB/sec * 2) with a buffer size of 1024 kBcomparing with threadsI was equal, even after 3684070359 bytes and reading for 602391 ms (5,83MB/sec * 2) with a buffer size of 8 kBI was equal, even after 3684070359 bytes and reading for 523156 ms (6,72MB/sec * 2) with a buffer size of 16 kBI was equal, even after 3684070359 bytes and reading for 527547 ms (6,66MB/sec * 2) with a buffer size of 32 kBI was equal, even after 3684070359 bytes and reading for 276750 ms (12,69MB/sec * 2) with a buffer size of 64 kBI was equal, even after 3684070359 bytes and reading for 493172 ms (7,12MB/sec * 2) with a buffer size of 128 kBI was equal, even after 3684070359 bytes and reading for 696781 ms (5,04MB/sec * 2) with a buffer size of 256 kBI was equal, even after 3684070359 bytes and reading for 727953 ms (4,83MB/sec * 2) with a buffer size of 512 kBI was equal, even after 3684070359 bytes and reading for 741000 ms (4,74MB/sec * 2) with a buffer size of 1024 kBcomparing with nioI was equal, even after 3684070360 bytes and reading for 661313 ms (5,31MB/sec * 2) with a buffer size of 8 kBI was equal, even after 3684070360 bytes and reading for 656156 ms (5,35MB/sec * 2) with a buffer size of 16 kBI was equal, even after 3684070360 bytes and reading for 491781 ms (7,14MB/sec * 2) with a buffer size of 32 kBI was equal, even after 3684070360 bytes and reading for 317360 ms (11,07MB/sec * 2) with a buffer size of 64 kBI was equal, even after 3684070360 bytes and reading for 643078 ms (5,46MB/sec * 2) with a buffer size of 128 kBI was equal, even after 3684070360 bytes and reading for 865016 ms (4,06MB/sec * 2) with a buffer size of 256 kBI was equal, even after 3684070360 bytes and reading for 716796 ms (4,90MB/sec * 2) with a buffer size of 512 kBI was equal, even after 3684070360 bytes and reading for 652016 ms (5,39MB/sec * 2) with a buffer size of 1024 kB

使用的代码：

import junit.framework.Assert;import org.junit.Before;import org.junit.Test;import java.io.BufferedInputStream;import java.io.File;import java.io.FileInputStream;import java.io.IOException;import java.nio.ByteBuffer;import java.nio.channels.FileChannel;import java.text.DecimalFormat;import java.text.NumberFormat;import java.util.Arrays;import java.util.concurrent.*;public class FileCompare {    private static final int MIN_BUFFER_SIZE = 1024 * 8;    private static final int MAX_BUFFER_SIZE = 1024 * 1024;    private String fileName1;    private String fileName2;    private long start;    private long totalbytes;    @Before    public void createInputStream() {        fileName1 = "bigFile.1";        fileName2 = "bigFile.2";    }    @Test    public void compareTwoFiles() throws IOException {        System.out.println("comparing just with two streams");        int currentBufferSize = MIN_BUFFER_SIZE;        while (currentBufferSize <= MAX_BUFFER_SIZE) { compareWithBufferSize(currentBufferSize); currentBufferSize *= 2;        }    }    @Test    public void compareTwoFilesFutures()  throws IOException, ExecutionException, InterruptedException {        System.out.println("comparing with threads");        int myBufferSize = MIN_BUFFER_SIZE;        while (myBufferSize <= MAX_BUFFER_SIZE) { start = System.currentTimeMillis(); totalbytes = 0; compareWithBufferSizeFutures(myBufferSize); myBufferSize *= 2;        }    }    @Test    public void compareTwoFilesNio() throws IOException {        System.out.println("comparing with nio");        int myBufferSize = MIN_BUFFER_SIZE;        while (myBufferSize <= MAX_BUFFER_SIZE) { start = System.currentTimeMillis(); totalbytes = 0; boolean wasEqual = isEqualsNio(myBufferSize); if (wasEqual) {     printAfterEquals(myBufferSize); } else {     Assert.fail("files were not equal"); } myBufferSize *= 2;        }    }    private void compareWithBufferSize(int myBufferSize) throws IOException {        final BufferedInputStream inputStream1 =     new BufferedInputStream(  new FileInputStream(new File(fileName1)),  myBufferSize);        byte[] buff1 = new byte[myBufferSize];        final BufferedInputStream inputStream2 =     new BufferedInputStream(  new FileInputStream(new File(fileName2)),  myBufferSize);        byte[] buff2 = new byte[myBufferSize];        int read1;        start = System.currentTimeMillis();        totalbytes = 0;        while ((read1 = inputStream1.read(buff1)) != -1) { totalbytes += read1; int read2 = inputStream2.read(buff2); if (read1 != read2) {     break; } if (!Arrays.equals(buff1, buff2)) {     break; }        }        if (read1 == -1) { printAfterEquals(myBufferSize);        } else { Assert.fail("files were not equal");        }        inputStream1.close();        inputStream2.close();    }    private void compareWithBufferSizeFutures(int myBufferSize) throws ExecutionException, InterruptedException, IOException {        final BufferedInputStream inputStream1 =     new BufferedInputStream(  new FileInputStream(          new File(fileName1)),  myBufferSize);        final BufferedInputStream inputStream2 =     new BufferedInputStream(  new FileInputStream(          new File(fileName2)),  myBufferSize);        final boolean wasEqual = isEqualsParallel(myBufferSize, inputStream1, inputStream2);        if (wasEqual) { printAfterEquals(myBufferSize);        } else { Assert.fail("files were not equal");        }        inputStream1.close();        inputStream2.close();    }    private boolean isEqualsParallel(int myBufferSize , final BufferedInputStream inputStream1 , final BufferedInputStream inputStream2) throws InterruptedException, ExecutionException {        final byte[] buff1Even = new byte[myBufferSize];        final byte[] buff1Odd = new byte[myBufferSize];        final byte[] buff2Even = new byte[myBufferSize];        final byte[] buff2Odd = new byte[myBufferSize];        final Callable<Integer> read1Even = new Callable<Integer>() { public Integer call() throws Exception {     return inputStream1.read(buff1Even); }        };        final Callable<Integer> read2Even = new Callable<Integer>() { public Integer call() throws Exception {     return inputStream2.read(buff2Even); }        };        final Callable<Integer> read1Odd = new Callable<Integer>() { public Integer call() throws Exception {     return inputStream1.read(buff1Odd); }        };        final Callable<Integer> read2Odd = new Callable<Integer>() { public Integer call() throws Exception {     return inputStream2.read(buff2Odd); }        };        final Callable<Boolean> oddEqualsArray = new Callable<Boolean>() { public Boolean call() throws Exception {     return Arrays.equals(buff1Odd, buff2Odd); }        };        final Callable<Boolean> evenEqualsArray = new Callable<Boolean>() { public Boolean call() throws Exception {     return Arrays.equals(buff1Even, buff2Even); }        };        ExecutorService executor = Executors.newCachedThreadPool();        boolean isEven = true;        Future<Integer> read1 = null;        Future<Integer> read2 = null;        Future<Boolean> isEqual = null;        int lastSize = 0;        while (true) { if (isEqual != null) {     if (!isEqual.get()) {         return false;     } else if (lastSize == -1) {         return true;     } } if (read1 != null) {     lastSize = read1.get();     totalbytes += lastSize;     final int size2 = read2.get();     if (lastSize != size2) {         return false;     } } isEven = !isEven; if (isEven) {     if (read1 != null) {         isEqual = executor.submit(oddEqualsArray);     }     read1 = executor.submit(read1Even);     read2 = executor.submit(read2Even); } else {     if (read1 != null) {         isEqual = executor.submit(evenEqualsArray);     }     read1 = executor.submit(read1Odd);     read2 = executor.submit(read2Odd); }        }    }    private boolean isEqualsNio(int myBufferSize) throws IOException {        FileChannel first = null, seconde = null;        try { first = new FileInputStream(fileName1).getChannel(); seconde = new FileInputStream(fileName2).getChannel(); if (first.size() != seconde.size()) {     return false; } ByteBuffer firstBuffer = ByteBuffer.allocateDirect(myBufferSize); ByteBuffer secondBuffer = ByteBuffer.allocateDirect(myBufferSize); int firstRead, secondRead; while (first.position() < first.size()) {     firstRead = first.read(firstBuffer);     totalbytes += firstRead;     secondRead = seconde.read(secondBuffer);     if (firstRead != secondRead) {         return false;     }     if (!nioBuffersEqual(firstBuffer, secondBuffer, firstRead)) {         return false;     } } return true;        } finally { if (first != null) {     first.close(); } if (seconde != null) {     seconde.close(); }        }    }    private static boolean nioBuffersEqual(ByteBuffer first, ByteBuffer second, final int length) {        if (first.limit() != second.limit() || length > first.limit()) { return false;        }        first.rewind();        second.rewind();        for (int i = 0; i < length; i++) { if (first.get() != second.get()) {     return false; }        }        return true;    }    private void printAfterEquals(int myBufferSize) {        NumberFormat nf = new DecimalFormat("#.00");        final long dur = System.currentTimeMillis() - start;        double seconds = dur / 1000d;        double megabytes = totalbytes / 1024 / 1024;        double rate = (megabytes) / seconds;        System.out.println("I was equal, even after " + totalbytes     + " bytes and reading for " + dur     + " ms (" + nf.format(rate) + "MB/sec * 2)" +     " with a buffer size of " + myBufferSize / 1024 + " kB");    }}