package org.apache.lucene.index;

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;

import java.io.File;
import java.io.IOException;

import static org.apache.lucene.store.TestHelper.isSimpleFSIndexInput;
import static org.apache.lucene.store.TestHelper.isSimpleFSIndexInputOpen;

public class TestCompoundFile extends LuceneTestCase
{
    private Directory dir;

    @Override
    public void setUp() throws Exception {
       super.setUp();
       File file = createTempDir("testIndex");
       // use a simple FSDir here, to be sure to have SimpleFSInputs
       dir = new SimpleFSDirectory(file,null);
    }

    @Override
    public void tearDown() throws Exception {
       dir.close();
       super.tearDown();
    }

    /** Creates a file of the specified size with random data. */
    private void createRandomFile(Directory dir, String name, int size)
    throws IOException
    {
        IndexOutput os = dir.createOutput(name, newIOContext(random()));
        for (int i=0; i<size; i++) {
            byte b = (byte) (Math.random() * 256);
            os.writeByte(b);
        }
        os.close();
    }

    /** Creates a file of the specified size with sequential data. The first
     *  byte is written as the start byte provided. All subsequent bytes are
     *  computed as start + offset where offset is the number of the byte.
     */
    private void createSequenceFile(Directory dir,
                                    String name,
                                    byte start,
                                    int size)
    throws IOException
    {
        IndexOutput os = dir.createOutput(name, newIOContext(random()));
        for (int i=0; i < size; i++) {
            os.writeByte(start);
            start ++;
        }
        os.close();
    }


    private void assertSameStreams(String msg,
                                   IndexInput expected,
                                   IndexInput test)
    throws IOException
    {
        assertNotNull(msg + " null expected", expected);
        assertNotNull(msg + " null test", test);
        assertEquals(msg + " length", expected.length(), test.length());
        assertEquals(msg + " position", expected.getFilePointer(),
                                        test.getFilePointer());

        byte expectedBuffer[] = new byte[512];
        byte testBuffer[] = new byte[expectedBuffer.length];

        long remainder = expected.length() - expected.getFilePointer();
        while(remainder > 0) {
            int readLen = (int) Math.min(remainder, expectedBuffer.length);
            expected.readBytes(expectedBuffer, 0, readLen);
            test.readBytes(testBuffer, 0, readLen);
            assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer,
                testBuffer, 0, readLen);
            remainder -= readLen;
        }
    }


    private void assertSameStreams(String msg,
                                   IndexInput expected,
                                   IndexInput actual,
                                   long seekTo)
    throws IOException
    {
        if(seekTo >= 0 && seekTo < expected.length())
        {
            expected.seek(seekTo);
            actual.seek(seekTo);
            assertSameStreams(msg + ", seek(mid)", expected, actual);
        }
    }



    private void assertSameSeekBehavior(String msg,
                                        IndexInput expected,
                                        IndexInput actual)
    throws IOException
    {
        // seek to 0
        long point = 0;
        assertSameStreams(msg + ", seek(0)", expected, actual, point);

        // seek to middle
        point = expected.length() / 2l;
        assertSameStreams(msg + ", seek(mid)", expected, actual, point);

        // seek to end - 2
        point = expected.length() - 2;
        assertSameStreams(msg + ", seek(end-2)", expected, actual, point);

        // seek to end - 1
        point = expected.length() - 1;
        assertSameStreams(msg + ", seek(end-1)", expected, actual, point);

        // seek to the end
        point = expected.length();
        assertSameStreams(msg + ", seek(end)", expected, actual, point);

        // seek past end
        point = expected.length() + 1;
        assertSameStreams(msg + ", seek(end+1)", expected, actual, point);
    }


    private void assertEqualArrays(String msg,
                                   byte[] expected,
                                   byte[] test,
                                   int start,
                                   int len)
    {
        assertNotNull(msg + " null expected", expected);
        assertNotNull(msg + " null test", test);

        for (int i=start; i<len; i++) {
            assertEquals(msg + " " + i, expected[i], test[i]);
        }
    }


    // ===========================================================
    //  Tests of the basic CompoundFile functionality
    // ===========================================================


    /** This test creates compound file based on a single file.
     *  Files of different sizes are tested: 0, 1, 10, 100 bytes.
     */
    public void testSingleFile() throws IOException {
        int data[] = new int[] { 0, 1, 10, 100 };
        for (int i=0; i<data.length; i++) {
            String name = "t" + data[i];
            createSequenceFile(dir, name, (byte) 0, data[i]);
            CompoundFileDirectory csw = new CompoundFileDirectory(dir, name + ".cfs", newIOContext(random()), true);
            dir.copy(csw, name, name, newIOContext(random()));
            csw.close();

            CompoundFileDirectory csr = new CompoundFileDirectory(dir, name + ".cfs", newIOContext(random()), false);
            IndexInput expected = dir.openInput(name, newIOContext(random()));
            IndexInput actual = csr.openInput(name, newIOContext(random()));
            assertSameStreams(name, expected, actual);
            assertSameSeekBehavior(name, expected, actual);
            expected.close();
            actual.close();
            csr.close();
        }
    }


    /** This test creates compound file based on two files.
     *
     */
    public void testTwoFiles() throws IOException {
        createSequenceFile(dir, "d1", (byte) 0, 15);
        createSequenceFile(dir, "d2", (byte) 0, 114);

        CompoundFileDirectory csw = new CompoundFileDirectory(dir, "d.cfs", newIOContext(random()), true);
        dir.copy(csw, "d1", "d1", newIOContext(random()));
        dir.copy(csw, "d2", "d2", newIOContext(random()));
        csw.close();

        CompoundFileDirectory csr = new CompoundFileDirectory(dir, "d.cfs", newIOContext(random()), false);
        IndexInput expected = dir.openInput("d1", newIOContext(random()));
        IndexInput actual = csr.openInput("d1", newIOContext(random()));
        assertSameStreams("d1", expected, actual);
        assertSameSeekBehavior("d1", expected, actual);
        expected.close();
        actual.close();

        expected = dir.openInput("d2", newIOContext(random()));
        actual = csr.openInput("d2", newIOContext(random()));
        assertSameStreams("d2", expected, actual);
        assertSameSeekBehavior("d2", expected, actual);
        expected.close();
        actual.close();
        csr.close();
    }

    /** This test creates a compound file based on a large number of files of
     *  various length. The file content is generated randomly. The sizes range
     *  from 0 to 1Mb. Some of the sizes are selected to test the buffering
     *  logic in the file reading code. For this the chunk variable is set to
     *  the length of the buffer used internally by the compound file logic.
     */
    public void testRandomFiles() throws IOException {
        // Setup the test segment
        String segment = "test";
        int chunk = 1024; // internal buffer size used by the stream
        createRandomFile(dir, segment + ".zero", 0);
        createRandomFile(dir, segment + ".one", 1);
        createRandomFile(dir, segment + ".ten", 10);
        createRandomFile(dir, segment + ".hundred", 100);
        createRandomFile(dir, segment + ".big1", chunk);
        createRandomFile(dir, segment + ".big2", chunk - 1);
        createRandomFile(dir, segment + ".big3", chunk + 1);
        createRandomFile(dir, segment + ".big4", 3 * chunk);
        createRandomFile(dir, segment + ".big5", 3 * chunk - 1);
        createRandomFile(dir, segment + ".big6", 3 * chunk + 1);
        createRandomFile(dir, segment + ".big7", 1000 * chunk);

        // Setup extraneous files
        createRandomFile(dir, "onetwothree", 100);
        createRandomFile(dir, segment + ".notIn", 50);
        createRandomFile(dir, segment + ".notIn2", 51);

        // Now test
        CompoundFileDirectory csw = new CompoundFileDirectory(dir, "test.cfs", newIOContext(random()), true);
        final String data[] = new String[] {
            ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3",
            ".big4", ".big5", ".big6", ".big7"
        };
        for (int i=0; i<data.length; i++) {
            String fileName = segment + data[i];
            dir.copy(csw, fileName, fileName, newIOContext(random()));
        }
        csw.close();

        CompoundFileDirectory csr = new CompoundFileDirectory(dir, "test.cfs", newIOContext(random()), false);
        for (int i=0; i<data.length; i++) {
            IndexInput check = dir.openInput(segment + data[i], newIOContext(random()));
            IndexInput test = csr.openInput(segment + data[i], newIOContext(random()));
            assertSameStreams(data[i], check, test);
            assertSameSeekBehavior(data[i], check, test);
            test.close();
            check.close();
        }
        csr.close();
    }


    /** Setup a larger compound file with a number of components, each of
     *  which is a sequential file (so that we can easily tell that we are
     *  reading in the right byte). The methods sets up 20 files - f0 to f19,
     *  the size of each file is 1000 bytes.
     */
    private void setUp_2() throws IOException {
        CompoundFileDirectory cw = new CompoundFileDirectory(dir, "f.comp", newIOContext(random()), true);
        for (int i=0; i<20; i++) {
            createSequenceFile(dir, "f" + i, (byte) 0, 2000);
            String fileName = "f" + i;
            dir.copy(cw, fileName, fileName, newIOContext(random()));
        }
        cw.close();
    }


    public void testReadAfterClose() throws IOException {
        demo_FSIndexInputBug(dir, "test");
    }

    private void demo_FSIndexInputBug(Directory fsdir, String file)
    throws IOException
    {
        // Setup the test file - we need more than 1024 bytes
        IndexOutput os = fsdir.createOutput(file, IOContext.DEFAULT);
        for(int i=0; i<2000; i++) {
            os.writeByte((byte) i);
        }
        os.close();

        IndexInput in = fsdir.openInput(file, IOContext.DEFAULT);

        // This read primes the buffer in IndexInput
        in.readByte();

        // Close the file
        in.close();

        // ERROR: this call should fail, but succeeds because the buffer
        // is still filled
        in.readByte();

        // ERROR: this call should fail, but succeeds for some reason as well
        in.seek(1099);

        try {
            // OK: this call correctly fails. We are now past the 1024 internal
            // buffer, so an actual IO is attempted, which fails
            in.readByte();
            fail("expected readByte() to throw exception");
        } catch (IOException e) {
          // expected exception
        }
    }

    public void testClonedStreamsClosing() throws IOException {
        setUp_2();
        CompoundFileDirectory cr = new CompoundFileDirectory(dir, "f.comp", newIOContext(random()), false);

        // basic clone
        IndexInput expected = dir.openInput("f11", newIOContext(random()));

        // this test only works for FSIndexInput
        assertTrue(isSimpleFSIndexInput(expected));
        assertTrue(isSimpleFSIndexInputOpen(expected));

        IndexInput one = cr.openInput("f11", newIOContext(random()));

        IndexInput two = one.clone();

        assertSameStreams("basic clone one", expected, one);
        expected.seek(0);
        assertSameStreams("basic clone two", expected, two);

        // Now close the first stream
        one.close();

        // The following should really fail since we couldn't expect to
        // access a file once close has been called on it (regardless of
        // buffering and/or clone magic)
        expected.seek(0);
        two.seek(0);
        assertSameStreams("basic clone two/2", expected, two);


        // Now close the compound reader
        cr.close();

        // The following may also fail since the compound stream is closed
        expected.seek(0);
        two.seek(0);
        //assertSameStreams("basic clone two/3", expected, two);


        // Now close the second clone
        two.close();
        expected.seek(0);
        two.seek(0);
        //assertSameStreams("basic clone two/4", expected, two);

        expected.close();
    }


    /** This test opens two files from a compound stream and verifies that
     *  their file positions are independent of each other.
     */
    public void testRandomAccess() throws IOException {
        setUp_2();
        CompoundFileDirectory cr = new CompoundFileDirectory(dir, "f.comp", newIOContext(random()), false);

        // Open two files
        IndexInput e1 = dir.openInput("f11", newIOContext(random()));
        IndexInput e2 = dir.openInput("f3", newIOContext(random()));

        IndexInput a1 = cr.openInput("f11", newIOContext(random()));
        IndexInput a2 = dir.openInput("f3", newIOContext(random()));

        // Seek the first pair
        e1.seek(100);
        a1.seek(100);
        assertEquals(100, e1.getFilePointer());
        assertEquals(100, a1.getFilePointer());
        byte be1 = e1.readByte();
        byte ba1 = a1.readByte();
        assertEquals(be1, ba1);

        // Now seek the second pair
        e2.seek(1027);
        a2.seek(1027);
        assertEquals(1027, e2.getFilePointer());
        assertEquals(1027, a2.getFilePointer());
        byte be2 = e2.readByte();
        byte ba2 = a2.readByte();
        assertEquals(be2, ba2);

        // Now make sure the first one didn't move
        assertEquals(101, e1.getFilePointer());
        assertEquals(101, a1.getFilePointer());
        be1 = e1.readByte();
        ba1 = a1.readByte();
        assertEquals(be1, ba1);

        // Now more the first one again, past the buffer length
        e1.seek(1910);
        a1.seek(1910);
        assertEquals(1910, e1.getFilePointer());
        assertEquals(1910, a1.getFilePointer());
        be1 = e1.readByte();
        ba1 = a1.readByte();
        assertEquals(be1, ba1);

        // Now make sure the second set didn't move
        assertEquals(1028, e2.getFilePointer());
        assertEquals(1028, a2.getFilePointer());
        be2 = e2.readByte();
        ba2 = a2.readByte();
        assertEquals(be2, ba2);

        // Move the second set back, again cross the buffer size
        e2.seek(17);
        a2.seek(17);
        assertEquals(17, e2.getFilePointer());
        assertEquals(17, a2.getFilePointer());
        be2 = e2.readByte();
        ba2 = a2.readByte();
        assertEquals(be2, ba2);

        // Finally, make sure the first set didn't move
        // Now make sure the first one didn't move
        assertEquals(1911, e1.getFilePointer());
        assertEquals(1911, a1.getFilePointer());
        be1 = e1.readByte();
        ba1 = a1.readByte();
        assertEquals(be1, ba1);

        e1.close();
        e2.close();
        a1.close();
        a2.close();
        cr.close();
    }

    /** This test opens two files from a compound stream and verifies that
     *  their file positions are independent of each other.
     */
    public void testRandomAccessClones() throws IOException {
        setUp_2();
        CompoundFileDirectory cr = new CompoundFileDirectory(dir, "f.comp", newIOContext(random()), false);

        // Open two files
        IndexInput e1 = cr.openInput("f11", newIOContext(random()));
        IndexInput e2 = cr.openInput("f3", newIOContext(random()));

        IndexInput a1 = e1.clone();
        IndexInput a2 = e2.clone();

        // Seek the first pair
        e1.seek(100);
        a1.seek(100);
        assertEquals(100, e1.getFilePointer());
        assertEquals(100, a1.getFilePointer());
        byte be1 = e1.readByte();
        byte ba1 = a1.readByte();
        assertEquals(be1, ba1);

        // Now seek the second pair
        e2.seek(1027);
        a2.seek(1027);
        assertEquals(1027, e2.getFilePointer());
        assertEquals(1027, a2.getFilePointer());
        byte be2 = e2.readByte();
        byte ba2 = a2.readByte();
        assertEquals(be2, ba2);

        // Now make sure the first one didn't move
        assertEquals(101, e1.getFilePointer());
        assertEquals(101, a1.getFilePointer());
        be1 = e1.readByte();
        ba1 = a1.readByte();
        assertEquals(be1, ba1);

        // Now more the first one again, past the buffer length
        e1.seek(1910);
        a1.seek(1910);
        assertEquals(1910, e1.getFilePointer());
        assertEquals(1910, a1.getFilePointer());
        be1 = e1.readByte();
        ba1 = a1.readByte();
        assertEquals(be1, ba1);

        // Now make sure the second set didn't move
        assertEquals(1028, e2.getFilePointer());
        assertEquals(1028, a2.getFilePointer());
        be2 = e2.readByte();
        ba2 = a2.readByte();
        assertEquals(be2, ba2);

        // Move the second set back, again cross the buffer size
        e2.seek(17);
        a2.seek(17);
        assertEquals(17, e2.getFilePointer());
        assertEquals(17, a2.getFilePointer());
        be2 = e2.readByte();
        ba2 = a2.readByte();
        assertEquals(be2, ba2);

        // Finally, make sure the first set didn't move
        // Now make sure the first one didn't move
        assertEquals(1911, e1.getFilePointer());
        assertEquals(1911, a1.getFilePointer());
        be1 = e1.readByte();
        ba1 = a1.readByte();
        assertEquals(be1, ba1);

        e1.close();
        e2.close();
        a1.close();
        a2.close();
        cr.close();
    }


    public void testFileNotFound() throws IOException {
        setUp_2();
        CompoundFileDirectory cr = new CompoundFileDirectory(dir, "f.comp", newIOContext(random()), false);

        // Open two files
        try {
            cr.openInput("bogus", newIOContext(random()));
            fail("File not found");

        } catch (IOException e) {
            /* success */
            //System.out.println("SUCCESS: File Not Found: " + e);
        }

        cr.close();
    }


    public void testReadPastEOF() throws IOException {
        setUp_2();
        CompoundFileDirectory cr = new CompoundFileDirectory(dir, "f.comp", newIOContext(random()), false);
        IndexInput is = cr.openInput("f2", newIOContext(random()));
        is.seek(is.length() - 10);
        byte b[] = new byte[100];
        is.readBytes(b, 0, 10);

        try {
            is.readByte();
            fail("Single byte read past end of file");
        } catch (IOException e) {
            /* success */
            //System.out.println("SUCCESS: single byte read past end of file: " + e);
        }

        is.seek(is.length() - 10);
        try {
            is.readBytes(b, 0, 50);
            fail("Block read past end of file");
        } catch (IOException e) {
            /* success */
            //System.out.println("SUCCESS: block read past end of file: " + e);
        }

        is.close();
        cr.close();
    }

    /** This test that writes larger than the size of the buffer output
     * will correctly increment the file pointer.
     */
    public void testLargeWrites() throws IOException {
        IndexOutput os = dir.createOutput("testBufferStart.txt", newIOContext(random()));

        byte[] largeBuf = new byte[2048];
        for (int i=0; i<largeBuf.length; i++) {
            largeBuf[i] = (byte) (Math.random() * 256);
        }

        long currentPos = os.getFilePointer();
        os.writeBytes(largeBuf, largeBuf.length);

        try {
            assertEquals(currentPos + largeBuf.length, os.getFilePointer());
        } finally {
            os.close();
        }

    }
    
   public void testAddExternalFile() throws IOException {
       createSequenceFile(dir, "d1", (byte) 0, 15);

       Directory newDir = newDirectory();
       CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
       dir.copy(csw, "d1", "d1", newIOContext(random()));
       csw.close();

       CompoundFileDirectory csr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
       IndexInput expected = dir.openInput("d1", newIOContext(random()));
       IndexInput actual = csr.openInput("d1", newIOContext(random()));
       assertSameStreams("d1", expected, actual);
       assertSameSeekBehavior("d1", expected, actual);
       expected.close();
       actual.close();
       csr.close();
       
       newDir.close();
   }
   
   
  public void testAppend() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    int size = 5 + random().nextInt(128);
    for (int j = 0; j < 2; j++) {
      IndexOutput os = csw.createOutput("seg_" + j + "_foo.txt", newIOContext(random()));
      for (int i = 0; i < size; i++) {
        os.writeInt(i*j);
      }
      os.close();
      String[] listAll = newDir.listAll();
      assertEquals(1, listAll.length);
      assertEquals("d.cfs", listAll[0]);
    }
    createSequenceFile(dir, "d1", (byte) 0, 15);
    dir.copy(csw, "d1", "d1", newIOContext(random()));
    String[] listAll = newDir.listAll();
    assertEquals(1, listAll.length);
    assertEquals("d.cfs", listAll[0]);
    csw.close();
    CompoundFileDirectory csr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    for (int j = 0; j < 2; j++) {
      IndexInput openInput = csr.openInput("seg_" + j + "_foo.txt", newIOContext(random()));
      assertEquals(size * 4, openInput.length());
      for (int i = 0; i < size; i++) {
        assertEquals(i*j, openInput.readInt());
      }

      openInput.close();

    }
    IndexInput expected = dir.openInput("d1", newIOContext(random()));
    IndexInput actual = csr.openInput("d1", newIOContext(random()));
    assertSameStreams("d1", expected, actual);
    assertSameSeekBehavior("d1", expected, actual);
    expected.close();
    actual.close();
    csr.close();
    newDir.close();
  }
  
  public void testAppendTwice() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    createSequenceFile(newDir, "d1", (byte) 0, 15);
    IndexOutput out = csw.createOutput("d.xyz", newIOContext(random()));
    out.writeInt(0);
    out.close();
    assertEquals(1, csw.listAll().length);
    assertEquals("d.xyz", csw.listAll()[0]);
   
    csw.close();

    CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    assertEquals(1, cfr.listAll().length);
    assertEquals("d.xyz", cfr.listAll()[0]);
    cfr.close();
    newDir.close();
  }
  
  public void testEmptyCFS() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    csw.close();

    CompoundFileDirectory csr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    assertEquals(0, csr.listAll().length);
    csr.close();

    newDir.close();
  }
  
  public void testReadNestedCFP() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    CompoundFileDirectory nested = new CompoundFileDirectory(newDir, "b.cfs", newIOContext(random()), true);
    IndexOutput out = nested.createOutput("b.xyz", newIOContext(random()));
    IndexOutput out1 = nested.createOutput("b_1.xyz", newIOContext(random()));
    out.writeInt(0);
    out1.writeInt(1);
    out.close();
    out1.close();
    nested.close();
    newDir.copy(csw, "b.cfs", "b.cfs", newIOContext(random()));
    newDir.copy(csw, "b.cfe", "b.cfe", newIOContext(random()));
    newDir.deleteFile("b.cfs");
    newDir.deleteFile("b.cfe");
    csw.close();
    
    assertEquals(2, newDir.listAll().length);
    csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    
    assertEquals(2, csw.listAll().length);
    nested = new CompoundFileDirectory(csw, "b.cfs", newIOContext(random()), false);
    
    assertEquals(2, nested.listAll().length);
    IndexInput openInput = nested.openInput("b.xyz", newIOContext(random()));
    assertEquals(0, openInput.readInt());
    openInput.close();
    openInput = nested.openInput("b_1.xyz", newIOContext(random()));
    assertEquals(1, openInput.readInt());
    openInput.close();
    nested.close();
    csw.close();
    newDir.close();
  }
  
  public void testDoubleClose() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    IndexOutput out = csw.createOutput("d.xyz", newIOContext(random()));
    out.writeInt(0);
    out.close();
    
    csw.close();
    // close a second time - must have no effect according to Closeable
    csw.close();
    
    csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    IndexInput openInput = csw.openInput("d.xyz", newIOContext(random()));
    assertEquals(0, openInput.readInt());
    openInput.close();
    csw.close();
    // close a second time - must have no effect according to Closeable
    csw.close();
    
    newDir.close();
    
  }

  // Make sure we don't somehow use more than 1 descriptor
  // when reading a CFS with many subs:
  public void testManySubFiles() throws IOException {

    final Directory d = newFSDirectory(createTempDir("CFSManySubFiles"));
    final int FILE_COUNT = atLeast(500);

    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      IndexOutput out = d.createOutput("file." + fileIdx, newIOContext(random()));
      out.writeByte((byte) fileIdx);
      out.close();
    }
    
    final CompoundFileDirectory cfd = new CompoundFileDirectory(d, "c.cfs", newIOContext(random()), true);
    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      final String fileName = "file." + fileIdx;
      d.copy(cfd, fileName, fileName, newIOContext(random()));
    }
    cfd.close();

    final IndexInput[] ins = new IndexInput[FILE_COUNT];
    final CompoundFileDirectory cfr = new CompoundFileDirectory(d, "c.cfs", newIOContext(random()), false);
    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      ins[fileIdx] = cfr.openInput("file." + fileIdx, newIOContext(random()));
    }

    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      assertEquals((byte) fileIdx, ins[fileIdx].readByte());
    }

    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      ins[fileIdx].close();
    }
    cfr.close();
    d.close();
  }
  
  public void testListAll() throws Exception {
    Directory dir = newDirectory();
    // riw should sometimes create docvalues fields, etc
    RandomIndexWriter riw = new RandomIndexWriter(random(), dir);
    Document doc = new Document();
    // these fields should sometimes get term vectors, etc
    Field idField = newStringField("id", "", Field.Store.NO);
    Field bodyField = newTextField("body", "", Field.Store.NO);
    doc.add(idField);
    doc.add(bodyField);
    for (int i = 0; i < 100; i++) {
      idField.setStringValue(Integer.toString(i));
      bodyField.setStringValue(TestUtil.randomUnicodeString(random()));
      riw.addDocument(doc);
      if (random().nextInt(7) == 0) {
        riw.commit();
      }
    }
    riw.close();
    checkFiles(dir);
    dir.close();
  }
  
  // checks that we can open all files returned by listAll!
  private void checkFiles(Directory dir) throws IOException {
    for (String file : dir.listAll()) {
      if (file.endsWith(IndexFileNames.COMPOUND_FILE_EXTENSION)) {
        CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
        checkFiles(cfsDir); // recurse into cfs
        cfsDir.close();
      }
      IndexInput in = null;
      boolean success = false;
      try {
        in = dir.openInput(file, newIOContext(random()));
        success = true;
      } finally {
        if (success) {
          IOUtils.close(in);
        } else {
          IOUtils.closeWhileHandlingException(in);
        }
      }
    }
  }
}
