org.apache.bookkeeper.proto.BookieServer - java examples

Here are the examples of the java api org.apache.bookkeeper.proto.BookieServer taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

114 Examples 7

19 View Complete Implementation : BookieZKExpireTest.java
Copyright Apache License 2.0
Author : apache
@SuppressWarnings("deprecation")
@Test
public void testBookieServerZKExpireBehaviour() throws Exception {
    BookieServer server = null;
    try {
        File f = createTempDir("bookieserver", "test");
        HashSet<Thread> threadset = new HashSet<Thread>();
        int threadCount = Thread.activeCount();
        Thread[] threads = new Thread[threadCount * 2];
        threadCount = Thread.enumerate(threads);
        for (int i = 0; i < threadCount; i++) {
            if (threads[i].getName().indexOf("SendThread") != -1) {
                threadset.add(threads[i]);
            }
        }
        ServerConfiguration conf = newServerConfiguration(PortManager.nextFreePort(), f, new File[] { f });
        server = new BookieServer(conf);
        server.start();
        int secondsToWait = 5;
        while (!server.isRunning()) {
            Thread.sleep(1000);
            if (secondsToWait-- <= 0) {
                fail("Bookie never started");
            }
        }
        Thread sendthread = null;
        threadCount = Thread.activeCount();
        threads = new Thread[threadCount * 2];
        threadCount = Thread.enumerate(threads);
        for (int i = 0; i < threadCount; i++) {
            if (threads[i].getName().indexOf("SendThread") != -1 && !threadset.contains(threads[i])) {
                sendthread = threads[i];
                break;
            }
        }
        replacedertNotNull("Send thread not found", sendthread);
        sendthread.suspend();
        Thread.sleep(2 * conf.getZkTimeout());
        sendthread.resume();
        // allow watcher thread to run
        Thread.sleep(3000);
        replacedertTrue("Bookie should not shutdown on losing zk session", server.isBookieRunning());
        replacedertTrue("Bookie Server should not shutdown on losing zk session", server.isRunning());
    } finally {
        server.shutdown();
    }
}

19 View Complete Implementation : EtcdBKClusterTestBase.java
Copyright Apache License 2.0
Author : apache
private static BookieServer startBookie(ServerConfiguration conf) throws Exception {
    conf.setAutoRecoveryDaemonEnabled(true);
    TestStatsProvider provider = new TestStatsProvider();
    BookieServer server = new BookieServer(conf, provider.getStatsLogger(""));
    server.start();
    return server;
}

19 View Complete Implementation : TestAuth.java
Copyright Apache License 2.0
Author : apache
/**
 * Test authentication.
 */
@RunWith(Parameterized.clreplaced)
public clreplaced TestAuth extends BookKeeperClusterTestCase {

    static final Logger LOG = LoggerFactory.getLogger(TestAuth.clreplaced);

    public static final String TEST_AUTH_PROVIDER_PLUGIN_NAME = "TestAuthProviderPlugin";

    private static final byte[] PreplacedWD = "testPreplacedwd".getBytes();

    private static final byte[] ENTRY = "TestEntry".getBytes();

    private static final byte[] SUCCESS_RESPONSE = { 1 };

    private static final byte[] FAILURE_RESPONSE = { 2 };

    private static final byte[] PAYLOAD_MESSAGE = { 3 };

    enum ProtocolVersion {

        ProtocolV2, ProtocolV3
    }

    @Parameters
    public static Collection<Object[]> configs() {
        return Arrays.asList(new Object[][] { { ProtocolVersion.ProtocolV2 }, { ProtocolVersion.ProtocolV3 } });
    }

    private final ProtocolVersion protocolVersion;

    public TestAuth(ProtocolVersion protocolVersion) {
        // start them later when auth providers are configured
        super(0);
        this.protocolVersion = protocolVersion;
    }

    protected ClientConfiguration newClientConfiguration() {
        ClientConfiguration conf = super.newClientConfiguration();
        conf.setUseV2WireProtocol(protocolVersion == ProtocolVersion.ProtocolV2);
        return conf;
    }

    // we preplaced in ledgerId because the method may throw exceptions
    private void connectAndWriteToBookie(ClientConfiguration conf, AtomicLong ledgerWritten) throws Exception {
        LOG.info("Connecting to bookie");
        try (BookKeeper bkc = new BookKeeper(conf, zkc);
            LedgerHandle l = bkc.createLedger(1, 1, DigestType.CRC32, PreplacedWD)) {
            ledgerWritten.set(l.getId());
            l.addEntry(ENTRY);
        }
    }

    /**
     * check if the entry exists. Restart the bookie to allow
     * access
     */
    private int entryCount(long ledgerId, ServerConfiguration bookieConf, ClientConfiguration clientConf) throws Exception {
        LOG.info("Counting entries in {}", ledgerId);
        for (ServerConfiguration conf : bsConfs) {
            conf.setBookieAuthProviderFactoryClreplaced(AlwaysSucceedBookieAuthProviderFactory.clreplaced.getName());
        }
        clientConf.setClientAuthProviderFactoryClreplaced(SendUntilCompleteClientAuthProviderFactory.clreplaced.getName());
        restartBookies();
        int count = 0;
        try (BookKeeper bkc = new BookKeeper(clientConf, zkc);
            LedgerHandle lh = bkc.openLedger(ledgerId, DigestType.CRC32, PreplacedWD)) {
            if (lh.getLastAddConfirmed() < 0) {
                return 0;
            }
            Enumeration<LedgerEntry> e = lh.readEntries(0, lh.getLastAddConfirmed());
            while (e.hasMoreElements()) {
                count++;
                replacedertTrue("Should match what we wrote", Arrays.equals(e.nextElement().getEntry(), ENTRY));
            }
        }
        return count;
    }

    /**
     * Test an connection will authorize with a single message
     * to the server and a single response.
     */
    @Test
    public void testSingleMessageAuth() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced(AlwaysSucceedBookieAuthProviderFactory.clreplaced.getName());
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced(SendUntilCompleteClientAuthProviderFactory.clreplaced.getName());
        startAndStoreBookie(bookieConf);
        AtomicLong ledgerId = new AtomicLong(-1);
        // should succeed
        connectAndWriteToBookie(clientConf, ledgerId);
        replacedertFalse(ledgerId.get() == -1);
        replacedertEquals("Should have entry", 1, entryCount(ledgerId.get(), bookieConf, clientConf));
    }

    @Test
    public void testCloseMethodCalledOnAuthProvider() throws Exception {
        LogCloseCallsBookieAuthProviderFactory.closeCountersOnFactory.set(0);
        LogCloseCallsBookieAuthProviderFactory.closeCountersOnConnections.set(0);
        LogCloseCallsBookieAuthProviderFactory.initCountersOnFactory.set(0);
        LogCloseCallsBookieAuthProviderFactory.initCountersOnConnections.set(0);
        LogCloseCallsClientAuthProviderFactory.initCountersOnFactory.set(0);
        LogCloseCallsClientAuthProviderFactory.closeCountersOnFactory.set(0);
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced(LogCloseCallsBookieAuthProviderFactory.clreplaced.getName());
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced(LogCloseCallsClientAuthProviderFactory.clreplaced.getName());
        startAndStoreBookie(bookieConf);
        AtomicLong ledgerId = new AtomicLong(-1);
        // should succeed
        connectAndWriteToBookie(clientConf, ledgerId);
        replacedertFalse(ledgerId.get() == -1);
        replacedertEquals("Should have entry", 1, entryCount(ledgerId.get(), bookieConf, clientConf));
        for (BookieServer bks : bs) {
            bks.shutdown();
        }
        replacedertEquals(LogCloseCallsBookieAuthProviderFactory.initCountersOnConnections.get(), LogCloseCallsBookieAuthProviderFactory.closeCountersOnConnections.get());
        replacedertTrue(LogCloseCallsBookieAuthProviderFactory.initCountersOnConnections.get() > 0);
        replacedertEquals(1, LogCloseCallsBookieAuthProviderFactory.initCountersOnFactory.get());
        replacedertEquals(1, LogCloseCallsBookieAuthProviderFactory.closeCountersOnFactory.get());
        replacedertEquals(LogCloseCallsClientAuthProviderFactory.initCountersOnConnections.get(), LogCloseCallsClientAuthProviderFactory.closeCountersOnConnections.get());
        replacedertTrue(LogCloseCallsClientAuthProviderFactory.initCountersOnConnections.get() > 0);
        replacedertEquals(1, LogCloseCallsClientAuthProviderFactory.initCountersOnFactory.get());
        replacedertEquals(1, LogCloseCallsClientAuthProviderFactory.closeCountersOnFactory.get());
    }

    /**
     * Test that when the bookie provider sends a failure message
     * the client will not be able to write.
     */
    @Test
    public void testSingleMessageAuthFailure() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced(AlwaysFailBookieAuthProviderFactory.clreplaced.getName());
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced(SendUntilCompleteClientAuthProviderFactory.clreplaced.getName());
        startAndStoreBookie(bookieConf);
        AtomicLong ledgerId = new AtomicLong(-1);
        try {
            // should fail
            connectAndWriteToBookie(clientConf, ledgerId);
            fail("Shouldn't get this far");
        } catch (BKException.BKUnauthorizedAccessException bke) {
        // client shouldnt be able to find enough bookies to
        // write
        }
        replacedertFalse(ledgerId.get() == -1);
        replacedertEquals("Shouldn't have entry", 0, entryCount(ledgerId.get(), bookieConf, clientConf));
    }

    /**
     * Test that authentication works when the providers
     * exchange multiple messages.
     */
    @Test
    public void testMultiMessageAuth() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced(SucceedAfter3BookieAuthProviderFactory.clreplaced.getName());
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced(SendUntilCompleteClientAuthProviderFactory.clreplaced.getName());
        AtomicLong ledgerId = new AtomicLong(-1);
        startAndStoreBookie(bookieConf);
        // should succeed
        connectAndWriteToBookie(clientConf, ledgerId);
        replacedertFalse(ledgerId.get() == -1);
        replacedertEquals("Should have entry", 1, entryCount(ledgerId.get(), bookieConf, clientConf));
    }

    /**
     * Test that when the bookie provider sends a failure message
     * the client will not be able to write.
     */
    @Test
    public void testMultiMessageAuthFailure() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced(FailAfter3BookieAuthProviderFactory.clreplaced.getName());
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced(SendUntilCompleteClientAuthProviderFactory.clreplaced.getName());
        startAndStoreBookie(bookieConf);
        AtomicLong ledgerId = new AtomicLong(-1);
        try {
            // should fail
            connectAndWriteToBookie(clientConf, ledgerId);
            fail("Shouldn't get this far");
        } catch (BKException.BKUnauthorizedAccessException bke) {
        // bookie should have sent a negative response before
        // breaking the conneciton
        }
        replacedertFalse(ledgerId.get() == -1);
        replacedertEquals("Shouldn't have entry", 0, entryCount(ledgerId.get(), bookieConf, clientConf));
    }

    /**
     * Test that when the bookie and the client have a different
     * plugin configured, no messages will get through.
     */
    @Test
    public void testDifferentPluginFailure() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced(DifferentPluginBookieAuthProviderFactory.clreplaced.getName());
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced(SendUntilCompleteClientAuthProviderFactory.clreplaced.getName());
        startAndStoreBookie(bookieConf);
        AtomicLong ledgerId = new AtomicLong(-1);
        try {
            // should fail
            connectAndWriteToBookie(clientConf, ledgerId);
            fail("Shouldn't get this far");
        } catch (BKException.BKUnauthorizedAccessException bke) {
            // bookie should have sent a negative response before
            // breaking the conneciton
            replacedertEquals(ProtocolVersion.ProtocolV3, protocolVersion);
        } catch (BKException.BKNotEnoughBookiesException nebe) {
            // With V2 we don't get the authorization error, but rather just
            // fail to write to bookies.
            replacedertEquals(ProtocolVersion.ProtocolV2, protocolVersion);
        }
        replacedertFalse(ledgerId.get() == -1);
        replacedertEquals("Shouldn't have entry", 0, entryCount(ledgerId.get(), bookieConf, clientConf));
    }

    /**
     * Test that when the plugin clreplaced does exist, but
     * doesn't implement the interface, we fail predictably.
     */
    @Test
    public void testExistantButNotValidPlugin() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced("java.lang.String");
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced("java.lang.String");
        try {
            startAndStoreBookie(bookieConf);
            fail("Shouldn't get this far");
        } catch (RuntimeException e) {
            // received correct exception
            replacedertTrue("Wrong exception thrown", e.getMessage().contains("not " + BookieAuthProvider.Factory.clreplaced.getName()));
        }
        try {
            BookKeeper bkc = new BookKeeper(clientConf, zkc);
            fail("Shouldn't get this far");
        } catch (RuntimeException e) {
            // received correct exception
            replacedertTrue("Wrong exception thrown", e.getMessage().contains("not " + ClientAuthProvider.Factory.clreplaced.getName()));
        }
    }

    /**
     * Test that when the plugin clreplaced does not exist,
     * the bookie will not start and the client will
     * break.
     */
    @Test
    public void testNonExistantPlugin() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced("NonExistantClreplacedNameForTestingAuthPlugins");
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced("NonExistantClreplacedNameForTestingAuthPlugins");
        try {
            startAndStoreBookie(bookieConf);
            fail("Shouldn't get this far");
        } catch (RuntimeException e) {
            // received correct exception
            replacedertEquals("Wrong exception thrown", e.getCause().getClreplaced(), ClreplacedNotFoundException.clreplaced);
        }
        try {
            BookKeeper bkc = new BookKeeper(clientConf, zkc);
            fail("Shouldn't get this far");
        } catch (RuntimeException e) {
            // received correct exception
            replacedertEquals("Wrong exception thrown", e.getCause().getClreplaced(), ClreplacedNotFoundException.clreplaced);
        }
    }

    /**
     * Test that when the plugin on the bookie crashes, the client doesn't
     * hang also, but it cannot write in any case.
     */
    @Test
    public void testCrashDuringAuth() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced(CrashAfter3BookieAuthProviderFactory.clreplaced.getName());
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced(SendUntilCompleteClientAuthProviderFactory.clreplaced.getName());
        startAndStoreBookie(bookieConf);
        AtomicLong ledgerId = new AtomicLong(-1);
        try {
            connectAndWriteToBookie(clientConf, ledgerId);
            fail("Shouldn't get this far");
        } catch (BKException.BKNotEnoughBookiesException bke) {
        // bookie won't respond, request will timeout, and then
        // we wont be able to find a replacement
        }
        replacedertFalse(ledgerId.get() == -1);
        replacedertEquals("Shouldn't have entry", 0, entryCount(ledgerId.get(), bookieConf, clientConf));
    }

    /**
     * Test that when a bookie simply stops replying during auth, the client doesn't
     * hang also, but it cannot write in any case.
     */
    @Test
    public void testCrashType2DuringAuth() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced(CrashType2After3BookieAuthProviderFactory.clreplaced.getName());
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced(SendUntilCompleteClientAuthProviderFactory.clreplaced.getName());
        crashType2bookieInstance = startAndStoreBookie(bookieConf);
        AtomicLong ledgerId = new AtomicLong(-1);
        try {
            connectAndWriteToBookie(clientConf, ledgerId);
            fail("Shouldn't get this far");
        } catch (BKException.BKNotEnoughBookiesException bke) {
        // bookie won't respond, request will timeout, and then
        // we wont be able to find a replacement
        }
        replacedertFalse(ledgerId.get() == -1);
        replacedertEquals("Shouldn't have entry", 0, entryCount(ledgerId.get(), bookieConf, clientConf));
    }

    /**
     * Client will try to perform authentication but bookies are not configured.
     */
    @Test
    public void testClientWithAuthAndBookieWithDisabledAuth() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        replacedertNull(bookieConf.getBookieAuthProviderFactoryClreplaced());
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced(SendUntilCompleteClientAuthProviderFactory.clreplaced.getName());
        startAndStoreBookie(bookieConf);
        AtomicLong ledgerId = new AtomicLong(-1);
        // should succeed
        connectAndWriteToBookie(clientConf, ledgerId);
        replacedertFalse(ledgerId.get() == -1);
        replacedertEquals("Should have entry", 1, entryCount(ledgerId.get(), bookieConf, clientConf));
    }

    /**
     * The plugin will drop the connection from the bookie side.
     */
    @Test
    public void testDropConnectionFromBookieAuthPlugin() throws Exception {
        ServerConfiguration bookieConf = newServerConfiguration();
        bookieConf.setBookieAuthProviderFactoryClreplaced(DropConnectionBookieAuthProviderFactory.clreplaced.getName());
        ClientConfiguration clientConf = newClientConfiguration();
        clientConf.setClientAuthProviderFactoryClreplaced(SendUntilCompleteClientAuthProviderFactory.clreplaced.getName());
        startAndStoreBookie(bookieConf);
        AtomicLong ledgerId = new AtomicLong(-1);
        try {
            // should fail
            connectAndWriteToBookie(clientConf, ledgerId);
            fail();
        } catch (BKNotEnoughBookiesException error) {
        }
    }

    BookieServer startAndStoreBookie(ServerConfiguration conf) throws Exception {
        bsConfs.add(conf);
        BookieServer s = startBookie(conf);
        bs.add(s);
        return s;
    }

    /**
     * Factory for a bookie that always succeeds.
     */
    public static clreplaced AlwaysSucceedBookieAuthProviderFactory implements BookieAuthProvider.Factory {

        @Override
        public String getPluginName() {
            return TEST_AUTH_PROVIDER_PLUGIN_NAME;
        }

        @Override
        public void init(ServerConfiguration conf) {
        }

        @Override
        public BookieAuthProvider newProvider(BookieConnectionPeer addr, final AuthCallbacks.GenericCallback<Void> completeCb) {
            return new BookieAuthProvider() {

                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                    addr.setAuthorizedId(new BookKeeperPrincipal("test-principal"));
                    cb.operationComplete(BKException.Code.OK, AuthToken.wrap(SUCCESS_RESPONSE));
                    completeCb.operationComplete(BKException.Code.OK, null);
                }
            };
        }
    }

    private static clreplaced LogCloseCallsBookieAuthProviderFactory implements BookieAuthProvider.Factory {

        private static AtomicInteger closeCountersOnFactory = new AtomicInteger();

        private static AtomicInteger closeCountersOnConnections = new AtomicInteger();

        private static AtomicInteger initCountersOnFactory = new AtomicInteger();

        private static AtomicInteger initCountersOnConnections = new AtomicInteger();

        @Override
        public void init(ServerConfiguration conf) throws IOException {
            initCountersOnFactory.incrementAndGet();
        }

        @Override
        public void close() {
            closeCountersOnFactory.incrementAndGet();
        }

        @Override
        public BookieAuthProvider newProvider(BookieConnectionPeer connection, AuthCallbacks.GenericCallback<Void> completeCb) {
            return new BookieAuthProvider() {

                {
                    completeCb.operationComplete(BKException.Code.OK, null);
                    initCountersOnConnections.incrementAndGet();
                }

                @Override
                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                }

                @Override
                public void close() {
                    closeCountersOnConnections.incrementAndGet();
                }
            };
        }

        @Override
        public String getPluginName() {
            return TEST_AUTH_PROVIDER_PLUGIN_NAME;
        }
    }

    /**
     * Factory for a bookie that drops connections.
     */
    public static clreplaced DropConnectionBookieAuthProviderFactory implements BookieAuthProvider.Factory {

        @Override
        public String getPluginName() {
            return TEST_AUTH_PROVIDER_PLUGIN_NAME;
        }

        @Override
        public void init(ServerConfiguration conf) {
        }

        @Override
        public BookieAuthProvider newProvider(BookieConnectionPeer addr, final AuthCallbacks.GenericCallback<Void> completeCb) {
            return new BookieAuthProvider() {

                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                    addr.disconnect();
                }
            };
        }
    }

    /**
     * Factory for a bookie that always fails.
     */
    public static clreplaced AlwaysFailBookieAuthProviderFactory implements BookieAuthProvider.Factory {

        @Override
        public String getPluginName() {
            return TEST_AUTH_PROVIDER_PLUGIN_NAME;
        }

        @Override
        public void init(ServerConfiguration conf) {
        }

        @Override
        public BookieAuthProvider newProvider(BookieConnectionPeer addr, final AuthCallbacks.GenericCallback<Void> completeCb) {
            return new BookieAuthProvider() {

                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                    addr.setAuthorizedId(new BookKeeperPrincipal("test-principal"));
                    cb.operationComplete(BKException.Code.OK, AuthToken.wrap(FAILURE_RESPONSE));
                    completeCb.operationComplete(BKException.Code.UnauthorizedAccessException, null);
                }
            };
        }
    }

    private static clreplaced LogCloseCallsClientAuthProviderFactory implements ClientAuthProvider.Factory {

        private static AtomicInteger initCountersOnFactory = new AtomicInteger();

        private static AtomicInteger initCountersOnConnections = new AtomicInteger();

        private static AtomicInteger closeCountersOnFactory = new AtomicInteger();

        private static AtomicInteger closeCountersOnConnections = new AtomicInteger();

        @Override
        public void init(ClientConfiguration conf) throws IOException {
            initCountersOnFactory.incrementAndGet();
        }

        @Override
        public ClientAuthProvider newProvider(ClientConnectionPeer connection, AuthCallbacks.GenericCallback<Void> completeCb) {
            return new ClientAuthProvider() {

                @Override
                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                }

                @Override
                public void close() {
                    closeCountersOnConnections.incrementAndGet();
                }

                @Override
                public void init(AuthCallbacks.GenericCallback<AuthToken> cb) {
                    initCountersOnConnections.incrementAndGet();
                    completeCb.operationComplete(BKException.Code.OK, null);
                }
            };
        }

        @Override
        public String getPluginName() {
            return TEST_AUTH_PROVIDER_PLUGIN_NAME;
        }

        @Override
        public void close() {
            closeCountersOnFactory.incrementAndGet();
        }
    }

    /**
     * Factory for bookie that will send until complete.
     */
    private static clreplaced SendUntilCompleteClientAuthProviderFactory implements ClientAuthProvider.Factory {

        @Override
        public String getPluginName() {
            return TEST_AUTH_PROVIDER_PLUGIN_NAME;
        }

        @Override
        public void init(ClientConfiguration conf) {
        }

        @Override
        public ClientAuthProvider newProvider(ClientConnectionPeer addr, final AuthCallbacks.GenericCallback<Void> completeCb) {
            return new ClientAuthProvider() {

                public void init(AuthCallbacks.GenericCallback<AuthToken> cb) {
                    cb.operationComplete(BKException.Code.OK, AuthToken.wrap(PAYLOAD_MESSAGE));
                }

                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                    byte[] type = m.getData();
                    if (Arrays.equals(type, SUCCESS_RESPONSE)) {
                        addr.setAuthorizedId(new BookKeeperPrincipal("test-client-principal"));
                        completeCb.operationComplete(BKException.Code.OK, null);
                    } else if (Arrays.equals(type, FAILURE_RESPONSE)) {
                        completeCb.operationComplete(BKException.Code.UnauthorizedAccessException, null);
                    } else {
                        cb.operationComplete(BKException.Code.OK, AuthToken.wrap(PAYLOAD_MESSAGE));
                    }
                }
            };
        }
    }

    /**
     * Factory for bookie that succeeds after three messages.
     */
    public static clreplaced SucceedAfter3BookieAuthProviderFactory implements BookieAuthProvider.Factory {

        AtomicInteger numMessages = new AtomicInteger(0);

        @Override
        public String getPluginName() {
            return TEST_AUTH_PROVIDER_PLUGIN_NAME;
        }

        @Override
        public void init(ServerConfiguration conf) {
        }

        @Override
        public BookieAuthProvider newProvider(BookieConnectionPeer addr, final AuthCallbacks.GenericCallback<Void> completeCb) {
            return new BookieAuthProvider() {

                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                    if (numMessages.incrementAndGet() == 3) {
                        addr.setAuthorizedId(new BookKeeperPrincipal("test-principal"));
                        cb.operationComplete(BKException.Code.OK, AuthToken.wrap(SUCCESS_RESPONSE));
                        completeCb.operationComplete(BKException.Code.OK, null);
                    } else {
                        cb.operationComplete(BKException.Code.OK, AuthToken.wrap(PAYLOAD_MESSAGE));
                    }
                }
            };
        }
    }

    /**
     * Factory for bookie that fails after three messages.
     */
    public static clreplaced FailAfter3BookieAuthProviderFactory implements BookieAuthProvider.Factory {

        AtomicInteger numMessages = new AtomicInteger(0);

        @Override
        public String getPluginName() {
            return TEST_AUTH_PROVIDER_PLUGIN_NAME;
        }

        @Override
        public void init(ServerConfiguration conf) {
        }

        @Override
        public BookieAuthProvider newProvider(BookieConnectionPeer addr, final AuthCallbacks.GenericCallback<Void> completeCb) {
            return new BookieAuthProvider() {

                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                    if (numMessages.incrementAndGet() == 3) {
                        addr.setAuthorizedId(new BookKeeperPrincipal("test-principal"));
                        cb.operationComplete(BKException.Code.OK, AuthToken.wrap(FAILURE_RESPONSE));
                        completeCb.operationComplete(BKException.Code.UnauthorizedAccessException, null);
                    } else {
                        cb.operationComplete(BKException.Code.OK, AuthToken.wrap(PAYLOAD_MESSAGE));
                    }
                }
            };
        }
    }

    /**
     * Factory for crashing the bookie after 3 messages with an auth provider.
     */
    public static clreplaced CrashAfter3BookieAuthProviderFactory implements BookieAuthProvider.Factory {

        AtomicInteger numMessages = new AtomicInteger(0);

        @Override
        public String getPluginName() {
            return TEST_AUTH_PROVIDER_PLUGIN_NAME;
        }

        @Override
        public void init(ServerConfiguration conf) {
        }

        @Override
        public BookieAuthProvider newProvider(BookieConnectionPeer addr, final AuthCallbacks.GenericCallback<Void> completeCb) {
            return new BookieAuthProvider() {

                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                    if (numMessages.incrementAndGet() == 3) {
                        throw new RuntimeException("Do bad things to the bookie");
                    } else {
                        addr.setAuthorizedId(new BookKeeperPrincipal("test-principal"));
                        cb.operationComplete(BKException.Code.OK, AuthToken.wrap(PAYLOAD_MESSAGE));
                    }
                }
            };
        }
    }

    private static BookieServer crashType2bookieInstance = null;

    /**
     * Factory for a bookie with CrashType2 after three messages.
     */
    public static clreplaced CrashType2After3BookieAuthProviderFactory implements BookieAuthProvider.Factory {

        AtomicInteger numMessages = new AtomicInteger(0);

        @Override
        public String getPluginName() {
            return TEST_AUTH_PROVIDER_PLUGIN_NAME;
        }

        @Override
        public void init(ServerConfiguration conf) {
        }

        @Override
        public BookieAuthProvider newProvider(BookieConnectionPeer addr, final AuthCallbacks.GenericCallback<Void> completeCb) {
            return new BookieAuthProvider() {

                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                    if (numMessages.incrementAndGet() != 3) {
                        cb.operationComplete(BKException.Code.OK, AuthToken.wrap(PAYLOAD_MESSAGE));
                        return;
                    }
                    crashType2bookieInstance.suspendProcessing();
                }
            };
        }
    }

    /**
     * Factory for a DifferentAuthProviderPlugin.
     */
    public static clreplaced DifferentPluginBookieAuthProviderFactory implements BookieAuthProvider.Factory {

        @Override
        public String getPluginName() {
            return "DifferentAuthProviderPlugin";
        }

        @Override
        public void init(ServerConfiguration conf) {
        }

        @Override
        public BookieAuthProvider newProvider(BookieConnectionPeer addr, final AuthCallbacks.GenericCallback<Void> completeCb) {
            return new BookieAuthProvider() {

                public void process(AuthToken m, AuthCallbacks.GenericCallback<AuthToken> cb) {
                    cb.operationComplete(BKException.Code.OK, AuthToken.wrap(FAILURE_RESPONSE));
                    completeCb.operationComplete(BKException.Code.OK, null);
                }
            };
        }
    }
}

19 View Complete Implementation : TestMain.java
Copyright Apache License 2.0
Author : apache
@Test
public void testIgnoreExtraServerComponentsStartupFailures() throws Exception {
    ServerConfiguration serverConf = new ServerConfiguration().setAutoRecoveryDaemonEnabled(false).setHttpServerEnabled(false).setExtraServerComponents(new String[] { "bad-server-component" }).setIgnoreExtraServerComponentsStartupFailures(true);
    BookieConfiguration conf = new BookieConfiguration(serverConf);
    BookieServer mockServer = PowerMockito.mock(BookieServer.clreplaced);
    whenNew(BookieServer.clreplaced).withArguments(any(ServerConfiguration.clreplaced), any(StatsLogger.clreplaced)).thenReturn(mockServer);
    LifecycleComponentStack stack = buildBookieServer(conf);
    replacedertEquals(2, stack.getNumComponents());
    stack.start();
    verify(mockServer, times(1)).start();
    stack.stop();
    stack.close();
    verify(mockServer, times(1)).shutdown();
}

19 View Complete Implementation : ZKTestEnv.java
Copyright Apache License 2.0
Author : diennea
public clreplaced ZKTestEnv implements AutoCloseable {

    TestingZookeeperServerEmbedded zkServer;

    BookieServer bookie;

    Path path;

    public ZKTestEnv(Path path) throws Exception {
        zkServer = new TestingZookeeperServerEmbedded(1282, path.toFile());
        zkServer.start();
        this.path = path;
    }

    public void startBookie() throws Exception {
        if (bookie != null) {
            throw new Exception("bookie already started!");
        }
        ServerConfiguration conf = new ServerConfiguration();
        conf.setBookiePort(5621);
        conf.setUseHostNameAsBookieID(true);
        Path targetDir = path.resolve("bookie_data");
        conf.setZkServers("localhost:1282");
        conf.setLedgerDirNames(new String[] { targetDir.toAbsolutePath().toString() });
        conf.setLedgerManagerFactoryClreplaced(HierarchicalLedgerManagerFactory.clreplaced);
        conf.setZkLedgersRootPath(BOOKKEEPER_ZK_LEDGERS_ROOT_PATH_DEFAULT);
        int numJournals = 1;
        String[] journals = new String[numJournals];
        for (int i = 0; i < journals.length; i++) {
            Path jpath = targetDir.resolve("journal-" + i);
            journals[i] = jpath.toAbsolutePath().toString();
        }
        conf.setNumAddWorkerThreads(8);
        // new in 4.6
        conf.setMaxPendingReadRequestPerThread(10000);
        // new in 4.6
        conf.setMaxPendingAddRequestPerThread(20000);
        // new in 4.6, do not wait for fsync on journal
        conf.setJournalSyncData(false);
        conf.setJournalDirsName(journals);
        conf.setFlushInterval(100);
        conf.setJournalFlushWhenQueueEmpty(true);
        // conf.setSkipListSizeLimit(1024*1024*1024);
        conf.setProperty("journalMaxGroupWaitMSec", 10);
        // conf.setProperty("journalBufferedWritesThreshold", 1024);
        conf.setAutoRecoveryDaemonEnabled(false);
        conf.setEnableLocalTransport(true);
        conf.setAllowLoopback(true);
        BookKeeperAdmin.format(conf, false, false);
        Clreplaced<? extends StatsProvider> statsProviderClreplaced = conf.getStatsProviderClreplaced();
        StatsProvider statsProvider = ReflectionUtils.newInstance(statsProviderClreplaced);
        statsProvider.start(conf);
        this.bookie = new BookieServer(conf, statsProvider.getStatsLogger(""));
        this.bookie.start();
        TestUtils.waitForCondition(bookie::isRunning, NOOP, 100);
        System.out.println("[BOOKIE] started at " + bookie);
    }

    public BookieServer getBookie() {
        return bookie;
    }

    public String getAddress() {
        return "localhost:1282";
    }

    public int getTimeout() {
        return 40000;
    }

    public String getPath() {
        return "/test";
    }

    public void stopBookie() throws Exception {
        bookie.shutdown();
        bookie.join();
        bookie = null;
    }

    @Override
    public void close() throws Exception {
        try {
            if (bookie != null) {
                bookie.shutdown();
            }
        } catch (Throwable t) {
        }
        try {
            if (zkServer != null) {
                zkServer.close();
            }
        } catch (Throwable t) {
        }
    }
}

19 View Complete Implementation : LocalBookKeeper.java
Copyright Apache License 2.0
Author : apache
/**
 * Local Bookkeeper.
 */
public clreplaced LocalBookKeeper {

    protected static final Logger LOG = LoggerFactory.getLogger(LocalBookKeeper.clreplaced);

    public static final int CONNECTION_TIMEOUT = 30000;

    private static String newMetadataServiceUri(String zkServers, int port, String layout, String ledgerPath) {
        return "zk+" + layout + "://" + zkServers + ":" + port + ledgerPath;
    }

    int numberOfBookies;

    public LocalBookKeeper() {
        this(3);
    }

    public LocalBookKeeper(int numberOfBookies) {
        this(numberOfBookies, 5000, new ServerConfiguration(), defaultLocalBookiesConfigDir);
    }

    public LocalBookKeeper(int numberOfBookies, int initialPort, ServerConfiguration baseConf, String localBookiesConfigDirName) {
        this.numberOfBookies = numberOfBookies;
        this.initialPort = initialPort;
        this.localBookiesConfigDir = new File(localBookiesConfigDirName);
        this.baseConf = baseConf;
        LOG.info("Running {} bookie(s) on zkServer {}.", this.numberOfBookies);
    }

    private static String zooKeeperDefaultHost = "127.0.0.1";

    private static int zooKeeperDefaultPort = 2181;

    private static int zkSessionTimeOut = 5000;

    private static Integer bookieDefaultInitialPort = 5000;

    private static String defaultLocalBookiesConfigDir = "/tmp/localbookies-config";

    // BookKeeper variables
    File[] journalDirs;

    BookieServer[] bs;

    ServerConfiguration[] bsConfs;

    Integer initialPort = 5000;

    private ServerConfiguration baseConf;

    File localBookiesConfigDir;

    /**
     * @param maxCC
     *          Max Concurrency of Client
     * @param zookeeperPort
     *          ZooKeeper Server Port
     */
    public static ZooKeeperServerShim runZookeeper(int maxCC, int zookeeperPort) throws IOException {
        File zkTmpDir = IOUtils.createTempDir("zookeeper", "localbookkeeper");
        return runZookeeper(maxCC, zookeeperPort, zkTmpDir);
    }

    public static ZooKeeperServerShim runZookeeper(int maxCC, int zookeeperPort, File zkDir) throws IOException {
        LOG.info("Starting ZK server");
        ZooKeeperServerShim server = ZooKeeperServerShimFactory.createServer(zkDir, zkDir, zookeeperPort, maxCC);
        server.start();
        boolean b = waitForServerUp(InetAddress.getLoopbackAddress().getHostAddress() + ":" + zookeeperPort, CONNECTION_TIMEOUT);
        if (LOG.isDebugEnabled()) {
            LOG.debug("ZooKeeper server up: {}", b);
        }
        return server;
    }

    @SuppressWarnings("deprecation")
    private void initializeZookeeper(String zkHost, int zkPort) throws IOException {
        LOG.info("Instantiate ZK Client");
        // initialize the zk client with values
        try (ZooKeeperClient zkc = ZooKeeperClient.newBuilder().connectString(zkHost + ":" + zkPort).sessionTimeoutMs(zkSessionTimeOut).build()) {
            String zkLedgersRootPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf);
            ZkUtils.createFullPathOptimistic(zkc, zkLedgersRootPath, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
            List<Op> multiOps = Lists.newArrayListWithExpectedSize(2);
            multiOps.add(Op.create(zkLedgersRootPath + "/" + AVAILABLE_NODE, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
            multiOps.add(Op.create(zkLedgersRootPath + "/" + AVAILABLE_NODE + "/" + READONLY, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
            zkc.multi(multiOps);
        // No need to create an entry for each requested bookie anymore as the
        // BookieServers will register themselves with ZooKeeper on startup.
        } catch (KeeperException e) {
            LOG.error("Exception while creating znodes", e);
            throw new IOException("Error creating znodes : ", e);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            LOG.error("Interrupted while creating znodes", e);
            throw new IOException("Error creating znodes : ", e);
        }
    }

    private static void cleanupDirectories(List<File> dirs) throws IOException {
        for (File dir : dirs) {
            FileUtils.deleteDirectory(dir);
        }
    }

    private List<File> runBookies(String dirSuffix) throws IOException, KeeperException, InterruptedException, BookieException, UnavailableException, CompatibilityException, SecurityException, BKException, ConfigurationException {
        List<File> tempDirs = new ArrayList<File>();
        try {
            runBookies(tempDirs, dirSuffix);
            return tempDirs;
        } catch (IOException ioe) {
            cleanupDirectories(tempDirs);
            throw ioe;
        } catch (KeeperException ke) {
            cleanupDirectories(tempDirs);
            throw ke;
        } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
            cleanupDirectories(tempDirs);
            throw ie;
        } catch (BookieException be) {
            cleanupDirectories(tempDirs);
            throw be;
        } catch (UnavailableException ue) {
            cleanupDirectories(tempDirs);
            throw ue;
        } catch (CompatibilityException ce) {
            cleanupDirectories(tempDirs);
            throw ce;
        }
    }

    @SuppressWarnings("deprecation")
    private void runBookies(List<File> tempDirs, String dirSuffix) throws IOException, KeeperException, InterruptedException, BookieException, UnavailableException, CompatibilityException, SecurityException, BKException, ConfigurationException {
        LOG.info("Starting Bookie(s)");
        // Create Bookie Servers (B1, B2, B3)
        journalDirs = new File[numberOfBookies];
        bs = new BookieServer[numberOfBookies];
        bsConfs = new ServerConfiguration[numberOfBookies];
        if (localBookiesConfigDir.exists() && localBookiesConfigDir.isFile()) {
            throw new IOException("Unable to create LocalBookiesConfigDir, since there is a file at " + localBookiesConfigDir.getAbsolutePath());
        }
        if (!localBookiesConfigDir.exists() && !localBookiesConfigDir.mkdirs()) {
            throw new IOException("Unable to create LocalBookiesConfigDir - " + localBookiesConfigDir.getAbsolutePath());
        }
        for (int i = 0; i < numberOfBookies; i++) {
            if (null == baseConf.getJournalDirNameWithoutDefault()) {
                journalDirs[i] = IOUtils.createTempDir("localbookkeeper" + Integer.toString(i), dirSuffix);
                tempDirs.add(journalDirs[i]);
            } else {
                journalDirs[i] = new File(baseConf.getJournalDirName(), "bookie" + Integer.toString(i));
            }
            if (journalDirs[i].exists()) {
                if (journalDirs[i].isDirectory()) {
                    FileUtils.deleteDirectory(journalDirs[i]);
                } else if (!journalDirs[i].delete()) {
                    throw new IOException("Couldn't cleanup bookie journal dir " + journalDirs[i]);
                }
            }
            if (!journalDirs[i].mkdirs()) {
                throw new IOException("Couldn't create bookie journal dir " + journalDirs[i]);
            }
            String[] ledgerDirs = baseConf.getLedgerDirWithoutDefault();
            if ((null == ledgerDirs) || (0 == ledgerDirs.length)) {
                ledgerDirs = new String[] { journalDirs[i].getPath() };
            } else {
                for (int l = 0; l < ledgerDirs.length; l++) {
                    File dir = new File(ledgerDirs[l], "bookie" + Integer.toString(i));
                    if (dir.exists()) {
                        if (dir.isDirectory()) {
                            FileUtils.deleteDirectory(dir);
                        } else if (!dir.delete()) {
                            throw new IOException("Couldn't cleanup bookie ledger dir " + dir);
                        }
                    }
                    if (!dir.mkdirs()) {
                        throw new IOException("Couldn't create bookie ledger dir " + dir);
                    }
                    ledgerDirs[l] = dir.getPath();
                }
            }
            bsConfs[i] = new ServerConfiguration((ServerConfiguration) baseConf.clone());
            // If the caller specified ephemeral ports then use ephemeral ports for all
            // the bookies else use numBookie ports starting at initialPort
            if (0 == initialPort) {
                bsConfs[i].setBookiePort(0);
            } else {
                bsConfs[i].setBookiePort(initialPort + i);
            }
            if (null == baseConf.getMetadataServiceUriUnchecked()) {
                bsConfs[i].setMetadataServiceUri(baseConf.getMetadataServiceUri());
            }
            bsConfs[i].setJournalDirName(journalDirs[i].getPath());
            bsConfs[i].setLedgerDirNames(ledgerDirs);
            // write config into file before start so we can know what's wrong if start failed
            String fileName = Bookie.getBookieAddress(bsConfs[i]).toString() + ".conf";
            serializeLocalBookieConfig(bsConfs[i], fileName);
            bs[i] = new BookieServer(bsConfs[i]);
            bs[i].start();
        }
        /*
         * baseconf.conf is needed because for executing any BookieShell command
         * of Metadata/Zookeeper Operation nature we need a valid conf file
         * having correct zk details and this could be used for running any such
         * bookieshell commands if bookieid is not provided as parameter to
         * bookkeeper shell operation. for eg:
         * "./bookkeeper shell localbookie listbookies -rw". But for execution
         * shell command of bookie Operation nature we need to provide bookieid,
         * for eg "./bookkeeper shell -localbookie 10.3.27.190:5000 lastmark",
         * so this shell command would use '10.3.27.190:5000.conf' file
         */
        ServerConfiguration baseConfWithCorrectZKServers = new ServerConfiguration((ServerConfiguration) baseConf.clone());
        if (null == baseConf.getMetadataServiceUriUnchecked()) {
            baseConfWithCorrectZKServers.setMetadataServiceUri(baseConf.getMetadataServiceUri());
        }
        serializeLocalBookieConfig(baseConfWithCorrectZKServers, "baseconf.conf");
    }

    public static void startLocalBookies(String zkHost, int zkPort, int numBookies, boolean shouldStartZK, int initialBookiePort) throws Exception {
        ServerConfiguration conf = new ServerConfiguration();
        startLocalBookiesInternal(conf, zkHost, zkPort, numBookies, shouldStartZK, initialBookiePort, true, "test", null, defaultLocalBookiesConfigDir);
    }

    public static void startLocalBookies(String zkHost, int zkPort, int numBookies, boolean shouldStartZK, int initialBookiePort, ServerConfiguration conf) throws Exception {
        startLocalBookiesInternal(conf, zkHost, zkPort, numBookies, shouldStartZK, initialBookiePort, true, "test", null, defaultLocalBookiesConfigDir);
    }

    public static void startLocalBookies(String zkHost, int zkPort, int numBookies, boolean shouldStartZK, int initialBookiePort, String dirSuffix) throws Exception {
        ServerConfiguration conf = new ServerConfiguration();
        startLocalBookiesInternal(conf, zkHost, zkPort, numBookies, shouldStartZK, initialBookiePort, true, dirSuffix, null, defaultLocalBookiesConfigDir);
    }

    @SuppressWarnings("deprecation")
    static void startLocalBookiesInternal(ServerConfiguration conf, String zkHost, int zkPort, int numBookies, boolean shouldStartZK, int initialBookiePort, boolean stopOnExit, String dirSuffix, String zkDataDir, String localBookiesConfigDirName) throws Exception {
        conf.setMetadataServiceUri(newMetadataServiceUri(zkHost, zkPort, conf.getLedgerManagerLayoutStringFromFactoryClreplaced(), conf.getZkLedgersRootPath()));
        LocalBookKeeper lb = new LocalBookKeeper(numBookies, initialBookiePort, conf, localBookiesConfigDirName);
        ZooKeeperServerShim zks = null;
        File zkTmpDir = null;
        List<File> bkTmpDirs = null;
        try {
            if (shouldStartZK) {
                File zkDataDirFile = null;
                if (zkDataDir != null) {
                    zkDataDirFile = new File(zkDataDir);
                    if (zkDataDirFile.exists() && zkDataDirFile.isFile()) {
                        throw new IOException("Unable to create zkDataDir, since there is a file at " + zkDataDirFile.getAbsolutePath());
                    }
                    if (!zkDataDirFile.exists() && !zkDataDirFile.mkdirs()) {
                        throw new IOException("Unable to create zkDataDir - " + zkDataDirFile.getAbsolutePath());
                    }
                }
                zkTmpDir = IOUtils.createTempDir("zookeeper", dirSuffix, zkDataDirFile);
                zkTmpDir.deleteOnExit();
                zks = LocalBookKeeper.runZookeeper(1000, zkPort, zkTmpDir);
            }
            lb.initializeZookeeper(zkHost, zkPort);
            bkTmpDirs = lb.runBookies(dirSuffix);
            try {
                while (true) {
                    Thread.sleep(5000);
                }
            } catch (InterruptedException ie) {
                Thread.currentThread().interrupt();
                if (stopOnExit) {
                    lb.shutdownBookies();
                    if (null != zks) {
                        zks.stop();
                    }
                }
                throw ie;
            }
        } catch (Exception e) {
            LOG.error("Failed to run {} bookies : zk ensemble = '{}:{}'", numBookies, zkHost, zkPort, e);
            throw e;
        } finally {
            if (stopOnExit) {
                cleanupDirectories(bkTmpDirs);
                if (null != zkTmpDir) {
                    FileUtils.deleteDirectory(zkTmpDir);
                }
            }
        }
    }

    /**
     * Serializes the config object to the specified file in localBookiesConfigDir.
     *
     * @param localBookieConfig
     *         config object which has to be serialized
     * @param fileName
     *         name of the file
     * @throws IOException
     */
    private void serializeLocalBookieConfig(ServerConfiguration localBookieConfig, String fileName) throws IOException {
        File localBookieConfFile = new File(localBookiesConfigDir, fileName);
        if (localBookieConfFile.exists() && !localBookieConfFile.delete()) {
            throw new IOException("Unable to delete the existing LocalBookieConfigFile - " + localBookieConfFile.getAbsolutePath());
        }
        if (!localBookieConfFile.createNewFile()) {
            throw new IOException("Unable to create new File - " + localBookieConfFile.getAbsolutePath());
        }
        Iterator<String> keys = localBookieConfig.getKeys();
        try (PrintWriter writer = new PrintWriter(localBookieConfFile, "UTF-8")) {
            while (keys.hasNext()) {
                String key = keys.next();
                String[] values = localBookieConfig.getStringArray(key);
                StringBuilder concatenatedValue = new StringBuilder(values[0]);
                for (int i = 1; i < values.length; i++) {
                    concatenatedValue.append(",").append(values[i]);
                }
                writer.println(key + "=" + concatenatedValue.toString());
            }
        }
    }

    public static void main(String[] args) {
        try {
            if (args.length < 1) {
                usage();
                System.exit(-1);
            }
            int numBookies = 0;
            try {
                numBookies = Integer.parseInt(args[0]);
            } catch (NumberFormatException nfe) {
                LOG.error("Unrecognized number-of-bookies: {}", args[0]);
                usage();
                System.exit(-1);
            }
            ServerConfiguration conf = new ServerConfiguration();
            conf.setAllowLoopback(true);
            if (args.length >= 2) {
                String confFile = args[1];
                try {
                    conf.loadConf(new File(confFile).toURI().toURL());
                    LOG.info("Using configuration file " + confFile);
                } catch (Exception e) {
                    // load conf failed
                    LOG.warn("Error loading configuration file " + confFile, e);
                }
            }
            String zkDataDir = null;
            if (args.length >= 3) {
                zkDataDir = args[2];
            }
            String localBookiesConfigDirName = defaultLocalBookiesConfigDir;
            if (args.length >= 4) {
                localBookiesConfigDirName = args[3];
            }
            startLocalBookiesInternal(conf, zooKeeperDefaultHost, zooKeeperDefaultPort, numBookies, true, bookieDefaultInitialPort, false, "test", zkDataDir, localBookiesConfigDirName);
        } catch (Exception e) {
            LOG.error("Exiting LocalBookKeeper because of exception in main method", e);
            e.printStackTrace();
            /*
             * This is needed because, some non-daemon thread (probably in ZK or
             * some other dependent service) is preventing the JVM from exiting, though
             * there is exception in main thread.
             */
            System.exit(-1);
        }
    }

    private static void usage() {
        System.err.println("Usage: LocalBookKeeper number-of-bookies [path to bookie config] " + "[path to create ZK data directory at] [path to LocalBookiesConfigDir]");
    }

    public static boolean waitForServerUp(String hp, long timeout) {
        long start = System.currentTimeMillis();
        String[] split = hp.split(":");
        String host = split[0];
        int port = Integer.parseInt(split[1]);
        while (true) {
            try {
                Socket sock = new Socket(host, port);
                BufferedReader reader = null;
                try {
                    OutputStream outstream = sock.getOutputStream();
                    outstream.write("stat".getBytes(UTF_8));
                    outstream.flush();
                    reader = new BufferedReader(new InputStreamReader(sock.getInputStream(), UTF_8));
                    String line = reader.readLine();
                    if (line != null && line.startsWith("Zookeeper version:")) {
                        LOG.info("Server UP");
                        return true;
                    }
                } finally {
                    sock.close();
                    if (reader != null) {
                        reader.close();
                    }
                }
            } catch (IOException e) {
                // ignore as this is expected
                LOG.info("server " + hp + " not up " + e);
            }
            if (System.currentTimeMillis() > start + timeout) {
                break;
            }
            try {
                Thread.sleep(250);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            // ignore
            }
        }
        return false;
    }

    public void shutdownBookies() {
        for (BookieServer bookieServer : bs) {
            bookieServer.shutdown();
        }
    }
}

19 View Complete Implementation : BookKeeperClusterTestCase.java
Copyright Apache License 2.0
Author : apache
/**
 * Will starts the auto recovery process for the bookie servers. One auto
 * recovery process per each bookie server, if isAutoRecoveryEnabled is
 * enabled.
 */
public void startReplicationService() throws Exception {
    int index = -1;
    for (BookieServer bserver : bs) {
        startAutoRecovery(bserver, bsConfs.get(++index));
    }
}

19 View Complete Implementation : BKHttpServiceProvider.java
Copyright Apache License 2.0
Author : apache
/**
 * Bookkeeper based implementation of HttpServiceProvider,
 * which provide bookkeeper services to handle http requests
 * from different http endpoints.
 *
 * <p>TODO: eliminate the direct usage of zookeeper here {@link https://github.com/apache/bookkeeper/issues/1332}
 */
@Slf4j
public clreplaced BKHttpServiceProvider implements HttpServiceProvider {

    private final StatsProvider statsProvider;

    private final BookieServer bookieServer;

    private final AutoRecoveryMain autoRecovery;

    private final ServerConfiguration serverConf;

    private final ZooKeeper zk;

    private final BookKeeperAdmin bka;

    private final ExecutorService executor;

    private BKHttpServiceProvider(BookieServer bookieServer, AutoRecoveryMain autoRecovery, ServerConfiguration serverConf, StatsProvider statsProvider) throws IOException, KeeperException, InterruptedException, BKException {
        this.bookieServer = bookieServer;
        this.autoRecovery = autoRecovery;
        this.serverConf = serverConf;
        this.statsProvider = statsProvider;
        String zkServers = ZKMetadataDriverBase.resolveZkServers(serverConf);
        this.zk = ZooKeeperClient.newBuilder().connectString(zkServers).sessionTimeoutMs(serverConf.getZkTimeout()).build();
        ClientConfiguration clientConfiguration = new ClientConfiguration(serverConf);
        this.bka = new BookKeeperAdmin(clientConfiguration);
        this.executor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("BKHttpServiceThread").setDaemon(true).build());
    }

    @Override
    public void close() throws IOException {
        try {
            executor.shutdown();
            if (bka != null) {
                bka.close();
            }
            if (zk != null) {
                zk.close();
            }
        } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
            log.error("Interruption while closing BKHttpServiceProvider", ie);
            throw new IOException("Interruption while closing BKHttpServiceProvider", ie);
        } catch (BKException e) {
            log.error("Error while closing BKHttpServiceProvider", e);
            throw new IOException("Error while closing BKHttpServiceProvider", e);
        }
    }

    private ServerConfiguration getServerConf() {
        return serverConf;
    }

    private Auditor getAuditor() {
        return autoRecovery == null ? null : autoRecovery.getAuditor();
    }

    private Bookie getBookie() {
        return bookieServer == null ? null : bookieServer.getBookie();
    }

    /**
     * Builder for HttpServiceProvider.
     */
    public static clreplaced Builder {

        BookieServer bookieServer = null;

        AutoRecoveryMain autoRecovery = null;

        ServerConfiguration serverConf = null;

        StatsProvider statsProvider = null;

        public Builder setBookieServer(BookieServer bookieServer) {
            this.bookieServer = bookieServer;
            return this;
        }

        public Builder setAutoRecovery(AutoRecoveryMain autoRecovery) {
            this.autoRecovery = autoRecovery;
            return this;
        }

        public Builder setServerConfiguration(ServerConfiguration conf) {
            this.serverConf = conf;
            return this;
        }

        public Builder setStatsProvider(StatsProvider statsProvider) {
            this.statsProvider = statsProvider;
            return this;
        }

        public BKHttpServiceProvider build() throws IOException, KeeperException, InterruptedException, BKException {
            return new BKHttpServiceProvider(bookieServer, autoRecovery, serverConf, statsProvider);
        }
    }

    @Override
    public HttpEndpointService provideHttpEndpointService(ApiType type) {
        ServerConfiguration configuration = getServerConf();
        if (configuration == null) {
            return new ErrorHttpService();
        }
        switch(type) {
            case HEARTBEAT:
                return new HeartbeatService();
            case SERVER_CONFIG:
                return new ConfigurationService(configuration);
            case METRICS:
                return new MetricsService(configuration, statsProvider);
            // ledger
            case DELETE_LEDGER:
                return new DeleteLedgerService(configuration);
            case LIST_LEDGER:
                return new ListLedgerService(configuration, bookieServer);
            case GET_LEDGER_META:
                return new GetLedgerMetaService(configuration, bookieServer);
            case READ_LEDGER_ENTRY:
                return new ReadLedgerEntryService(configuration, bka);
            // bookie
            case LIST_BOOKIES:
                return new ListBookiesService(configuration, bka);
            case LIST_BOOKIE_INFO:
                return new ListBookieInfoService(configuration);
            case LAST_LOG_MARK:
                return new GetLastLogMarkService(configuration);
            case LIST_DISK_FILE:
                return new ListDiskFilesService(configuration);
            case EXPAND_STORAGE:
                return new ExpandStorageService(configuration);
            case GC:
                return new TriggerGCService(configuration, bookieServer);
            case GC_DETAILS:
                return new GCDetailsService(configuration, bookieServer);
            case BOOKIE_STATE:
                return new BookieStateService(bookieServer.getBookie());
            case BOOKIE_IS_READY:
                return new BookieIsReadyService(bookieServer.getBookie());
            // autorecovery
            case RECOVERY_BOOKIE:
                return new RecoveryBookieService(configuration, bka, executor);
            case LIST_UNDER_REPLICATED_LEDGER:
                return new ListUnderReplicatedLedgerService(configuration, bookieServer);
            case WHO_IS_AUDITOR:
                return new WhoIsAuditorService(configuration, zk);
            case TRIGGER_AUDIT:
                return new TriggerAuditService(configuration, bka);
            case LOST_BOOKIE_RECOVERY_DELAY:
                return new LostBookieRecoveryDelayService(configuration, bka);
            case DECOMMISSION:
                return new DecommissionService(configuration, bka, executor);
            default:
                return new ConfigurationService(configuration);
        }
    }
}

19 View Complete Implementation : BookieService.java
Copyright Apache License 2.0
Author : apache
/**
 * A {@link org.apache.bookkeeper.common.component.LifecycleComponent} that runs
 * a {@link org.apache.bookkeeper.proto.BookieServer}.
 */
@Accessors(fluent = true)
@Slf4j
public clreplaced BookieService extends AbstractLifecycleComponent<BookieConfiguration> {

    @Getter
    private final ServerConfiguration serverConf;

    private BookieServer bs;

    public BookieService(BookieConfiguration conf, StatsLogger statsLogger) {
        super("bookie-server", conf, statsLogger);
        this.serverConf = new ServerConfiguration();
        this.serverConf.loadConf(conf.getUnderlyingConf());
    }

    @Override
    protected void doStart() {
        List<File> indexDirs;
        if (null == serverConf.getIndexDirs()) {
            indexDirs = Collections.emptyList();
        } else {
            indexDirs = Arrays.asList(serverConf.getIndexDirs());
        }
        log.info("Hello, I'm your bookie, listening on port {} :" + " metadata service uri = {}, journals = {}, ledgers = {}, index = {}", serverConf.getBookiePort(), serverConf.getMetadataServiceUriUnchecked(), Arrays.asList(serverConf.getJournalDirNames()), Arrays.asList(serverConf.getLedgerDirs()), indexDirs);
        try {
            this.bs = new BookieServer(serverConf, statsLogger);
            bs.start();
            log.info("Started bookie server successfully.");
        } catch (Exception e) {
            throw new RuntimeException("Failed to start bookie server", e);
        }
    }

    @Override
    protected void doStop() {
        if (null != bs) {
            bs.shutdown();
        }
    }

    @Override
    protected void doClose() throws IOException {
    // no-op
    }
}

19 View Complete Implementation : AuditorBookieTest.java
Copyright Apache License 2.0
Author : apache
private BookieServer waitForNewAuditor(BookieServer auditor) throws Exception {
    BookieServer newAuditor = null;
    int retryCount = 8;
    while (retryCount > 0) {
        List<BookieServer> auditors = getAuditorBookie();
        if (auditors.size() > 0) {
            newAuditor = auditors.get(0);
            if (auditor != newAuditor) {
                break;
            }
        }
        Thread.sleep(500);
        retryCount--;
    }
    replacedertNotNull("New Auditor is not reelected after auditor crashes", newAuditor);
    verifyAuditor();
    return newAuditor;
}

19 View Complete Implementation : TestMain.java
Copyright Apache License 2.0
Author : apache
@Test
public void testExtraServerComponentsStartupFailures() throws Exception {
    ServerConfiguration serverConf = new ServerConfiguration().setAutoRecoveryDaemonEnabled(false).setHttpServerEnabled(false).setExtraServerComponents(new String[] { "bad-server-component" }).setIgnoreExtraServerComponentsStartupFailures(false);
    BookieConfiguration conf = new BookieConfiguration(serverConf);
    BookieServer mockServer = PowerMockito.mock(BookieServer.clreplaced);
    whenNew(BookieServer.clreplaced).withArguments(any(ServerConfiguration.clreplaced), any(StatsLogger.clreplaced)).thenReturn(mockServer);
    try {
        buildBookieServer(conf);
        fail("Should fail to start bookie server if `ignoreExtraServerComponentsStartupFailures` is set to false");
    } catch (RuntimeException re) {
        replacedertTrue(re.getCause() instanceof ClreplacedNotFoundException);
    }
}

19 View Complete Implementation : ZKTestEnv.java
Copyright Apache License 2.0
Author : diennea
public clreplaced ZKTestEnv implements AutoCloseable {

    static {
        System.setProperty("zookeeper.admin.enableServer", "true");
        System.setProperty("zookeeper.admin.serverPort", "0");
    }

    TestingZookeeperServerEmbedded zkServer;

    BookieServer bookie;

    Path path;

    public ZKTestEnv(Path path) throws Exception {
        zkServer = new TestingZookeeperServerEmbedded(1282, path.toFile());
        zkServer.start();
        this.path = path;
    }

    public void startBookie() throws Exception {
        ServerConfiguration conf = new ServerConfiguration();
        conf.setBookiePort(5621);
        conf.setUseHostNameAsBookieID(true);
        Path targetDir = path.resolve("bookie_data");
        conf.setZkServers("localhost:1282");
        conf.setLedgerDirNames(new String[] { targetDir.toAbsolutePath().toString() });
        conf.setJournalDirName(targetDir.toAbsolutePath().toString());
        conf.setFlushInterval(1000);
        conf.setJournalFlushWhenQueueEmpty(true);
        conf.setGcWaitTime(10);
        conf.setAutoRecoveryDaemonEnabled(false);
        // in unit tests we do not need real network for bookies
        conf.setEnableLocalTransport(true);
        // conf.setDisableServerSocketBind(true);
        conf.setAllowLoopback(true);
        BookKeeperAdmin.format(conf, false, true);
        this.bookie = new BookieServer(conf);
        this.bookie.start();
    }

    public void stopBookie() throws InterruptedException {
        this.bookie.shutdown();
        this.bookie.join();
    }

    public String getAddress() {
        return "localhost:1282";
    }

    public int getTimeout() {
        return 40000;
    }

    public String getPath() {
        return "/dodotest";
    }

    @Override
    public void close() throws Exception {
        try {
            if (bookie != null) {
                bookie.shutdown();
            }
        } catch (Throwable t) {
        }
        try {
            if (zkServer != null) {
                zkServer.close();
            }
        } catch (Throwable t) {
        }
    }
}

19 View Complete Implementation : BookieService.java
Copyright Apache License 2.0
Author : apache
/**
 * A {@link ServerLifecycleComponent} that starts the core bookie server.
 */
public clreplaced BookieService extends ServerLifecycleComponent {

    public static final String NAME = "bookie-server";

    private final BookieServer server;

    public BookieService(BookieConfiguration conf, StatsLogger statsLogger) throws Exception {
        super(NAME, conf, statsLogger);
        this.server = new BookieServer(conf.getServerConf(), statsLogger);
    }

    @Override
    public void setExceptionHandler(UncaughtExceptionHandler handler) {
        super.setExceptionHandler(handler);
        server.setExceptionHandler(handler);
    }

    public BookieServer getServer() {
        return server;
    }

    @Override
    protected void doStart() {
        try {
            this.server.start();
        } catch (InterruptedException exc) {
            throw new RuntimeException("Failed to start bookie server", exc);
        }
    }

    @Override
    protected void doStop() {
    // no-op
    }

    @Override
    protected void doClose() throws IOException {
        this.server.shutdown();
    }
}

19 View Complete Implementation : BookieClientTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Test the bookie client.
 */
public clreplaced BookieClientTest {

    BookieServer bs;

    File tmpDir;

    public int port = 13645;

    public EventLoopGroup eventLoopGroup;

    public OrderedExecutor executor;

    private ScheduledExecutorService scheduler;

    @Before
    public void setUp() throws Exception {
        tmpDir = IOUtils.createTempDir("bookieClient", "test");
        // Since this test does not rely on the BookKeeper client needing to
        // know via ZooKeeper which Bookies are available, okay, so preplaced in null
        // for the zkServers input parameter when constructing the BookieServer.
        ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
        conf.setBookiePort(port).setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new String[] { tmpDir.getPath() }).setMetadataServiceUri(null);
        bs = new BookieServer(conf);
        bs.start();
        eventLoopGroup = new NioEventLoopGroup();
        executor = OrderedExecutor.newBuilder().name("BKClientOrderedSafeExecutor").numThreads(2).build();
        scheduler = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("BookKeeperClientScheduler"));
    }

    @After
    public void tearDown() throws Exception {
        scheduler.shutdown();
        bs.shutdown();
        recursiveDelete(tmpDir);
        eventLoopGroup.shutdownGracefully();
        executor.shutdown();
    }

    private static void recursiveDelete(File dir) {
        File[] children = dir.listFiles();
        if (children != null) {
            for (File child : children) {
                recursiveDelete(child);
            }
        }
        dir.delete();
    }

    static clreplaced ResultStruct {

        int rc = -123456;

        ByteBuffer entry;
    }

    ReadEntryCallback recb = new ReadEntryCallback() {

        public void readEntryComplete(int rc, long ledgerId, long entryId, ByteBuf bb, Object ctx) {
            ResultStruct rs = (ResultStruct) ctx;
            synchronized (rs) {
                rs.rc = rc;
                if (BKException.Code.OK == rc && bb != null) {
                    bb.readerIndex(24);
                    rs.entry = bb.nioBuffer();
                }
                rs.notifyAll();
            }
        }
    };

    WriteCallback wrcb = new WriteCallback() {

        public void writeComplete(int rc, long ledgerId, long entryId, BookieSocketAddress addr, Object ctx) {
            if (ctx != null) {
                synchronized (ctx) {
                    if (ctx instanceof ResultStruct) {
                        ResultStruct rs = (ResultStruct) ctx;
                        rs.rc = rc;
                    }
                    ctx.notifyAll();
                }
            }
        }
    };

    @Test
    public void testWriteGaps() throws Exception {
        final Object notifyObject = new Object();
        byte[] preplacedwd = new byte[20];
        Arrays.fill(preplacedwd, (byte) 'a');
        BookieSocketAddress addr = bs.getLocalAddress();
        ResultStruct arc = new ResultStruct();
        BookieClient bc = new BookieClientImpl(new ClientConfiguration(), eventLoopGroup, UnpooledByteBufAllocator.DEFAULT, executor, scheduler, NullStatsLogger.INSTANCE);
        ByteBufList bb = createByteBuffer(1, 1, 1);
        bc.addEntry(addr, 1, preplacedwd, 1, bb, wrcb, arc, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE);
        synchronized (arc) {
            arc.wait(1000);
            replacedertEquals(0, arc.rc);
            bc.readEntry(addr, 1, 1, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(0, arc.rc);
            replacedertEquals(1, arc.entry.getInt());
        }
        bb = createByteBuffer(2, 1, 2);
        bc.addEntry(addr, 1, preplacedwd, 2, bb, wrcb, null, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE);
        bb = createByteBuffer(3, 1, 3);
        bc.addEntry(addr, 1, preplacedwd, 3, bb, wrcb, null, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE);
        bb = createByteBuffer(5, 1, 5);
        bc.addEntry(addr, 1, preplacedwd, 5, bb, wrcb, null, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE);
        bb = createByteBuffer(7, 1, 7);
        bc.addEntry(addr, 1, preplacedwd, 7, bb, wrcb, null, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE);
        synchronized (notifyObject) {
            bb = createByteBuffer(11, 1, 11);
            bc.addEntry(addr, 1, preplacedwd, 11, bb, wrcb, notifyObject, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE);
            notifyObject.wait();
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 6, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(BKException.Code.NoSuchEntryException, arc.rc);
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 7, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(0, arc.rc);
            replacedertEquals(7, arc.entry.getInt(), BookieProtocol.FLAG_NONE);
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 1, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(0, arc.rc);
            replacedertEquals(1, arc.entry.getInt());
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 2, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(0, arc.rc);
            replacedertEquals(2, arc.entry.getInt());
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 3, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(0, arc.rc);
            replacedertEquals(3, arc.entry.getInt());
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 4, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(BKException.Code.NoSuchEntryException, arc.rc);
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 11, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(0, arc.rc);
            replacedertEquals(11, arc.entry.getInt());
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 5, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(0, arc.rc);
            replacedertEquals(5, arc.entry.getInt());
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 10, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(BKException.Code.NoSuchEntryException, arc.rc);
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 12, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(BKException.Code.NoSuchEntryException, arc.rc);
        }
        synchronized (arc) {
            bc.readEntry(addr, 1, 13, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(BKException.Code.NoSuchEntryException, arc.rc);
        }
    }

    private ByteBufList createByteBuffer(int i, long lid, long eid) {
        ByteBuf bb = Unpooled.buffer(4 + 24);
        bb.writeLong(lid);
        bb.writeLong(eid);
        bb.writeLong(eid - 1);
        bb.writeInt(i);
        return ByteBufList.get(bb);
    }

    @Test
    public void testNoLedger() throws Exception {
        ResultStruct arc = new ResultStruct();
        BookieSocketAddress addr = bs.getLocalAddress();
        BookieClient bc = new BookieClientImpl(new ClientConfiguration(), eventLoopGroup, UnpooledByteBufAllocator.DEFAULT, executor, scheduler, NullStatsLogger.INSTANCE);
        synchronized (arc) {
            bc.readEntry(addr, 2, 13, recb, arc, BookieProtocol.FLAG_NONE);
            arc.wait(1000);
            replacedertEquals(BKException.Code.NoSuchLedgerExistsException, arc.rc);
        }
    }

    @Test
    public void testGetBookieInfoWithLimitStatsLogging() throws IOException, InterruptedException {
        testGetBookieInfo(true);
    }

    @Test
    public void testGetBookieInfoWithoutLimitStatsLogging() throws IOException, InterruptedException {
        testGetBookieInfo(false);
    }

    public void testGetBookieInfo(boolean limitStatsLogging) throws IOException, InterruptedException {
        BookieSocketAddress addr = bs.getLocalAddress();
        ClientConfiguration clientConf = new ClientConfiguration();
        clientConf.setLimitStatsLogging(limitStatsLogging);
        TestStatsProvider statsProvider = new TestStatsProvider();
        TestStatsLogger statsLogger = statsProvider.getStatsLogger("");
        BookieClient bc = new BookieClientImpl(clientConf, new NioEventLoopGroup(), UnpooledByteBufAllocator.DEFAULT, executor, scheduler, statsLogger);
        long flags = BookkeeperProtocol.GetBookieInfoRequest.Flags.FREE_DISK_SPACE_VALUE | BookkeeperProtocol.GetBookieInfoRequest.Flags.TOTAL_DISK_CAPACITY_VALUE;
        clreplaced CallbackObj {

            int rc;

            long requested;

            long freeDiskSpace, totalDiskCapacity;

            CountDownLatch latch = new CountDownLatch(1);

            CallbackObj(long requested) {
                this.requested = requested;
                this.rc = 0;
                this.freeDiskSpace = 0L;
                this.totalDiskCapacity = 0L;
            }
        }
        CallbackObj obj = new CallbackObj(flags);
        bc.getBookieInfo(addr, flags, new GetBookieInfoCallback() {

            @Override
            public void getBookieInfoComplete(int rc, BookieInfo bInfo, Object ctx) {
                CallbackObj obj = (CallbackObj) ctx;
                obj.rc = rc;
                if (rc == Code.OK) {
                    if ((obj.requested & BookkeeperProtocol.GetBookieInfoRequest.Flags.FREE_DISK_SPACE_VALUE) != 0) {
                        obj.freeDiskSpace = bInfo.getFreeDiskSpace();
                    }
                    if ((obj.requested & BookkeeperProtocol.GetBookieInfoRequest.Flags.TOTAL_DISK_CAPACITY_VALUE) != 0) {
                        obj.totalDiskCapacity = bInfo.getTotalDiskSpace();
                    }
                }
                obj.latch.countDown();
            }
        }, obj);
        obj.latch.await();
        System.out.println("Return code: " + obj.rc + "FreeDiskSpace: " + obj.freeDiskSpace + " TotalCapacity: " + obj.totalDiskCapacity);
        replacedertTrue("GetBookieInfo failed with error " + obj.rc, obj.rc == Code.OK);
        replacedertTrue("GetBookieInfo failed with error " + obj.rc, obj.freeDiskSpace <= obj.totalDiskCapacity);
        replacedertTrue("GetBookieInfo failed with error " + obj.rc, obj.totalDiskCapacity > 0);
        TestOpStatsLogger perChannelBookieClientScopeOfThisAddr = (TestOpStatsLogger) statsLogger.scope(BookKeeperClientStats.CHANNEL_SCOPE).scope(PerChannelBookieClient.buildStatsLoggerScopeName(addr)).getOpStatsLogger(BookKeeperClientStats.GET_BOOKIE_INFO_OP);
        int expectedBookieInfoSuccessCount = (limitStatsLogging) ? 0 : 1;
        replacedertEquals("BookieInfoSuccessCount", expectedBookieInfoSuccessCount, perChannelBookieClientScopeOfThisAddr.getSuccessCount());
    }
}

19 View Complete Implementation : GetLedgerMetaService.java
Copyright Apache License 2.0
Author : apache
/**
 * HttpEndpointService that handle Bookkeeper get ledger metadata related http request.
 * The GET method will get the ledger metadata for given "ledger_id".
 */
public clreplaced GetLedgerMetaService implements HttpEndpointService {

    static final Logger LOG = LoggerFactory.getLogger(GetLedgerMetaService.clreplaced);

    protected ServerConfiguration conf;

    protected BookieServer bookieServer;

    private final LedgerMetadataSerDe serDe;

    public GetLedgerMetaService(ServerConfiguration conf, BookieServer bookieServer) {
        checkNotNull(conf);
        this.conf = conf;
        this.bookieServer = bookieServer;
        this.serDe = new LedgerMetadataSerDe();
    }

    @Override
    public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
        HttpServiceResponse response = new HttpServiceResponse();
        Map<String, String> params = request.getParams();
        if (HttpServer.Method.GET == request.getMethod() && (params != null) && params.containsKey("ledger_id")) {
            Long ledgerId = Long.parseLong(params.get("ledger_id"));
            LedgerManagerFactory mFactory = bookieServer.getBookie().getLedgerManagerFactory();
            LedgerManager manager = mFactory.newLedgerManager();
            // output <ledgerId: ledgerMetadata>
            Map<String, Object> output = Maps.newHashMap();
            LedgerMetadata md = manager.readLedgerMetadata(ledgerId).get().getValue();
            output.put(ledgerId.toString(), md);
            manager.close();
            String jsonResponse = JsonUtil.toJson(output);
            LOG.debug("output body:" + jsonResponse);
            response.setBody(jsonResponse);
            response.setCode(HttpServer.StatusCode.OK);
            return response;
        } else {
            response.setCode(HttpServer.StatusCode.NOT_FOUND);
            response.setBody("Not found method. Should be GET method");
            return response;
        }
    }
}

19 View Complete Implementation : BookKeeperClusterTestCase.java
Copyright Apache License 2.0
Author : apache
/**
 * Kill bookie by index and verify that it's stopped.
 *
 * @param index index of bookie to kill
 *
 * @return configuration of killed bookie
 */
public ServerConfiguration killBookieAndWaitForZK(int index) throws Exception {
    if (index >= bs.size()) {
        throw new IOException("Bookie does not exist");
    }
    BookieServer server = bs.get(index);
    ServerConfiguration ret = killBookie(index);
    while (zkc.exists(ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf) + "/" + AVAILABLE_NODE + "/" + server.getLocalAddress().toString(), false) != null) {
        Thread.sleep(500);
    }
    return ret;
}

19 View Complete Implementation : TestAuth.java
Copyright Apache License 2.0
Author : apache
BookieServer startAndStoreBookie(ServerConfiguration conf) throws Exception {
    bsConfs.add(conf);
    BookieServer s = startBookie(conf);
    bs.add(s);
    return s;
}

19 View Complete Implementation : LocalBookiesRegistryTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testAccessibleLocalBookiesRegistry() throws Exception {
    replacedertEquals(3, bs.size());
    for (BookieServer bk : bs) {
        replacedertTrue(LocalBookiesRegistry.isLocalBookie(bk.getLocalAddress()));
    }
}

19 View Complete Implementation : ListUnderReplicatedLedgerService.java
Copyright Apache License 2.0
Author : apache
/**
 * HttpEndpointService that handle Bookkeeper list under replicated ledger related http request.
 *
 * <p>The GET method will list all ledger_ids of under replicated ledger.
 * User can filer wanted ledger by set parameter "missingreplica" and "excludingmissingreplica"
 */
public clreplaced ListUnderReplicatedLedgerService implements HttpEndpointService {

    static final Logger LOG = LoggerFactory.getLogger(ListUnderReplicatedLedgerService.clreplaced);

    protected ServerConfiguration conf;

    protected BookieServer bookieServer;

    public ListUnderReplicatedLedgerService(ServerConfiguration conf, BookieServer bookieServer) {
        checkNotNull(conf);
        this.conf = conf;
        this.bookieServer = bookieServer;
    }

    /*
     * Print the node which holds the auditor lock.
     */
    @Override
    public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
        HttpServiceResponse response = new HttpServiceResponse();
        // parameter as this: ?missingreplica=<bookie_address>&excludingmissingreplica=<bookid_address>
        Map<String, String> params = request.getParams();
        if (HttpServer.Method.GET == request.getMethod()) {
            final String includingBookieId;
            final String excludingBookieId;
            boolean printMissingReplica = false;
            if (params != null && params.containsKey("missingreplica")) {
                includingBookieId = params.get("missingreplica");
            } else {
                includingBookieId = null;
            }
            if (params != null && params.containsKey("excludingmissingreplica")) {
                excludingBookieId = params.get("excludingmissingreplica");
            } else {
                excludingBookieId = null;
            }
            if (params != null && params.containsKey("printmissingreplica")) {
                printMissingReplica = true;
            }
            Predicate<List<String>> predicate = null;
            if (!StringUtils.isBlank(includingBookieId) && !StringUtils.isBlank(excludingBookieId)) {
                predicate = replicasList -> (replicasList.contains(includingBookieId) && !replicasList.contains(excludingBookieId));
            } else if (!StringUtils.isBlank(includingBookieId)) {
                predicate = replicasList -> replicasList.contains(includingBookieId);
            } else if (!StringUtils.isBlank(excludingBookieId)) {
                predicate = replicasList -> !replicasList.contains(excludingBookieId);
            }
            try {
                boolean hasURLedgers = false;
                List<Long> outputLedgers = null;
                Map<Long, List<String>> outputLedgersWithMissingReplica = null;
                LedgerManagerFactory mFactory = bookieServer.getBookie().getLedgerManagerFactory();
                LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager();
                Iterator<UnderreplicatedLedger> iter = underreplicationManager.listLedgersToRereplicate(predicate);
                hasURLedgers = iter.hasNext();
                if (hasURLedgers) {
                    if (printMissingReplica) {
                        outputLedgersWithMissingReplica = new LinkedHashMap<Long, List<String>>();
                    } else {
                        outputLedgers = Lists.newArrayList();
                    }
                }
                while (iter.hasNext()) {
                    if (printMissingReplica) {
                        UnderreplicatedLedger underreplicatedLedger = iter.next();
                        outputLedgersWithMissingReplica.put(underreplicatedLedger.getLedgerId(), underreplicatedLedger.getReplicaList());
                    } else {
                        outputLedgers.add(iter.next().getLedgerId());
                    }
                }
                if (!hasURLedgers) {
                    response.setCode(HttpServer.StatusCode.NOT_FOUND);
                    response.setBody("No under replicated ledgers found");
                    return response;
                } else {
                    response.setCode(HttpServer.StatusCode.OK);
                    String jsonResponse = JsonUtil.toJson(printMissingReplica ? outputLedgersWithMissingReplica : outputLedgers);
                    LOG.debug("output body: " + jsonResponse);
                    response.setBody(jsonResponse);
                    return response;
                }
            } catch (Exception e) {
                LOG.error("Exception occurred while listing under replicated ledgers", e);
                response.setCode(HttpServer.StatusCode.NOT_FOUND);
                response.setBody("Exception when get." + e.getMessage());
                return response;
            }
        } else {
            response.setCode(HttpServer.StatusCode.NOT_FOUND);
            response.setBody("Not found method. Should be GET method");
            return response;
        }
    }
}

19 View Complete Implementation : EtcdBKClusterTestBase.java
Copyright Apache License 2.0
Author : apache
private static void startNumBookies(int numBookies) throws Exception {
    for (int i = 0; i < numBookies; i++) {
        ServerConfiguration conf = newServerConfiguration();
        log.info("Starting new bookie on port : {}", conf.getBookiePort());
        BookieServer server = startBookie(conf);
        synchronized (BOOKIES) {
            BOOKIES.add(server);
        }
    }
}

19 View Complete Implementation : AuditorBookieTest.java
Copyright Apache License 2.0
Author : apache
private List<BookieServer> getAuditorBookie() throws Exception {
    List<BookieServer> auditors = new LinkedList<BookieServer>();
    byte[] data = zkc.getData(electionPath, false, null);
    replacedertNotNull("Auditor election failed", data);
    for (BookieServer bks : bs) {
        if (new String(data).contains(bks.getLocalAddress().getPort() + "")) {
            auditors.add(bks);
        }
    }
    return auditors;
}

19 View Complete Implementation : RackAwareTest.java
Copyright Apache License 2.0
Author : apache
@AfterClreplaced
protected void shutdown() throws Exception {
    super.shutdown();
    for (BookieServer bs : bookies) {
        bs.shutdown();
    }
    bookies.clear();
}

19 View Complete Implementation : TestBookKeeperSpeculativeRead.java
Copyright Apache License 2.0
Author : aliyun-beta
/**
 * Sleep a bookie until I count down the latch
 *
 * @param latch
 *          latch to wait on
 * @param bookie
 *          bookie server
 * @throws Exception
 */
private void sleepBookie(final CountDownLatch latch, final BookieServer bookie) throws Exception {
    Thread sleeper = new Thread() {

        public void run() {
            try {
                bookie.suspendProcessing();
                latch.await(2, TimeUnit.MINUTES);
                bookie.resumeProcessing();
            } catch (Exception e) {
                LOG.error("Error suspending bookie", e);
            }
        }
    };
    sleeper.setName("BookieServerSleeper-" + bookie.getBookie().getId());
    sleeper.start();
}

19 View Complete Implementation : EmbeddedBookie.java
Copyright Apache License 2.0
Author : diennea
/**
 * Utility for starting embedded Apache BookKeeper Server (Bookie)
 *
 * @author enrico.olivelli
 */
public clreplaced EmbeddedBookie implements AutoCloseable {

    private static final Logger LOG = Logger.getLogger(EmbeddedBookie.clreplaced.getName());

    private final ServerConfiguration configuration;

    private final Path baseDirectory;

    private BookieServer bookieServer;

    private StatsProvider statsProvider;

    public EmbeddedBookie(Path baseDirectory, ServerConfiguration configuration) {
        this.configuration = configuration;
        this.baseDirectory = baseDirectory;
    }

    public void start() throws Exception {
        org.apache.bookkeeper.conf.ServerConfiguration conf = new org.apache.bookkeeper.conf.ServerConfiguration();
        conf.setZkTimeout(configuration.getInt(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT_DEFAULT));
        String zkServers = configuration.getString(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS_DEFAULT);
        String zkLedgersRootPath = configuration.getString(ServerConfiguration.PROPERTY_BOOKKEEPER_ZK_LEDGERS_ROOT_PATH, ServerConfiguration.PROPERTY_BOOKKEEPER_ZK_LEDGERS_ROOT_PATH_DEFAULT);
        String metadataServiceUri = "zk+null://" + zkServers.replace(",", ";") + "" + zkLedgersRootPath;
        LOG.log(Level.INFO, "Embeeded Bookie will use metadataServiceUri: {0}", metadataServiceUri);
        conf.setMetadataServiceUri(metadataServiceUri);
        conf.setStatisticsEnabled(true);
        conf.setProperty("codahaleStatsJmxEndpoint", "BlobIt_Bookie");
        conf.setStatsProviderClreplaced(PrometheusMetricsProvider.clreplaced);
        conf.setNumAddWorkerThreads(8);
        // new in 4.6
        conf.setMaxPendingReadRequestPerThread(10000);
        // new in 4.6
        conf.setMaxPendingAddRequestPerThread(20000);
        // by default we will not require fsync on journal, set this to true if you have only one machine
        conf.setJournalSyncData(false);
        int port = configuration.getInt(ServerConfiguration.PROPERTY_BOOKKEEPER_BOOKIE_PORT, ServerConfiguration.PROPERTY_BOOKKEEPER_BOOKIE_PORT_DEFAULT);
        conf.setUseHostNameAsBookieID(true);
        Path bookie_dir = baseDirectory.resolve("bookie");
        if (port <= 0) {
            Integer _port = readLocalBookiePort(bookie_dir);
            if (_port == null) {
                _port = NetworkUtils.replacedignFirstFreePort();
                LOG.log(Level.SEVERE, "As configuration parameter " + ServerConfiguration.PROPERTY_BOOKKEEPER_BOOKIE_PORT + " is {0},I have choosen to listen on port {1}." + " Set to a positive number in order to use a fixed port", new Object[] { Integer.toString(port), Integer.toString(_port) });
                persistLocalBookiePort(bookie_dir, _port);
            }
            port = _port;
        }
        conf.setBookiePort(port);
        Files.createDirectories(bookie_dir);
        Path bookie_data_dir = bookie_dir.resolve("bookie_data").toAbsolutePath();
        Path bookie_journal_dir = bookie_dir.resolve("bookie_journal").toAbsolutePath();
        Files.createDirectories(bookie_data_dir);
        Files.createDirectories(bookie_journal_dir);
        conf.setLedgerDirNames(new String[] { bookie_data_dir.toString() });
        conf.setJournalDirName(bookie_journal_dir.toString());
        conf.setFlushInterval(1000);
        conf.setMaxBackupJournals(5);
        conf.setMaxJournalSizeMB(1048);
        conf.setEnableLocalTransport(true);
        // default 200ms
        conf.setProperty("journalMaxGroupWaitMSec", 10L);
        conf.setJournalFlushWhenQueueEmpty(true);
        conf.setAutoRecoveryDaemonEnabled(true);
        conf.setLedgerManagerFactoryClreplaced(HierarchicalLedgerManagerFactory.clreplaced);
        conf.setHttpServerEnabled(true);
        conf.setProperty("httpServerClreplaced", ServletHttpServer.clreplaced.getName());
        // do not start an additional Jetty for metrics
        conf.setProperty("prometheusStatsHttpEnable", "false");
        for (String key : configuration.keys()) {
            if (key.startsWith("bookie.")) {
                String bookieConf = key.substring("bookie.".length());
                String value = configuration.getString(key, null);
                conf.addProperty(bookieConf, value);
                LOG.log(Level.CONFIG, "config {0} remapped to {1}={2}", new Object[] { key, bookieConf, value });
            }
        }
        long _start = System.currentTimeMillis();
        LOG.severe("Booting Apache Bookkeeper on port " + port);
        Files.createDirectories(bookie_dir);
        dumpBookieConfiguration(bookie_dir, conf);
        boolean forcemetaformat = configuration.getBoolean("bookie.forcemetaformat", false);
        LOG.log(Level.CONFIG, "bookie.forcemetaformat={0}", forcemetaformat);
        boolean result = BookKeeperAdmin.format(conf, false, forcemetaformat);
        if (result) {
            LOG.info("BookKeeperAdmin.format: created a new workspace on ZK");
        } else {
            LOG.info("BookKeeperAdmin.format: ZK space does not need an format operation");
        }
        boolean forceformat = configuration.getBoolean("bookie.forceformat", false);
        LOG.log(Level.CONFIG, "bookie.forceformat={0}", forceformat);
        if (forceformat) {
            result = Bookie.format(conf, false, forceformat);
            if (result) {
                LOG.info("Bookie.format: formatter applied to local bookie");
            } else {
                LOG.info("Bookie.format: local boookie did not need formatting");
            }
        }
        Clreplaced<? extends StatsProvider> statsProviderClreplaced = conf.getStatsProviderClreplaced();
        statsProvider = ReflectionUtils.newInstance(statsProviderClreplaced);
        statsProvider.start(conf);
        bookieServer = new BookieServer(conf, statsProvider.getStatsLogger(""));
        HttpService httpService = null;
        LOG.log(Level.INFO, "Bookie httpServerEnabled:{0}", conf.isHttpServerEnabled());
        if (conf.isHttpServerEnabled()) {
            BKHttpServiceProvider provider = new BKHttpServiceProvider.Builder().setBookieServer(bookieServer).setServerConfiguration(conf).setStatsProvider(statsProvider).build();
            httpService = new HttpService(provider, new BookieConfiguration(conf), statsProvider.getStatsLogger(""));
            httpService.start();
        }
        bookieServer.start();
        for (int i = 0; i < 100; i++) {
            if (bookieServer.getBookie().isRunning()) {
                LOG.info("Apache Bookkeeper started");
                break;
            }
            Thread.sleep(500);
        }
        long _stop = System.currentTimeMillis();
        LOG.severe("Booting Apache Bookkeeper finished. Time " + (_stop - _start) + " ms");
    }

    private void dumpBookieConfiguration(Path bookie_dir, org.apache.bookkeeper.conf.ServerConfiguration conf) throws IOException {
        // dump actual BookKeeper configuration in order to use bookkeeper shell
        Path actual_bookkeeper_configuration = bookie_dir.resolve("embedded.bookie.properties");
        StringBuilder builder = new StringBuilder();
        for (Iterator<String> key_it = conf.getKeys(); key_it.hasNext(); ) {
            String key = key_it.next();
            Object value = conf.getProperty(key);
            if (value instanceof Collection) {
                value = ((Collection) value).stream().map(String::valueOf).collect(Collectors.joining(","));
            }
            builder.append(key + "=" + value + "\n");
        }
        Files.write(actual_bookkeeper_configuration, builder.toString().getBytes(StandardCharsets.UTF_8), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE);
        LOG.severe("Dumped actual Bookie configuration to " + actual_bookkeeper_configuration.toAbsolutePath());
    }

    @Override
    public void close() {
        if (bookieServer != null) {
            LOG.info("Apache Bookkeeper stopping");
            try {
                bookieServer.shutdown();
                bookieServer.join();
            } catch (InterruptedException err) {
                Thread.currentThread().interrupt();
            } finally {
                bookieServer = null;
            }
        }
        if (statsProvider != null) {
            statsProvider.stop();
        }
    }

    public Integer readLocalBookiePort(Path dataPath) throws IOException {
        Path file = dataPath.resolve("bookie_port");
        try {
            LOG.log(Level.SEVERE, "Looking for local port into file {0}", file);
            if (!Files.isRegularFile(file)) {
                LOG.log(Level.SEVERE, "Cannot find file {0}", file);
                return null;
            }
            List<String> lines = Files.readAllLines(file, StandardCharsets.UTF_8);
            for (String line : lines) {
                // skip comments and empty lines
                if (line.startsWith("#") || line.isEmpty()) {
                    continue;
                }
                int res = Integer.parseInt(line);
                LOG.log(Level.SEVERE, "Found local port {0} into file {1}", new Object[] { Integer.toString(res), file });
                return res;
            }
            throw new IOException("Cannot find any valid line inside file " + file.toAbsolutePath());
        } catch (IOException error) {
            LOG.log(Level.SEVERE, "Error while reading file " + file.toAbsolutePath(), error);
            throw error;
        }
    }

    public void persistLocalBookiePort(Path dataPath, int port) throws IOException {
        Files.createDirectories(dataPath);
        Path file = dataPath.resolve("bookie_port");
        StringBuilder message = new StringBuilder();
        message.append("# This file contains the port of the bookie used by this node\n");
        message.append("# Do not change the contents of this file, otherwise the beheaviour of the system will\n");
        message.append("# lead eventually to data loss\n");
        message.append("# \n");
        message.append("# Any line which starts with '#' and and blank line will be ignored\n");
        message.append("# The system will consider the first non-blank line as port\n");
        message.append("\n\n");
        message.append(port);
        Files.write(file, message.toString().getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE_NEW);
    }
}

19 View Complete Implementation : GSSAPIBookKeeperTest.java
Copyright Apache License 2.0
Author : apache
BookieServer startAndStoreBookie(ServerConfiguration conf) throws Exception {
    System.setProperty(SaslConstants.SASL_SERVICE_NAME, non_default_sasl_service_name);
    bsConfs.add(conf);
    BookieServer s = startBookie(conf);
    bs.add(s);
    return s;
}

19 View Complete Implementation : ListLedgerService.java
Copyright Apache License 2.0
Author : apache
/**
 * HttpEndpointService that handle Bookkeeper list ledger related http request.
 *
 * <p>The GET method will list all ledger_ids in this bookkeeper cluster.
 * User can choose print metadata of each ledger or not by set parameter "print_metadata"
 */
public clreplaced ListLedgerService implements HttpEndpointService {

    static final Logger LOG = LoggerFactory.getLogger(ListLedgerService.clreplaced);

    protected ServerConfiguration conf;

    protected BookieServer bookieServer;

    private final LedgerMetadataSerDe serDe;

    public ListLedgerService(ServerConfiguration conf, BookieServer bookieServer) {
        checkNotNull(conf);
        this.conf = conf;
        this.bookieServer = bookieServer;
        this.serDe = new LedgerMetadataSerDe();
    }

    // Number of LedgerMetadata contains in each page
    static final int LIST_LEDGER_BATCH_SIZE = 100;

    private void keepLedgerMetadata(long ledgerId, CompletableFuture<Versioned<LedgerMetadata>> future, LinkedHashMap<String, String> output) throws Exception {
        LedgerMetadata md = future.get().getValue();
        output.put(Long.valueOf(ledgerId).toString(), new String(serDe.serialize(md), UTF_8));
    }

    @Override
    public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
        HttpServiceResponse response = new HttpServiceResponse();
        // GET
        // parameter could be like: print_metadata=true&page=PageIndex
        if (HttpServer.Method.GET == request.getMethod()) {
            Map<String, String> params = request.getParams();
            // default not print metadata
            boolean printMeta = (params != null) && params.containsKey("print_metadata") && params.get("print_metadata").equals("true");
            // Page index should start from 1;
            int pageIndex = (printMeta && params.containsKey("page")) ? Integer.parseInt(params.get("page")) : -1;
            LedgerManagerFactory mFactory = bookieServer.getBookie().getLedgerManagerFactory();
            LedgerManager manager = mFactory.newLedgerManager();
            LedgerManager.LedgerRangeIterator iter = manager.getLedgerRanges(0);
            // output <ledgerId: ledgerMetadata>
            LinkedHashMap<String, String> output = Maps.newLinkedHashMap();
            // futures for readLedgerMetadata for each page.
            Map<Long, CompletableFuture<Versioned<LedgerMetadata>>> futures = new LinkedHashMap<>(LIST_LEDGER_BATCH_SIZE);
            if (printMeta) {
                int ledgerIndex = 0;
                // start and end ledger index for wanted page.
                int startLedgerIndex = 0;
                int endLedgerIndex = 0;
                if (pageIndex > 0) {
                    startLedgerIndex = (pageIndex - 1) * LIST_LEDGER_BATCH_SIZE;
                    endLedgerIndex = startLedgerIndex + LIST_LEDGER_BATCH_SIZE - 1;
                }
                // get metadata
                while (iter.hasNext()) {
                    LedgerManager.LedgerRange r = iter.next();
                    for (Long lid : r.getLedgers()) {
                        ledgerIndex++;
                        if (// no actual page parameter provided
                        endLedgerIndex == 0 || (ledgerIndex >= startLedgerIndex && ledgerIndex <= endLedgerIndex)) {
                            futures.put(lid, manager.readLedgerMetadata(lid));
                        }
                    }
                    if (futures.size() >= LIST_LEDGER_BATCH_SIZE) {
                        for (Map.Entry<Long, CompletableFuture<Versioned<LedgerMetadata>>> e : futures.entrySet()) {
                            keepLedgerMetadata(e.getKey(), e.getValue(), output);
                        }
                        futures.clear();
                    }
                }
                for (Map.Entry<Long, CompletableFuture<Versioned<LedgerMetadata>>> e : futures.entrySet()) {
                    keepLedgerMetadata(e.getKey(), e.getValue(), output);
                }
                futures.clear();
            } else {
                while (iter.hasNext()) {
                    LedgerManager.LedgerRange r = iter.next();
                    for (Long lid : r.getLedgers()) {
                        output.put(lid.toString(), null);
                    }
                }
            }
            manager.close();
            String jsonResponse = JsonUtil.toJson(output);
            LOG.debug("output body:" + jsonResponse);
            response.setBody(jsonResponse);
            response.setCode(HttpServer.StatusCode.OK);
            return response;
        } else {
            response.setCode(HttpServer.StatusCode.NOT_FOUND);
            response.setBody("Not found method. Should be GET method");
            return response;
        }
    }
}

19 View Complete Implementation : ZKTestEnv.java
Copyright Apache License 2.0
Author : diennea
public clreplaced ZKTestEnv implements AutoCloseable {

    TestingServer zkServer;

    BookieServer bookie;

    Path path;

    public ZKTestEnv(Path path) throws Exception {
        zkServer = new TestingServer(1282, path.toFile(), true);
        this.path = path;
    }

    public void startBookie() throws Exception {
        ServerConfiguration conf = new ServerConfiguration();
        conf.setBookiePort(5621);
        conf.setUseHostNameAsBookieID(true);
        Path targetDir = path.resolve("bookie_data");
        conf.setZkServers("localhost:1282");
        conf.setLedgerDirNames(new String[] { targetDir.toAbsolutePath().toString() });
        conf.setJournalDirName(targetDir.toAbsolutePath().toString());
        conf.setFlushInterval(1000);
        conf.setAutoRecoveryDaemonEnabled(false);
        conf.setAllowLoopback(true);
        BookKeeperAdmin.format(conf, false, true);
        this.bookie = new BookieServer(conf);
        this.bookie.start();
    }

    public String getAddress() {
        return "localhost:1282";
    }

    public int getTimeout() {
        return 40000;
    }

    public String getPath() {
        return "/dodotest";
    }

    @Override
    public void close() throws Exception {
        try {
            if (bookie != null) {
                bookie.shutdown();
            }
        } catch (Throwable t) {
        }
        try {
            if (zkServer != null) {
                zkServer.close();
            }
        } catch (Throwable t) {
        }
    }
}

19 View Complete Implementation : GCDetailsService.java
Copyright Apache License 2.0
Author : apache
/**
 * HttpEndpointService that handle get garbage collection details service.
 *
 * <p>Get Garbage Collection status, the output would be like:
 *        [ {
 *           "forceCompacting" : false,
 *           "majorCompacting" : false,
 *           "minorCompacting" : false,
 *           "lastMajorCompactionTime" : 1544578144944,
 *           "lastMinorCompactionTime" : 1544578144944,
 *           "majorCompactionCounter" : 1,
 *           "minorCompactionCounter" : 0
 *         } ]
 */
public clreplaced GCDetailsService implements HttpEndpointService {

    static final Logger LOG = LoggerFactory.getLogger(GCDetailsService.clreplaced);

    protected ServerConfiguration conf;

    protected BookieServer bookieServer;

    public GCDetailsService(ServerConfiguration conf, BookieServer bookieServer) {
        checkNotNull(conf);
        checkNotNull(bookieServer);
        this.conf = conf;
        this.bookieServer = bookieServer;
    }

    @Override
    public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
        HttpServiceResponse response = new HttpServiceResponse();
        if (HttpServer.Method.GET == request.getMethod()) {
            List<GarbageCollectionStatus> details = bookieServer.getBookie().getLedgerStorage().getGarbageCollectionStatus();
            String jsonResponse = JsonUtil.toJson(details);
            if (LOG.isDebugEnabled()) {
                LOG.debug("output body:" + jsonResponse);
            }
            response.setBody(jsonResponse);
            response.setCode(HttpServer.StatusCode.OK);
            return response;
        } else {
            response.setCode(HttpServer.StatusCode.NOT_FOUND);
            response.setBody("Only support GET method to retrieve GC details." + " If you want to trigger gc, send a POST to gc endpoint.");
            return response;
        }
    }
}

19 View Complete Implementation : TriggerGCService.java
Copyright Apache License 2.0
Author : apache
/**
 * HttpEndpointService that handle force trigger GC requests.
 *
 * <p>The PUT method will force trigger GC on current bookie, and make GC run at backend.
 *
 * <p>The GET method will get the force triggered GC running or not.
 * Output would be like:
 *        {
 *           "is_in_force_gc" : "false"
 *        }
 */
public clreplaced TriggerGCService implements HttpEndpointService {

    static final Logger LOG = LoggerFactory.getLogger(TriggerGCService.clreplaced);

    protected ServerConfiguration conf;

    protected BookieServer bookieServer;

    public TriggerGCService(ServerConfiguration conf, BookieServer bookieServer) {
        checkNotNull(conf);
        checkNotNull(bookieServer);
        this.conf = conf;
        this.bookieServer = bookieServer;
    }

    @Override
    public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
        HttpServiceResponse response = new HttpServiceResponse();
        if (HttpServer.Method.PUT == request.getMethod()) {
            bookieServer.getBookie().getLedgerStorage().forceGC();
            String output = "Triggered GC on BookieServer: " + bookieServer.toString();
            String jsonResponse = JsonUtil.toJson(output);
            if (LOG.isDebugEnabled()) {
                LOG.debug("output body:" + jsonResponse);
            }
            response.setBody(jsonResponse);
            response.setCode(HttpServer.StatusCode.OK);
            return response;
        } else if (HttpServer.Method.GET == request.getMethod()) {
            Boolean isInForceGC = bookieServer.getBookie().getLedgerStorage().isInForceGC();
            Pair<String, String> output = Pair.of("is_in_force_gc", isInForceGC.toString());
            String jsonResponse = JsonUtil.toJson(output);
            if (LOG.isDebugEnabled()) {
                LOG.debug("output body:" + jsonResponse);
            }
            response.setBody(jsonResponse);
            response.setCode(HttpServer.StatusCode.OK);
            return response;
        } else {
            response.setCode(HttpServer.StatusCode.NOT_FOUND);
            response.setBody("Not found method. Should be PUT to trigger GC, Or GET to get Force GC state.");
            return response;
        }
    }
}

19 View Complete Implementation : TestMain.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBuildBookieServer() throws Exception {
    ServerConfiguration serverConf = new ServerConfiguration().setAutoRecoveryDaemonEnabled(false).setHttpServerEnabled(false).setExtraServerComponents(new String[] { TestComponent.clreplaced.getName() });
    BookieConfiguration conf = new BookieConfiguration(serverConf);
    BookieServer mockServer = PowerMockito.mock(BookieServer.clreplaced);
    whenNew(BookieServer.clreplaced).withArguments(any(ServerConfiguration.clreplaced), any(StatsLogger.clreplaced)).thenReturn(mockServer);
    LifecycleComponentStack stack = buildBookieServer(conf);
    replacedertEquals(3, stack.getNumComponents());
    replacedertTrue(stack.getComponent(2) instanceof TestComponent);
    stack.start();
    verify(mockServer, times(1)).start();
    stack.stop();
    stack.close();
    verify(mockServer, times(1)).shutdown();
}

18 View Complete Implementation : TestBookKeeperJournalManager.java
Copyright Apache License 2.0
Author : aliyun-beta
public clreplaced TestBookKeeperJournalManager {

    static final Log LOG = LogFactory.getLog(TestBookKeeperJournalManager.clreplaced);

    private static final long DEFAULT_SEGMENT_SIZE = 1000;

    protected static Configuration conf = new Configuration();

    private ZooKeeper zkc;

    private static BKJMUtil bkutil;

    static int numBookies = 3;

    private BookieServer newBookie;

    @BeforeClreplaced
    public static void setupBookkeeper() throws Exception {
        bkutil = new BKJMUtil(numBookies);
        bkutil.start();
    }

    @AfterClreplaced
    public static void teardownBookkeeper() throws Exception {
        bkutil.teardown();
    }

    @Before
    public void setup() throws Exception {
        zkc = BKJMUtil.connectZooKeeper();
    }

    @After
    public void teardown() throws Exception {
        zkc.close();
        if (newBookie != null) {
            newBookie.shutdown();
        }
    }

    private NamespaceInfo newNSInfo() {
        Random r = new Random();
        return new NamespaceInfo(r.nextInt(), "testCluster", "TestBPID", -1);
    }

    @Test
    public void testSimpleWrite() throws Exception {
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
        bkjm.format(nsi);
        EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        for (long i = 1; i <= 100; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.close();
        bkjm.finalizeLogSegment(1, 100);
        String zkpath = bkjm.finalizedLedgerZNode(1, 100);
        replacedertNotNull(zkc.exists(zkpath, false));
        replacedertNull(zkc.exists(bkjm.inprogressZNode(1), false));
    }

    @Test
    public void testNumberOfTransactions() throws Exception {
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
        bkjm.format(nsi);
        EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        for (long i = 1; i <= 100; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.close();
        bkjm.finalizeLogSegment(1, 100);
        long numTrans = bkjm.getNumberOfTransactions(1, true);
        replacedertEquals(100, numTrans);
    }

    @Test
    public void testNumberOfTransactionsWithGaps() throws Exception {
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-gaps"), nsi);
        bkjm.format(nsi);
        long txid = 1;
        for (long i = 0; i < 3; i++) {
            long start = txid;
            EditLogOutputStream out = bkjm.startLogSegment(start, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
            for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
                FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
                op.setTransactionId(txid++);
                out.write(op);
            }
            out.close();
            bkjm.finalizeLogSegment(start, txid - 1);
            replacedertNotNull(zkc.exists(bkjm.finalizedLedgerZNode(start, txid - 1), false));
        }
        zkc.delete(bkjm.finalizedLedgerZNode(DEFAULT_SEGMENT_SIZE + 1, DEFAULT_SEGMENT_SIZE * 2), -1);
        long numTrans = bkjm.getNumberOfTransactions(1, true);
        replacedertEquals(DEFAULT_SEGMENT_SIZE, numTrans);
        try {
            numTrans = bkjm.getNumberOfTransactions(DEFAULT_SEGMENT_SIZE + 1, true);
            fail("Should have thrown corruption exception by this point");
        } catch (JournalManager.CorruptionException ce) {
        // if we get here, everything is going good
        }
        numTrans = bkjm.getNumberOfTransactions((DEFAULT_SEGMENT_SIZE * 2) + 1, true);
        replacedertEquals(DEFAULT_SEGMENT_SIZE, numTrans);
    }

    @Test
    public void testNumberOfTransactionsWithInprogressAtEnd() throws Exception {
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-inprogressAtEnd"), nsi);
        bkjm.format(nsi);
        long txid = 1;
        for (long i = 0; i < 3; i++) {
            long start = txid;
            EditLogOutputStream out = bkjm.startLogSegment(start, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
            for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
                FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
                op.setTransactionId(txid++);
                out.write(op);
            }
            out.close();
            bkjm.finalizeLogSegment(start, (txid - 1));
            replacedertNotNull(zkc.exists(bkjm.finalizedLedgerZNode(start, (txid - 1)), false));
        }
        long start = txid;
        EditLogOutputStream out = bkjm.startLogSegment(start, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(txid++);
            out.write(op);
        }
        out.setReadyToFlush();
        out.flush();
        out.abort();
        out.close();
        long numTrans = bkjm.getNumberOfTransactions(1, true);
        replacedertEquals((txid - 1), numTrans);
    }

    /**
     * Create a bkjm namespace, write a journal from txid 1, close stream.
     * Try to create a new journal from txid 1. Should throw an exception.
     */
    @Test
    public void testWriteRestartFrom1() throws Exception {
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-restartFrom1"), nsi);
        bkjm.format(nsi);
        long txid = 1;
        long start = txid;
        EditLogOutputStream out = bkjm.startLogSegment(txid, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(txid++);
            out.write(op);
        }
        out.close();
        bkjm.finalizeLogSegment(start, (txid - 1));
        txid = 1;
        try {
            out = bkjm.startLogSegment(txid, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
            fail("Shouldn't be able to start another journal from " + txid + " when one already exists");
        } catch (Exception ioe) {
            LOG.info("Caught exception as expected", ioe);
        }
        // test border case
        txid = DEFAULT_SEGMENT_SIZE;
        try {
            out = bkjm.startLogSegment(txid, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
            fail("Shouldn't be able to start another journal from " + txid + " when one already exists");
        } catch (IOException ioe) {
            LOG.info("Caught exception as expected", ioe);
        }
        // open journal continuing from before
        txid = DEFAULT_SEGMENT_SIZE + 1;
        start = txid;
        out = bkjm.startLogSegment(start, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        replacedertNotNull(out);
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(txid++);
            out.write(op);
        }
        out.close();
        bkjm.finalizeLogSegment(start, (txid - 1));
        // open journal arbitarily far in the future
        txid = DEFAULT_SEGMENT_SIZE * 4;
        out = bkjm.startLogSegment(txid, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        replacedertNotNull(out);
    }

    @Test
    public void testTwoWriters() throws Exception {
        long start = 1;
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm1 = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi);
        bkjm1.format(nsi);
        BookKeeperJournalManager bkjm2 = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi);
        EditLogOutputStream out1 = bkjm1.startLogSegment(start, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        try {
            bkjm2.startLogSegment(start, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
            fail("Shouldn't have been able to open the second writer");
        } catch (IOException ioe) {
            LOG.info("Caught exception as expected", ioe);
        } finally {
            out1.close();
        }
    }

    @Test
    public void testSimpleRead() throws Exception {
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-simpleread"), nsi);
        bkjm.format(nsi);
        final long numTransactions = 10000;
        EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        ;
        for (long i = 1; i <= numTransactions; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.close();
        bkjm.finalizeLogSegment(1, numTransactions);
        List<EditLogInputStream> in = new ArrayList<EditLogInputStream>();
        bkjm.selectInputStreams(in, 1, true);
        try {
            replacedertEquals(numTransactions, FSEditLogTestUtil.countTransactionsInStream(in.get(0)));
        } finally {
            in.get(0).close();
        }
    }

    @Test
    public void testSimpleRecovery() throws Exception {
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery"), nsi);
        bkjm.format(nsi);
        EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        ;
        for (long i = 1; i <= 100; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.setReadyToFlush();
        out.flush();
        out.abort();
        out.close();
        replacedertNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false));
        replacedertNotNull(zkc.exists(bkjm.inprogressZNode(1), false));
        bkjm.recoverUnfinalizedSegments();
        replacedertNotNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false));
        replacedertNull(zkc.exists(bkjm.inprogressZNode(1), false));
    }

    /**
     * Test that if enough bookies fail to prevent an ensemble,
     * writes the bookkeeper will fail. Test that when once again
     * an ensemble is available, it can continue to write.
     */
    @Test
    public void testAllBookieFailure() throws Exception {
        // bookie to fail
        newBookie = bkutil.newBookie();
        BookieServer replacementBookie = null;
        try {
            int ensembleSize = numBookies + 1;
            replacedertEquals("New bookie didn't start", ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10));
            // ensure that the journal manager has to use all bookies,
            // so that a failure will fail the journal manager
            Configuration conf = new Configuration();
            conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE, ensembleSize);
            conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, ensembleSize);
            long txid = 1;
            NamespaceInfo nsi = newNSInfo();
            BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-allbookiefailure"), nsi);
            bkjm.format(nsi);
            EditLogOutputStream out = bkjm.startLogSegment(txid, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
            for (long i = 1; i <= 3; i++) {
                FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
                op.setTransactionId(txid++);
                out.write(op);
            }
            out.setReadyToFlush();
            out.flush();
            newBookie.shutdown();
            replacedertEquals("New bookie didn't die", numBookies, bkutil.checkBookiesUp(numBookies, 10));
            try {
                for (long i = 1; i <= 3; i++) {
                    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
                    op.setTransactionId(txid++);
                    out.write(op);
                }
                out.setReadyToFlush();
                out.flush();
                fail("should not get to this stage");
            } catch (IOException ioe) {
                LOG.debug("Error writing to bookkeeper", ioe);
                replacedertTrue("Invalid exception message", ioe.getMessage().contains("Failed to write to bookkeeper"));
            }
            replacementBookie = bkutil.newBookie();
            replacedertEquals("New bookie didn't start", numBookies + 1, bkutil.checkBookiesUp(numBookies + 1, 10));
            bkjm.recoverUnfinalizedSegments();
            out = bkjm.startLogSegment(txid, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
            for (long i = 1; i <= 3; i++) {
                FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
                op.setTransactionId(txid++);
                out.write(op);
            }
            out.setReadyToFlush();
            out.flush();
        } catch (Exception e) {
            LOG.error("Exception in test", e);
            throw e;
        } finally {
            if (replacementBookie != null) {
                replacementBookie.shutdown();
            }
            newBookie.shutdown();
            if (bkutil.checkBookiesUp(numBookies, 30) != numBookies) {
                LOG.warn("Not all bookies from this test shut down, expect errors");
            }
        }
    }

    /**
     * Test that a BookKeeper JM can continue to work across the
     * failure of a bookie. This should be handled transparently
     * by bookkeeper.
     */
    @Test
    public void testOneBookieFailure() throws Exception {
        newBookie = bkutil.newBookie();
        BookieServer replacementBookie = null;
        try {
            int ensembleSize = numBookies + 1;
            replacedertEquals("New bookie didn't start", ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10));
            // ensure that the journal manager has to use all bookies,
            // so that a failure will fail the journal manager
            Configuration conf = new Configuration();
            conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE, ensembleSize);
            conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, ensembleSize);
            long txid = 1;
            NamespaceInfo nsi = newNSInfo();
            BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"), nsi);
            bkjm.format(nsi);
            EditLogOutputStream out = bkjm.startLogSegment(txid, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
            for (long i = 1; i <= 3; i++) {
                FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
                op.setTransactionId(txid++);
                out.write(op);
            }
            out.setReadyToFlush();
            out.flush();
            replacementBookie = bkutil.newBookie();
            replacedertEquals("replacement bookie didn't start", ensembleSize + 1, bkutil.checkBookiesUp(ensembleSize + 1, 10));
            newBookie.shutdown();
            replacedertEquals("New bookie didn't die", ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10));
            for (long i = 1; i <= 3; i++) {
                FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
                op.setTransactionId(txid++);
                out.write(op);
            }
            out.setReadyToFlush();
            out.flush();
        } catch (Exception e) {
            LOG.error("Exception in test", e);
            throw e;
        } finally {
            if (replacementBookie != null) {
                replacementBookie.shutdown();
            }
            newBookie.shutdown();
            if (bkutil.checkBookiesUp(numBookies, 30) != numBookies) {
                LOG.warn("Not all bookies from this test shut down, expect errors");
            }
        }
    }

    /**
     * If a journal manager has an empty inprogress node, ensure that we throw an
     * error, as this should not be possible, and some third party has corrupted
     * the zookeeper state
     */
    @Test
    public void testEmptyInprogressNode() throws Exception {
        URI uri = BKJMUtil.createJournalURI("/hdfsjournal-emptyInprogress");
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        bkjm.format(nsi);
        EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        ;
        for (long i = 1; i <= 100; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.close();
        bkjm.finalizeLogSegment(1, 100);
        out = bkjm.startLogSegment(101, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        out.close();
        bkjm.close();
        String inprogressZNode = bkjm.inprogressZNode(101);
        zkc.setData(inprogressZNode, new byte[0], -1);
        bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        try {
            bkjm.recoverUnfinalizedSegments();
            fail("Should have failed. There should be no way of creating" + " an empty inprogess znode");
        } catch (IOException e) {
            // correct behaviour
            replacedertTrue("Exception different than expected", e.getMessage().contains("Invalid/Incomplete data in znode"));
        } finally {
            bkjm.close();
        }
    }

    /**
     * If a journal manager has an corrupt inprogress node, ensure that we throw
     * an error, as this should not be possible, and some third party has
     * corrupted the zookeeper state
     */
    @Test
    public void testCorruptInprogressNode() throws Exception {
        URI uri = BKJMUtil.createJournalURI("/hdfsjournal-corruptInprogress");
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        bkjm.format(nsi);
        EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        ;
        for (long i = 1; i <= 100; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.close();
        bkjm.finalizeLogSegment(1, 100);
        out = bkjm.startLogSegment(101, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        out.close();
        bkjm.close();
        String inprogressZNode = bkjm.inprogressZNode(101);
        zkc.setData(inprogressZNode, "WholeLottaJunk".getBytes(), -1);
        bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        try {
            bkjm.recoverUnfinalizedSegments();
            fail("Should have failed. There should be no way of creating" + " an empty inprogess znode");
        } catch (IOException e) {
            // correct behaviour
            replacedertTrue("Exception different than expected", e.getMessage().contains("has no field named"));
        } finally {
            bkjm.close();
        }
    }

    /**
     * Cases can occur where we create a segment but crash before we even have the
     * chance to write the START_SEGMENT op. If this occurs we should warn, but
     * load as normal
     */
    @Test
    public void testEmptyInprogressLedger() throws Exception {
        URI uri = BKJMUtil.createJournalURI("/hdfsjournal-emptyInprogressLedger");
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        bkjm.format(nsi);
        EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        ;
        for (long i = 1; i <= 100; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.close();
        bkjm.finalizeLogSegment(1, 100);
        out = bkjm.startLogSegment(101, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        out.close();
        bkjm.close();
        bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        bkjm.recoverUnfinalizedSegments();
        out = bkjm.startLogSegment(101, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        for (long i = 1; i <= 100; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.close();
        bkjm.finalizeLogSegment(101, 200);
        bkjm.close();
    }

    /**
     * Test that if we fail between finalizing an inprogress and deleting the
     * corresponding inprogress znode.
     */
    @Test
    public void testRefinalizeAlreadyFinalizedInprogress() throws Exception {
        URI uri = BKJMUtil.createJournalURI("/hdfsjournal-refinalizeInprogressLedger");
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        bkjm.format(nsi);
        EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        ;
        for (long i = 1; i <= 100; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.close();
        bkjm.close();
        String inprogressZNode = bkjm.inprogressZNode(1);
        String finalizedZNode = bkjm.finalizedLedgerZNode(1, 100);
        replacedertNotNull("inprogress znode doesn't exist", zkc.exists(inprogressZNode, null));
        replacedertNull("finalized znode exists", zkc.exists(finalizedZNode, null));
        byte[] inprogressData = zkc.getData(inprogressZNode, false, null);
        // finalize
        bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        bkjm.recoverUnfinalizedSegments();
        bkjm.close();
        replacedertNull("inprogress znode exists", zkc.exists(inprogressZNode, null));
        replacedertNotNull("finalized znode doesn't exist", zkc.exists(finalizedZNode, null));
        zkc.create(inprogressZNode, inprogressData, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
        // should work fine
        bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        bkjm.recoverUnfinalizedSegments();
        bkjm.close();
    }

    /**
     * Tests that the edit log file meta data reading from ZooKeeper should be
     * able to handle the NoNodeException. bkjm.getInputStream(fromTxId,
     * inProgressOk) should suppress the NoNodeException and continue. HDFS-3441.
     */
    @Test
    public void testEditLogFileNotExistsWhenReadingMetadata() throws Exception {
        URI uri = BKJMUtil.createJournalURI("/hdfsjournal-editlogfile");
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        bkjm.format(nsi);
        try {
            // start new inprogress log segment with txid=1
            // and write transactions till txid=50
            String zkpath1 = startAndFinalizeLogSegment(bkjm, 1, 50);
            // start new inprogress log segment with txid=51
            // and write transactions till txid=100
            String zkpath2 = startAndFinalizeLogSegment(bkjm, 51, 100);
            // read the metadata from ZK. Here simulating the situation
            // when reading,the edit log metadata can be removed by purger thread.
            ZooKeeper zkspy = spy(BKJMUtil.connectZooKeeper());
            bkjm.setZooKeeper(zkspy);
            Mockito.doThrow(new KeeperException.NoNodeException(zkpath2 + " doesn't exists")).when(zkspy).getData(zkpath2, false, null);
            List<EditLogLedgerMetadata> ledgerList = bkjm.getLedgerList(false);
            replacedertEquals("List contains the metadata of non exists path.", 1, ledgerList.size());
            replacedertEquals("LogLedgerMetadata contains wrong zk paths.", zkpath1, ledgerList.get(0).getZkPath());
        } finally {
            bkjm.close();
        }
    }

    private enum ThreadStatus {

        COMPLETED, GOODEXCEPTION, BADEXCEPTION
    }

    /**
     * Tests that concurrent calls to format will still allow one to succeed.
     */
    @Test
    public void testConcurrentFormat() throws Exception {
        final URI uri = BKJMUtil.createJournalURI("/hdfsjournal-concurrentformat");
        final NamespaceInfo nsi = newNSInfo();
        // populate with data first
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
        bkjm.format(nsi);
        for (int i = 1; i < 100 * 2; i += 2) {
            bkjm.startLogSegment(i, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
            bkjm.finalizeLogSegment(i, i + 1);
        }
        bkjm.close();
        final int numThreads = 40;
        List<Callable<ThreadStatus>> threads = new ArrayList<Callable<ThreadStatus>>();
        final CyclicBarrier barrier = new CyclicBarrier(numThreads);
        for (int i = 0; i < numThreads; i++) {
            threads.add(new Callable<ThreadStatus>() {

                public ThreadStatus call() {
                    BookKeeperJournalManager bkjm = null;
                    try {
                        bkjm = new BookKeeperJournalManager(conf, uri, nsi);
                        barrier.await();
                        bkjm.format(nsi);
                        return ThreadStatus.COMPLETED;
                    } catch (IOException ioe) {
                        LOG.info("Exception formatting ", ioe);
                        return ThreadStatus.GOODEXCEPTION;
                    } catch (InterruptedException ie) {
                        LOG.error("Interrupted. Something is broken", ie);
                        Thread.currentThread().interrupt();
                        return ThreadStatus.BADEXCEPTION;
                    } catch (Exception e) {
                        LOG.error("Some other bad exception", e);
                        return ThreadStatus.BADEXCEPTION;
                    } finally {
                        if (bkjm != null) {
                            try {
                                bkjm.close();
                            } catch (IOException ioe) {
                                LOG.error("Error closing journal manager", ioe);
                            }
                        }
                    }
                }
            });
        }
        ExecutorService service = Executors.newFixedThreadPool(numThreads);
        List<Future<ThreadStatus>> statuses = service.invokeAll(threads, 60, TimeUnit.SECONDS);
        int numCompleted = 0;
        for (Future<ThreadStatus> s : statuses) {
            replacedertTrue(s.isDone());
            replacedertTrue("Thread threw invalid exception", s.get() == ThreadStatus.COMPLETED || s.get() == ThreadStatus.GOODEXCEPTION);
            if (s.get() == ThreadStatus.COMPLETED) {
                numCompleted++;
            }
        }
        LOG.info("Completed " + numCompleted + " formats");
        replacedertTrue("No thread managed to complete formatting", numCompleted > 0);
    }

    @Test(timeout = 120000)
    public void testDefaultAckQuorum() throws Exception {
        newBookie = bkutil.newBookie();
        int ensembleSize = numBookies + 1;
        int quorumSize = numBookies + 1;
        // ensure that the journal manager has to use all bookies,
        // so that a failure will fail the journal manager
        Configuration conf = new Configuration();
        conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE, ensembleSize);
        conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, quorumSize);
        // sets 2 secs
        conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_SEC, 2);
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"), nsi);
        bkjm.format(nsi);
        CountDownLatch sleepLatch = new CountDownLatch(1);
        sleepBookie(sleepLatch, newBookie);
        EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        int numTransactions = 100;
        for (long i = 1; i <= numTransactions; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        try {
            out.close();
            bkjm.finalizeLogSegment(1, numTransactions);
            List<EditLogInputStream> in = new ArrayList<EditLogInputStream>();
            bkjm.selectInputStreams(in, 1, true);
            try {
                replacedertEquals(numTransactions, FSEditLogTestUtil.countTransactionsInStream(in.get(0)));
            } finally {
                in.get(0).close();
            }
            fail("Should throw exception as not enough non-faulty bookies available!");
        } catch (IOException ioe) {
        // expected
        }
    }

    /**
     * Test ack quorum feature supported by bookkeeper. Keep ack quorum bookie
     * alive and sleep all the other bookies. Now the client would wait for the
     * acknowledgement from the ack size bookies and after receiving the success
     * response will continue writing. Non ack client will hang long time to add
     * entries.
     */
    @Test(timeout = 120000)
    public void testAckQuorum() throws Exception {
        // slow bookie
        newBookie = bkutil.newBookie();
        // make quorum size and ensemble size same to avoid the interleave writing
        // of the ledger entries
        int ensembleSize = numBookies + 1;
        int quorumSize = numBookies + 1;
        int ackSize = numBookies;
        // ensure that the journal manager has to use all bookies,
        // so that a failure will fail the journal manager
        Configuration conf = new Configuration();
        conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE, ensembleSize);
        conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, quorumSize);
        conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ACK_QUORUM_SIZE, ackSize);
        // sets 60 minutes
        conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_SEC, 3600);
        NamespaceInfo nsi = newNSInfo();
        BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"), nsi);
        bkjm.format(nsi);
        CountDownLatch sleepLatch = new CountDownLatch(1);
        sleepBookie(sleepLatch, newBookie);
        EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        int numTransactions = 100;
        for (long i = 1; i <= numTransactions; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.close();
        bkjm.finalizeLogSegment(1, numTransactions);
        List<EditLogInputStream> in = new ArrayList<EditLogInputStream>();
        bkjm.selectInputStreams(in, 1, true);
        try {
            replacedertEquals(numTransactions, FSEditLogTestUtil.countTransactionsInStream(in.get(0)));
        } finally {
            sleepLatch.countDown();
            in.get(0).close();
            bkjm.close();
        }
    }

    /**
     * Sleep a bookie until I count down the latch
     *
     * @param latch
     *          Latch to wait on
     * @param bookie
     *          bookie server
     * @throws Exception
     */
    private void sleepBookie(final CountDownLatch l, final BookieServer bookie) throws Exception {
        Thread sleeper = new Thread() {

            public void run() {
                try {
                    bookie.suspendProcessing();
                    l.await(60, TimeUnit.SECONDS);
                    bookie.resumeProcessing();
                } catch (Exception e) {
                    LOG.error("Error suspending bookie", e);
                }
            }
        };
        sleeper.setName("BookieServerSleeper-" + bookie.getBookie().getId());
        sleeper.start();
    }

    private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm, int startTxid, int endTxid) throws IOException, KeeperException, InterruptedException {
        EditLogOutputStream out = bkjm.startLogSegment(startTxid, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
        for (long i = startTxid; i <= endTxid; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            op.setTransactionId(i);
            out.write(op);
        }
        out.close();
        // finalize the inprogress_1 log segment.
        bkjm.finalizeLogSegment(startTxid, endTxid);
        String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
        replacedertNotNull(zkc.exists(zkpath1, false));
        replacedertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));
        return zkpath1;
    }
}

18 View Complete Implementation : TestBookKeeperSpeculativeRead.java
Copyright Apache License 2.0
Author : aliyun-beta
@AfterClreplaced
public static void teardownBookkeeper() throws Exception {
    bkutil.teardown();
    for (BookieServer bk : bks) {
        bk.shutdown();
    }
}

18 View Complete Implementation : LocalBookKeeper.java
Copyright Apache License 2.0
Author : apache
public void shutdownBookies() {
    for (BookieServer bookieServer : bs) {
        bookieServer.shutdown();
    }
}

18 View Complete Implementation : GcOverreplicatedLedgerTest.java
Copyright Apache License 2.0
Author : apache
private BookieSocketAddress getBookieNotInEnsemble(LedgerMetadata ledgerMetadata) throws UnknownHostException {
    List<BookieSocketAddress> allAddresses = Lists.newArrayList();
    for (BookieServer bk : bs) {
        allAddresses.add(bk.getLocalAddress());
    }
    SortedMap<Long, ? extends List<BookieSocketAddress>> ensembles = ledgerMetadata.getAllEnsembles();
    for (List<BookieSocketAddress> fragmentEnsembles : ensembles.values()) {
        allAddresses.removeAll(fragmentEnsembles);
    }
    replacedert.replacedertEquals(allAddresses.size(), 1);
    return allAddresses.get(0);
}

18 View Complete Implementation : BookKeeperDiskSpaceWeightedLedgerPlacementTest.java
Copyright Apache License 2.0
Author : apache
private BookieServer replaceBookieWithCustomFreeDiskSpaceBookie(BookKeeperCheckInfoReader client, BookieServer bookie, final long freeDiskSpace) throws Exception {
    for (int i = 0; i < bs.size(); i++) {
        if (bs.get(i).getLocalAddress().equals(bookie.getLocalAddress())) {
            return replaceBookieWithCustomFreeDiskSpaceBookie(client, i, freeDiskSpace);
        }
    }
    return null;
}

18 View Complete Implementation : AuditorBookieTest.java
Copyright Apache License 2.0
Author : apache
private void startAuditorElectors() throws Exception {
    for (BookieServer bserver : bs) {
        String addr = bserver.getLocalAddress().toString();
        startAuditorElector(addr);
    }
}

18 View Complete Implementation : AuditorBookieTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Test should ensure only one should act as Auditor. Starting/shutdown
 * other than auditor bookie shouldn't initiate re-election and multiple
 * auditors.
 */
@Test
public void testEnsureOnlySingleAuditor() throws Exception {
    BookieServer auditor = verifyAuditor();
    // shutdown bookie which is not an auditor
    int indexOf = bs.indexOf(auditor);
    int bkIndexDownBookie;
    if (indexOf < bs.size() - 1) {
        bkIndexDownBookie = indexOf + 1;
    } else {
        bkIndexDownBookie = indexOf - 1;
    }
    shutdownBookie(bs.get(bkIndexDownBookie));
    startNewBookie();
    startNewBookie();
    // grace period for the auditor re-election if any
    BookieServer newAuditor = waitForNewAuditor(auditor);
    replacedertSame("Auditor re-election is not happened for auditor failure!", auditor, newAuditor);
}

18 View Complete Implementation : AuditorBookieTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Test Auditor crashes should trigger re-election and another bookie should
 * take over the auditor ship.
 */
@Test
public void testSuccessiveAuditorCrashes() throws Exception {
    BookieServer auditor = verifyAuditor();
    shutdownBookie(auditor);
    BookieServer newAuditor1 = waitForNewAuditor(auditor);
    bs.remove(auditor);
    shutdownBookie(newAuditor1);
    BookieServer newAuditor2 = waitForNewAuditor(newAuditor1);
    replacedertNotSame("Auditor re-election is not happened for auditor failure!", auditor, newAuditor2);
    bs.remove(newAuditor1);
}

18 View Complete Implementation : AuditorLedgerCheckerTest.java
Copyright Apache License 2.0
Author : apache
private Auditor getAuditorBookiesAuditor() throws Exception {
    BookieServer auditorBookieServer = getAuditorBookie();
    String bookieAddr = auditorBookieServer.getLocalAddress().toString();
    return auditorElectors.get(bookieAddr).auditor;
}

18 View Complete Implementation : BookieAutoRecoveryTest.java
Copyright Apache License 2.0
Author : apache
private void verifyLedgerEnsembleMetadataAfterReplication(BookieServer newBookieServer, LedgerHandle lh, int ledgerReplicaIndex) throws Exception {
    LedgerHandle openLedger = bkc.openLedger(lh.getId(), digestType, PreplacedWD);
    BookieSocketAddress inetSocketAddress = openLedger.getLedgerMetadata().getAllEnsembles().get(0L).get(ledgerReplicaIndex);
    replacedertEquals("Rereplication has been failed and ledgerReplicaIndex :" + ledgerReplicaIndex, newBookieServer.getLocalAddress(), inetSocketAddress);
    openLedger.close();
}

18 View Complete Implementation : BookKeeperClusterTestCase.java
Copyright Apache License 2.0
Author : apache
/**
 * Get bookie configuration for bookie.
 */
public ServerConfiguration getBkConf(BookieSocketAddress addr) throws Exception {
    int bkIndex = 0;
    for (BookieServer server : bs) {
        if (server.getLocalAddress().equals(addr)) {
            break;
        }
        ++bkIndex;
    }
    if (bkIndex < bs.size()) {
        return bsConfs.get(bkIndex);
    }
    return null;
}

18 View Complete Implementation : TestBookKeeperJournalManager.java
Copyright Apache License 2.0
Author : aliyun-beta
/**
 * Sleep a bookie until I count down the latch
 *
 * @param latch
 *          Latch to wait on
 * @param bookie
 *          bookie server
 * @throws Exception
 */
private void sleepBookie(final CountDownLatch l, final BookieServer bookie) throws Exception {
    Thread sleeper = new Thread() {

        public void run() {
            try {
                bookie.suspendProcessing();
                l.await(60, TimeUnit.SECONDS);
                bookie.resumeProcessing();
            } catch (Exception e) {
                LOG.error("Error suspending bookie", e);
            }
        }
    };
    sleeper.setName("BookieServerSleeper-" + bookie.getBookie().getId());
    sleeper.start();
}

17 View Complete Implementation : TestAuth.java
Copyright Apache License 2.0
Author : apache
@Test
public void testCloseMethodCalledOnAuthProvider() throws Exception {
    LogCloseCallsBookieAuthProviderFactory.closeCountersOnFactory.set(0);
    LogCloseCallsBookieAuthProviderFactory.closeCountersOnConnections.set(0);
    LogCloseCallsBookieAuthProviderFactory.initCountersOnFactory.set(0);
    LogCloseCallsBookieAuthProviderFactory.initCountersOnConnections.set(0);
    LogCloseCallsClientAuthProviderFactory.initCountersOnFactory.set(0);
    LogCloseCallsClientAuthProviderFactory.closeCountersOnFactory.set(0);
    ServerConfiguration bookieConf = newServerConfiguration();
    bookieConf.setBookieAuthProviderFactoryClreplaced(LogCloseCallsBookieAuthProviderFactory.clreplaced.getName());
    ClientConfiguration clientConf = newClientConfiguration();
    clientConf.setClientAuthProviderFactoryClreplaced(LogCloseCallsClientAuthProviderFactory.clreplaced.getName());
    startAndStoreBookie(bookieConf);
    AtomicLong ledgerId = new AtomicLong(-1);
    // should succeed
    connectAndWriteToBookie(clientConf, ledgerId);
    replacedertFalse(ledgerId.get() == -1);
    replacedertEquals("Should have entry", 1, entryCount(ledgerId.get(), bookieConf, clientConf));
    for (BookieServer bks : bs) {
        bks.shutdown();
    }
    replacedertEquals(LogCloseCallsBookieAuthProviderFactory.initCountersOnConnections.get(), LogCloseCallsBookieAuthProviderFactory.closeCountersOnConnections.get());
    replacedertTrue(LogCloseCallsBookieAuthProviderFactory.initCountersOnConnections.get() > 0);
    replacedertEquals(1, LogCloseCallsBookieAuthProviderFactory.initCountersOnFactory.get());
    replacedertEquals(1, LogCloseCallsBookieAuthProviderFactory.closeCountersOnFactory.get());
    replacedertEquals(LogCloseCallsClientAuthProviderFactory.initCountersOnConnections.get(), LogCloseCallsClientAuthProviderFactory.closeCountersOnConnections.get());
    replacedertTrue(LogCloseCallsClientAuthProviderFactory.initCountersOnConnections.get() > 0);
    replacedertEquals(1, LogCloseCallsClientAuthProviderFactory.initCountersOnFactory.get());
    replacedertEquals(1, LogCloseCallsClientAuthProviderFactory.closeCountersOnFactory.get());
}

17 View Complete Implementation : UpdateCookieCmdTest.java
Copyright Apache License 2.0
Author : apache
/**
 * updatecookie to ipaddress.
 */
@Test
public void testUpdateCookieHostnameToIpAddress() throws Exception {
    updateCookie("-bookieId", "hostname", true);
    updateCookie("-b", "ip", false);
    // start bookie to ensure everything works fine
    ServerConfiguration conf = bsConfs.get(0);
    BookieServer restartBookie = startBookie(conf);
    restartBookie.shutdown();
}

17 View Complete Implementation : BookKeeperDiskSpaceWeightedLedgerPlacementTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Test to show that weight based selection honors the disk weight of bookies and also adapts
 * when the bookies's weight changes.
 */
@FlakyTest("https://github.com/apache/bookkeeper/issues/503")
public void testDiskSpaceWeightedBookieSelectionWithChangingWeights() throws Exception {
    long freeDiskSpace = 1000000L;
    int multiple = 3;
    ClientConfiguration conf = new ClientConfiguration();
    conf.setDiskWeightBasedPlacementEnabled(true).setGetBookieInfoRetryIntervalSeconds(1, TimeUnit.SECONDS).setBookieMaxWeightMultipleForWeightBasedPlacement(multiple).setMetadataServiceUri(zkUtil.getMetadataServiceUri());
    final BookKeeperCheckInfoReader client = new BookKeeperCheckInfoReader(conf);
    for (int i = 0; i < numBookies; i++) {
        // the first 8 bookies have freeDiskSpace of 1MB; While the remaining 2 have 3MB
        if (i < numBookies - 2) {
            replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, freeDiskSpace);
        } else {
            replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, multiple * freeDiskSpace);
        }
    }
    Map<BookieSocketAddress, Integer> m = new HashMap<BookieSocketAddress, Integer>();
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPreplacedwd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsembleAt(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    // make sure that bookies with higher weight(the last 2 bookies) are chosen 3X as often as the median;
    // since the number of ledgers created is small (2000), we allow a range of 2X to 4X instead of the exact 3X
    for (int i = 0; i < numBookies - 2; i++) {
        double ratio1 = (double) m.get(bs.get(numBookies - 2).getLocalAddress()) / (double) m.get(bs.get(i).getLocalAddress());
        replacedertTrue("Weigheted placement is not honored: " + Math.abs(ratio1 - multiple), Math.abs(ratio1 - multiple) < 1);
        double ratio2 = (double) m.get(bs.get(numBookies - 1).getLocalAddress()) / (double) m.get(bs.get(i).getLocalAddress());
        replacedertTrue("Weigheted placement is not honored: " + Math.abs(ratio2 - multiple), Math.abs(ratio2 - multiple) < 1);
    }
    // Restart the bookies in such a way that the first 2 bookies go from 1MB to 3MB free space and the last
    // 2 bookies go from 3MB to 1MB
    BookieServer server1 = bs.get(0);
    BookieServer server2 = bs.get(1);
    BookieServer server3 = bs.get(numBookies - 2);
    BookieServer server4 = bs.get(numBookies - 1);
    server1 = replaceBookieWithCustomFreeDiskSpaceBookie(client, server1, multiple * freeDiskSpace);
    server2 = replaceBookieWithCustomFreeDiskSpaceBookie(client, server2, multiple * freeDiskSpace);
    server3 = replaceBookieWithCustomFreeDiskSpaceBookie(client, server3, freeDiskSpace);
    server4 = replaceBookieWithCustomFreeDiskSpaceBookie(client, server4, freeDiskSpace);
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPreplacedwd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsembleAt(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    // make sure that bookies with higher weight(the last 2 bookies) are chosen 3X as often as the median;
    // since the number of ledgers created is small (2000), we allow a range of 2X to 4X instead of the exact 3X
    for (int i = 0; i < numBookies; i++) {
        if (server1.getLocalAddress().equals(bs.get(i).getLocalAddress()) || server2.getLocalAddress().equals(bs.get(i).getLocalAddress())) {
            continue;
        }
        double ratio1 = (double) m.get(server1.getLocalAddress()) / (double) m.get(bs.get(i).getLocalAddress());
        replacedertTrue("Weigheted placement is not honored: " + Math.abs(ratio1 - multiple), Math.abs(ratio1 - multiple) < 1);
        double ratio2 = (double) m.get(server2.getLocalAddress()) / (double) m.get(bs.get(i).getLocalAddress());
        replacedertTrue("Weigheted placement is not honored: " + Math.abs(ratio2 - multiple), Math.abs(ratio2 - multiple) < 1);
    }
    client.close();
}

17 View Complete Implementation : BookKeeperDiskSpaceWeightedLedgerPlacementTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Test to show that weight based selection honors the disk weight of bookies.
 */
@FlakyTest("https://github.com/apache/bookkeeper/issues/503")
public void testDiskSpaceWeightedBookieSelection() throws Exception {
    long freeDiskSpace = 1000000L;
    int multiple = 3;
    ClientConfiguration conf = new ClientConfiguration();
    conf.setDiskWeightBasedPlacementEnabled(true).setGetBookieInfoRetryIntervalSeconds(1, TimeUnit.SECONDS).setBookieMaxWeightMultipleForWeightBasedPlacement(multiple).setMetadataServiceUri(zkUtil.getMetadataServiceUri());
    final BookKeeperCheckInfoReader client = new BookKeeperCheckInfoReader(conf);
    for (int i = 0; i < numBookies; i++) {
        // the first 8 bookies have freeDiskSpace of 1MB; While the remaining 2 have 3MB
        if (i < numBookies - 2) {
            replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, freeDiskSpace);
        } else {
            replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, multiple * freeDiskSpace);
        }
    }
    Map<BookieSocketAddress, Integer> m = new HashMap<BookieSocketAddress, Integer>();
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPreplacedwd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsembleAt(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    client.close();
    // make sure that bookies with higher weight(the last 2 bookies) are chosen 3X as often as the median;
    // since the number of ledgers created is small (2000), we allow a range of 2X to 4X instead of the exact 3X
    for (int i = 0; i < numBookies - 2; i++) {
        double ratio1 = (double) m.get(bs.get(numBookies - 2).getLocalAddress()) / (double) m.get(bs.get(i).getLocalAddress());
        replacedertTrue("Weigheted placement is not honored: " + Math.abs(ratio1 - multiple), Math.abs(ratio1 - multiple) < 1);
        double ratio2 = (double) m.get(bs.get(numBookies - 1).getLocalAddress()) / (double) m.get(bs.get(i).getLocalAddress());
        replacedertTrue("Weigheted placement is not honored: " + Math.abs(ratio2 - multiple), Math.abs(ratio2 - multiple) < 1);
    }
}

17 View Complete Implementation : BookKeeperDiskSpaceWeightedLedgerPlacementTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Test to show that weight based selection honors the disk weight of bookies and also adapts
 * when bookies go away permanently.
 */
@FlakyTest("https://github.com/apache/bookkeeper/issues/503")
public void testDiskSpaceWeightedBookieSelectionWithBookiesDying() throws Exception {
    long freeDiskSpace = 1000000L;
    int multiple = 3;
    ClientConfiguration conf = new ClientConfiguration();
    conf.setDiskWeightBasedPlacementEnabled(true).setGetBookieInfoRetryIntervalSeconds(1, TimeUnit.SECONDS).setBookieMaxWeightMultipleForWeightBasedPlacement(multiple).setMetadataServiceUri(zkUtil.getMetadataServiceUri());
    final BookKeeperCheckInfoReader client = new BookKeeperCheckInfoReader(conf);
    for (int i = 0; i < numBookies; i++) {
        // the first 8 bookies have freeDiskSpace of 1MB; While the remaining 2 have 1GB
        if (i < numBookies - 2) {
            replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, freeDiskSpace);
        } else {
            replaceBookieWithCustomFreeDiskSpaceBookie(client, 0, multiple * freeDiskSpace);
        }
    }
    Map<BookieSocketAddress, Integer> m = new HashMap<BookieSocketAddress, Integer>();
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPreplacedwd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsembleAt(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    // make sure that bookies with higher weight are chosen 3X as often as the median;
    // since the number of ledgers is small (2000), there may be variation
    double ratio1 = (double) m.get(bs.get(numBookies - 2).getLocalAddress()) / (double) m.get(bs.get(0).getLocalAddress());
    replacedertTrue("Weigheted placement is not honored: " + Math.abs(ratio1 - multiple), Math.abs(ratio1 - multiple) < 1);
    double ratio2 = (double) m.get(bs.get(numBookies - 1).getLocalAddress()) / (double) m.get(bs.get(1).getLocalAddress());
    replacedertTrue("Weigheted placement is not honored: " + Math.abs(ratio2 - multiple), Math.abs(ratio2 - multiple) < 1);
    // Bring down the 2 bookies that had higher weight; after this the allocation to all
    // the remaining bookies should be uniform
    for (BookieServer b : bs) {
        m.put(b.getLocalAddress(), 0);
    }
    BookieServer server1 = bs.get(numBookies - 2);
    BookieServer server2 = bs.get(numBookies - 1);
    killBookieAndWaitForZK(numBookies - 1);
    killBookieAndWaitForZK(numBookies - 2);
    for (int i = 0; i < 2000; i++) {
        LedgerHandle lh = client.createLedger(3, 3, DigestType.CRC32, "testPreplacedwd".getBytes());
        for (BookieSocketAddress b : lh.getLedgerMetadata().getEnsembleAt(0)) {
            m.put(b, m.get(b) + 1);
        }
    }
    // make sure that bookies with higher weight are chosen 3X as often as the median;
    for (int i = 0; i < numBookies - 3; i++) {
        double delta = Math.abs((double) m.get(bs.get(i).getLocalAddress()) - (double) m.get(bs.get(i + 1).getLocalAddress()));
        delta = (delta * 100) / (double) m.get(bs.get(i + 1).getLocalAddress());
        // the deviation should be less than 30%
        replacedertTrue("Weigheted placement is not honored: " + delta, delta <= 30);
    }
    // since the following 2 bookies were down, they shouldn't ever be selected
    replacedertTrue("Weigheted placement is not honored" + m.get(server1.getLocalAddress()), m.get(server1.getLocalAddress()) == 0);
    replacedertTrue("Weigheted placement is not honored" + m.get(server2.getLocalAddress()), m.get(server2.getLocalAddress()) == 0);
    client.close();
}

17 View Complete Implementation : AuditorBookieTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Test the vote is deleting from the ZooKeeper during shutdown.
 */
@Test
public void testShutdown() throws Exception {
    BookieServer auditor = verifyAuditor();
    shutdownBookie(auditor);
    // waiting for new auditor
    BookieServer newAuditor = waitForNewAuditor(auditor);
    replacedertNotSame("Auditor re-election is not happened for auditor failure!", auditor, newAuditor);
    int indexOfDownBookie = bs.indexOf(auditor);
    bs.remove(indexOfDownBookie);
    bsConfs.remove(indexOfDownBookie);
    List<String> children = zkc.getChildren(electionPath, false);
    for (String child : children) {
        byte[] data = zkc.getData(electionPath + '/' + child, false, null);
        String bookieIP = new String(data);
        String addr = auditor.getLocalAddress().toString();
        replacedertFalse("AuditorElection cleanup fails", bookieIP.contains(addr));
    }
}

17 View Complete Implementation : AuditorBookieTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Test restarting the entire bookie cluster. It shouldn't create multiple
 * bookie auditors.
 */
@Test
public void testBookieClusterRestart() throws Exception {
    BookieServer auditor = verifyAuditor();
    for (AuditorElector auditorElector : auditorElectors.values()) {
        replacedertTrue("Auditor elector is not running!", auditorElector.isRunning());
    }
    stopBKCluster();
    stopAuditorElectors();
    startBKCluster(zkUtil.getMetadataServiceUri());
    startAuditorElectors();
    BookieServer newAuditor = waitForNewAuditor(auditor);
    replacedertNotSame("Auditor re-election is not happened for auditor failure!", auditor, newAuditor);
}

17 View Complete Implementation : AuditorBookieTest.java
Copyright Apache License 2.0
Author : apache
private void shutdownBookie(BookieServer bkServer) throws Exception {
    String addr = bkServer.getLocalAddress().toString();
    LOG.debug("Shutting down bookie:" + addr);
    // shutdown bookie which is an auditor
    bkServer.shutdown();
    // stopping corresponding auditor elector
    auditorElectors.get(addr).shutdown();
}